logicaffeine_language/lexer.rs
1//! Two-stage lexer for LOGOS natural language input.
2//!
3//! The lexer transforms natural language text into a token stream suitable
4//! for parsing. It operates in two stages:
5//!
6//! ## Stage 1: Line Lexer
7//!
8//! The [`LineLexer`] handles structural concerns:
9//!
10//! - **Indentation**: Tracks indent levels, emits `Indent`/`Dedent` tokens
11//! - **Block boundaries**: Identifies significant whitespace
12//! - **Content extraction**: Passes line content to Stage 2
13//!
14//! ## Stage 2: Word Lexer
15//!
16//! The [`Lexer`] performs word-level tokenization:
17//!
18//! - **Vocabulary lookup**: Identifies words via the lexicon database
19//! - **Morphological analysis**: Handles inflection (verb tenses, plurals)
20//! - **Ambiguity resolution**: Uses priority rules for ambiguous words
21//!
22//! ## Ambiguity Rules
23//!
24//! When a word matches multiple lexicon entries, priority determines the token:
25//!
26//! 1. **Quantifiers** over nouns ("some" → Quantifier, not Noun)
27//! 2. **Determiners** over adjectives ("the" → Determiner, not Adjective)
28//! 3. **Verbs** over nouns for -ing/-ed forms ("running" → Verb)
29//!
30//! ## Example
31//!
32//! ```text
33//! Input: "Every cat sleeps."
34//! Output: [Quantifier("every"), Noun("cat"), Verb("sleeps"), Period]
35//! ```
36
37use crate::lexicon::{self, Aspect, Definiteness, Lexicon, Time};
38use crate::token::{
39 BlockType, CalendarUnit, FocusKind, MeasureKind, RawKind, Span, Token, TokenType,
40};
41use logicaffeine_base::Interner;
42
43// ============================================================================
44// Stage 1: Line Lexer (Spec §2.5.2)
45// ============================================================================
46
47/// Tokens emitted by the LineLexer (Stage 1).
48/// Handles structural tokens (Indent, Dedent, Newline) while treating
49/// all other content as opaque for Stage 2 word classification.
50#[derive(Debug, Clone, PartialEq)]
51pub enum LineToken {
52 /// Block increased indentation
53 Indent,
54 /// Block decreased indentation
55 Dedent,
56 /// Logical newline (statement boundary) - reserved for future use
57 Newline,
58 /// Content to be further tokenized (line content, trimmed)
59 Content {
60 text: String,
61 start: usize,
62 end: usize,
63 },
64}
65
66/// Stage 1 Lexer: Handles only lines, indentation, and structural tokens.
67/// Treats all other text as opaque `Content` for the Stage 2 WordLexer.
68pub struct LineLexer<'a> {
69 source: &'a str,
70 bytes: &'a [u8],
71 indent_stack: Vec<usize>,
72 pending_dedents: usize,
73 position: usize,
74 /// True if we need to emit Content for current line
75 has_pending_content: bool,
76 pending_content_start: usize,
77 pending_content_end: usize,
78 pending_content_text: String,
79 /// True after we've finished processing all lines
80 finished_lines: bool,
81 /// True if we've emitted at least one Indent (need to emit Dedents at EOF)
82 emitted_indent: bool,
83 /// Escape block body byte ranges to skip (start_byte, end_byte)
84 escape_body_ranges: Vec<(usize, usize)>,
85}
86
87impl<'a> LineLexer<'a> {
88 pub fn new(source: &'a str) -> Self {
89 Self {
90 source,
91 bytes: source.as_bytes(),
92 indent_stack: vec![0],
93 pending_dedents: 0,
94 position: 0,
95 has_pending_content: false,
96 pending_content_start: 0,
97 pending_content_end: 0,
98 pending_content_text: String::new(),
99 finished_lines: false,
100 emitted_indent: false,
101 escape_body_ranges: Vec::new(),
102 }
103 }
104
105 pub fn with_escape_ranges(source: &'a str, escape_body_ranges: Vec<(usize, usize)>) -> Self {
106 Self {
107 source,
108 bytes: source.as_bytes(),
109 indent_stack: vec![0],
110 pending_dedents: 0,
111 position: 0,
112 has_pending_content: false,
113 pending_content_start: 0,
114 pending_content_end: 0,
115 pending_content_text: String::new(),
116 finished_lines: false,
117 emitted_indent: false,
118 escape_body_ranges,
119 }
120 }
121
122 /// Check if a byte position falls within an escape body range.
123 fn is_in_escape_body(&self, pos: usize) -> bool {
124 self.escape_body_ranges
125 .iter()
126 .any(|(start, end)| pos >= *start && pos < *end)
127 }
128
129 /// Calculate indentation level at current position (at start of line).
130 /// Returns (indent_level, content_start_pos).
131 fn measure_indent(&self, line_start: usize) -> (usize, usize) {
132 let mut indent = 0;
133 let mut pos = line_start;
134
135 while pos < self.bytes.len() {
136 match self.bytes[pos] {
137 b' ' => {
138 indent += 1;
139 pos += 1;
140 }
141 b'\t' => {
142 indent += 4; // Tab = 4 spaces
143 pos += 1;
144 }
145 _ => break,
146 }
147 }
148
149 (indent, pos)
150 }
151
152 /// Read content from current position until end of line or EOF.
153 /// Returns (content_text, content_start, content_end, next_line_start).
154 fn read_line_content(&self, content_start: usize) -> (String, usize, usize, usize) {
155 let mut pos = content_start;
156
157 // Find end of line
158 while pos < self.bytes.len() && self.bytes[pos] != b'\n' {
159 pos += 1;
160 }
161
162 let content_end = pos;
163 let text = self.source[content_start..content_end]
164 .trim_end()
165 .to_string();
166
167 // Move past newline if present
168 let next_line_start = if pos < self.bytes.len() && self.bytes[pos] == b'\n' {
169 pos + 1
170 } else {
171 pos
172 };
173
174 (text, content_start, content_end, next_line_start)
175 }
176
177 /// Check if the line starting at `pos` is blank (only whitespace).
178 fn is_blank_line(&self, line_start: usize) -> bool {
179 let mut pos = line_start;
180 while pos < self.bytes.len() {
181 match self.bytes[pos] {
182 b' ' | b'\t' => pos += 1,
183 b'\n' => return true,
184 _ => return false,
185 }
186 }
187 true // EOF counts as blank
188 }
189
190 /// Process the next line and update internal state.
191 /// Returns true if we have tokens to emit, false if we're done.
192 fn process_next_line(&mut self) -> bool {
193 // Skip blank lines
194 while self.position < self.bytes.len() && self.is_blank_line(self.position) {
195 // Skip to next line
196 while self.position < self.bytes.len() && self.bytes[self.position] != b'\n' {
197 self.position += 1;
198 }
199 if self.position < self.bytes.len() {
200 self.position += 1; // Skip the newline
201 }
202 }
203
204 // Check if we've reached EOF
205 if self.position >= self.bytes.len() {
206 self.finished_lines = true;
207 // Emit remaining dedents at EOF
208 if self.indent_stack.len() > 1 {
209 self.pending_dedents = self.indent_stack.len() - 1;
210 self.indent_stack.truncate(1);
211 }
212 return self.pending_dedents > 0;
213 }
214
215 // Measure indentation of current line
216 let (line_indent, content_start) = self.measure_indent(self.position);
217
218 // Read line content
219 let (text, start, end, next_pos) = self.read_line_content(content_start);
220
221 // Skip if content is empty (shouldn't happen after blank line skip, but be safe)
222 if text.is_empty() {
223 self.position = next_pos;
224 return self.process_next_line();
225 }
226
227 let current_indent = *self.indent_stack.last().unwrap();
228
229 // Handle indentation changes
230 if line_indent > current_indent {
231 // Indent: push new level
232 self.indent_stack.push(line_indent);
233 self.emitted_indent = true;
234 // Store content to emit after Indent
235 self.has_pending_content = true;
236 self.pending_content_text = text;
237 self.pending_content_start = start;
238 self.pending_content_end = end;
239 self.position = next_pos;
240 // We'll emit Indent first, then Content
241 return true;
242 } else if line_indent < current_indent {
243 // Dedent: pop until we match
244 while self.indent_stack.len() > 1 {
245 let top = *self.indent_stack.last().unwrap();
246 if line_indent < top {
247 self.indent_stack.pop();
248 self.pending_dedents += 1;
249 } else {
250 break;
251 }
252 }
253 // Store content to emit after Dedents
254 self.has_pending_content = true;
255 self.pending_content_text = text;
256 self.pending_content_start = start;
257 self.pending_content_end = end;
258 self.position = next_pos;
259 return true;
260 } else {
261 // Same indentation level
262 self.has_pending_content = true;
263 self.pending_content_text = text;
264 self.pending_content_start = start;
265 self.pending_content_end = end;
266 self.position = next_pos;
267 return true;
268 }
269 }
270}
271
272impl<'a> Iterator for LineLexer<'a> {
273 type Item = LineToken;
274
275 fn next(&mut self) -> Option<LineToken> {
276 // 1. Emit pending dedents first
277 if self.pending_dedents > 0 {
278 self.pending_dedents -= 1;
279 return Some(LineToken::Dedent);
280 }
281
282 // 2. Emit pending content
283 if self.has_pending_content {
284 self.has_pending_content = false;
285 let text = std::mem::take(&mut self.pending_content_text);
286 let start = self.pending_content_start;
287 let end = self.pending_content_end;
288 return Some(LineToken::Content { text, start, end });
289 }
290
291 // 3. Check if we need to emit Indent (after pushing to stack)
292 // This happens when we detected an indent but haven't emitted the token yet
293 // We need to check if indent_stack was just modified
294
295 // 4. Process next line
296 if !self.finished_lines {
297 let had_indent = self.indent_stack.len();
298 if self.process_next_line() {
299 // Check if we added an indent level
300 if self.indent_stack.len() > had_indent {
301 return Some(LineToken::Indent);
302 }
303 // Check if we have pending dedents
304 if self.pending_dedents > 0 {
305 self.pending_dedents -= 1;
306 return Some(LineToken::Dedent);
307 }
308 // Otherwise emit content
309 if self.has_pending_content {
310 self.has_pending_content = false;
311 let text = std::mem::take(&mut self.pending_content_text);
312 let start = self.pending_content_start;
313 let end = self.pending_content_end;
314 return Some(LineToken::Content { text, start, end });
315 }
316 } else if self.pending_dedents > 0 {
317 // EOF with pending dedents
318 self.pending_dedents -= 1;
319 return Some(LineToken::Dedent);
320 }
321 }
322
323 // 5. Emit any remaining dedents at EOF
324 if self.pending_dedents > 0 {
325 self.pending_dedents -= 1;
326 return Some(LineToken::Dedent);
327 }
328
329 None
330 }
331}
332
333// ============================================================================
334// Stage 2: Word Lexer (existing Lexer)
335// ============================================================================
336
337#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
338pub enum LexerMode {
339 #[default]
340 Declarative, // Logic, Theorems, Definitions
341 Imperative, // Main, Functions, Code
342}
343
344pub struct Lexer<'a> {
345 words: Vec<WordItem>,
346 pos: usize,
347 lexicon: Lexicon,
348 interner: &'a mut Interner,
349 input_len: usize,
350 in_let_context: bool,
351 mode: LexerMode,
352 source: String,
353 /// Escape block body byte ranges: (skip_start, skip_end) for filtering LineLexer events
354 escape_body_ranges: Vec<(usize, usize)>,
355}
356
357struct WordItem {
358 word: String,
359 trailing_punct: Option<char>,
360 start: usize,
361 end: usize,
362 punct_pos: Option<usize>,
363}
364
365impl<'a> Lexer<'a> {
366 /// Creates a new lexer for the given input text.
367 ///
368 /// The lexer will tokenize natural language text according to the
369 /// lexicon database, performing morphological analysis and ambiguity
370 /// resolution.
371 ///
372 /// # Arguments
373 ///
374 /// * `input` - The natural language text to tokenize
375 /// * `interner` - String interner for efficient symbol handling
376 ///
377 /// # Example
378 ///
379 /// ```
380 /// use logicaffeine_language::lexer::Lexer;
381 /// use logicaffeine_base::Interner;
382 ///
383 /// let mut interner = Interner::new();
384 /// let mut lexer = Lexer::new("Every cat sleeps.", &mut interner);
385 /// let tokens = lexer.tokenize();
386 ///
387 /// assert_eq!(tokens.len(), 5); // Quantifier, Noun, Verb, Period, EOI
388 /// ```
389 pub fn new(input: &str, interner: &'a mut Interner) -> Self {
390 let input = Self::blank_optional_header_colons(input);
391 let escape_ranges = Self::find_escape_block_ranges(&input);
392 let escape_body_ranges: Vec<(usize, usize)> = escape_ranges
393 .iter()
394 .map(|(_, end, content_start, _, _, _)| (*content_start, *end))
395 .collect();
396 let words = Self::split_into_words(&input, &escape_ranges);
397 let input_len = input.len();
398
399 Lexer {
400 words,
401 pos: 0,
402 lexicon: Lexicon::new(),
403 interner,
404 input_len,
405 in_let_context: false,
406 mode: LexerMode::Declarative,
407 source: input,
408 escape_body_ranges,
409 }
410 }
411
412 /// Every `## <Header>` whose own grammar does not already read a trailing `:`
413 /// (`BlockType::accepts_optional_trailing_colon`: `Main`, `Component`, `Store`, `Page`,
414 /// `Error page` — never the ones that already require one, like `Directive`/`Route`, or
415 /// `Test`, whose colon has a different, load-bearing job) accepts one anyway, silently: a
416 /// habitual trailing `:` at the end of exactly one of these header lines is blanked to a
417 /// space BEFORE tokenizing, so it never becomes a `Colon` token at all.
418 ///
419 /// Done here, on the raw source, rather than as a token-stream pass: LOGOS does not emit a
420 /// `Newline` token between two ordinary statements at the same indentation (only `Indent`/
421 /// `Dedent` mark structure, and a table row gets its own explicit one) — `## Main:\nShow
422 /// "hi".` tokenizes as `BlockHeader, Colon, Show, …` with NO token between the colon and
423 /// `Show` to search up to. The source string, in contrast, always has the `\n` that ends the
424 /// header's own line, so "the header's own line" is unambiguous here and nowhere else. A
425 /// byte-for-byte blank (never a delete) keeps every later span exactly where it already was.
426 fn blank_optional_header_colons(input: &str) -> String {
427 const ACCEPTING: &[&str] = &["main", "component", "store", "page", "error"];
428 let mut out = input.as_bytes().to_vec();
429 let mut line_start = 0usize;
430 for line_end in input
431 .match_indices('\n')
432 .map(|(i, _)| i)
433 .chain(std::iter::once(input.len()))
434 {
435 let line = &input[line_start..line_end];
436 let trimmed_start = line.trim_start();
437 if let Some(after_hashes) = trimmed_start.strip_prefix("##") {
438 let first_word = after_hashes
439 .trim_start()
440 .split(|c: char| c.is_whitespace() || c == ':')
441 .next()
442 .unwrap_or("");
443 if ACCEPTING.contains(&first_word.to_ascii_lowercase().as_str()) {
444 let trimmed_end = line.trim_end();
445 if let Some(':') = trimmed_end.chars().next_back() {
446 let colon_offset = line_start + trimmed_end.len() - 1;
447 out[colon_offset] = b' ';
448 }
449 }
450 }
451 line_start = line_end + 1;
452 }
453 String::from_utf8(out).expect("blanking a `:` to a space cannot break UTF-8")
454 }
455
456 /// A lexer for a fragment that is CODE, not prose — an LGX template hole, a directive
457 /// value, a filter argument. Ordinary [`Lexer::new`] starts in declarative mode and only
458 /// switches on a `## Main` / `## To` / `## Component` header, which a bare fragment has
459 /// none of; a hole like `todo's id` or `a . b` must tokenize the imperative way.
460 pub fn new_imperative(input: &str, interner: &'a mut Interner) -> Self {
461 let mut lexer = Self::new(input, interner);
462 lexer.mode = LexerMode::Imperative;
463 lexer
464 }
465
466 /// Pre-scan source text for escape block bodies.
467 /// Returns (skip_start_byte, skip_end_byte, content_start_byte, raw_code) tuples.
468 /// `skip_start` is the line start (for byte skipping in split_into_words).
469 /// `content_start` is after leading whitespace (for token span alignment with Indent events).
470 fn find_escape_block_ranges(source: &str) -> Vec<(usize, usize, usize, String, char, usize)> {
471 let mut ranges = Vec::new();
472 let lines: Vec<&str> = source.split('\n').collect();
473 let mut line_starts: Vec<usize> = Vec::with_capacity(lines.len());
474 let mut pos = 0;
475 for line in &lines {
476 line_starts.push(pos);
477 pos += line.len() + 1; // +1 for the newline
478 }
479
480 let mut i = 0;
481 while i < lines.len() {
482 let trimmed = lines[i].trim();
483 // Check if this line contains an escape header: "Escape to Rust:"
484 // Matches both statement position (whole line) and expression position
485 // (e.g., "Let x: Int be Escape to Rust:")
486 let lower = trimmed.to_lowercase();
487 // An `Escape to <Language>:` header, an LGX `View:` / `Style:` section, or a
488 // `## To … -> View:` template function — each opens a body captured verbatim.
489 let raw_kind: Option<char> = if lower == "escape to rust:"
490 || lower.ends_with(" escape to rust:")
491 || (lower.starts_with("escape to ") && lower.ends_with(':'))
492 {
493 Some('E')
494 } else if trimmed == "Style:" {
495 Some('S')
496 } else if trimmed == "Head:" {
497 // A `Head:` body is ALWAYS markup — there is no legacy indentation
498 // dialect for it — so it needs none of the `View:` fallback below.
499 Some('H')
500 } else if trimmed == "View:"
501 || (trimmed.starts_with("## ") && trimmed.ends_with("-> View:"))
502 {
503 Some('V')
504 } else {
505 None
506 };
507 if let Some(kind) = raw_kind {
508 // Find the body: subsequent lines with deeper indentation
509 let header_indent = Self::measure_indent_static(lines[i]);
510 i += 1;
511
512 // Skip blank lines to find the first body line
513 let mut body_start_line = i;
514 while body_start_line < lines.len() && lines[body_start_line].trim().is_empty() {
515 body_start_line += 1;
516 }
517
518 if body_start_line >= lines.len() {
519 // No body found
520 continue;
521 }
522
523 let base_indent = Self::measure_indent_static(lines[body_start_line]);
524 if base_indent <= header_indent {
525 // No indented body
526 continue;
527 }
528 // A view body is captured raw only when it IS markup: the legacy indentation DSL's
529 // bodies begin with an element word (`div [class="x"]:`), and a `-> View:` function
530 // or route may answer through statements (`Return answer(who).`) — neither begins
531 // with `<` or `{`.
532 if kind == 'V' {
533 let first = lines[body_start_line].trim_start();
534 if !(first.starts_with('<') || first.starts_with('{')) {
535 continue;
536 }
537 }
538
539 // Capture all lines at base_indent or deeper
540 let body_byte_start = line_starts[body_start_line];
541 let mut body_end_line = body_start_line;
542 let mut code_lines: Vec<String> = Vec::new();
543
544 let mut j = body_start_line;
545 while j < lines.len() {
546 let line = lines[j];
547 if line.trim().is_empty() {
548 // Blank lines are preserved
549 code_lines.push(String::new());
550 body_end_line = j;
551 j += 1;
552 continue;
553 }
554 let line_indent = Self::measure_indent_static(line);
555 if line_indent < base_indent {
556 break;
557 }
558 // Strip base indentation
559 let stripped = Self::strip_indent(line, base_indent);
560 code_lines.push(stripped);
561 body_end_line = j;
562 j += 1;
563 }
564
565 // Trim trailing empty lines from code
566 while code_lines.last().map_or(false, |l| l.is_empty()) {
567 code_lines.pop();
568 }
569
570 if !code_lines.is_empty() {
571 let body_byte_end = if body_end_line + 1 < lines.len() {
572 line_starts[body_end_line + 1]
573 } else {
574 source.len()
575 };
576 // Compute content start (after leading whitespace of first body line)
577 let content_start =
578 body_byte_start + Self::leading_whitespace_bytes(lines[body_start_line]);
579 let raw_code = code_lines.join("\n");
580 ranges.push((
581 body_byte_start,
582 body_byte_end,
583 content_start,
584 raw_code,
585 kind,
586 base_indent,
587 ));
588 }
589
590 i = j;
591 } else {
592 i += 1;
593 }
594 }
595
596 ranges
597 }
598
599 /// The whitespace-delimited words after the current `## Header` word on its line,
600 /// read verbatim from the source (a URL path must not be split into LOGOS
601 /// punctuation), each with the byte offset just past it. A peek: consumes nothing.
602 fn header_line_words(&self) -> Vec<(String, usize)> {
603 let hdr_end = self.words[self.pos].end;
604 let line_end = self.source[hdr_end..]
605 .find('\n')
606 .map(|k| hdr_end + k)
607 .unwrap_or(self.source.len());
608 let mut out = Vec::new();
609 let mut cursor = hdr_end;
610 while cursor < line_end {
611 let rest = &self.source[cursor..line_end];
612 let skip = rest.len() - rest.trim_start().len();
613 let start = cursor + skip;
614 if start >= line_end {
615 break;
616 }
617 let len = self.source[start..line_end]
618 .find(char::is_whitespace)
619 .unwrap_or(line_end - start);
620 out.push((self.source[start..start + len].to_string(), start + len));
621 cursor = start + len;
622 }
623 out
624 }
625
626 /// Advance `pos` past every split word that ends at or before `end` (the tokenize
627 /// loop's own advance then steps past the last one), like the `## Test:` capture.
628 fn consume_header_through(&mut self, end: usize) {
629 while self.pos + 1 < self.words.len() && self.words[self.pos + 1].start < end {
630 self.pos += 1;
631 }
632 }
633
634 /// Recognize an LGX header by the SHAPE of its line, not its first word alone, so a
635 /// prose heading (`## Error Propagation`, `## Before you begin`) stays prose:
636 /// `## Page /path`, `## Route METHOD /path`, `## Store Name`, `## Directive name (…)`,
637 /// `## Before each request`, `## Redirect /a -> /b`, `## Error page 404`. The raw path
638 /// words are interned and consumed; everything after them lexes normally.
639 fn try_lgx_header(&mut self, lowered: &str) -> Option<BlockType> {
640 let words = self.header_line_words();
641 let word = |i: usize| words.get(i).map(|(w, _)| w.as_str()).unwrap_or("");
642 let end_of = |i: usize| words.get(i).map(|(_, e)| *e).unwrap_or(0);
643 let is_path = |w: &str| w.starts_with('/');
644 let is_method = |w: &str| !w.is_empty() && w.chars().all(|c| c.is_ascii_uppercase());
645 match lowered {
646 "page" if is_path(word(0)) => {
647 let path = self.interner.intern(word(0).trim_end_matches('.'));
648 self.consume_header_through(end_of(0));
649 Some(BlockType::Page { path })
650 }
651 "route" if is_method(word(0)) && is_path(word(1)) => {
652 let method = self.interner.intern(word(0));
653 let path = self.interner.intern(word(1).trim_end_matches('.'));
654 self.consume_header_through(end_of(1));
655 Some(BlockType::Route { method, path })
656 }
657 "store"
658 if word(0)
659 .chars()
660 .next()
661 .is_some_and(|c| c.is_ascii_uppercase()) =>
662 {
663 Some(BlockType::Store)
664 }
665 // `## Session` takes no name: a program has one session, the way it has one router.
666 // The guard keeps `## Session Something` out — that is a typo or a misunderstanding,
667 // and falling through to prose would silently discard the block.
668 "session" if word(0).is_empty() => Some(BlockType::Session),
669 "directive"
670 if !word(0).is_empty()
671 && word(0)
672 .chars()
673 .next()
674 .is_some_and(|c| c.is_ascii_lowercase())
675 && words.iter().any(|(w, _)| w.contains('(')) =>
676 {
677 Some(BlockType::Directive)
678 }
679 "before" if word(0) == "each" && word(1).trim_end_matches(':') == "request" => {
680 self.consume_header_through(end_of(1));
681 Some(BlockType::Middleware)
682 }
683 "redirect" if is_path(word(0)) && word(1) == "->" && is_path(word(2)) => {
684 let from = self.interner.intern(word(0));
685 let to = self.interner.intern(word(2).trim_end_matches('.'));
686 self.consume_header_through(end_of(2));
687 Some(BlockType::Redirect { from, to })
688 }
689 "error" if word(0) == "page" => {
690 let status = word(1).trim_end_matches('.').parse::<u16>().ok()?;
691 self.consume_header_through(end_of(1));
692 Some(BlockType::ErrorPage { status })
693 }
694 _ => None,
695 }
696 }
697
698 /// Count leading whitespace bytes in a line.
699 fn leading_whitespace_bytes(line: &str) -> usize {
700 let mut count = 0;
701 for c in line.chars() {
702 match c {
703 ' ' | '\t' => count += c.len_utf8(),
704 _ => break,
705 }
706 }
707 count
708 }
709
710 /// Measure indent of a line (static helper for pre-scan).
711 fn measure_indent_static(line: &str) -> usize {
712 let mut indent = 0;
713 for c in line.chars() {
714 match c {
715 ' ' => indent += 1,
716 '\t' => indent += 4,
717 _ => break,
718 }
719 }
720 indent
721 }
722
723 /// Strip `count` leading spaces/tabs from a line.
724 fn strip_indent(line: &str, count: usize) -> String {
725 let mut stripped = 0;
726 let mut byte_pos = 0;
727 for (i, c) in line.char_indices() {
728 if stripped >= count {
729 byte_pos = i;
730 break;
731 }
732 match c {
733 ' ' => {
734 stripped += 1;
735 byte_pos = i + 1;
736 }
737 '\t' => {
738 stripped += 4;
739 byte_pos = i + 1;
740 }
741 _ => {
742 byte_pos = i;
743 break;
744 }
745 }
746 }
747 if stripped < count {
748 byte_pos = line.len();
749 }
750 line[byte_pos..].to_string()
751 }
752
753 fn split_into_words(
754 input: &str,
755 escape_ranges: &[(usize, usize, usize, String, char, usize)],
756 ) -> Vec<WordItem> {
757 let mut items = Vec::new();
758 let mut current_word = String::new();
759 let mut word_start = 0;
760 let chars: Vec<char> = input.chars().collect();
761 let mut char_idx = 0;
762 let mut skip_count = 0;
763 // Inside `[` … `]` a digit-flanked comma is a list separator, not a
764 // thousands separator — `[1,2,3]` is three elements. Money literals
765 // (`$125,000`) keep their commas at any depth; line-local so an
766 // unbalanced bracket never poisons the rest of the input.
767 let mut bracket_depth: usize = 0;
768 // Track byte offset for escape range matching
769 let mut skip_to_byte: Option<usize> = None;
770
771 for (i, c) in input.char_indices() {
772 if skip_count > 0 {
773 skip_count -= 1;
774 char_idx += 1;
775 continue;
776 }
777 // Skip bytes inside escape block bodies
778 if let Some(end) = skip_to_byte {
779 if i < end {
780 char_idx += 1;
781 continue;
782 }
783 skip_to_byte = None;
784 word_start = i;
785 }
786 // Check if this byte position starts an escape block body
787 if let Some((_, end, content_start, raw_code, kind, indent)) =
788 escape_ranges.iter().find(|(s, _, _, _, _, _)| i == *s)
789 {
790 // Flush any pending word
791 if !current_word.is_empty() {
792 items.push(WordItem {
793 word: std::mem::take(&mut current_word),
794 trailing_punct: None,
795 start: word_start,
796 end: i,
797 punct_pos: None,
798 });
799 }
800 // Emit the entire block as a single \x00ESC: marker
801 // Use content_start (after whitespace) for span alignment with Indent events
802 items.push(WordItem {
803 word: match kind {
804 'E' => format!("\x00ESC:{}", raw_code),
805 // `\x00LGX:<kind>:<content_start>:<indent>:<body>` — decoded in
806 // `tokenize` (`\x00RAW:` is the raw STRING literal marker, a different
807 // thing). `indent` is what was stripped from each body line, so a byte
808 // offset inside the body maps back to its place in the file.
809 k => format!("\x00LGX:{k}:{content_start}:{indent}:{raw_code}"),
810 },
811 trailing_punct: None,
812 start: *content_start,
813 end: *end,
814 punct_pos: None,
815 });
816 skip_to_byte = Some(*end);
817 word_start = *end;
818 char_idx += 1;
819 continue;
820 }
821 let next_pos = i + c.len_utf8();
822 match c {
823 // "8:15 pm" — a space between a H:MM time and "am"/"pm" is part of
824 // the time literal: skip the flush so "pm" joins "8:15" → "8:15pm".
825 ' ' if Self::is_time_space_before_ampm(¤t_word, &chars, char_idx) => {}
826 ' ' | '\t' | '\n' | '\r' => {
827 if c == '\n' {
828 bracket_depth = 0;
829 }
830 if !current_word.is_empty() {
831 items.push(WordItem {
832 word: std::mem::take(&mut current_word),
833 trailing_punct: None,
834 start: word_start,
835 end: i,
836 punct_pos: None,
837 });
838 }
839 word_start = next_pos;
840 }
841 '.' => {
842 // Check if this is a decimal point (digit before and after)
843 let prev_is_digit = !current_word.is_empty()
844 && current_word
845 .chars()
846 .last()
847 .map_or(false, |ch| ch.is_ascii_digit());
848 let next_is_digit =
849 char_idx + 1 < chars.len() && chars[char_idx + 1].is_ascii_digit();
850
851 // Field-access / UFCS DOT: an identifier char (letter/`_`) or a
852 // closing `)`/`]` immediately before, and an identifier-start
853 // immediately after — no whitespace either side. `p.x`, `xs.f(a)`,
854 // `(e).m`. Digit-glue wins (`5.sqrt` stays `5` + period + `sqrt`,
855 // `5.0` stays a decimal), and `Show x.` stays a sentence (the next
856 // char is a newline, not an identifier). Mode is resolved later:
857 // `classify_with_lookahead` maps the `\x00DOT` marker to `Dot`
858 // (imperative) or a sentence `Period` (declarative — prose keeps
859 // `e.g.` exactly as today).
860 let prev_ident = current_word
861 .chars()
862 .last()
863 .map_or(false, |ch| ch.is_alphabetic() || ch == '_');
864 let prev_close = current_word.is_empty()
865 && char_idx > 0
866 && matches!(chars[char_idx - 1], ')' | ']');
867 let next_ident = char_idx + 1 < chars.len()
868 && (chars[char_idx + 1].is_alphabetic() || chars[char_idx + 1] == '_');
869
870 if prev_is_digit && next_is_digit {
871 // This is a decimal point, include it in the current word
872 current_word.push(c);
873 } else if (prev_ident || prev_close) && next_ident {
874 // Flush the receiver, then the mode-deferred dot marker.
875 if !current_word.is_empty() {
876 items.push(WordItem {
877 word: std::mem::take(&mut current_word),
878 trailing_punct: None,
879 start: word_start,
880 end: i,
881 punct_pos: None,
882 });
883 }
884 items.push(WordItem {
885 word: "\x00DOT".to_string(),
886 trailing_punct: None,
887 start: i,
888 end: next_pos,
889 punct_pos: None,
890 });
891 word_start = next_pos;
892 } else {
893 // This is a sentence period
894 if !current_word.is_empty() {
895 items.push(WordItem {
896 word: std::mem::take(&mut current_word),
897 trailing_punct: Some(c),
898 start: word_start,
899 end: i,
900 punct_pos: Some(i),
901 });
902 } else {
903 items.push(WordItem {
904 word: String::new(),
905 trailing_punct: Some(c),
906 start: i,
907 end: next_pos,
908 punct_pos: Some(i),
909 });
910 }
911 word_start = next_pos;
912 }
913 }
914 '#' => {
915 // Check for ## block header (markdown-style)
916 if char_idx + 1 < chars.len() && chars[char_idx + 1] == '#' {
917 // This is a ## block header
918 // Skip the second # and capture the next word as a block header
919 if !current_word.is_empty() {
920 items.push(WordItem {
921 word: std::mem::take(&mut current_word),
922 trailing_punct: None,
923 start: word_start,
924 end: i,
925 punct_pos: None,
926 });
927 }
928 // Skip whitespace after ##
929 let header_start = i;
930 let mut j = char_idx + 2;
931 while j < chars.len() && (chars[j] == ' ' || chars[j] == '\t') {
932 j += 1;
933 }
934 // Capture the block type word
935 let mut block_word = String::from("##");
936 while j < chars.len() && chars[j].is_alphabetic() {
937 block_word.push(chars[j]);
938 j += 1;
939 }
940 if block_word.len() > 2 {
941 items.push(WordItem {
942 word: block_word,
943 trailing_punct: None,
944 start: header_start,
945 end: header_start + (j - char_idx),
946 punct_pos: None,
947 });
948 }
949 skip_count = j - char_idx - 1;
950 word_start = header_start + (j - char_idx);
951 } else {
952 // Single # - treat as comment, skip to end of line
953 // Count how many chars to skip (without modifying char_idx here -
954 // the main loop's skip handler will increment it)
955 let mut look_ahead = char_idx + 1;
956 while look_ahead < chars.len() && chars[look_ahead] != '\n' {
957 skip_count += 1;
958 look_ahead += 1;
959 }
960 if !current_word.is_empty() {
961 items.push(WordItem {
962 word: std::mem::take(&mut current_word),
963 trailing_punct: None,
964 start: word_start,
965 end: i,
966 punct_pos: None,
967 });
968 }
969 word_start = look_ahead + 1; // Start after the newline
970 }
971 }
972 // String literals: "hello world" or """multi-line"""
973 '"' => {
974 // `r"…"` RAW string: the adjacent `r` prefix is part of
975 // the literal, not a word — backslashes stay verbatim
976 // (paths, regexes). Only the exact word `r` glued to the
977 // quote triggers it; `Show r.` (a variable) is untouched.
978 let raw_string = current_word == "r";
979 if raw_string {
980 current_word.clear();
981 } else if !current_word.is_empty() {
982 items.push(WordItem {
983 word: std::mem::take(&mut current_word),
984 trailing_punct: None,
985 start: word_start,
986 end: i,
987 punct_pos: None,
988 });
989 }
990
991 // Check for triple-quote: """
992 if char_idx + 2 < chars.len()
993 && chars[char_idx + 1] == '"'
994 && chars[char_idx + 2] == '"'
995 {
996 let string_start = i;
997 let mut j = char_idx + 3; // skip opening """
998 // Skip optional newline after opening """
999 if j < chars.len() && chars[j] == '\n' {
1000 j += 1;
1001 }
1002 let mut raw_content = String::new();
1003 // Scan until closing """
1004 while j < chars.len() {
1005 if j + 2 < chars.len()
1006 && chars[j] == '"'
1007 && chars[j + 1] == '"'
1008 && chars[j + 2] == '"'
1009 {
1010 break;
1011 }
1012 raw_content.push(chars[j]);
1013 j += 1;
1014 }
1015 // Strip trailing newline before closing """
1016 if raw_content.ends_with('\n') {
1017 raw_content.pop();
1018 }
1019 // Dedent: find minimum common indentation and strip it
1020 let dedented = Self::dedent_triple_quote(&raw_content);
1021 let end_pos = if j + 2 < chars.len() {
1022 j + 3
1023 } else {
1024 chars.len()
1025 };
1026 items.push(WordItem {
1027 word: format!("\x00STR:{}", dedented),
1028 trailing_punct: None,
1029 start: string_start,
1030 end: end_pos,
1031 punct_pos: None,
1032 });
1033 // Skip past the closing """
1034 if j + 2 < chars.len() {
1035 skip_count = (j + 2) - char_idx;
1036 } else {
1037 skip_count = chars.len() - 1 - char_idx;
1038 }
1039 word_start = end_pos;
1040 } else {
1041 // Single-quoted string: scan until closing quote
1042 let string_start = i;
1043 let mut j = char_idx + 1;
1044 let mut string_content = String::new();
1045 while j < chars.len() && chars[j] != '"' {
1046 if chars[j] == '\\' && j + 1 < chars.len() && !raw_string {
1047 // DECODE the escape (`"a\nb"` is two lines,
1048 // not the letters a-n-b). Unknown escapes and
1049 // malformed \u{…} stay verbatim rather than
1050 // silently dropping the backslash.
1051 j += 1;
1052 match chars[j] {
1053 'n' => string_content.push('\n'),
1054 't' => string_content.push('\t'),
1055 'r' => string_content.push('\r'),
1056 '0' => string_content.push('\0'),
1057 '\\' => string_content.push('\\'),
1058 '"' => string_content.push('"'),
1059 'u' if j + 1 < chars.len() && chars[j + 1] == '{' => {
1060 let mut k = j + 2;
1061 let mut hex = String::new();
1062 while k < chars.len() && chars[k] != '}' {
1063 hex.push(chars[k]);
1064 k += 1;
1065 }
1066 match u32::from_str_radix(&hex, 16)
1067 .ok()
1068 .and_then(char::from_u32)
1069 {
1070 Some(c) if k < chars.len() => {
1071 string_content.push(c);
1072 j = k;
1073 }
1074 _ => {
1075 string_content.push('\\');
1076 string_content.push('u');
1077 }
1078 }
1079 }
1080 other => {
1081 string_content.push('\\');
1082 string_content.push(other);
1083 }
1084 }
1085 } else {
1086 string_content.push(chars[j]);
1087 }
1088 j += 1;
1089 }
1090
1091 // Create a special marker for string literals
1092 // We prefix with a special character to identify in tokenize()
1093 //
1094 // `end` and the following `word_start` are BYTE offsets
1095 // (Span fields are byte positions, like `start`/`i`); `j`
1096 // is a CHAR index, so convert through the byte length of
1097 // the covered source chars. Emitting `j + 1` (a char
1098 // index) produced a byte-start / char-end span that
1099 // underflows downstream on multibyte input.
1100 let end_char = if j < chars.len() { j + 1 } else { j };
1101 let end_byte = string_start
1102 + chars[char_idx..end_char]
1103 .iter()
1104 .map(|c| c.len_utf8())
1105 .sum::<usize>();
1106 // `r"..."` is VERBATIM: braces are literal, never an
1107 // interpolation hole — the `\x00RAW:` marker tells Stage 2
1108 // to skip both the hole scan and the `{{`→`{` collapse.
1109 let marker = if raw_string { "\x00RAW:" } else { "\x00STR:" };
1110 items.push(WordItem {
1111 word: format!("{marker}{string_content}"),
1112 trailing_punct: None,
1113 start: string_start,
1114 end: end_byte,
1115 punct_pos: None,
1116 });
1117
1118 // Skip past the closing quote
1119 if j < chars.len() {
1120 skip_count = j - char_idx;
1121 } else {
1122 skip_count = j - char_idx - 1;
1123 }
1124 word_start = end_byte;
1125 }
1126 }
1127 // Character literals with backticks: `x`
1128 '`' => {
1129 // Push any pending word
1130 if !current_word.is_empty() {
1131 items.push(WordItem {
1132 word: std::mem::take(&mut current_word),
1133 trailing_punct: None,
1134 start: word_start,
1135 end: i,
1136 punct_pos: None,
1137 });
1138 }
1139
1140 // Scan for character content and closing backtick
1141 let char_start = i;
1142 let mut j = char_idx + 1;
1143 let mut char_content = String::new();
1144
1145 if j < chars.len() {
1146 if chars[j] == '\\' && j + 1 < chars.len() {
1147 // Escape sequence
1148 j += 1;
1149 let escaped_char = match chars[j] {
1150 'n' => '\n',
1151 't' => '\t',
1152 'r' => '\r',
1153 '\\' => '\\',
1154 '`' => '`',
1155 '0' => '\0',
1156 c => c,
1157 };
1158 char_content.push(escaped_char);
1159 j += 1;
1160 } else if chars[j] != '`' {
1161 // Regular character
1162 char_content.push(chars[j]);
1163 j += 1;
1164 }
1165 }
1166
1167 // Expect closing backtick
1168 if j < chars.len() && chars[j] == '`' {
1169 j += 1; // skip closing backtick
1170 }
1171
1172 // Create a special marker for char literals
1173 items.push(WordItem {
1174 word: format!("\x00CHAR:{}", char_content),
1175 trailing_punct: None,
1176 start: char_start,
1177 end: if j <= chars.len() {
1178 char_start + (j - char_idx)
1179 } else {
1180 char_start + 1
1181 },
1182 punct_pos: None,
1183 });
1184
1185 if j > char_idx + 1 {
1186 skip_count = j - char_idx - 1;
1187 }
1188 word_start = char_start + (j - char_idx);
1189 }
1190 // Handle -> as a single token for return type syntax
1191 '-' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '>' => {
1192 // Push any pending word first
1193 if !current_word.is_empty() {
1194 items.push(WordItem {
1195 word: std::mem::take(&mut current_word),
1196 trailing_punct: None,
1197 start: word_start,
1198 end: i,
1199 punct_pos: None,
1200 });
1201 }
1202 // Push -> as its own word
1203 items.push(WordItem {
1204 word: "->".to_string(),
1205 trailing_punct: None,
1206 start: i,
1207 end: i + 2,
1208 punct_pos: None,
1209 });
1210 skip_count = 1; // Skip the '>' character
1211 word_start = i + 2;
1212 }
1213 // Grand Challenge: Handle <= as a single token
1214 '<' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1215 if !current_word.is_empty() {
1216 items.push(WordItem {
1217 word: std::mem::take(&mut current_word),
1218 trailing_punct: None,
1219 start: word_start,
1220 end: i,
1221 punct_pos: None,
1222 });
1223 }
1224 items.push(WordItem {
1225 word: "<=".to_string(),
1226 trailing_punct: None,
1227 start: i,
1228 end: i + 2,
1229 punct_pos: None,
1230 });
1231 skip_count = 1;
1232 word_start = i + 2;
1233 }
1234 // Grand Challenge: Handle >= as a single token
1235 '>' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1236 if !current_word.is_empty() {
1237 items.push(WordItem {
1238 word: std::mem::take(&mut current_word),
1239 trailing_punct: None,
1240 start: word_start,
1241 end: i,
1242 punct_pos: None,
1243 });
1244 }
1245 items.push(WordItem {
1246 word: ">=".to_string(),
1247 trailing_punct: None,
1248 start: i,
1249 end: i + 2,
1250 punct_pos: None,
1251 });
1252 skip_count = 1;
1253 word_start = i + 2;
1254 }
1255 // Handle == as a single token
1256 '=' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1257 if !current_word.is_empty() {
1258 items.push(WordItem {
1259 word: std::mem::take(&mut current_word),
1260 trailing_punct: None,
1261 start: word_start,
1262 end: i,
1263 punct_pos: None,
1264 });
1265 }
1266 items.push(WordItem {
1267 word: "==".to_string(),
1268 trailing_punct: None,
1269 start: i,
1270 end: i + 2,
1271 punct_pos: None,
1272 });
1273 skip_count = 1;
1274 word_start = i + 2;
1275 }
1276 // Handle != as a single token
1277 '!' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1278 if !current_word.is_empty() {
1279 items.push(WordItem {
1280 word: std::mem::take(&mut current_word),
1281 trailing_punct: None,
1282 start: word_start,
1283 end: i,
1284 punct_pos: None,
1285 });
1286 }
1287 items.push(WordItem {
1288 word: "!=".to_string(),
1289 trailing_punct: None,
1290 start: i,
1291 end: i + 2,
1292 punct_pos: None,
1293 });
1294 skip_count = 1;
1295 word_start = i + 2;
1296 }
1297 // Special handling for '-' in ISO-8601 dates (YYYY-MM-DD)
1298 '-' if Self::is_date_hyphen(¤t_word, &chars, char_idx) => {
1299 // This hyphen is part of a date, include it in the word
1300 current_word.push(c);
1301 }
1302 // Special handling for ':' in time literals (9:30am, 11:45pm)
1303 ':' if Self::is_time_colon(¤t_word, &chars, char_idx) => {
1304 // This colon is part of a time, include it in the word
1305 current_word.push(c);
1306 }
1307 // Namespaced type reference `Alias::Type` (no intervening spaces):
1308 // the `::` glues an identifier segment on each side into ONE word, so
1309 // the interned token text is exactly the cross-module registry name
1310 // (`Geometry::Point`) that construction and type annotations consume.
1311 // A lone `:` (a block colon `xs[1]:`, a `label:`) stays punctuation —
1312 // only a `::` between identifier characters is the path separator.
1313 ':' if Self::is_namespace_colon(¤t_word, &chars, char_idx) => {
1314 current_word.push(c);
1315 current_word.push(chars[char_idx + 1]);
1316 skip_count = 1;
1317 }
1318 // Hyphenated compounds ("weak-until", "highest-priority"):
1319 // a hyphen BETWEEN letters is part of the word, never the
1320 // arithmetic minus (which is space- or digit-adjacent).
1321 '-' if char_idx > 0
1322 && chars[char_idx - 1].is_alphabetic()
1323 && char_idx + 1 < chars.len()
1324 && chars[char_idx + 1].is_alphabetic() =>
1325 {
1326 current_word.push(c);
1327 }
1328 // Scientific notation: 4.84e+00, 1.66E-03, 2.5e-2
1329 '+' | '-' if Self::is_exponent_sign(¤t_word, &chars, char_idx) => {
1330 current_word.push(c);
1331 }
1332 // Alphanumeric codes ("AV-435", "FRZ-192", "I-5"): a hyphen with
1333 // an UPPERCASE-code word before it and a DIGIT after is part of
1334 // the identifier, not an arithmetic minus. The all-uppercase gate
1335 // keeps lowercase variables ("x-1") and digit arithmetic ("5-3")
1336 // emitting a Minus.
1337 '-' if char_idx > 0
1338 && chars[char_idx - 1].is_alphabetic()
1339 && char_idx + 1 < chars.len()
1340 && chars[char_idx + 1].is_ascii_digit()
1341 && !current_word.is_empty()
1342 && current_word.chars().all(|ch| ch.is_ascii_uppercase()) =>
1343 {
1344 current_word.push(c);
1345 }
1346 // Letter grades ("B+", "A+"): a '+' directly after an
1347 // uppercase-letter word, NOT followed by a digit (arithmetic
1348 // "B+2"), is part of the grade. A spaced "B + C" flushes "B"
1349 // before the '+', so only the glued grade form is caught.
1350 '+' if !current_word.is_empty()
1351 && current_word.chars().all(|ch| ch.is_ascii_uppercase())
1352 && (char_idx + 1 >= chars.len() || !chars[char_idx + 1].is_ascii_digit()) =>
1353 {
1354 current_word.push(c);
1355 }
1356 // Thousands separator inside a number: "125,000", "1,234,567".
1357 // A comma flanked by digits is part of the numeral, not a clause
1358 // separator — except inside a bracketed list, where `[1,2,3]`
1359 // means three elements. A money word (`$125,000`) keeps its
1360 // commas even there.
1361 ',' if char_idx > 0
1362 && chars[char_idx - 1].is_ascii_digit()
1363 && char_idx + 1 < chars.len()
1364 && chars[char_idx + 1].is_ascii_digit()
1365 && (bracket_depth == 0
1366 || current_word
1367 .chars()
1368 .next()
1369 .map_or(false, Self::is_currency_symbol)) =>
1370 {
1371 current_word.push(c);
1372 }
1373 // Date separator inside a numeral: "04/2024", "12/25", "04/2024"
1374 // (MM/YYYY, MM/DD). A slash flanked by digits is part of the date
1375 // token, not an arithmetic division (rare in NL clues).
1376 '/' if char_idx > 0
1377 && chars[char_idx - 1].is_ascii_digit()
1378 && char_idx + 1 < chars.len()
1379 && chars[char_idx + 1].is_ascii_digit() =>
1380 {
1381 current_word.push(c);
1382 }
1383 // "28-inch", "6-year-old", "12-year-old" — a hyphen with a DIGIT
1384 // before and a LETTER after is orthographic (= "28 inch"), so
1385 // split into separate words here (flush the number, skip the
1386 // hyphen as a word boundary) rather than emitting an arithmetic
1387 // Minus. A digit-digit hyphen ("5-3") stays Minus (letter-after
1388 // required); a letter-letter hyphen ("well-known") is handled above.
1389 '-' if char_idx > 0
1390 && chars[char_idx - 1].is_ascii_digit()
1391 && char_idx + 1 < chars.len()
1392 && chars[char_idx + 1].is_alphabetic() =>
1393 {
1394 if !current_word.is_empty() {
1395 items.push(WordItem {
1396 word: std::mem::take(&mut current_word),
1397 trailing_punct: None,
1398 start: word_start,
1399 end: i,
1400 punct_pos: None,
1401 });
1402 }
1403 word_start = next_pos;
1404 }
1405 // `**` exponentiation — a single two-char token (before the
1406 // generic punct arm and the `*=` arm; `**` is `*` then `*`).
1407 '*' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '*' => {
1408 if !current_word.is_empty() {
1409 items.push(WordItem {
1410 word: std::mem::take(&mut current_word),
1411 trailing_punct: None,
1412 start: word_start,
1413 end: i,
1414 punct_pos: None,
1415 });
1416 }
1417 items.push(WordItem {
1418 word: "**".to_string(),
1419 trailing_punct: None,
1420 start: i,
1421 end: i + 2,
1422 punct_pos: None,
1423 });
1424 skip_count = 1;
1425 word_start = i + 2;
1426 }
1427 // `//` floor division — a single two-char token (before the generic
1428 // punct arm and the `/=` arm; `//` is `/` then `/`, not `/` then `=`).
1429 // A digit-flanked `/` is already claimed above as a date separator, so
1430 // this only fires on a genuine `x // y`.
1431 '/' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '/' => {
1432 if !current_word.is_empty() {
1433 items.push(WordItem {
1434 word: std::mem::take(&mut current_word),
1435 trailing_punct: None,
1436 start: word_start,
1437 end: i,
1438 punct_pos: None,
1439 });
1440 }
1441 items.push(WordItem {
1442 word: "//".to_string(),
1443 trailing_punct: None,
1444 start: i,
1445 end: i + 2,
1446 punct_pos: None,
1447 });
1448 skip_count = 1;
1449 word_start = i + 2;
1450 }
1451 // Compound assignment `+= -= *= /= %=` — a single two-char token
1452 // (before the generic punct arm so `+` etc. don't split first).
1453 // `-=` is distinct from `->` (next is `=`, not `>`); `/=` from `//`.
1454 '+' | '-' | '*' | '/' | '%'
1455 if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' =>
1456 {
1457 if !current_word.is_empty() {
1458 items.push(WordItem {
1459 word: std::mem::take(&mut current_word),
1460 trailing_punct: None,
1461 start: word_start,
1462 end: i,
1463 punct_pos: None,
1464 });
1465 }
1466 items.push(WordItem {
1467 word: format!("{c}="),
1468 trailing_punct: None,
1469 start: i,
1470 end: i + 2,
1471 punct_pos: None,
1472 });
1473 skip_count = 1;
1474 word_start = i + 2;
1475 }
1476 '(' | ')' | '[' | ']' | '{' | '}' | '|' | '~' | '^' | ',' | '?' | '!' | ':'
1477 | '+' | '-' | '*' | '/' | '%' | '<' | '>' | '=' => {
1478 match c {
1479 // `{…}` literals are bracket contexts for the comma
1480 // rule too: `{1,2}` is a two-element set, not `{12}`.
1481 '[' | '{' => bracket_depth += 1,
1482 ']' | '}' => bracket_depth = bracket_depth.saturating_sub(1),
1483 _ => {}
1484 }
1485 if !current_word.is_empty() {
1486 items.push(WordItem {
1487 word: std::mem::take(&mut current_word),
1488 trailing_punct: Some(c),
1489 start: word_start,
1490 end: i,
1491 punct_pos: Some(i),
1492 });
1493 } else {
1494 items.push(WordItem {
1495 word: String::new(),
1496 trailing_punct: Some(c),
1497 start: i,
1498 end: next_pos,
1499 punct_pos: Some(i),
1500 });
1501 }
1502 word_start = next_pos;
1503 }
1504 '\'' => {
1505 // Handle contractions: expand "don't" → "do" + "not", etc.
1506 let remaining: String = chars[char_idx + 1..].iter().collect();
1507 let remaining_lower = remaining.to_lowercase();
1508
1509 // Possessive on a bracketed sub-expression: `foo(x)'s field`,
1510 // `items[0]'s value`. The receiver was already flushed by its
1511 // closing `)`/`]`, so `current_word` is empty here — mirror the
1512 // `(e).m` dot rule (`prev_close`) and emit a possessive marker,
1513 // consuming the clitic `s`. Field access after a value is always
1514 // the genitive, never the `is`-clitic (that attaches only to
1515 // pronouns / wh-words, which never end in `)`/`]`).
1516 let prev_close = current_word.is_empty()
1517 && char_idx > 0
1518 && matches!(chars[char_idx - 1], ')' | ']');
1519 let clitic_s = matches!(chars.get(char_idx + 1), Some('s' | 'S'))
1520 && chars.get(char_idx + 2).map_or(true, |c| !c.is_alphabetic());
1521 if prev_close && clitic_s {
1522 items.push(WordItem {
1523 word: "\x00POSS".to_string(),
1524 trailing_punct: None,
1525 start: i,
1526 end: next_pos + 1,
1527 punct_pos: None,
1528 });
1529 word_start = next_pos + 1;
1530 skip_count = 1;
1531 } else if remaining_lower.starts_with("t ")
1532 || remaining_lower.starts_with("t.")
1533 || remaining_lower.starts_with("t,")
1534 || remaining_lower == "t"
1535 || (char_idx + 1 < chars.len()
1536 && chars[char_idx + 1] == 't'
1537 && (char_idx + 2 >= chars.len()
1538 || !chars[char_idx + 2].is_alphabetic()))
1539 {
1540 // The splitter broke the word at the apostrophe, so
1541 // `current_word` is the n't-contraction STEM ("isn",
1542 // "won"). Which stems contract — and what they expand
1543 // to ("won" is suppletive: "will not") — is lexical
1544 // knowledge, so the lexicon owns the table; here we
1545 // only apply its expansion.
1546 let word_lower = current_word.to_lowercase();
1547 if let Some(expansion) =
1548 crate::lexicon::lookup_negative_contraction(&word_lower)
1549 {
1550 let words: Vec<&str> = expansion.split_whitespace().collect();
1551 let last_idx = words.len().saturating_sub(1);
1552 for (idx, part) in words.iter().enumerate() {
1553 items.push(WordItem {
1554 word: (*part).to_string(),
1555 trailing_punct: None,
1556 start: if idx == 0 { word_start } else { i },
1557 end: if idx == last_idx { i + 2 } else { i },
1558 punct_pos: None,
1559 });
1560 }
1561 current_word.clear();
1562 word_start = next_pos + 1;
1563 skip_count = 1;
1564 } else {
1565 // Unknown contraction, split normally
1566 if !current_word.is_empty() {
1567 items.push(WordItem {
1568 word: std::mem::take(&mut current_word),
1569 trailing_punct: Some('\''),
1570 start: word_start,
1571 end: i,
1572 punct_pos: Some(i),
1573 });
1574 }
1575 word_start = next_pos;
1576 }
1577 } else {
1578 // Not a 't contraction, handle normally
1579 if !current_word.is_empty() {
1580 items.push(WordItem {
1581 word: std::mem::take(&mut current_word),
1582 trailing_punct: Some('\''),
1583 start: word_start,
1584 end: i,
1585 punct_pos: Some(i),
1586 });
1587 }
1588 word_start = next_pos;
1589 }
1590 }
1591 // Currency-symbol money literal: a `$ € £ ¥` directly before a digit starts a money
1592 // word (`$19.99`, `€5`, `¥100`), so the symbol survives into the word for
1593 // `classify_with_lookahead` to read as `MoneyLiteral`. A lone or non-numeric symbol
1594 // (`$ each`) still falls through to the default arm and is dropped, as before.
1595 c if Self::is_currency_symbol(c)
1596 && char_idx + 1 < chars.len()
1597 && chars[char_idx + 1].is_ascii_digit() =>
1598 {
1599 if !current_word.is_empty() {
1600 items.push(WordItem {
1601 word: std::mem::take(&mut current_word),
1602 trailing_punct: None,
1603 start: word_start,
1604 end: i,
1605 punct_pos: None,
1606 });
1607 }
1608 word_start = i;
1609 current_word.push(c);
1610 }
1611 c if c.is_alphabetic()
1612 || c.is_ascii_digit()
1613 || (c == '.'
1614 && !current_word.is_empty()
1615 && current_word.chars().all(|ch| ch.is_ascii_digit()))
1616 || c == '_' =>
1617 {
1618 if current_word.is_empty() {
1619 word_start = i;
1620 }
1621 current_word.push(c);
1622 }
1623 '&' => {
1624 // "&" is COLOR coordination ("black & red" → emit "and" →
1625 // Black ∧ Red) when its neighbour is LOWERCASE, but a FIRM-NAME
1626 // joiner ("Leach & Mccall", "Ingram & Kemp") when CAPITALIZED —
1627 // there it is dropped so the proper-name absorber joins the
1628 // names (Leach_Mccall), the prior behaviour. Decide on the next
1629 // word's case.
1630 if !current_word.is_empty() {
1631 items.push(WordItem {
1632 word: std::mem::take(&mut current_word),
1633 trailing_punct: None,
1634 start: word_start,
1635 end: i,
1636 punct_pos: None,
1637 });
1638 }
1639 let next_cap = chars[char_idx + 1..]
1640 .iter()
1641 .find(|ch| !ch.is_whitespace())
1642 .map_or(false, |ch| ch.is_ascii_uppercase());
1643 if !next_cap {
1644 // Mode-deferred: `classify_with_lookahead` resolves the
1645 // marker to prose "and" (Declarative) or the bitwise
1646 // `&` operator (Imperative).
1647 items.push(WordItem {
1648 word: "\x00AMP".to_string(),
1649 trailing_punct: None,
1650 start: i,
1651 end: next_pos,
1652 punct_pos: None,
1653 });
1654 }
1655 word_start = next_pos;
1656 }
1657 _ => {
1658 word_start = next_pos;
1659 }
1660 }
1661 char_idx += 1;
1662 }
1663
1664 if !current_word.is_empty() {
1665 items.push(WordItem {
1666 word: current_word,
1667 trailing_punct: None,
1668 start: word_start,
1669 end: input.len(),
1670 punct_pos: None,
1671 });
1672 }
1673
1674 items
1675 }
1676
1677 fn peek_word(&self, offset: usize) -> Option<&str> {
1678 self.words.get(self.pos + offset).map(|w| w.word.as_str())
1679 }
1680
1681 /// Check if the previous word is a determiner (every, each, some, all, any, no, the, a, an).
1682 fn prev_token_is_determiner(&self) -> bool {
1683 if self.pos == 0 {
1684 return false;
1685 }
1686 if let Some(prev) = self.words.get(self.pos - 1) {
1687 matches!(
1688 prev.word.to_lowercase().as_str(),
1689 "every" | "each" | "some" | "all" | "any" | "no" | "the" | "a" | "an"
1690 )
1691 } else {
1692 false
1693 }
1694 }
1695
1696 /// Whether the previous word closes a sentence (ends with `.`/`!`/`?`), so the
1697 /// current word is sentence-initial. Used to tell a capitalized modal that opens a
1698 /// question ("Will Alice win?") from a capitalized proper name mid-clause ("started
1699 /// by Will Waters") — a function word never capitalizes mid-sentence.
1700 fn prev_word_ends_sentence(&self) -> bool {
1701 if self.pos == 0 {
1702 return true;
1703 }
1704 self.words
1705 .get(self.pos - 1)
1706 .and_then(|w| w.word.chars().last())
1707 .map_or(true, |c| matches!(c, '.' | '!' | '?' | ':' | ';'))
1708 }
1709
1710 /// Whether the previous word is a bare number ("30 minutes", "1 second"),
1711 /// used to disambiguate the singular clock units "second"/"minute" (which are
1712 /// also an ordinal and an adjective) — they are time units only after a count.
1713 fn prev_word_is_numeric(&self) -> bool {
1714 if self.pos == 0 {
1715 return false;
1716 }
1717 self.words
1718 .get(self.pos - 1)
1719 .map(|p| {
1720 let w = p.word.trim_start_matches('$').replace(',', "");
1721 !w.is_empty() && w.chars().all(|c| c.is_ascii_digit() || c == '.')
1722 })
1723 .unwrap_or(false)
1724 }
1725
1726 fn next_token_is_copula(&self) -> bool {
1727 if let Some(next) = self.peek_word(1) {
1728 matches!(next.to_lowercase().as_str(), "is" | "are" | "was" | "were")
1729 } else {
1730 false
1731 }
1732 }
1733
1734 fn peek_sequence(&self, expected: &[&str]) -> bool {
1735 for (i, &exp) in expected.iter().enumerate() {
1736 match self.peek_word(i + 1) {
1737 Some(w) if w.to_lowercase() == exp => continue,
1738 _ => return false,
1739 }
1740 }
1741 true
1742 }
1743
1744 fn consume_words(&mut self, count: usize) {
1745 self.pos += count;
1746 }
1747
1748 /// Tokenizes the input text and returns a vector of [`Token`]s.
1749 ///
1750 /// Each token includes its type, the interned lexeme, and the source
1751 /// span for error reporting. Words are classified according to the
1752 /// lexicon database with priority-based ambiguity resolution.
1753 ///
1754 /// # Returns
1755 ///
1756 /// A vector of tokens representing the input. The final token is
1757 /// typically `TokenType::Eof`.
1758 pub fn tokenize(&mut self) -> Vec<Token> {
1759 let mut tokens = Vec::new();
1760 // Markdown-table row tracking: a table row is a LINE construct, but the
1761 // word stream erases line breaks. While lexing a `|`-led line, the
1762 // first item past a real newline gets an explicit Newline token so the
1763 // parser can see where a row ends (case tables, the in-language test
1764 // framework).
1765 let mut prev_item_end: Option<usize> = None;
1766 let mut in_table_line = false;
1767
1768 while self.pos < self.words.len() {
1769 let item = &self.words[self.pos];
1770 let word = item.word.clone();
1771 let trailing_punct = item.trailing_punct;
1772 let word_start = item.start;
1773 let word_end = item.end;
1774 let punct_pos = item.punct_pos;
1775
1776 if matches!(self.mode, LexerMode::Imperative) {
1777 let gap_has_newline = prev_item_end
1778 .map(|pe| {
1779 self.source
1780 .get(pe..word_start)
1781 .is_some_and(|gap| gap.contains('\n'))
1782 })
1783 .unwrap_or(false);
1784 if in_table_line && gap_has_newline {
1785 let sym = self.interner.intern("");
1786 tokens.push(Token::new(
1787 TokenType::Newline,
1788 sym,
1789 Span::new(word_start, word_start),
1790 ));
1791 in_table_line = false;
1792 }
1793 let is_pipe = word.is_empty() && trailing_punct == Some('|');
1794 if is_pipe && !in_table_line && (gap_has_newline || prev_item_end.is_none()) {
1795 in_table_line = true;
1796 }
1797 }
1798 prev_item_end = Some(word_end);
1799
1800 if word.is_empty() {
1801 if let Some(punct) = trailing_punct {
1802 let kind = match punct {
1803 '(' => TokenType::LParen,
1804 ')' => TokenType::RParen,
1805 '[' => TokenType::LBracket,
1806 ']' => TokenType::RBracket,
1807 '{' => TokenType::LBrace,
1808 '}' => TokenType::RBrace,
1809 // Bitwise symbols exist only in IMPERATIVE code; in
1810 // prose they stay dropped (the old behavior).
1811 '|' if matches!(self.mode, LexerMode::Imperative) => TokenType::VBar,
1812 '~' if matches!(self.mode, LexerMode::Imperative) => TokenType::Tilde,
1813 '^' if matches!(self.mode, LexerMode::Imperative) => TokenType::Caret,
1814 ',' => TokenType::Comma,
1815 ':' => TokenType::Colon,
1816 '.' | '?' => {
1817 self.in_let_context = false;
1818 TokenType::Period
1819 }
1820 '!' => TokenType::Exclamation,
1821 '+' => TokenType::Plus,
1822 '-' => TokenType::Minus,
1823 '*' => TokenType::Star,
1824 '/' => TokenType::Slash,
1825 '%' => TokenType::Percent,
1826 '<' => TokenType::Lt,
1827 '>' => TokenType::Gt,
1828 '=' => TokenType::Assign,
1829 _ => {
1830 self.pos += 1;
1831 continue;
1832 }
1833 };
1834 let lexeme = self.interner.intern(&punct.to_string());
1835 let span = Span::new(word_start, word_end);
1836 // Collapse consecutive Periods — an abbreviation's own period
1837 // followed by the sentence period ("Dorsey Assoc.." → "Assoc"
1838 // + one Period). The extra "." would otherwise strand the parse.
1839 let dup_period = kind == TokenType::Period
1840 && tokens
1841 .last()
1842 .map_or(false, |t: &Token| t.kind == TokenType::Period);
1843 if !dup_period {
1844 tokens.push(Token::new(kind, lexeme, span));
1845 }
1846 }
1847 self.pos += 1;
1848 continue;
1849 }
1850
1851 // Raw string literal (`r"..."`): verbatim — braces are literal, so
1852 // neither the interpolation scan nor the `{{`→`{` collapse applies.
1853 if word.starts_with("\x00RAW:") {
1854 let content = &word[5..];
1855 let span = Span::new(word_start, word_end);
1856 let sym = self.interner.intern(content);
1857 tokens.push(Token::new(TokenType::StringLiteral(sym), sym, span));
1858 self.pos += 1;
1859 continue;
1860 }
1861
1862 // Check for string literal marker (pre-tokenized in Stage 1)
1863 if word.starts_with("\x00STR:") {
1864 let content = &word[5..]; // Skip the marker prefix
1865 let span = Span::new(word_start, word_end);
1866 if Self::has_unescaped_brace(content) {
1867 let sym = self.interner.intern(content);
1868 tokens.push(Token::new(TokenType::InterpolatedString(sym), sym, span));
1869 } else {
1870 // Collapse {{ → { and }} → } for plain strings
1871 let normalized = content.replace("{{", "{").replace("}}", "}");
1872 let sym = self.interner.intern(&normalized);
1873 tokens.push(Token::new(TokenType::StringLiteral(sym), sym, span));
1874 }
1875 self.pos += 1;
1876 continue;
1877 }
1878
1879 // Check for character literal marker
1880 if word.starts_with("\x00CHAR:") {
1881 let content = &word[6..]; // Skip the marker prefix
1882 let sym = self.interner.intern(content);
1883 let span = Span::new(word_start, word_end);
1884 tokens.push(Token::new(TokenType::CharLiteral(sym), sym, span));
1885 self.pos += 1;
1886 continue;
1887 }
1888
1889 // An LGX raw section (pre-captured `View:` / `Style:` body): `\x00LGX:<kind>:<offset>:<body>`.
1890 if let Some(rest) = word.strip_prefix("\x00LGX:") {
1891 let mut parts = rest.splitn(4, ':');
1892 let kind = match parts.next() {
1893 Some("S") => RawKind::Style,
1894 Some("H") => RawKind::Head,
1895 _ => RawKind::View,
1896 };
1897 let content_start: usize = parts
1898 .next()
1899 .and_then(|n| n.parse().ok())
1900 .unwrap_or(word_start);
1901 let indent: usize = parts.next().and_then(|n| n.parse().ok()).unwrap_or(0);
1902 let body = parts.next().unwrap_or("");
1903 let sym = self.interner.intern(body);
1904 tokens.push(Token::new(
1905 TokenType::RawBlock {
1906 kind,
1907 body: sym,
1908 content_start,
1909 indent,
1910 },
1911 sym,
1912 Span::new(word_start, word_end),
1913 ));
1914 self.pos += 1;
1915 continue;
1916 }
1917 // Check for escape block marker (pre-captured raw foreign code)
1918 if word.starts_with("\x00ESC:") {
1919 let content = &word[5..]; // Skip the "\x00ESC:" prefix
1920 let sym = self.interner.intern(content);
1921 let span = Span::new(word_start, word_end);
1922 tokens.push(Token::new(TokenType::EscapeBlock(sym), sym, span));
1923 self.pos += 1;
1924 continue;
1925 }
1926
1927 // "exactly N" / "precisely N" — a redundant exactness marker on a following
1928 // count or measure. The count/measure is already exact in the FOL (a value,
1929 // not a ≥/≤ bound), so the word adds no constraint; dropping it lets the
1930 // count object parse ("serves exactly 2 people" ≡ "serves 2 people") with
1931 // zero meaning loss. (Approximative "about N" is a Preposition and keeps its
1932 // own About-relation; "at least/most N" are handled separately as bounds.)
1933 if matches!(word.to_lowercase().as_str(), "exactly" | "precisely")
1934 && self.peek_word(1).map_or(false, |w| {
1935 let w = w.trim_start_matches('$');
1936 w.chars().next().map_or(false, |c| c.is_ascii_digit())
1937 || crate::lexicon::word_to_number(&w.to_lowercase()).is_some()
1938 })
1939 {
1940 self.pos += 1;
1941 continue;
1942 }
1943
1944 let first_word = self.pos;
1945 let kind = self.classify_with_lookahead(&word);
1946 // A token read from several words ("at least three", "all but finitely many") covers
1947 // its whole phrase: its lexeme and span run to its last word, and the punctuation
1948 // after that word follows it. A block header's further words are its data (a route's
1949 // method and path, a test's name), not part of its phrase.
1950 let multi_word = self.pos > first_word && !matches!(kind, TokenType::BlockHeader { .. });
1951 let (word, trailing_punct, word_end, punct_pos) = if multi_word {
1952 let last = &self.words[self.pos];
1953 let phrase = self
1954 .source
1955 .get(word_start..last.end)
1956 .unwrap_or(&word)
1957 .to_string();
1958 (phrase, last.trailing_punct, last.end, last.punct_pos)
1959 } else {
1960 (word, trailing_punct, word_end, punct_pos)
1961 };
1962 // A subject pronoun or WH-relativizer takes the copula clitic "'s" = "is"
1963 // ("who's going", "he's …"), never the possessive 's (their genitive is the
1964 // dedicated form whose / its / his). Captured from the host's lexical
1965 // CATEGORY before `kind` is moved into the token, so the contraction rule
1966 // below is data-driven, not a word list.
1967 let host_takes_copula_clitic = matches!(
1968 kind,
1969 TokenType::Pronoun {
1970 case: crate::lexicon::Case::Subject,
1971 ..
1972 } | TokenType::Who
1973 | TokenType::That
1974 | TokenType::What
1975 | TokenType::Where
1976 );
1977 let lexeme = self.interner.intern(&word);
1978 let span = Span::new(word_start, word_end);
1979 tokens.push(Token::new(kind, lexeme, span));
1980
1981 if let Some(punct) = trailing_punct {
1982 if punct == '\'' {
1983 if let Some(next_item) = self.words.get(self.pos + 1) {
1984 if next_item.word.to_lowercase() == "s" {
1985 // The contraction "'s" = "is" iff the host takes the copula
1986 // clitic AND the next word is not a past-participle/non-
1987 // progressive verb — that case is the perfect "has"
1988 // ("he's found/been"), a separate unsupported feature, and
1989 // emitting "is" there would misread it as the passive "is
1990 // found". Verb aspect comes from the lexicon (data-driven).
1991 let next_is_past_verb = self
1992 .words
1993 .get(self.pos + 2)
1994 .map(|a| a.word.to_lowercase())
1995 .and_then(|w| self.lexicon.lookup_verb(&w))
1996 .map_or(false, |v| v.aspect != crate::lexicon::Aspect::Progressive);
1997 let (poss_kind, poss_text) =
1998 if host_takes_copula_clitic && !next_is_past_verb {
1999 (TokenType::Is, "is")
2000 } else {
2001 (TokenType::Possessive, "'s")
2002 };
2003 let poss_lexeme = self.interner.intern(poss_text);
2004 let poss_start = punct_pos.unwrap_or(word_end);
2005 let poss_end = next_item.end;
2006 tokens.push(Token::new(
2007 poss_kind,
2008 poss_lexeme,
2009 Span::new(poss_start, poss_end),
2010 ));
2011 self.pos += 1;
2012 if let Some(s_punct) = next_item.trailing_punct {
2013 let kind = match s_punct {
2014 '(' => TokenType::LParen,
2015 ')' => TokenType::RParen,
2016 '[' => TokenType::LBracket,
2017 ']' => TokenType::RBracket,
2018 '{' => TokenType::LBrace,
2019 '}' => TokenType::RBrace,
2020 '|' if matches!(self.mode, LexerMode::Imperative) => {
2021 TokenType::VBar
2022 }
2023 '~' if matches!(self.mode, LexerMode::Imperative) => {
2024 TokenType::Tilde
2025 }
2026 '^' if matches!(self.mode, LexerMode::Imperative) => {
2027 TokenType::Caret
2028 }
2029 ',' => TokenType::Comma,
2030 ':' => TokenType::Colon,
2031 '.' | '?' => TokenType::Period,
2032 '!' => TokenType::Exclamation,
2033 '+' => TokenType::Plus,
2034 '-' => TokenType::Minus,
2035 '*' => TokenType::Star,
2036 '/' => TokenType::Slash,
2037 '%' => TokenType::Percent,
2038 '<' => TokenType::Lt,
2039 '>' => TokenType::Gt,
2040 '=' => TokenType::Assign,
2041 _ => {
2042 self.pos += 1;
2043 continue;
2044 }
2045 };
2046 let s_punct_pos = next_item.punct_pos.unwrap_or(next_item.end);
2047 let lexeme = self.interner.intern(&s_punct.to_string());
2048 tokens.push(Token::new(
2049 kind,
2050 lexeme,
2051 Span::new(s_punct_pos, s_punct_pos + 1),
2052 ));
2053 }
2054 self.pos += 1;
2055 continue;
2056 }
2057 }
2058 self.pos += 1;
2059 continue;
2060 }
2061
2062 // An abbreviation's own dot ("Mr.", "Dr.", "153 ft.", "Mt.") is NOT a
2063 // sentence terminator when more text follows — emitting a Period there
2064 // strands the rest of the clause. Which words abbreviate is lexical
2065 // (lexicon `abbreviations`); suppress the dot only mid-clue, so a
2066 // genuine clue-final abbreviation ("…on Main St.") keeps its terminator.
2067 // Prose only: in code every word is an identifier and a full stop ends
2068 // its statement — `Return m.` is a sentence about a variable named `m`.
2069 if punct == '.'
2070 && self.mode != LexerMode::Imperative
2071 && lexicon::is_abbreviation(&word.to_lowercase())
2072 && self.words.get(self.pos + 1).is_some()
2073 {
2074 self.pos += 1;
2075 continue;
2076 }
2077 let kind = match punct {
2078 '(' => TokenType::LParen,
2079 ')' => TokenType::RParen,
2080 '[' => TokenType::LBracket,
2081 ']' => TokenType::RBracket,
2082 '{' => TokenType::LBrace,
2083 '}' => TokenType::RBrace,
2084 '|' if matches!(self.mode, LexerMode::Imperative) => TokenType::VBar,
2085 '~' if matches!(self.mode, LexerMode::Imperative) => TokenType::Tilde,
2086 '^' if matches!(self.mode, LexerMode::Imperative) => TokenType::Caret,
2087 ',' => TokenType::Comma,
2088 ':' => TokenType::Colon,
2089 '.' | '?' => {
2090 self.in_let_context = false;
2091 TokenType::Period
2092 }
2093 '!' => TokenType::Exclamation,
2094 '+' => TokenType::Plus,
2095 '-' => TokenType::Minus,
2096 '*' => TokenType::Star,
2097 '/' => TokenType::Slash,
2098 '%' => TokenType::Percent,
2099 '<' => TokenType::Lt,
2100 '>' => TokenType::Gt,
2101 '=' => TokenType::Assign,
2102 _ => {
2103 self.pos += 1;
2104 continue;
2105 }
2106 };
2107 let p_start = punct_pos.unwrap_or(word_end);
2108 let lexeme = self.interner.intern(&punct.to_string());
2109 tokens.push(Token::new(kind, lexeme, Span::new(p_start, p_start + 1)));
2110 }
2111
2112 self.pos += 1;
2113 }
2114
2115 let eof_lexeme = self.interner.intern("");
2116 let eof_span = Span::new(self.input_len, self.input_len);
2117 tokens.push(Token::new(TokenType::EOF, eof_lexeme, eof_span));
2118
2119 self.insert_indentation_tokens(tokens)
2120 }
2121
2122 /// Insert Indent/Dedent tokens using LineLexer's two-pass architecture (Spec §2.5.2).
2123 ///
2124 /// Phase 1: LineLexer determines the structural layout (where indents/dedents occur)
2125 /// Phase 2: We correlate these with word token positions
2126 fn insert_indentation_tokens(&mut self, tokens: Vec<Token>) -> Vec<Token> {
2127 let mut result = Vec::new();
2128 let empty_sym = self.interner.intern("");
2129
2130 // Phase 1: Run LineLexer to determine structural positions
2131 let line_lexer = LineLexer::new(&self.source);
2132 let line_tokens: Vec<LineToken> = line_lexer.collect();
2133
2134 // Build a list of (byte_position, is_indent) for structural tokens
2135 // Position is where the NEXT Content starts after the Indent/Dedent
2136 let mut structural_events: Vec<(usize, bool)> = Vec::new(); // (byte_pos, true=Indent, false=Dedent)
2137 let mut pending_indents = 0usize;
2138 let mut pending_dedents = 0usize;
2139
2140 for line_token in &line_tokens {
2141 match line_token {
2142 LineToken::Indent => {
2143 pending_indents += 1;
2144 }
2145 LineToken::Dedent => {
2146 pending_dedents += 1;
2147 }
2148 LineToken::Content { start, .. } => {
2149 // Emit pending dedents first (they come BEFORE the content)
2150 for _ in 0..pending_dedents {
2151 structural_events.push((*start, false)); // false = Dedent
2152 }
2153 pending_dedents = 0;
2154
2155 // Emit pending indents (they also come BEFORE the content)
2156 for _ in 0..pending_indents {
2157 structural_events.push((*start, true)); // true = Indent
2158 }
2159 pending_indents = 0;
2160 }
2161 LineToken::Newline => {}
2162 }
2163 }
2164
2165 // Handle any remaining dedents at EOF
2166 for _ in 0..pending_dedents {
2167 structural_events.push((self.input_len, false));
2168 }
2169
2170 // Filter out structural events from within escape block bodies.
2171 // The LineLexer sees raw Rust code lines and generates spurious Indent/Dedent
2172 // events for their indentation changes. We keep exactly the boundary events
2173 // (Indent at body start, Dedent at body end) but remove internal ones.
2174 if !self.escape_body_ranges.is_empty() {
2175 // For each escape body range, find the first Indent at the body start and
2176 // track that we're inside the range. Filter out all events strictly inside
2177 // the range except for the first Indent and events at/after the end.
2178 let mut filtered = Vec::new();
2179 for &(pos, is_indent) in &structural_events {
2180 let is_inside_escape_body = self.escape_body_ranges.iter().any(|(start, end)| {
2181 // Strictly inside the body (not at start boundary and not at/after end)
2182 pos > *start && pos < *end
2183 });
2184 if !is_inside_escape_body {
2185 filtered.push((pos, is_indent));
2186 }
2187 }
2188 structural_events = filtered;
2189 }
2190
2191 // Filter out structural events from within multi-line string literals.
2192 // Triple-quote strings span multiple lines; their internal indentation
2193 // must not generate Indent/Dedent tokens.
2194 {
2195 let string_spans: Vec<(usize, usize)> = tokens
2196 .iter()
2197 .filter(|t| {
2198 matches!(
2199 t.kind,
2200 TokenType::StringLiteral(_) | TokenType::InterpolatedString(_)
2201 )
2202 })
2203 .filter(|t| t.span.end.saturating_sub(t.span.start) > 6) // only multi-line strings (""" adds >=6 chars)
2204 .map(|t| (t.span.start, t.span.end))
2205 .collect();
2206 if !string_spans.is_empty() {
2207 structural_events.retain(|&(pos, _)| {
2208 !string_spans
2209 .iter()
2210 .any(|(start, end)| pos > *start && pos < *end)
2211 });
2212 }
2213 }
2214
2215 // Bracket line-continuation: a collection literal / call / parenthesised expression may span
2216 // lines with the continuation lines INDENTED (`[\n 1,\n 2,\n]`). The LineLexer sees that
2217 // indentation and emits Indent/Dedent, which would break element parsing. Drop every
2218 // structural event that falls strictly inside an unclosed `(`/`[`/`{` … `)`/`]`/`}` span. Both
2219 // the opening Indent and the matching Dedent lie inside the span, so they are dropped as a
2220 // balanced pair — the enclosing block level is preserved.
2221 {
2222 let mut bracket_ranges: Vec<(usize, usize)> = Vec::new();
2223 let mut open_stack: Vec<usize> = Vec::new();
2224 for t in &tokens {
2225 match t.kind {
2226 TokenType::LParen | TokenType::LBracket | TokenType::LBrace => {
2227 open_stack.push(t.span.start);
2228 }
2229 TokenType::RParen | TokenType::RBracket | TokenType::RBrace => {
2230 if let Some(open) = open_stack.pop() {
2231 bracket_ranges.push((open, t.span.end));
2232 }
2233 }
2234 _ => {}
2235 }
2236 }
2237 if !bracket_ranges.is_empty() {
2238 structural_events.retain(|&(pos, _)| {
2239 !bracket_ranges
2240 .iter()
2241 .any(|(start, end)| pos > *start && pos < *end)
2242 });
2243 }
2244 }
2245
2246 // Sort events by position, with dedents before indents at same position
2247 structural_events.sort_by(|a, b| {
2248 if a.0 != b.0 {
2249 a.0.cmp(&b.0)
2250 } else {
2251 // Dedents (false) before Indents (true) at same position
2252 a.1.cmp(&b.1)
2253 }
2254 });
2255
2256 // Phase 2: Insert structural tokens at the right positions
2257 // Strategy: For each word token, check if any structural events should be inserted
2258 // before it (based on byte position)
2259
2260 let mut event_idx = 0;
2261 let mut last_colon_pos: Option<usize> = None;
2262
2263 for token in tokens.iter() {
2264 let token_start = token.span.start;
2265
2266 // Insert any structural tokens that should come BEFORE this token
2267 while event_idx < structural_events.len() {
2268 let (event_pos, is_indent) = structural_events[event_idx];
2269
2270 // Insert structural tokens before this token if the event position <= token start
2271 if event_pos <= token_start {
2272 let span = if is_indent {
2273 // Indent is inserted after the preceding Colon
2274 Span::new(
2275 last_colon_pos.unwrap_or(event_pos),
2276 last_colon_pos.unwrap_or(event_pos),
2277 )
2278 } else {
2279 Span::new(event_pos, event_pos)
2280 };
2281 let kind = if is_indent {
2282 TokenType::Indent
2283 } else {
2284 TokenType::Dedent
2285 };
2286 result.push(Token::new(kind, empty_sym, span));
2287 event_idx += 1;
2288 } else {
2289 break;
2290 }
2291 }
2292
2293 result.push(token.clone());
2294
2295 // Track colon positions for Indent span calculation
2296 if token.kind == TokenType::Colon && self.is_end_of_line(token.span.end) {
2297 last_colon_pos = Some(token.span.end);
2298 }
2299 }
2300
2301 // Insert any remaining structural tokens (typically Dedents at EOF)
2302 while event_idx < structural_events.len() {
2303 let (event_pos, is_indent) = structural_events[event_idx];
2304 let span = Span::new(event_pos, event_pos);
2305 let kind = if is_indent {
2306 TokenType::Indent
2307 } else {
2308 TokenType::Dedent
2309 };
2310 result.push(Token::new(kind, empty_sym, span));
2311 event_idx += 1;
2312 }
2313
2314 // Ensure EOF is at the end
2315 let eof_pos = result.iter().position(|t| t.kind == TokenType::EOF);
2316 if let Some(pos) = eof_pos {
2317 let eof = result.remove(pos);
2318 result.push(eof);
2319 }
2320
2321 result
2322 }
2323
2324 /// Check if position is at end of line (only whitespace until newline)
2325 fn is_end_of_line(&self, from_pos: usize) -> bool {
2326 let bytes = self.source.as_bytes();
2327 let mut pos = from_pos;
2328 while pos < bytes.len() {
2329 match bytes[pos] {
2330 b' ' | b'\t' => pos += 1,
2331 b'\n' => return true,
2332 _ => return false,
2333 }
2334 }
2335 true // End of input is also end of line
2336 }
2337
2338 fn measure_next_line_indent(&self, from_pos: usize) -> Option<usize> {
2339 let bytes = self.source.as_bytes();
2340 let mut pos = from_pos;
2341
2342 while pos < bytes.len() && bytes[pos] != b'\n' {
2343 pos += 1;
2344 }
2345
2346 if pos >= bytes.len() {
2347 return None;
2348 }
2349
2350 pos += 1;
2351
2352 let mut indent = 0;
2353 while pos < bytes.len() {
2354 match bytes[pos] {
2355 b' ' => indent += 1,
2356 b'\t' => indent += 4,
2357 b'\n' => {
2358 indent = 0;
2359 }
2360 _ => break,
2361 }
2362 pos += 1;
2363 }
2364
2365 if pos >= bytes.len() {
2366 return None;
2367 }
2368
2369 Some(indent)
2370 }
2371
2372 fn word_to_number(word: &str) -> Option<u32> {
2373 lexicon::word_to_number(&word.to_lowercase())
2374 }
2375
2376 /// Check if a hyphen at the current position is part of an ISO-8601 date.
2377 ///
2378 /// Detects patterns like:
2379 /// - "2026-" followed by "05-20" → first hyphen of date
2380 /// - "2026-05-" followed by "20" → second hyphen of date
2381 fn is_date_hyphen(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2382 // Current word must be all digits (year or year-month)
2383 let word_chars: Vec<char> = current_word.chars().collect();
2384
2385 // Check for first hyphen pattern: YYYY- followed by MM-DD
2386 if word_chars.len() == 4 && word_chars.iter().all(|c| c.is_ascii_digit()) {
2387 // Check if followed by exactly 2 digits, hyphen, 2 digits
2388 if char_idx + 5 < chars.len()
2389 && chars[char_idx + 1].is_ascii_digit()
2390 && chars[char_idx + 2].is_ascii_digit()
2391 && chars[char_idx + 3] == '-'
2392 && chars[char_idx + 4].is_ascii_digit()
2393 && chars[char_idx + 5].is_ascii_digit()
2394 {
2395 return true;
2396 }
2397 }
2398
2399 // Check for second hyphen pattern: YYYY-MM- followed by DD
2400 if word_chars.len() == 7
2401 && word_chars[0..4].iter().all(|c| c.is_ascii_digit())
2402 && word_chars[4] == '-'
2403 && word_chars[5..7].iter().all(|c| c.is_ascii_digit())
2404 {
2405 // Check if followed by exactly 2 digits
2406 if char_idx + 2 < chars.len()
2407 && chars[char_idx + 1].is_ascii_digit()
2408 && chars[char_idx + 2].is_ascii_digit()
2409 {
2410 // Make sure we're not followed by more digits (would be a longer number)
2411 let next_not_digit =
2412 char_idx + 3 >= chars.len() || !chars[char_idx + 3].is_ascii_digit();
2413 if next_not_digit {
2414 return true;
2415 }
2416 }
2417 }
2418
2419 false
2420 }
2421
2422 /// Check if a colon is part of a time literal (e.g., 9:30am, 11:45pm).
2423 ///
2424 /// Detects patterns like:
2425 /// - "9:" followed by "30am" or "30pm"
2426 /// - "11:" followed by "45pm"
2427 fn is_time_colon(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2428 // Current word must be 1-2 digits (hour)
2429 let word_chars: Vec<char> = current_word.chars().collect();
2430 if word_chars.is_empty() || word_chars.len() > 2 {
2431 return false;
2432 }
2433 if !word_chars.iter().all(|c| c.is_ascii_digit()) {
2434 return false;
2435 }
2436
2437 // Followed by exactly 2 digits (the minutes), then "am"/"pm" — which may
2438 // come immediately ("9:30am") or after a space ("8:15 pm").
2439 if char_idx + 3 < chars.len()
2440 && chars[char_idx + 1].is_ascii_digit()
2441 && chars[char_idx + 2].is_ascii_digit()
2442 {
2443 let suffix_start = if chars.get(char_idx + 3) == Some(&' ') {
2444 char_idx + 4
2445 } else {
2446 char_idx + 3
2447 };
2448 if suffix_start + 1 < chars.len() {
2449 let next_two: String = chars[suffix_start..suffix_start + 2].iter().collect();
2450 let lower = next_two.to_lowercase();
2451 if (lower == "am" || lower == "pm")
2452 && (suffix_start + 2 >= chars.len() || !chars[suffix_start + 2].is_alphabetic())
2453 {
2454 return true;
2455 }
2456 }
2457 }
2458
2459 false
2460 }
2461
2462 /// Whether the `:` at `char_idx` opens a `::` namespace separator inside a
2463 /// qualified type reference (`Geometry::Point`, `A::B::C`). True only when the
2464 /// colon is doubled and sits between identifier characters — an identifier
2465 /// character (letter/digit/underscore) immediately precedes the first colon and
2466 /// an identifier START (letter/underscore) immediately follows the second. This
2467 /// keeps a lone `:` a block/label colon; only the glued `Ident::Ident` form is a
2468 /// path, and the same rule chains across each `::` in a multi-segment path.
2469 fn is_namespace_colon(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2470 // The previous emitted character must be part of the left identifier segment.
2471 let prev_is_ident = current_word
2472 .chars()
2473 .next_back()
2474 .map_or(false, |c| c.is_alphanumeric() || c == '_');
2475 if !prev_is_ident {
2476 return false;
2477 }
2478 // Doubled colon.
2479 if chars.get(char_idx + 1) != Some(&':') {
2480 return false;
2481 }
2482 // The character after `::` must begin an identifier segment.
2483 chars
2484 .get(char_idx + 2)
2485 .map_or(false, |c| c.is_alphabetic() || *c == '_')
2486 }
2487
2488 /// Whether a space separates a `H:MM` clock time from a trailing "am"/"pm"
2489 /// ("8:15 pm") — the space is part of the time literal, so the tokenizer keeps
2490 /// "am"/"pm" attached to the time rather than splitting on it.
2491 fn is_time_space_before_ampm(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2492 let valid_time = match current_word.find(':') {
2493 Some(p) => {
2494 let hour = ¤t_word[..p];
2495 let min = ¤t_word[p + 1..];
2496 (1..=2).contains(&hour.len())
2497 && hour.chars().all(|c| c.is_ascii_digit())
2498 && min.len() == 2
2499 && min.chars().all(|c| c.is_ascii_digit())
2500 }
2501 None => false,
2502 };
2503 if !valid_time {
2504 return false;
2505 }
2506 let next_two: String = chars
2507 .get(char_idx + 1..char_idx + 3)
2508 .map(|s| s.iter().collect())
2509 .unwrap_or_default();
2510 let lower = next_two.to_lowercase();
2511 (lower == "am" || lower == "pm")
2512 && chars.get(char_idx + 3).map_or(true, |c| !c.is_alphabetic())
2513 }
2514
2515 /// Check if a string contains an unescaped `{` (i.e., not part of `{{`).
2516 /// Used to distinguish `InterpolatedString` from `StringLiteral`.
2517 fn has_unescaped_brace(content: &str) -> bool {
2518 let bytes = content.as_bytes();
2519 let mut i = 0;
2520 while i < bytes.len() {
2521 if bytes[i] == b'{' {
2522 if i + 1 < bytes.len() && bytes[i + 1] == b'{' {
2523 i += 2;
2524 } else {
2525 return true;
2526 }
2527 } else {
2528 i += 1;
2529 }
2530 }
2531 false
2532 }
2533
2534 /// Check if a `+` or `-` at the current position is the sign of a scientific notation exponent.
2535 ///
2536 /// Detects patterns like:
2537 /// - "4.84e+" followed by "00" → exponent sign in `4.84e+00`
2538 /// - "2.5e-" followed by "2" → exponent sign in `2.5e-2`
2539 fn is_exponent_sign(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2540 // Word must end with e/E
2541 if !current_word.ends_with('e') && !current_word.ends_with('E') {
2542 return false;
2543 }
2544 // Before e/E must contain a digit (ensures it's a number, not a bare "e")
2545 let before_e = ¤t_word[..current_word.len() - 1];
2546 if before_e.is_empty() || !before_e.chars().next().unwrap().is_ascii_digit() {
2547 return false;
2548 }
2549 // Next char must be a digit (the exponent value)
2550 char_idx + 1 < chars.len() && chars[char_idx + 1].is_ascii_digit()
2551 }
2552
2553 /// Dedent a triple-quoted string: strip the common leading whitespace from each line.
2554 /// Joins lines with literal newline characters (not escape sequences).
2555 fn dedent_triple_quote(raw: &str) -> String {
2556 let lines: Vec<&str> = raw.lines().collect();
2557 if lines.is_empty() {
2558 return String::new();
2559 }
2560 // Find minimum indentation of non-empty lines
2561 let min_indent = lines
2562 .iter()
2563 .filter(|l| !l.trim().is_empty())
2564 .map(|l| l.len() - l.trim_start().len())
2565 .min()
2566 .unwrap_or(0);
2567 // Strip that indentation and join with actual newlines
2568 lines
2569 .iter()
2570 .map(|l| {
2571 if l.len() >= min_indent {
2572 &l[min_indent..]
2573 } else {
2574 l.trim()
2575 }
2576 })
2577 .collect::<Vec<_>>()
2578 .join("\n")
2579 }
2580
2581 fn is_numeric_literal(word: &str) -> bool {
2582 if word.is_empty() {
2583 return false;
2584 }
2585 let chars: Vec<char> = word.chars().collect();
2586 let first = chars[0];
2587 if first.is_ascii_digit() {
2588 // Numeric literal: starts with digit (may have underscore separators like 1_000)
2589 return true;
2590 }
2591 // Symbolic numbers: only recognize known mathematical symbols
2592 // (aleph, omega, beth) followed by underscore and digits
2593 if let Some(underscore_pos) = word.rfind('_') {
2594 let before_underscore = &word[..underscore_pos];
2595 let after_underscore = &word[underscore_pos + 1..];
2596 // Must be a known mathematical symbol prefix AND digits after underscore
2597 let is_math_symbol = matches!(
2598 before_underscore.to_lowercase().as_str(),
2599 "aleph" | "omega" | "beth"
2600 );
2601 if is_math_symbol
2602 && !after_underscore.is_empty()
2603 && after_underscore.chars().all(|c| c.is_ascii_digit())
2604 {
2605 return true;
2606 }
2607 }
2608 false
2609 }
2610
2611 /// Parse a duration literal with SI suffix.
2612 ///
2613 /// Returns Some((nanoseconds, unit_str)) if the word is a valid duration literal,
2614 /// None otherwise.
2615 ///
2616 /// Supported suffixes:
2617 /// - ns: nanoseconds
2618 /// - us, μs: microseconds
2619 /// - ms: milliseconds
2620 /// - s, sec: seconds
2621 /// - min: minutes
2622 /// - h, hr: hours
2623 fn parse_duration_literal(word: &str) -> Option<(i64, &str)> {
2624 if word.is_empty() || !word.chars().next()?.is_ascii_digit() {
2625 return None;
2626 }
2627
2628 // SI suffix table with multipliers to nanoseconds
2629 const SUFFIXES: &[(&str, i64)] = &[
2630 ("ns", 1),
2631 ("μs", 1_000),
2632 ("us", 1_000),
2633 ("ms", 1_000_000),
2634 ("sec", 1_000_000_000),
2635 ("s", 1_000_000_000),
2636 ("min", 60_000_000_000),
2637 ("hr", 3_600_000_000_000),
2638 ("h", 3_600_000_000_000),
2639 ];
2640
2641 // Try each suffix (longer suffixes first to avoid partial matches)
2642 for (suffix, multiplier) in SUFFIXES {
2643 if word.ends_with(suffix) {
2644 let num_part = &word[..word.len() - suffix.len()];
2645 // Parse the numeric part (may have underscore separators)
2646 let cleaned: String = num_part.chars().filter(|c| *c != '_').collect();
2647 if let Ok(n) = cleaned.parse::<i64>() {
2648 return Some((n.saturating_mul(*multiplier), *suffix));
2649 }
2650 }
2651 }
2652
2653 None
2654 }
2655
2656 /// Parse an ISO-8601 date literal (YYYY-MM-DD).
2657 ///
2658 /// Returns Some(days_since_epoch) if the word is a valid date literal,
2659 /// None otherwise.
2660 fn parse_date_literal(word: &str) -> Option<i32> {
2661 // Must match pattern: YYYY-MM-DD
2662 if word.len() != 10 {
2663 return None;
2664 }
2665
2666 let bytes = word.as_bytes();
2667
2668 // Check format: 4 digits, hyphen, 2 digits, hyphen, 2 digits
2669 if bytes[4] != b'-' || bytes[7] != b'-' {
2670 return None;
2671 }
2672
2673 // Parse year, month, day
2674 let year: i32 = word[0..4].parse().ok()?;
2675 let month: u32 = word[5..7].parse().ok()?;
2676 let day: u32 = word[8..10].parse().ok()?;
2677
2678 // Reject calendar-impossible dates (e.g. 2026-02-30, 2026-04-31): the
2679 // Howard Hinnant day count below would otherwise silently map them onto
2680 // a real but DIFFERENT day.
2681 if month < 1 || month > 12 || day < 1 {
2682 return None;
2683 }
2684 let is_leap = (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0);
2685 let max_day = match month {
2686 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
2687 4 | 6 | 9 | 11 => 30,
2688 2 => {
2689 if is_leap {
2690 29
2691 } else {
2692 28
2693 }
2694 }
2695 _ => return None,
2696 };
2697 if day > max_day {
2698 return None;
2699 }
2700
2701 // Convert to days since Unix epoch using Howard Hinnant's algorithm
2702 // https://howardhinnant.github.io/date_algorithms.html
2703 let y = if month <= 2 { year - 1 } else { year };
2704 let era = if y >= 0 { y / 400 } else { (y - 399) / 400 };
2705 let yoe = (y - era * 400) as u32;
2706 let m = month;
2707 let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + day - 1;
2708 let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
2709 let days = era * 146097 + doe as i32 - 719468;
2710
2711 Some(days)
2712 }
2713
2714 /// Parse a time-of-day literal.
2715 ///
2716 /// Supported formats:
2717 /// - 12-hour with am/pm: "4pm", "9am", "12pm"
2718 /// - 12-hour with minutes: "9:30am", "11:45pm"
2719 /// - Special words: "noon" (12:00), "midnight" (00:00)
2720 ///
2721 /// Returns Some(nanos_from_midnight) if valid, None otherwise.
2722 fn parse_time_literal(word: &str) -> Option<i64> {
2723 let lower = word.to_lowercase();
2724
2725 // Handle special time words
2726 if lower == "noon" {
2727 return Some(12i64 * 3600 * 1_000_000_000);
2728 }
2729 if lower == "midnight" {
2730 return Some(0);
2731 }
2732
2733 // Handle 12-hour formats: "4pm", "9am", "9:30am", "11:45pm"
2734 let is_pm = lower.ends_with("pm");
2735 let is_am = lower.ends_with("am");
2736
2737 if !is_pm && !is_am {
2738 return None;
2739 }
2740
2741 // Strip the am/pm suffix
2742 let time_part = &lower[..lower.len() - 2];
2743
2744 // Check for hour:minute format
2745 let (hour, minute): (i64, i64) = if let Some(colon_idx) = time_part.find(':') {
2746 let hour_str = &time_part[..colon_idx];
2747 let min_str = &time_part[colon_idx + 1..];
2748 let h: i64 = hour_str.parse().ok()?;
2749 let m: i64 = min_str.parse().ok()?;
2750 (h, m)
2751 } else {
2752 // Just hour: "4pm", "9am"
2753 let h: i64 = time_part.parse().ok()?;
2754 (h, 0)
2755 };
2756
2757 // Validate ranges
2758 if hour < 1 || hour > 12 || minute < 0 || minute > 59 {
2759 return None;
2760 }
2761
2762 // Convert to 24-hour format
2763 let hour_24 = if is_am {
2764 if hour == 12 {
2765 0
2766 } else {
2767 hour
2768 } // 12am = midnight = 0
2769 } else {
2770 if hour == 12 {
2771 12
2772 } else {
2773 hour + 12
2774 } // 12pm = noon = 12, 4pm = 16
2775 };
2776
2777 // Convert to nanoseconds from midnight
2778 let nanos = (hour_24 * 3600 + minute * 60) * 1_000_000_000;
2779 Some(nanos)
2780 }
2781
2782 /// True for the currency symbols that prefix a money literal (`$ € £ ¥`).
2783 fn is_currency_symbol(c: char) -> bool {
2784 matches!(c, '$' | '€' | '£' | '¥')
2785 }
2786
2787 /// The ISO-4217 code a leading currency symbol denotes — `$`→USD, `€`→EUR, `£`→GBP, `¥`→JPY (the
2788 /// dominant reading of each symbol). Unknown symbols yield `None`.
2789 fn currency_for_symbol(c: char) -> Option<&'static str> {
2790 Some(match c {
2791 '$' => "USD",
2792 '€' => "EUR",
2793 '£' => "GBP",
2794 '¥' => "JPY",
2795 _ => return None,
2796 })
2797 }
2798
2799 fn classify_with_lookahead(&mut self, word: &str) -> TokenType {
2800 // The `&` character, deferred by the word-splitter: prose keeps the
2801 // coordination reading (`black & red` → and); imperative code gets
2802 // the bitwise operator.
2803 if word == "\x00AMP" {
2804 return if matches!(self.mode, LexerMode::Imperative) {
2805 TokenType::Amp
2806 } else {
2807 TokenType::And
2808 };
2809 }
2810 if word == "\x00DOT" {
2811 // Imperative: the field-access / UFCS operator. Declarative/prose:
2812 // a plain sentence period (so `e.g.` and abbreviations read as today).
2813 return if matches!(self.mode, LexerMode::Imperative) {
2814 TokenType::Dot
2815 } else {
2816 TokenType::Period
2817 };
2818 }
2819 if word == "\x00POSS" {
2820 // The genitive clitic on a bracketed receiver (`foo(x)'s field`),
2821 // emitted by the splitter's `prev_close` possessive rule.
2822 return TokenType::Possessive;
2823 }
2824 // Handle block headers (##Theorem, ##Main, etc.)
2825 if word.starts_with("##") {
2826 let block_name = &word[2..];
2827 let lowered = block_name.to_lowercase();
2828 let lgx_header = self.try_lgx_header(&lowered);
2829 let block_type = if let Some(bt) = lgx_header {
2830 bt
2831 } else {
2832 match lowered.as_str() {
2833 "theorem" => BlockType::Theorem,
2834 "main" => BlockType::Main,
2835 "definition" => BlockType::Definition,
2836 "define" => BlockType::Define, // Vernacular-logic predicate definition (Rung 0a)
2837 "axiom" => BlockType::Axiom, // Formal first-order axiom (the seam for Tarski)
2838 "theory" => BlockType::Theory, // Named development grouping axioms + theorems
2839 "proof" => BlockType::Proof,
2840 "example" => BlockType::Example,
2841 "logic" => BlockType::Logic,
2842 "note" => BlockType::Note,
2843 "to" => BlockType::Function, // Function definition block
2844 "component" => BlockType::Component, // Reactive UI component block
2845 "a" | "an" => BlockType::TypeDef, // Inline type definitions: ## A Point has:
2846 "policy" => BlockType::Policy, // Security policy definitions
2847 "requires" => BlockType::Requires, // External crate dependencies
2848 "hardware" => BlockType::Hardware, // Signal declarations
2849 "property" => BlockType::Property, // Temporal assertions
2850 "no" => BlockType::No, // Optimization annotation: ## No Memo, ## No TCO, etc.
2851 "tier" => BlockType::Tier, // Tiered-optimizer pin: ## Tier specialize eager, etc.
2852 "require" => BlockType::Require, // Optimization demand: ## Require memo, ## Require closedform
2853
2854 "test" => {
2855 // `## Test: <prose>` — the prose after the colon is the
2856 // test's NAME: raw prose, never tokenized as code. Capture
2857 // the rest of the source line verbatim and skip its words.
2858 let hdr_end = self.words[self.pos].end;
2859 let line_end = self.source[hdr_end..]
2860 .find('\n')
2861 .map(|k| hdr_end + k)
2862 .unwrap_or(self.source.len());
2863 let raw = self.source[hdr_end..line_end]
2864 .trim()
2865 .trim_start_matches(':')
2866 .trim();
2867 let name = self.interner.intern(raw);
2868 // Leave `pos` on the line's last word; the tokenize loop's
2869 // own advance steps past it.
2870 while self.pos + 1 < self.words.len()
2871 && self.words[self.pos + 1].start < line_end
2872 {
2873 self.pos += 1;
2874 }
2875 BlockType::Test { name }
2876 }
2877 other => {
2878 // A near-miss of a CONSEQUENTIAL header is a probable
2879 // typo — `## Mian` silently becoming prose is the bug
2880 // class where a whole program runs to empty output.
2881 // Prose-type names (note/example/logic) and the short
2882 // forms are excluded, so ordinary literate headings
2883 // (`## Notes`, `## Design`) keep working.
2884 const CONSEQUENTIAL: &[&str] = &[
2885 "main",
2886 "theorem",
2887 "definition",
2888 "define",
2889 "axiom",
2890 "theory",
2891 "proof",
2892 "policy",
2893 "requires",
2894 "hardware",
2895 "property",
2896 "tier",
2897 "test",
2898 "component",
2899 "page",
2900 "route",
2901 "store",
2902 "directive",
2903 "redirect",
2904 ];
2905 if let Some(similar) = crate::suggest::find_similar(other, CONSEQUENTIAL, 2)
2906 {
2907 let found = self.interner.intern(block_name);
2908 let suggestion = self.interner.intern(similar);
2909 BlockType::SuspectedTypo { found, suggestion }
2910 } else {
2911 BlockType::Note // Unknown block types stay literate prose
2912 }
2913 }
2914 }
2915 };
2916
2917 // Update lexer mode based on block type
2918 self.mode = match block_type {
2919 BlockType::Main
2920 | BlockType::Function
2921 | BlockType::Component
2922 | BlockType::Test { .. }
2923 | BlockType::Page { .. }
2924 | BlockType::Store
2925 | BlockType::Directive
2926 | BlockType::Route { .. }
2927 | BlockType::Middleware
2928 | BlockType::ErrorPage { .. } => LexerMode::Imperative,
2929 _ => LexerMode::Declarative,
2930 };
2931
2932 return TokenType::BlockHeader { block_type };
2933 }
2934
2935 let lower = word.to_lowercase();
2936
2937 if lower == "each" && self.peek_sequence(&["other"]) {
2938 self.consume_words(1);
2939 return TokenType::Reciprocal;
2940 }
2941
2942 // The infinitary determiners, `infinitely often` and `infinitely close to` are one token
2943 // each, as `at least n` is.
2944 match lower.as_str() {
2945 "infinitely" if self.peek_sequence(&["many"]) => {
2946 self.consume_words(1);
2947 return TokenType::InfinitelyMany;
2948 }
2949 "infinitely" if self.peek_sequence(&["often"]) => {
2950 self.consume_words(1);
2951 return TokenType::InfinitelyOften;
2952 }
2953 "infinitely" if self.peek_sequence(&["close", "to"]) => {
2954 self.consume_words(2);
2955 return TokenType::Preposition(self.interner.intern("InfinitelyClose"));
2956 }
2957 "finitely" if self.peek_sequence(&["many"]) => {
2958 self.consume_words(1);
2959 return TokenType::FinitelyMany;
2960 }
2961 "only" if self.peek_sequence(&["finitely", "many"]) => {
2962 self.consume_words(2);
2963 return TokenType::FinitelyMany;
2964 }
2965 "all" if self.peek_sequence(&["but", "finitely", "many"]) => {
2966 self.consume_words(3);
2967 return TokenType::AllButFinitelyMany;
2968 }
2969 _ => {}
2970 }
2971
2972 if lower == "to" {
2973 if let Some(next) = self.peek_word(1) {
2974 if self.is_verb_like(next) {
2975 return TokenType::To;
2976 }
2977 }
2978 let sym = self.interner.intern("to");
2979 return TokenType::Preposition(sym);
2980 }
2981
2982 if lower == "at" {
2983 if let Some(next) = self.peek_word(1) {
2984 let next_lower = next.to_lowercase();
2985 if next_lower == "least" {
2986 if let Some(num_word) = self.peek_word(2) {
2987 if let Some(n) = Self::word_to_number(num_word) {
2988 self.consume_words(2);
2989 return TokenType::AtLeast(n);
2990 }
2991 }
2992 }
2993 if next_lower == "most" {
2994 if let Some(num_word) = self.peek_word(2) {
2995 if let Some(n) = Self::word_to_number(num_word) {
2996 self.consume_words(2);
2997 return TokenType::AtMost(n);
2998 }
2999 }
3000 }
3001 }
3002 }
3003
3004 // "Exactly N" → Cardinal(N) — same as bare number but explicit
3005 if lower == "exactly" {
3006 if let Some(num_word) = self.peek_word(1) {
3007 if let Some(n) = Self::word_to_number(num_word) {
3008 self.consume_words(1);
3009 return TokenType::Cardinal(n);
3010 }
3011 }
3012 }
3013
3014 if let Some(n) = Self::word_to_number(&lower) {
3015 return TokenType::Cardinal(n);
3016 }
3017
3018 // Check for duration literal first (e.g., "500ms", "2s", "50ns")
3019 if let Some((nanos, unit)) = Self::parse_duration_literal(word) {
3020 let unit_sym = self.interner.intern(unit);
3021 return TokenType::DurationLiteral {
3022 nanos,
3023 original_unit: unit_sym,
3024 };
3025 }
3026
3027 // Check for ISO-8601 date literal (e.g., "2026-05-20")
3028 if let Some(days) = Self::parse_date_literal(word) {
3029 return TokenType::DateLiteral { days };
3030 }
3031
3032 // Check for time-of-day literal (e.g., "4pm", "9:30am", "noon", "midnight")
3033 if let Some(nanos_from_midnight) = Self::parse_time_literal(word) {
3034 return TokenType::TimeLiteral {
3035 nanos_from_midnight,
3036 };
3037 }
3038
3039 // Currency-symbol money literal: `$19.99`, `€5`, `£10`, `¥100`, `$1,250.50`. The symbol
3040 // resolves to its ISO-4217 code; the magnitude keeps its digits and decimal point (thousands
3041 // separators stripped). A `MoneyLiteral` carries both, so the parser builds an exact
3042 // currency-tagged value rather than dropping the symbol. ONLY in imperative code — in
3043 // natural-language clauses `$25,000` is a bare magnitude (a logic-grid constraint), handled
3044 // by the magnitude-stripping block below; the symbol there is just orthography.
3045 if self.mode == LexerMode::Imperative {
3046 if let Some(first) = word.chars().next() {
3047 if let Some(code) = Self::currency_for_symbol(first) {
3048 let cleaned: String = word
3049 .chars()
3050 .skip(1)
3051 .filter(|c| c.is_ascii_digit() || *c == '.')
3052 .collect();
3053 if cleaned.starts_with(|c: char| c.is_ascii_digit()) {
3054 let amount = self.interner.intern(&cleaned);
3055 let currency = self.interner.intern(code);
3056 return TokenType::MoneyLiteral { amount, currency };
3057 }
3058 }
3059 }
3060 }
3061
3062 // Currency / comma-grouped numerals: "$125,000", "€2.60", "1,234". Strip the currency
3063 // marker and thousands separators; in a natural-language clause the puzzle constraint is the
3064 // bare magnitude (125000, 2.60). (Imperative code took the `MoneyLiteral` path above.)
3065 if word.starts_with(|c: char| Self::is_currency_symbol(c))
3066 || (word.starts_with(|c: char| c.is_ascii_digit()) && word.contains(','))
3067 {
3068 let cleaned: String = word
3069 .chars()
3070 .filter(|c| c.is_ascii_digit() || *c == '.')
3071 .collect();
3072 if cleaned.starts_with(|c: char| c.is_ascii_digit()) {
3073 let sym = self.interner.intern(&cleaned);
3074 return TokenType::Number(sym);
3075 }
3076 }
3077
3078 if Self::is_numeric_literal(word) {
3079 let sym = self.interner.intern(word);
3080 return TokenType::Number(sym);
3081 }
3082
3083 if lower == "if" && self.peek_sequence(&["and", "only", "if"]) {
3084 self.consume_words(3);
3085 return TokenType::Iff;
3086 }
3087
3088 if lower == "is" {
3089 if self.peek_sequence(&["equal", "to"]) {
3090 self.consume_words(2);
3091 return TokenType::Identity;
3092 }
3093 if self.peek_sequence(&["identical", "to"]) {
3094 self.consume_words(2);
3095 return TokenType::Identity;
3096 }
3097 }
3098
3099 if (lower == "a" || lower == "an") && word.chars().next().unwrap().is_uppercase() {
3100 // Capitalized "A" or "An" - disambiguate article vs proper name
3101 // Heuristic: articles are followed by nouns/adjectives, not verbs or keywords
3102 if let Some(next) = self.peek_word(1) {
3103 let next_lower = next.to_lowercase();
3104 let next_starts_lowercase = next
3105 .chars()
3106 .next()
3107 .map(|c| c.is_lowercase())
3108 .unwrap_or(false);
3109
3110 // If followed by logical keyword, treat as proper name (propositional variable)
3111 if matches!(next_lower.as_str(), "if" | "and" | "or" | "implies" | "iff") {
3112 let sym = self.interner.intern(word);
3113 return TokenType::ProperName(sym);
3114 }
3115
3116 // If next word is ONLY a verb (like "has", "is", "ran"), A is likely a name
3117 // Exception: gerunds (like "running") can follow articles
3118 // Exception: words in disambiguation_not_verbs (like "red") are not verbs
3119 // Exception: words that are also nouns/adjectives (like "fire") can follow articles
3120 let is_verb = self.lexicon.lookup_verb(&next_lower).is_some()
3121 && !lexicon::is_disambiguation_not_verb(&next_lower);
3122 let is_gerund = next_lower.ends_with("ing");
3123 let is_also_noun_or_adj =
3124 self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower);
3125 if is_verb && !is_gerund && !is_also_noun_or_adj {
3126 let sym = self.interner.intern(word);
3127 return TokenType::ProperName(sym);
3128 }
3129
3130 // Definition pattern: "A [TypeName] is a..." or "A [TypeName] has:" - treat A as article
3131 // even when TypeName is capitalized and unknown
3132 if let Some(third) = self.peek_word(2) {
3133 let third_lower = third.to_lowercase();
3134 // "has" for struct definitions: "A Point has:"
3135 if third_lower == "is" || third_lower == "are" || third_lower == "has" {
3136 return TokenType::Article(Definiteness::Indefinite);
3137 }
3138 }
3139
3140 // It's an article if next word is:
3141 // - A known noun or adjective, or
3142 // - Lowercase (likely a common word we don't recognize)
3143 let is_content_word =
3144 self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower);
3145 if is_content_word || next_starts_lowercase {
3146 return TokenType::Article(Definiteness::Indefinite);
3147 }
3148 }
3149 let sym = self.interner.intern(word);
3150 return TokenType::ProperName(sym);
3151 }
3152
3153 self.classify_word(word)
3154 }
3155
3156 fn is_noun_like(&self, word: &str) -> bool {
3157 if lexicon::is_noun_pattern(word) || lexicon::is_common_noun(word) {
3158 return true;
3159 }
3160 if word.ends_with("er") || word.ends_with("ian") || word.ends_with("ist") {
3161 return true;
3162 }
3163 false
3164 }
3165
3166 fn is_adjective_like(&self, word: &str) -> bool {
3167 lexicon::is_adjective(word) || lexicon::is_non_intersective(word)
3168 }
3169
3170 fn classify_word(&mut self, word: &str) -> TokenType {
3171 let lower = word.to_lowercase();
3172 let first_char = word.chars().next().unwrap();
3173
3174 // Disambiguate "that" as determiner vs complementizer/relativizer.
3175 // "that dog" → Article(Distal); "I know that he ran" → That.
3176 // After a nominal antecedent ("the vessel that saw …"), "that" heads a
3177 // relative clause even when the clause verb is also noun-like ("saw"),
3178 // so it must NOT collapse to a demonstrative there.
3179 if lower == "that" {
3180 if let Some(next) = self.peek_word(1) {
3181 let next_lower = next.to_lowercase();
3182 // A verb-capable next word ("that saw …", "that played …") makes
3183 // "that" a relativizer/complementizer even when that word is also
3184 // noun-like — the demonstrative reading needs a pure-noun head.
3185 let next_is_verb = self.lexicon.lookup_verb(&next_lower).is_some();
3186 if !next_is_verb
3187 && (self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower))
3188 {
3189 return TokenType::Article(Definiteness::Distal);
3190 }
3191 }
3192 }
3193
3194 // Arrow token for return type syntax
3195 if word == "->" {
3196 return TokenType::Arrow;
3197 }
3198
3199 // Grand Challenge: Comparison operator tokens
3200 if word == "<=" {
3201 return TokenType::LtEq;
3202 }
3203 if word == ">=" {
3204 return TokenType::GtEq;
3205 }
3206 match word {
3207 "+=" => return TokenType::PlusEq,
3208 "-=" => return TokenType::MinusEq,
3209 "*=" => return TokenType::StarEq,
3210 "/=" => return TokenType::SlashEq,
3211 "%=" => return TokenType::PercentEq,
3212 "**" => return TokenType::StarStar,
3213 "//" => return TokenType::SlashSlash,
3214 _ => {}
3215 }
3216 if word == "==" {
3217 return TokenType::EqEq;
3218 }
3219 if word == "!=" {
3220 return TokenType::NotEq;
3221 }
3222 if word == "<" {
3223 return TokenType::Lt;
3224 }
3225 if word == ">" {
3226 return TokenType::Gt;
3227 }
3228 // Single = for assignment (must come after == check)
3229 if word == "=" {
3230 return TokenType::Assign;
3231 }
3232
3233 if let Some(kind) = lexicon::lookup_keyword(&lower) {
3234 // A capitalized modal/auxiliary MID-sentence is a proper name, not the
3235 // function word ("the startup started by Will Waters", "the May deadline").
3236 // Modals/auxiliaries never capitalize mid-clause, so the keyword reading is
3237 // spurious there; sentence-initial stays the keyword (a question "Will Alice
3238 // win?"). This is the same name/keyword collision resolved for item/native.
3239 let is_modal = matches!(
3240 kind,
3241 TokenType::Must
3242 | TokenType::Shall
3243 | TokenType::Should
3244 | TokenType::Can
3245 | TokenType::May
3246 | TokenType::Cannot
3247 | TokenType::Would
3248 | TokenType::Could
3249 | TokenType::Might
3250 );
3251 if is_modal && first_char.is_uppercase() && !self.prev_word_ends_sentence() {
3252 return TokenType::ProperName(self.interner.intern(word));
3253 }
3254 // In Declarative (NL) mode, "from" and "for" are prepositions, not Logos keywords.
3255 // They are listed in the prepositions section of the lexicon and will be
3256 // correctly re-classified by the is_preposition() check below if we skip here.
3257 let kind = match (kind, self.mode) {
3258 (TokenType::From, LexerMode::Declarative) => {
3259 // A QUALIFIED IMPORT "Type from Module" — both neighbours are
3260 // capitalised identifiers — keeps the `From` keyword even in
3261 // declarative text (there are zero "Capitalised from
3262 // Capitalised" sequences in NL clues). Otherwise "from" is an
3263 // ordinary NL preposition ("the species FROM Australia").
3264 let prev_cap = self.pos > 0
3265 && self
3266 .words
3267 .get(self.pos - 1)
3268 .and_then(|w| w.word.chars().next())
3269 .map_or(false, |c| c.is_uppercase());
3270 let next_cap = self
3271 .words
3272 .get(self.pos + 1)
3273 .and_then(|w| w.word.chars().next())
3274 .map_or(false, |c| c.is_uppercase());
3275 if prev_cap && next_cap {
3276 TokenType::From
3277 } else {
3278 let sym = self.interner.intern("from");
3279 TokenType::Preposition(sym)
3280 }
3281 }
3282 (TokenType::For, LexerMode::Declarative) => {
3283 let sym = self.interner.intern("for");
3284 TokenType::Preposition(sym)
3285 }
3286 (other, _) => other,
3287 };
3288 return kind;
3289 }
3290
3291 if let Some(kind) = lexicon::lookup_pronoun(&lower) {
3292 return kind;
3293 }
3294
3295 if let Some(def) = lexicon::lookup_article(&lower) {
3296 return TokenType::Article(def);
3297 }
3298
3299 if let Some(time) = lexicon::lookup_auxiliary(&lower) {
3300 // A capitalized auxiliary MID-sentence is a proper name ("started by Will
3301 // Waters"), not the function word — auxiliaries never capitalize mid-clause.
3302 if first_char.is_uppercase() && !self.prev_word_ends_sentence() {
3303 return TokenType::ProperName(self.interner.intern(word));
3304 }
3305 return TokenType::Auxiliary(time);
3306 }
3307
3308 // Handle imperative keywords that might conflict with prepositions
3309 match lower.as_str() {
3310 "call" => return TokenType::Call,
3311 "in" if self.mode == LexerMode::Imperative => return TokenType::In,
3312 // Zone keywords (must come before is_preposition check)
3313 "inside" if self.mode == LexerMode::Imperative => return TokenType::Inside,
3314 // "at" for chunk access (must come before is_preposition check)
3315 "at" if self.mode == LexerMode::Imperative => return TokenType::At,
3316 // "into" for pipe send (must come before is_preposition check)
3317 "into" if self.mode == LexerMode::Imperative => return TokenType::Into,
3318 // Temporal span operator (must come before is_preposition check)
3319 "before" => return TokenType::Before,
3320 _ => {}
3321 }
3322
3323 // "per" is the rate preposition ("$2.50 per pound", "10 miles per
3324 // hour"). It is not in the general preposition lexicon, and must not be
3325 // mistaken for a measure unit or an unknown noun.
3326 if lower == "per" {
3327 let sym = self.interner.intern("per");
3328 return TokenType::Preposition(sym);
3329 }
3330
3331 // `During <gerund>:` is a trace-span statement in imperative code — it
3332 // must win over the preposition reading (constitution Part 2). Prose
3333 // keeps `during` a preposition (declarative mode).
3334 if lower == "during" && self.mode == LexerMode::Imperative {
3335 return TokenType::During;
3336 }
3337
3338 // A word that is BOTH a verb and a preposition ("like") stays
3339 // ambiguous — the dual-class block below emits the alternatives.
3340 // Lexicon-disambiguated NON-verbs ("during") are pure prepositions
3341 // even when morphology over-derives a verb reading.
3342 if lexicon::is_preposition(&lower)
3343 && (self.lexicon.lookup_verb(&lower).is_none()
3344 || lexicon::is_disambiguation_not_verb(&lower))
3345 {
3346 let sym = self.interner.intern(&lower);
3347 return TokenType::Preposition(sym);
3348 }
3349
3350 match lower.as_str() {
3351 "equals" => return TokenType::Equals,
3352 // "item"/"items" is the indexing keyword in imperative code (where a
3353 // variable index like "item i of arr" is the whole point and prose nouns
3354 // never appear), and in declarative text ONLY when an index number follows
3355 // ("item 1 of list", "items 2 through 5"). Otherwise, in declarative text,
3356 // it is the ordinary English noun ("the blue item", "the item made of
3357 // gold"), which leaked as the keyword before and stranded the noun after an
3358 // adjective. Mode separates code from prose; the following number is the
3359 // finer signal within prose.
3360 "item" | "items"
3361 if self.mode == LexerMode::Imperative
3362 || self.peek_word(1).map_or(false, |w| {
3363 w.chars().next().map_or(false, |c| c.is_ascii_digit())
3364 || crate::lexicon::word_to_number(&w.to_lowercase()).is_some()
3365 }) =>
3366 {
3367 return if lower == "item" {
3368 TokenType::Item
3369 } else {
3370 TokenType::Items
3371 };
3372 }
3373 // Mutability keyword for `mut x = 5` syntax
3374 "mut" if self.mode == LexerMode::Imperative => return TokenType::Mut,
3375 "let" => {
3376 self.in_let_context = true;
3377 return TokenType::Let;
3378 }
3379 "set" => {
3380 // Check if "set" is used as a type (followed by "of") - "Set of Int"
3381 // This takes priority over the assignment keyword
3382 if self
3383 .peek_word(1)
3384 .map_or(false, |w| w.to_lowercase() == "of")
3385 {
3386 // It's a type like "Set of Int" - don't return keyword, let it be a noun
3387 } else if self.mode == LexerMode::Imperative {
3388 // In Imperative mode, treat "set" as the assignment keyword
3389 return TokenType::Set;
3390 } else {
3391 // In Declarative mode, check positions 2-5 for "to"
3392 // (handles field access like "set p's x to")
3393 for offset in 2..=5 {
3394 if self
3395 .peek_word(offset)
3396 .map_or(false, |w| w.to_lowercase() == "to")
3397 {
3398 return TokenType::Set;
3399 }
3400 }
3401 }
3402 }
3403 "return" => return TokenType::Return,
3404 "break" => return TokenType::Break,
3405 "xor" => return TokenType::Xor,
3406 "shifted" => return TokenType::Shifted,
3407 "be" if self.in_let_context => {
3408 self.in_let_context = false;
3409 return TokenType::Be;
3410 }
3411 "while" => return TokenType::While,
3412 "assert" => return TokenType::Assert,
3413 "trust" => return TokenType::Trust,
3414 // Imperative-only: these are common English words ("the proof requires
3415 // …", "this ensures …"), so keep them as plain words in declarative mode.
3416 "require" if self.mode == LexerMode::Imperative => return TokenType::Require,
3417 "requires" if self.mode == LexerMode::Imperative => return TokenType::Requires,
3418 "ensures" if self.mode == LexerMode::Imperative => return TokenType::Ensures,
3419 "check" => return TokenType::Check,
3420 // Theorem keywords (Declarative mode - for theorem blocks).
3421 // In IMPERATIVE mode `given` is the test-fixture keyword
3422 // (`Given a file "…" containing "…".`) — same token, both modes.
3423 "given" => return TokenType::Given,
3424 // Test-framework keywords (Imperative mode only — `## Test` bodies).
3425 // Declarative prose
3426 // keeps them as ordinary verbs.
3427 "expect" if self.mode == LexerMode::Imperative => return TokenType::Expect,
3428 "advance" if self.mode == LexerMode::Imperative => return TokenType::Advance,
3429 "skip" if self.mode == LexerMode::Imperative => return TokenType::Skip,
3430 // Observation statements (constitution Part 2), imperative-only —
3431 // declarative prose keeps them as ordinary verbs/prepositions.
3432 "note" if self.mode == LexerMode::Imperative => return TokenType::Note,
3433 "warn" if self.mode == LexerMode::Imperative => return TokenType::Warn,
3434 "record" if self.mode == LexerMode::Imperative => return TokenType::Record,
3435 // ("during" is handled before the is_preposition check above.)
3436 "prove" if self.mode == LexerMode::Declarative => return TokenType::Prove,
3437 "auto" if self.mode == LexerMode::Declarative => return TokenType::Auto,
3438 // P2P Networking keywords (Imperative mode only)
3439 "listen" if self.mode == LexerMode::Imperative => return TokenType::Listen,
3440 "connect" if self.mode == LexerMode::Imperative => return TokenType::NetConnect,
3441 "sleep" if self.mode == LexerMode::Imperative => return TokenType::Sleep,
3442 // GossipSub keywords (Imperative mode only)
3443 "sync" if self.mode == LexerMode::Imperative => return TokenType::Sync,
3444 // Persistence keywords
3445 "mount" if self.mode == LexerMode::Imperative => return TokenType::Mount,
3446 "persistent" => return TokenType::Persistent, // Works in type expressions
3447 "combined" if self.mode == LexerMode::Imperative => return TokenType::Combined,
3448 "followed" if self.mode == LexerMode::Imperative => return TokenType::Followed,
3449 // Go-like Concurrency keywords (Imperative mode only)
3450 // Note: "first" and "after" are NOT keywords - they're checked via lookahead in parser
3451 // to avoid conflicting with their use as variable names
3452 "launch" if self.mode == LexerMode::Imperative => return TokenType::Launch,
3453 "task" if self.mode == LexerMode::Imperative => return TokenType::Task,
3454 "pipe" if self.mode == LexerMode::Imperative => return TokenType::Pipe,
3455 "receive" if self.mode == LexerMode::Imperative => return TokenType::Receive,
3456 "stop" if self.mode == LexerMode::Imperative => return TokenType::Stop,
3457 "try" if self.mode == LexerMode::Imperative => return TokenType::Try,
3458 "into" if self.mode == LexerMode::Imperative => return TokenType::Into,
3459 "native" if self.mode == LexerMode::Imperative => return TokenType::Native,
3460 "escape" if self.mode == LexerMode::Imperative => return TokenType::Escape,
3461 "from" => return TokenType::From,
3462 "otherwise" => return TokenType::Otherwise,
3463 // Phase 30c: Else/elif as aliases for Otherwise/Otherwise If
3464 "else" => return TokenType::Else,
3465 "elif" => return TokenType::Elif,
3466 // Sum type definition (Declarative mode only - for enum "either...or...")
3467 "either" if self.mode == LexerMode::Declarative => return TokenType::Either,
3468 // Pattern matching statement
3469 "inspect" if self.mode == LexerMode::Imperative => return TokenType::Inspect,
3470 // Constructor keyword (Imperative mode only)
3471 "new" if self.mode == LexerMode::Imperative => return TokenType::New,
3472 // Only emit Give/Show as keywords in Imperative mode
3473 // In Declarative mode, they fall through to lexicon lookup as verbs
3474 "give" if self.mode == LexerMode::Imperative => return TokenType::Give,
3475 "show" if self.mode == LexerMode::Imperative => return TokenType::Show,
3476 // Collection operation keywords (Imperative mode only)
3477 "push" if self.mode == LexerMode::Imperative => return TokenType::Push,
3478 "pop" if self.mode == LexerMode::Imperative => return TokenType::Pop,
3479 "copy" if self.mode == LexerMode::Imperative => return TokenType::Copy,
3480 "through" if self.mode == LexerMode::Imperative => return TokenType::Through,
3481 "length" if self.mode == LexerMode::Imperative => return TokenType::Length,
3482 "at" if self.mode == LexerMode::Imperative => return TokenType::At,
3483 // Set operation keywords (Imperative mode only)
3484 "add" if self.mode == LexerMode::Imperative => return TokenType::Add,
3485 "remove" if self.mode == LexerMode::Imperative => return TokenType::Remove,
3486 "contains" if self.mode == LexerMode::Imperative => return TokenType::Contains,
3487 "union" if self.mode == LexerMode::Imperative => return TokenType::Union,
3488 "intersection" if self.mode == LexerMode::Imperative => return TokenType::Intersection,
3489 // Zone keywords (Imperative mode only)
3490 "inside" if self.mode == LexerMode::Imperative => return TokenType::Inside,
3491 "zone" if self.mode == LexerMode::Imperative => return TokenType::Zone,
3492 "called" if self.mode == LexerMode::Imperative => return TokenType::Called,
3493 "size" if self.mode == LexerMode::Imperative => return TokenType::Size,
3494 "mapped" if self.mode == LexerMode::Imperative => return TokenType::Mapped,
3495 // Structured Concurrency keywords (Imperative mode only)
3496 "attempt" if self.mode == LexerMode::Imperative => return TokenType::Attempt,
3497 "following" if self.mode == LexerMode::Imperative => return TokenType::Following,
3498 "simultaneously" if self.mode == LexerMode::Imperative => {
3499 return TokenType::Simultaneously
3500 }
3501 // IO keywords (Imperative mode only)
3502 "read" if self.mode == LexerMode::Imperative => return TokenType::Read,
3503 "write" if self.mode == LexerMode::Imperative => return TokenType::Write,
3504 "console" if self.mode == LexerMode::Imperative => return TokenType::Console,
3505 "file" if self.mode == LexerMode::Imperative => return TokenType::File,
3506 // Agent System keywords (Imperative mode only)
3507 "spawn" if self.mode == LexerMode::Imperative => return TokenType::Spawn,
3508 "send" if self.mode == LexerMode::Imperative => return TokenType::Send,
3509 "await" if self.mode == LexerMode::Imperative => return TokenType::Await,
3510 // Serialization keyword (works in Definition blocks too)
3511 "portable" => return TokenType::Portable,
3512 // Sipping Protocol keywords (Imperative mode only)
3513 "manifest" if self.mode == LexerMode::Imperative => return TokenType::Manifest,
3514 "chunk" if self.mode == LexerMode::Imperative => return TokenType::Chunk,
3515 // CRDT keywords
3516 "shared" => return TokenType::Shared, // Works in Definition blocks like Portable
3517 "merge" if self.mode == LexerMode::Imperative => return TokenType::Merge,
3518 "converge" if self.mode == LexerMode::Imperative => return TokenType::Converge,
3519 "swap" if self.mode == LexerMode::Imperative => return TokenType::Swap,
3520 "increase" if self.mode == LexerMode::Imperative => return TokenType::Increase,
3521 // Extended CRDT keywords
3522 "decrease" if self.mode == LexerMode::Imperative => return TokenType::Decrease,
3523 "append" if self.mode == LexerMode::Imperative => return TokenType::Append,
3524 "resolve" if self.mode == LexerMode::Imperative => return TokenType::Resolve,
3525 "values" if self.mode == LexerMode::Imperative => return TokenType::Values,
3526 // Type keywords (work in both modes like "Shared"):
3527 "tally" => return TokenType::Tally,
3528 "sharedset" => return TokenType::SharedSet,
3529 "sharedsequence" => return TokenType::SharedSequence,
3530 "collaborativesequence" => return TokenType::CollaborativeSequence,
3531 "sharedmap" => return TokenType::SharedMap,
3532 "divergent" => return TokenType::Divergent,
3533 "removewins" => return TokenType::RemoveWins,
3534 "addwins" => return TokenType::AddWins,
3535 "yata" => return TokenType::YATA,
3536 // Calendar / clock time unit words (Span expressions). The singular
3537 // "second"/"minute" are ambiguous (ordinal "second base", adjective
3538 // "minute detail") so they are time units only after a count; the
3539 // plurals and "hour" are unambiguous time units.
3540 "seconds" => return TokenType::CalendarUnit(CalendarUnit::Second),
3541 "minutes" => return TokenType::CalendarUnit(CalendarUnit::Minute),
3542 "second" if self.prev_word_is_numeric() => {
3543 return TokenType::CalendarUnit(CalendarUnit::Second)
3544 }
3545 "minute" if self.prev_word_is_numeric() => {
3546 return TokenType::CalendarUnit(CalendarUnit::Minute)
3547 }
3548 "hour" | "hours" => return TokenType::CalendarUnit(CalendarUnit::Hour),
3549 "day" | "days" => return TokenType::CalendarUnit(CalendarUnit::Day),
3550 "week" | "weeks" => return TokenType::CalendarUnit(CalendarUnit::Week),
3551 "month" | "months" => return TokenType::CalendarUnit(CalendarUnit::Month),
3552 "year" | "years" => return TokenType::CalendarUnit(CalendarUnit::Year),
3553 // Sub-second clock units: unlike bare "second"/"minute", these compound words have
3554 // no other common English reading, so they need no preceding-number guard — the
3555 // English-word spelling of the same durations the compact `100ms`/`50ns` literals
3556 // already cover (LOGOS_QUICKGUIDE.md's Duration row).
3557 "millisecond" | "milliseconds" => {
3558 return TokenType::CalendarUnit(CalendarUnit::Millisecond)
3559 }
3560 "microsecond" | "microseconds" => {
3561 return TokenType::CalendarUnit(CalendarUnit::Microsecond)
3562 }
3563 "nanosecond" | "nanoseconds" => {
3564 return TokenType::CalendarUnit(CalendarUnit::Nanosecond)
3565 }
3566 // Span-related keywords (note: "before" is handled earlier to avoid preposition conflict)
3567 "ago" => return TokenType::Ago,
3568 "hence" => return TokenType::Hence,
3569 "if" => return TokenType::If,
3570 "only" => return TokenType::Focus(FocusKind::Only),
3571 "even" => return TokenType::Focus(FocusKind::Even),
3572 "just"
3573 if self.peek_word(1).map_or(false, |w| {
3574 !self.is_verb_like(w)
3575 || w.to_lowercase() == "john"
3576 || w.chars().next().map_or(false, |c| c.is_uppercase())
3577 }) =>
3578 {
3579 return TokenType::Focus(FocusKind::Just)
3580 }
3581 "much" => return TokenType::Measure(MeasureKind::Much),
3582 "little" => return TokenType::Measure(MeasureKind::Little),
3583 _ => {}
3584 }
3585
3586 if lexicon::is_scopal_adverb(&lower) {
3587 let sym = self.interner.intern(&Self::capitalize(&lower));
3588 return TokenType::ScopalAdverb(sym);
3589 }
3590
3591 if lexicon::is_temporal_adverb(&lower) && !self.prev_token_is_determiner() {
3592 let sym = self.interner.intern(&Self::capitalize(&lower));
3593 return TokenType::TemporalAdverb(sym);
3594 }
3595
3596 if lexicon::is_non_intersective(&lower) {
3597 let sym = self.interner.intern(&Self::capitalize(&lower));
3598 return TokenType::NonIntersectiveAdjective(sym);
3599 }
3600
3601 if lexicon::is_adverb(&lower) {
3602 let sym = self.interner.intern(&Self::capitalize(&lower));
3603 return TokenType::Adverb(sym);
3604 }
3605 // A capitalized "-ly" word is normally a proper name ("Billy", "Holly",
3606 // "Kelly's stamp"), not an adverb — UNLESS it is a sentence-initial
3607 // manner adverb modifying a following pronoun ("Quickly he ran").
3608 let cap_ly_is_name = word.chars().next().map_or(false, |c| c.is_uppercase())
3609 && !self.peek_word(1).map_or(false, |w| {
3610 matches!(
3611 w.to_lowercase().as_str(),
3612 "he" | "she" | "it" | "they" | "we" | "i" | "you" | "who"
3613 )
3614 });
3615 // A capital letter ANYWHERE past the first character (`bOnly`, `isEmpty`) is camelCase —
3616 // never a natural English word, however its tail happens to be spelled. No real adverb is
3617 // written this way, so a chosen code identifier that merely ends the same way as one
3618 // ("bOnly" ends in the same four letters as "only") must never be mistaken for it; without
3619 // this guard `bOnly`/`aOnly` classified as `Adverb` while the plain `onlyB` (which never
3620 // reaches this "-ly" check at all) classified as an ordinary identifier — the same word
3621 // choice landing on two different token kinds by accident of which end the suffix fell on.
3622 let has_internal_capital = word.chars().skip(1).any(|c| c.is_uppercase());
3623 if lower.ends_with("ly")
3624 && !lexicon::is_not_adverb(&lower)
3625 && !lexicon::is_common_noun(&lower)
3626 && lower.len() > 4
3627 && !cap_ly_is_name
3628 && !has_internal_capital
3629 {
3630 let sym = self.interner.intern(&Self::capitalize(&lower));
3631 return TokenType::Adverb(sym);
3632 }
3633
3634 if let Some(base) = self.try_parse_superlative(&lower) {
3635 let sym = self.interner.intern(&base);
3636 return TokenType::Superlative(sym);
3637 }
3638
3639 // Handle irregular comparatives (less, more, better, worse). "fewer" is
3640 // the count counterpart of "less" — same decreasing direction — and maps
3641 // to the same Decreasing base so it grades like "less" ("received fewer
3642 // votes than Y" → Less(Vote(X), Vote(Y))) without turning bare "few" into
3643 // an adjective (which would break its quantifier reading, "few cookies").
3644 let irregular_comparative = match lower.as_str() {
3645 "less" => Some("Little"),
3646 "fewer" => Some("Little"),
3647 "more" => Some("Much"),
3648 "better" => Some("Good"),
3649 "worse" => Some("Bad"),
3650 _ => None,
3651 };
3652 if let Some(base) = irregular_comparative {
3653 let sym = self.interner.intern(base);
3654 return TokenType::Comparative(sym);
3655 }
3656
3657 if let Some(base) = self.try_parse_comparative(&lower) {
3658 // A word that is ALSO a common noun ("stranger" — the noun vs the
3659 // comparative of "strange") is a comparative only in comparative
3660 // position, signalled by a following "than" ("is stranger than").
3661 // Elsewhere ("the stranger", "Let stranger be …") it is the noun, so
3662 // defer to noun classification below — mirroring the performative /
3663 // base-verb noun-deference above.
3664 let next_is_than = self
3665 .peek_word(1)
3666 .map_or(false, |w| w.eq_ignore_ascii_case("than"));
3667 if !lexicon::is_common_noun(&lower) || next_is_than {
3668 let sym = self.interner.intern(&base);
3669 return TokenType::Comparative(sym);
3670 }
3671 // else fall through to noun classification
3672 }
3673
3674 if lexicon::is_performative(&lower) {
3675 // If the word is also a common noun AND follows a determiner or precedes a copula,
3676 // don't force performative reading.
3677 // "every request holds" → request is a noun, not a performative verb.
3678 // "If request is asserted" → request is a noun (subject before copula).
3679 // "I promise to come" → promise IS a performative verb.
3680 let after_determiner = self.prev_token_is_determiner();
3681 let before_copula = self.next_token_is_copula();
3682 if !lexicon::is_common_noun(&lower) || (!after_determiner && !before_copula) {
3683 let sym = self.interner.intern(&Self::capitalize(&lower));
3684 return TokenType::Performative(sym);
3685 }
3686 // Fall through to noun/verb disambiguation below
3687 }
3688
3689 if lexicon::is_base_verb_early(&lower) {
3690 // If the word is also a common noun AND follows a determiner or precedes a copula,
3691 // don't force verb reading.
3692 // "every grant holds" → grant is a noun, not a verb.
3693 // "If grant is low" → grant is a noun (subject before copula).
3694 let after_determiner = self.prev_token_is_determiner();
3695 let before_copula = self.next_token_is_copula();
3696 if !lexicon::is_common_noun(&lower) || (!after_determiner && !before_copula) {
3697 let sym = self.interner.intern(&Self::capitalize(&lower));
3698 let class = lexicon::lookup_verb_class(&lower);
3699 return TokenType::Verb {
3700 lemma: sym,
3701 time: Time::Present,
3702 aspect: Aspect::Simple,
3703 class,
3704 };
3705 }
3706 // Fall through to noun/verb disambiguation below
3707 }
3708
3709 // Check for gerunds/progressive verbs BEFORE ProperName check
3710 // "Running" at start of sentence should be Verb, not ProperName
3711 if lower.ends_with("ing") && lower.len() > 4 {
3712 // Participial ADJECTIVE in attributive position ("the
3713 // corresponding output line"): the word has an adjective entry
3714 // and a content word follows it. -ing PREPOSITIONS ("during")
3715 // skip the early return too — the dual-class block below emits
3716 // their Ambiguous alternatives.
3717 let attributive = self.is_adjective_like(&lower)
3718 && self
3719 .peek_word(1)
3720 .map(|next| {
3721 let next_lower = next.to_lowercase();
3722 self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower)
3723 })
3724 .unwrap_or(false);
3725 if !attributive && !lexicon::is_preposition(&lower) {
3726 if let Some(entry) = self.lexicon.lookup_verb(&lower) {
3727 let sym = self.interner.intern(&entry.lemma);
3728 return TokenType::Verb {
3729 lemma: sym,
3730 time: entry.time,
3731 aspect: entry.aspect,
3732 class: entry.class,
3733 };
3734 }
3735 }
3736 }
3737
3738 if first_char.is_uppercase() {
3739 // Smart Lexicon: Check if this capitalized word is actually a common noun
3740 // Only apply for sentence-initial words (followed by verb) to avoid
3741 // breaking type definitions like "A Point has:"
3742 //
3743 // Pattern: "Farmers walk." → Farmers is plural of Farmer (common noun)
3744 // Pattern: "A Point has:" → Point is a type name (proper name)
3745 if let Some(next) = self.peek_word(1) {
3746 let next_lower = next.to_lowercase();
3747 // If next word is a verb, this capitalized word is likely a subject noun
3748 let is_followed_by_verb = self.lexicon.lookup_verb(&next_lower).is_some()
3749 || matches!(
3750 next_lower.as_str(),
3751 "is" | "are" | "was" | "were" | "has" | "have" | "had"
3752 );
3753
3754 if is_followed_by_verb {
3755 // Check if lowercase version is a derivable common noun
3756 if let Some(analysis) = lexicon::analyze_word(&lower) {
3757 match analysis {
3758 lexicon::WordAnalysis::Noun(meta)
3759 if meta.number == lexicon::Number::Plural =>
3760 {
3761 // It's a plural noun - definitely a common noun
3762 let sym = self.interner.intern(&lower);
3763 return TokenType::Noun(sym);
3764 }
3765 lexicon::WordAnalysis::DerivedNoun {
3766 number: lexicon::Number::Plural,
3767 ..
3768 } => {
3769 // Derived plural agentive noun (e.g., "Bloggers")
3770 let sym = self.interner.intern(&lower);
3771 return TokenType::Noun(sym);
3772 }
3773 _ => {
3774 // Singular nouns at sentence start could still be proper names
3775 // e.g., "John walks." vs "Farmer walks."
3776 }
3777 }
3778 }
3779 }
3780 }
3781
3782 let sym = self.interner.intern(word);
3783 return TokenType::ProperName(sym);
3784 }
3785
3786 let verb_entry = self.lexicon.lookup_verb(&lower);
3787 // Irregular plurals ("children", "mice") are not in the common-noun
3788 // table; morphological analysis identifies them. Restricted to
3789 // non-verb words so "rains" stays a pure verb.
3790 let is_noun = lexicon::is_common_noun(&lower)
3791 || (verb_entry.is_none()
3792 && matches!(
3793 lexicon::analyze_word(&lower),
3794 Some(lexicon::WordAnalysis::Noun(_))
3795 ));
3796 let is_adj = self.is_adjective_like(&lower);
3797 let is_disambiguated = lexicon::is_disambiguation_not_verb(&lower);
3798
3799 // Ambiguous: word is Verb AND (Noun OR Adjective OR Preposition),
3800 // not disambiguated
3801 let is_prep = lexicon::is_preposition(&lower);
3802 if verb_entry.is_some() && (is_noun || is_adj || is_prep) && !is_disambiguated {
3803 let entry = verb_entry.unwrap();
3804 let verb_token = TokenType::Verb {
3805 lemma: self.interner.intern(&entry.lemma),
3806 time: entry.time,
3807 aspect: entry.aspect,
3808 class: entry.class,
3809 };
3810
3811 let mut alternatives = Vec::new();
3812 if is_noun {
3813 alternatives.push(TokenType::Noun(self.interner.intern(word)));
3814 }
3815 if is_adj {
3816 alternatives.push(TokenType::Adjective(self.interner.intern(word)));
3817 }
3818 if is_prep {
3819 alternatives.push(TokenType::Preposition(self.interner.intern(&lower)));
3820 }
3821
3822 return TokenType::Ambiguous {
3823 primary: Box::new(verb_token),
3824 alternatives,
3825 };
3826 }
3827
3828 // Disambiguated away from verb — only when there is an alternate reading.
3829 // If the word has no noun or adjective reading, the lexicon verb entry wins.
3830 if let Some(entry) = &verb_entry {
3831 if is_disambiguated {
3832 let sym = self.interner.intern(word);
3833 if is_noun {
3834 return TokenType::Noun(sym);
3835 }
3836 if is_adj {
3837 return TokenType::Adjective(sym);
3838 }
3839 // No alternate reading: honour the lexicon (e.g. "led" = past of "lead").
3840 return TokenType::Verb {
3841 lemma: self.interner.intern(&entry.lemma),
3842 time: entry.time,
3843 aspect: entry.aspect,
3844 class: entry.class,
3845 };
3846 }
3847 }
3848
3849 // Pure verb
3850 if let Some(entry) = verb_entry {
3851 let sym = self.interner.intern(&entry.lemma);
3852 return TokenType::Verb {
3853 lemma: sym,
3854 time: entry.time,
3855 aspect: entry.aspect,
3856 class: entry.class,
3857 };
3858 }
3859
3860 // Pure noun
3861 if is_noun {
3862 let sym = self.interner.intern(word);
3863 return TokenType::Noun(sym);
3864 }
3865
3866 // Pure adjective — a word the lexicon knows only as an adjective
3867 // ("happy", "dangerous", "tall"). Without this the word falls through
3868 // to the Noun fallback and predicative/attributive adjective readings
3869 // are lost (e.g. "John seems happy." strands "happy").
3870 if is_adj {
3871 let sym = self.interner.intern(word);
3872 return TokenType::Adjective(sym);
3873 }
3874
3875 if lexicon::is_base_verb(&lower) {
3876 let sym = self.interner.intern(&Self::capitalize(&lower));
3877 let class = lexicon::lookup_verb_class(&lower);
3878 return TokenType::Verb {
3879 lemma: sym,
3880 time: Time::Present,
3881 aspect: Aspect::Simple,
3882 class,
3883 };
3884 }
3885
3886 if lower.ends_with("ian")
3887 || lower.ends_with("er")
3888 || lower == "logic"
3889 || lower == "time"
3890 || lower == "men"
3891 || lower == "book"
3892 || lower == "house"
3893 || lower == "code"
3894 || lower == "user"
3895 {
3896 let sym = self.interner.intern(word);
3897 return TokenType::Noun(sym);
3898 }
3899
3900 if lexicon::is_particle(&lower) {
3901 let sym = self.interner.intern(&lower);
3902 return TokenType::Particle(sym);
3903 }
3904
3905 // Unknown "-ed" word → regular past-tense verb ("aired", "studied",
3906 // "debuted"). English marks the past tense morphologically, so an
3907 // otherwise-unrecognized "-ed" content word is overwhelmingly a verb.
3908 // Kept Ambiguous with a Noun reading so it can still head an NP where a
3909 // noun is expected; the parser picks the verb reading in VP position.
3910 if lower.len() >= 4 && lower.ends_with("ed") {
3911 let stem = if lower.ends_with("ied") {
3912 format!("{}y", &lower[..lower.len() - 3])
3913 } else {
3914 lower[..lower.len() - 2].to_string()
3915 };
3916 let verb_token = TokenType::Verb {
3917 lemma: self.interner.intern(&Self::capitalize(&stem)),
3918 time: Time::Past,
3919 aspect: Aspect::Simple,
3920 class: lexicon::lookup_verb_class(&stem),
3921 };
3922 return TokenType::Ambiguous {
3923 primary: Box::new(verb_token),
3924 alternatives: vec![TokenType::Noun(self.interner.intern(word))],
3925 };
3926 }
3927
3928 // Unknown lowercase words default to Noun. In natural language contexts
3929 // (puzzle clues, prose) unknown content words are overwhelmingly nouns —
3930 // domain-specific items like dance styles, place names, object names, etc.
3931 // Genuine adjectives and adverbs are caught by earlier checks.
3932 let sym = self.interner.intern(word);
3933 TokenType::Noun(sym)
3934 }
3935
3936 fn capitalize(s: &str) -> String {
3937 let mut chars = s.chars();
3938 match chars.next() {
3939 None => String::new(),
3940 Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
3941 }
3942 }
3943
3944 pub fn is_collective_verb(lemma: &str) -> bool {
3945 lexicon::is_collective_verb(&lemma.to_lowercase())
3946 }
3947
3948 pub fn is_mixed_verb(lemma: &str) -> bool {
3949 lexicon::is_mixed_verb(&lemma.to_lowercase())
3950 }
3951
3952 pub fn is_distributive_verb(lemma: &str) -> bool {
3953 lexicon::is_distributive_verb(&lemma.to_lowercase())
3954 }
3955
3956 pub fn is_intensional_predicate(lemma: &str) -> bool {
3957 lexicon::is_intensional_predicate(&lemma.to_lowercase())
3958 }
3959
3960 pub fn is_opaque_verb(lemma: &str) -> bool {
3961 lexicon::is_opaque_verb(&lemma.to_lowercase())
3962 }
3963
3964 pub fn is_ditransitive_verb(lemma: &str) -> bool {
3965 lexicon::is_ditransitive_verb(&lemma.to_lowercase())
3966 }
3967
3968 fn is_verb_like(&self, word: &str) -> bool {
3969 let lower = word.to_lowercase();
3970 if lexicon::is_infinitive_verb(&lower) {
3971 return true;
3972 }
3973 if let Some(entry) = self.lexicon.lookup_verb(&lower) {
3974 return entry.lemma.len() > 0;
3975 }
3976 false
3977 }
3978
3979 pub fn is_subject_control_verb(lemma: &str) -> bool {
3980 lexicon::is_subject_control_verb(&lemma.to_lowercase())
3981 }
3982
3983 pub fn is_raising_verb(lemma: &str) -> bool {
3984 lexicon::is_raising_verb(&lemma.to_lowercase())
3985 }
3986
3987 pub fn is_object_control_verb(lemma: &str) -> bool {
3988 lexicon::is_object_control_verb(&lemma.to_lowercase())
3989 }
3990
3991 pub fn is_weather_verb(lemma: &str) -> bool {
3992 matches!(
3993 lemma.to_lowercase().as_str(),
3994 "rain" | "snow" | "hail" | "thunder" | "pour"
3995 )
3996 }
3997
3998 fn try_parse_superlative(&self, word: &str) -> Option<String> {
3999 if !word.ends_with("est") || word.len() < 5 {
4000 return None;
4001 }
4002
4003 let base = &word[..word.len() - 3];
4004
4005 // Guard on the CHARACTER count, not the byte length: a single multibyte code point
4006 // (e.g. `ߑ`, 2 bytes / 1 char) passes a `base.len() >= 2` byte check but has only one
4007 // char, making `chars[chars.len() - 2]` underflow `usize`. Slice by the last char's
4008 // UTF-8 width so a doubled multibyte letter never cuts a code point mid-way.
4009 let chars: Vec<char> = base.chars().collect();
4010 if chars.len() >= 2 {
4011 let last = chars[chars.len() - 1];
4012 let second_last = chars[chars.len() - 2];
4013 if last == second_last && !"aeiou".contains(last) {
4014 let stem = &base[..base.len() - last.len_utf8()];
4015 if lexicon::is_gradable_adjective(stem) {
4016 return Some(Self::capitalize(stem));
4017 }
4018 }
4019 }
4020
4021 if base.ends_with("i") {
4022 let stem = format!("{}y", &base[..base.len() - 1]);
4023 if lexicon::is_gradable_adjective(&stem) {
4024 return Some(Self::capitalize(&stem));
4025 }
4026 }
4027
4028 if lexicon::is_gradable_adjective(base) {
4029 return Some(Self::capitalize(base));
4030 }
4031
4032 None
4033 }
4034
4035 fn try_parse_comparative(&self, word: &str) -> Option<String> {
4036 if !word.ends_with("er") || word.len() < 4 {
4037 return None;
4038 }
4039
4040 let base = &word[..word.len() - 2];
4041
4042 // Same byte-vs-char hazard as try_parse_superlative: a single multibyte code point is
4043 // >= 2 bytes but only 1 char, so a `base.len() >= 2` byte guard lets `chars.len() - 2`
4044 // underflow. Guard on the char count; slice by the last char's UTF-8 width.
4045 let chars: Vec<char> = base.chars().collect();
4046 if chars.len() >= 2 {
4047 let last = chars[chars.len() - 1];
4048 let second_last = chars[chars.len() - 2];
4049 if last == second_last && !"aeiou".contains(last) {
4050 let stem = &base[..base.len() - last.len_utf8()];
4051 if lexicon::is_gradable_adjective(stem) {
4052 return Some(Self::capitalize(stem));
4053 }
4054 }
4055 }
4056
4057 if base.ends_with("i") {
4058 let stem = format!("{}y", &base[..base.len() - 1]);
4059 if lexicon::is_gradable_adjective(&stem) {
4060 return Some(Self::capitalize(&stem));
4061 }
4062 }
4063
4064 if lexicon::is_gradable_adjective(base) {
4065 return Some(Self::capitalize(base));
4066 }
4067
4068 // Silent-e adjectives drop the 'e' before -er: "wide"→"wider",
4069 // "large"→"larger", "late"→"later". Recover the base by restoring it.
4070 let with_e = format!("{}e", base);
4071 if lexicon::is_gradable_adjective(&with_e) {
4072 return Some(Self::capitalize(&with_e));
4073 }
4074
4075 None
4076 }
4077}
4078
4079#[cfg(test)]
4080#[path = "../../logicaffeine_tests/unit/logicaffeine_language/lexer/tests.rs"]
4081mod tests;