Skip to main content

logicaffeine_language/
lexer.rs

1//! Two-stage lexer for LOGOS natural language input.
2//!
3//! The lexer transforms natural language text into a token stream suitable
4//! for parsing. It operates in two stages:
5//!
6//! ## Stage 1: Line Lexer
7//!
8//! The [`LineLexer`] handles structural concerns:
9//!
10//! - **Indentation**: Tracks indent levels, emits `Indent`/`Dedent` tokens
11//! - **Block boundaries**: Identifies significant whitespace
12//! - **Content extraction**: Passes line content to Stage 2
13//!
14//! ## Stage 2: Word Lexer
15//!
16//! The [`Lexer`] performs word-level tokenization:
17//!
18//! - **Vocabulary lookup**: Identifies words via the lexicon database
19//! - **Morphological analysis**: Handles inflection (verb tenses, plurals)
20//! - **Ambiguity resolution**: Uses priority rules for ambiguous words
21//!
22//! ## Ambiguity Rules
23//!
24//! When a word matches multiple lexicon entries, priority determines the token:
25//!
26//! 1. **Quantifiers** over nouns ("some" → Quantifier, not Noun)
27//! 2. **Determiners** over adjectives ("the" → Determiner, not Adjective)
28//! 3. **Verbs** over nouns for -ing/-ed forms ("running" → Verb)
29//!
30//! ## Example
31//!
32//! ```text
33//! Input:  "Every cat sleeps."
34//! Output: [Quantifier("every"), Noun("cat"), Verb("sleeps"), Period]
35//! ```
36
37use crate::lexicon::{self, Aspect, Definiteness, Lexicon, Time};
38use crate::token::{
39    BlockType, CalendarUnit, FocusKind, MeasureKind, RawKind, Span, Token, TokenType,
40};
41use logicaffeine_base::Interner;
42
43// ============================================================================
44// Stage 1: Line Lexer (Spec §2.5.2)
45// ============================================================================
46
47/// Tokens emitted by the LineLexer (Stage 1).
48/// Handles structural tokens (Indent, Dedent, Newline) while treating
49/// all other content as opaque for Stage 2 word classification.
50#[derive(Debug, Clone, PartialEq)]
51pub enum LineToken {
52    /// Block increased indentation
53    Indent,
54    /// Block decreased indentation
55    Dedent,
56    /// Logical newline (statement boundary) - reserved for future use
57    Newline,
58    /// Content to be further tokenized (line content, trimmed)
59    Content {
60        text: String,
61        start: usize,
62        end: usize,
63    },
64}
65
66/// Stage 1 Lexer: Handles only lines, indentation, and structural tokens.
67/// Treats all other text as opaque `Content` for the Stage 2 WordLexer.
68pub struct LineLexer<'a> {
69    source: &'a str,
70    bytes: &'a [u8],
71    indent_stack: Vec<usize>,
72    pending_dedents: usize,
73    position: usize,
74    /// True if we need to emit Content for current line
75    has_pending_content: bool,
76    pending_content_start: usize,
77    pending_content_end: usize,
78    pending_content_text: String,
79    /// True after we've finished processing all lines
80    finished_lines: bool,
81    /// True if we've emitted at least one Indent (need to emit Dedents at EOF)
82    emitted_indent: bool,
83    /// Escape block body byte ranges to skip (start_byte, end_byte)
84    escape_body_ranges: Vec<(usize, usize)>,
85}
86
87impl<'a> LineLexer<'a> {
88    pub fn new(source: &'a str) -> Self {
89        Self {
90            source,
91            bytes: source.as_bytes(),
92            indent_stack: vec![0],
93            pending_dedents: 0,
94            position: 0,
95            has_pending_content: false,
96            pending_content_start: 0,
97            pending_content_end: 0,
98            pending_content_text: String::new(),
99            finished_lines: false,
100            emitted_indent: false,
101            escape_body_ranges: Vec::new(),
102        }
103    }
104
105    pub fn with_escape_ranges(source: &'a str, escape_body_ranges: Vec<(usize, usize)>) -> Self {
106        Self {
107            source,
108            bytes: source.as_bytes(),
109            indent_stack: vec![0],
110            pending_dedents: 0,
111            position: 0,
112            has_pending_content: false,
113            pending_content_start: 0,
114            pending_content_end: 0,
115            pending_content_text: String::new(),
116            finished_lines: false,
117            emitted_indent: false,
118            escape_body_ranges,
119        }
120    }
121
122    /// Check if a byte position falls within an escape body range.
123    fn is_in_escape_body(&self, pos: usize) -> bool {
124        self.escape_body_ranges
125            .iter()
126            .any(|(start, end)| pos >= *start && pos < *end)
127    }
128
129    /// Calculate indentation level at current position (at start of line).
130    /// Returns (indent_level, content_start_pos).
131    fn measure_indent(&self, line_start: usize) -> (usize, usize) {
132        let mut indent = 0;
133        let mut pos = line_start;
134
135        while pos < self.bytes.len() {
136            match self.bytes[pos] {
137                b' ' => {
138                    indent += 1;
139                    pos += 1;
140                }
141                b'\t' => {
142                    indent += 4; // Tab = 4 spaces
143                    pos += 1;
144                }
145                _ => break,
146            }
147        }
148
149        (indent, pos)
150    }
151
152    /// Read content from current position until end of line or EOF.
153    /// Returns (content_text, content_start, content_end, next_line_start).
154    fn read_line_content(&self, content_start: usize) -> (String, usize, usize, usize) {
155        let mut pos = content_start;
156
157        // Find end of line
158        while pos < self.bytes.len() && self.bytes[pos] != b'\n' {
159            pos += 1;
160        }
161
162        let content_end = pos;
163        let text = self.source[content_start..content_end]
164            .trim_end()
165            .to_string();
166
167        // Move past newline if present
168        let next_line_start = if pos < self.bytes.len() && self.bytes[pos] == b'\n' {
169            pos + 1
170        } else {
171            pos
172        };
173
174        (text, content_start, content_end, next_line_start)
175    }
176
177    /// Check if the line starting at `pos` is blank (only whitespace).
178    fn is_blank_line(&self, line_start: usize) -> bool {
179        let mut pos = line_start;
180        while pos < self.bytes.len() {
181            match self.bytes[pos] {
182                b' ' | b'\t' => pos += 1,
183                b'\n' => return true,
184                _ => return false,
185            }
186        }
187        true // EOF counts as blank
188    }
189
190    /// Process the next line and update internal state.
191    /// Returns true if we have tokens to emit, false if we're done.
192    fn process_next_line(&mut self) -> bool {
193        // Skip blank lines
194        while self.position < self.bytes.len() && self.is_blank_line(self.position) {
195            // Skip to next line
196            while self.position < self.bytes.len() && self.bytes[self.position] != b'\n' {
197                self.position += 1;
198            }
199            if self.position < self.bytes.len() {
200                self.position += 1; // Skip the newline
201            }
202        }
203
204        // Check if we've reached EOF
205        if self.position >= self.bytes.len() {
206            self.finished_lines = true;
207            // Emit remaining dedents at EOF
208            if self.indent_stack.len() > 1 {
209                self.pending_dedents = self.indent_stack.len() - 1;
210                self.indent_stack.truncate(1);
211            }
212            return self.pending_dedents > 0;
213        }
214
215        // Measure indentation of current line
216        let (line_indent, content_start) = self.measure_indent(self.position);
217
218        // Read line content
219        let (text, start, end, next_pos) = self.read_line_content(content_start);
220
221        // Skip if content is empty (shouldn't happen after blank line skip, but be safe)
222        if text.is_empty() {
223            self.position = next_pos;
224            return self.process_next_line();
225        }
226
227        let current_indent = *self.indent_stack.last().unwrap();
228
229        // Handle indentation changes
230        if line_indent > current_indent {
231            // Indent: push new level
232            self.indent_stack.push(line_indent);
233            self.emitted_indent = true;
234            // Store content to emit after Indent
235            self.has_pending_content = true;
236            self.pending_content_text = text;
237            self.pending_content_start = start;
238            self.pending_content_end = end;
239            self.position = next_pos;
240            // We'll emit Indent first, then Content
241            return true;
242        } else if line_indent < current_indent {
243            // Dedent: pop until we match
244            while self.indent_stack.len() > 1 {
245                let top = *self.indent_stack.last().unwrap();
246                if line_indent < top {
247                    self.indent_stack.pop();
248                    self.pending_dedents += 1;
249                } else {
250                    break;
251                }
252            }
253            // Store content to emit after Dedents
254            self.has_pending_content = true;
255            self.pending_content_text = text;
256            self.pending_content_start = start;
257            self.pending_content_end = end;
258            self.position = next_pos;
259            return true;
260        } else {
261            // Same indentation level
262            self.has_pending_content = true;
263            self.pending_content_text = text;
264            self.pending_content_start = start;
265            self.pending_content_end = end;
266            self.position = next_pos;
267            return true;
268        }
269    }
270}
271
272impl<'a> Iterator for LineLexer<'a> {
273    type Item = LineToken;
274
275    fn next(&mut self) -> Option<LineToken> {
276        // 1. Emit pending dedents first
277        if self.pending_dedents > 0 {
278            self.pending_dedents -= 1;
279            return Some(LineToken::Dedent);
280        }
281
282        // 2. Emit pending content
283        if self.has_pending_content {
284            self.has_pending_content = false;
285            let text = std::mem::take(&mut self.pending_content_text);
286            let start = self.pending_content_start;
287            let end = self.pending_content_end;
288            return Some(LineToken::Content { text, start, end });
289        }
290
291        // 3. Check if we need to emit Indent (after pushing to stack)
292        // This happens when we detected an indent but haven't emitted the token yet
293        // We need to check if indent_stack was just modified
294
295        // 4. Process next line
296        if !self.finished_lines {
297            let had_indent = self.indent_stack.len();
298            if self.process_next_line() {
299                // Check if we added an indent level
300                if self.indent_stack.len() > had_indent {
301                    return Some(LineToken::Indent);
302                }
303                // Check if we have pending dedents
304                if self.pending_dedents > 0 {
305                    self.pending_dedents -= 1;
306                    return Some(LineToken::Dedent);
307                }
308                // Otherwise emit content
309                if self.has_pending_content {
310                    self.has_pending_content = false;
311                    let text = std::mem::take(&mut self.pending_content_text);
312                    let start = self.pending_content_start;
313                    let end = self.pending_content_end;
314                    return Some(LineToken::Content { text, start, end });
315                }
316            } else if self.pending_dedents > 0 {
317                // EOF with pending dedents
318                self.pending_dedents -= 1;
319                return Some(LineToken::Dedent);
320            }
321        }
322
323        // 5. Emit any remaining dedents at EOF
324        if self.pending_dedents > 0 {
325            self.pending_dedents -= 1;
326            return Some(LineToken::Dedent);
327        }
328
329        None
330    }
331}
332
333// ============================================================================
334// Stage 2: Word Lexer (existing Lexer)
335// ============================================================================
336
337#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
338pub enum LexerMode {
339    #[default]
340    Declarative, // Logic, Theorems, Definitions
341    Imperative, // Main, Functions, Code
342}
343
344pub struct Lexer<'a> {
345    words: Vec<WordItem>,
346    pos: usize,
347    lexicon: Lexicon,
348    interner: &'a mut Interner,
349    input_len: usize,
350    in_let_context: bool,
351    mode: LexerMode,
352    source: String,
353    /// Escape block body byte ranges: (skip_start, skip_end) for filtering LineLexer events
354    escape_body_ranges: Vec<(usize, usize)>,
355}
356
357struct WordItem {
358    word: String,
359    trailing_punct: Option<char>,
360    start: usize,
361    end: usize,
362    punct_pos: Option<usize>,
363}
364
365impl<'a> Lexer<'a> {
366    /// Creates a new lexer for the given input text.
367    ///
368    /// The lexer will tokenize natural language text according to the
369    /// lexicon database, performing morphological analysis and ambiguity
370    /// resolution.
371    ///
372    /// # Arguments
373    ///
374    /// * `input` - The natural language text to tokenize
375    /// * `interner` - String interner for efficient symbol handling
376    ///
377    /// # Example
378    ///
379    /// ```
380    /// use logicaffeine_language::lexer::Lexer;
381    /// use logicaffeine_base::Interner;
382    ///
383    /// let mut interner = Interner::new();
384    /// let mut lexer = Lexer::new("Every cat sleeps.", &mut interner);
385    /// let tokens = lexer.tokenize();
386    ///
387    /// assert_eq!(tokens.len(), 5); // Quantifier, Noun, Verb, Period, EOI
388    /// ```
389    pub fn new(input: &str, interner: &'a mut Interner) -> Self {
390        let input = Self::blank_optional_header_colons(input);
391        let escape_ranges = Self::find_escape_block_ranges(&input);
392        let escape_body_ranges: Vec<(usize, usize)> = escape_ranges
393            .iter()
394            .map(|(_, end, content_start, _, _, _)| (*content_start, *end))
395            .collect();
396        let words = Self::split_into_words(&input, &escape_ranges);
397        let input_len = input.len();
398
399        Lexer {
400            words,
401            pos: 0,
402            lexicon: Lexicon::new(),
403            interner,
404            input_len,
405            in_let_context: false,
406            mode: LexerMode::Declarative,
407            source: input,
408            escape_body_ranges,
409        }
410    }
411
412    /// Every `## <Header>` whose own grammar does not already read a trailing `:`
413    /// (`BlockType::accepts_optional_trailing_colon`: `Main`, `Component`, `Store`, `Page`,
414    /// `Error page` — never the ones that already require one, like `Directive`/`Route`, or
415    /// `Test`, whose colon has a different, load-bearing job) accepts one anyway, silently: a
416    /// habitual trailing `:` at the end of exactly one of these header lines is blanked to a
417    /// space BEFORE tokenizing, so it never becomes a `Colon` token at all.
418    ///
419    /// Done here, on the raw source, rather than as a token-stream pass: LOGOS does not emit a
420    /// `Newline` token between two ordinary statements at the same indentation (only `Indent`/
421    /// `Dedent` mark structure, and a table row gets its own explicit one) — `## Main:\nShow
422    /// "hi".` tokenizes as `BlockHeader, Colon, Show, …` with NO token between the colon and
423    /// `Show` to search up to. The source string, in contrast, always has the `\n` that ends the
424    /// header's own line, so "the header's own line" is unambiguous here and nowhere else. A
425    /// byte-for-byte blank (never a delete) keeps every later span exactly where it already was.
426    fn blank_optional_header_colons(input: &str) -> String {
427        const ACCEPTING: &[&str] = &["main", "component", "store", "page", "error"];
428        let mut out = input.as_bytes().to_vec();
429        let mut line_start = 0usize;
430        for line_end in input
431            .match_indices('\n')
432            .map(|(i, _)| i)
433            .chain(std::iter::once(input.len()))
434        {
435            let line = &input[line_start..line_end];
436            let trimmed_start = line.trim_start();
437            if let Some(after_hashes) = trimmed_start.strip_prefix("##") {
438                let first_word = after_hashes
439                    .trim_start()
440                    .split(|c: char| c.is_whitespace() || c == ':')
441                    .next()
442                    .unwrap_or("");
443                if ACCEPTING.contains(&first_word.to_ascii_lowercase().as_str()) {
444                    let trimmed_end = line.trim_end();
445                    if let Some(':') = trimmed_end.chars().next_back() {
446                        let colon_offset = line_start + trimmed_end.len() - 1;
447                        out[colon_offset] = b' ';
448                    }
449                }
450            }
451            line_start = line_end + 1;
452        }
453        String::from_utf8(out).expect("blanking a `:` to a space cannot break UTF-8")
454    }
455
456    /// A lexer for a fragment that is CODE, not prose — an LGX template hole, a directive
457    /// value, a filter argument. Ordinary [`Lexer::new`] starts in declarative mode and only
458    /// switches on a `## Main` / `## To` / `## Component` header, which a bare fragment has
459    /// none of; a hole like `todo's id` or `a . b` must tokenize the imperative way.
460    pub fn new_imperative(input: &str, interner: &'a mut Interner) -> Self {
461        let mut lexer = Self::new(input, interner);
462        lexer.mode = LexerMode::Imperative;
463        lexer
464    }
465
466    /// Pre-scan source text for escape block bodies.
467    /// Returns (skip_start_byte, skip_end_byte, content_start_byte, raw_code) tuples.
468    /// `skip_start` is the line start (for byte skipping in split_into_words).
469    /// `content_start` is after leading whitespace (for token span alignment with Indent events).
470    fn find_escape_block_ranges(source: &str) -> Vec<(usize, usize, usize, String, char, usize)> {
471        let mut ranges = Vec::new();
472        let lines: Vec<&str> = source.split('\n').collect();
473        let mut line_starts: Vec<usize> = Vec::with_capacity(lines.len());
474        let mut pos = 0;
475        for line in &lines {
476            line_starts.push(pos);
477            pos += line.len() + 1; // +1 for the newline
478        }
479
480        let mut i = 0;
481        while i < lines.len() {
482            let trimmed = lines[i].trim();
483            // Check if this line contains an escape header: "Escape to Rust:"
484            // Matches both statement position (whole line) and expression position
485            // (e.g., "Let x: Int be Escape to Rust:")
486            let lower = trimmed.to_lowercase();
487            // An `Escape to <Language>:` header, an LGX `View:` / `Style:` section, or a
488            // `## To … -> View:` template function — each opens a body captured verbatim.
489            let raw_kind: Option<char> = if lower == "escape to rust:"
490                || lower.ends_with(" escape to rust:")
491                || (lower.starts_with("escape to ") && lower.ends_with(':'))
492            {
493                Some('E')
494            } else if trimmed == "Style:" {
495                Some('S')
496            } else if trimmed == "Head:" {
497                // A `Head:` body is ALWAYS markup — there is no legacy indentation
498                // dialect for it — so it needs none of the `View:` fallback below.
499                Some('H')
500            } else if trimmed == "View:"
501                || (trimmed.starts_with("## ") && trimmed.ends_with("-> View:"))
502            {
503                Some('V')
504            } else {
505                None
506            };
507            if let Some(kind) = raw_kind {
508                // Find the body: subsequent lines with deeper indentation
509                let header_indent = Self::measure_indent_static(lines[i]);
510                i += 1;
511
512                // Skip blank lines to find the first body line
513                let mut body_start_line = i;
514                while body_start_line < lines.len() && lines[body_start_line].trim().is_empty() {
515                    body_start_line += 1;
516                }
517
518                if body_start_line >= lines.len() {
519                    // No body found
520                    continue;
521                }
522
523                let base_indent = Self::measure_indent_static(lines[body_start_line]);
524                if base_indent <= header_indent {
525                    // No indented body
526                    continue;
527                }
528                // A view body is captured raw only when it IS markup: the legacy indentation DSL's
529                // bodies begin with an element word (`div [class="x"]:`), and a `-> View:` function
530                // or route may answer through statements (`Return answer(who).`) — neither begins
531                // with `<` or `{`.
532                if kind == 'V' {
533                    let first = lines[body_start_line].trim_start();
534                    if !(first.starts_with('<') || first.starts_with('{')) {
535                        continue;
536                    }
537                }
538
539                // Capture all lines at base_indent or deeper
540                let body_byte_start = line_starts[body_start_line];
541                let mut body_end_line = body_start_line;
542                let mut code_lines: Vec<String> = Vec::new();
543
544                let mut j = body_start_line;
545                while j < lines.len() {
546                    let line = lines[j];
547                    if line.trim().is_empty() {
548                        // Blank lines are preserved
549                        code_lines.push(String::new());
550                        body_end_line = j;
551                        j += 1;
552                        continue;
553                    }
554                    let line_indent = Self::measure_indent_static(line);
555                    if line_indent < base_indent {
556                        break;
557                    }
558                    // Strip base indentation
559                    let stripped = Self::strip_indent(line, base_indent);
560                    code_lines.push(stripped);
561                    body_end_line = j;
562                    j += 1;
563                }
564
565                // Trim trailing empty lines from code
566                while code_lines.last().map_or(false, |l| l.is_empty()) {
567                    code_lines.pop();
568                }
569
570                if !code_lines.is_empty() {
571                    let body_byte_end = if body_end_line + 1 < lines.len() {
572                        line_starts[body_end_line + 1]
573                    } else {
574                        source.len()
575                    };
576                    // Compute content start (after leading whitespace of first body line)
577                    let content_start =
578                        body_byte_start + Self::leading_whitespace_bytes(lines[body_start_line]);
579                    let raw_code = code_lines.join("\n");
580                    ranges.push((
581                        body_byte_start,
582                        body_byte_end,
583                        content_start,
584                        raw_code,
585                        kind,
586                        base_indent,
587                    ));
588                }
589
590                i = j;
591            } else {
592                i += 1;
593            }
594        }
595
596        ranges
597    }
598
599    /// The whitespace-delimited words after the current `## Header` word on its line,
600    /// read verbatim from the source (a URL path must not be split into LOGOS
601    /// punctuation), each with the byte offset just past it. A peek: consumes nothing.
602    fn header_line_words(&self) -> Vec<(String, usize)> {
603        let hdr_end = self.words[self.pos].end;
604        let line_end = self.source[hdr_end..]
605            .find('\n')
606            .map(|k| hdr_end + k)
607            .unwrap_or(self.source.len());
608        let mut out = Vec::new();
609        let mut cursor = hdr_end;
610        while cursor < line_end {
611            let rest = &self.source[cursor..line_end];
612            let skip = rest.len() - rest.trim_start().len();
613            let start = cursor + skip;
614            if start >= line_end {
615                break;
616            }
617            let len = self.source[start..line_end]
618                .find(char::is_whitespace)
619                .unwrap_or(line_end - start);
620            out.push((self.source[start..start + len].to_string(), start + len));
621            cursor = start + len;
622        }
623        out
624    }
625
626    /// Advance `pos` past every split word that ends at or before `end` (the tokenize
627    /// loop's own advance then steps past the last one), like the `## Test:` capture.
628    fn consume_header_through(&mut self, end: usize) {
629        while self.pos + 1 < self.words.len() && self.words[self.pos + 1].start < end {
630            self.pos += 1;
631        }
632    }
633
634    /// Recognize an LGX header by the SHAPE of its line, not its first word alone, so a
635    /// prose heading (`## Error Propagation`, `## Before you begin`) stays prose:
636    /// `## Page /path`, `## Route METHOD /path`, `## Store Name`, `## Directive name (…)`,
637    /// `## Before each request`, `## Redirect /a -> /b`, `## Error page 404`. The raw path
638    /// words are interned and consumed; everything after them lexes normally.
639    fn try_lgx_header(&mut self, lowered: &str) -> Option<BlockType> {
640        let words = self.header_line_words();
641        let word = |i: usize| words.get(i).map(|(w, _)| w.as_str()).unwrap_or("");
642        let end_of = |i: usize| words.get(i).map(|(_, e)| *e).unwrap_or(0);
643        let is_path = |w: &str| w.starts_with('/');
644        let is_method = |w: &str| !w.is_empty() && w.chars().all(|c| c.is_ascii_uppercase());
645        match lowered {
646            "page" if is_path(word(0)) => {
647                let path = self.interner.intern(word(0).trim_end_matches('.'));
648                self.consume_header_through(end_of(0));
649                Some(BlockType::Page { path })
650            }
651            "route" if is_method(word(0)) && is_path(word(1)) => {
652                let method = self.interner.intern(word(0));
653                let path = self.interner.intern(word(1).trim_end_matches('.'));
654                self.consume_header_through(end_of(1));
655                Some(BlockType::Route { method, path })
656            }
657            "store"
658                if word(0)
659                    .chars()
660                    .next()
661                    .is_some_and(|c| c.is_ascii_uppercase()) =>
662            {
663                Some(BlockType::Store)
664            }
665            // `## Session` takes no name: a program has one session, the way it has one router.
666            // The guard keeps `## Session Something` out — that is a typo or a misunderstanding,
667            // and falling through to prose would silently discard the block.
668            "session" if word(0).is_empty() => Some(BlockType::Session),
669            "directive"
670                if !word(0).is_empty()
671                    && word(0)
672                        .chars()
673                        .next()
674                        .is_some_and(|c| c.is_ascii_lowercase())
675                    && words.iter().any(|(w, _)| w.contains('(')) =>
676            {
677                Some(BlockType::Directive)
678            }
679            "before" if word(0) == "each" && word(1).trim_end_matches(':') == "request" => {
680                self.consume_header_through(end_of(1));
681                Some(BlockType::Middleware)
682            }
683            "redirect" if is_path(word(0)) && word(1) == "->" && is_path(word(2)) => {
684                let from = self.interner.intern(word(0));
685                let to = self.interner.intern(word(2).trim_end_matches('.'));
686                self.consume_header_through(end_of(2));
687                Some(BlockType::Redirect { from, to })
688            }
689            "error" if word(0) == "page" => {
690                let status = word(1).trim_end_matches('.').parse::<u16>().ok()?;
691                self.consume_header_through(end_of(1));
692                Some(BlockType::ErrorPage { status })
693            }
694            _ => None,
695        }
696    }
697
698    /// Count leading whitespace bytes in a line.
699    fn leading_whitespace_bytes(line: &str) -> usize {
700        let mut count = 0;
701        for c in line.chars() {
702            match c {
703                ' ' | '\t' => count += c.len_utf8(),
704                _ => break,
705            }
706        }
707        count
708    }
709
710    /// Measure indent of a line (static helper for pre-scan).
711    fn measure_indent_static(line: &str) -> usize {
712        let mut indent = 0;
713        for c in line.chars() {
714            match c {
715                ' ' => indent += 1,
716                '\t' => indent += 4,
717                _ => break,
718            }
719        }
720        indent
721    }
722
723    /// Strip `count` leading spaces/tabs from a line.
724    fn strip_indent(line: &str, count: usize) -> String {
725        let mut stripped = 0;
726        let mut byte_pos = 0;
727        for (i, c) in line.char_indices() {
728            if stripped >= count {
729                byte_pos = i;
730                break;
731            }
732            match c {
733                ' ' => {
734                    stripped += 1;
735                    byte_pos = i + 1;
736                }
737                '\t' => {
738                    stripped += 4;
739                    byte_pos = i + 1;
740                }
741                _ => {
742                    byte_pos = i;
743                    break;
744                }
745            }
746        }
747        if stripped < count {
748            byte_pos = line.len();
749        }
750        line[byte_pos..].to_string()
751    }
752
753    fn split_into_words(
754        input: &str,
755        escape_ranges: &[(usize, usize, usize, String, char, usize)],
756    ) -> Vec<WordItem> {
757        let mut items = Vec::new();
758        let mut current_word = String::new();
759        let mut word_start = 0;
760        let chars: Vec<char> = input.chars().collect();
761        let mut char_idx = 0;
762        let mut skip_count = 0;
763        // Inside `[` … `]` a digit-flanked comma is a list separator, not a
764        // thousands separator — `[1,2,3]` is three elements. Money literals
765        // (`$125,000`) keep their commas at any depth; line-local so an
766        // unbalanced bracket never poisons the rest of the input.
767        let mut bracket_depth: usize = 0;
768        // Track byte offset for escape range matching
769        let mut skip_to_byte: Option<usize> = None;
770
771        for (i, c) in input.char_indices() {
772            if skip_count > 0 {
773                skip_count -= 1;
774                char_idx += 1;
775                continue;
776            }
777            // Skip bytes inside escape block bodies
778            if let Some(end) = skip_to_byte {
779                if i < end {
780                    char_idx += 1;
781                    continue;
782                }
783                skip_to_byte = None;
784                word_start = i;
785            }
786            // Check if this byte position starts an escape block body
787            if let Some((_, end, content_start, raw_code, kind, indent)) =
788                escape_ranges.iter().find(|(s, _, _, _, _, _)| i == *s)
789            {
790                // Flush any pending word
791                if !current_word.is_empty() {
792                    items.push(WordItem {
793                        word: std::mem::take(&mut current_word),
794                        trailing_punct: None,
795                        start: word_start,
796                        end: i,
797                        punct_pos: None,
798                    });
799                }
800                // Emit the entire block as a single \x00ESC: marker
801                // Use content_start (after whitespace) for span alignment with Indent events
802                items.push(WordItem {
803                    word: match kind {
804                        'E' => format!("\x00ESC:{}", raw_code),
805                        // `\x00LGX:<kind>:<content_start>:<indent>:<body>` — decoded in
806                        // `tokenize` (`\x00RAW:` is the raw STRING literal marker, a different
807                        // thing). `indent` is what was stripped from each body line, so a byte
808                        // offset inside the body maps back to its place in the file.
809                        k => format!("\x00LGX:{k}:{content_start}:{indent}:{raw_code}"),
810                    },
811                    trailing_punct: None,
812                    start: *content_start,
813                    end: *end,
814                    punct_pos: None,
815                });
816                skip_to_byte = Some(*end);
817                word_start = *end;
818                char_idx += 1;
819                continue;
820            }
821            let next_pos = i + c.len_utf8();
822            match c {
823                // "8:15 pm" — a space between a H:MM time and "am"/"pm" is part of
824                // the time literal: skip the flush so "pm" joins "8:15" → "8:15pm".
825                ' ' if Self::is_time_space_before_ampm(&current_word, &chars, char_idx) => {}
826                ' ' | '\t' | '\n' | '\r' => {
827                    if c == '\n' {
828                        bracket_depth = 0;
829                    }
830                    if !current_word.is_empty() {
831                        items.push(WordItem {
832                            word: std::mem::take(&mut current_word),
833                            trailing_punct: None,
834                            start: word_start,
835                            end: i,
836                            punct_pos: None,
837                        });
838                    }
839                    word_start = next_pos;
840                }
841                '.' => {
842                    // Check if this is a decimal point (digit before and after)
843                    let prev_is_digit = !current_word.is_empty()
844                        && current_word
845                            .chars()
846                            .last()
847                            .map_or(false, |ch| ch.is_ascii_digit());
848                    let next_is_digit =
849                        char_idx + 1 < chars.len() && chars[char_idx + 1].is_ascii_digit();
850
851                    // Field-access / UFCS DOT: an identifier char (letter/`_`) or a
852                    // closing `)`/`]` immediately before, and an identifier-start
853                    // immediately after — no whitespace either side. `p.x`, `xs.f(a)`,
854                    // `(e).m`. Digit-glue wins (`5.sqrt` stays `5` + period + `sqrt`,
855                    // `5.0` stays a decimal), and `Show x.` stays a sentence (the next
856                    // char is a newline, not an identifier). Mode is resolved later:
857                    // `classify_with_lookahead` maps the `\x00DOT` marker to `Dot`
858                    // (imperative) or a sentence `Period` (declarative — prose keeps
859                    // `e.g.` exactly as today).
860                    let prev_ident = current_word
861                        .chars()
862                        .last()
863                        .map_or(false, |ch| ch.is_alphabetic() || ch == '_');
864                    let prev_close = current_word.is_empty()
865                        && char_idx > 0
866                        && matches!(chars[char_idx - 1], ')' | ']');
867                    let next_ident = char_idx + 1 < chars.len()
868                        && (chars[char_idx + 1].is_alphabetic() || chars[char_idx + 1] == '_');
869
870                    if prev_is_digit && next_is_digit {
871                        // This is a decimal point, include it in the current word
872                        current_word.push(c);
873                    } else if (prev_ident || prev_close) && next_ident {
874                        // Flush the receiver, then the mode-deferred dot marker.
875                        if !current_word.is_empty() {
876                            items.push(WordItem {
877                                word: std::mem::take(&mut current_word),
878                                trailing_punct: None,
879                                start: word_start,
880                                end: i,
881                                punct_pos: None,
882                            });
883                        }
884                        items.push(WordItem {
885                            word: "\x00DOT".to_string(),
886                            trailing_punct: None,
887                            start: i,
888                            end: next_pos,
889                            punct_pos: None,
890                        });
891                        word_start = next_pos;
892                    } else {
893                        // This is a sentence period
894                        if !current_word.is_empty() {
895                            items.push(WordItem {
896                                word: std::mem::take(&mut current_word),
897                                trailing_punct: Some(c),
898                                start: word_start,
899                                end: i,
900                                punct_pos: Some(i),
901                            });
902                        } else {
903                            items.push(WordItem {
904                                word: String::new(),
905                                trailing_punct: Some(c),
906                                start: i,
907                                end: next_pos,
908                                punct_pos: Some(i),
909                            });
910                        }
911                        word_start = next_pos;
912                    }
913                }
914                '#' => {
915                    // Check for ## block header (markdown-style)
916                    if char_idx + 1 < chars.len() && chars[char_idx + 1] == '#' {
917                        // This is a ## block header
918                        // Skip the second # and capture the next word as a block header
919                        if !current_word.is_empty() {
920                            items.push(WordItem {
921                                word: std::mem::take(&mut current_word),
922                                trailing_punct: None,
923                                start: word_start,
924                                end: i,
925                                punct_pos: None,
926                            });
927                        }
928                        // Skip whitespace after ##
929                        let header_start = i;
930                        let mut j = char_idx + 2;
931                        while j < chars.len() && (chars[j] == ' ' || chars[j] == '\t') {
932                            j += 1;
933                        }
934                        // Capture the block type word
935                        let mut block_word = String::from("##");
936                        while j < chars.len() && chars[j].is_alphabetic() {
937                            block_word.push(chars[j]);
938                            j += 1;
939                        }
940                        if block_word.len() > 2 {
941                            items.push(WordItem {
942                                word: block_word,
943                                trailing_punct: None,
944                                start: header_start,
945                                end: header_start + (j - char_idx),
946                                punct_pos: None,
947                            });
948                        }
949                        skip_count = j - char_idx - 1;
950                        word_start = header_start + (j - char_idx);
951                    } else {
952                        // Single # - treat as comment, skip to end of line
953                        // Count how many chars to skip (without modifying char_idx here -
954                        // the main loop's skip handler will increment it)
955                        let mut look_ahead = char_idx + 1;
956                        while look_ahead < chars.len() && chars[look_ahead] != '\n' {
957                            skip_count += 1;
958                            look_ahead += 1;
959                        }
960                        if !current_word.is_empty() {
961                            items.push(WordItem {
962                                word: std::mem::take(&mut current_word),
963                                trailing_punct: None,
964                                start: word_start,
965                                end: i,
966                                punct_pos: None,
967                            });
968                        }
969                        word_start = look_ahead + 1; // Start after the newline
970                    }
971                }
972                // String literals: "hello world" or """multi-line"""
973                '"' => {
974                    // `r"…"` RAW string: the adjacent `r` prefix is part of
975                    // the literal, not a word — backslashes stay verbatim
976                    // (paths, regexes). Only the exact word `r` glued to the
977                    // quote triggers it; `Show r.` (a variable) is untouched.
978                    let raw_string = current_word == "r";
979                    if raw_string {
980                        current_word.clear();
981                    } else if !current_word.is_empty() {
982                        items.push(WordItem {
983                            word: std::mem::take(&mut current_word),
984                            trailing_punct: None,
985                            start: word_start,
986                            end: i,
987                            punct_pos: None,
988                        });
989                    }
990
991                    // Check for triple-quote: """
992                    if char_idx + 2 < chars.len()
993                        && chars[char_idx + 1] == '"'
994                        && chars[char_idx + 2] == '"'
995                    {
996                        let string_start = i;
997                        let mut j = char_idx + 3; // skip opening """
998                                                  // Skip optional newline after opening """
999                        if j < chars.len() && chars[j] == '\n' {
1000                            j += 1;
1001                        }
1002                        let mut raw_content = String::new();
1003                        // Scan until closing """
1004                        while j < chars.len() {
1005                            if j + 2 < chars.len()
1006                                && chars[j] == '"'
1007                                && chars[j + 1] == '"'
1008                                && chars[j + 2] == '"'
1009                            {
1010                                break;
1011                            }
1012                            raw_content.push(chars[j]);
1013                            j += 1;
1014                        }
1015                        // Strip trailing newline before closing """
1016                        if raw_content.ends_with('\n') {
1017                            raw_content.pop();
1018                        }
1019                        // Dedent: find minimum common indentation and strip it
1020                        let dedented = Self::dedent_triple_quote(&raw_content);
1021                        let end_pos = if j + 2 < chars.len() {
1022                            j + 3
1023                        } else {
1024                            chars.len()
1025                        };
1026                        items.push(WordItem {
1027                            word: format!("\x00STR:{}", dedented),
1028                            trailing_punct: None,
1029                            start: string_start,
1030                            end: end_pos,
1031                            punct_pos: None,
1032                        });
1033                        // Skip past the closing """
1034                        if j + 2 < chars.len() {
1035                            skip_count = (j + 2) - char_idx;
1036                        } else {
1037                            skip_count = chars.len() - 1 - char_idx;
1038                        }
1039                        word_start = end_pos;
1040                    } else {
1041                        // Single-quoted string: scan until closing quote
1042                        let string_start = i;
1043                        let mut j = char_idx + 1;
1044                        let mut string_content = String::new();
1045                        while j < chars.len() && chars[j] != '"' {
1046                            if chars[j] == '\\' && j + 1 < chars.len() && !raw_string {
1047                                // DECODE the escape (`"a\nb"` is two lines,
1048                                // not the letters a-n-b). Unknown escapes and
1049                                // malformed \u{…} stay verbatim rather than
1050                                // silently dropping the backslash.
1051                                j += 1;
1052                                match chars[j] {
1053                                    'n' => string_content.push('\n'),
1054                                    't' => string_content.push('\t'),
1055                                    'r' => string_content.push('\r'),
1056                                    '0' => string_content.push('\0'),
1057                                    '\\' => string_content.push('\\'),
1058                                    '"' => string_content.push('"'),
1059                                    'u' if j + 1 < chars.len() && chars[j + 1] == '{' => {
1060                                        let mut k = j + 2;
1061                                        let mut hex = String::new();
1062                                        while k < chars.len() && chars[k] != '}' {
1063                                            hex.push(chars[k]);
1064                                            k += 1;
1065                                        }
1066                                        match u32::from_str_radix(&hex, 16)
1067                                            .ok()
1068                                            .and_then(char::from_u32)
1069                                        {
1070                                            Some(c) if k < chars.len() => {
1071                                                string_content.push(c);
1072                                                j = k;
1073                                            }
1074                                            _ => {
1075                                                string_content.push('\\');
1076                                                string_content.push('u');
1077                                            }
1078                                        }
1079                                    }
1080                                    other => {
1081                                        string_content.push('\\');
1082                                        string_content.push(other);
1083                                    }
1084                                }
1085                            } else {
1086                                string_content.push(chars[j]);
1087                            }
1088                            j += 1;
1089                        }
1090
1091                        // Create a special marker for string literals
1092                        // We prefix with a special character to identify in tokenize()
1093                        //
1094                        // `end` and the following `word_start` are BYTE offsets
1095                        // (Span fields are byte positions, like `start`/`i`); `j`
1096                        // is a CHAR index, so convert through the byte length of
1097                        // the covered source chars. Emitting `j + 1` (a char
1098                        // index) produced a byte-start / char-end span that
1099                        // underflows downstream on multibyte input.
1100                        let end_char = if j < chars.len() { j + 1 } else { j };
1101                        let end_byte = string_start
1102                            + chars[char_idx..end_char]
1103                                .iter()
1104                                .map(|c| c.len_utf8())
1105                                .sum::<usize>();
1106                        // `r"..."` is VERBATIM: braces are literal, never an
1107                        // interpolation hole — the `\x00RAW:` marker tells Stage 2
1108                        // to skip both the hole scan and the `{{`→`{` collapse.
1109                        let marker = if raw_string { "\x00RAW:" } else { "\x00STR:" };
1110                        items.push(WordItem {
1111                            word: format!("{marker}{string_content}"),
1112                            trailing_punct: None,
1113                            start: string_start,
1114                            end: end_byte,
1115                            punct_pos: None,
1116                        });
1117
1118                        // Skip past the closing quote
1119                        if j < chars.len() {
1120                            skip_count = j - char_idx;
1121                        } else {
1122                            skip_count = j - char_idx - 1;
1123                        }
1124                        word_start = end_byte;
1125                    }
1126                }
1127                // Character literals with backticks: `x`
1128                '`' => {
1129                    // Push any pending word
1130                    if !current_word.is_empty() {
1131                        items.push(WordItem {
1132                            word: std::mem::take(&mut current_word),
1133                            trailing_punct: None,
1134                            start: word_start,
1135                            end: i,
1136                            punct_pos: None,
1137                        });
1138                    }
1139
1140                    // Scan for character content and closing backtick
1141                    let char_start = i;
1142                    let mut j = char_idx + 1;
1143                    let mut char_content = String::new();
1144
1145                    if j < chars.len() {
1146                        if chars[j] == '\\' && j + 1 < chars.len() {
1147                            // Escape sequence
1148                            j += 1;
1149                            let escaped_char = match chars[j] {
1150                                'n' => '\n',
1151                                't' => '\t',
1152                                'r' => '\r',
1153                                '\\' => '\\',
1154                                '`' => '`',
1155                                '0' => '\0',
1156                                c => c,
1157                            };
1158                            char_content.push(escaped_char);
1159                            j += 1;
1160                        } else if chars[j] != '`' {
1161                            // Regular character
1162                            char_content.push(chars[j]);
1163                            j += 1;
1164                        }
1165                    }
1166
1167                    // Expect closing backtick
1168                    if j < chars.len() && chars[j] == '`' {
1169                        j += 1; // skip closing backtick
1170                    }
1171
1172                    // Create a special marker for char literals
1173                    items.push(WordItem {
1174                        word: format!("\x00CHAR:{}", char_content),
1175                        trailing_punct: None,
1176                        start: char_start,
1177                        end: if j <= chars.len() {
1178                            char_start + (j - char_idx)
1179                        } else {
1180                            char_start + 1
1181                        },
1182                        punct_pos: None,
1183                    });
1184
1185                    if j > char_idx + 1 {
1186                        skip_count = j - char_idx - 1;
1187                    }
1188                    word_start = char_start + (j - char_idx);
1189                }
1190                // Handle -> as a single token for return type syntax
1191                '-' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '>' => {
1192                    // Push any pending word first
1193                    if !current_word.is_empty() {
1194                        items.push(WordItem {
1195                            word: std::mem::take(&mut current_word),
1196                            trailing_punct: None,
1197                            start: word_start,
1198                            end: i,
1199                            punct_pos: None,
1200                        });
1201                    }
1202                    // Push -> as its own word
1203                    items.push(WordItem {
1204                        word: "->".to_string(),
1205                        trailing_punct: None,
1206                        start: i,
1207                        end: i + 2,
1208                        punct_pos: None,
1209                    });
1210                    skip_count = 1; // Skip the '>' character
1211                    word_start = i + 2;
1212                }
1213                // Grand Challenge: Handle <= as a single token
1214                '<' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1215                    if !current_word.is_empty() {
1216                        items.push(WordItem {
1217                            word: std::mem::take(&mut current_word),
1218                            trailing_punct: None,
1219                            start: word_start,
1220                            end: i,
1221                            punct_pos: None,
1222                        });
1223                    }
1224                    items.push(WordItem {
1225                        word: "<=".to_string(),
1226                        trailing_punct: None,
1227                        start: i,
1228                        end: i + 2,
1229                        punct_pos: None,
1230                    });
1231                    skip_count = 1;
1232                    word_start = i + 2;
1233                }
1234                // Grand Challenge: Handle >= as a single token
1235                '>' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1236                    if !current_word.is_empty() {
1237                        items.push(WordItem {
1238                            word: std::mem::take(&mut current_word),
1239                            trailing_punct: None,
1240                            start: word_start,
1241                            end: i,
1242                            punct_pos: None,
1243                        });
1244                    }
1245                    items.push(WordItem {
1246                        word: ">=".to_string(),
1247                        trailing_punct: None,
1248                        start: i,
1249                        end: i + 2,
1250                        punct_pos: None,
1251                    });
1252                    skip_count = 1;
1253                    word_start = i + 2;
1254                }
1255                // Handle == as a single token
1256                '=' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1257                    if !current_word.is_empty() {
1258                        items.push(WordItem {
1259                            word: std::mem::take(&mut current_word),
1260                            trailing_punct: None,
1261                            start: word_start,
1262                            end: i,
1263                            punct_pos: None,
1264                        });
1265                    }
1266                    items.push(WordItem {
1267                        word: "==".to_string(),
1268                        trailing_punct: None,
1269                        start: i,
1270                        end: i + 2,
1271                        punct_pos: None,
1272                    });
1273                    skip_count = 1;
1274                    word_start = i + 2;
1275                }
1276                // Handle != as a single token
1277                '!' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' => {
1278                    if !current_word.is_empty() {
1279                        items.push(WordItem {
1280                            word: std::mem::take(&mut current_word),
1281                            trailing_punct: None,
1282                            start: word_start,
1283                            end: i,
1284                            punct_pos: None,
1285                        });
1286                    }
1287                    items.push(WordItem {
1288                        word: "!=".to_string(),
1289                        trailing_punct: None,
1290                        start: i,
1291                        end: i + 2,
1292                        punct_pos: None,
1293                    });
1294                    skip_count = 1;
1295                    word_start = i + 2;
1296                }
1297                // Special handling for '-' in ISO-8601 dates (YYYY-MM-DD)
1298                '-' if Self::is_date_hyphen(&current_word, &chars, char_idx) => {
1299                    // This hyphen is part of a date, include it in the word
1300                    current_word.push(c);
1301                }
1302                // Special handling for ':' in time literals (9:30am, 11:45pm)
1303                ':' if Self::is_time_colon(&current_word, &chars, char_idx) => {
1304                    // This colon is part of a time, include it in the word
1305                    current_word.push(c);
1306                }
1307                // Namespaced type reference `Alias::Type` (no intervening spaces):
1308                // the `::` glues an identifier segment on each side into ONE word, so
1309                // the interned token text is exactly the cross-module registry name
1310                // (`Geometry::Point`) that construction and type annotations consume.
1311                // A lone `:` (a block colon `xs[1]:`, a `label:`) stays punctuation —
1312                // only a `::` between identifier characters is the path separator.
1313                ':' if Self::is_namespace_colon(&current_word, &chars, char_idx) => {
1314                    current_word.push(c);
1315                    current_word.push(chars[char_idx + 1]);
1316                    skip_count = 1;
1317                }
1318                // Hyphenated compounds ("weak-until", "highest-priority"):
1319                // a hyphen BETWEEN letters is part of the word, never the
1320                // arithmetic minus (which is space- or digit-adjacent).
1321                '-' if char_idx > 0
1322                    && chars[char_idx - 1].is_alphabetic()
1323                    && char_idx + 1 < chars.len()
1324                    && chars[char_idx + 1].is_alphabetic() =>
1325                {
1326                    current_word.push(c);
1327                }
1328                // Scientific notation: 4.84e+00, 1.66E-03, 2.5e-2
1329                '+' | '-' if Self::is_exponent_sign(&current_word, &chars, char_idx) => {
1330                    current_word.push(c);
1331                }
1332                // Alphanumeric codes ("AV-435", "FRZ-192", "I-5"): a hyphen with
1333                // an UPPERCASE-code word before it and a DIGIT after is part of
1334                // the identifier, not an arithmetic minus. The all-uppercase gate
1335                // keeps lowercase variables ("x-1") and digit arithmetic ("5-3")
1336                // emitting a Minus.
1337                '-' if char_idx > 0
1338                    && chars[char_idx - 1].is_alphabetic()
1339                    && char_idx + 1 < chars.len()
1340                    && chars[char_idx + 1].is_ascii_digit()
1341                    && !current_word.is_empty()
1342                    && current_word.chars().all(|ch| ch.is_ascii_uppercase()) =>
1343                {
1344                    current_word.push(c);
1345                }
1346                // Letter grades ("B+", "A+"): a '+' directly after an
1347                // uppercase-letter word, NOT followed by a digit (arithmetic
1348                // "B+2"), is part of the grade. A spaced "B + C" flushes "B"
1349                // before the '+', so only the glued grade form is caught.
1350                '+' if !current_word.is_empty()
1351                    && current_word.chars().all(|ch| ch.is_ascii_uppercase())
1352                    && (char_idx + 1 >= chars.len() || !chars[char_idx + 1].is_ascii_digit()) =>
1353                {
1354                    current_word.push(c);
1355                }
1356                // Thousands separator inside a number: "125,000", "1,234,567".
1357                // A comma flanked by digits is part of the numeral, not a clause
1358                // separator — except inside a bracketed list, where `[1,2,3]`
1359                // means three elements. A money word (`$125,000`) keeps its
1360                // commas even there.
1361                ',' if char_idx > 0
1362                    && chars[char_idx - 1].is_ascii_digit()
1363                    && char_idx + 1 < chars.len()
1364                    && chars[char_idx + 1].is_ascii_digit()
1365                    && (bracket_depth == 0
1366                        || current_word
1367                            .chars()
1368                            .next()
1369                            .map_or(false, Self::is_currency_symbol)) =>
1370                {
1371                    current_word.push(c);
1372                }
1373                // Date separator inside a numeral: "04/2024", "12/25", "04/2024"
1374                // (MM/YYYY, MM/DD). A slash flanked by digits is part of the date
1375                // token, not an arithmetic division (rare in NL clues).
1376                '/' if char_idx > 0
1377                    && chars[char_idx - 1].is_ascii_digit()
1378                    && char_idx + 1 < chars.len()
1379                    && chars[char_idx + 1].is_ascii_digit() =>
1380                {
1381                    current_word.push(c);
1382                }
1383                // "28-inch", "6-year-old", "12-year-old" — a hyphen with a DIGIT
1384                // before and a LETTER after is orthographic (= "28 inch"), so
1385                // split into separate words here (flush the number, skip the
1386                // hyphen as a word boundary) rather than emitting an arithmetic
1387                // Minus. A digit-digit hyphen ("5-3") stays Minus (letter-after
1388                // required); a letter-letter hyphen ("well-known") is handled above.
1389                '-' if char_idx > 0
1390                    && chars[char_idx - 1].is_ascii_digit()
1391                    && char_idx + 1 < chars.len()
1392                    && chars[char_idx + 1].is_alphabetic() =>
1393                {
1394                    if !current_word.is_empty() {
1395                        items.push(WordItem {
1396                            word: std::mem::take(&mut current_word),
1397                            trailing_punct: None,
1398                            start: word_start,
1399                            end: i,
1400                            punct_pos: None,
1401                        });
1402                    }
1403                    word_start = next_pos;
1404                }
1405                // `**` exponentiation — a single two-char token (before the
1406                // generic punct arm and the `*=` arm; `**` is `*` then `*`).
1407                '*' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '*' => {
1408                    if !current_word.is_empty() {
1409                        items.push(WordItem {
1410                            word: std::mem::take(&mut current_word),
1411                            trailing_punct: None,
1412                            start: word_start,
1413                            end: i,
1414                            punct_pos: None,
1415                        });
1416                    }
1417                    items.push(WordItem {
1418                        word: "**".to_string(),
1419                        trailing_punct: None,
1420                        start: i,
1421                        end: i + 2,
1422                        punct_pos: None,
1423                    });
1424                    skip_count = 1;
1425                    word_start = i + 2;
1426                }
1427                // `//` floor division — a single two-char token (before the generic
1428                // punct arm and the `/=` arm; `//` is `/` then `/`, not `/` then `=`).
1429                // A digit-flanked `/` is already claimed above as a date separator, so
1430                // this only fires on a genuine `x // y`.
1431                '/' if char_idx + 1 < chars.len() && chars[char_idx + 1] == '/' => {
1432                    if !current_word.is_empty() {
1433                        items.push(WordItem {
1434                            word: std::mem::take(&mut current_word),
1435                            trailing_punct: None,
1436                            start: word_start,
1437                            end: i,
1438                            punct_pos: None,
1439                        });
1440                    }
1441                    items.push(WordItem {
1442                        word: "//".to_string(),
1443                        trailing_punct: None,
1444                        start: i,
1445                        end: i + 2,
1446                        punct_pos: None,
1447                    });
1448                    skip_count = 1;
1449                    word_start = i + 2;
1450                }
1451                // Compound assignment `+= -= *= /= %=` — a single two-char token
1452                // (before the generic punct arm so `+` etc. don't split first).
1453                // `-=` is distinct from `->` (next is `=`, not `>`); `/=` from `//`.
1454                '+' | '-' | '*' | '/' | '%'
1455                    if char_idx + 1 < chars.len() && chars[char_idx + 1] == '=' =>
1456                {
1457                    if !current_word.is_empty() {
1458                        items.push(WordItem {
1459                            word: std::mem::take(&mut current_word),
1460                            trailing_punct: None,
1461                            start: word_start,
1462                            end: i,
1463                            punct_pos: None,
1464                        });
1465                    }
1466                    items.push(WordItem {
1467                        word: format!("{c}="),
1468                        trailing_punct: None,
1469                        start: i,
1470                        end: i + 2,
1471                        punct_pos: None,
1472                    });
1473                    skip_count = 1;
1474                    word_start = i + 2;
1475                }
1476                '(' | ')' | '[' | ']' | '{' | '}' | '|' | '~' | '^' | ',' | '?' | '!' | ':'
1477                | '+' | '-' | '*' | '/' | '%' | '<' | '>' | '=' => {
1478                    match c {
1479                        // `{…}` literals are bracket contexts for the comma
1480                        // rule too: `{1,2}` is a two-element set, not `{12}`.
1481                        '[' | '{' => bracket_depth += 1,
1482                        ']' | '}' => bracket_depth = bracket_depth.saturating_sub(1),
1483                        _ => {}
1484                    }
1485                    if !current_word.is_empty() {
1486                        items.push(WordItem {
1487                            word: std::mem::take(&mut current_word),
1488                            trailing_punct: Some(c),
1489                            start: word_start,
1490                            end: i,
1491                            punct_pos: Some(i),
1492                        });
1493                    } else {
1494                        items.push(WordItem {
1495                            word: String::new(),
1496                            trailing_punct: Some(c),
1497                            start: i,
1498                            end: next_pos,
1499                            punct_pos: Some(i),
1500                        });
1501                    }
1502                    word_start = next_pos;
1503                }
1504                '\'' => {
1505                    // Handle contractions: expand "don't" → "do" + "not", etc.
1506                    let remaining: String = chars[char_idx + 1..].iter().collect();
1507                    let remaining_lower = remaining.to_lowercase();
1508
1509                    // Possessive on a bracketed sub-expression: `foo(x)'s field`,
1510                    // `items[0]'s value`. The receiver was already flushed by its
1511                    // closing `)`/`]`, so `current_word` is empty here — mirror the
1512                    // `(e).m` dot rule (`prev_close`) and emit a possessive marker,
1513                    // consuming the clitic `s`. Field access after a value is always
1514                    // the genitive, never the `is`-clitic (that attaches only to
1515                    // pronouns / wh-words, which never end in `)`/`]`).
1516                    let prev_close = current_word.is_empty()
1517                        && char_idx > 0
1518                        && matches!(chars[char_idx - 1], ')' | ']');
1519                    let clitic_s = matches!(chars.get(char_idx + 1), Some('s' | 'S'))
1520                        && chars.get(char_idx + 2).map_or(true, |c| !c.is_alphabetic());
1521                    if prev_close && clitic_s {
1522                        items.push(WordItem {
1523                            word: "\x00POSS".to_string(),
1524                            trailing_punct: None,
1525                            start: i,
1526                            end: next_pos + 1,
1527                            punct_pos: None,
1528                        });
1529                        word_start = next_pos + 1;
1530                        skip_count = 1;
1531                    } else if remaining_lower.starts_with("t ")
1532                        || remaining_lower.starts_with("t.")
1533                        || remaining_lower.starts_with("t,")
1534                        || remaining_lower == "t"
1535                        || (char_idx + 1 < chars.len()
1536                            && chars[char_idx + 1] == 't'
1537                            && (char_idx + 2 >= chars.len()
1538                                || !chars[char_idx + 2].is_alphabetic()))
1539                    {
1540                        // The splitter broke the word at the apostrophe, so
1541                        // `current_word` is the n't-contraction STEM ("isn",
1542                        // "won"). Which stems contract — and what they expand
1543                        // to ("won" is suppletive: "will not") — is lexical
1544                        // knowledge, so the lexicon owns the table; here we
1545                        // only apply its expansion.
1546                        let word_lower = current_word.to_lowercase();
1547                        if let Some(expansion) =
1548                            crate::lexicon::lookup_negative_contraction(&word_lower)
1549                        {
1550                            let words: Vec<&str> = expansion.split_whitespace().collect();
1551                            let last_idx = words.len().saturating_sub(1);
1552                            for (idx, part) in words.iter().enumerate() {
1553                                items.push(WordItem {
1554                                    word: (*part).to_string(),
1555                                    trailing_punct: None,
1556                                    start: if idx == 0 { word_start } else { i },
1557                                    end: if idx == last_idx { i + 2 } else { i },
1558                                    punct_pos: None,
1559                                });
1560                            }
1561                            current_word.clear();
1562                            word_start = next_pos + 1;
1563                            skip_count = 1;
1564                        } else {
1565                            // Unknown contraction, split normally
1566                            if !current_word.is_empty() {
1567                                items.push(WordItem {
1568                                    word: std::mem::take(&mut current_word),
1569                                    trailing_punct: Some('\''),
1570                                    start: word_start,
1571                                    end: i,
1572                                    punct_pos: Some(i),
1573                                });
1574                            }
1575                            word_start = next_pos;
1576                        }
1577                    } else {
1578                        // Not a 't contraction, handle normally
1579                        if !current_word.is_empty() {
1580                            items.push(WordItem {
1581                                word: std::mem::take(&mut current_word),
1582                                trailing_punct: Some('\''),
1583                                start: word_start,
1584                                end: i,
1585                                punct_pos: Some(i),
1586                            });
1587                        }
1588                        word_start = next_pos;
1589                    }
1590                }
1591                // Currency-symbol money literal: a `$ € £ ¥` directly before a digit starts a money
1592                // word (`$19.99`, `€5`, `¥100`), so the symbol survives into the word for
1593                // `classify_with_lookahead` to read as `MoneyLiteral`. A lone or non-numeric symbol
1594                // (`$ each`) still falls through to the default arm and is dropped, as before.
1595                c if Self::is_currency_symbol(c)
1596                    && char_idx + 1 < chars.len()
1597                    && chars[char_idx + 1].is_ascii_digit() =>
1598                {
1599                    if !current_word.is_empty() {
1600                        items.push(WordItem {
1601                            word: std::mem::take(&mut current_word),
1602                            trailing_punct: None,
1603                            start: word_start,
1604                            end: i,
1605                            punct_pos: None,
1606                        });
1607                    }
1608                    word_start = i;
1609                    current_word.push(c);
1610                }
1611                c if c.is_alphabetic()
1612                    || c.is_ascii_digit()
1613                    || (c == '.'
1614                        && !current_word.is_empty()
1615                        && current_word.chars().all(|ch| ch.is_ascii_digit()))
1616                    || c == '_' =>
1617                {
1618                    if current_word.is_empty() {
1619                        word_start = i;
1620                    }
1621                    current_word.push(c);
1622                }
1623                '&' => {
1624                    // "&" is COLOR coordination ("black & red" → emit "and" →
1625                    // Black ∧ Red) when its neighbour is LOWERCASE, but a FIRM-NAME
1626                    // joiner ("Leach & Mccall", "Ingram & Kemp") when CAPITALIZED —
1627                    // there it is dropped so the proper-name absorber joins the
1628                    // names (Leach_Mccall), the prior behaviour. Decide on the next
1629                    // word's case.
1630                    if !current_word.is_empty() {
1631                        items.push(WordItem {
1632                            word: std::mem::take(&mut current_word),
1633                            trailing_punct: None,
1634                            start: word_start,
1635                            end: i,
1636                            punct_pos: None,
1637                        });
1638                    }
1639                    let next_cap = chars[char_idx + 1..]
1640                        .iter()
1641                        .find(|ch| !ch.is_whitespace())
1642                        .map_or(false, |ch| ch.is_ascii_uppercase());
1643                    if !next_cap {
1644                        // Mode-deferred: `classify_with_lookahead` resolves the
1645                        // marker to prose "and" (Declarative) or the bitwise
1646                        // `&` operator (Imperative).
1647                        items.push(WordItem {
1648                            word: "\x00AMP".to_string(),
1649                            trailing_punct: None,
1650                            start: i,
1651                            end: next_pos,
1652                            punct_pos: None,
1653                        });
1654                    }
1655                    word_start = next_pos;
1656                }
1657                _ => {
1658                    word_start = next_pos;
1659                }
1660            }
1661            char_idx += 1;
1662        }
1663
1664        if !current_word.is_empty() {
1665            items.push(WordItem {
1666                word: current_word,
1667                trailing_punct: None,
1668                start: word_start,
1669                end: input.len(),
1670                punct_pos: None,
1671            });
1672        }
1673
1674        items
1675    }
1676
1677    fn peek_word(&self, offset: usize) -> Option<&str> {
1678        self.words.get(self.pos + offset).map(|w| w.word.as_str())
1679    }
1680
1681    /// Check if the previous word is a determiner (every, each, some, all, any, no, the, a, an).
1682    fn prev_token_is_determiner(&self) -> bool {
1683        if self.pos == 0 {
1684            return false;
1685        }
1686        if let Some(prev) = self.words.get(self.pos - 1) {
1687            matches!(
1688                prev.word.to_lowercase().as_str(),
1689                "every" | "each" | "some" | "all" | "any" | "no" | "the" | "a" | "an"
1690            )
1691        } else {
1692            false
1693        }
1694    }
1695
1696    /// Whether the previous word closes a sentence (ends with `.`/`!`/`?`), so the
1697    /// current word is sentence-initial. Used to tell a capitalized modal that opens a
1698    /// question ("Will Alice win?") from a capitalized proper name mid-clause ("started
1699    /// by Will Waters") — a function word never capitalizes mid-sentence.
1700    fn prev_word_ends_sentence(&self) -> bool {
1701        if self.pos == 0 {
1702            return true;
1703        }
1704        self.words
1705            .get(self.pos - 1)
1706            .and_then(|w| w.word.chars().last())
1707            .map_or(true, |c| matches!(c, '.' | '!' | '?' | ':' | ';'))
1708    }
1709
1710    /// Whether the previous word is a bare number ("30 minutes", "1 second"),
1711    /// used to disambiguate the singular clock units "second"/"minute" (which are
1712    /// also an ordinal and an adjective) — they are time units only after a count.
1713    fn prev_word_is_numeric(&self) -> bool {
1714        if self.pos == 0 {
1715            return false;
1716        }
1717        self.words
1718            .get(self.pos - 1)
1719            .map(|p| {
1720                let w = p.word.trim_start_matches('$').replace(',', "");
1721                !w.is_empty() && w.chars().all(|c| c.is_ascii_digit() || c == '.')
1722            })
1723            .unwrap_or(false)
1724    }
1725
1726    fn next_token_is_copula(&self) -> bool {
1727        if let Some(next) = self.peek_word(1) {
1728            matches!(next.to_lowercase().as_str(), "is" | "are" | "was" | "were")
1729        } else {
1730            false
1731        }
1732    }
1733
1734    fn peek_sequence(&self, expected: &[&str]) -> bool {
1735        for (i, &exp) in expected.iter().enumerate() {
1736            match self.peek_word(i + 1) {
1737                Some(w) if w.to_lowercase() == exp => continue,
1738                _ => return false,
1739            }
1740        }
1741        true
1742    }
1743
1744    fn consume_words(&mut self, count: usize) {
1745        self.pos += count;
1746    }
1747
1748    /// Tokenizes the input text and returns a vector of [`Token`]s.
1749    ///
1750    /// Each token includes its type, the interned lexeme, and the source
1751    /// span for error reporting. Words are classified according to the
1752    /// lexicon database with priority-based ambiguity resolution.
1753    ///
1754    /// # Returns
1755    ///
1756    /// A vector of tokens representing the input. The final token is
1757    /// typically `TokenType::Eof`.
1758    pub fn tokenize(&mut self) -> Vec<Token> {
1759        let mut tokens = Vec::new();
1760        // Markdown-table row tracking: a table row is a LINE construct, but the
1761        // word stream erases line breaks. While lexing a `|`-led line, the
1762        // first item past a real newline gets an explicit Newline token so the
1763        // parser can see where a row ends (case tables, the in-language test
1764        // framework).
1765        let mut prev_item_end: Option<usize> = None;
1766        let mut in_table_line = false;
1767
1768        while self.pos < self.words.len() {
1769            let item = &self.words[self.pos];
1770            let word = item.word.clone();
1771            let trailing_punct = item.trailing_punct;
1772            let word_start = item.start;
1773            let word_end = item.end;
1774            let punct_pos = item.punct_pos;
1775
1776            if matches!(self.mode, LexerMode::Imperative) {
1777                let gap_has_newline = prev_item_end
1778                    .map(|pe| {
1779                        self.source
1780                            .get(pe..word_start)
1781                            .is_some_and(|gap| gap.contains('\n'))
1782                    })
1783                    .unwrap_or(false);
1784                if in_table_line && gap_has_newline {
1785                    let sym = self.interner.intern("");
1786                    tokens.push(Token::new(
1787                        TokenType::Newline,
1788                        sym,
1789                        Span::new(word_start, word_start),
1790                    ));
1791                    in_table_line = false;
1792                }
1793                let is_pipe = word.is_empty() && trailing_punct == Some('|');
1794                if is_pipe && !in_table_line && (gap_has_newline || prev_item_end.is_none()) {
1795                    in_table_line = true;
1796                }
1797            }
1798            prev_item_end = Some(word_end);
1799
1800            if word.is_empty() {
1801                if let Some(punct) = trailing_punct {
1802                    let kind = match punct {
1803                        '(' => TokenType::LParen,
1804                        ')' => TokenType::RParen,
1805                        '[' => TokenType::LBracket,
1806                        ']' => TokenType::RBracket,
1807                        '{' => TokenType::LBrace,
1808                        '}' => TokenType::RBrace,
1809                        // Bitwise symbols exist only in IMPERATIVE code; in
1810                        // prose they stay dropped (the old behavior).
1811                        '|' if matches!(self.mode, LexerMode::Imperative) => TokenType::VBar,
1812                        '~' if matches!(self.mode, LexerMode::Imperative) => TokenType::Tilde,
1813                        '^' if matches!(self.mode, LexerMode::Imperative) => TokenType::Caret,
1814                        ',' => TokenType::Comma,
1815                        ':' => TokenType::Colon,
1816                        '.' | '?' => {
1817                            self.in_let_context = false;
1818                            TokenType::Period
1819                        }
1820                        '!' => TokenType::Exclamation,
1821                        '+' => TokenType::Plus,
1822                        '-' => TokenType::Minus,
1823                        '*' => TokenType::Star,
1824                        '/' => TokenType::Slash,
1825                        '%' => TokenType::Percent,
1826                        '<' => TokenType::Lt,
1827                        '>' => TokenType::Gt,
1828                        '=' => TokenType::Assign,
1829                        _ => {
1830                            self.pos += 1;
1831                            continue;
1832                        }
1833                    };
1834                    let lexeme = self.interner.intern(&punct.to_string());
1835                    let span = Span::new(word_start, word_end);
1836                    // Collapse consecutive Periods — an abbreviation's own period
1837                    // followed by the sentence period ("Dorsey Assoc.." → "Assoc"
1838                    // + one Period). The extra "." would otherwise strand the parse.
1839                    let dup_period = kind == TokenType::Period
1840                        && tokens
1841                            .last()
1842                            .map_or(false, |t: &Token| t.kind == TokenType::Period);
1843                    if !dup_period {
1844                        tokens.push(Token::new(kind, lexeme, span));
1845                    }
1846                }
1847                self.pos += 1;
1848                continue;
1849            }
1850
1851            // Raw string literal (`r"..."`): verbatim — braces are literal, so
1852            // neither the interpolation scan nor the `{{`→`{` collapse applies.
1853            if word.starts_with("\x00RAW:") {
1854                let content = &word[5..];
1855                let span = Span::new(word_start, word_end);
1856                let sym = self.interner.intern(content);
1857                tokens.push(Token::new(TokenType::StringLiteral(sym), sym, span));
1858                self.pos += 1;
1859                continue;
1860            }
1861
1862            // Check for string literal marker (pre-tokenized in Stage 1)
1863            if word.starts_with("\x00STR:") {
1864                let content = &word[5..]; // Skip the marker prefix
1865                let span = Span::new(word_start, word_end);
1866                if Self::has_unescaped_brace(content) {
1867                    let sym = self.interner.intern(content);
1868                    tokens.push(Token::new(TokenType::InterpolatedString(sym), sym, span));
1869                } else {
1870                    // Collapse {{ → { and }} → } for plain strings
1871                    let normalized = content.replace("{{", "{").replace("}}", "}");
1872                    let sym = self.interner.intern(&normalized);
1873                    tokens.push(Token::new(TokenType::StringLiteral(sym), sym, span));
1874                }
1875                self.pos += 1;
1876                continue;
1877            }
1878
1879            // Check for character literal marker
1880            if word.starts_with("\x00CHAR:") {
1881                let content = &word[6..]; // Skip the marker prefix
1882                let sym = self.interner.intern(content);
1883                let span = Span::new(word_start, word_end);
1884                tokens.push(Token::new(TokenType::CharLiteral(sym), sym, span));
1885                self.pos += 1;
1886                continue;
1887            }
1888
1889            // An LGX raw section (pre-captured `View:` / `Style:` body): `\x00LGX:<kind>:<offset>:<body>`.
1890            if let Some(rest) = word.strip_prefix("\x00LGX:") {
1891                let mut parts = rest.splitn(4, ':');
1892                let kind = match parts.next() {
1893                    Some("S") => RawKind::Style,
1894                    Some("H") => RawKind::Head,
1895                    _ => RawKind::View,
1896                };
1897                let content_start: usize = parts
1898                    .next()
1899                    .and_then(|n| n.parse().ok())
1900                    .unwrap_or(word_start);
1901                let indent: usize = parts.next().and_then(|n| n.parse().ok()).unwrap_or(0);
1902                let body = parts.next().unwrap_or("");
1903                let sym = self.interner.intern(body);
1904                tokens.push(Token::new(
1905                    TokenType::RawBlock {
1906                        kind,
1907                        body: sym,
1908                        content_start,
1909                        indent,
1910                    },
1911                    sym,
1912                    Span::new(word_start, word_end),
1913                ));
1914                self.pos += 1;
1915                continue;
1916            }
1917            // Check for escape block marker (pre-captured raw foreign code)
1918            if word.starts_with("\x00ESC:") {
1919                let content = &word[5..]; // Skip the "\x00ESC:" prefix
1920                let sym = self.interner.intern(content);
1921                let span = Span::new(word_start, word_end);
1922                tokens.push(Token::new(TokenType::EscapeBlock(sym), sym, span));
1923                self.pos += 1;
1924                continue;
1925            }
1926
1927            // "exactly N" / "precisely N" — a redundant exactness marker on a following
1928            // count or measure. The count/measure is already exact in the FOL (a value,
1929            // not a ≥/≤ bound), so the word adds no constraint; dropping it lets the
1930            // count object parse ("serves exactly 2 people" ≡ "serves 2 people") with
1931            // zero meaning loss. (Approximative "about N" is a Preposition and keeps its
1932            // own About-relation; "at least/most N" are handled separately as bounds.)
1933            if matches!(word.to_lowercase().as_str(), "exactly" | "precisely")
1934                && self.peek_word(1).map_or(false, |w| {
1935                    let w = w.trim_start_matches('$');
1936                    w.chars().next().map_or(false, |c| c.is_ascii_digit())
1937                        || crate::lexicon::word_to_number(&w.to_lowercase()).is_some()
1938                })
1939            {
1940                self.pos += 1;
1941                continue;
1942            }
1943
1944            let first_word = self.pos;
1945            let kind = self.classify_with_lookahead(&word);
1946            // A token read from several words ("at least three", "all but finitely many") covers
1947            // its whole phrase: its lexeme and span run to its last word, and the punctuation
1948            // after that word follows it. A block header's further words are its data (a route's
1949            // method and path, a test's name), not part of its phrase.
1950            let multi_word = self.pos > first_word && !matches!(kind, TokenType::BlockHeader { .. });
1951            let (word, trailing_punct, word_end, punct_pos) = if multi_word {
1952                let last = &self.words[self.pos];
1953                let phrase = self
1954                    .source
1955                    .get(word_start..last.end)
1956                    .unwrap_or(&word)
1957                    .to_string();
1958                (phrase, last.trailing_punct, last.end, last.punct_pos)
1959            } else {
1960                (word, trailing_punct, word_end, punct_pos)
1961            };
1962            // A subject pronoun or WH-relativizer takes the copula clitic "'s" = "is"
1963            // ("who's going", "he's …"), never the possessive 's (their genitive is the
1964            // dedicated form whose / its / his). Captured from the host's lexical
1965            // CATEGORY before `kind` is moved into the token, so the contraction rule
1966            // below is data-driven, not a word list.
1967            let host_takes_copula_clitic = matches!(
1968                kind,
1969                TokenType::Pronoun {
1970                    case: crate::lexicon::Case::Subject,
1971                    ..
1972                } | TokenType::Who
1973                    | TokenType::That
1974                    | TokenType::What
1975                    | TokenType::Where
1976            );
1977            let lexeme = self.interner.intern(&word);
1978            let span = Span::new(word_start, word_end);
1979            tokens.push(Token::new(kind, lexeme, span));
1980
1981            if let Some(punct) = trailing_punct {
1982                if punct == '\'' {
1983                    if let Some(next_item) = self.words.get(self.pos + 1) {
1984                        if next_item.word.to_lowercase() == "s" {
1985                            // The contraction "'s" = "is" iff the host takes the copula
1986                            // clitic AND the next word is not a past-participle/non-
1987                            // progressive verb — that case is the perfect "has"
1988                            // ("he's found/been"), a separate unsupported feature, and
1989                            // emitting "is" there would misread it as the passive "is
1990                            // found". Verb aspect comes from the lexicon (data-driven).
1991                            let next_is_past_verb = self
1992                                .words
1993                                .get(self.pos + 2)
1994                                .map(|a| a.word.to_lowercase())
1995                                .and_then(|w| self.lexicon.lookup_verb(&w))
1996                                .map_or(false, |v| v.aspect != crate::lexicon::Aspect::Progressive);
1997                            let (poss_kind, poss_text) =
1998                                if host_takes_copula_clitic && !next_is_past_verb {
1999                                    (TokenType::Is, "is")
2000                                } else {
2001                                    (TokenType::Possessive, "'s")
2002                                };
2003                            let poss_lexeme = self.interner.intern(poss_text);
2004                            let poss_start = punct_pos.unwrap_or(word_end);
2005                            let poss_end = next_item.end;
2006                            tokens.push(Token::new(
2007                                poss_kind,
2008                                poss_lexeme,
2009                                Span::new(poss_start, poss_end),
2010                            ));
2011                            self.pos += 1;
2012                            if let Some(s_punct) = next_item.trailing_punct {
2013                                let kind = match s_punct {
2014                                    '(' => TokenType::LParen,
2015                                    ')' => TokenType::RParen,
2016                                    '[' => TokenType::LBracket,
2017                                    ']' => TokenType::RBracket,
2018                                    '{' => TokenType::LBrace,
2019                                    '}' => TokenType::RBrace,
2020                                    '|' if matches!(self.mode, LexerMode::Imperative) => {
2021                                        TokenType::VBar
2022                                    }
2023                                    '~' if matches!(self.mode, LexerMode::Imperative) => {
2024                                        TokenType::Tilde
2025                                    }
2026                                    '^' if matches!(self.mode, LexerMode::Imperative) => {
2027                                        TokenType::Caret
2028                                    }
2029                                    ',' => TokenType::Comma,
2030                                    ':' => TokenType::Colon,
2031                                    '.' | '?' => TokenType::Period,
2032                                    '!' => TokenType::Exclamation,
2033                                    '+' => TokenType::Plus,
2034                                    '-' => TokenType::Minus,
2035                                    '*' => TokenType::Star,
2036                                    '/' => TokenType::Slash,
2037                                    '%' => TokenType::Percent,
2038                                    '<' => TokenType::Lt,
2039                                    '>' => TokenType::Gt,
2040                                    '=' => TokenType::Assign,
2041                                    _ => {
2042                                        self.pos += 1;
2043                                        continue;
2044                                    }
2045                                };
2046                                let s_punct_pos = next_item.punct_pos.unwrap_or(next_item.end);
2047                                let lexeme = self.interner.intern(&s_punct.to_string());
2048                                tokens.push(Token::new(
2049                                    kind,
2050                                    lexeme,
2051                                    Span::new(s_punct_pos, s_punct_pos + 1),
2052                                ));
2053                            }
2054                            self.pos += 1;
2055                            continue;
2056                        }
2057                    }
2058                    self.pos += 1;
2059                    continue;
2060                }
2061
2062                // An abbreviation's own dot ("Mr.", "Dr.", "153 ft.", "Mt.") is NOT a
2063                // sentence terminator when more text follows — emitting a Period there
2064                // strands the rest of the clause. Which words abbreviate is lexical
2065                // (lexicon `abbreviations`); suppress the dot only mid-clue, so a
2066                // genuine clue-final abbreviation ("…on Main St.") keeps its terminator.
2067                // Prose only: in code every word is an identifier and a full stop ends
2068                // its statement — `Return m.` is a sentence about a variable named `m`.
2069                if punct == '.'
2070                    && self.mode != LexerMode::Imperative
2071                    && lexicon::is_abbreviation(&word.to_lowercase())
2072                    && self.words.get(self.pos + 1).is_some()
2073                {
2074                    self.pos += 1;
2075                    continue;
2076                }
2077                let kind = match punct {
2078                    '(' => TokenType::LParen,
2079                    ')' => TokenType::RParen,
2080                    '[' => TokenType::LBracket,
2081                    ']' => TokenType::RBracket,
2082                    '{' => TokenType::LBrace,
2083                    '}' => TokenType::RBrace,
2084                    '|' if matches!(self.mode, LexerMode::Imperative) => TokenType::VBar,
2085                    '~' if matches!(self.mode, LexerMode::Imperative) => TokenType::Tilde,
2086                    '^' if matches!(self.mode, LexerMode::Imperative) => TokenType::Caret,
2087                    ',' => TokenType::Comma,
2088                    ':' => TokenType::Colon,
2089                    '.' | '?' => {
2090                        self.in_let_context = false;
2091                        TokenType::Period
2092                    }
2093                    '!' => TokenType::Exclamation,
2094                    '+' => TokenType::Plus,
2095                    '-' => TokenType::Minus,
2096                    '*' => TokenType::Star,
2097                    '/' => TokenType::Slash,
2098                    '%' => TokenType::Percent,
2099                    '<' => TokenType::Lt,
2100                    '>' => TokenType::Gt,
2101                    '=' => TokenType::Assign,
2102                    _ => {
2103                        self.pos += 1;
2104                        continue;
2105                    }
2106                };
2107                let p_start = punct_pos.unwrap_or(word_end);
2108                let lexeme = self.interner.intern(&punct.to_string());
2109                tokens.push(Token::new(kind, lexeme, Span::new(p_start, p_start + 1)));
2110            }
2111
2112            self.pos += 1;
2113        }
2114
2115        let eof_lexeme = self.interner.intern("");
2116        let eof_span = Span::new(self.input_len, self.input_len);
2117        tokens.push(Token::new(TokenType::EOF, eof_lexeme, eof_span));
2118
2119        self.insert_indentation_tokens(tokens)
2120    }
2121
2122    /// Insert Indent/Dedent tokens using LineLexer's two-pass architecture (Spec §2.5.2).
2123    ///
2124    /// Phase 1: LineLexer determines the structural layout (where indents/dedents occur)
2125    /// Phase 2: We correlate these with word token positions
2126    fn insert_indentation_tokens(&mut self, tokens: Vec<Token>) -> Vec<Token> {
2127        let mut result = Vec::new();
2128        let empty_sym = self.interner.intern("");
2129
2130        // Phase 1: Run LineLexer to determine structural positions
2131        let line_lexer = LineLexer::new(&self.source);
2132        let line_tokens: Vec<LineToken> = line_lexer.collect();
2133
2134        // Build a list of (byte_position, is_indent) for structural tokens
2135        // Position is where the NEXT Content starts after the Indent/Dedent
2136        let mut structural_events: Vec<(usize, bool)> = Vec::new(); // (byte_pos, true=Indent, false=Dedent)
2137        let mut pending_indents = 0usize;
2138        let mut pending_dedents = 0usize;
2139
2140        for line_token in &line_tokens {
2141            match line_token {
2142                LineToken::Indent => {
2143                    pending_indents += 1;
2144                }
2145                LineToken::Dedent => {
2146                    pending_dedents += 1;
2147                }
2148                LineToken::Content { start, .. } => {
2149                    // Emit pending dedents first (they come BEFORE the content)
2150                    for _ in 0..pending_dedents {
2151                        structural_events.push((*start, false)); // false = Dedent
2152                    }
2153                    pending_dedents = 0;
2154
2155                    // Emit pending indents (they also come BEFORE the content)
2156                    for _ in 0..pending_indents {
2157                        structural_events.push((*start, true)); // true = Indent
2158                    }
2159                    pending_indents = 0;
2160                }
2161                LineToken::Newline => {}
2162            }
2163        }
2164
2165        // Handle any remaining dedents at EOF
2166        for _ in 0..pending_dedents {
2167            structural_events.push((self.input_len, false));
2168        }
2169
2170        // Filter out structural events from within escape block bodies.
2171        // The LineLexer sees raw Rust code lines and generates spurious Indent/Dedent
2172        // events for their indentation changes. We keep exactly the boundary events
2173        // (Indent at body start, Dedent at body end) but remove internal ones.
2174        if !self.escape_body_ranges.is_empty() {
2175            // For each escape body range, find the first Indent at the body start and
2176            // track that we're inside the range. Filter out all events strictly inside
2177            // the range except for the first Indent and events at/after the end.
2178            let mut filtered = Vec::new();
2179            for &(pos, is_indent) in &structural_events {
2180                let is_inside_escape_body = self.escape_body_ranges.iter().any(|(start, end)| {
2181                    // Strictly inside the body (not at start boundary and not at/after end)
2182                    pos > *start && pos < *end
2183                });
2184                if !is_inside_escape_body {
2185                    filtered.push((pos, is_indent));
2186                }
2187            }
2188            structural_events = filtered;
2189        }
2190
2191        // Filter out structural events from within multi-line string literals.
2192        // Triple-quote strings span multiple lines; their internal indentation
2193        // must not generate Indent/Dedent tokens.
2194        {
2195            let string_spans: Vec<(usize, usize)> = tokens
2196                .iter()
2197                .filter(|t| {
2198                    matches!(
2199                        t.kind,
2200                        TokenType::StringLiteral(_) | TokenType::InterpolatedString(_)
2201                    )
2202                })
2203                .filter(|t| t.span.end.saturating_sub(t.span.start) > 6) // only multi-line strings (""" adds >=6 chars)
2204                .map(|t| (t.span.start, t.span.end))
2205                .collect();
2206            if !string_spans.is_empty() {
2207                structural_events.retain(|&(pos, _)| {
2208                    !string_spans
2209                        .iter()
2210                        .any(|(start, end)| pos > *start && pos < *end)
2211                });
2212            }
2213        }
2214
2215        // Bracket line-continuation: a collection literal / call / parenthesised expression may span
2216        // lines with the continuation lines INDENTED (`[\n    1,\n    2,\n]`). The LineLexer sees that
2217        // indentation and emits Indent/Dedent, which would break element parsing. Drop every
2218        // structural event that falls strictly inside an unclosed `(`/`[`/`{` … `)`/`]`/`}` span. Both
2219        // the opening Indent and the matching Dedent lie inside the span, so they are dropped as a
2220        // balanced pair — the enclosing block level is preserved.
2221        {
2222            let mut bracket_ranges: Vec<(usize, usize)> = Vec::new();
2223            let mut open_stack: Vec<usize> = Vec::new();
2224            for t in &tokens {
2225                match t.kind {
2226                    TokenType::LParen | TokenType::LBracket | TokenType::LBrace => {
2227                        open_stack.push(t.span.start);
2228                    }
2229                    TokenType::RParen | TokenType::RBracket | TokenType::RBrace => {
2230                        if let Some(open) = open_stack.pop() {
2231                            bracket_ranges.push((open, t.span.end));
2232                        }
2233                    }
2234                    _ => {}
2235                }
2236            }
2237            if !bracket_ranges.is_empty() {
2238                structural_events.retain(|&(pos, _)| {
2239                    !bracket_ranges
2240                        .iter()
2241                        .any(|(start, end)| pos > *start && pos < *end)
2242                });
2243            }
2244        }
2245
2246        // Sort events by position, with dedents before indents at same position
2247        structural_events.sort_by(|a, b| {
2248            if a.0 != b.0 {
2249                a.0.cmp(&b.0)
2250            } else {
2251                // Dedents (false) before Indents (true) at same position
2252                a.1.cmp(&b.1)
2253            }
2254        });
2255
2256        // Phase 2: Insert structural tokens at the right positions
2257        // Strategy: For each word token, check if any structural events should be inserted
2258        // before it (based on byte position)
2259
2260        let mut event_idx = 0;
2261        let mut last_colon_pos: Option<usize> = None;
2262
2263        for token in tokens.iter() {
2264            let token_start = token.span.start;
2265
2266            // Insert any structural tokens that should come BEFORE this token
2267            while event_idx < structural_events.len() {
2268                let (event_pos, is_indent) = structural_events[event_idx];
2269
2270                // Insert structural tokens before this token if the event position <= token start
2271                if event_pos <= token_start {
2272                    let span = if is_indent {
2273                        // Indent is inserted after the preceding Colon
2274                        Span::new(
2275                            last_colon_pos.unwrap_or(event_pos),
2276                            last_colon_pos.unwrap_or(event_pos),
2277                        )
2278                    } else {
2279                        Span::new(event_pos, event_pos)
2280                    };
2281                    let kind = if is_indent {
2282                        TokenType::Indent
2283                    } else {
2284                        TokenType::Dedent
2285                    };
2286                    result.push(Token::new(kind, empty_sym, span));
2287                    event_idx += 1;
2288                } else {
2289                    break;
2290                }
2291            }
2292
2293            result.push(token.clone());
2294
2295            // Track colon positions for Indent span calculation
2296            if token.kind == TokenType::Colon && self.is_end_of_line(token.span.end) {
2297                last_colon_pos = Some(token.span.end);
2298            }
2299        }
2300
2301        // Insert any remaining structural tokens (typically Dedents at EOF)
2302        while event_idx < structural_events.len() {
2303            let (event_pos, is_indent) = structural_events[event_idx];
2304            let span = Span::new(event_pos, event_pos);
2305            let kind = if is_indent {
2306                TokenType::Indent
2307            } else {
2308                TokenType::Dedent
2309            };
2310            result.push(Token::new(kind, empty_sym, span));
2311            event_idx += 1;
2312        }
2313
2314        // Ensure EOF is at the end
2315        let eof_pos = result.iter().position(|t| t.kind == TokenType::EOF);
2316        if let Some(pos) = eof_pos {
2317            let eof = result.remove(pos);
2318            result.push(eof);
2319        }
2320
2321        result
2322    }
2323
2324    /// Check if position is at end of line (only whitespace until newline)
2325    fn is_end_of_line(&self, from_pos: usize) -> bool {
2326        let bytes = self.source.as_bytes();
2327        let mut pos = from_pos;
2328        while pos < bytes.len() {
2329            match bytes[pos] {
2330                b' ' | b'\t' => pos += 1,
2331                b'\n' => return true,
2332                _ => return false,
2333            }
2334        }
2335        true // End of input is also end of line
2336    }
2337
2338    fn measure_next_line_indent(&self, from_pos: usize) -> Option<usize> {
2339        let bytes = self.source.as_bytes();
2340        let mut pos = from_pos;
2341
2342        while pos < bytes.len() && bytes[pos] != b'\n' {
2343            pos += 1;
2344        }
2345
2346        if pos >= bytes.len() {
2347            return None;
2348        }
2349
2350        pos += 1;
2351
2352        let mut indent = 0;
2353        while pos < bytes.len() {
2354            match bytes[pos] {
2355                b' ' => indent += 1,
2356                b'\t' => indent += 4,
2357                b'\n' => {
2358                    indent = 0;
2359                }
2360                _ => break,
2361            }
2362            pos += 1;
2363        }
2364
2365        if pos >= bytes.len() {
2366            return None;
2367        }
2368
2369        Some(indent)
2370    }
2371
2372    fn word_to_number(word: &str) -> Option<u32> {
2373        lexicon::word_to_number(&word.to_lowercase())
2374    }
2375
2376    /// Check if a hyphen at the current position is part of an ISO-8601 date.
2377    ///
2378    /// Detects patterns like:
2379    /// - "2026-" followed by "05-20" → first hyphen of date
2380    /// - "2026-05-" followed by "20" → second hyphen of date
2381    fn is_date_hyphen(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2382        // Current word must be all digits (year or year-month)
2383        let word_chars: Vec<char> = current_word.chars().collect();
2384
2385        // Check for first hyphen pattern: YYYY- followed by MM-DD
2386        if word_chars.len() == 4 && word_chars.iter().all(|c| c.is_ascii_digit()) {
2387            // Check if followed by exactly 2 digits, hyphen, 2 digits
2388            if char_idx + 5 < chars.len()
2389                && chars[char_idx + 1].is_ascii_digit()
2390                && chars[char_idx + 2].is_ascii_digit()
2391                && chars[char_idx + 3] == '-'
2392                && chars[char_idx + 4].is_ascii_digit()
2393                && chars[char_idx + 5].is_ascii_digit()
2394            {
2395                return true;
2396            }
2397        }
2398
2399        // Check for second hyphen pattern: YYYY-MM- followed by DD
2400        if word_chars.len() == 7
2401            && word_chars[0..4].iter().all(|c| c.is_ascii_digit())
2402            && word_chars[4] == '-'
2403            && word_chars[5..7].iter().all(|c| c.is_ascii_digit())
2404        {
2405            // Check if followed by exactly 2 digits
2406            if char_idx + 2 < chars.len()
2407                && chars[char_idx + 1].is_ascii_digit()
2408                && chars[char_idx + 2].is_ascii_digit()
2409            {
2410                // Make sure we're not followed by more digits (would be a longer number)
2411                let next_not_digit =
2412                    char_idx + 3 >= chars.len() || !chars[char_idx + 3].is_ascii_digit();
2413                if next_not_digit {
2414                    return true;
2415                }
2416            }
2417        }
2418
2419        false
2420    }
2421
2422    /// Check if a colon is part of a time literal (e.g., 9:30am, 11:45pm).
2423    ///
2424    /// Detects patterns like:
2425    /// - "9:" followed by "30am" or "30pm"
2426    /// - "11:" followed by "45pm"
2427    fn is_time_colon(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2428        // Current word must be 1-2 digits (hour)
2429        let word_chars: Vec<char> = current_word.chars().collect();
2430        if word_chars.is_empty() || word_chars.len() > 2 {
2431            return false;
2432        }
2433        if !word_chars.iter().all(|c| c.is_ascii_digit()) {
2434            return false;
2435        }
2436
2437        // Followed by exactly 2 digits (the minutes), then "am"/"pm" — which may
2438        // come immediately ("9:30am") or after a space ("8:15 pm").
2439        if char_idx + 3 < chars.len()
2440            && chars[char_idx + 1].is_ascii_digit()
2441            && chars[char_idx + 2].is_ascii_digit()
2442        {
2443            let suffix_start = if chars.get(char_idx + 3) == Some(&' ') {
2444                char_idx + 4
2445            } else {
2446                char_idx + 3
2447            };
2448            if suffix_start + 1 < chars.len() {
2449                let next_two: String = chars[suffix_start..suffix_start + 2].iter().collect();
2450                let lower = next_two.to_lowercase();
2451                if (lower == "am" || lower == "pm")
2452                    && (suffix_start + 2 >= chars.len() || !chars[suffix_start + 2].is_alphabetic())
2453                {
2454                    return true;
2455                }
2456            }
2457        }
2458
2459        false
2460    }
2461
2462    /// Whether the `:` at `char_idx` opens a `::` namespace separator inside a
2463    /// qualified type reference (`Geometry::Point`, `A::B::C`). True only when the
2464    /// colon is doubled and sits between identifier characters — an identifier
2465    /// character (letter/digit/underscore) immediately precedes the first colon and
2466    /// an identifier START (letter/underscore) immediately follows the second. This
2467    /// keeps a lone `:` a block/label colon; only the glued `Ident::Ident` form is a
2468    /// path, and the same rule chains across each `::` in a multi-segment path.
2469    fn is_namespace_colon(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2470        // The previous emitted character must be part of the left identifier segment.
2471        let prev_is_ident = current_word
2472            .chars()
2473            .next_back()
2474            .map_or(false, |c| c.is_alphanumeric() || c == '_');
2475        if !prev_is_ident {
2476            return false;
2477        }
2478        // Doubled colon.
2479        if chars.get(char_idx + 1) != Some(&':') {
2480            return false;
2481        }
2482        // The character after `::` must begin an identifier segment.
2483        chars
2484            .get(char_idx + 2)
2485            .map_or(false, |c| c.is_alphabetic() || *c == '_')
2486    }
2487
2488    /// Whether a space separates a `H:MM` clock time from a trailing "am"/"pm"
2489    /// ("8:15 pm") — the space is part of the time literal, so the tokenizer keeps
2490    /// "am"/"pm" attached to the time rather than splitting on it.
2491    fn is_time_space_before_ampm(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2492        let valid_time = match current_word.find(':') {
2493            Some(p) => {
2494                let hour = &current_word[..p];
2495                let min = &current_word[p + 1..];
2496                (1..=2).contains(&hour.len())
2497                    && hour.chars().all(|c| c.is_ascii_digit())
2498                    && min.len() == 2
2499                    && min.chars().all(|c| c.is_ascii_digit())
2500            }
2501            None => false,
2502        };
2503        if !valid_time {
2504            return false;
2505        }
2506        let next_two: String = chars
2507            .get(char_idx + 1..char_idx + 3)
2508            .map(|s| s.iter().collect())
2509            .unwrap_or_default();
2510        let lower = next_two.to_lowercase();
2511        (lower == "am" || lower == "pm")
2512            && chars.get(char_idx + 3).map_or(true, |c| !c.is_alphabetic())
2513    }
2514
2515    /// Check if a string contains an unescaped `{` (i.e., not part of `{{`).
2516    /// Used to distinguish `InterpolatedString` from `StringLiteral`.
2517    fn has_unescaped_brace(content: &str) -> bool {
2518        let bytes = content.as_bytes();
2519        let mut i = 0;
2520        while i < bytes.len() {
2521            if bytes[i] == b'{' {
2522                if i + 1 < bytes.len() && bytes[i + 1] == b'{' {
2523                    i += 2;
2524                } else {
2525                    return true;
2526                }
2527            } else {
2528                i += 1;
2529            }
2530        }
2531        false
2532    }
2533
2534    /// Check if a `+` or `-` at the current position is the sign of a scientific notation exponent.
2535    ///
2536    /// Detects patterns like:
2537    /// - "4.84e+" followed by "00" → exponent sign in `4.84e+00`
2538    /// - "2.5e-" followed by "2"  → exponent sign in `2.5e-2`
2539    fn is_exponent_sign(current_word: &str, chars: &[char], char_idx: usize) -> bool {
2540        // Word must end with e/E
2541        if !current_word.ends_with('e') && !current_word.ends_with('E') {
2542            return false;
2543        }
2544        // Before e/E must contain a digit (ensures it's a number, not a bare "e")
2545        let before_e = &current_word[..current_word.len() - 1];
2546        if before_e.is_empty() || !before_e.chars().next().unwrap().is_ascii_digit() {
2547            return false;
2548        }
2549        // Next char must be a digit (the exponent value)
2550        char_idx + 1 < chars.len() && chars[char_idx + 1].is_ascii_digit()
2551    }
2552
2553    /// Dedent a triple-quoted string: strip the common leading whitespace from each line.
2554    /// Joins lines with literal newline characters (not escape sequences).
2555    fn dedent_triple_quote(raw: &str) -> String {
2556        let lines: Vec<&str> = raw.lines().collect();
2557        if lines.is_empty() {
2558            return String::new();
2559        }
2560        // Find minimum indentation of non-empty lines
2561        let min_indent = lines
2562            .iter()
2563            .filter(|l| !l.trim().is_empty())
2564            .map(|l| l.len() - l.trim_start().len())
2565            .min()
2566            .unwrap_or(0);
2567        // Strip that indentation and join with actual newlines
2568        lines
2569            .iter()
2570            .map(|l| {
2571                if l.len() >= min_indent {
2572                    &l[min_indent..]
2573                } else {
2574                    l.trim()
2575                }
2576            })
2577            .collect::<Vec<_>>()
2578            .join("\n")
2579    }
2580
2581    fn is_numeric_literal(word: &str) -> bool {
2582        if word.is_empty() {
2583            return false;
2584        }
2585        let chars: Vec<char> = word.chars().collect();
2586        let first = chars[0];
2587        if first.is_ascii_digit() {
2588            // Numeric literal: starts with digit (may have underscore separators like 1_000)
2589            return true;
2590        }
2591        // Symbolic numbers: only recognize known mathematical symbols
2592        // (aleph, omega, beth) followed by underscore and digits
2593        if let Some(underscore_pos) = word.rfind('_') {
2594            let before_underscore = &word[..underscore_pos];
2595            let after_underscore = &word[underscore_pos + 1..];
2596            // Must be a known mathematical symbol prefix AND digits after underscore
2597            let is_math_symbol = matches!(
2598                before_underscore.to_lowercase().as_str(),
2599                "aleph" | "omega" | "beth"
2600            );
2601            if is_math_symbol
2602                && !after_underscore.is_empty()
2603                && after_underscore.chars().all(|c| c.is_ascii_digit())
2604            {
2605                return true;
2606            }
2607        }
2608        false
2609    }
2610
2611    /// Parse a duration literal with SI suffix.
2612    ///
2613    /// Returns Some((nanoseconds, unit_str)) if the word is a valid duration literal,
2614    /// None otherwise.
2615    ///
2616    /// Supported suffixes:
2617    /// - ns: nanoseconds
2618    /// - us, μs: microseconds
2619    /// - ms: milliseconds
2620    /// - s, sec: seconds
2621    /// - min: minutes
2622    /// - h, hr: hours
2623    fn parse_duration_literal(word: &str) -> Option<(i64, &str)> {
2624        if word.is_empty() || !word.chars().next()?.is_ascii_digit() {
2625            return None;
2626        }
2627
2628        // SI suffix table with multipliers to nanoseconds
2629        const SUFFIXES: &[(&str, i64)] = &[
2630            ("ns", 1),
2631            ("μs", 1_000),
2632            ("us", 1_000),
2633            ("ms", 1_000_000),
2634            ("sec", 1_000_000_000),
2635            ("s", 1_000_000_000),
2636            ("min", 60_000_000_000),
2637            ("hr", 3_600_000_000_000),
2638            ("h", 3_600_000_000_000),
2639        ];
2640
2641        // Try each suffix (longer suffixes first to avoid partial matches)
2642        for (suffix, multiplier) in SUFFIXES {
2643            if word.ends_with(suffix) {
2644                let num_part = &word[..word.len() - suffix.len()];
2645                // Parse the numeric part (may have underscore separators)
2646                let cleaned: String = num_part.chars().filter(|c| *c != '_').collect();
2647                if let Ok(n) = cleaned.parse::<i64>() {
2648                    return Some((n.saturating_mul(*multiplier), *suffix));
2649                }
2650            }
2651        }
2652
2653        None
2654    }
2655
2656    /// Parse an ISO-8601 date literal (YYYY-MM-DD).
2657    ///
2658    /// Returns Some(days_since_epoch) if the word is a valid date literal,
2659    /// None otherwise.
2660    fn parse_date_literal(word: &str) -> Option<i32> {
2661        // Must match pattern: YYYY-MM-DD
2662        if word.len() != 10 {
2663            return None;
2664        }
2665
2666        let bytes = word.as_bytes();
2667
2668        // Check format: 4 digits, hyphen, 2 digits, hyphen, 2 digits
2669        if bytes[4] != b'-' || bytes[7] != b'-' {
2670            return None;
2671        }
2672
2673        // Parse year, month, day
2674        let year: i32 = word[0..4].parse().ok()?;
2675        let month: u32 = word[5..7].parse().ok()?;
2676        let day: u32 = word[8..10].parse().ok()?;
2677
2678        // Reject calendar-impossible dates (e.g. 2026-02-30, 2026-04-31): the
2679        // Howard Hinnant day count below would otherwise silently map them onto
2680        // a real but DIFFERENT day.
2681        if month < 1 || month > 12 || day < 1 {
2682            return None;
2683        }
2684        let is_leap = (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0);
2685        let max_day = match month {
2686            1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
2687            4 | 6 | 9 | 11 => 30,
2688            2 => {
2689                if is_leap {
2690                    29
2691                } else {
2692                    28
2693                }
2694            }
2695            _ => return None,
2696        };
2697        if day > max_day {
2698            return None;
2699        }
2700
2701        // Convert to days since Unix epoch using Howard Hinnant's algorithm
2702        // https://howardhinnant.github.io/date_algorithms.html
2703        let y = if month <= 2 { year - 1 } else { year };
2704        let era = if y >= 0 { y / 400 } else { (y - 399) / 400 };
2705        let yoe = (y - era * 400) as u32;
2706        let m = month;
2707        let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + day - 1;
2708        let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
2709        let days = era * 146097 + doe as i32 - 719468;
2710
2711        Some(days)
2712    }
2713
2714    /// Parse a time-of-day literal.
2715    ///
2716    /// Supported formats:
2717    /// - 12-hour with am/pm: "4pm", "9am", "12pm"
2718    /// - 12-hour with minutes: "9:30am", "11:45pm"
2719    /// - Special words: "noon" (12:00), "midnight" (00:00)
2720    ///
2721    /// Returns Some(nanos_from_midnight) if valid, None otherwise.
2722    fn parse_time_literal(word: &str) -> Option<i64> {
2723        let lower = word.to_lowercase();
2724
2725        // Handle special time words
2726        if lower == "noon" {
2727            return Some(12i64 * 3600 * 1_000_000_000);
2728        }
2729        if lower == "midnight" {
2730            return Some(0);
2731        }
2732
2733        // Handle 12-hour formats: "4pm", "9am", "9:30am", "11:45pm"
2734        let is_pm = lower.ends_with("pm");
2735        let is_am = lower.ends_with("am");
2736
2737        if !is_pm && !is_am {
2738            return None;
2739        }
2740
2741        // Strip the am/pm suffix
2742        let time_part = &lower[..lower.len() - 2];
2743
2744        // Check for hour:minute format
2745        let (hour, minute): (i64, i64) = if let Some(colon_idx) = time_part.find(':') {
2746            let hour_str = &time_part[..colon_idx];
2747            let min_str = &time_part[colon_idx + 1..];
2748            let h: i64 = hour_str.parse().ok()?;
2749            let m: i64 = min_str.parse().ok()?;
2750            (h, m)
2751        } else {
2752            // Just hour: "4pm", "9am"
2753            let h: i64 = time_part.parse().ok()?;
2754            (h, 0)
2755        };
2756
2757        // Validate ranges
2758        if hour < 1 || hour > 12 || minute < 0 || minute > 59 {
2759            return None;
2760        }
2761
2762        // Convert to 24-hour format
2763        let hour_24 = if is_am {
2764            if hour == 12 {
2765                0
2766            } else {
2767                hour
2768            } // 12am = midnight = 0
2769        } else {
2770            if hour == 12 {
2771                12
2772            } else {
2773                hour + 12
2774            } // 12pm = noon = 12, 4pm = 16
2775        };
2776
2777        // Convert to nanoseconds from midnight
2778        let nanos = (hour_24 * 3600 + minute * 60) * 1_000_000_000;
2779        Some(nanos)
2780    }
2781
2782    /// True for the currency symbols that prefix a money literal (`$ € £ ¥`).
2783    fn is_currency_symbol(c: char) -> bool {
2784        matches!(c, '$' | '€' | '£' | '¥')
2785    }
2786
2787    /// The ISO-4217 code a leading currency symbol denotes — `$`→USD, `€`→EUR, `£`→GBP, `¥`→JPY (the
2788    /// dominant reading of each symbol). Unknown symbols yield `None`.
2789    fn currency_for_symbol(c: char) -> Option<&'static str> {
2790        Some(match c {
2791            '$' => "USD",
2792            '€' => "EUR",
2793            '£' => "GBP",
2794            '¥' => "JPY",
2795            _ => return None,
2796        })
2797    }
2798
2799    fn classify_with_lookahead(&mut self, word: &str) -> TokenType {
2800        // The `&` character, deferred by the word-splitter: prose keeps the
2801        // coordination reading (`black & red` → and); imperative code gets
2802        // the bitwise operator.
2803        if word == "\x00AMP" {
2804            return if matches!(self.mode, LexerMode::Imperative) {
2805                TokenType::Amp
2806            } else {
2807                TokenType::And
2808            };
2809        }
2810        if word == "\x00DOT" {
2811            // Imperative: the field-access / UFCS operator. Declarative/prose:
2812            // a plain sentence period (so `e.g.` and abbreviations read as today).
2813            return if matches!(self.mode, LexerMode::Imperative) {
2814                TokenType::Dot
2815            } else {
2816                TokenType::Period
2817            };
2818        }
2819        if word == "\x00POSS" {
2820            // The genitive clitic on a bracketed receiver (`foo(x)'s field`),
2821            // emitted by the splitter's `prev_close` possessive rule.
2822            return TokenType::Possessive;
2823        }
2824        // Handle block headers (##Theorem, ##Main, etc.)
2825        if word.starts_with("##") {
2826            let block_name = &word[2..];
2827            let lowered = block_name.to_lowercase();
2828            let lgx_header = self.try_lgx_header(&lowered);
2829            let block_type = if let Some(bt) = lgx_header {
2830                bt
2831            } else {
2832                match lowered.as_str() {
2833                    "theorem" => BlockType::Theorem,
2834                    "main" => BlockType::Main,
2835                    "definition" => BlockType::Definition,
2836                    "define" => BlockType::Define, // Vernacular-logic predicate definition (Rung 0a)
2837                    "axiom" => BlockType::Axiom,   // Formal first-order axiom (the seam for Tarski)
2838                    "theory" => BlockType::Theory, // Named development grouping axioms + theorems
2839                    "proof" => BlockType::Proof,
2840                    "example" => BlockType::Example,
2841                    "logic" => BlockType::Logic,
2842                    "note" => BlockType::Note,
2843                    "to" => BlockType::Function, // Function definition block
2844                    "component" => BlockType::Component, // Reactive UI component block
2845                    "a" | "an" => BlockType::TypeDef, // Inline type definitions: ## A Point has:
2846                    "policy" => BlockType::Policy, // Security policy definitions
2847                    "requires" => BlockType::Requires, // External crate dependencies
2848                    "hardware" => BlockType::Hardware, // Signal declarations
2849                    "property" => BlockType::Property, // Temporal assertions
2850                    "no" => BlockType::No, // Optimization annotation: ## No Memo, ## No TCO, etc.
2851                    "tier" => BlockType::Tier, // Tiered-optimizer pin: ## Tier specialize eager, etc.
2852                    "require" => BlockType::Require, // Optimization demand: ## Require memo, ## Require closedform
2853
2854                    "test" => {
2855                        // `## Test: <prose>` — the prose after the colon is the
2856                        // test's NAME: raw prose, never tokenized as code. Capture
2857                        // the rest of the source line verbatim and skip its words.
2858                        let hdr_end = self.words[self.pos].end;
2859                        let line_end = self.source[hdr_end..]
2860                            .find('\n')
2861                            .map(|k| hdr_end + k)
2862                            .unwrap_or(self.source.len());
2863                        let raw = self.source[hdr_end..line_end]
2864                            .trim()
2865                            .trim_start_matches(':')
2866                            .trim();
2867                        let name = self.interner.intern(raw);
2868                        // Leave `pos` on the line's last word; the tokenize loop's
2869                        // own advance steps past it.
2870                        while self.pos + 1 < self.words.len()
2871                            && self.words[self.pos + 1].start < line_end
2872                        {
2873                            self.pos += 1;
2874                        }
2875                        BlockType::Test { name }
2876                    }
2877                    other => {
2878                        // A near-miss of a CONSEQUENTIAL header is a probable
2879                        // typo — `## Mian` silently becoming prose is the bug
2880                        // class where a whole program runs to empty output.
2881                        // Prose-type names (note/example/logic) and the short
2882                        // forms are excluded, so ordinary literate headings
2883                        // (`## Notes`, `## Design`) keep working.
2884                        const CONSEQUENTIAL: &[&str] = &[
2885                            "main",
2886                            "theorem",
2887                            "definition",
2888                            "define",
2889                            "axiom",
2890                            "theory",
2891                            "proof",
2892                            "policy",
2893                            "requires",
2894                            "hardware",
2895                            "property",
2896                            "tier",
2897                            "test",
2898                            "component",
2899                            "page",
2900                            "route",
2901                            "store",
2902                            "directive",
2903                            "redirect",
2904                        ];
2905                        if let Some(similar) = crate::suggest::find_similar(other, CONSEQUENTIAL, 2)
2906                        {
2907                            let found = self.interner.intern(block_name);
2908                            let suggestion = self.interner.intern(similar);
2909                            BlockType::SuspectedTypo { found, suggestion }
2910                        } else {
2911                            BlockType::Note // Unknown block types stay literate prose
2912                        }
2913                    }
2914                }
2915            };
2916
2917            // Update lexer mode based on block type
2918            self.mode = match block_type {
2919                BlockType::Main
2920                | BlockType::Function
2921                | BlockType::Component
2922                | BlockType::Test { .. }
2923                | BlockType::Page { .. }
2924                | BlockType::Store
2925                | BlockType::Directive
2926                | BlockType::Route { .. }
2927                | BlockType::Middleware
2928                | BlockType::ErrorPage { .. } => LexerMode::Imperative,
2929                _ => LexerMode::Declarative,
2930            };
2931
2932            return TokenType::BlockHeader { block_type };
2933        }
2934
2935        let lower = word.to_lowercase();
2936
2937        if lower == "each" && self.peek_sequence(&["other"]) {
2938            self.consume_words(1);
2939            return TokenType::Reciprocal;
2940        }
2941
2942        // The infinitary determiners, `infinitely often` and `infinitely close to` are one token
2943        // each, as `at least n` is.
2944        match lower.as_str() {
2945            "infinitely" if self.peek_sequence(&["many"]) => {
2946                self.consume_words(1);
2947                return TokenType::InfinitelyMany;
2948            }
2949            "infinitely" if self.peek_sequence(&["often"]) => {
2950                self.consume_words(1);
2951                return TokenType::InfinitelyOften;
2952            }
2953            "infinitely" if self.peek_sequence(&["close", "to"]) => {
2954                self.consume_words(2);
2955                return TokenType::Preposition(self.interner.intern("InfinitelyClose"));
2956            }
2957            "finitely" if self.peek_sequence(&["many"]) => {
2958                self.consume_words(1);
2959                return TokenType::FinitelyMany;
2960            }
2961            "only" if self.peek_sequence(&["finitely", "many"]) => {
2962                self.consume_words(2);
2963                return TokenType::FinitelyMany;
2964            }
2965            "all" if self.peek_sequence(&["but", "finitely", "many"]) => {
2966                self.consume_words(3);
2967                return TokenType::AllButFinitelyMany;
2968            }
2969            _ => {}
2970        }
2971
2972        if lower == "to" {
2973            if let Some(next) = self.peek_word(1) {
2974                if self.is_verb_like(next) {
2975                    return TokenType::To;
2976                }
2977            }
2978            let sym = self.interner.intern("to");
2979            return TokenType::Preposition(sym);
2980        }
2981
2982        if lower == "at" {
2983            if let Some(next) = self.peek_word(1) {
2984                let next_lower = next.to_lowercase();
2985                if next_lower == "least" {
2986                    if let Some(num_word) = self.peek_word(2) {
2987                        if let Some(n) = Self::word_to_number(num_word) {
2988                            self.consume_words(2);
2989                            return TokenType::AtLeast(n);
2990                        }
2991                    }
2992                }
2993                if next_lower == "most" {
2994                    if let Some(num_word) = self.peek_word(2) {
2995                        if let Some(n) = Self::word_to_number(num_word) {
2996                            self.consume_words(2);
2997                            return TokenType::AtMost(n);
2998                        }
2999                    }
3000                }
3001            }
3002        }
3003
3004        // "Exactly N" → Cardinal(N) — same as bare number but explicit
3005        if lower == "exactly" {
3006            if let Some(num_word) = self.peek_word(1) {
3007                if let Some(n) = Self::word_to_number(num_word) {
3008                    self.consume_words(1);
3009                    return TokenType::Cardinal(n);
3010                }
3011            }
3012        }
3013
3014        if let Some(n) = Self::word_to_number(&lower) {
3015            return TokenType::Cardinal(n);
3016        }
3017
3018        // Check for duration literal first (e.g., "500ms", "2s", "50ns")
3019        if let Some((nanos, unit)) = Self::parse_duration_literal(word) {
3020            let unit_sym = self.interner.intern(unit);
3021            return TokenType::DurationLiteral {
3022                nanos,
3023                original_unit: unit_sym,
3024            };
3025        }
3026
3027        // Check for ISO-8601 date literal (e.g., "2026-05-20")
3028        if let Some(days) = Self::parse_date_literal(word) {
3029            return TokenType::DateLiteral { days };
3030        }
3031
3032        // Check for time-of-day literal (e.g., "4pm", "9:30am", "noon", "midnight")
3033        if let Some(nanos_from_midnight) = Self::parse_time_literal(word) {
3034            return TokenType::TimeLiteral {
3035                nanos_from_midnight,
3036            };
3037        }
3038
3039        // Currency-symbol money literal: `$19.99`, `€5`, `£10`, `¥100`, `$1,250.50`. The symbol
3040        // resolves to its ISO-4217 code; the magnitude keeps its digits and decimal point (thousands
3041        // separators stripped). A `MoneyLiteral` carries both, so the parser builds an exact
3042        // currency-tagged value rather than dropping the symbol. ONLY in imperative code — in
3043        // natural-language clauses `$25,000` is a bare magnitude (a logic-grid constraint), handled
3044        // by the magnitude-stripping block below; the symbol there is just orthography.
3045        if self.mode == LexerMode::Imperative {
3046            if let Some(first) = word.chars().next() {
3047                if let Some(code) = Self::currency_for_symbol(first) {
3048                    let cleaned: String = word
3049                        .chars()
3050                        .skip(1)
3051                        .filter(|c| c.is_ascii_digit() || *c == '.')
3052                        .collect();
3053                    if cleaned.starts_with(|c: char| c.is_ascii_digit()) {
3054                        let amount = self.interner.intern(&cleaned);
3055                        let currency = self.interner.intern(code);
3056                        return TokenType::MoneyLiteral { amount, currency };
3057                    }
3058                }
3059            }
3060        }
3061
3062        // Currency / comma-grouped numerals: "$125,000", "€2.60", "1,234". Strip the currency
3063        // marker and thousands separators; in a natural-language clause the puzzle constraint is the
3064        // bare magnitude (125000, 2.60). (Imperative code took the `MoneyLiteral` path above.)
3065        if word.starts_with(|c: char| Self::is_currency_symbol(c))
3066            || (word.starts_with(|c: char| c.is_ascii_digit()) && word.contains(','))
3067        {
3068            let cleaned: String = word
3069                .chars()
3070                .filter(|c| c.is_ascii_digit() || *c == '.')
3071                .collect();
3072            if cleaned.starts_with(|c: char| c.is_ascii_digit()) {
3073                let sym = self.interner.intern(&cleaned);
3074                return TokenType::Number(sym);
3075            }
3076        }
3077
3078        if Self::is_numeric_literal(word) {
3079            let sym = self.interner.intern(word);
3080            return TokenType::Number(sym);
3081        }
3082
3083        if lower == "if" && self.peek_sequence(&["and", "only", "if"]) {
3084            self.consume_words(3);
3085            return TokenType::Iff;
3086        }
3087
3088        if lower == "is" {
3089            if self.peek_sequence(&["equal", "to"]) {
3090                self.consume_words(2);
3091                return TokenType::Identity;
3092            }
3093            if self.peek_sequence(&["identical", "to"]) {
3094                self.consume_words(2);
3095                return TokenType::Identity;
3096            }
3097        }
3098
3099        if (lower == "a" || lower == "an") && word.chars().next().unwrap().is_uppercase() {
3100            // Capitalized "A" or "An" - disambiguate article vs proper name
3101            // Heuristic: articles are followed by nouns/adjectives, not verbs or keywords
3102            if let Some(next) = self.peek_word(1) {
3103                let next_lower = next.to_lowercase();
3104                let next_starts_lowercase = next
3105                    .chars()
3106                    .next()
3107                    .map(|c| c.is_lowercase())
3108                    .unwrap_or(false);
3109
3110                // If followed by logical keyword, treat as proper name (propositional variable)
3111                if matches!(next_lower.as_str(), "if" | "and" | "or" | "implies" | "iff") {
3112                    let sym = self.interner.intern(word);
3113                    return TokenType::ProperName(sym);
3114                }
3115
3116                // If next word is ONLY a verb (like "has", "is", "ran"), A is likely a name
3117                // Exception: gerunds (like "running") can follow articles
3118                // Exception: words in disambiguation_not_verbs (like "red") are not verbs
3119                // Exception: words that are also nouns/adjectives (like "fire") can follow articles
3120                let is_verb = self.lexicon.lookup_verb(&next_lower).is_some()
3121                    && !lexicon::is_disambiguation_not_verb(&next_lower);
3122                let is_gerund = next_lower.ends_with("ing");
3123                let is_also_noun_or_adj =
3124                    self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower);
3125                if is_verb && !is_gerund && !is_also_noun_or_adj {
3126                    let sym = self.interner.intern(word);
3127                    return TokenType::ProperName(sym);
3128                }
3129
3130                // Definition pattern: "A [TypeName] is a..." or "A [TypeName] has:" - treat A as article
3131                // even when TypeName is capitalized and unknown
3132                if let Some(third) = self.peek_word(2) {
3133                    let third_lower = third.to_lowercase();
3134                    // "has" for struct definitions: "A Point has:"
3135                    if third_lower == "is" || third_lower == "are" || third_lower == "has" {
3136                        return TokenType::Article(Definiteness::Indefinite);
3137                    }
3138                }
3139
3140                // It's an article if next word is:
3141                // - A known noun or adjective, or
3142                // - Lowercase (likely a common word we don't recognize)
3143                let is_content_word =
3144                    self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower);
3145                if is_content_word || next_starts_lowercase {
3146                    return TokenType::Article(Definiteness::Indefinite);
3147                }
3148            }
3149            let sym = self.interner.intern(word);
3150            return TokenType::ProperName(sym);
3151        }
3152
3153        self.classify_word(word)
3154    }
3155
3156    fn is_noun_like(&self, word: &str) -> bool {
3157        if lexicon::is_noun_pattern(word) || lexicon::is_common_noun(word) {
3158            return true;
3159        }
3160        if word.ends_with("er") || word.ends_with("ian") || word.ends_with("ist") {
3161            return true;
3162        }
3163        false
3164    }
3165
3166    fn is_adjective_like(&self, word: &str) -> bool {
3167        lexicon::is_adjective(word) || lexicon::is_non_intersective(word)
3168    }
3169
3170    fn classify_word(&mut self, word: &str) -> TokenType {
3171        let lower = word.to_lowercase();
3172        let first_char = word.chars().next().unwrap();
3173
3174        // Disambiguate "that" as determiner vs complementizer/relativizer.
3175        // "that dog" → Article(Distal); "I know that he ran" → That.
3176        // After a nominal antecedent ("the vessel that saw …"), "that" heads a
3177        // relative clause even when the clause verb is also noun-like ("saw"),
3178        // so it must NOT collapse to a demonstrative there.
3179        if lower == "that" {
3180            if let Some(next) = self.peek_word(1) {
3181                let next_lower = next.to_lowercase();
3182                // A verb-capable next word ("that saw …", "that played …") makes
3183                // "that" a relativizer/complementizer even when that word is also
3184                // noun-like — the demonstrative reading needs a pure-noun head.
3185                let next_is_verb = self.lexicon.lookup_verb(&next_lower).is_some();
3186                if !next_is_verb
3187                    && (self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower))
3188                {
3189                    return TokenType::Article(Definiteness::Distal);
3190                }
3191            }
3192        }
3193
3194        // Arrow token for return type syntax
3195        if word == "->" {
3196            return TokenType::Arrow;
3197        }
3198
3199        // Grand Challenge: Comparison operator tokens
3200        if word == "<=" {
3201            return TokenType::LtEq;
3202        }
3203        if word == ">=" {
3204            return TokenType::GtEq;
3205        }
3206        match word {
3207            "+=" => return TokenType::PlusEq,
3208            "-=" => return TokenType::MinusEq,
3209            "*=" => return TokenType::StarEq,
3210            "/=" => return TokenType::SlashEq,
3211            "%=" => return TokenType::PercentEq,
3212            "**" => return TokenType::StarStar,
3213            "//" => return TokenType::SlashSlash,
3214            _ => {}
3215        }
3216        if word == "==" {
3217            return TokenType::EqEq;
3218        }
3219        if word == "!=" {
3220            return TokenType::NotEq;
3221        }
3222        if word == "<" {
3223            return TokenType::Lt;
3224        }
3225        if word == ">" {
3226            return TokenType::Gt;
3227        }
3228        // Single = for assignment (must come after == check)
3229        if word == "=" {
3230            return TokenType::Assign;
3231        }
3232
3233        if let Some(kind) = lexicon::lookup_keyword(&lower) {
3234            // A capitalized modal/auxiliary MID-sentence is a proper name, not the
3235            // function word ("the startup started by Will Waters", "the May deadline").
3236            // Modals/auxiliaries never capitalize mid-clause, so the keyword reading is
3237            // spurious there; sentence-initial stays the keyword (a question "Will Alice
3238            // win?"). This is the same name/keyword collision resolved for item/native.
3239            let is_modal = matches!(
3240                kind,
3241                TokenType::Must
3242                    | TokenType::Shall
3243                    | TokenType::Should
3244                    | TokenType::Can
3245                    | TokenType::May
3246                    | TokenType::Cannot
3247                    | TokenType::Would
3248                    | TokenType::Could
3249                    | TokenType::Might
3250            );
3251            if is_modal && first_char.is_uppercase() && !self.prev_word_ends_sentence() {
3252                return TokenType::ProperName(self.interner.intern(word));
3253            }
3254            // In Declarative (NL) mode, "from" and "for" are prepositions, not Logos keywords.
3255            // They are listed in the prepositions section of the lexicon and will be
3256            // correctly re-classified by the is_preposition() check below if we skip here.
3257            let kind = match (kind, self.mode) {
3258                (TokenType::From, LexerMode::Declarative) => {
3259                    // A QUALIFIED IMPORT "Type from Module" — both neighbours are
3260                    // capitalised identifiers — keeps the `From` keyword even in
3261                    // declarative text (there are zero "Capitalised from
3262                    // Capitalised" sequences in NL clues). Otherwise "from" is an
3263                    // ordinary NL preposition ("the species FROM Australia").
3264                    let prev_cap = self.pos > 0
3265                        && self
3266                            .words
3267                            .get(self.pos - 1)
3268                            .and_then(|w| w.word.chars().next())
3269                            .map_or(false, |c| c.is_uppercase());
3270                    let next_cap = self
3271                        .words
3272                        .get(self.pos + 1)
3273                        .and_then(|w| w.word.chars().next())
3274                        .map_or(false, |c| c.is_uppercase());
3275                    if prev_cap && next_cap {
3276                        TokenType::From
3277                    } else {
3278                        let sym = self.interner.intern("from");
3279                        TokenType::Preposition(sym)
3280                    }
3281                }
3282                (TokenType::For, LexerMode::Declarative) => {
3283                    let sym = self.interner.intern("for");
3284                    TokenType::Preposition(sym)
3285                }
3286                (other, _) => other,
3287            };
3288            return kind;
3289        }
3290
3291        if let Some(kind) = lexicon::lookup_pronoun(&lower) {
3292            return kind;
3293        }
3294
3295        if let Some(def) = lexicon::lookup_article(&lower) {
3296            return TokenType::Article(def);
3297        }
3298
3299        if let Some(time) = lexicon::lookup_auxiliary(&lower) {
3300            // A capitalized auxiliary MID-sentence is a proper name ("started by Will
3301            // Waters"), not the function word — auxiliaries never capitalize mid-clause.
3302            if first_char.is_uppercase() && !self.prev_word_ends_sentence() {
3303                return TokenType::ProperName(self.interner.intern(word));
3304            }
3305            return TokenType::Auxiliary(time);
3306        }
3307
3308        // Handle imperative keywords that might conflict with prepositions
3309        match lower.as_str() {
3310            "call" => return TokenType::Call,
3311            "in" if self.mode == LexerMode::Imperative => return TokenType::In,
3312            // Zone keywords (must come before is_preposition check)
3313            "inside" if self.mode == LexerMode::Imperative => return TokenType::Inside,
3314            // "at" for chunk access (must come before is_preposition check)
3315            "at" if self.mode == LexerMode::Imperative => return TokenType::At,
3316            // "into" for pipe send (must come before is_preposition check)
3317            "into" if self.mode == LexerMode::Imperative => return TokenType::Into,
3318            // Temporal span operator (must come before is_preposition check)
3319            "before" => return TokenType::Before,
3320            _ => {}
3321        }
3322
3323        // "per" is the rate preposition ("$2.50 per pound", "10 miles per
3324        // hour"). It is not in the general preposition lexicon, and must not be
3325        // mistaken for a measure unit or an unknown noun.
3326        if lower == "per" {
3327            let sym = self.interner.intern("per");
3328            return TokenType::Preposition(sym);
3329        }
3330
3331        // `During <gerund>:` is a trace-span statement in imperative code — it
3332        // must win over the preposition reading (constitution Part 2). Prose
3333        // keeps `during` a preposition (declarative mode).
3334        if lower == "during" && self.mode == LexerMode::Imperative {
3335            return TokenType::During;
3336        }
3337
3338        // A word that is BOTH a verb and a preposition ("like") stays
3339        // ambiguous — the dual-class block below emits the alternatives.
3340        // Lexicon-disambiguated NON-verbs ("during") are pure prepositions
3341        // even when morphology over-derives a verb reading.
3342        if lexicon::is_preposition(&lower)
3343            && (self.lexicon.lookup_verb(&lower).is_none()
3344                || lexicon::is_disambiguation_not_verb(&lower))
3345        {
3346            let sym = self.interner.intern(&lower);
3347            return TokenType::Preposition(sym);
3348        }
3349
3350        match lower.as_str() {
3351            "equals" => return TokenType::Equals,
3352            // "item"/"items" is the indexing keyword in imperative code (where a
3353            // variable index like "item i of arr" is the whole point and prose nouns
3354            // never appear), and in declarative text ONLY when an index number follows
3355            // ("item 1 of list", "items 2 through 5"). Otherwise, in declarative text,
3356            // it is the ordinary English noun ("the blue item", "the item made of
3357            // gold"), which leaked as the keyword before and stranded the noun after an
3358            // adjective. Mode separates code from prose; the following number is the
3359            // finer signal within prose.
3360            "item" | "items"
3361                if self.mode == LexerMode::Imperative
3362                    || self.peek_word(1).map_or(false, |w| {
3363                        w.chars().next().map_or(false, |c| c.is_ascii_digit())
3364                            || crate::lexicon::word_to_number(&w.to_lowercase()).is_some()
3365                    }) =>
3366            {
3367                return if lower == "item" {
3368                    TokenType::Item
3369                } else {
3370                    TokenType::Items
3371                };
3372            }
3373            // Mutability keyword for `mut x = 5` syntax
3374            "mut" if self.mode == LexerMode::Imperative => return TokenType::Mut,
3375            "let" => {
3376                self.in_let_context = true;
3377                return TokenType::Let;
3378            }
3379            "set" => {
3380                // Check if "set" is used as a type (followed by "of") - "Set of Int"
3381                // This takes priority over the assignment keyword
3382                if self
3383                    .peek_word(1)
3384                    .map_or(false, |w| w.to_lowercase() == "of")
3385                {
3386                    // It's a type like "Set of Int" - don't return keyword, let it be a noun
3387                } else if self.mode == LexerMode::Imperative {
3388                    // In Imperative mode, treat "set" as the assignment keyword
3389                    return TokenType::Set;
3390                } else {
3391                    // In Declarative mode, check positions 2-5 for "to"
3392                    // (handles field access like "set p's x to")
3393                    for offset in 2..=5 {
3394                        if self
3395                            .peek_word(offset)
3396                            .map_or(false, |w| w.to_lowercase() == "to")
3397                        {
3398                            return TokenType::Set;
3399                        }
3400                    }
3401                }
3402            }
3403            "return" => return TokenType::Return,
3404            "break" => return TokenType::Break,
3405            "xor" => return TokenType::Xor,
3406            "shifted" => return TokenType::Shifted,
3407            "be" if self.in_let_context => {
3408                self.in_let_context = false;
3409                return TokenType::Be;
3410            }
3411            "while" => return TokenType::While,
3412            "assert" => return TokenType::Assert,
3413            "trust" => return TokenType::Trust,
3414            // Imperative-only: these are common English words ("the proof requires
3415            // …", "this ensures …"), so keep them as plain words in declarative mode.
3416            "require" if self.mode == LexerMode::Imperative => return TokenType::Require,
3417            "requires" if self.mode == LexerMode::Imperative => return TokenType::Requires,
3418            "ensures" if self.mode == LexerMode::Imperative => return TokenType::Ensures,
3419            "check" => return TokenType::Check,
3420            // Theorem keywords (Declarative mode - for theorem blocks).
3421            // In IMPERATIVE mode `given` is the test-fixture keyword
3422            // (`Given a file "…" containing "…".`) — same token, both modes.
3423            "given" => return TokenType::Given,
3424            // Test-framework keywords (Imperative mode only — `## Test` bodies).
3425            // Declarative prose
3426            // keeps them as ordinary verbs.
3427            "expect" if self.mode == LexerMode::Imperative => return TokenType::Expect,
3428            "advance" if self.mode == LexerMode::Imperative => return TokenType::Advance,
3429            "skip" if self.mode == LexerMode::Imperative => return TokenType::Skip,
3430            // Observation statements (constitution Part 2), imperative-only —
3431            // declarative prose keeps them as ordinary verbs/prepositions.
3432            "note" if self.mode == LexerMode::Imperative => return TokenType::Note,
3433            "warn" if self.mode == LexerMode::Imperative => return TokenType::Warn,
3434            "record" if self.mode == LexerMode::Imperative => return TokenType::Record,
3435            // ("during" is handled before the is_preposition check above.)
3436            "prove" if self.mode == LexerMode::Declarative => return TokenType::Prove,
3437            "auto" if self.mode == LexerMode::Declarative => return TokenType::Auto,
3438            // P2P Networking keywords (Imperative mode only)
3439            "listen" if self.mode == LexerMode::Imperative => return TokenType::Listen,
3440            "connect" if self.mode == LexerMode::Imperative => return TokenType::NetConnect,
3441            "sleep" if self.mode == LexerMode::Imperative => return TokenType::Sleep,
3442            // GossipSub keywords (Imperative mode only)
3443            "sync" if self.mode == LexerMode::Imperative => return TokenType::Sync,
3444            // Persistence keywords
3445            "mount" if self.mode == LexerMode::Imperative => return TokenType::Mount,
3446            "persistent" => return TokenType::Persistent, // Works in type expressions
3447            "combined" if self.mode == LexerMode::Imperative => return TokenType::Combined,
3448            "followed" if self.mode == LexerMode::Imperative => return TokenType::Followed,
3449            // Go-like Concurrency keywords (Imperative mode only)
3450            // Note: "first" and "after" are NOT keywords - they're checked via lookahead in parser
3451            // to avoid conflicting with their use as variable names
3452            "launch" if self.mode == LexerMode::Imperative => return TokenType::Launch,
3453            "task" if self.mode == LexerMode::Imperative => return TokenType::Task,
3454            "pipe" if self.mode == LexerMode::Imperative => return TokenType::Pipe,
3455            "receive" if self.mode == LexerMode::Imperative => return TokenType::Receive,
3456            "stop" if self.mode == LexerMode::Imperative => return TokenType::Stop,
3457            "try" if self.mode == LexerMode::Imperative => return TokenType::Try,
3458            "into" if self.mode == LexerMode::Imperative => return TokenType::Into,
3459            "native" if self.mode == LexerMode::Imperative => return TokenType::Native,
3460            "escape" if self.mode == LexerMode::Imperative => return TokenType::Escape,
3461            "from" => return TokenType::From,
3462            "otherwise" => return TokenType::Otherwise,
3463            // Phase 30c: Else/elif as aliases for Otherwise/Otherwise If
3464            "else" => return TokenType::Else,
3465            "elif" => return TokenType::Elif,
3466            // Sum type definition (Declarative mode only - for enum "either...or...")
3467            "either" if self.mode == LexerMode::Declarative => return TokenType::Either,
3468            // Pattern matching statement
3469            "inspect" if self.mode == LexerMode::Imperative => return TokenType::Inspect,
3470            // Constructor keyword (Imperative mode only)
3471            "new" if self.mode == LexerMode::Imperative => return TokenType::New,
3472            // Only emit Give/Show as keywords in Imperative mode
3473            // In Declarative mode, they fall through to lexicon lookup as verbs
3474            "give" if self.mode == LexerMode::Imperative => return TokenType::Give,
3475            "show" if self.mode == LexerMode::Imperative => return TokenType::Show,
3476            // Collection operation keywords (Imperative mode only)
3477            "push" if self.mode == LexerMode::Imperative => return TokenType::Push,
3478            "pop" if self.mode == LexerMode::Imperative => return TokenType::Pop,
3479            "copy" if self.mode == LexerMode::Imperative => return TokenType::Copy,
3480            "through" if self.mode == LexerMode::Imperative => return TokenType::Through,
3481            "length" if self.mode == LexerMode::Imperative => return TokenType::Length,
3482            "at" if self.mode == LexerMode::Imperative => return TokenType::At,
3483            // Set operation keywords (Imperative mode only)
3484            "add" if self.mode == LexerMode::Imperative => return TokenType::Add,
3485            "remove" if self.mode == LexerMode::Imperative => return TokenType::Remove,
3486            "contains" if self.mode == LexerMode::Imperative => return TokenType::Contains,
3487            "union" if self.mode == LexerMode::Imperative => return TokenType::Union,
3488            "intersection" if self.mode == LexerMode::Imperative => return TokenType::Intersection,
3489            // Zone keywords (Imperative mode only)
3490            "inside" if self.mode == LexerMode::Imperative => return TokenType::Inside,
3491            "zone" if self.mode == LexerMode::Imperative => return TokenType::Zone,
3492            "called" if self.mode == LexerMode::Imperative => return TokenType::Called,
3493            "size" if self.mode == LexerMode::Imperative => return TokenType::Size,
3494            "mapped" if self.mode == LexerMode::Imperative => return TokenType::Mapped,
3495            // Structured Concurrency keywords (Imperative mode only)
3496            "attempt" if self.mode == LexerMode::Imperative => return TokenType::Attempt,
3497            "following" if self.mode == LexerMode::Imperative => return TokenType::Following,
3498            "simultaneously" if self.mode == LexerMode::Imperative => {
3499                return TokenType::Simultaneously
3500            }
3501            // IO keywords (Imperative mode only)
3502            "read" if self.mode == LexerMode::Imperative => return TokenType::Read,
3503            "write" if self.mode == LexerMode::Imperative => return TokenType::Write,
3504            "console" if self.mode == LexerMode::Imperative => return TokenType::Console,
3505            "file" if self.mode == LexerMode::Imperative => return TokenType::File,
3506            // Agent System keywords (Imperative mode only)
3507            "spawn" if self.mode == LexerMode::Imperative => return TokenType::Spawn,
3508            "send" if self.mode == LexerMode::Imperative => return TokenType::Send,
3509            "await" if self.mode == LexerMode::Imperative => return TokenType::Await,
3510            // Serialization keyword (works in Definition blocks too)
3511            "portable" => return TokenType::Portable,
3512            // Sipping Protocol keywords (Imperative mode only)
3513            "manifest" if self.mode == LexerMode::Imperative => return TokenType::Manifest,
3514            "chunk" if self.mode == LexerMode::Imperative => return TokenType::Chunk,
3515            // CRDT keywords
3516            "shared" => return TokenType::Shared, // Works in Definition blocks like Portable
3517            "merge" if self.mode == LexerMode::Imperative => return TokenType::Merge,
3518            "converge" if self.mode == LexerMode::Imperative => return TokenType::Converge,
3519            "swap" if self.mode == LexerMode::Imperative => return TokenType::Swap,
3520            "increase" if self.mode == LexerMode::Imperative => return TokenType::Increase,
3521            // Extended CRDT keywords
3522            "decrease" if self.mode == LexerMode::Imperative => return TokenType::Decrease,
3523            "append" if self.mode == LexerMode::Imperative => return TokenType::Append,
3524            "resolve" if self.mode == LexerMode::Imperative => return TokenType::Resolve,
3525            "values" if self.mode == LexerMode::Imperative => return TokenType::Values,
3526            // Type keywords (work in both modes like "Shared"):
3527            "tally" => return TokenType::Tally,
3528            "sharedset" => return TokenType::SharedSet,
3529            "sharedsequence" => return TokenType::SharedSequence,
3530            "collaborativesequence" => return TokenType::CollaborativeSequence,
3531            "sharedmap" => return TokenType::SharedMap,
3532            "divergent" => return TokenType::Divergent,
3533            "removewins" => return TokenType::RemoveWins,
3534            "addwins" => return TokenType::AddWins,
3535            "yata" => return TokenType::YATA,
3536            // Calendar / clock time unit words (Span expressions). The singular
3537            // "second"/"minute" are ambiguous (ordinal "second base", adjective
3538            // "minute detail") so they are time units only after a count; the
3539            // plurals and "hour" are unambiguous time units.
3540            "seconds" => return TokenType::CalendarUnit(CalendarUnit::Second),
3541            "minutes" => return TokenType::CalendarUnit(CalendarUnit::Minute),
3542            "second" if self.prev_word_is_numeric() => {
3543                return TokenType::CalendarUnit(CalendarUnit::Second)
3544            }
3545            "minute" if self.prev_word_is_numeric() => {
3546                return TokenType::CalendarUnit(CalendarUnit::Minute)
3547            }
3548            "hour" | "hours" => return TokenType::CalendarUnit(CalendarUnit::Hour),
3549            "day" | "days" => return TokenType::CalendarUnit(CalendarUnit::Day),
3550            "week" | "weeks" => return TokenType::CalendarUnit(CalendarUnit::Week),
3551            "month" | "months" => return TokenType::CalendarUnit(CalendarUnit::Month),
3552            "year" | "years" => return TokenType::CalendarUnit(CalendarUnit::Year),
3553            // Sub-second clock units: unlike bare "second"/"minute", these compound words have
3554            // no other common English reading, so they need no preceding-number guard — the
3555            // English-word spelling of the same durations the compact `100ms`/`50ns` literals
3556            // already cover (LOGOS_QUICKGUIDE.md's Duration row).
3557            "millisecond" | "milliseconds" => {
3558                return TokenType::CalendarUnit(CalendarUnit::Millisecond)
3559            }
3560            "microsecond" | "microseconds" => {
3561                return TokenType::CalendarUnit(CalendarUnit::Microsecond)
3562            }
3563            "nanosecond" | "nanoseconds" => {
3564                return TokenType::CalendarUnit(CalendarUnit::Nanosecond)
3565            }
3566            // Span-related keywords (note: "before" is handled earlier to avoid preposition conflict)
3567            "ago" => return TokenType::Ago,
3568            "hence" => return TokenType::Hence,
3569            "if" => return TokenType::If,
3570            "only" => return TokenType::Focus(FocusKind::Only),
3571            "even" => return TokenType::Focus(FocusKind::Even),
3572            "just"
3573                if self.peek_word(1).map_or(false, |w| {
3574                    !self.is_verb_like(w)
3575                        || w.to_lowercase() == "john"
3576                        || w.chars().next().map_or(false, |c| c.is_uppercase())
3577                }) =>
3578            {
3579                return TokenType::Focus(FocusKind::Just)
3580            }
3581            "much" => return TokenType::Measure(MeasureKind::Much),
3582            "little" => return TokenType::Measure(MeasureKind::Little),
3583            _ => {}
3584        }
3585
3586        if lexicon::is_scopal_adverb(&lower) {
3587            let sym = self.interner.intern(&Self::capitalize(&lower));
3588            return TokenType::ScopalAdverb(sym);
3589        }
3590
3591        if lexicon::is_temporal_adverb(&lower) && !self.prev_token_is_determiner() {
3592            let sym = self.interner.intern(&Self::capitalize(&lower));
3593            return TokenType::TemporalAdverb(sym);
3594        }
3595
3596        if lexicon::is_non_intersective(&lower) {
3597            let sym = self.interner.intern(&Self::capitalize(&lower));
3598            return TokenType::NonIntersectiveAdjective(sym);
3599        }
3600
3601        if lexicon::is_adverb(&lower) {
3602            let sym = self.interner.intern(&Self::capitalize(&lower));
3603            return TokenType::Adverb(sym);
3604        }
3605        // A capitalized "-ly" word is normally a proper name ("Billy", "Holly",
3606        // "Kelly's stamp"), not an adverb — UNLESS it is a sentence-initial
3607        // manner adverb modifying a following pronoun ("Quickly he ran").
3608        let cap_ly_is_name = word.chars().next().map_or(false, |c| c.is_uppercase())
3609            && !self.peek_word(1).map_or(false, |w| {
3610                matches!(
3611                    w.to_lowercase().as_str(),
3612                    "he" | "she" | "it" | "they" | "we" | "i" | "you" | "who"
3613                )
3614            });
3615        // A capital letter ANYWHERE past the first character (`bOnly`, `isEmpty`) is camelCase —
3616        // never a natural English word, however its tail happens to be spelled. No real adverb is
3617        // written this way, so a chosen code identifier that merely ends the same way as one
3618        // ("bOnly" ends in the same four letters as "only") must never be mistaken for it; without
3619        // this guard `bOnly`/`aOnly` classified as `Adverb` while the plain `onlyB` (which never
3620        // reaches this "-ly" check at all) classified as an ordinary identifier — the same word
3621        // choice landing on two different token kinds by accident of which end the suffix fell on.
3622        let has_internal_capital = word.chars().skip(1).any(|c| c.is_uppercase());
3623        if lower.ends_with("ly")
3624            && !lexicon::is_not_adverb(&lower)
3625            && !lexicon::is_common_noun(&lower)
3626            && lower.len() > 4
3627            && !cap_ly_is_name
3628            && !has_internal_capital
3629        {
3630            let sym = self.interner.intern(&Self::capitalize(&lower));
3631            return TokenType::Adverb(sym);
3632        }
3633
3634        if let Some(base) = self.try_parse_superlative(&lower) {
3635            let sym = self.interner.intern(&base);
3636            return TokenType::Superlative(sym);
3637        }
3638
3639        // Handle irregular comparatives (less, more, better, worse). "fewer" is
3640        // the count counterpart of "less" — same decreasing direction — and maps
3641        // to the same Decreasing base so it grades like "less" ("received fewer
3642        // votes than Y" → Less(Vote(X), Vote(Y))) without turning bare "few" into
3643        // an adjective (which would break its quantifier reading, "few cookies").
3644        let irregular_comparative = match lower.as_str() {
3645            "less" => Some("Little"),
3646            "fewer" => Some("Little"),
3647            "more" => Some("Much"),
3648            "better" => Some("Good"),
3649            "worse" => Some("Bad"),
3650            _ => None,
3651        };
3652        if let Some(base) = irregular_comparative {
3653            let sym = self.interner.intern(base);
3654            return TokenType::Comparative(sym);
3655        }
3656
3657        if let Some(base) = self.try_parse_comparative(&lower) {
3658            // A word that is ALSO a common noun ("stranger" — the noun vs the
3659            // comparative of "strange") is a comparative only in comparative
3660            // position, signalled by a following "than" ("is stranger than").
3661            // Elsewhere ("the stranger", "Let stranger be …") it is the noun, so
3662            // defer to noun classification below — mirroring the performative /
3663            // base-verb noun-deference above.
3664            let next_is_than = self
3665                .peek_word(1)
3666                .map_or(false, |w| w.eq_ignore_ascii_case("than"));
3667            if !lexicon::is_common_noun(&lower) || next_is_than {
3668                let sym = self.interner.intern(&base);
3669                return TokenType::Comparative(sym);
3670            }
3671            // else fall through to noun classification
3672        }
3673
3674        if lexicon::is_performative(&lower) {
3675            // If the word is also a common noun AND follows a determiner or precedes a copula,
3676            // don't force performative reading.
3677            // "every request holds" → request is a noun, not a performative verb.
3678            // "If request is asserted" → request is a noun (subject before copula).
3679            // "I promise to come" → promise IS a performative verb.
3680            let after_determiner = self.prev_token_is_determiner();
3681            let before_copula = self.next_token_is_copula();
3682            if !lexicon::is_common_noun(&lower) || (!after_determiner && !before_copula) {
3683                let sym = self.interner.intern(&Self::capitalize(&lower));
3684                return TokenType::Performative(sym);
3685            }
3686            // Fall through to noun/verb disambiguation below
3687        }
3688
3689        if lexicon::is_base_verb_early(&lower) {
3690            // If the word is also a common noun AND follows a determiner or precedes a copula,
3691            // don't force verb reading.
3692            // "every grant holds" → grant is a noun, not a verb.
3693            // "If grant is low" → grant is a noun (subject before copula).
3694            let after_determiner = self.prev_token_is_determiner();
3695            let before_copula = self.next_token_is_copula();
3696            if !lexicon::is_common_noun(&lower) || (!after_determiner && !before_copula) {
3697                let sym = self.interner.intern(&Self::capitalize(&lower));
3698                let class = lexicon::lookup_verb_class(&lower);
3699                return TokenType::Verb {
3700                    lemma: sym,
3701                    time: Time::Present,
3702                    aspect: Aspect::Simple,
3703                    class,
3704                };
3705            }
3706            // Fall through to noun/verb disambiguation below
3707        }
3708
3709        // Check for gerunds/progressive verbs BEFORE ProperName check
3710        // "Running" at start of sentence should be Verb, not ProperName
3711        if lower.ends_with("ing") && lower.len() > 4 {
3712            // Participial ADJECTIVE in attributive position ("the
3713            // corresponding output line"): the word has an adjective entry
3714            // and a content word follows it. -ing PREPOSITIONS ("during")
3715            // skip the early return too — the dual-class block below emits
3716            // their Ambiguous alternatives.
3717            let attributive = self.is_adjective_like(&lower)
3718                && self
3719                    .peek_word(1)
3720                    .map(|next| {
3721                        let next_lower = next.to_lowercase();
3722                        self.is_noun_like(&next_lower) || self.is_adjective_like(&next_lower)
3723                    })
3724                    .unwrap_or(false);
3725            if !attributive && !lexicon::is_preposition(&lower) {
3726                if let Some(entry) = self.lexicon.lookup_verb(&lower) {
3727                    let sym = self.interner.intern(&entry.lemma);
3728                    return TokenType::Verb {
3729                        lemma: sym,
3730                        time: entry.time,
3731                        aspect: entry.aspect,
3732                        class: entry.class,
3733                    };
3734                }
3735            }
3736        }
3737
3738        if first_char.is_uppercase() {
3739            // Smart Lexicon: Check if this capitalized word is actually a common noun
3740            // Only apply for sentence-initial words (followed by verb) to avoid
3741            // breaking type definitions like "A Point has:"
3742            //
3743            // Pattern: "Farmers walk." → Farmers is plural of Farmer (common noun)
3744            // Pattern: "A Point has:" → Point is a type name (proper name)
3745            if let Some(next) = self.peek_word(1) {
3746                let next_lower = next.to_lowercase();
3747                // If next word is a verb, this capitalized word is likely a subject noun
3748                let is_followed_by_verb = self.lexicon.lookup_verb(&next_lower).is_some()
3749                    || matches!(
3750                        next_lower.as_str(),
3751                        "is" | "are" | "was" | "were" | "has" | "have" | "had"
3752                    );
3753
3754                if is_followed_by_verb {
3755                    // Check if lowercase version is a derivable common noun
3756                    if let Some(analysis) = lexicon::analyze_word(&lower) {
3757                        match analysis {
3758                            lexicon::WordAnalysis::Noun(meta)
3759                                if meta.number == lexicon::Number::Plural =>
3760                            {
3761                                // It's a plural noun - definitely a common noun
3762                                let sym = self.interner.intern(&lower);
3763                                return TokenType::Noun(sym);
3764                            }
3765                            lexicon::WordAnalysis::DerivedNoun {
3766                                number: lexicon::Number::Plural,
3767                                ..
3768                            } => {
3769                                // Derived plural agentive noun (e.g., "Bloggers")
3770                                let sym = self.interner.intern(&lower);
3771                                return TokenType::Noun(sym);
3772                            }
3773                            _ => {
3774                                // Singular nouns at sentence start could still be proper names
3775                                // e.g., "John walks." vs "Farmer walks."
3776                            }
3777                        }
3778                    }
3779                }
3780            }
3781
3782            let sym = self.interner.intern(word);
3783            return TokenType::ProperName(sym);
3784        }
3785
3786        let verb_entry = self.lexicon.lookup_verb(&lower);
3787        // Irregular plurals ("children", "mice") are not in the common-noun
3788        // table; morphological analysis identifies them. Restricted to
3789        // non-verb words so "rains" stays a pure verb.
3790        let is_noun = lexicon::is_common_noun(&lower)
3791            || (verb_entry.is_none()
3792                && matches!(
3793                    lexicon::analyze_word(&lower),
3794                    Some(lexicon::WordAnalysis::Noun(_))
3795                ));
3796        let is_adj = self.is_adjective_like(&lower);
3797        let is_disambiguated = lexicon::is_disambiguation_not_verb(&lower);
3798
3799        // Ambiguous: word is Verb AND (Noun OR Adjective OR Preposition),
3800        // not disambiguated
3801        let is_prep = lexicon::is_preposition(&lower);
3802        if verb_entry.is_some() && (is_noun || is_adj || is_prep) && !is_disambiguated {
3803            let entry = verb_entry.unwrap();
3804            let verb_token = TokenType::Verb {
3805                lemma: self.interner.intern(&entry.lemma),
3806                time: entry.time,
3807                aspect: entry.aspect,
3808                class: entry.class,
3809            };
3810
3811            let mut alternatives = Vec::new();
3812            if is_noun {
3813                alternatives.push(TokenType::Noun(self.interner.intern(word)));
3814            }
3815            if is_adj {
3816                alternatives.push(TokenType::Adjective(self.interner.intern(word)));
3817            }
3818            if is_prep {
3819                alternatives.push(TokenType::Preposition(self.interner.intern(&lower)));
3820            }
3821
3822            return TokenType::Ambiguous {
3823                primary: Box::new(verb_token),
3824                alternatives,
3825            };
3826        }
3827
3828        // Disambiguated away from verb — only when there is an alternate reading.
3829        // If the word has no noun or adjective reading, the lexicon verb entry wins.
3830        if let Some(entry) = &verb_entry {
3831            if is_disambiguated {
3832                let sym = self.interner.intern(word);
3833                if is_noun {
3834                    return TokenType::Noun(sym);
3835                }
3836                if is_adj {
3837                    return TokenType::Adjective(sym);
3838                }
3839                // No alternate reading: honour the lexicon (e.g. "led" = past of "lead").
3840                return TokenType::Verb {
3841                    lemma: self.interner.intern(&entry.lemma),
3842                    time: entry.time,
3843                    aspect: entry.aspect,
3844                    class: entry.class,
3845                };
3846            }
3847        }
3848
3849        // Pure verb
3850        if let Some(entry) = verb_entry {
3851            let sym = self.interner.intern(&entry.lemma);
3852            return TokenType::Verb {
3853                lemma: sym,
3854                time: entry.time,
3855                aspect: entry.aspect,
3856                class: entry.class,
3857            };
3858        }
3859
3860        // Pure noun
3861        if is_noun {
3862            let sym = self.interner.intern(word);
3863            return TokenType::Noun(sym);
3864        }
3865
3866        // Pure adjective — a word the lexicon knows only as an adjective
3867        // ("happy", "dangerous", "tall"). Without this the word falls through
3868        // to the Noun fallback and predicative/attributive adjective readings
3869        // are lost (e.g. "John seems happy." strands "happy").
3870        if is_adj {
3871            let sym = self.interner.intern(word);
3872            return TokenType::Adjective(sym);
3873        }
3874
3875        if lexicon::is_base_verb(&lower) {
3876            let sym = self.interner.intern(&Self::capitalize(&lower));
3877            let class = lexicon::lookup_verb_class(&lower);
3878            return TokenType::Verb {
3879                lemma: sym,
3880                time: Time::Present,
3881                aspect: Aspect::Simple,
3882                class,
3883            };
3884        }
3885
3886        if lower.ends_with("ian")
3887            || lower.ends_with("er")
3888            || lower == "logic"
3889            || lower == "time"
3890            || lower == "men"
3891            || lower == "book"
3892            || lower == "house"
3893            || lower == "code"
3894            || lower == "user"
3895        {
3896            let sym = self.interner.intern(word);
3897            return TokenType::Noun(sym);
3898        }
3899
3900        if lexicon::is_particle(&lower) {
3901            let sym = self.interner.intern(&lower);
3902            return TokenType::Particle(sym);
3903        }
3904
3905        // Unknown "-ed" word → regular past-tense verb ("aired", "studied",
3906        // "debuted"). English marks the past tense morphologically, so an
3907        // otherwise-unrecognized "-ed" content word is overwhelmingly a verb.
3908        // Kept Ambiguous with a Noun reading so it can still head an NP where a
3909        // noun is expected; the parser picks the verb reading in VP position.
3910        if lower.len() >= 4 && lower.ends_with("ed") {
3911            let stem = if lower.ends_with("ied") {
3912                format!("{}y", &lower[..lower.len() - 3])
3913            } else {
3914                lower[..lower.len() - 2].to_string()
3915            };
3916            let verb_token = TokenType::Verb {
3917                lemma: self.interner.intern(&Self::capitalize(&stem)),
3918                time: Time::Past,
3919                aspect: Aspect::Simple,
3920                class: lexicon::lookup_verb_class(&stem),
3921            };
3922            return TokenType::Ambiguous {
3923                primary: Box::new(verb_token),
3924                alternatives: vec![TokenType::Noun(self.interner.intern(word))],
3925            };
3926        }
3927
3928        // Unknown lowercase words default to Noun. In natural language contexts
3929        // (puzzle clues, prose) unknown content words are overwhelmingly nouns —
3930        // domain-specific items like dance styles, place names, object names, etc.
3931        // Genuine adjectives and adverbs are caught by earlier checks.
3932        let sym = self.interner.intern(word);
3933        TokenType::Noun(sym)
3934    }
3935
3936    fn capitalize(s: &str) -> String {
3937        let mut chars = s.chars();
3938        match chars.next() {
3939            None => String::new(),
3940            Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
3941        }
3942    }
3943
3944    pub fn is_collective_verb(lemma: &str) -> bool {
3945        lexicon::is_collective_verb(&lemma.to_lowercase())
3946    }
3947
3948    pub fn is_mixed_verb(lemma: &str) -> bool {
3949        lexicon::is_mixed_verb(&lemma.to_lowercase())
3950    }
3951
3952    pub fn is_distributive_verb(lemma: &str) -> bool {
3953        lexicon::is_distributive_verb(&lemma.to_lowercase())
3954    }
3955
3956    pub fn is_intensional_predicate(lemma: &str) -> bool {
3957        lexicon::is_intensional_predicate(&lemma.to_lowercase())
3958    }
3959
3960    pub fn is_opaque_verb(lemma: &str) -> bool {
3961        lexicon::is_opaque_verb(&lemma.to_lowercase())
3962    }
3963
3964    pub fn is_ditransitive_verb(lemma: &str) -> bool {
3965        lexicon::is_ditransitive_verb(&lemma.to_lowercase())
3966    }
3967
3968    fn is_verb_like(&self, word: &str) -> bool {
3969        let lower = word.to_lowercase();
3970        if lexicon::is_infinitive_verb(&lower) {
3971            return true;
3972        }
3973        if let Some(entry) = self.lexicon.lookup_verb(&lower) {
3974            return entry.lemma.len() > 0;
3975        }
3976        false
3977    }
3978
3979    pub fn is_subject_control_verb(lemma: &str) -> bool {
3980        lexicon::is_subject_control_verb(&lemma.to_lowercase())
3981    }
3982
3983    pub fn is_raising_verb(lemma: &str) -> bool {
3984        lexicon::is_raising_verb(&lemma.to_lowercase())
3985    }
3986
3987    pub fn is_object_control_verb(lemma: &str) -> bool {
3988        lexicon::is_object_control_verb(&lemma.to_lowercase())
3989    }
3990
3991    pub fn is_weather_verb(lemma: &str) -> bool {
3992        matches!(
3993            lemma.to_lowercase().as_str(),
3994            "rain" | "snow" | "hail" | "thunder" | "pour"
3995        )
3996    }
3997
3998    fn try_parse_superlative(&self, word: &str) -> Option<String> {
3999        if !word.ends_with("est") || word.len() < 5 {
4000            return None;
4001        }
4002
4003        let base = &word[..word.len() - 3];
4004
4005        // Guard on the CHARACTER count, not the byte length: a single multibyte code point
4006        // (e.g. `ߑ`, 2 bytes / 1 char) passes a `base.len() >= 2` byte check but has only one
4007        // char, making `chars[chars.len() - 2]` underflow `usize`. Slice by the last char's
4008        // UTF-8 width so a doubled multibyte letter never cuts a code point mid-way.
4009        let chars: Vec<char> = base.chars().collect();
4010        if chars.len() >= 2 {
4011            let last = chars[chars.len() - 1];
4012            let second_last = chars[chars.len() - 2];
4013            if last == second_last && !"aeiou".contains(last) {
4014                let stem = &base[..base.len() - last.len_utf8()];
4015                if lexicon::is_gradable_adjective(stem) {
4016                    return Some(Self::capitalize(stem));
4017                }
4018            }
4019        }
4020
4021        if base.ends_with("i") {
4022            let stem = format!("{}y", &base[..base.len() - 1]);
4023            if lexicon::is_gradable_adjective(&stem) {
4024                return Some(Self::capitalize(&stem));
4025            }
4026        }
4027
4028        if lexicon::is_gradable_adjective(base) {
4029            return Some(Self::capitalize(base));
4030        }
4031
4032        None
4033    }
4034
4035    fn try_parse_comparative(&self, word: &str) -> Option<String> {
4036        if !word.ends_with("er") || word.len() < 4 {
4037            return None;
4038        }
4039
4040        let base = &word[..word.len() - 2];
4041
4042        // Same byte-vs-char hazard as try_parse_superlative: a single multibyte code point is
4043        // >= 2 bytes but only 1 char, so a `base.len() >= 2` byte guard lets `chars.len() - 2`
4044        // underflow. Guard on the char count; slice by the last char's UTF-8 width.
4045        let chars: Vec<char> = base.chars().collect();
4046        if chars.len() >= 2 {
4047            let last = chars[chars.len() - 1];
4048            let second_last = chars[chars.len() - 2];
4049            if last == second_last && !"aeiou".contains(last) {
4050                let stem = &base[..base.len() - last.len_utf8()];
4051                if lexicon::is_gradable_adjective(stem) {
4052                    return Some(Self::capitalize(stem));
4053                }
4054            }
4055        }
4056
4057        if base.ends_with("i") {
4058            let stem = format!("{}y", &base[..base.len() - 1]);
4059            if lexicon::is_gradable_adjective(&stem) {
4060                return Some(Self::capitalize(&stem));
4061            }
4062        }
4063
4064        if lexicon::is_gradable_adjective(base) {
4065            return Some(Self::capitalize(base));
4066        }
4067
4068        // Silent-e adjectives drop the 'e' before -er: "wide"→"wider",
4069        // "large"→"larger", "late"→"later". Recover the base by restoring it.
4070        let with_e = format!("{}e", base);
4071        if lexicon::is_gradable_adjective(&with_e) {
4072            return Some(Self::capitalize(&with_e));
4073        }
4074
4075        None
4076    }
4077}
4078
4079#[cfg(test)]
4080#[path = "../../logicaffeine_tests/unit/logicaffeine_language/lexer/tests.rs"]
4081mod tests;