logicaffeine_lsp/
semantic_tokens.rs

1use std::collections::{HashMap, HashSet};
2use std::sync::OnceLock;
3
4use tower_lsp::lsp_types::{
5    SemanticToken, SemanticTokenType, SemanticTokensLegend, SemanticTokenModifier,
6};
7
8use logicaffeine_language::token::{BlockType, Span, Token, TokenType};
9
10use crate::document::DocumentState;
11use crate::index::DefinitionKind;
12use crate::line_index::LineIndex;
13
14/// Our semantic token types, registered with the client.
15/// The legend is APPEND-ONLY — indices are a wire contract with client themes
16/// (pinned by `tests/locks.rs`).
17pub const TOKEN_TYPES: &[SemanticTokenType] = &[
18    SemanticTokenType::KEYWORD,    // 0
19    SemanticTokenType::TYPE,       // 1
20    SemanticTokenType::FUNCTION,   // 2
21    SemanticTokenType::VARIABLE,   // 3
22    SemanticTokenType::STRING,     // 4
23    SemanticTokenType::NUMBER,     // 5
24    SemanticTokenType::OPERATOR,   // 6
25    SemanticTokenType::NAMESPACE,  // 7
26    SemanticTokenType::MODIFIER,   // 8
27    SemanticTokenType::PROPERTY,   // 9
28    SemanticTokenType::COMMENT,    // 10
29    SemanticTokenType::PARAMETER,  // 11
30    SemanticTokenType::ENUM_MEMBER, // 12
31];
32
33pub const TOKEN_MODIFIERS: &[SemanticTokenModifier] = &[
34    SemanticTokenModifier::DECLARATION,     // bit 0
35    SemanticTokenModifier::READONLY,        // bit 1
36    SemanticTokenModifier::MODIFICATION,    // bit 2 — write sites (Set/Increase/Push targets)
37    SemanticTokenModifier::DEFAULT_LIBRARY, // bit 3 — stdlib prelude names
38];
39
40pub const TYPE_KEYWORD: u32 = 0;
41pub const TYPE_TYPE: u32 = 1;
42pub const TYPE_FUNCTION: u32 = 2;
43pub const TYPE_VARIABLE: u32 = 3;
44pub const TYPE_STRING: u32 = 4;
45pub const TYPE_NUMBER: u32 = 5;
46pub const TYPE_OPERATOR: u32 = 6;
47pub const TYPE_NAMESPACE: u32 = 7;
48pub const TYPE_MODIFIER: u32 = 8;
49pub const TYPE_PROPERTY: u32 = 9;
50pub const TYPE_COMMENT: u32 = 10;
51pub const TYPE_PARAMETER: u32 = 11;
52pub const TYPE_ENUM_MEMBER: u32 = 12;
53
54pub const MOD_DECLARATION: u32 = 1 << 0;
55pub const MOD_READONLY: u32 = 1 << 1;
56pub const MOD_MODIFICATION: u32 = 1 << 2;
57pub const MOD_DEFAULT_LIBRARY: u32 = 1 << 3;
58
59pub fn legend() -> SemanticTokensLegend {
60    SemanticTokensLegend {
61        token_types: TOKEN_TYPES.to_vec(),
62        token_modifiers: TOKEN_MODIFIERS.to_vec(),
63    }
64}
65
66/// Encode a document's tokens with the full resolution overlay.
67///
68/// The base layer is part-of-speech classification ([`classify_token`]);
69/// the overlay upgrades identifiers through the `SymbolIndex` — a name that
70/// resolves to a parameter IS a parameter, at declaration and at every
71/// reference — and adds the modifier facts: DECLARATION only where
72/// `token.span == definition.span`, READONLY for immutable `Let`s,
73/// MODIFICATION on write sites, DEFAULT_LIBRARY on stdlib prelude names.
74/// Prose inside `## Note`/`## Example` blocks recedes to COMMENT.
75pub fn encode_document_tokens(doc: &DocumentState) -> Vec<SemanticToken> {
76    let overlay = ResolutionOverlay::build(doc);
77    let spans = paint_spans(doc, &overlay, None);
78    encode_spans(&spans, &doc.line_index)
79}
80
81/// The classified paint spans for the document: one per token, EXCEPT
82/// interpolated strings, whose `{expr}` interiors expand into real code
83/// spans — string segments stay strings, braces paint as operators, and the
84/// interior re-lexes and resolves against the document's own index.
85fn paint_spans(
86    doc: &DocumentState,
87    overlay: &ResolutionOverlay,
88    range: Option<(usize, usize)>,
89) -> Vec<(Span, u32, u32)> {
90    let mut spans = Vec::new();
91    for token in &doc.tokens {
92        if let Some((start, end)) = range {
93            if token.span.end <= start || token.span.start >= end {
94                continue;
95            }
96        }
97        let (token_type, modifiers) = overlay.classify(token, doc);
98        let Some(token_type) = token_type else { continue };
99        if matches!(token.kind, TokenType::InterpolatedString(_)) && token_type == TYPE_STRING {
100            expand_interpolation(doc, token, &mut spans);
101        } else {
102            spans.push((token.span, token_type, modifiers));
103        }
104    }
105    spans
106}
107
108/// Split one interpolated-string token into string segments, `{`/`}`
109/// operators, and re-lexed interior code (`{{` escapes stay string).
110fn expand_interpolation(doc: &DocumentState, token: &Token, out: &mut Vec<(Span, u32, u32)>) {
111    let Some(src) = doc.source.get(token.span.start..token.span.end) else {
112        out.push((token.span, TYPE_STRING, 0));
113        return;
114    };
115    let base = token.span.start;
116    let bytes = src.as_bytes();
117    let mut segment_start = 0usize;
118    let mut i = 0usize;
119    while i < bytes.len() {
120        match bytes[i] {
121            b'{' if bytes.get(i + 1) == Some(&b'{') => i += 2,
122            b'{' => {
123                let Some(close_rel) = src[i + 1..].find('}') else {
124                    i += 1;
125                    continue;
126                };
127                let close = i + 1 + close_rel;
128                if i > segment_start {
129                    out.push((Span::new(base + segment_start, base + i), TYPE_STRING, 0));
130                }
131                out.push((Span::new(base + i, base + i + 1), TYPE_OPERATOR, 0));
132                paint_fragment(doc, &src[i + 1..close], base + i + 1, out);
133                out.push((Span::new(base + close, base + close + 1), TYPE_OPERATOR, 0));
134                segment_start = close + 1;
135                i = close + 1;
136            }
137            _ => i += 1,
138        }
139    }
140    if segment_start < src.len() {
141        out.push((Span::new(base + segment_start, base + src.len()), TYPE_STRING, 0));
142    }
143}
144
145/// Paint an interpolation interior: lex the fragment with the REAL lexer,
146/// classify by part of speech, and upgrade identifier-like tokens through
147/// the document's definitions — a `{name}` paints exactly like `name` would
148/// outside the string.
149fn paint_fragment(
150    doc: &DocumentState,
151    fragment: &str,
152    offset: usize,
153    out: &mut Vec<(Span, u32, u32)>,
154) {
155    let mut interner = logicaffeine_base::Interner::new();
156    let mut lexer = logicaffeine_language::lexer::Lexer::new(fragment, &mut interner);
157    let tokens = lexer.tokenize();
158    for token in tokens {
159        if token.span.end > fragment.len() || token.span.start >= token.span.end {
160            continue;
161        }
162        let (token_type, mut modifiers) = classify_token(&token.kind);
163        let Some(mut token_type) = token_type else { continue };
164        // The same surface-form resolution the overlay applies (verbs and
165        // ambiguous words resolve by what they LOOK like, not their lemma).
166        if let Some(name) = crate::index::resolve_token_name(&token, &interner) {
167            if let Some(def) = doc.symbol_index.definitions_of(name).first() {
168                token_type = semantic_type_for(&def.kind);
169                modifiers = readonly_bit(def);
170            }
171        }
172        out.push((
173            Span::new(offset + token.span.start, offset + token.span.end),
174            token_type,
175            modifiers,
176        ));
177    }
178}
179
180/// Delta-encode classified spans into the LSP wire form.
181fn encode_spans(spans: &[(Span, u32, u32)], line_index: &LineIndex) -> Vec<SemanticToken> {
182    let mut result = Vec::with_capacity(spans.len());
183    let mut prev_line = 0u32;
184    let mut prev_start = 0u32;
185    for (span, token_type, modifiers) in spans {
186        let pos = line_index.position(span.start);
187        let length = line_index.utf16_length(span.start, span.end);
188        if length == 0 {
189            continue;
190        }
191        let delta_line = pos.line - prev_line;
192        let delta_start =
193            if delta_line == 0 { pos.character - prev_start } else { pos.character };
194        result.push(SemanticToken {
195            delta_line,
196            delta_start,
197            length,
198            token_type: *token_type,
199            token_modifiers_bitset: *modifiers,
200        });
201        prev_line = pos.line;
202        prev_start = pos.character;
203    }
204    result
205}
206
207struct ResolutionOverlay {
208    /// Definition-site span starts → definition index.
209    decl_at: HashMap<usize, usize>,
210    /// Resolved reference span starts → definition index.
211    ref_at: HashMap<usize, usize>,
212    /// Span starts of write targets (Set/Increase/Push-to/…).
213    mutation_at: HashSet<usize>,
214    /// `## Note` / `## Example` block extents.
215    prose: Vec<Span>,
216    /// Every registered type name (primitives included): `Int` in an
217    /// annotation is a TYPE even though no definition site exists for it.
218    type_names: HashSet<String>,
219}
220
221impl ResolutionOverlay {
222    fn build(doc: &DocumentState) -> Self {
223        let index = &doc.symbol_index;
224
225        let mut decl_at = HashMap::new();
226        for (i, def) in index.definitions.iter().enumerate() {
227            if def.span != Span::default() {
228                decl_at.insert(def.span.start, i);
229            }
230        }
231
232        let mut ref_at = HashMap::new();
233        for reference in &index.references {
234            if let Some(def_idx) = reference.definition_idx {
235                ref_at.insert(reference.span.start, def_idx);
236            }
237        }
238
239        let prose = index
240            .block_spans
241            .iter()
242            .filter(|(_, block_type, _)| {
243                matches!(block_type, BlockType::Note | BlockType::Example)
244            })
245            .map(|(_, _, span)| *span)
246            .collect();
247
248        let type_names = doc
249            .type_registry
250            .iter_types()
251            .map(|(sym, _)| doc.interner.resolve(*sym).to_string())
252            .collect();
253
254        ResolutionOverlay {
255            decl_at,
256            ref_at,
257            mutation_at: find_mutation_targets(&doc.tokens),
258            prose,
259            type_names,
260        }
261    }
262
263    fn classify(&self, token: &Token, doc: &DocumentState) -> (Option<u32>, u32) {
264        // Headers keep their identity even when they open a prose block.
265        if matches!(token.kind, TokenType::BlockHeader { .. }) {
266            return classify_token(&token.kind);
267        }
268
269        // Documentation prose recedes: everything inside Note/Example is comment.
270        let start = token.span.start;
271        if self.prose.iter().any(|p| start > p.start && start < p.end) {
272            return (Some(TYPE_COMMENT), 0);
273        }
274
275        let (mut token_type, mut modifiers) = classify_token(&token.kind);
276
277        if let Some(&def_idx) = self.decl_at.get(&start) {
278            let def = &doc.symbol_index.definitions[def_idx];
279            token_type = Some(semantic_type_for(&def.kind));
280            modifiers = MOD_DECLARATION | readonly_bit(def);
281        } else if let Some(&def_idx) = self.ref_at.get(&start) {
282            let def = &doc.symbol_index.definitions[def_idx];
283            token_type = Some(semantic_type_for(&def.kind));
284            modifiers = readonly_bit(def);
285        } else if is_identifier_like(&token.kind)
286            && self.type_names.contains(doc.interner.resolve(token.lexeme))
287        {
288            // A registered type name with no definition site (primitives:
289            // `Int`, `Text`, …) is a TYPE wherever it appears.
290            token_type = Some(TYPE_TYPE);
291            modifiers &= !MOD_DECLARATION;
292        } else if matches!(
293            doc.interner.resolve(token.lexeme),
294            "mutable" | "mut"
295        ) {
296            // The mutability marker lexes as an English noun; it IS a
297            // storage modifier (the grammar layer agrees).
298            token_type = Some(TYPE_MODIFIER);
299            modifiers = 0;
300        } else if matches!(token.kind, TokenType::ProperName(_)) {
301            // Base layer marks ProperName as a declaration; without a resolved
302            // definition at this exact span it is a reference, not a declaration.
303            modifiers &= !MOD_DECLARATION;
304        }
305
306        if self.mutation_at.contains(&start) {
307            modifiers |= MOD_MODIFICATION;
308        }
309        if is_identifier_like(&token.kind)
310            && prelude_names().contains(doc.interner.resolve(token.lexeme))
311        {
312            modifiers |= MOD_DEFAULT_LIBRARY;
313        }
314
315        (token_type, modifiers)
316    }
317}
318
319fn readonly_bit(def: &crate::index::Definition) -> u32 {
320    match def.mutable {
321        Some(false) => MOD_READONLY,
322        _ => 0,
323    }
324}
325
326/// The semantic token type an identifier gets once resolution names its kind.
327/// Wildcard-free: a new `DefinitionKind` must decide its color here.
328pub fn semantic_type_for(kind: &DefinitionKind) -> u32 {
329    match kind {
330        DefinitionKind::Variable => TYPE_VARIABLE,
331        DefinitionKind::Function => TYPE_FUNCTION,
332        DefinitionKind::Struct => TYPE_TYPE,
333        DefinitionKind::Enum => TYPE_TYPE,
334        DefinitionKind::Field => TYPE_PROPERTY,
335        DefinitionKind::Parameter => TYPE_PARAMETER,
336        DefinitionKind::Block => TYPE_NAMESPACE,
337        DefinitionKind::Variant => TYPE_ENUM_MEMBER,
338        DefinitionKind::Theorem => TYPE_NAMESPACE,
339    }
340}
341
342fn is_identifier_like(kind: &TokenType) -> bool {
343    matches!(
344        kind,
345        TokenType::Identifier
346            | TokenType::ProperName(_)
347            | TokenType::Noun(_)
348            | TokenType::Verb { .. }
349            | TokenType::Adjective(_)
350    )
351}
352
353/// The stdlib prelude vocabulary (`md5`, `uuidV3`, `flush`, …), computed once.
354fn prelude_names() -> &'static HashSet<String> {
355    static NAMES: OnceLock<HashSet<String>> = OnceLock::new();
356    NAMES.get_or_init(|| {
357        logicaffeine_compile::loader::prelude_vocabulary()
358            .into_iter()
359            .collect()
360    })
361}
362
363/// Span starts of tokens that a statement WRITES:
364/// `Set <x> to …` / `Set … at <k> to …`-style targets take the identifier
365/// nearest the write (the last one before `to`, or before `at` when a keyed
366/// write follows), `Increase`/`Decrease <x>`, `Push`/`Add`/`Append … to <x>`,
367/// `Remove`/`Pop … from <x>`.
368pub fn find_mutation_targets(tokens: &[Token]) -> HashSet<usize> {
369    let mut targets = HashSet::new();
370    let mut i = 0;
371    while i < tokens.len() {
372        match &tokens[i].kind {
373            TokenType::Set => {
374                let mut last_ident: Option<usize> = None;
375                let mut before_at: Option<usize> = None;
376                let mut j = i + 1;
377                while j < tokens.len() {
378                    match &tokens[j].kind {
379                        TokenType::To => break,
380                        TokenType::At => before_at = last_ident,
381                        TokenType::Period => break,
382                        kind if is_identifier_like(kind) => {
383                            last_ident = Some(tokens[j].span.start)
384                        }
385                        _ => {}
386                    }
387                    j += 1;
388                }
389                if let Some(start) = before_at.or(last_ident) {
390                    targets.insert(start);
391                }
392                i = j;
393            }
394            TokenType::Increase | TokenType::Decrease => {
395                if let Some(next) = tokens[i + 1..]
396                    .iter()
397                    .find(|t| is_identifier_like(&t.kind))
398                {
399                    targets.insert(next.span.start);
400                }
401                i += 1;
402            }
403            TokenType::Push | TokenType::Add | TokenType::Append => {
404                // The collection after `to` is what changes.
405                if let Some(start) = ident_after_keyword(tokens, i, &TokenType::To) {
406                    targets.insert(start);
407                }
408                i += 1;
409            }
410            TokenType::Pop | TokenType::Remove => {
411                if let Some(start) = ident_after_keyword(tokens, i, &TokenType::From) {
412                    targets.insert(start);
413                }
414                i += 1;
415            }
416            _ => i += 1,
417        }
418    }
419    targets
420}
421
422/// The first identifier after `keyword` within the current sentence.
423fn ident_after_keyword(tokens: &[Token], from: usize, keyword: &TokenType) -> Option<usize> {
424    let mut seen_keyword = false;
425    for token in &tokens[from + 1..] {
426        match &token.kind {
427            TokenType::Period => return None,
428            kind if kind == keyword => seen_keyword = true,
429            kind if seen_keyword && is_identifier_like(kind) => {
430                return Some(token.span.start)
431            }
432            _ => {}
433        }
434    }
435    None
436}
437
438/// Convert a token stream to LSP semantic tokens (delta-encoded) using the
439/// base part-of-speech layer only. [`encode_document_tokens`] is the
440/// resolution-aware entry point.
441pub fn encode_tokens(tokens: &[Token], line_index: &LineIndex) -> Vec<SemanticToken> {
442    encode_with(tokens, line_index, |token| classify_token(&token.kind))
443}
444
445/// Resolution-aware encoding restricted to tokens overlapping a byte range.
446/// The first emitted token stays absolute from the document start, exactly as
447/// the LSP range response requires.
448pub fn encode_document_tokens_in_range(
449    doc: &DocumentState,
450    start_offset: usize,
451    end_offset: usize,
452) -> Vec<SemanticToken> {
453    let overlay = ResolutionOverlay::build(doc);
454    let spans = paint_spans(doc, &overlay, Some((start_offset, end_offset)));
455    encode_spans(&spans, &doc.line_index)
456}
457
458/// The minimal single-splice edit list turning `prev` into `next`.
459///
460/// Deltas are compared raw: a positional shift changes the first token after
461/// the edited region, so prefix/suffix agreement on the encoded stream is
462/// exactly the correct splice boundary. Offsets are in INTEGERS (5 per
463/// token), as the LSP delta contract requires.
464pub fn semantic_token_edits(
465    prev: &[SemanticToken],
466    next: &[SemanticToken],
467) -> Vec<tower_lsp::lsp_types::SemanticTokensEdit> {
468    let prefix = prev
469        .iter()
470        .zip(next.iter())
471        .take_while(|(a, b)| a == b)
472        .count();
473    if prefix == prev.len() && prefix == next.len() {
474        return Vec::new();
475    }
476    let max_suffix = prev.len().min(next.len()) - prefix;
477    let suffix = prev
478        .iter()
479        .rev()
480        .zip(next.iter().rev())
481        .take_while(|(a, b)| a == b)
482        .count()
483        .min(max_suffix);
484
485    vec![tower_lsp::lsp_types::SemanticTokensEdit {
486        start: (prefix * 5) as u32,
487        delete_count: ((prev.len() - prefix - suffix) * 5) as u32,
488        data: Some(next[prefix..next.len() - suffix].to_vec()),
489    }]
490}
491
492fn encode_with(
493    tokens: &[Token],
494    line_index: &LineIndex,
495    classify: impl Fn(&Token) -> (Option<u32>, u32),
496) -> Vec<SemanticToken> {
497    let mut result = Vec::with_capacity(tokens.len());
498    let mut prev_line = 0u32;
499    let mut prev_start = 0u32;
500
501    for token in tokens {
502        let (token_type, modifiers) = classify(token);
503        let token_type = match token_type {
504            Some(t) => t,
505            None => continue, // Skip tokens we don't highlight
506        };
507
508        let pos = line_index.position(token.span.start);
509        let length = line_index.utf16_length(token.span.start, token.span.end);
510
511        if length == 0 {
512            continue;
513        }
514
515        let delta_line = pos.line - prev_line;
516        let delta_start = if delta_line == 0 {
517            pos.character - prev_start
518        } else {
519            pos.character
520        };
521
522        result.push(SemanticToken {
523            delta_line,
524            delta_start,
525            length,
526            token_type,
527            token_modifiers_bitset: modifiers,
528        });
529
530        prev_line = pos.line;
531        prev_start = pos.character;
532    }
533
534    result
535}
536
537/// Map a `TokenType` to a semantic token type index and modifier bitset.
538///
539/// Delegates the CLASS decision to the language crate's
540/// [`logicaffeine_language::token_class`] — the single classification truth
541/// every surface (this server, the terminal REPL) shares — and adds the
542/// legend-level concerns: index mapping and the ProperName declaration bit.
543/// `tests/locks.rs` pins every classification inside the advertised legend.
544pub fn classify_token(kind: &TokenType) -> (Option<u32>, u32) {
545    use logicaffeine_language::token_class::{classify, TokenClass};
546    let class = classify(kind);
547    let index = class.map(|c| match c {
548        TokenClass::Keyword => TYPE_KEYWORD,
549        TokenClass::Type => TYPE_TYPE,
550        TokenClass::Function => TYPE_FUNCTION,
551        TokenClass::Variable => TYPE_VARIABLE,
552        TokenClass::String => TYPE_STRING,
553        TokenClass::Number => TYPE_NUMBER,
554        TokenClass::Operator => TYPE_OPERATOR,
555        TokenClass::Namespace => TYPE_NAMESPACE,
556        TokenClass::Modifier => TYPE_MODIFIER,
557        TokenClass::Property => TYPE_PROPERTY,
558        TokenClass::Comment => TYPE_COMMENT,
559        TokenClass::Parameter => TYPE_PARAMETER,
560        TokenClass::EnumMember => TYPE_ENUM_MEMBER,
561    });
562    let modifiers = if matches!(kind, TokenType::ProperName(_)) {
563        MOD_DECLARATION
564    } else {
565        0
566    };
567    (index, modifiers)
568}
569
570#[cfg(test)]
571mod tests {
572    use super::*;
573    use logicaffeine_base::Interner;
574    use logicaffeine_language::token::Span;
575
576    #[test]
577    fn all_token_types_classified() {
578        // This test ensures that adding new TokenType variants forces an update
579        // to classify_token. If this test compiles, all variants are handled.
580        let mut interner = Interner::new();
581        let sym = interner.intern("test");
582
583        let test_tokens = vec![
584            TokenType::Let,
585            TokenType::Noun(sym),
586            TokenType::Verb {
587                lemma: sym,
588                time: logicaffeine_language::lexicon::Time::Present,
589                aspect: logicaffeine_language::lexicon::Aspect::Simple,
590                class: logicaffeine_language::lexicon::VerbClass::Activity,
591            },
592            TokenType::StringLiteral(sym),
593            TokenType::Number(sym),
594            TokenType::BlockHeader { block_type: logicaffeine_language::token::BlockType::Main },
595        ];
596
597        for tt in &test_tokens {
598            let (ty, _) = classify_token(tt);
599            assert!(ty.is_some(), "Token {:?} should be classified", tt);
600        }
601    }
602
603    #[test]
604    fn keywords_classified_as_keyword() {
605        let keywords = [
606            TokenType::Let, TokenType::Set, TokenType::Return,
607            TokenType::While, TokenType::Repeat, TokenType::Push,
608        ];
609        for kw in &keywords {
610            let (ty, _) = classify_token(kw);
611            assert_eq!(ty, Some(0), "Keyword {:?} should map to type 0", kw);
612        }
613    }
614
615    #[test]
616    fn operators_classified_correctly() {
617        let ops = [
618            TokenType::Plus, TokenType::Minus, TokenType::Star,
619            TokenType::Lt, TokenType::Gt, TokenType::EqEq,
620        ];
621        for op in &ops {
622            let (ty, _) = classify_token(op);
623            assert_eq!(ty, Some(6), "Operator {:?} should map to type 6", op);
624        }
625    }
626
627    #[test]
628    fn string_literal_classified() {
629        let mut interner = Interner::new();
630        let sym = interner.intern("hello");
631        let (ty, _) = classify_token(&TokenType::StringLiteral(sym));
632        assert_eq!(ty, Some(4), "String literal should map to type 4");
633    }
634
635    #[test]
636    fn number_literal_classified() {
637        let mut interner = Interner::new();
638        let sym = interner.intern("42");
639        let (ty, _) = classify_token(&TokenType::Number(sym));
640        assert_eq!(ty, Some(5), "Number should map to type 5");
641    }
642
643    #[test]
644    fn structural_tokens_skipped() {
645        let skipped = [
646            TokenType::Indent, TokenType::Dedent,
647            TokenType::Newline, TokenType::EOF,
648        ];
649        for tt in &skipped {
650            let (ty, _) = classify_token(tt);
651            assert_eq!(ty, None, "Structural token {:?} should be skipped", tt);
652        }
653    }
654
655    #[test]
656    fn zero_length_tokens_skipped_in_encoding() {
657        let line_index = LineIndex::new("Let x be 5.");
658        let mut interner = Interner::new();
659        let sym = interner.intern("");
660        let tokens = vec![
661            Token::new(TokenType::Indent, sym, Span::new(0, 0)),
662        ];
663        let encoded = encode_tokens(&tokens, &line_index);
664        assert!(encoded.is_empty(), "Zero-length tokens should be skipped");
665    }
666
667    #[test]
668    fn multi_line_delta_encoding() {
669        let line_index = LineIndex::new("Let x\nbe 5.");
670        let mut interner = Interner::new();
671        let let_sym = interner.intern("Let");
672        let be_sym = interner.intern("be");
673
674        let tokens = vec![
675            Token::new(TokenType::Let, let_sym, Span::new(0, 3)),
676            Token::new(TokenType::Be, be_sym, Span::new(6, 8)),
677        ];
678
679        let encoded = encode_tokens(&tokens, &line_index);
680        assert_eq!(encoded.len(), 2);
681        assert_eq!(encoded[0].delta_line, 0);
682        assert_eq!(encoded[1].delta_line, 1, "Second token should be on next line");
683        assert_eq!(encoded[1].delta_start, 0, "After line change, delta_start resets");
684    }
685
686    #[test]
687    fn block_header_classified_as_namespace() {
688        let (ty, _) = classify_token(&TokenType::BlockHeader {
689            block_type: logicaffeine_language::token::BlockType::Main,
690        });
691        assert_eq!(ty, Some(7), "Block header should map to type 7 (namespace)");
692    }
693
694    #[test]
695    fn noun_classified_as_type() {
696        let mut interner = Interner::new();
697        let sym = interner.intern("person");
698        let (ty, _) = classify_token(&TokenType::Noun(sym));
699        assert_eq!(ty, Some(1), "Noun should map to type 1 (TYPE)");
700    }
701
702    #[test]
703    fn proper_name_has_declaration_modifier() {
704        let mut interner = Interner::new();
705        let sym = interner.intern("Alice");
706        let (ty, mods) = classify_token(&TokenType::ProperName(sym));
707        assert_eq!(ty, Some(3), "ProperName should map to type 3 (VARIABLE)");
708        assert_eq!(mods, 1, "ProperName should have DECLARATION modifier (bit 0)");
709    }
710
711    #[test]
712    fn ambiguous_uses_primary() {
713        let mut interner = Interner::new();
714        let sym = interner.intern("test");
715        let primary = Box::new(TokenType::Noun(sym));
716        let alternatives = vec![TokenType::Identifier];
717        let (ty, _) = classify_token(&TokenType::Ambiguous { primary, alternatives });
718        assert_eq!(ty, Some(1), "Ambiguous wrapping Noun should classify as type 1");
719    }
720
721    #[test]
722    fn encode_tokens_utf16_length() {
723        // 'é' is 2 bytes in UTF-8 but 1 UTF-16 code unit
724        let source = "café";
725        let line_index = LineIndex::new(source);
726        let mut interner = Interner::new();
727        let sym = interner.intern("café");
728        let tokens = vec![
729            Token::new(TokenType::Identifier, sym, Span::new(0, 5)), // 5 bytes: c(1)+a(1)+f(1)+é(2)
730        ];
731        let encoded = encode_tokens(&tokens, &line_index);
732        assert_eq!(encoded.len(), 1);
733        assert_eq!(encoded[0].length, 4, "UTF-16 length of 'café' should be 4, not 5 bytes");
734    }
735
736    #[test]
737    fn classify_ambiguous_nested_has_depth_guard() {
738        let mut interner = Interner::new();
739        let sym = interner.intern("test");
740        // Create a deeply nested Ambiguous chain: Ambiguous(Ambiguous(Ambiguous(Noun)))
741        let inner = TokenType::Noun(sym);
742        let mid = TokenType::Ambiguous {
743            primary: Box::new(inner),
744            alternatives: vec![TokenType::Identifier],
745        };
746        let outer = TokenType::Ambiguous {
747            primary: Box::new(mid),
748            alternatives: vec![TokenType::Identifier],
749        };
750        let (ty, _) = classify_token(&outer);
751        assert_eq!(ty, Some(1), "Nested Ambiguous wrapping Noun should still classify as type 1 (TYPE)");
752    }
753
754    #[test]
755    fn classify_ambiguous_too_deep_returns_none() {
756        let mut interner = Interner::new();
757        let sym = interner.intern("test");
758        // Create a chain deeper than the depth limit
759        let mut current = TokenType::Noun(sym);
760        for _ in 0..10 {
761            current = TokenType::Ambiguous {
762                primary: Box::new(current),
763                alternatives: vec![TokenType::Identifier],
764            };
765        }
766        // Should not stack overflow; should return None for excessive depth
767        let (ty, _) = classify_token(&current);
768        assert_eq!(ty, None, "Excessively nested Ambiguous should return None");
769    }
770
771    #[test]
772    fn delta_encoding() {
773        let line_index = LineIndex::new("Let x be 5.");
774        let mut interner = Interner::new();
775        let let_sym = interner.intern("Let");
776        let x_sym = interner.intern("x");
777
778        let tokens = vec![
779            Token::new(TokenType::Let, let_sym, Span::new(0, 3)),
780            Token::new(TokenType::Identifier, x_sym, Span::new(4, 5)),
781        ];
782
783        let encoded = encode_tokens(&tokens, &line_index);
784        assert_eq!(encoded.len(), 2);
785        assert_eq!(encoded[0].delta_line, 0);
786        assert_eq!(encoded[0].delta_start, 0);
787        assert_eq!(encoded[0].length, 3);
788        assert_eq!(encoded[1].delta_line, 0);
789        assert_eq!(encoded[1].delta_start, 4);
790        assert_eq!(encoded[1].length, 1);
791    }
792}
logicaffeine_lsp/semantic_tokens.rs

logicaffeine_lsp/
semantic_tokens.rs
