Skip to main content

logicaffeine_language/
token.rs

1//! Token types for the LOGOS lexer and parser.
2//!
3//! This module defines the vocabulary of the LOGOS language at the token level.
4//! Tokens represent the atomic syntactic units produced by the lexer and consumed
5//! by the parser.
6//!
7//! ## Token Categories
8//!
9//! | Category | Examples | Description |
10//! |----------|----------|-------------|
11//! | **Quantifiers** | every, some, no | Bind variables over domains |
12//! | **Determiners** | the, a, this | Select referents |
13//! | **Nouns** | cat, philosopher | Predicates over individuals |
14//! | **Verbs** | runs, loves | Relations between arguments |
15//! | **Adjectives** | red, happy | Modify noun denotations |
16//! | **Connectives** | and, or, implies | Combine propositions |
17//! | **Pronouns** | he, she, it | Resolve to antecedents |
18//!
19//! ## Block Types
20//!
21//! LOGOS uses markdown-style block headers for structured documents:
22//!
23//! - `## Theorem`: Declares a proposition to be proved
24//! - `## Proof`: Contains the proof steps
25//! - `## Definition`: Introduces new terminology
26//! - `## Main`: Program entry point
27
28use logicaffeine_base::Symbol;
29use logicaffeine_lexicon::{Aspect, Case, Definiteness, Gender, Number, Time, VerbClass};
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
32pub struct Span {
33    pub start: usize,
34    pub end: usize,
35}
36
37impl Span {
38    pub fn new(start: usize, end: usize) -> Self {
39        Self { start, end }
40    }
41}
42
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum PresupKind {
45    Stop,
46    Start,
47    Regret,
48    Continue,
49    Realize,
50    Know,
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum FocusKind {
55    Only,
56    Even,
57    Just,
58    /// it-cleft / pseudo-cleft: "It was John who left." — focus + exhaustivity.
59    Cleft,
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub enum MeasureKind {
64    Much,
65    Little,
66}
67
68/// Calendar and clock time units for Span/Duration expressions.
69///
70/// `Day`/`Week`/`Month`/`Year` are variable-length CALENDAR units (they feed a `Span`);
71/// `Hour`/`Minute`/`Second`/`Millisecond`/`Microsecond`/`Nanosecond` are fixed CLOCK units
72/// (they feed a nanosecond `Duration`) — see `parse_span_literal_from_num`'s own split. One
73/// enum drives the whole word-form duration grammar (`3 seconds`, `1 millisecond`, `5 days
74/// and 2 hours`); the compact suffix spellings (`100ms`, `50ns`) are a separate lexer path.
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum CalendarUnit {
77    Second,
78    Minute,
79    Hour,
80    Day,
81    Week,
82    Month,
83    Year,
84    Millisecond,
85    Microsecond,
86    Nanosecond,
87}
88
89/// Document structure block type markers.
90///
91/// LOGOS uses markdown-style `## Header` syntax to delimit different
92/// sections of a program or proof document.
93#[derive(Debug, Clone, Copy, PartialEq, Eq)]
94pub enum BlockType {
95    /// An unknown `## Header` a small edit distance from a CONSEQUENTIAL
96    /// code header (`## Mian` → `Main`) — the parser fails loudly with the
97    /// suggestion instead of silently treating the whole block as prose.
98    SuspectedTypo { found: Symbol, suggestion: Symbol },
99    /// `## Theorem` - Declares a proposition to be proved.
100    Theorem,
101    /// `## Main` - Program entry point for imperative code.
102    Main,
103    /// `## Definition` - Introduces new terminology or type definitions.
104    Definition,
105    /// `## Define` - Mints a vernacular-logic predicate definition that the
106    /// prover unfolds (Rung 0a). Distinct from `## Definition` (type defs).
107    Define,
108    /// `## Axiom` - Declares a named first-order axiom in formal notation
109    /// (`## Axiom name: for all a b, Cong(a,b,b,a).`). Its body is parsed by the
110    /// formal-formula parser and registered as a shared premise for later theorems —
111    /// the seam for an axiomatic base like Tarski geometry.
112    Axiom,
113    /// `## Theory` - Names a development that groups the `## Axiom`s and `## Theorem`s
114    /// that follow it (`## Theory Tarski`).
115    Theory,
116    /// `## Proof` - Contains proof steps for a theorem.
117    Proof,
118    /// `## Example` - Illustrative examples.
119    Example,
120    /// `## Logic` - Direct logical notation input.
121    Logic,
122    /// `## Note` - Explanatory documentation.
123    Note,
124    /// `## To` - Function definition block.
125    Function,
126    /// `## Component <Name> (props)` - An LGX component: `State:`/`Actions:`/`View:`/
127    /// `Style:` sections lowered to a `__mount_<Name>`/`__render_<Name>` pair the runtime
128    /// drives through turns (an action runs, the view repaints, the DOM commits).
129    Component,
130    /// Inline type definition: `## A Point has:` or `## A Color is one of:`.
131    TypeDef,
132    /// `## Session` — the fields a visitor's session carries between requests.
133    ///
134    /// A RECORD, declared in the same words a record is (`a user, which is Text.`), which the
135    /// router loads from a signed cookie before dispatch and saves back when a route wrote to it.
136    /// Distinct from `## Store`, which is shared state inside ONE running page: a session belongs
137    /// to a visitor and outlives the request, and lives on the server.
138    Session,
139    /// `## Policy` - Security policy rule definitions.
140    Policy,
141    /// `## Requires` - External crate dependency declarations.
142    Requires,
143    /// `## Hardware` - Signal declarations for hardware verification.
144    Hardware,
145    /// `## Property` - Temporal assertions for hardware verification.
146    Property,
147    /// `## No` - Optimization annotation (followed by Memo, TCO, Peephole, Borrow, or Optimize).
148    No,
149    /// `## Tier` - Tiered-optimizer pin: `## Tier <opt> <eager|t1|t2|t3|never>` overrides
150    /// the hotness tier at which that optimization runs (HOTSWAP §8).
151    Tier,
152    /// `## Require <opt>` - Optimization DEMAND: the inverse of `## No`. Compilation
153    /// FAILS if the named optimization did not fire for this program (so a program
154    /// can declare it depends on an optimization, and losing it becomes an error).
155    Require,
156    /// `## Test: <prose sentence>` - An independent test entry point with an
157    /// imperative body. The prose name is captured RAW at the lexer (never
158    /// tokenized as code) and carried on the header — it is the test's
159    /// identity.
160    Test { name: Symbol },
161    /// `## Page /path (params)` — an LGX page component mounted at `path`. The path is
162    /// read RAW off the header line (a URL is not LOGOS words); the typed params that
163    /// follow lex normally.
164    Page { path: Symbol },
165    /// `## Store <Name> [persisted as "key" | shared on "topic"]` — a shared model.
166    Store,
167    /// `## Directive <name> (el: Element) and (value: T):` — a custom attribute behavior.
168    Directive,
169    /// `## Route <METHOD> /path (params) -> T:` — a typed HTTP route; method and path are
170    /// read RAW off the header line, the params and return type lex normally.
171    Route { method: Symbol, path: Symbol },
172    /// `## Before each request (r: Request) -> Option of Response:` — middleware.
173    Middleware,
174    /// `## Redirect /from -> /to.` — both paths read raw; the whole line is the header.
175    Redirect { from: Symbol, to: Symbol },
176    /// `## Error page <status>` — the view served for an HTTP error status.
177    ErrorPage { status: u16 },
178}
179
180/// Which LGX section a [`TokenType::RawBlock`] body belongs to.
181#[derive(Debug, Clone, Copy, PartialEq, Eq)]
182pub enum RawKind {
183    /// A `View:` body (or a `-> View:` function body): HTML with `{…}` template tags.
184    View,
185    /// A `Style:` body: CSS.
186    Style,
187    /// A `Head:` body: the head HTML a page declares for its own document
188    /// (title, description, social card, canonical). Parsed by the same
189    /// template machinery as a `View:`, because it is HTML with holes.
190    Head,
191}
192
193#[derive(Debug, Clone, PartialEq, Eq)]
194pub enum TokenType {
195    // Document Structure
196    BlockHeader {
197        block_type: BlockType,
198    },
199
200    // Quantifiers
201    All,
202    No,
203    Some,
204    Any,
205    Both, // Correlative conjunction marker: "both X and Y"
206    Most,
207    Few,
208    Many,
209    Cardinal(u32),
210    AtLeast(u32),
211    AtMost(u32),
212    /// "infinitely many": ∃^∞.
213    InfinitelyMany,
214    /// "finitely many", "only finitely many": ∃^<∞.
215    FinitelyMany,
216    /// "all but finitely many": ∀^∞, the cofinite quantifier.
217    AllButFinitelyMany,
218
219    // Negative Polarity Items (NPIs)
220    Anything,
221    Anyone,
222    Nothing,
223    Nobody,
224    NoOne,
225    Nowhere,
226    Ever,
227    Never,
228
229    /// "infinitely often": G F, the scope recurring without end.
230    InfinitelyOften,
231
232    // Logical Connectives
233    And,
234    Or,
235    If,
236    Then,
237    Not,
238    Iff,
239    Because,
240    /// Concessive subordinator: "although"/"though"/"even though".
241    Although,
242    /// Temporal binary connective: "P until Q"
243    Until,
244    /// Temporal binary connective: "P release Q" (dual of Until)
245    Release,
246    /// Temporal binary connective: "P weak-until Q" (Until or Always)
247    WeakUntil,
248    /// Compiler-generated implication (e.g., quantifier restrictions from Kripke lowering).
249    /// Distinguished from `If` which represents user-written conditionals.
250    /// This separation gives downstream passes (KG extraction, SVA synthesis)
251    /// irrefutable provenance: `If` = user intent, `Implies` = compiler glue.
252    Implies,
253
254    // Modal Operators
255    Must,
256    Shall,
257    Should,
258    Can,
259    May,
260    Cannot,
261    Would,
262    Could,
263    Might,
264    Had,
265
266    // Imperative Statement Keywords
267    Let,
268    Set,
269    /// "Swap x and y." — the zero-garbage reversible register exchange (`Op::RevSwap`).
270    Swap,
271    Return,
272    /// Exits the innermost while loop: `Break.`
273    Break,
274    Be,
275    While,
276    Repeat,
277    For,
278    In,
279    From,
280    Assert,
281    /// Documented assertion with justification string.
282    Trust,
283    /// Enforced runtime invariant: `Require that <cond>.` → a hard `assert!`
284    /// (survives release, unlike `Assert` → `debug_assert!`).
285    Require,
286    /// Function precondition clause: `Requires <check>.` (checked at entry).
287    Requires,
288    /// Function postcondition clause: `Ensures <check>.` (checked before return).
289    Ensures,
290    /// Test-only assertion: `Expect that <cond>.` — records-and-continues
291    /// inside `## Test`/`## Example` bodies; refused elsewhere.
292    Expect,
293    /// Virtual-clock control: `Advance time by <duration>.` (test-only).
294    Advance,
295    /// Loud skip: `Skip this test because "<reason>".` (test-only).
296    Skip,
297    /// Observation statement (developer): `Note that <clause>.`
298    Note,
299    /// Observation statement (operator): `Warn that <clause>.`
300    Warn,
301    /// Observation statement (audit): `Record that <clause>.`
302    Record,
303    /// Trace span block: `During <gerund>:` (statement-initial, imperative).
304    During,
305    Otherwise,
306    /// Alias for `Otherwise` - Pythonic else clause
307    Else,
308    /// Python-style else-if shorthand
309    Elif,
310    Call,
311    /// Constructor keyword for struct instantiation.
312    New,
313    /// Sum type definition keyword.
314    Either,
315    /// Pattern matching statement keyword.
316    Inspect,
317    /// Native function modifier for FFI bindings.
318    Native,
319    /// Escape hatch header keyword: "Escape to Rust:"
320    Escape,
321    /// Raw code block captured verbatim from an escape hatch body.
322    /// The Symbol holds the interned raw foreign code (indentation-stripped).
323    EscapeBlock(Symbol),
324    /// An LGX `View:` / `Style:` section body captured verbatim (base indentation
325    /// stripped, inner indentation and blank lines preserved). `content_start` is the byte
326    /// offset of the body's first character in the source and `indent` the whitespace
327    /// stripped from each of its lines, so an offset inside the body maps back to the file
328    /// and a template error points into the HTML.
329    RawBlock {
330        kind: RawKind,
331        body: Symbol,
332        content_start: usize,
333        indent: usize,
334    },
335
336    // Theorem Keywords
337    /// Premise marker in theorem blocks.
338    Given,
339    /// Goal marker in theorem blocks.
340    Prove,
341    /// Automatic proof strategy directive.
342    Auto,
343
344    // IO Keywords
345    /// "Read input from..."
346    Read,
347    /// "Write x to file..."
348    Write,
349    /// "...from the console"
350    Console,
351    /// "...from file..." or "...to file..."
352    File,
353
354    // Ownership Keywords (Move/Borrow Semantics)
355    /// Move ownership: "Give x to processor"
356    Give,
357    /// Immutable borrow: "Show x to console"
358    Show,
359
360    // Collection Operations
361    /// "Push x to items"
362    Push,
363    /// "Pop from items"
364    Pop,
365    /// "copy of slice" → slice.to_vec()
366    Copy,
367    /// "items 1 through 3" → inclusive slice
368    Through,
369    /// "length of items" → items.len()
370    Length,
371    /// "items at i" → `items[i]`
372    At,
373
374    // Set Operations
375    /// "Add x to set" (insert)
376    Add,
377    /// "Remove x from set"
378    Remove,
379    /// "set contains x"
380    Contains,
381    /// "a union b"
382    Union,
383    /// "a intersection b"
384    Intersection,
385
386    // Memory Management (Zones)
387    /// "Inside a new zone..."
388    Inside,
389    /// "...zone called..."
390    Zone,
391    /// "...called 'Scratch'"
392    Called,
393    /// "...of size 1 MB"
394    Size,
395    /// "...mapped from 'file.bin'"
396    Mapped,
397
398    // Structured Concurrency
399    /// "Attempt all of the following:" → concurrent (async, I/O-bound)
400    Attempt,
401    /// "the following"
402    Following,
403    /// "Simultaneously:" → parallel (CPU-bound)
404    Simultaneously,
405
406    // Agent System (Actor Model)
407    /// "Spawn a Worker called 'w1'" → create agent
408    Spawn,
409    /// "Send Ping to 'agent'" → send message to agent
410    Send,
411    /// "Await response from 'agent' into result" → receive message
412    Await,
413
414    // Serialization
415    /// "A Message is Portable and has:" → serde derives
416    Portable,
417
418    // Sipping Protocol
419    /// "the manifest of Zone" → FileSipper manifest
420    Manifest,
421    /// "the chunk at N in Zone" → FileSipper chunk
422    Chunk,
423
424    // CRDT Keywords
425    /// "A Counter is Shared and has:" → CRDT struct
426    Shared,
427    /// "Merge remote into local" → CRDT merge
428    Merge,
429    /// "Converge remote into local" → CRDT idempotent lattice join
430    Converge,
431    /// "Increase x's count by 10" → GCounter increment
432    Increase,
433
434    // Extended CRDT Keywords
435    /// "Decrease x's count by 5" → PNCounter decrement
436    Decrease,
437    /// "which is a Tally" → PNCounter type
438    Tally,
439    /// "which is a SharedSet of T" → ORSet type
440    SharedSet,
441    /// "which is a SharedSequence of T" → RGA type
442    SharedSequence,
443    /// "which is a CollaborativeSequence of T" → YATA type
444    CollaborativeSequence,
445    /// "which is a SharedMap from K to V" → ORMap type
446    SharedMap,
447    /// "which is a Divergent T" → MVRegister type
448    Divergent,
449    /// "Append x to seq" → RGA append
450    Append,
451    /// "Resolve x to value" → MVRegister resolve
452    Resolve,
453    /// "(RemoveWins)" → ORSet bias
454    RemoveWins,
455    /// "(AddWins)" → ORSet bias (default)
456    AddWins,
457    /// "(YATA)" → Sequence algorithm
458    YATA,
459    /// "x's values" → MVRegister values accessor
460    Values,
461
462    // Security Keywords
463    /// "Check that user is admin" → mandatory runtime guard
464    Check,
465
466    // P2P Networking Keywords
467    /// "Listen on \[addr\]" → bind to network address
468    Listen,
469    /// "Connect to \[addr\]" → dial a peer (NetConnect to avoid conflict)
470    NetConnect,
471    /// "Sleep N." → pause execution for N milliseconds
472    Sleep,
473
474    // GossipSub Keywords
475    /// "Sync x on 'topic'" → automatic CRDT replication
476    Sync,
477
478    // Persistence Keywords
479    /// "Mount x at \[path\]" → load/create persistent CRDT from journal
480    Mount,
481    /// "Persistent Counter" → type wrapped with journaling
482    Persistent,
483    /// "x combined with y" → string concatenation
484    Combined,
485    /// "a followed by b" → sequence concatenation (merge two sequences into one)
486    Followed,
487
488    // Go-like Concurrency Keywords
489    /// "Launch a task to..." → spawn green thread
490    Launch,
491    /// "a task" → identifier for task context
492    Task,
493    /// "Pipe of Type" → channel creation
494    Pipe,
495    /// "Receive from pipe" → recv from channel
496    Receive,
497    /// "Stop handle" → abort task
498    Stop,
499    /// "Try to send/receive" → non-blocking variant
500    Try,
501    /// "Send value into pipe" → channel send
502    Into,
503    /// "Await the first of:" → select statement
504    First,
505    /// "After N seconds:" → timeout branch
506    After,
507
508    // Block Scoping
509    Colon,
510    Indent,
511    Dedent,
512    Newline,
513
514    // Content Words
515    Noun(Symbol),
516    Adjective(Symbol),
517    NonIntersectiveAdjective(Symbol),
518    Adverb(Symbol),
519    ScopalAdverb(Symbol),
520    TemporalAdverb(Symbol),
521    Verb {
522        lemma: Symbol,
523        time: Time,
524        aspect: Aspect,
525        class: VerbClass,
526    },
527    ProperName(Symbol),
528
529    /// Lexically ambiguous token (e.g., "fish" as noun or verb).
530    ///
531    /// The parser tries the primary interpretation first, then alternatives
532    /// if parsing fails. Used for parse forest generation.
533    Ambiguous {
534        primary: Box<TokenType>,
535        alternatives: Vec<TokenType>,
536    },
537
538    // Speech Acts (Performatives)
539    Performative(Symbol),
540    Exclamation,
541
542    // Articles (Definiteness)
543    Article(Definiteness),
544
545    // Temporal Auxiliaries
546    Auxiliary(Time),
547
548    // Copula & Functional
549    Is,
550    Are,
551    Was,
552    Were,
553    That,
554    Who,
555    What,
556    Where,
557    Whose,
558    When,
559    Why,
560    Does,
561    Do,
562
563    // Identity & Reflexive (FOL)
564    Identity,
565    Equals,
566    Reflexive,
567    Reciprocal,
568    /// Pairwise list coordination: "A and B respectively love C and D"
569    Respectively,
570
571    // Pronouns (Discourse)
572    Pronoun {
573        gender: Gender,
574        number: Number,
575        case: Case,
576    },
577
578    // Prepositions (for N-ary relations)
579    Preposition(Symbol),
580
581    // Phrasal Verb Particles (up, down, out, in, off, on, away)
582    Particle(Symbol),
583
584    // Comparatives & Superlatives (Pillar 3 - Degree Semantics)
585    Comparative(Symbol),
586    Superlative(Symbol),
587    Than,
588
589    // Control Verbs (Chomsky's Control Theory)
590    To,
591
592    // Presupposition Triggers (Austin/Strawson)
593    PresupTrigger(PresupKind),
594
595    // Focus Particles (Rooth)
596    Focus(FocusKind),
597
598    // Mass Noun Measure
599    Measure(MeasureKind),
600
601    // Numeric Literals (prover-ready: stores raw string for symbolic math)
602    Number(Symbol),
603
604    /// Currency-symbol money literal: `$19.99`, `€5`, `£10`, `¥100`. Carries the magnitude (digits +
605    /// optional decimal point, thousands separators stripped) and the resolved ISO-4217 code, so a
606    /// money-aware parser builds `money(..)` while a magnitude-only consumer can still read `amount`.
607    MoneyLiteral {
608        amount: Symbol,
609        currency: Symbol,
610    },
611
612    /// Duration literal with SI suffix: 500ms, 2s, 50ns
613    /// Stores the value normalized to nanoseconds and preserves the original unit.
614    DurationLiteral {
615        nanos: i64,
616        original_unit: Symbol,
617    },
618
619    /// Date literal in ISO-8601 format: 2026-05-20
620    /// Stores days since Unix epoch (1970-01-01).
621    DateLiteral {
622        days: i32,
623    },
624
625    /// Time-of-day literal: 4pm, 9:30am, noon, midnight
626    /// Stores nanoseconds from midnight (00:00:00).
627    TimeLiteral {
628        nanos_from_midnight: i64,
629    },
630
631    /// Calendar time unit word: day, week, month, year (or plurals)
632    /// Used in Span expressions like "3 days" or "2 months and 5 days"
633    CalendarUnit(CalendarUnit),
634
635    /// Postfix operator for relative past time: "3 days ago"
636    Ago,
637
638    /// Postfix operator for relative future time: "3 days hence"
639    Hence,
640
641    /// Binary operator for span subtraction from a date: "3 days before 2026-05-20"
642    Before,
643
644    /// String literal: `"hello world"`
645    StringLiteral(Symbol),
646
647    /// Interpolated string literal: `"Hello, {name}!"`
648    /// Contains raw content with {} holes preserved
649    InterpolatedString(Symbol),
650
651    // Character literal: `x` (backtick syntax)
652    CharLiteral(Symbol),
653
654    // Index Access (1-indexed)
655    Item,
656    Items,
657
658    // Possession (Genitive Case)
659    Possessive,
660
661    // Punctuation
662    LParen,
663    RParen,
664    LBracket,
665    RBracket,
666    /// `{` — map/set literal opener (`{k: v}`, `{a, b}`). Interpolation braces
667    /// never reach here (they are consumed inside the string-literal path).
668    LBrace,
669    /// `&` in IMPERATIVE code — bitwise AND on Int, intersection on Sets.
670    /// In prose the same character stays the coordination/firm-name joiner.
671    Amp,
672    /// `|` in imperative code — bitwise OR on Int, union on Sets. (`Pipe`
673    /// is taken by the channel keyword `Pipe of T`.)
674    VBar,
675    /// `~` in imperative code — bitwise complement (lowers to `x ^ -1`).
676    Tilde,
677    /// `^` in imperative code — bitwise XOR on Int, symmetric difference on
678    /// Sets (the word `xor` remains the English spelling).
679    Caret,
680    /// `}` — map/set literal closer.
681    RBrace,
682    Comma,
683    Period,
684    /// `.` as the FIELD-ACCESS / UFCS-method operator (imperative only): `p.x`
685    /// (≡ `p's x`) and `xs.f(a)` (≡ `f(xs, a)`). Distinguished from a sentence
686    /// `Period` in the lexer by the no-whitespace + identifier-on-both-sides rule.
687    Dot,
688
689    // Bitwise Operators
690    /// "x xor y" → bitwise XOR (`^`)
691    Xor,
692    /// "x shifted left/right by y" → bit shift (`<<`/`>>`)
693    Shifted,
694
695    // Arithmetic Operators
696    Plus,
697    Minus,
698    Star,
699    Slash,
700    Percent, // Modulo operator
701    /// Compound assignment operators — `x += e` desugars to `Set x to x <op> e`.
702    PlusEq,
703    MinusEq,
704    StarEq,
705    SlashEq,
706    PercentEq,
707    /// `**` — the exponentiation operator.
708    StarStar,
709    /// `//` — floor division (rounds toward negative infinity).
710    SlashSlash,
711
712    // Comparison Operators
713    /// `<`
714    Lt,
715    /// `>`
716    Gt,
717    /// `<=`
718    LtEq,
719    /// `>=`
720    GtEq,
721    /// `==`
722    EqEq,
723    /// `!=`
724    NotEq,
725
726    /// Arrow for return type syntax: `->`
727    Arrow,
728
729    /// Assignment operator `=` for `identifier = value` syntax
730    Assign,
731
732    /// Mutability keyword `mut` for explicit mutable declarations
733    Mut,
734
735    /// Generic identifier (for equals-style assignment)
736    Identifier,
737
738    EOF,
739}
740
741#[derive(Debug, Clone)]
742pub struct Token {
743    pub kind: TokenType,
744    pub lexeme: Symbol,
745    pub span: Span,
746}
747
748impl Token {
749    pub fn new(kind: TokenType, lexeme: Symbol, span: Span) -> Self {
750        Token { kind, lexeme, span }
751    }
752}
753
754impl TokenType {
755    pub const WH_WORDS: &'static [TokenType] = &[
756        TokenType::Who,
757        TokenType::What,
758        TokenType::Where,
759        TokenType::When,
760        TokenType::Why,
761    ];
762
763    /// A determiner beyond `all`, `no`, `some`, `any` and the counted ones, binding a quantifier
764    /// of its own: `most`, `few`, `many`, `infinitely many`, `finitely many`, `all but finitely
765    /// many`.
766    pub fn is_generalized_quantifier(&self) -> bool {
767        matches!(
768            self,
769            TokenType::Most
770                | TokenType::Few
771                | TokenType::Many
772                | TokenType::InfinitelyMany
773                | TokenType::FinitelyMany
774                | TokenType::AllButFinitelyMany
775        )
776    }
777
778    pub const MODALS: &'static [TokenType] = &[
779        TokenType::Must,
780        TokenType::Shall,
781        TokenType::Should,
782        TokenType::Can,
783        TokenType::May,
784        TokenType::Cannot,
785        TokenType::Would,
786        TokenType::Could,
787        TokenType::Might,
788    ];
789}
790
791#[cfg(test)]
792#[path = "../../logicaffeine_tests/unit/logicaffeine_language/token/tests.rs"]
793mod tests;