cel_parser/
lex_lexer.rs

1//! Flattens Rust token trees into CEL tokens while preserving source spans.
2//!
3//! Construct a [`LexLexer`] from a `proc_macro2::TokenStream` iterator and consume
4//! it as an iterator of [`Token`] values. Groups emit matching opening and closing
5//! delimiters, booleans become literals, and adjacent punctuation combines into
6//! the supported operators. Documentation attributes become comment tokens.
7//!
8//! ```
9//! use cel_parser::lex_lexer::{LexLexer, Token};
10//!
11//! let input: proc_macro2::TokenStream = "value && true".parse()?;
12//! let mut lexer = LexLexer::new(input.into_iter());
13//! assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "value"));
14//! assert!(matches!(lexer.next(), Some(Token::Punct { op, .. }) if op == "&&"));
15//! assert!(matches!(lexer.next(), Some(Token::Literal(_))));
16//! assert!(lexer.next().is_none());
17//! # Ok::<(), proc_macro2::LexError>(())
18//! ```
19//!
20//! # Error Handling
21//!
22//! A `#` commits to a documentation attribute: `#[doc = "..."]` or `#![doc = "..."]`.
23//! Other attribute shapes and literals Syn cannot parse produce [`Token::Error`].
24//! Errors consume the inspected token trees; iteration may continue with the
25//! remaining input, but does not roll back a malformed attribute.
26
27use proc_macro2::{Delimiter, Ident, Spacing, Span, TokenTree};
28use syn::Lit;
29
30/// Provides a token's source span for diagnostics.
31///
32/// # Examples
33///
34/// ```
35/// use cel_parser::lex_lexer::HasSpan;
36///
37/// let input: proc_macro2::TokenStream = "name".parse()?;
38/// let token = input.into_iter().next().unwrap();
39/// assert_eq!(HasSpan::span(&token).source_text().as_deref(), Some("name"));
40/// # Ok::<(), proc_macro2::LexError>(())
41/// ```
42pub trait HasSpan {
43    /// Returns the source span associated with this token.
44    ///
45    /// # Examples
46    ///
47    /// ```
48    /// use cel_parser::lex_lexer::HasSpan;
49    ///
50    /// let input: proc_macro2::TokenStream = "42".parse()?;
51    /// let token = input.into_iter().next().unwrap();
52    /// assert_eq!(HasSpan::span(&token).source_text().as_deref(), Some("42"));
53    /// # Ok::<(), proc_macro2::LexError>(())
54    /// ```
55    fn span(&self) -> Span;
56}
57
58/// Owns the unconsumed contents and matching close delimiter of one open group.
59struct GroupLevel {
60    iter: TokenStreamIter,
61    delimiter: Delimiter,
62    span: Span,
63}
64
65/// Owns a token stream's consuming iterator.
66///
67/// # Examples
68///
69/// ```
70/// use cel_parser::lex_lexer::TokenStreamIter;
71///
72/// let input: proc_macro2::TokenStream = "42".parse()?;
73/// let mut iter: TokenStreamIter = input.into_iter();
74/// assert!(iter.next().is_some());
75/// assert!(iter.next().is_none());
76/// # Ok::<(), proc_macro2::LexError>(())
77/// ```
78pub type TokenStreamIter = proc_macro2::token_stream::IntoIter;
79
80/// Stores one to three operator characters inline.
81///
82/// Equality with a string compares the complete character sequence, not a prefix.
83///
84/// # Examples
85///
86/// ```
87/// use cel_parser::lex_lexer::PunctOp;
88///
89/// let op = PunctOp::Two(['&', '&']);
90/// assert!(op == "&&");
91/// assert!(op != "&");
92/// assert!(op != "&&&");
93/// ```
94#[derive(Clone, Debug)]
95pub enum PunctOp {
96    /// Single character (e.g. `+`, `-`).
97    One(char),
98    /// Two characters (e.g. `&&`, `<=`, `..`).
99    Two([char; 2]),
100    /// Three characters (e.g. `..=`).
101    Three([char; 3]),
102}
103
104impl PartialEq<str> for PunctOp {
105    /// Returns whether the string contains exactly this operator's characters.
106    fn eq(&self, other: &str) -> bool {
107        let chars: &[char] = match self {
108            PunctOp::One(c) => std::slice::from_ref(c),
109            PunctOp::Two(chars) => chars,
110            PunctOp::Three(chars) => chars,
111        };
112        chars.iter().copied().eq(other.chars())
113    }
114}
115
116impl PartialEq<&str> for PunctOp {
117    /// Returns whether the borrowed string equals this operator's characters.
118    fn eq(&self, other: &&str) -> bool {
119        self.eq(*other)
120    }
121}
122
123/// Lazily flattens token trees and combines adjacent supported punctuation.
124///
125/// Each group emits matching delimiters, including `Delimiter::None`. Operators
126/// combine only across joint punctuation within the same group. Combined tokens
127/// retain the first punctuation's span; delimiters retain their group's span.
128///
129/// - Invariant: The group stack owns open groups in nesting order; lookahead and
130///   pending close tokens precede any further tokens from their parent iterator.
131/// - Complexity: Traversal visits O(n) token trees and retains O(d) group levels
132///   for nesting depth d, excluding token payloads and upstream iterator storage.
133///
134/// # Examples
135///
136/// ```
137/// use cel_parser::lex_lexer::{LexLexer, Token};
138///
139/// let input: proc_macro2::TokenStream = "(true)".parse()?;
140/// let mut lexer = LexLexer::new(input.into_iter());
141/// assert!(matches!(lexer.next(), Some(Token::OpenDelim { .. })));
142/// assert!(matches!(lexer.next(), Some(Token::Literal(syn::Lit::Bool(b))) if b.value));
143/// assert!(matches!(lexer.next(), Some(Token::CloseDelim { .. })));
144/// assert!(lexer.next().is_none());
145/// # Ok::<(), proc_macro2::LexError>(())
146/// ```
147pub struct LexLexer {
148    input: TokenStreamIter,
149    /// Stack of iterators for nested groups - allows lazy flattening.
150    /// Each entry tracks the iterator and its close delimiter info.
151    group_stack: Vec<GroupLevel>,
152    /// Pending close delimiter to emit when we've just exhausted a group iterator.
153    pending_close: Option<(Delimiter, Span)>,
154    /// Pending token that was consumed while looking ahead.
155    pending_token: Option<TokenTree>,
156}
157
158impl LexLexer {
159    /// Takes ownership of an iterator without consuming any tokens.
160    ///
161    /// # Examples
162    ///
163    /// ```
164    /// use cel_parser::lex_lexer::LexLexer;
165    ///
166    /// let lexer = LexLexer::new(proc_macro2::TokenStream::new().into_iter());
167    /// assert_eq!(lexer.count(), 0);
168    /// ```
169    pub fn new(input: TokenStreamIter) -> Self {
170        Self {
171            input,
172            group_stack: Vec::new(),
173            pending_close: None,
174            pending_token: None,
175        }
176    }
177
178    /// Parses a literal while preserving its span.
179    ///
180    /// Returns `Token::Error` if Syn cannot interpret the literal.
181    ///
182    /// - Complexity: O(n) time and space for a literal of n characters.
183    fn convert_literal(literal: proc_macro2::Literal) -> Token {
184        let span = literal.span();
185        match syn::parse2(TokenTree::Literal(literal).into()) {
186            Ok(literal) => Token::Literal(literal),
187            Err(error) => Token::Error {
188                message: error.to_string(),
189                span,
190            },
191        }
192    }
193
194    /// Converts boolean keywords to literals and preserves every other identifier.
195    ///
196    /// - Complexity: O(n) time for an identifier of n characters.
197    fn convert_identifier(ident: Ident) -> Token {
198        if ident == "true" || ident == "false" {
199            Token::Literal(Lit::Bool(syn::LitBool::new(ident == "true", ident.span())))
200        } else {
201            Token::Identifier(ident)
202        }
203    }
204
205    /// Returns whether two characters form a supported compound operator.
206    fn is_compound_operator(first: char, second: char) -> bool {
207        matches!(
208            (first, second),
209            ('&', '&')
210                | ('|', '|')
211                | ('=', '=')
212                | ('!', '=')
213                | ('<', '=')
214                | ('>', '=')
215                | ('<', '<')
216                | ('>', '>')
217                | ('-', '>')
218                | ('=', '>')
219                | (':', '=')
220                | ('.', '.')
221        )
222    }
223
224    /// Returns the next lookahead, innermost group, or root token, in that order.
225    ///
226    /// On group exhaustion, returns `None` and schedules its closing delimiter
227    /// before any further parent tokens.
228    fn next_token_tree(&mut self) -> Option<TokenTree> {
229        if let Some(token) = self.pending_token.take() {
230            return Some(token);
231        }
232
233        if let Some(level) = self.group_stack.last_mut() {
234            if let Some(tt) = level.iter.next() {
235                return Some(tt);
236            }
237            let level = self.group_stack.pop().unwrap();
238            self.pending_close = Some((level.delimiter, level.span));
239            return None;
240        }
241
242        self.input.next()
243    }
244
245    /// Consumes the longest supported operator beginning with `first`.
246    ///
247    /// Preserves the first punctuation's span and retains unmatched lookahead.
248    /// Only joint punctuation within the current group can extend the operator.
249    fn combine_punctuation(&mut self, first: proc_macro2::Punct) -> Token {
250        let ch = first.as_char();
251        let mut op = PunctOp::One(ch);
252        if first.spacing() == Spacing::Joint {
253            match self.next_token_tree() {
254                Some(TokenTree::Punct(second))
255                    if Self::is_compound_operator(ch, second.as_char()) =>
256                {
257                    let next_ch = second.as_char();
258                    op = PunctOp::Two([ch, next_ch]);
259                    if [ch, next_ch] == ['.', '.'] && second.spacing() == Spacing::Joint {
260                        match self.next_token_tree() {
261                            Some(TokenTree::Punct(third)) if third.as_char() == '=' => {
262                                op = PunctOp::Three(['.', '.', '=']);
263                            }
264                            other => self.pending_token = other,
265                        }
266                    }
267                }
268                other => self.pending_token = other,
269            }
270        }
271        Token::Punct {
272            op,
273            span: first.span(),
274        }
275    }
276
277    /// `doc_attribute_tail = ["!"] "[" "doc" "=" string_literal "]".`
278    ///
279    /// Returns a documentation token or `Token::Error` for any mismatch, consuming
280    /// the inspected token trees without rolling back.
281    ///
282    /// - Complexity: O(n) time and space for an attribute containing n characters.
283    fn parse_doc_comment_attribute(&mut self, hash_span: Span) -> Token {
284        let next = self.next_token_tree();
285        let (inner, group_token) = match next {
286            Some(TokenTree::Punct(p)) if p.as_char() == '!' => (true, self.next_token_tree()),
287            other => (false, other),
288        };
289        let group = match group_token {
290            Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Bracket => g,
291            Some(other) => {
292                return Token::Error {
293                    message: "expected `[` after `#`".to_string(),
294                    span: other.span(),
295                };
296            }
297            None => {
298                return Token::Error {
299                    message: "expected `[` after `#`".to_string(),
300                    span: hash_span,
301                };
302            }
303        };
304        let mut inner_tokens = group.stream().into_iter();
305        let ident_ok = match inner_tokens.next() {
306            Some(TokenTree::Ident(id)) => id == "doc",
307            _ => false,
308        };
309        if !ident_ok {
310            return Token::Error {
311                message: "expected `doc` after `#[`".to_string(),
312                span: group.span(),
313            };
314        }
315        let eq_ok = matches!(
316            inner_tokens.next(),
317            Some(TokenTree::Punct(p)) if p.as_char() == '='
318        );
319        if !eq_ok {
320            return Token::Error {
321                message: "expected `=` after `doc`".to_string(),
322                span: group.span(),
323            };
324        }
325        let text = match inner_tokens.next() {
326            Some(TokenTree::Literal(lit)) => {
327                let token_stream: proc_macro2::TokenStream = TokenTree::Literal(lit).into();
328                match syn::parse2::<Lit>(token_stream) {
329                    Ok(Lit::Str(s)) => s.value(),
330                    _ => {
331                        return Token::Error {
332                            message: "expected a string literal after `doc =`".to_string(),
333                            span: group.span(),
334                        };
335                    }
336                }
337            }
338            _ => {
339                return Token::Error {
340                    message: "expected a string literal after `doc =`".to_string(),
341                    span: group.span(),
342                };
343            }
344        };
345        if inner_tokens.next().is_some() {
346            return Token::Error {
347                message: "unexpected token after `doc = \"...\"`".to_string(),
348                span: group.span(),
349            };
350        }
351        Token::DocComment {
352            text,
353            inner,
354            span: hash_span,
355        }
356    }
357}
358
359/// Represents a parsed literal, including boolean keywords.
360///
361/// # Examples
362///
363/// ```
364/// use cel_parser::lex_lexer::{HasSpan, Literal};
365///
366/// let literal: Literal = syn::parse_str("true")?;
367/// assert!(matches!(literal, Literal::Bool(ref b) if b.value));
368/// let span = HasSpan::span(&literal);
369/// # Ok::<(), syn::Error>(())
370/// ```
371pub type Literal = Lit;
372
373impl HasSpan for Literal {
374    /// Returns the literal's span, including uninterpreted verbatim literals.
375    fn span(&self) -> Span {
376        syn::spanned::Spanned::span(self)
377    }
378}
379
380/// A flattened token that represents elements from a TokenTree stream.
381///
382/// Groups are flattened into OpenDelim and CloseDelim tokens, making parsing
383/// simpler by removing nesting from the token stream.
384///
385/// # Examples
386///
387/// ```
388/// use cel_parser::lex_lexer::{LexLexer, Token};
389///
390/// let input: proc_macro2::TokenStream = "name".parse()?;
391/// let token = LexLexer::new(input.into_iter()).next().unwrap();
392/// assert!(matches!(token, Token::Identifier(id) if id == "name"));
393/// # Ok::<(), proc_macro2::LexError>(())
394/// ```
395#[derive(Debug)]
396pub enum Token {
397    /// A literal value (integer, string, boolean, or float) with eager discrimination.
398    Literal(Literal),
399
400    /// An identifier.
401    Identifier(Ident),
402
403    /// A punctuation operator storing all one to three characters inline.
404    Punct {
405        /// The operator (e.g., "+", "&&", "<=").
406        op: PunctOp,
407        /// Span for error reporting.
408        span: Span,
409    },
410
411    /// Opening delimiter (flattened from Group).
412    OpenDelim {
413        /// The type of delimiter (Parenthesis, Brace, Bracket).
414        delimiter: Delimiter,
415        /// Span for error reporting.
416        span: Span,
417    },
418
419    /// Closing delimiter (flattened from Group).
420    CloseDelim {
421        /// The type of delimiter (Parenthesis, Brace, Bracket).
422        delimiter: Delimiter,
423        /// Span for error reporting.
424        span: Span,
425    },
426
427    /// A recognized `///`/`//!` doc comment, unwrapped from the `#[doc = "..."]`/
428    /// `#![doc = "..."]` attribute-shaped token sequence `proc_macro2` produces for it. General
429    /// `#[...]` attribute syntax is not supported — see `Token::Error`.
430    DocComment {
431        /// The comment's text: the doc-attribute string literal's value verbatim (including its
432        /// leading space), e.g. `" the total"` for `/// the total`.
433        text: String,
434        /// `true` for `//!`/`#![doc]` (inner), `false` for `///`/`#[doc]` (outer).
435        inner: bool,
436        /// The `#` token's span.
437        span: Span,
438    },
439
440    /// An unsupported attribute shape or a literal Syn cannot parse.
441    Error {
442        /// A description of what was expected instead.
443        message: String,
444        /// Where the mismatch was detected.
445        span: Span,
446    },
447}
448
449impl HasSpan for Token {
450    /// Returns the token's preserved source or diagnostic span.
451    fn span(&self) -> Span {
452        match self {
453            Token::Literal(lit) => lit.span(),
454            Token::Identifier(ident) => ident.span(),
455            Token::Punct { span, .. } => *span,
456            Token::OpenDelim { span, .. } => *span,
457            Token::CloseDelim { span, .. } => *span,
458            Token::DocComment { span, .. } => *span,
459            Token::Error { span, .. } => *span,
460        }
461    }
462}
463
464impl HasSpan for TokenTree {
465    /// Returns the source span of any token-tree variant.
466    fn span(&self) -> Span {
467        TokenTree::span(self)
468    }
469}
470
471impl Iterator for LexLexer {
472    type Item = Token;
473
474    /// Returns the next flattened token or `None` after exhausting the input.
475    ///
476    /// - Complexity: O(n) time and space for the consumed token payload; entering
477    ///   a group also retains its iterator and may grow the O(d) nesting stack.
478    fn next(&mut self) -> Option<Self::Item> {
479        if let Some((delimiter, span)) = self.pending_close.take() {
480            return Some(Token::CloseDelim { delimiter, span });
481        }
482
483        let token = match self.next_token_tree() {
484            Some(token) => token,
485            None => {
486                return self
487                    .pending_close
488                    .take()
489                    .map(|(delimiter, span)| Token::CloseDelim { delimiter, span });
490            }
491        };
492
493        Some(match token {
494            TokenTree::Group(group) => {
495                let delimiter = group.delimiter();
496                let span = group.span();
497                self.group_stack.push(GroupLevel {
498                    iter: group.stream().into_iter(),
499                    delimiter,
500                    span,
501                });
502                Token::OpenDelim { delimiter, span }
503            }
504            TokenTree::Punct(punct) if punct.as_char() == '#' => {
505                self.parse_doc_comment_attribute(punct.span())
506            }
507            TokenTree::Punct(punct) => self.combine_punctuation(punct),
508            TokenTree::Literal(literal) => Self::convert_literal(literal),
509            TokenTree::Ident(ident) => Self::convert_identifier(ident),
510        })
511    }
512}
513
514#[cfg(test)]
515mod tests {
516    use super::*;
517    use proc_macro2::TokenStream;
518    use std::str::FromStr;
519
520    /// Compares complete character sequences, including non-ASCII characters.
521    #[test]
522    fn punctuation_equality_compares_characters_without_prefix_matches() {
523        let cases = [
524            (PunctOp::One('+'), "+", true),
525            (PunctOp::One('+'), "++", false),
526            (PunctOp::One('+'), "", false),
527            (PunctOp::One('\u{a7}'), "\u{a7}", true),
528            (PunctOp::Two(['&', '&']), "&&", true),
529            (PunctOp::Two(['&', '&']), "&", false),
530            (PunctOp::Two(['&', '&']), "&&&", false),
531            (PunctOp::Two(['&', '&']), "&|", false),
532            (PunctOp::Three(['.', '.', '=']), "..=", true),
533            (PunctOp::Three(['.', '.', '=']), "..", false),
534            (PunctOp::Three(['.', '.', '=']), "..==", false),
535            (PunctOp::Three(['.', '.', '=']), "...", false),
536        ];
537        // Check both supported string comparison interfaces against the same contract.
538        for (op, text, expected) in cases {
539            assert_eq!(op == *text, expected, "{op:?} versus {text:?}");
540            assert_eq!(op == text, expected, "{op:?} versus {text:?}");
541        }
542    }
543
544    /// Preserves the source span of a literal not interpreted by Syn.
545    #[test]
546    fn verbatim_literal_retains_its_span() {
547        let literal = proc_macro2::Literal::from_str("42").unwrap();
548        let literal = Lit::Verbatim(literal);
549        assert_eq!(HasSpan::span(&literal).source_text().as_deref(), Some("42"));
550    }
551
552    /// Clones punctuation values without sharing mutable character storage.
553    #[test]
554    fn punctuation_clones_are_independent_values() {
555        let cases = [
556            (PunctOp::One('+'), "+"),
557            (PunctOp::Two(['&', '&']), "&&"),
558            (PunctOp::Three(['.', '.', '=']), "..="),
559        ];
560        // Mutate each original and its copy independently through their public variants.
561        for (mut original, text) in cases {
562            let mut copied = original.clone();
563            assert!(copied == text);
564            match &mut original {
565                PunctOp::One(c) => *c = '!',
566                PunctOp::Two(chars) => chars[0] = '!',
567                PunctOp::Three(chars) => chars[0] = '!',
568            }
569            assert!(copied == text);
570            copied = PunctOp::One('?');
571            assert!(copied == "?");
572            assert!(original != "?");
573            assert!(original != text);
574        }
575    }
576
577    /// Preserves literal values and spans across every supported literal category.
578    #[test]
579    fn literal_spans_preserve_the_source() {
580        // All literal variants use the same source-span contract.
581        for source in [
582            "42",
583            "1.5",
584            "\"text\"",
585            "b\"bytes\"",
586            "c\"cstr\"",
587            "'x'",
588            "b'x'",
589            "true",
590            "false",
591        ] {
592            let mut lexer = LexLexer::new(TokenStream::from_str(source).unwrap().into_iter());
593            let Some(Token::Literal(literal)) = lexer.next() else {
594                panic!("expected literal for {source}");
595            };
596            assert_eq!(
597                HasSpan::span(&literal).source_text().as_deref(),
598                Some(source)
599            );
600            assert_eq!(
601                HasSpan::span(&Token::Literal(literal))
602                    .source_text()
603                    .as_deref(),
604                Some(source)
605            );
606            assert!(lexer.next().is_none());
607        }
608    }
609
610    /// Preserves token-tree spans for groups, identifiers, punctuation, and literals.
611    #[test]
612    fn token_tree_spans_cover_each_variant() {
613        // Verify delegation for every token-tree variant.
614        for source in ["(x)", "name", "+", "42"] {
615            let tree = TokenStream::from_str(source)
616                .unwrap()
617                .into_iter()
618                .next()
619                .unwrap();
620            assert_eq!(HasSpan::span(&tree).source_text().as_deref(), Some(source));
621        }
622    }
623
624    /// Preserves diagnostic and delimiter spans rather than substituting call-site spans.
625    #[test]
626    fn flattened_tokens_preserve_their_associated_spans() {
627        let cases: &[(&str, &[&str])] = &[
628            ("(x)", &["(x)", "x", "(x)"]),
629            ("&&", &["&"]),
630            ("#[bad]", &["[bad]"]),
631        ];
632        // Group delimiters use group spans; combined punctuation uses its first character.
633        for &(source, spans) in cases {
634            let mut lexer = LexLexer::new(TokenStream::from_str(source).unwrap().into_iter());
635            for &expected in spans {
636                let token = lexer.next().unwrap();
637                assert_eq!(
638                    HasSpan::span(&token).source_text().as_deref(),
639                    Some(expected)
640                );
641            }
642            assert!(lexer.next().is_none());
643        }
644    }
645
646    /// Produces no tokens for empty input, including repeated exhaustion checks.
647    #[test]
648    fn empty_input_remains_exhausted() {
649        let mut lexer = LexLexer::new(TokenStream::new().into_iter());
650        assert!(lexer.next().is_none());
651        assert!(lexer.next().is_none());
652    }
653
654    /// Combines exactly the supported adjacent operators.
655    #[test]
656    fn combines_each_supported_operator() {
657        // Each operator must consume its complete spelling, but not the following identifier.
658        for text in [
659            "&&", "||", "==", "!=", "<=", ">=", "<<", ">>", "->", "=>", ":=", "..", "..=",
660        ] {
661            let input = TokenStream::from_str(&format!("{text} tail")).unwrap();
662            let mut lexer = LexLexer::new(input.into_iter());
663            assert!(matches!(lexer.next(), Some(Token::Punct { op, .. }) if op == text));
664            assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "tail"));
665            assert!(lexer.next().is_none());
666        }
667    }
668
669    /// Preserves uncombined lookahead tokens and respects punctuation spacing.
670    #[test]
671    fn punctuation_lookahead_preserves_the_remaining_stream() {
672        let cases: &[(&str, &[&str])] = &[
673            ("& &", &["&", "&"]),
674            (".. =", &["..", "="]),
675            ("..+", &["..", "+"]),
676            ("&&=", &["&&", "="]),
677            ("+=", &["+", "="]),
678            ("...", &["..", "."]),
679        ];
680        // Compare the entire punctuation stream so no lookahead token can disappear.
681        for &(input, expected) in cases {
682            let mut lexer = LexLexer::new(TokenStream::from_str(input).unwrap().into_iter());
683            for &text in expected {
684                assert!(matches!(lexer.next(), Some(Token::Punct { op, .. }) if op == text));
685            }
686            assert!(lexer.next().is_none(), "{input}");
687        }
688    }
689
690    /// Emits every empty group's matching delimiters before resuming its parent.
691    #[test]
692    fn empty_groups_preserve_delimiter_order() {
693        let mut lexer = LexLexer::new(TokenStream::from_str("([]{}) tail").unwrap().into_iter());
694        let expected = [
695            (true, Delimiter::Parenthesis),
696            (true, Delimiter::Bracket),
697            (false, Delimiter::Bracket),
698            (true, Delimiter::Brace),
699            (false, Delimiter::Brace),
700            (false, Delimiter::Parenthesis),
701        ];
702        // Verify both delimiter kind and nesting order.
703        for (open, expected_delimiter) in expected {
704            match lexer.next() {
705                Some(Token::OpenDelim { delimiter, .. }) if open => {
706                    assert_eq!(delimiter, expected_delimiter);
707                }
708                Some(Token::CloseDelim { delimiter, .. }) if !open => {
709                    assert_eq!(delimiter, expected_delimiter);
710                }
711                other => panic!("expected delimiter, got {other:?}"),
712            }
713        }
714        assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "tail"));
715        assert!(lexer.next().is_none());
716    }
717
718    /// Emits a group close after punctuation lookahead or an incomplete attribute.
719    #[test]
720    fn lookahead_does_not_consume_a_group_close() {
721        // Joint punctuation is constructible even when no following token exists.
722        for punctuation in ["&", "..", "#", "#!"] {
723            let group = proc_macro2::Group::new(
724                Delimiter::Parenthesis,
725                punctuation
726                    .chars()
727                    .map(|ch| TokenTree::Punct(proc_macro2::Punct::new(ch, Spacing::Joint)))
728                    .collect(),
729            );
730            let input: TokenStream = [
731                TokenTree::Group(group),
732                TokenTree::Ident(Ident::new("tail", Span::call_site())),
733            ]
734            .into_iter()
735            .collect();
736            let mut lexer = LexLexer::new(input.into_iter());
737            assert!(matches!(lexer.next(), Some(Token::OpenDelim { .. })));
738            match lexer.next() {
739                Some(Token::Punct { op, .. }) if !punctuation.starts_with('#') => {
740                    assert!(op == punctuation);
741                }
742                Some(Token::Error { message, .. }) if punctuation.starts_with('#') => {
743                    assert!(!message.is_empty());
744                }
745                other => panic!("expected punctuation or attribute error, got {other:?}"),
746            }
747            assert!(matches!(lexer.next(), Some(Token::CloseDelim { .. })));
748            assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "tail"));
749            assert!(lexer.next().is_none());
750        }
751    }
752
753    /// Emits delimiters for invisible groups without combining across their boundaries.
754    #[test]
755    fn invisible_groups_bound_operator_combining() {
756        let contents = [
757            TokenTree::Punct(proc_macro2::Punct::new('.', Spacing::Joint)),
758            TokenTree::Punct(proc_macro2::Punct::new('.', Spacing::Joint)),
759        ]
760        .into_iter()
761        .collect();
762        let input: TokenStream = [
763            TokenTree::Group(proc_macro2::Group::new(Delimiter::None, contents)),
764            TokenTree::Punct(proc_macro2::Punct::new('=', Spacing::Alone)),
765        ]
766        .into_iter()
767        .collect();
768        let mut lexer = LexLexer::new(input.into_iter());
769        assert!(matches!(
770            lexer.next(),
771            Some(Token::OpenDelim {
772                delimiter: Delimiter::None,
773                ..
774            })
775        ));
776        assert!(matches!(lexer.next(), Some(Token::Punct { op, .. }) if op == ".."));
777        assert!(matches!(
778            lexer.next(),
779            Some(Token::CloseDelim {
780                delimiter: Delimiter::None,
781                ..
782            })
783        ));
784        assert!(matches!(lexer.next(), Some(Token::Punct { op, .. }) if op == "="));
785        assert!(lexer.next().is_none());
786    }
787
788    /// Reports incomplete attributes at end of input without terminating iteration early.
789    #[test]
790    fn incomplete_attributes_at_end_of_input_report_errors() {
791        // Neither form supplies the required bracket group.
792        for source in ["#", "#!"] {
793            let mut lexer = LexLexer::new(TokenStream::from_str(source).unwrap().into_iter());
794            assert!(
795                matches!(lexer.next(), Some(Token::Error { message, .. }) if !message.is_empty())
796            );
797            assert!(lexer.next().is_none());
798            assert!(lexer.next().is_none());
799        }
800    }
801
802    /// Decodes explicit documentation strings while preserving the inner marker.
803    #[test]
804    fn explicit_documentation_attributes_decode_string_values() {
805        let input =
806            TokenStream::from_str(r##"#[doc = "line\nnext"] #![doc = r#" raw "#]"##).unwrap();
807        let mut lexer = LexLexer::new(input.into_iter());
808        assert!(
809            matches!(lexer.next(), Some(Token::DocComment { text, inner: false, .. })
810                if text == "line\nnext")
811        );
812        assert!(
813            matches!(lexer.next(), Some(Token::DocComment { text, inner: true, .. })
814                if text == " raw ")
815        );
816        assert!(lexer.next().is_none());
817    }
818
819    /// Reports each malformed attribute shape without losing the next outer token.
820    #[test]
821    fn malformed_attributes_report_errors_and_resume() {
822        // Attribute contents are untrusted input, not caller preconditions.
823        for input in [
824            "#[] tail",
825            "#[doc] tail",
826            "#[doc \"x\"] tail",
827            "#[doc =] tail",
828            "#[doc = value] tail",
829            "#[doc = b\"x\"] tail",
830            "#![doc = 1] tail",
831            "#[doc = \"x\" extra] tail",
832        ] {
833            let mut lexer = LexLexer::new(TokenStream::from_str(input).unwrap().into_iter());
834            assert!(
835                matches!(lexer.next(), Some(Token::Error { message, .. }) if !message.is_empty()),
836                "{input}"
837            );
838            assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "tail"));
839            assert!(lexer.next().is_none());
840        }
841    }
842
843    /// Recognizes booleans without treating raw or longer identifiers as literals.
844    #[test]
845    fn boolean_recognition_preserves_other_identifiers() {
846        let input = TokenStream::from_str("true false r#true true_value").unwrap();
847        let mut lexer = LexLexer::new(input.into_iter());
848        assert!(matches!(lexer.next(), Some(Token::Literal(Lit::Bool(lit))) if lit.value));
849        assert!(matches!(lexer.next(), Some(Token::Literal(Lit::Bool(lit))) if !lit.value));
850        assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "r#true"));
851        assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "true_value"));
852        assert!(lexer.next().is_none());
853    }
854
855    /// Recognizes an integer literal.
856    #[test]
857    fn test_literal_integer() {
858        let input = TokenStream::from_str("42").unwrap();
859        let mut lexer = LexLexer::new(input.into_iter());
860
861        let token = lexer.next().unwrap();
862        match token {
863            Token::Literal(Lit::Int(..)) => {}
864            _ => panic!("Expected integer literal, got {:?}", token),
865        }
866    }
867
868    /// Recognizes a string literal.
869    #[test]
870    fn test_literal_string() {
871        let input = TokenStream::from_str(r#""hello""#).unwrap();
872        let mut lexer = LexLexer::new(input.into_iter());
873
874        let token = lexer.next().unwrap();
875        match token {
876            Token::Literal(Lit::Str(..)) => {}
877            _ => panic!("Expected string literal"),
878        }
879    }
880
881    /// Recognizes both boolean keyword values.
882    #[test]
883    fn test_literal_boolean() {
884        // Test 'true' boolean literal
885        let input = TokenStream::from_str("true").unwrap();
886        let mut lexer = LexLexer::new(input.into_iter());
887
888        let token = lexer.next().unwrap();
889        match token {
890            Token::Literal(Lit::Bool(lit_bool)) => {
891                assert!(lit_bool.value);
892            }
893            _ => panic!("Expected boolean literal for 'true', got {:?}", token),
894        }
895
896        // Test 'false' boolean literal
897        let input = TokenStream::from_str("false").unwrap();
898        let mut lexer = LexLexer::new(input.into_iter());
899
900        let token = lexer.next().unwrap();
901        match token {
902            Token::Literal(Lit::Bool(lit_bool)) => {
903                assert!(!lit_bool.value);
904            }
905            _ => panic!("Expected boolean literal for 'false', got {:?}", token),
906        }
907    }
908
909    /// Recognizes a floating-point literal.
910    #[test]
911    fn test_literal_float() {
912        let input = TokenStream::from_str("3.14").unwrap();
913        let mut lexer = LexLexer::new(input.into_iter());
914
915        let token = lexer.next().unwrap();
916        match token {
917            Token::Literal(Lit::Float(..)) => {}
918            _ => panic!("Expected float literal"),
919        }
920    }
921
922    /// Preserves an identifier's spelling.
923    #[test]
924    fn test_identifier() {
925        let input = TokenStream::from_str("foo").unwrap();
926        let mut lexer = LexLexer::new(input.into_iter());
927
928        let token = lexer.next().unwrap();
929        match token {
930            Token::Identifier(ident) => {
931                assert_eq!(ident.to_string(), "foo");
932            }
933            _ => panic!("Expected identifier"),
934        }
935    }
936
937    /// Emits a single punctuation character.
938    #[test]
939    fn test_punct() {
940        let input = TokenStream::from_str("+").unwrap();
941        let mut lexer = LexLexer::new(input.into_iter());
942
943        let token = lexer.next().unwrap();
944        match token {
945            Token::Punct { op, .. } => {
946                assert!(op == "+");
947            }
948            _ => panic!("Expected punctuation"),
949        }
950    }
951
952    /// Combines a logical operator between identifiers.
953    #[test]
954    fn test_compound_operator() {
955        let input = TokenStream::from_str("a && b").unwrap();
956        let lexer = LexLexer::new(input.into_iter());
957
958        let tokens: Vec<_> = lexer.collect();
959        assert_eq!(tokens.len(), 3);
960
961        match &tokens[1] {
962            Token::Punct { op, .. } => {
963                assert!(op == "&&");
964            }
965            _ => panic!("Expected && operator"),
966        }
967    }
968
969    /// Emits matching delimiters around a group's flattened contents.
970    #[test]
971    fn test_group_flattening() {
972        let input = TokenStream::from_str("(10 + 20)").unwrap();
973        let lexer = LexLexer::new(input.into_iter());
974
975        // Should get: OpenDelim, Integer, Punct, Integer, CloseDelim
976        let tokens: Vec<_> = lexer.collect();
977        assert_eq!(tokens.len(), 5);
978
979        assert!(matches!(
980            tokens[0],
981            Token::OpenDelim {
982                delimiter: Delimiter::Parenthesis,
983                ..
984            }
985        ));
986        assert!(matches!(tokens[1], Token::Literal(Lit::Int(..))));
987        assert!(matches!(&tokens[2], Token::Punct { op, .. } if op == "+"));
988        assert!(matches!(tokens[3], Token::Literal(Lit::Int(..))));
989        assert!(matches!(
990            tokens[4],
991            Token::CloseDelim {
992                delimiter: Delimiter::Parenthesis,
993                ..
994            }
995        ));
996    }
997
998    /// Preserves the nesting order of groups and their contents.
999    #[test]
1000    fn test_nested_groups() {
1001        let input = TokenStream::from_str("(10 + (20 * 30))").unwrap();
1002        let lexer = LexLexer::new(input.into_iter());
1003
1004        let tokens: Vec<_> = lexer.collect();
1005
1006        // Should have: OpenDelim, 10, +, OpenDelim, 20, *, 30, CloseDelim, CloseDelim
1007        assert_eq!(tokens.len(), 9);
1008
1009        // Verify structure
1010        assert!(matches!(tokens[0], Token::OpenDelim { .. }));
1011        assert!(matches!(tokens[1], Token::Literal(Lit::Int(..))));
1012        assert!(matches!(&tokens[2], Token::Punct { op, .. } if op == "+"));
1013        assert!(matches!(tokens[3], Token::OpenDelim { .. }));
1014        assert!(matches!(tokens[4], Token::Literal(Lit::Int(..))));
1015        assert!(matches!(&tokens[5], Token::Punct { op, .. } if op == "*"));
1016        assert!(matches!(tokens[6], Token::Literal(Lit::Int(..))));
1017        assert!(matches!(tokens[7], Token::CloseDelim { .. }));
1018        assert!(matches!(tokens[8], Token::CloseDelim { .. }));
1019    }
1020
1021    /// Retains an identifier's source span.
1022    #[test]
1023    fn test_span_preservation() {
1024        let input = TokenStream::from_str("foo").unwrap();
1025        let mut lexer = LexLexer::new(input.into_iter());
1026
1027        let token = lexer.next().unwrap();
1028
1029        // HasSpan trait should provide span
1030        let span = HasSpan::span(&token);
1031        assert!(!span.source_text().unwrap_or_default().is_empty());
1032    }
1033
1034    /// Provides a source span through the token-tree trait implementation.
1035    #[test]
1036    fn test_haspan_trait_for_tokentree() {
1037        let input = TokenStream::from_str("42").unwrap();
1038        let tt = input.into_iter().next().unwrap();
1039
1040        assert_eq!(HasSpan::span(&tt).source_text().as_deref(), Some("42"));
1041    }
1042
1043    /// Preserves the order of identifiers, punctuation, and literals.
1044    #[test]
1045    fn test_mixed_tokens() {
1046        let input = TokenStream::from_str("foo + 42").unwrap();
1047        let lexer = LexLexer::new(input.into_iter());
1048
1049        let tokens: Vec<_> = lexer.collect();
1050        assert_eq!(tokens.len(), 3);
1051
1052        assert!(matches!(tokens[0], Token::Identifier(_)));
1053        assert!(matches!(&tokens[1], Token::Punct { op, .. } if op == "+"));
1054        assert!(matches!(tokens[2], Token::Literal(Lit::Int(..))));
1055    }
1056
1057    /// Emits an arrow as one operator.
1058    #[test]
1059    fn arrow_is_two_char_punct() {
1060        let stream: TokenStream = "->".parse().unwrap();
1061        let mut lexer = LexLexer::new(stream.into_iter());
1062        let tok = lexer.next().expect("one token");
1063        assert!(
1064            matches!(
1065                tok,
1066                Token::Punct {
1067                    op: PunctOp::Two(['-', '>']),
1068                    ..
1069                }
1070            ),
1071            "expected PunctOp::Two([\"->\"])), got {tok:?}"
1072        );
1073        assert!(lexer.next().is_none(), "expected no more tokens");
1074    }
1075
1076    /// Emits a fat arrow as one operator.
1077    #[test]
1078    fn fat_arrow_is_two_char_punct() {
1079        let stream: TokenStream = "=>".parse().unwrap();
1080        let mut lexer = LexLexer::new(stream.into_iter());
1081        let tok = lexer.next().expect("one token");
1082        assert!(
1083            matches!(
1084                tok,
1085                Token::Punct {
1086                    op: PunctOp::Two(['=', '>']),
1087                    ..
1088                }
1089            ),
1090            "expected PunctOp::Two([\"=>\"]), got {tok:?}"
1091        );
1092        assert!(lexer.next().is_none(), "expected no more tokens");
1093    }
1094
1095    /// Preserves outer documentation text and its outer marker.
1096    #[test]
1097    fn recognizes_an_outer_doc_comment() {
1098        let input = TokenStream::from_str("/// the total").unwrap();
1099        let mut lexer = LexLexer::new(input.into_iter());
1100        let token = lexer.next().unwrap();
1101        match token {
1102            Token::DocComment { text, inner, .. } => {
1103                assert_eq!(text, " the total");
1104                assert!(!inner);
1105            }
1106            other => panic!("expected DocComment, got {other:?}"),
1107        }
1108        assert!(lexer.next().is_none());
1109    }
1110
1111    /// Preserves inner documentation text and its inner marker.
1112    #[test]
1113    fn recognizes_an_inner_doc_comment() {
1114        let input = TokenStream::from_str("//! module docs").unwrap();
1115        let mut lexer = LexLexer::new(input.into_iter());
1116        let token = lexer.next().unwrap();
1117        match token {
1118            Token::DocComment { text, inner, .. } => {
1119                assert_eq!(text, " module docs");
1120                assert!(inner);
1121            }
1122            other => panic!("expected DocComment, got {other:?}"),
1123        }
1124    }
1125
1126    /// Resumes ordinary tokenization after documentation.
1127    #[test]
1128    fn doc_comment_run_is_followed_by_the_next_real_token() {
1129        let input = TokenStream::from_str("/// x\ncell").unwrap();
1130        let mut lexer = LexLexer::new(input.into_iter());
1131        assert!(matches!(lexer.next(), Some(Token::DocComment { .. })));
1132        match lexer.next() {
1133            Some(Token::Identifier(ident)) => assert_eq!(ident.to_string(), "cell"),
1134            other => panic!("expected Identifier(cell), got {other:?}"),
1135        }
1136    }
1137
1138    /// Rejects a non-documentation attribute.
1139    #[test]
1140    fn a_non_doc_attribute_is_a_lex_error() {
1141        let input = TokenStream::from_str("#[foo]").unwrap();
1142        let mut lexer = LexLexer::new(input.into_iter());
1143        let token = lexer.next().unwrap();
1144        assert!(
1145            matches!(token, Token::Error { .. }),
1146            "expected Error, got {token:?}"
1147        );
1148    }
1149
1150    /// Rejects a numeric documentation value.
1151    #[test]
1152    fn a_non_string_doc_value_is_a_lex_error() {
1153        let input = TokenStream::from_str("#[doc = 5]").unwrap();
1154        let mut lexer = LexLexer::new(input.into_iter());
1155        let token = lexer.next().unwrap();
1156        assert!(
1157            matches!(token, Token::Error { .. }),
1158            "expected Error, got {token:?}"
1159        );
1160    }
1161
1162    /// Rejects an attribute marker followed by an identifier instead of a group.
1163    #[test]
1164    fn a_bare_hash_with_no_group_is_a_lex_error() {
1165        let input = TokenStream::from_str("# foo").unwrap();
1166        let mut lexer = LexLexer::new(input.into_iter());
1167        let token = lexer.next().unwrap();
1168        assert!(
1169            matches!(token, Token::Error { .. }),
1170            "expected Error, got {token:?}"
1171        );
1172    }
1173
1174    /// Rejects an attribute group with parentheses instead of brackets.
1175    #[test]
1176    fn a_hash_led_group_with_the_wrong_delimiter_is_a_lex_error() {
1177        let input = TokenStream::from_str("#(foo)").unwrap();
1178        let mut lexer = LexLexer::new(input.into_iter());
1179        let token = lexer.next().unwrap();
1180        assert!(
1181            matches!(token, Token::Error { .. }),
1182            "expected Error, got {token:?}"
1183        );
1184    }
1185
1186    /// Rejects trailing tokens in a documentation attribute.
1187    #[test]
1188    fn extra_tokens_after_the_doc_string_are_a_lex_error() {
1189        let input = TokenStream::from_str("#[doc = \"x\", extra]").unwrap();
1190        let mut lexer = LexLexer::new(input.into_iter());
1191        let token = lexer.next().unwrap();
1192        assert!(
1193            matches!(token, Token::Error { .. }),
1194            "expected Error, got {token:?}"
1195        );
1196    }
1197
1198    /// Retains a documentation comment's source span.
1199    #[test]
1200    fn doc_comment_token_has_a_span() {
1201        let input = TokenStream::from_str("/// x").unwrap();
1202        let mut lexer = LexLexer::new(input.into_iter());
1203        let token = lexer.next().unwrap();
1204        let span = HasSpan::span(&token);
1205        assert!(!span.source_text().unwrap_or_default().is_empty());
1206    }
1207
1208    /// Emits the assignment operator as one token.
1209    #[test]
1210    fn walrus_operator_lexes_as_one_compound_token() {
1211        let stream: TokenStream = ":=".parse().unwrap();
1212        let mut lexer = LexLexer::new(stream.into_iter());
1213        let token = lexer.next().expect("one token");
1214        assert!(matches!(
1215            token,
1216            Token::Punct { ref op, .. } if op == ":="
1217        ));
1218        assert!(lexer.next().is_none());
1219    }
1220
1221    /// Preserves an identifier following a lone colon.
1222    #[test]
1223    fn colon_followed_by_non_equals_stays_two_separate_tokens() {
1224        let stream: TokenStream = ": x".parse().unwrap();
1225        let mut lexer = LexLexer::new(stream.into_iter());
1226        let first = lexer.next().expect("colon token");
1227        assert!(matches!(first, Token::Punct { ref op, .. } if op == ":"));
1228        assert!(matches!(lexer.next(), Some(Token::Identifier(id)) if id == "x"));
1229        assert!(lexer.next().is_none());
1230    }
1231
1232    /// Keeps the endpoints of a range as integer literals.
1233    #[test]
1234    fn digit_followed_by_double_dot_does_not_lex_as_a_float_literal() {
1235        let input = TokenStream::from_str("5..10").unwrap();
1236        let mut lexer = LexLexer::new(input.into_iter());
1237        match lexer.next() {
1238            Some(Token::Literal(Lit::Int(lit))) => {
1239                assert_eq!(lit.base10_parse::<i32>().unwrap(), 5)
1240            }
1241            other => panic!("expected an integer literal, got {other:?}"),
1242        }
1243        assert!(matches!(lexer.next(), Some(Token::Punct { op, .. }) if op == ".."));
1244        assert!(matches!(lexer.next(), Some(Token::Literal(Lit::Int(lit)))
1245                if lit.base10_parse::<i32>().unwrap() == 10));
1246        assert!(lexer.next().is_none());
1247    }
1248
1249    /// Emits an exclusive range operator as one token.
1250    #[test]
1251    fn double_dot_combines_into_two_char_punct() {
1252        let stream: TokenStream = "..".parse().unwrap();
1253        let mut lexer = LexLexer::new(stream.into_iter());
1254        let tok = lexer.next().expect("one token");
1255        assert!(
1256            matches!(
1257                tok,
1258                Token::Punct {
1259                    op: PunctOp::Two(['.', '.']),
1260                    ..
1261                }
1262            ),
1263            "expected PunctOp::Two(['.', '.']), got {tok:?}"
1264        );
1265        assert!(lexer.next().is_none(), "expected no more tokens");
1266    }
1267
1268    /// Emits an inclusive range operator as one token.
1269    #[test]
1270    fn double_dot_equals_combines_into_three_char_punct() {
1271        let stream: TokenStream = "..=".parse().unwrap();
1272        let mut lexer = LexLexer::new(stream.into_iter());
1273        let tok = lexer.next().expect("one token");
1274        assert!(
1275            matches!(
1276                tok,
1277                Token::Punct {
1278                    op: PunctOp::Three(['.', '.', '=']),
1279                    ..
1280                }
1281            ),
1282            "expected PunctOp::Three(['.', '.', '=']), got {tok:?}"
1283        );
1284        assert!(lexer.next().is_none(), "expected no more tokens");
1285    }
1286
1287    /// Preserves the literal after an exclusive range operator.
1288    #[test]
1289    fn double_dot_not_followed_by_equals_stays_two_char_and_does_not_consume_the_next_token() {
1290        let stream: TokenStream = "..5".parse().unwrap();
1291        let mut lexer = LexLexer::new(stream.into_iter());
1292        let first = lexer.next().expect("dot-dot token");
1293        assert!(
1294            matches!(
1295                first,
1296                Token::Punct {
1297                    op: PunctOp::Two(['.', '.']),
1298                    ..
1299                }
1300            ),
1301            "expected PunctOp::Two(['.', '.']), got {first:?}"
1302        );
1303        match lexer.next() {
1304            Some(Token::Literal(Lit::Int(lit))) => {
1305                assert_eq!(lit.base10_parse::<i32>().unwrap(), 5)
1306            }
1307            other => panic!("expected integer literal 5, got {other:?}"),
1308        }
1309    }
1310
1311    /// Preserves an integer field index after a single dot.
1312    #[test]
1313    fn single_dot_before_a_digit_is_unaffected_by_range_combining() {
1314        // Regression guard: tuple field access (`x.0`) must still lex as a lone `.` Punct
1315        // followed by an integer literal, not get swept into the new `..`/`..=` combining.
1316        let stream: TokenStream = ".0".parse().unwrap();
1317        let mut lexer = LexLexer::new(stream.into_iter());
1318        let first = lexer.next().expect("dot token");
1319        assert!(
1320            matches!(
1321                first,
1322                Token::Punct {
1323                    op: PunctOp::One('.'),
1324                    ..
1325                }
1326            ),
1327            "expected PunctOp::One('.'), got {first:?}"
1328        );
1329        assert!(matches!(lexer.next(), Some(Token::Literal(Lit::Int(lit)))
1330                if lit.base10_parse::<i32>().unwrap() == 0));
1331        assert!(lexer.next().is_none());
1332    }
1333}
cel_parser/lex_lexer.rs

cel_parser/
lex_lexer.rs
