cel_parser/

trivia.rs

//! Shared trivia recovery: turns the raw source text of a grammatically-constrained gap
//! (whitespace, comments, and a known fixed sequence of literal tokens) into an ordered sequence
//! of [`GapPiece`]s, and converts `proc_macro2` span positions to byte offsets. Used by
//! `cel-parser`'s own expression formatter and by `adam-lang`'s declaration formatter, so both
//! recover comments the same way. See <https://github.com/stlab/cel-rs/issues/201>.

use proc_macro2::LineColumn;

/// A recovered `//`/`/* */` comment, remembering which delimiter style the source used so a
/// formatter can reproduce it instead of normalizing every comment to one style.
///
/// # Examples
/// See [`scan_gap`] for comment recovery.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Comment {
    /// A line comment's text — its `//` and surrounding whitespace stripped. A producer that
    /// merges a run of consecutive `//` lines (e.g. adam-lang's trivia recovery) stores them
    /// joined with `\n`; cel-parser's `scan_gap` emits one `Line` per `//` line.
    Line(String),
    /// A `/* text */` block comment (single- or multi-line), inner text `\n`-joined with the
    /// outer `/*`/`*/` delimiters and per-line indentation stripped. Nested delimiters remain.
    Block(String),
}

/// One element of a scanned gap: either a recovered comment or one of the literal tokens the
/// caller declared it expected to find there.
///
/// # Examples
/// See [`scan_gap`] for ordered comment and punctuation pieces.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum GapPiece {
    /// A comment found in the gap.
    Comment(Comment),
    /// One expected literal token (an operator symbol, delimiter, comma, or keyword).
    Punct(&'static str),
}

/// Scans the raw text of a grammatically-constrained gap — whitespace, `//`/`/* */` comments, and
/// exactly the literal tokens in `expected`, in that order — returning every comment found plus
/// every `expected` token, interleaved in source order. Block comments nest; their inner
/// delimiters and text remain intact apart from per-line whitespace normalization.
///
/// - Postcondition: the result contains each element of `expected` exactly once, as a
///   `GapPiece::Punct`, in `expected`'s order, regardless of how much of `gap` actually matched
///   (an empty or synthetic `gap` still yields all expected tokens, so a hand-built `Expr` with no
///   real source re-synthesizes its separators).
///
/// - Complexity: O(n + e) time and space for n gap bytes and e expected-token bytes.
///
/// # Examples
/// ```
/// use cel_parser::trivia::{Comment, GapPiece, scan_gap};
/// assert_eq!(scan_gap("/* note */ + ", &["+"]), vec![
///     GapPiece::Comment(Comment::Block("note".into())),
///     GapPiece::Punct("+"),
/// ]);
/// ```
pub fn scan_gap(gap: &str, expected: &[&'static str]) -> Vec<GapPiece> {
    let mut pieces = Vec::new();
    let mut next = 0usize;
    let mut rest = gap;
    loop {
        rest = rest.trim_start();
        if rest.is_empty() {
            break;
        }
        if let Some(after) = rest.strip_prefix("//") {
            let end = after.find('\n').unwrap_or(after.len());
            pieces.push(GapPiece::Comment(Comment::Line(
                after[..end].trim().to_string(),
            )));
            rest = &after[end..];
        } else if let Some((inner, after)) = take_block_comment(rest) {
            pieces.push(GapPiece::Comment(Comment::Block(normalize_block(inner))));
            rest = after;
        } else if next < expected.len() && rest.starts_with(expected[next]) {
            pieces.push(GapPiece::Punct(expected[next]));
            rest = &rest[expected[next].len()..];
            next += 1;
        } else {
            // The gap didn't match the next expected token (synthetic/empty source, or an
            // unforeseen shape). Stop scanning; remaining expected tokens are synthesized below.
            break;
        }
    }
    while next < expected.len() {
        pieces.push(GapPiece::Punct(expected[next]));
        next += 1;
    }
    pieces
}

/// Splits a leading, nesting-aware block comment into its inner text and remaining source.
///
/// Returns `None` when `source` does not start with `/*`. An unterminated comment consumes
/// the remaining source as inner text, preserving trivia recovery's best-effort behavior.
///
/// - Complexity: O(n) in the leading comment's byte length, with O(1) additional space.
pub(crate) fn take_block_comment(source: &str) -> Option<(&str, &str)> {
    let inner = source.strip_prefix("/*")?;
    let bytes = inner.as_bytes();
    let mut depth = 1usize;
    let mut offset = 0usize;
    while offset + 1 < bytes.len() {
        match &bytes[offset..offset + 2] {
            b"/*" => {
                depth += 1;
                offset += 2;
            }
            b"*/" => {
                depth -= 1;
                if depth == 0 {
                    return Some((&inner[..offset], &inner[offset + 2..]));
                }
                offset += 2;
            }
            _ => offset += 1,
        }
    }
    Some((inner, ""))
}

/// Normalizes a block comment's inner text (between `/*` and `*/`): trims each line and joins the
/// non-empty ones with `\n`, matching `adam-lang`'s existing `Comment::Block` convention.
///
/// - Complexity: O(n) in `inner.len()`.
fn normalize_block(inner: &str) -> String {
    inner
        .lines()
        .map(str::trim)
        .filter(|l| !l.is_empty())
        .collect::<Vec<_>>()
        .join("\n")
}

/// Returns the byte offset of the start of each line in `source`: `result[line - 1]` is the start
/// of 1-based line `line` (matching [`proc_macro2::LineColumn::line`]'s convention).
///
/// - Complexity: O(n) in `source.len()`.
///
/// # Examples
/// See [`line_column_to_byte`] for conversion using the returned line index.
pub fn line_start_byte_offsets(source: &str) -> Vec<usize> {
    let mut offsets = vec![0usize];
    let mut byte = 0usize;
    for line in source.split_inclusive('\n') {
        byte += line.len();
        offsets.push(byte);
    }
    offsets
}

/// Converts a [`LineColumn`] (1-based line, 0-based character column) to a byte offset in
/// `source`, using `line_starts` (from [`line_start_byte_offsets`]).
///
/// - Precondition: `line_starts` was built from exactly `source`, and `pos.line - 1` is in range.
///
/// - Complexity: O(k) in `pos.column`.
///
/// # Examples
/// ```
/// use cel_parser::trivia::{line_column_to_byte, line_start_byte_offsets};
/// let source = "é\nβ";
/// let lines = line_start_byte_offsets(source);
/// let end = proc_macro2::LineColumn { line: 2, column: 1 };
/// assert_eq!(line_column_to_byte(source, &lines, end), source.len());
/// ```
pub fn line_column_to_byte(source: &str, line_starts: &[usize], pos: LineColumn) -> usize {
    debug_assert!(
        pos.line >= 1 && pos.line - 1 < line_starts.len(),
        "pos.line out of range for line_starts"
    );
    let line_start = line_starts[pos.line - 1];
    line_start
        + source[line_start..]
            .chars()
            .take(pos.column)
            .map(char::len_utf8)
            .sum::<usize>()
}

#[cfg(test)]
mod tests {
    use super::*;

    fn comments(pieces: Vec<GapPiece>) -> Vec<Comment> {
        pieces
            .into_iter()
            .filter_map(|p| match p {
                GapPiece::Comment(c) => Some(c),
                GapPiece::Punct(_) => None,
            })
            .collect()
    }

    fn puncts(pieces: &[GapPiece]) -> Vec<&'static str> {
        pieces
            .iter()
            .filter_map(|p| match p {
                GapPiece::Punct(s) => Some(*s),
                GapPiece::Comment(_) => None,
            })
            .collect()
    }

    #[test]
    fn empty_gap_yields_just_the_expected_tokens() {
        let pieces = scan_gap(" ", &["+"]);
        assert_eq!(puncts(&pieces), vec!["+"]);
        assert!(comments(pieces).is_empty());
    }

    #[test]
    fn a_block_comment_before_the_token_is_recovered_in_order() {
        let pieces = scan_gap(" /* a */ + ", &["+"]);
        assert!(matches!(pieces[0], GapPiece::Comment(Comment::Block(ref t)) if t == "a"));
        assert!(matches!(pieces[1], GapPiece::Punct("+")));
    }

    #[test]
    fn a_block_comment_after_the_token_is_recovered_in_order() {
        let pieces = scan_gap(" + /* b */ ", &["+"]);
        assert!(matches!(pieces[0], GapPiece::Punct("+")));
        assert!(matches!(pieces[1], GapPiece::Comment(Comment::Block(ref t)) if t == "b"));
    }

    #[test]
    fn comments_on_both_sides_of_the_token_keep_source_order() {
        let pieces = scan_gap(" /* a */ + /* b */ ", &["+"]);
        assert!(matches!(pieces[0], GapPiece::Comment(Comment::Block(ref t)) if t == "a"));
        assert!(matches!(pieces[1], GapPiece::Punct("+")));
        assert!(matches!(pieces[2], GapPiece::Comment(Comment::Block(ref t)) if t == "b"));
    }

    #[test]
    fn two_consecutive_block_comments_are_two_pieces() {
        let pieces = scan_gap(" /* a */ /* b */ + ", &["+"]);
        assert_eq!(comments(pieces.clone()).len(), 2);
        assert!(matches!(pieces[2], GapPiece::Punct("+")));
    }

    /// Verifies nested delimiters stay in one comment without swallowing adjacent tokens.
    #[test]
    fn nested_block_comments_preserve_text_and_token_order() {
        assert_eq!(
            scan_gap(
                " /* outer /* α /* deep */ β */ outer */ . /* after /* inner */ end */ ",
                &["."],
            ),
            vec![
                GapPiece::Comment(Comment::Block(
                    "outer /* α /* deep */ β */ outer".to_string(),
                )),
                GapPiece::Punct("."),
                GapPiece::Comment(Comment::Block("after /* inner */ end".to_string())),
            ],
        );
    }

    /// Verifies comment splitting handles empty, adjacent, Unicode, and incomplete input.
    #[test]
    fn block_comment_splitting_preserves_boundaries() {
        for (source, expected) in [
            ("", None),
            ("/", None),
            ("// line", None),
            ("/**/.x", Some(("", ".x"))),
            ("/*/**/*/.x", Some(("/**/", ".x"))),
            ("/* α /* β */ γ */δ", Some((" α /* β */ γ ", "δ"))),
            ("/* one *//* two */", Some((" one ", "/* two */"))),
            ("/*", Some(("", ""))),
            ("/* α", Some((" α", ""))),
            ("/* outer /* inner */", Some((" outer /* inner */", ""))),
            ("/*/*/", Some(("/*/", ""))),
        ] {
            assert_eq!(take_block_comment(source), expected, "{source}");
        }
    }

    #[test]
    fn two_consecutive_line_comments_are_two_pieces() {
        let pieces = scan_gap(" // a\n // b\n + ", &["+"]);
        assert_eq!(comments(pieces.clone()).len(), 2);
        assert!(matches!(pieces.last(), Some(GapPiece::Punct("+"))));
    }

    #[test]
    fn a_line_comment_is_recovered_as_comment_line() {
        let pieces = scan_gap(" + // hi\n", &["+"]);
        assert!(matches!(pieces[0], GapPiece::Punct("+")));
        assert!(matches!(pieces[1], GapPiece::Comment(Comment::Line(ref t)) if t == "hi"));
    }

    #[test]
    fn a_multi_line_block_comment_joins_its_inner_lines() {
        let pieces = scan_gap(" /*\n  one\n  two\n*/ + ", &["+"]);
        assert!(matches!(pieces[0], GapPiece::Comment(Comment::Block(ref t)) if t == "one\ntwo"));
    }

    #[test]
    fn multiple_expected_tokens_are_returned_in_order() {
        let pieces = scan_gap("} /* first */ else // second\n {", &["}", "else", "{"]);
        assert_eq!(
            pieces,
            vec![
                GapPiece::Punct("}"),
                GapPiece::Comment(Comment::Block("first".into())),
                GapPiece::Punct("else"),
                GapPiece::Comment(Comment::Line("second".into())),
                GapPiece::Punct("{"),
            ]
        );
    }

    #[test]
    fn missing_expected_token_is_still_synthesized() {
        // A synthetic/empty gap (hand-built Expr) must still yield every expected token.
        let pieces = scan_gap("", &["(", ")"]);
        assert_eq!(puncts(&pieces), vec!["(", ")"]);
    }

    #[test]
    fn line_column_to_byte_maps_a_position_on_the_second_line() {
        let source = "ab\ncd";
        let starts = line_start_byte_offsets(source);
        // line 2 (1-based), column 1 (0-based) => the 'd', byte offset 4.
        let pos = LineColumn { line: 2, column: 1 };
        assert_eq!(line_column_to_byte(source, &starts, pos), 4);
    }

    /// Maps empty, Unicode, newline, and final-line boundaries without splitting characters.
    #[test]
    fn position_conversion_covers_text_boundaries() {
        for (source, line, column, expected) in [
            ("", 1, 0, 0),
            ("é\nβ", 1, 1, 2),
            ("é\nβ", 2, 1, 5),
            ("x\n", 2, 0, 2),
        ] {
            assert_eq!(
                line_column_to_byte(
                    source,
                    &line_start_byte_offsets(source),
                    LineColumn { line, column }
                ),
                expected
            );
        }
    }
}