Skip to main content

sqlive/
token_scan.rs

1//! The tokens of SQL text with their byte ranges. sqlite3-parser's AST has no
2//! spans, so its lexer supplies the offsets that keep the app's spelling.
3
4use sqlite3_parser::lexer::{
5    sql::{TokenType, Tokenizer},
6    Scanner, Splitter,
7};
8use std::ops::Range;
9
10/// Each token of `sql`, without comments and whitespace, with its kind and
11/// its byte range in `sql`. SQLite's parameter names and unterminated block
12/// comments are accepted as SQLite accepts them.
13pub fn scan(sql: &str) -> rusqlite::Result<Vec<(TokenType, Range<usize>)>> {
14    let mut scanner = Scanner::new(SqliteTokenizer(Tokenizer::new()));
15    let mut tokens = Vec::new();
16    loop {
17        let (start, token, end) = scanner
18            .scan(sql.as_bytes())
19            .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?;
20        let Some((_, kind)) = token else { break };
21        tokens.push((kind, start..end));
22    }
23    Ok(tokens)
24}
25
26// sqlite3-parser omits SQLite's Tcl parameter suffixes and requires a closing
27// block comment. Keep those accepted spellings in the shared token source.
28struct SqliteTokenizer(Tokenizer);
29
30impl Splitter for SqliteTokenizer {
31    type Error = sqlite3_parser::lexer::sql::Error;
32    type TokenType = TokenType;
33
34    fn split<'a>(
35        &mut self,
36        data: &'a [u8],
37    ) -> Result<(Option<(&'a [u8], TokenType)>, usize), Self::Error> {
38        use sqlite3_parser::lexer::sql::Error;
39        if !matches!(data[0], b'$' | b'@' | b':' | b'#') {
40            return match self.0.split(data) {
41                Err(Error::UnterminatedBlockComment(_)) if data.len() > 2 => Ok((None, data.len())),
42                result => result,
43            };
44        }
45        let mut end = 1;
46        let mut named = false;
47        while let Some(&byte) = data.get(end) {
48            if byte.is_ascii_alphanumeric() || byte >= 0x80 || matches!(byte, b'_' | b'$') {
49                named = true;
50                end += 1;
51            } else if data[end..].starts_with(b"::") {
52                end += 2;
53            } else if byte == b'(' && named {
54                end += 1;
55                while data
56                    .get(end)
57                    .is_some_and(|b| !b.is_ascii_whitespace() && *b != b')' && *b != 0)
58                {
59                    end += 1;
60                }
61                if data.get(end) != Some(&b')') {
62                    return Err(Error::BadVariableName(None));
63                }
64                end += 1;
65                break;
66            } else {
67                break;
68            }
69        }
70        if !named {
71            return Err(Error::BadVariableName(None));
72        }
73        Ok((Some((&data[..end], TokenType::TK_VARIABLE)), end))
74    }
75}