1use sqlite3_parser::lexer::{
5 sql::{TokenType, Tokenizer},
6 Scanner, Splitter,
7};
8use std::ops::Range;
9
10pub fn scan(sql: &str) -> rusqlite::Result<Vec<(TokenType, Range<usize>)>> {
14 let mut scanner = Scanner::new(SqliteTokenizer(Tokenizer::new()));
15 let mut tokens = Vec::new();
16 loop {
17 let (start, token, end) = scanner
18 .scan(sql.as_bytes())
19 .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?;
20 let Some((_, kind)) = token else { break };
21 tokens.push((kind, start..end));
22 }
23 Ok(tokens)
24}
25
26struct SqliteTokenizer(Tokenizer);
29
30impl Splitter for SqliteTokenizer {
31 type Error = sqlite3_parser::lexer::sql::Error;
32 type TokenType = TokenType;
33
34 fn split<'a>(
35 &mut self,
36 data: &'a [u8],
37 ) -> Result<(Option<(&'a [u8], TokenType)>, usize), Self::Error> {
38 use sqlite3_parser::lexer::sql::Error;
39 if !matches!(data[0], b'$' | b'@' | b':' | b'#') {
40 return match self.0.split(data) {
41 Err(Error::UnterminatedBlockComment(_)) if data.len() > 2 => Ok((None, data.len())),
42 result => result,
43 };
44 }
45 let mut end = 1;
46 let mut named = false;
47 while let Some(&byte) = data.get(end) {
48 if byte.is_ascii_alphanumeric() || byte >= 0x80 || matches!(byte, b'_' | b'$') {
49 named = true;
50 end += 1;
51 } else if data[end..].starts_with(b"::") {
52 end += 2;
53 } else if byte == b'(' && named {
54 end += 1;
55 while data
56 .get(end)
57 .is_some_and(|b| !b.is_ascii_whitespace() && *b != b')' && *b != 0)
58 {
59 end += 1;
60 }
61 if data.get(end) != Some(&b')') {
62 return Err(Error::BadVariableName(None));
63 }
64 end += 1;
65 break;
66 } else {
67 break;
68 }
69 }
70 if !named {
71 return Err(Error::BadVariableName(None));
72 }
73 Ok((Some((&data[..end], TokenType::TK_VARIABLE)), end))
74 }
75}