mirror of
https://github.com/modrinth/code.git
synced 2026-09-03 05:25:58 +00:00
wip: search filter AST
This commit is contained in:
@@ -35,6 +35,7 @@ bitflags = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
censor = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
chumsky = { workspace = true }
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
clickhouse = { workspace = true, features = ["time", "uuid"] }
|
||||
color-eyre = { workspace = true }
|
||||
|
||||
@@ -167,6 +167,7 @@ vars! {
|
||||
TYPESENSE_INDEX_PREFIX: String = "labrinth";
|
||||
TYPESENSE_IMPORT_BATCH_SIZE: usize = 5000usize;
|
||||
TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize;
|
||||
TYPESENSE_USE_CACHE: bool = true;
|
||||
|
||||
// storage
|
||||
STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
|
||||
|
||||
@@ -0,0 +1,454 @@
|
||||
use std::fmt::{self, Display, Formatter};
|
||||
|
||||
use eyre::{Result, eyre};
|
||||
|
||||
use crate::search::SearchField;
|
||||
use crate::search::filter::{
|
||||
FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
|
||||
FilterPredicate,
|
||||
};
|
||||
|
||||
const MAX_DNF_CLAUSES: usize = 64;
|
||||
const MAX_FILTER_DEPTH: usize = 64;
|
||||
const MAX_FILTER_NODES: usize = 1024;
|
||||
const MAX_SERIALIZED_FILTER_BYTES: usize = 64 * 1024;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum FilterScope {
|
||||
Project,
|
||||
Version,
|
||||
Mixed,
|
||||
}
|
||||
|
||||
enum TypesenseFilter<'a> {
|
||||
And(Vec<Self>),
|
||||
Or(Vec<Self>),
|
||||
Predicate {
|
||||
predicate: &'a FilterPredicate,
|
||||
version: bool,
|
||||
},
|
||||
Join {
|
||||
collection: &'a str,
|
||||
filter: Box<Self>,
|
||||
},
|
||||
}
|
||||
|
||||
pub(super) fn serialize_filter(
|
||||
filter: &FilterExpr,
|
||||
versions_collection: &str,
|
||||
) -> Result<String> {
|
||||
let (nodes, depth) = filter_complexity(filter);
|
||||
if nodes > MAX_FILTER_NODES {
|
||||
return Err(eyre!("search filter has too many expressions"));
|
||||
}
|
||||
if depth > MAX_FILTER_DEPTH {
|
||||
return Err(eyre!("search filter is nested too deeply"));
|
||||
}
|
||||
|
||||
let filter = plan(filter, versions_collection)?;
|
||||
let serialized = filter.to_string();
|
||||
if serialized.len() > MAX_SERIALIZED_FILTER_BYTES {
|
||||
return Err(eyre!("search filter is too large"));
|
||||
}
|
||||
Ok(serialized)
|
||||
}
|
||||
|
||||
fn plan<'a>(
|
||||
filter: &'a FilterExpr,
|
||||
versions_collection: &'a str,
|
||||
) -> Result<TypesenseFilter<'a>> {
|
||||
match filter_scope(filter) {
|
||||
FilterScope::Project => lower(filter, false),
|
||||
FilterScope::Version => Ok(TypesenseFilter::Join {
|
||||
collection: versions_collection,
|
||||
filter: Box::new(lower(filter, true)?),
|
||||
}),
|
||||
FilterScope::Mixed => plan_mixed(filter, versions_collection),
|
||||
}
|
||||
}
|
||||
|
||||
fn plan_mixed<'a>(
|
||||
filter: &'a FilterExpr,
|
||||
versions_collection: &'a str,
|
||||
) -> Result<TypesenseFilter<'a>> {
|
||||
match filter {
|
||||
FilterExpr::Or(expressions) => expressions
|
||||
.iter()
|
||||
.map(|expression| plan(expression, versions_collection))
|
||||
.collect::<Result<Vec<_>>>()
|
||||
.map(TypesenseFilter::Or),
|
||||
FilterExpr::And(expressions)
|
||||
if expressions.iter().all(|expression| {
|
||||
filter_scope(expression) != FilterScope::Mixed
|
||||
}) =>
|
||||
{
|
||||
plan_partitioned_and(expressions, versions_collection)
|
||||
}
|
||||
_ => {
|
||||
let clauses = to_dnf(filter)?;
|
||||
clauses
|
||||
.into_iter()
|
||||
.map(|clause| plan_clause(clause, versions_collection))
|
||||
.collect::<Result<Vec<_>>>()
|
||||
.map(TypesenseFilter::Or)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn plan_partitioned_and<'a>(
|
||||
expressions: &'a [FilterExpr],
|
||||
versions_collection: &'a str,
|
||||
) -> Result<TypesenseFilter<'a>> {
|
||||
let mut project = Vec::new();
|
||||
let mut version = Vec::new();
|
||||
for expression in expressions {
|
||||
match filter_scope(expression) {
|
||||
FilterScope::Project => project.push(lower(expression, false)?),
|
||||
FilterScope::Version => version.push(lower(expression, true)?),
|
||||
FilterScope::Mixed => {
|
||||
return Err(eyre!("could not partition mixed search filter"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(filter) = and_filter(version) {
|
||||
project.push(TypesenseFilter::Join {
|
||||
collection: versions_collection,
|
||||
filter: Box::new(filter),
|
||||
});
|
||||
}
|
||||
and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
|
||||
}
|
||||
|
||||
fn plan_clause<'a>(
|
||||
predicates: Vec<&'a FilterPredicate>,
|
||||
versions_collection: &'a str,
|
||||
) -> Result<TypesenseFilter<'a>> {
|
||||
let mut project = Vec::new();
|
||||
let mut version = Vec::new();
|
||||
for predicate in predicates {
|
||||
let planned = lower_predicate(predicate)?;
|
||||
if is_version_filter_field(predicate.field.as_str()) {
|
||||
version.push(planned);
|
||||
} else {
|
||||
project.push(planned);
|
||||
}
|
||||
}
|
||||
if let Some(filter) = and_filter(version) {
|
||||
project.push(TypesenseFilter::Join {
|
||||
collection: versions_collection,
|
||||
filter: Box::new(filter),
|
||||
});
|
||||
}
|
||||
and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
|
||||
}
|
||||
|
||||
fn lower(filter: &FilterExpr, version: bool) -> Result<TypesenseFilter<'_>> {
|
||||
match filter {
|
||||
FilterExpr::And(expressions) => expressions
|
||||
.iter()
|
||||
.map(|expression| lower(expression, version))
|
||||
.collect::<Result<Vec<_>>>()
|
||||
.map(TypesenseFilter::And),
|
||||
FilterExpr::Or(expressions) => expressions
|
||||
.iter()
|
||||
.map(|expression| lower(expression, version))
|
||||
.collect::<Result<Vec<_>>>()
|
||||
.map(TypesenseFilter::Or),
|
||||
FilterExpr::Predicate(predicate) => {
|
||||
validate_predicate(predicate)?;
|
||||
Ok(TypesenseFilter::Predicate { predicate, version })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn lower_predicate(predicate: &FilterPredicate) -> Result<TypesenseFilter<'_>> {
|
||||
validate_predicate(predicate)?;
|
||||
Ok(TypesenseFilter::Predicate {
|
||||
predicate,
|
||||
version: is_version_filter_field(predicate.field.as_str()),
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_predicate(predicate: &FilterPredicate) -> Result<()> {
|
||||
if matches!(predicate.condition, FilterCondition::Exists { .. }) {
|
||||
return Err(eyre!(
|
||||
"filter field `{}` does not support `EXISTS`",
|
||||
predicate.field.as_str()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn filter_scope(filter: &FilterExpr) -> FilterScope {
|
||||
match filter {
|
||||
FilterExpr::Predicate(predicate) => {
|
||||
if is_version_filter_field(predicate.field.as_str()) {
|
||||
FilterScope::Version
|
||||
} else {
|
||||
FilterScope::Project
|
||||
}
|
||||
}
|
||||
FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
|
||||
let mut scopes = expressions.iter().map(filter_scope);
|
||||
let Some(first) = scopes.next() else {
|
||||
return FilterScope::Project;
|
||||
};
|
||||
if scopes.all(|scope| scope == first) {
|
||||
first
|
||||
} else {
|
||||
FilterScope::Mixed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_version_filter_field(field: &str) -> bool {
|
||||
<SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
|
||||
search_field.is_version_field()
|
||||
&& search_field.typesense_spec().path == field
|
||||
})
|
||||
}
|
||||
|
||||
fn to_dnf(filter: &FilterExpr) -> Result<Vec<Vec<&FilterPredicate>>> {
|
||||
match filter {
|
||||
FilterExpr::Predicate(predicate) => Ok(vec![vec![predicate]]),
|
||||
FilterExpr::Or(expressions) => {
|
||||
let mut clauses = Vec::new();
|
||||
for expression in expressions {
|
||||
clauses.extend(to_dnf(expression)?);
|
||||
if clauses.len() > MAX_DNF_CLAUSES {
|
||||
return Err(eyre!(
|
||||
"search filter has too many boolean clauses"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(clauses)
|
||||
}
|
||||
FilterExpr::And(expressions) => {
|
||||
let mut clauses = vec![Vec::new()];
|
||||
for expression in expressions {
|
||||
let right = to_dnf(expression)?;
|
||||
if clauses.len().saturating_mul(right.len()) > MAX_DNF_CLAUSES {
|
||||
return Err(eyre!(
|
||||
"search filter has too many boolean clauses"
|
||||
));
|
||||
}
|
||||
clauses = clauses
|
||||
.into_iter()
|
||||
.flat_map(|left| {
|
||||
right.iter().map(move |right| {
|
||||
let mut clause = left.clone();
|
||||
clause.extend(right);
|
||||
clause
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
Ok(clauses)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn and_filter(
|
||||
mut expressions: Vec<TypesenseFilter<'_>>,
|
||||
) -> Option<TypesenseFilter<'_>> {
|
||||
match expressions.len() {
|
||||
0 => None,
|
||||
1 => expressions.pop(),
|
||||
_ => Some(TypesenseFilter::And(expressions)),
|
||||
}
|
||||
}
|
||||
|
||||
fn filter_complexity(filter: &FilterExpr) -> (usize, usize) {
|
||||
match filter {
|
||||
FilterExpr::Predicate(_) => (1, 1),
|
||||
FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
|
||||
expressions.iter().map(filter_complexity).fold(
|
||||
(1, 1),
|
||||
|(nodes, depth), (child_nodes, child_depth)| {
|
||||
(nodes + child_nodes, depth.max(child_depth + 1))
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for TypesenseFilter<'_> {
|
||||
fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
|
||||
self.fmt_with_precedence(formatter, 0)
|
||||
}
|
||||
}
|
||||
|
||||
impl TypesenseFilter<'_> {
|
||||
fn precedence(&self) -> u8 {
|
||||
match self {
|
||||
Self::Or(_) => 1,
|
||||
Self::And(_) => 2,
|
||||
Self::Predicate { .. } | Self::Join { .. } => 3,
|
||||
}
|
||||
}
|
||||
|
||||
fn fmt_with_precedence(
|
||||
&self,
|
||||
formatter: &mut Formatter<'_>,
|
||||
parent_precedence: u8,
|
||||
) -> fmt::Result {
|
||||
let precedence = self.precedence();
|
||||
let parenthesize = precedence < parent_precedence;
|
||||
if parenthesize {
|
||||
formatter.write_str("(")?;
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::And(expressions) => {
|
||||
format_expressions(formatter, expressions, " && ", precedence)?;
|
||||
}
|
||||
Self::Or(expressions) => {
|
||||
format_expressions(formatter, expressions, " || ", precedence)?;
|
||||
}
|
||||
Self::Predicate { predicate, version } => {
|
||||
format_predicate(formatter, predicate, *version)?;
|
||||
}
|
||||
Self::Join { collection, filter } => {
|
||||
write!(formatter, "${collection}(")?;
|
||||
filter.fmt_with_precedence(formatter, 0)?;
|
||||
formatter.write_str(")")?;
|
||||
}
|
||||
}
|
||||
|
||||
if parenthesize {
|
||||
formatter.write_str(")")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn format_expressions(
|
||||
formatter: &mut Formatter<'_>,
|
||||
expressions: &[TypesenseFilter<'_>],
|
||||
separator: &str,
|
||||
precedence: u8,
|
||||
) -> fmt::Result {
|
||||
for (index, expression) in expressions.iter().enumerate() {
|
||||
if index != 0 {
|
||||
formatter.write_str(separator)?;
|
||||
}
|
||||
expression.fmt_with_precedence(formatter, precedence)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn format_predicate(
|
||||
formatter: &mut Formatter<'_>,
|
||||
predicate: &FilterPredicate,
|
||||
version: bool,
|
||||
) -> fmt::Result {
|
||||
formatter.write_str(predicate.field.as_str())?;
|
||||
match &predicate.condition {
|
||||
FilterCondition::Compare { comparison, value } => {
|
||||
let operator = match comparison {
|
||||
FilterComparison::Equal if version => ":",
|
||||
FilterComparison::Equal => ":=",
|
||||
FilterComparison::NotEqual => ":!=",
|
||||
FilterComparison::GreaterThan => ":>",
|
||||
FilterComparison::GreaterThanOrEqual => ":>=",
|
||||
FilterComparison::LessThan => ":<",
|
||||
FilterComparison::LessThanOrEqual => ":<=",
|
||||
};
|
||||
formatter.write_str(operator)?;
|
||||
format_literal(formatter, value)
|
||||
}
|
||||
FilterCondition::In { values, negated } => {
|
||||
formatter.write_str(if *negated { ":!=" } else { ":" })?;
|
||||
formatter.write_str("[")?;
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
if index != 0 {
|
||||
formatter.write_str(",")?;
|
||||
}
|
||||
format_literal(formatter, value)?;
|
||||
}
|
||||
formatter.write_str("]")
|
||||
}
|
||||
FilterCondition::Exists { .. } => unreachable!(
|
||||
"unsupported predicates are rejected before serialization"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_literal(
|
||||
formatter: &mut Formatter<'_>,
|
||||
literal: &FilterLiteral,
|
||||
) -> fmt::Result {
|
||||
match literal {
|
||||
FilterLiteral::String(value) => {
|
||||
formatter.write_str("`")?;
|
||||
for character in value.chars() {
|
||||
if character == '`' {
|
||||
formatter.write_str("\\")?;
|
||||
}
|
||||
write!(formatter, "{character}")?;
|
||||
}
|
||||
formatter.write_str("`")
|
||||
}
|
||||
FilterLiteral::Number(value) => formatter.write_str(value),
|
||||
FilterLiteral::Bool(value) => Display::fmt(value, formatter),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::serialize_filter;
|
||||
use crate::search::filter::{normalize, parse_expression};
|
||||
|
||||
fn serialize(input: &str) -> String {
|
||||
let filter = normalize(parse_expression(input).unwrap());
|
||||
serialize_filter(&filter, "versions").unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn project_filters_do_not_join_versions() {
|
||||
assert_eq!(serialize("license = MIT"), "license:=`MIT`");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correlated_version_filters_share_one_join() {
|
||||
assert_eq!(
|
||||
serialize("categories = fabric AND game_versions = 1.21"),
|
||||
"$versions(categories:`fabric` && game_versions:1.21)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_boolean_filters_preserve_version_correlation() {
|
||||
let filter = serialize(
|
||||
"(license = MIT OR categories = fabric) AND game_versions = 1.21",
|
||||
);
|
||||
|
||||
assert_eq!(filter.matches("$versions(").count(), 2);
|
||||
assert!(filter.contains("categories:`fabric` && game_versions:1.21"));
|
||||
assert!(filter.contains("license:=`MIT`"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_values_are_escaped() {
|
||||
assert_eq!(
|
||||
serialize(r#"license = "value, with (syntax) and `tick`""#),
|
||||
r#"license:=`value, with (syntax) and \`tick\``"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cartesian_version_filter_uses_one_join() {
|
||||
let filter = serialize(
|
||||
"(project_types = modpack AND game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
|
||||
(project_types = modpack AND game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
|
||||
(project_types = modpack AND game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
|
||||
(project_types = modpack AND game_versions = 1.21.1 AND categories = forge AND categories = technology)",
|
||||
);
|
||||
|
||||
assert_eq!(filter.matches("$versions(").count(), 1);
|
||||
assert!(filter.contains("categories:[`fabric`,`forge`]"));
|
||||
assert!(filter.contains("game_versions:[`1.20.1`,`1.21.1`]"));
|
||||
assert!(filter.contains("categories:`technology`"));
|
||||
}
|
||||
}
|
||||
@@ -1,425 +0,0 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use eyre::{Result, WrapErr, eyre};
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::search::SearchField;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct JoinedFilterClause {
|
||||
project: Vec<String>,
|
||||
version: Vec<String>,
|
||||
}
|
||||
|
||||
pub(super) fn rewrite_filter_for_join(
|
||||
filter: &str,
|
||||
versions_collection: &str,
|
||||
) -> Result<String> {
|
||||
const MAX_CLAUSES: usize = 256;
|
||||
|
||||
fn parse(expression: &str) -> Result<Vec<JoinedFilterClause>> {
|
||||
let expression = trim_outer_parentheses(expression.trim());
|
||||
|
||||
let or_parts = split_top_level(expression, "||");
|
||||
if or_parts.len() > 1 {
|
||||
let mut clauses = Vec::new();
|
||||
for part in or_parts {
|
||||
clauses.extend(parse(part)?);
|
||||
if clauses.len() > MAX_CLAUSES {
|
||||
return Err(eyre!(
|
||||
"search filter has too many boolean clauses"
|
||||
));
|
||||
}
|
||||
}
|
||||
return Ok(clauses);
|
||||
}
|
||||
|
||||
let and_parts = split_top_level(expression, "&&");
|
||||
if and_parts.len() > 1 {
|
||||
let mut clauses = vec![JoinedFilterClause::default()];
|
||||
for part in and_parts {
|
||||
let right = parse(part)?;
|
||||
if clauses.len().saturating_mul(right.len()) > MAX_CLAUSES {
|
||||
return Err(eyre!(
|
||||
"search filter has too many boolean clauses"
|
||||
));
|
||||
}
|
||||
clauses = clauses
|
||||
.into_iter()
|
||||
.cartesian_product(right)
|
||||
.map(|(mut left, right)| {
|
||||
left.project.extend(right.project);
|
||||
left.version.extend(right.version);
|
||||
left
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
return Ok(clauses);
|
||||
}
|
||||
|
||||
let field = filter_field(expression).ok_or_else(|| {
|
||||
eyre!("could not determine filter field in `{expression}`")
|
||||
})?;
|
||||
let mut clause = JoinedFilterClause::default();
|
||||
if is_version_filter_field(field) {
|
||||
clause.version.push(version_filter_expression(expression));
|
||||
} else {
|
||||
clause.project.push(expression.to_string());
|
||||
}
|
||||
Ok(vec![clause])
|
||||
}
|
||||
|
||||
let clauses = parse(filter)?;
|
||||
Ok(clauses
|
||||
.into_iter()
|
||||
.map(|clause| {
|
||||
let mut parts = clause.project;
|
||||
if !clause.version.is_empty() {
|
||||
parts.push(format!(
|
||||
"${versions_collection}({})",
|
||||
clause.version.join(" && ")
|
||||
));
|
||||
}
|
||||
if parts.len() == 1 {
|
||||
parts.pop().unwrap_or_default()
|
||||
} else {
|
||||
format!("({})", parts.join(" && "))
|
||||
}
|
||||
})
|
||||
.join(" || "))
|
||||
}
|
||||
|
||||
fn is_version_filter_field(field: &str) -> bool {
|
||||
<SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
|
||||
search_field.is_version_field()
|
||||
&& search_field.typesense_spec().path == field
|
||||
})
|
||||
}
|
||||
|
||||
fn version_filter_expression(expression: &str) -> String {
|
||||
let Some((field, value)) = expression.split_once(':') else {
|
||||
return expression.to_string();
|
||||
};
|
||||
let Some(value) = value.strip_prefix('=') else {
|
||||
return expression.to_string();
|
||||
};
|
||||
format!("{field}:{value}")
|
||||
}
|
||||
|
||||
fn filter_field(expression: &str) -> Option<&str> {
|
||||
let operator = expression.find(':')?;
|
||||
let field = expression[..operator].trim();
|
||||
(!field.is_empty()
|
||||
&& field.chars().all(|character| {
|
||||
character.is_ascii_alphanumeric() || "_.".contains(character)
|
||||
}))
|
||||
.then_some(field)
|
||||
}
|
||||
|
||||
fn trim_outer_parentheses(mut expression: &str) -> &str {
|
||||
while expression.starts_with('(')
|
||||
&& expression.ends_with(')')
|
||||
&& matching_outer_parentheses(expression)
|
||||
{
|
||||
expression = expression[1..expression.len() - 1].trim();
|
||||
}
|
||||
expression
|
||||
}
|
||||
|
||||
fn matching_outer_parentheses(expression: &str) -> bool {
|
||||
let mut depth = 0;
|
||||
let mut quote = None;
|
||||
let mut escaped = false;
|
||||
|
||||
for (index, character) in expression.char_indices() {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
if character == '\\' {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
if let Some(active_quote) = quote {
|
||||
if character == active_quote {
|
||||
quote = None;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if matches!(character, '\'' | '"' | '`') {
|
||||
quote = Some(character);
|
||||
continue;
|
||||
}
|
||||
match character {
|
||||
'(' => depth += 1,
|
||||
')' => {
|
||||
depth -= 1;
|
||||
if depth == 0 && index + character.len_utf8() < expression.len()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
depth == 0
|
||||
}
|
||||
|
||||
fn split_top_level<'a>(expression: &'a str, operator: &str) -> Vec<&'a str> {
|
||||
let mut parts = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut parentheses = 0;
|
||||
let mut brackets = 0;
|
||||
let mut quote = None;
|
||||
let mut escaped = false;
|
||||
let bytes = expression.as_bytes();
|
||||
let mut index = 0;
|
||||
|
||||
while index < bytes.len() {
|
||||
let character = expression[index..].chars().next().unwrap_or_default();
|
||||
let width = character.len_utf8();
|
||||
if escaped {
|
||||
escaped = false;
|
||||
index += width;
|
||||
continue;
|
||||
}
|
||||
if character == '\\' {
|
||||
escaped = true;
|
||||
index += width;
|
||||
continue;
|
||||
}
|
||||
if let Some(active_quote) = quote {
|
||||
if character == active_quote {
|
||||
quote = None;
|
||||
}
|
||||
index += width;
|
||||
continue;
|
||||
}
|
||||
if matches!(character, '\'' | '"' | '`') {
|
||||
quote = Some(character);
|
||||
index += width;
|
||||
continue;
|
||||
}
|
||||
match character {
|
||||
'(' => parentheses += 1,
|
||||
')' => parentheses -= 1,
|
||||
'[' => brackets += 1,
|
||||
']' => brackets -= 1,
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if parentheses == 0
|
||||
&& brackets == 0
|
||||
&& expression[index..].starts_with(operator)
|
||||
{
|
||||
parts.push(expression[start..index].trim());
|
||||
index += operator.len();
|
||||
start = index;
|
||||
continue;
|
||||
}
|
||||
index += width;
|
||||
}
|
||||
|
||||
if parts.is_empty() {
|
||||
vec![expression]
|
||||
} else {
|
||||
parts.push(expression[start..].trim());
|
||||
parts
|
||||
}
|
||||
}
|
||||
|
||||
/// Translates a Meilisearch filter expression into Typesense `filter_by`
|
||||
/// syntax.
|
||||
///
|
||||
/// Transformations (applied in order):
|
||||
/// 1. `field (NOT )IN [v1, v2]` → `field:[v1, v2]` / `field:!=[v1, v2]`
|
||||
/// 2. `field op value` for op ∈ {`!=`, `>=`, `<=`, `>`, `<`, `=`}
|
||||
/// → `field:op value`
|
||||
/// 3. `AND` / `OR` (case-insensitive) → `&&` / `||`
|
||||
pub(super) fn meili_to_typesense(filter: &str) -> String {
|
||||
static IN_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?IN\s*\[([^\]]*)\]",
|
||||
)
|
||||
.expect("valid regex")
|
||||
});
|
||||
static EXISTS_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?EXISTS\b")
|
||||
.expect("valid regex")
|
||||
});
|
||||
static CMP_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"([a-zA-Z_.][a-zA-Z0-9_.]*)\s*(!=|>=|<=|>|<|=)\s*")
|
||||
.expect("valid regex")
|
||||
});
|
||||
static AND_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bAND\b").expect("valid regex"));
|
||||
static OR_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bOR\b").expect("valid regex"));
|
||||
|
||||
// Step 1 – IN / NOT IN
|
||||
let s = IN_RE.replace_all(filter, |caps: ®ex::Captures<'_>| {
|
||||
let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
||||
let is_not = caps.get(2).is_some();
|
||||
let values = caps.get(3).map(|m| m.as_str()).unwrap_or_default();
|
||||
if is_not {
|
||||
format!("{field}:!=[{values}]")
|
||||
} else {
|
||||
format!("{field}:[{values}]")
|
||||
}
|
||||
});
|
||||
|
||||
let s = EXISTS_RE.replace_all(&s, |caps: ®ex::Captures<'_>| {
|
||||
let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
||||
let is_not = caps.get(2).is_some();
|
||||
|
||||
match field {
|
||||
"minecraft_java_server.ping.data" => format!(
|
||||
"minecraft_java_server.is_online:= {}",
|
||||
if is_not { "false" } else { "true" }
|
||||
),
|
||||
_ => caps
|
||||
.get(0)
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
}
|
||||
});
|
||||
|
||||
// Step 2 – comparison operators (field op value → field:op value).
|
||||
let s = CMP_RE.replace_all(&s, |caps: ®ex::Captures<'_>| {
|
||||
let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
||||
let op = caps.get(2).map(|m| m.as_str()).unwrap_or_default();
|
||||
format!("{field}:{op} ")
|
||||
});
|
||||
|
||||
// Step 3 – logical operators
|
||||
let s = AND_RE.replace_all(&s, " && ");
|
||||
let s = OR_RE.replace_all(&s, " || ");
|
||||
s.into_owned()
|
||||
}
|
||||
|
||||
/// Converts the legacy Meilisearch `facets` JSON array into a Typesense
|
||||
/// `filter_by` string. The outer array items are AND-ed together; the inner
|
||||
/// array items are OR-ed together.
|
||||
pub(super) fn facets_to_typesense(facets_json: &str) -> Result<String> {
|
||||
let facets = serde_json::from_str::<Vec<Vec<Value>>>(facets_json)
|
||||
.wrap_err("failed to parse facets JSON")?;
|
||||
|
||||
let and_parts: Vec<String> = facets
|
||||
.into_iter()
|
||||
.map(|or_group| {
|
||||
let or_parts: Vec<String> = or_group
|
||||
.into_iter()
|
||||
.map(|facet| {
|
||||
let conditions: Vec<String> = if facet.is_array() {
|
||||
serde_json::from_value::<Vec<String>>(facet)
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
vec![
|
||||
serde_json::from_value::<String>(facet)
|
||||
.unwrap_or_default(),
|
||||
]
|
||||
};
|
||||
let and_conds: Vec<String> = conditions
|
||||
.into_iter()
|
||||
.map(|condition| {
|
||||
condition_to_typesense_filter(&condition)
|
||||
})
|
||||
.collect();
|
||||
if and_conds.len() == 1 {
|
||||
and_conds.into_iter().next().unwrap_or_default()
|
||||
} else {
|
||||
format!("({})", and_conds.join(" && "))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
if or_parts.len() == 1 {
|
||||
or_parts.into_iter().next().unwrap_or_default()
|
||||
} else {
|
||||
format!("({})", or_parts.join(" || "))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(and_parts.join(" && "))
|
||||
}
|
||||
|
||||
/// Converts a single facet condition such as `"categories:mods"`,
|
||||
/// `"categories=mods"`, or `"downloads!=100"` into a Typesense filter clause.
|
||||
fn condition_to_typesense_filter(condition: &str) -> String {
|
||||
// Match multi-character operators before their single-character prefixes,
|
||||
// and range/inequality operators before the plain `=` equality arm.
|
||||
for operator in ["!=", ">=", "<=", ">", "<"] {
|
||||
if let Some((field, value)) = condition.split_once(operator) {
|
||||
return format!("{}:{} {}", field.trim(), operator, value.trim());
|
||||
}
|
||||
}
|
||||
if let Some((field, value)) = condition.split_once(':') {
|
||||
return format!("{}:= {}", field.trim(), value.trim());
|
||||
}
|
||||
if let Some((field, value)) = condition.split_once('=') {
|
||||
return format!("{}:= {}", field.trim(), value.trim());
|
||||
}
|
||||
condition.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::rewrite_filter_for_join;
|
||||
|
||||
#[test]
|
||||
fn project_filters_do_not_join_versions() {
|
||||
assert_eq!(
|
||||
rewrite_filter_for_join("license:= MIT", "versions").unwrap(),
|
||||
"license:= MIT"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correlated_version_filters_share_one_join() {
|
||||
assert_eq!(
|
||||
rewrite_filter_for_join(
|
||||
"categories:= fabric && game_versions:= 1.21",
|
||||
"versions",
|
||||
)
|
||||
.unwrap(),
|
||||
"$versions(categories: fabric && game_versions: 1.21)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn project_and_version_filters_are_partitioned() {
|
||||
assert_eq!(
|
||||
rewrite_filter_for_join(
|
||||
"license:= MIT && categories:= fabric",
|
||||
"versions",
|
||||
)
|
||||
.unwrap(),
|
||||
"(license:= MIT && $versions(categories: fabric))"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_boolean_filters_preserve_version_correlation() {
|
||||
assert_eq!(
|
||||
rewrite_filter_for_join(
|
||||
"(license:= MIT || categories:= fabric) && game_versions:= 1.21",
|
||||
"versions",
|
||||
)
|
||||
.unwrap(),
|
||||
"(license:= MIT && $versions(game_versions: 1.21)) || $versions(categories: fabric && game_versions: 1.21)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_categories_use_inherited_version_categories() {
|
||||
assert_eq!(
|
||||
rewrite_filter_for_join("categories:!= fabric", "versions")
|
||||
.unwrap(),
|
||||
"$versions(categories:!= fabric)"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -103,6 +103,12 @@
|
||||
//! is faster than two. It's even faster if you search for more facets like
|
||||
//! category, game version, loader, and environment.
|
||||
//!
|
||||
//! Filters are parsed into a backend-independent AST before they reach this
|
||||
//! module. Its normalization pass compacts Cartesian products such as many
|
||||
//! game versions combined with many loaders into `IN` predicates. The
|
||||
//! Typesense filter planner then puts maximal version-only expressions into a
|
||||
//! single join, avoiding repeated joins and deeply expanded boolean trees.
|
||||
//!
|
||||
//! ### Whole-field tokenization
|
||||
//!
|
||||
//! For fields like a version's `categories`, `environment`, `game_versions`
|
||||
@@ -117,6 +123,12 @@
|
||||
//!
|
||||
//! For this, we use the `:` operator instead of `:=` to tell
|
||||
//! Typesense to treat this as an exact token match.
|
||||
//!
|
||||
//! ### Query caching
|
||||
//!
|
||||
//! Typesense caches identical search responses when `TYPESENSE_USE_CACHE` is
|
||||
//! enabled. This avoids repeating joins for popular queries at the cost of
|
||||
//! results remaining stale for Typesense's default 60-second cache lifetime.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
@@ -136,6 +148,9 @@ use crate::search::backend::{
|
||||
SearchIndex, combined_search_filters, parse_search_index,
|
||||
parse_search_request,
|
||||
};
|
||||
use crate::search::filter::{
|
||||
FilterExpr, from_legacy_v2_facets_json, normalize, parse_expression,
|
||||
};
|
||||
use crate::search::indexing::index_local;
|
||||
use crate::search::{
|
||||
ResultSearchProject, SearchBackend, SearchField, SearchIndexUpdate,
|
||||
@@ -144,11 +159,9 @@ use crate::search::{
|
||||
};
|
||||
use crate::util::error::Context;
|
||||
|
||||
use self::filter_rewrite::{
|
||||
facets_to_typesense, meili_to_typesense, rewrite_filter_for_join,
|
||||
};
|
||||
use self::filter::serialize_filter;
|
||||
|
||||
mod filter_rewrite;
|
||||
mod filter;
|
||||
|
||||
const DELETE_FILTER_ID_BATCH_SIZE: usize = 256;
|
||||
|
||||
@@ -161,6 +174,7 @@ pub struct TypesenseConfig {
|
||||
pub index_chunk_size: i64,
|
||||
pub import_batch_size: usize,
|
||||
pub delete_batch_size: usize,
|
||||
pub use_cache: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
@@ -291,6 +305,7 @@ impl TypesenseConfig {
|
||||
index_chunk_size: ENV.SEARCH_INDEX_CHUNK_SIZE,
|
||||
import_batch_size: ENV.TYPESENSE_IMPORT_BATCH_SIZE,
|
||||
delete_batch_size: ENV.TYPESENSE_DELETE_BATCH_SIZE,
|
||||
use_cache: ENV.TYPESENSE_USE_CACHE,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -905,38 +920,23 @@ impl Typesense {
|
||||
versions_collection: &str,
|
||||
) -> Result<Option<String>, ApiError> {
|
||||
let facet_part = if let Some(facets_json) = info.facets.as_deref() {
|
||||
Some(
|
||||
facets_to_typesense(facets_json)
|
||||
.wrap_request_err("failed to parse facets")?,
|
||||
)
|
||||
from_legacy_v2_facets_json(facets_json)
|
||||
.wrap_request_err("failed to parse facets")?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let new_filters_part =
|
||||
info.new_filters.as_deref().map(meili_to_typesense);
|
||||
let filter_part = combined_search_filters(info)
|
||||
.filter(|filter| !filter.trim().is_empty())
|
||||
.map(|filter| parse_expression(&filter))
|
||||
.transpose()
|
||||
.wrap_request_err("failed to parse filters")?;
|
||||
|
||||
let legacy_part = if info.new_filters.is_none() {
|
||||
combined_search_filters(info).map(|f| meili_to_typesense(&f))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let filter_part = new_filters_part.or(legacy_part);
|
||||
|
||||
let filter = match (facet_part, filter_part) {
|
||||
(Some(f), Some(l)) if !f.is_empty() && !l.is_empty() => {
|
||||
Some(format!("({f}) && ({l})"))
|
||||
}
|
||||
(Some(f), _) if !f.is_empty() => Some(f),
|
||||
(_, Some(l)) if !l.is_empty() => Some(l),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
filter
|
||||
FilterExpr::and([facet_part, filter_part].into_iter().flatten())
|
||||
.map(normalize)
|
||||
.map(|filter| {
|
||||
rewrite_filter_for_join(&filter, versions_collection)
|
||||
.wrap_request_err("failed to rewrite search filter")
|
||||
serialize_filter(&filter, versions_collection)
|
||||
.wrap_request_err("failed to build search filter")
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
@@ -1114,6 +1114,7 @@ impl SearchBackend for Typesense {
|
||||
let resp = self
|
||||
.client
|
||||
.request(Method::POST, "/multi_search")
|
||||
.query(&[("use_cache", self.config.use_cache)])
|
||||
.json(&json!({
|
||||
"searches": [
|
||||
serde_json::Map::from_iter(
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub enum FilterExpr {
|
||||
And(Vec<Self>),
|
||||
Or(Vec<Self>),
|
||||
Predicate(FilterPredicate),
|
||||
}
|
||||
|
||||
impl FilterExpr {
|
||||
pub fn and(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
|
||||
let mut expressions = expressions.into_iter().collect::<Vec<_>>();
|
||||
match expressions.len() {
|
||||
0 => None,
|
||||
1 => expressions.pop(),
|
||||
_ => Some(Self::And(expressions)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn or(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
|
||||
let mut expressions = expressions.into_iter().collect::<Vec<_>>();
|
||||
match expressions.len() {
|
||||
0 => None,
|
||||
1 => expressions.pop(),
|
||||
_ => Some(Self::Or(expressions)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct FilterPredicate {
|
||||
pub field: FilterField,
|
||||
pub condition: FilterCondition,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct FilterField(String);
|
||||
|
||||
impl FilterField {
|
||||
pub fn new(field: impl Into<String>) -> Self {
|
||||
Self(field.into())
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub enum FilterCondition {
|
||||
Compare {
|
||||
comparison: FilterComparison,
|
||||
value: FilterLiteral,
|
||||
},
|
||||
In {
|
||||
values: Vec<FilterLiteral>,
|
||||
negated: bool,
|
||||
},
|
||||
Exists {
|
||||
negated: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub enum FilterComparison {
|
||||
Equal,
|
||||
NotEqual,
|
||||
GreaterThan,
|
||||
GreaterThanOrEqual,
|
||||
LessThan,
|
||||
LessThanOrEqual,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub enum FilterLiteral {
|
||||
String(String),
|
||||
Number(String),
|
||||
Bool(bool),
|
||||
}
|
||||
|
||||
impl FilterLiteral {
|
||||
pub(super) fn from_bare(value: String) -> Self {
|
||||
if value.eq_ignore_ascii_case("true") {
|
||||
Self::Bool(true)
|
||||
} else if value.eq_ignore_ascii_case("false") {
|
||||
Self::Bool(false)
|
||||
} else if value.parse::<f64>().is_ok() {
|
||||
Self::Number(value)
|
||||
} else {
|
||||
Self::String(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::{FilterExpr, FilterParseError, parse_expression};
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LegacyV2FacetsError {
|
||||
#[error("invalid facets JSON")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("facet condition must be a string")]
|
||||
InvalidCondition,
|
||||
#[error(transparent)]
|
||||
Filter(#[from] FilterParseError),
|
||||
}
|
||||
|
||||
pub fn from_legacy_v2_facets_json(
|
||||
input: &str,
|
||||
) -> Result<Option<FilterExpr>, LegacyV2FacetsError> {
|
||||
let facets = serde_json::from_str::<Vec<Vec<Value>>>(input)?;
|
||||
let mut groups = Vec::new();
|
||||
|
||||
for or_group in facets {
|
||||
let mut alternatives = Vec::new();
|
||||
for facet in or_group {
|
||||
let expression = match facet {
|
||||
Value::String(condition) => Some(parse_condition(&condition)?),
|
||||
Value::Array(conditions) => {
|
||||
let mut predicates = Vec::new();
|
||||
for condition in conditions {
|
||||
let condition = condition
|
||||
.as_str()
|
||||
.ok_or(LegacyV2FacetsError::InvalidCondition)?;
|
||||
predicates.push(parse_condition(condition)?);
|
||||
}
|
||||
FilterExpr::and(predicates)
|
||||
}
|
||||
_ => return Err(LegacyV2FacetsError::InvalidCondition),
|
||||
};
|
||||
if let Some(expression) = expression {
|
||||
alternatives.push(expression);
|
||||
}
|
||||
}
|
||||
if let Some(expression) = FilterExpr::or(alternatives) {
|
||||
groups.push(expression);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(FilterExpr::and(groups))
|
||||
}
|
||||
|
||||
fn parse_condition(condition: &str) -> Result<FilterExpr, LegacyV2FacetsError> {
|
||||
if ["!=", ">=", "<=", ">", "<", "="]
|
||||
.iter()
|
||||
.any(|operator| condition.contains(operator))
|
||||
{
|
||||
parse_expression(condition).map_err(Into::into)
|
||||
} else if let Some((field, value)) = condition.split_once(':') {
|
||||
parse_expression(&format!("{} = {}", field.trim(), value.trim()))
|
||||
.map_err(Into::into)
|
||||
} else {
|
||||
parse_expression(condition).map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::from_legacy_v2_facets_json;
|
||||
use crate::search::filter::FilterExpr;
|
||||
|
||||
#[test]
|
||||
fn converts_v2_boolean_structure() {
|
||||
let expression = from_legacy_v2_facets_json(
|
||||
r#"[["categories:fabric", "categories:forge"], [["game_versions:1.21", "project_types:mod"]]]"#,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let FilterExpr::And(groups) = expression else {
|
||||
panic!("expected outer facets to be joined with AND");
|
||||
};
|
||||
assert!(matches!(groups[0], FilterExpr::Or(_)));
|
||||
assert!(matches!(groups[1], FilterExpr::And(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_colons_inside_comparison_values() {
|
||||
from_legacy_v2_facets_json(
|
||||
r#"[["license='https://example.com/license'"]]"#,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
mod ast;
|
||||
mod legacy_v2;
|
||||
mod normalize;
|
||||
mod parse;
|
||||
|
||||
pub use ast::{
|
||||
FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
|
||||
FilterPredicate,
|
||||
};
|
||||
pub use legacy_v2::from_legacy_v2_facets_json;
|
||||
pub use normalize::normalize;
|
||||
pub use parse::{FilterParseError, parse_expression};
|
||||
@@ -0,0 +1,226 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use super::{
|
||||
FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
|
||||
FilterPredicate,
|
||||
};
|
||||
|
||||
pub fn normalize(expression: FilterExpr) -> FilterExpr {
|
||||
match expression {
|
||||
FilterExpr::And(expressions) => normalize_and(expressions),
|
||||
FilterExpr::Or(expressions) => normalize_or(expressions),
|
||||
FilterExpr::Predicate(predicate) => normalize_predicate(predicate),
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_predicate(predicate: FilterPredicate) -> FilterExpr {
|
||||
if predicate.field.as_str() == "minecraft_java_server.ping.data"
|
||||
&& let FilterCondition::Exists { negated } = predicate.condition
|
||||
{
|
||||
return FilterExpr::Predicate(FilterPredicate {
|
||||
field: FilterField::new("minecraft_java_server.is_online"),
|
||||
condition: FilterCondition::Compare {
|
||||
comparison: FilterComparison::Equal,
|
||||
value: FilterLiteral::Bool(!negated),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
FilterExpr::Predicate(predicate)
|
||||
}
|
||||
|
||||
fn normalize_and(expressions: Vec<FilterExpr>) -> FilterExpr {
|
||||
let mut normalized = expressions
|
||||
.into_iter()
|
||||
.map(normalize)
|
||||
.flat_map(|expression| match expression {
|
||||
FilterExpr::And(children) => children,
|
||||
expression => vec![expression],
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort();
|
||||
normalized.dedup();
|
||||
|
||||
FilterExpr::and(normalized).expect("an AND expression is non-empty")
|
||||
}
|
||||
|
||||
fn normalize_or(expressions: Vec<FilterExpr>) -> FilterExpr {
|
||||
let mut normalized = expressions
|
||||
.into_iter()
|
||||
.map(normalize)
|
||||
.flat_map(|expression| match expression {
|
||||
FilterExpr::Or(children) => children,
|
||||
expression => vec![expression],
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort();
|
||||
normalized.dedup();
|
||||
|
||||
if let Some(expression) = compact_cartesian_product(&normalized) {
|
||||
return expression;
|
||||
}
|
||||
|
||||
FilterExpr::or(normalized).expect("an OR expression is non-empty")
|
||||
}
|
||||
|
||||
fn compact_cartesian_product(expressions: &[FilterExpr]) -> Option<FilterExpr> {
|
||||
let clauses = expressions
|
||||
.iter()
|
||||
.map(predicate_clause)
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
if clauses.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let common = clauses
|
||||
.iter()
|
||||
.skip(1)
|
||||
.fold(clauses[0].clone(), |common, clause| {
|
||||
common.intersection(clause).cloned().collect()
|
||||
});
|
||||
let remaining = clauses
|
||||
.iter()
|
||||
.map(|clause| clause.difference(&common).cloned().collect::<Vec<_>>())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if remaining.iter().any(Vec::is_empty) {
|
||||
return FilterExpr::and(common.into_iter().map(FilterExpr::Predicate));
|
||||
}
|
||||
|
||||
let mut values_by_field =
|
||||
BTreeMap::<FilterField, BTreeSet<FilterLiteral>>::new();
|
||||
let expected_fields = remaining[0]
|
||||
.iter()
|
||||
.map(equality_parts)
|
||||
.collect::<Option<Vec<_>>>()?
|
||||
.into_iter()
|
||||
.map(|(field, _)| field.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
let mut unique_clauses = BTreeSet::new();
|
||||
for clause in &remaining {
|
||||
let parts = clause
|
||||
.iter()
|
||||
.map(equality_parts)
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
let fields = parts
|
||||
.iter()
|
||||
.map(|(field, _)| (*field).clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
if fields != expected_fields || fields.len() != parts.len() {
|
||||
return None;
|
||||
}
|
||||
for (field, value) in parts {
|
||||
values_by_field
|
||||
.entry(field.clone())
|
||||
.or_default()
|
||||
.insert(value.clone());
|
||||
}
|
||||
unique_clauses.insert(clause.clone());
|
||||
}
|
||||
|
||||
let combinations = values_by_field
|
||||
.values()
|
||||
.try_fold(1usize, |count, values| count.checked_mul(values.len()))?;
|
||||
if combinations != unique_clauses.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let compacted = values_by_field.into_iter().map(|(field, values)| {
|
||||
let values = values.into_iter().collect::<Vec<_>>();
|
||||
let condition = if values.len() == 1 {
|
||||
FilterCondition::Compare {
|
||||
comparison: FilterComparison::Equal,
|
||||
value: values.into_iter().next().expect("one value exists"),
|
||||
}
|
||||
} else {
|
||||
FilterCondition::In {
|
||||
values,
|
||||
negated: false,
|
||||
}
|
||||
};
|
||||
FilterPredicate { field, condition }
|
||||
});
|
||||
|
||||
FilterExpr::and(
|
||||
common
|
||||
.into_iter()
|
||||
.chain(compacted)
|
||||
.map(FilterExpr::Predicate),
|
||||
)
|
||||
}
|
||||
|
||||
fn predicate_clause(
|
||||
expression: &FilterExpr,
|
||||
) -> Option<BTreeSet<FilterPredicate>> {
|
||||
match expression {
|
||||
FilterExpr::Predicate(predicate) => {
|
||||
Some(BTreeSet::from([predicate.clone()]))
|
||||
}
|
||||
FilterExpr::And(expressions) => expressions
|
||||
.iter()
|
||||
.map(|expression| match expression {
|
||||
FilterExpr::Predicate(predicate) => Some(predicate.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
FilterExpr::Or(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn equality_parts(
|
||||
predicate: &FilterPredicate,
|
||||
) -> Option<(&FilterField, &FilterLiteral)> {
|
||||
match &predicate.condition {
|
||||
FilterCondition::Compare {
|
||||
comparison: FilterComparison::Equal,
|
||||
value,
|
||||
} => Some((&predicate.field, value)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize;
|
||||
use crate::search::filter::{
|
||||
FilterCondition, FilterExpr, FilterField, FilterLiteral,
|
||||
FilterPredicate, parse_expression,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn compacts_cartesian_product() {
|
||||
let expression = parse_expression(
|
||||
"(game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
|
||||
(game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
|
||||
(game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
|
||||
(game_versions = 1.21.1 AND categories = forge AND categories = technology)",
|
||||
)
|
||||
.unwrap();
|
||||
let FilterExpr::And(predicates) = normalize(expression) else {
|
||||
panic!("expected a compacted conjunction");
|
||||
};
|
||||
|
||||
assert_eq!(predicates.len(), 3);
|
||||
assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
|
||||
field: FilterField::new("categories"),
|
||||
condition: FilterCondition::In {
|
||||
values: vec![
|
||||
FilterLiteral::String("fabric".into()),
|
||||
FilterLiteral::String("forge".into()),
|
||||
],
|
||||
negated: false,
|
||||
},
|
||||
})));
|
||||
assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
|
||||
field: FilterField::new("game_versions"),
|
||||
condition: FilterCondition::In {
|
||||
values: vec![
|
||||
FilterLiteral::String("1.20.1".into()),
|
||||
FilterLiteral::String("1.21.1".into()),
|
||||
],
|
||||
negated: false,
|
||||
},
|
||||
})));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
use std::ops::Range;
|
||||
|
||||
use chumsky::{Parser, prelude::*};
|
||||
use thiserror::Error;
|
||||
|
||||
use super::{
|
||||
FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
|
||||
FilterPredicate,
|
||||
};
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
#[error("invalid filter at byte {position}: {message}")]
|
||||
pub struct FilterParseError {
|
||||
position: usize,
|
||||
message: String,
|
||||
}
|
||||
|
||||
fn keyword(
|
||||
keyword: &'static str,
|
||||
) -> BoxedParser<'static, char, (), Simple<char>> {
|
||||
keyword
|
||||
.chars()
|
||||
.fold(empty().ignored().boxed(), |parser, character| {
|
||||
parser
|
||||
.then_ignore(one_of([
|
||||
character.to_ascii_lowercase(),
|
||||
character.to_ascii_uppercase(),
|
||||
]))
|
||||
.boxed()
|
||||
})
|
||||
}
|
||||
|
||||
fn quoted_literal(
|
||||
quote: char,
|
||||
) -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
|
||||
let escaped = just('\\').ignore_then(any());
|
||||
let character = escaped.or(filter(move |character| {
|
||||
*character != quote && *character != '\\'
|
||||
}));
|
||||
|
||||
character
|
||||
.repeated()
|
||||
.collect::<String>()
|
||||
.delimited_by(just(quote), just(quote))
|
||||
.map(FilterLiteral::String)
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn literal_parser() -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
|
||||
let quoted = choice((
|
||||
quoted_literal('\''),
|
||||
quoted_literal('"'),
|
||||
quoted_literal('`'),
|
||||
));
|
||||
let bare = filter(|character: &char| {
|
||||
!character.is_whitespace() && !",[]()".contains(*character)
|
||||
})
|
||||
.repeated()
|
||||
.at_least(1)
|
||||
.collect::<String>()
|
||||
.map(FilterLiteral::from_bare);
|
||||
|
||||
quoted.or(bare).padded().boxed()
|
||||
}
|
||||
|
||||
fn parser() -> impl Parser<char, FilterExpr, Error = Simple<char>> {
|
||||
let field = filter(|character: &char| {
|
||||
character.is_ascii_alphabetic() || "_.".contains(*character)
|
||||
})
|
||||
.then(
|
||||
filter(|character: &char| {
|
||||
character.is_ascii_alphanumeric() || "_.".contains(*character)
|
||||
})
|
||||
.repeated(),
|
||||
)
|
||||
.map(|(first, rest)| {
|
||||
FilterField::new(std::iter::once(first).chain(rest).collect::<String>())
|
||||
})
|
||||
.padded();
|
||||
|
||||
let literal = literal_parser();
|
||||
let list = literal
|
||||
.clone()
|
||||
.separated_by(just(',').padded())
|
||||
.at_least(1)
|
||||
.allow_trailing()
|
||||
.delimited_by(just('[').padded(), just(']').padded());
|
||||
|
||||
let comparison = choice((
|
||||
just("!=").to(FilterComparison::NotEqual),
|
||||
just(">=").to(FilterComparison::GreaterThanOrEqual),
|
||||
just("<=").to(FilterComparison::LessThanOrEqual),
|
||||
just('>').to(FilterComparison::GreaterThan),
|
||||
just('<').to(FilterComparison::LessThan),
|
||||
just('=').to(FilterComparison::Equal),
|
||||
))
|
||||
.padded()
|
||||
.then(literal.clone())
|
||||
.map(|(comparison, value)| FilterCondition::Compare { comparison, value });
|
||||
|
||||
let not_in = keyword("NOT")
|
||||
.padded()
|
||||
.ignore_then(keyword("IN"))
|
||||
.padded()
|
||||
.ignore_then(list.clone())
|
||||
.map(|values| FilterCondition::In {
|
||||
values,
|
||||
negated: true,
|
||||
});
|
||||
let in_list = keyword("IN").padded().ignore_then(list).map(|values| {
|
||||
FilterCondition::In {
|
||||
values,
|
||||
negated: false,
|
||||
}
|
||||
});
|
||||
let not_exists = keyword("NOT")
|
||||
.padded()
|
||||
.ignore_then(keyword("EXISTS"))
|
||||
.map(|()| FilterCondition::Exists { negated: true });
|
||||
let exists =
|
||||
keyword("EXISTS").map(|()| FilterCondition::Exists { negated: false });
|
||||
|
||||
let predicate = field
|
||||
.then(choice((not_in, in_list, not_exists, exists, comparison)))
|
||||
.map(|(field, condition)| {
|
||||
FilterExpr::Predicate(FilterPredicate { field, condition })
|
||||
});
|
||||
|
||||
recursive(|expression| {
|
||||
let atom = predicate
|
||||
.clone()
|
||||
.or(expression.delimited_by(just('(').padded(), just(')').padded()))
|
||||
.padded();
|
||||
let and = atom
|
||||
.clone()
|
||||
.then(keyword("AND").padded().ignore_then(atom).repeated())
|
||||
.map(|(first, rest)| {
|
||||
FilterExpr::and(std::iter::once(first).chain(rest))
|
||||
.expect("an expression always contains one operand")
|
||||
});
|
||||
|
||||
and.clone()
|
||||
.then(keyword("OR").padded().ignore_then(and).repeated())
|
||||
.map(|(first, rest)| {
|
||||
FilterExpr::or(std::iter::once(first).chain(rest))
|
||||
.expect("an expression always contains one operand")
|
||||
})
|
||||
})
|
||||
.padded()
|
||||
.then_ignore(end())
|
||||
}
|
||||
|
||||
pub fn parse_expression(input: &str) -> Result<FilterExpr, FilterParseError> {
|
||||
parser().parse(input).map_err(|errors| {
|
||||
let error = errors
|
||||
.into_iter()
|
||||
.next()
|
||||
.unwrap_or_else(|| Simple::custom(0..0, "invalid filter"));
|
||||
let Range { start, .. } = error.span();
|
||||
FilterParseError {
|
||||
position: start,
|
||||
message: error.to_string(),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_expression;
|
||||
use crate::search::filter::{FilterCondition, FilterExpr, FilterLiteral};
|
||||
|
||||
#[test]
|
||||
fn parses_boolean_precedence() {
|
||||
let expression = parse_expression(
|
||||
"license = MIT OR downloads >= 100 AND open_source = true",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let FilterExpr::Or(expressions) = expression else {
|
||||
panic!("expected an OR expression");
|
||||
};
|
||||
assert_eq!(expressions.len(), 2);
|
||||
assert!(matches!(expressions[1], FilterExpr::And(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_quoted_and_list_values() {
|
||||
let expression = parse_expression(
|
||||
r#"name = "value with spaces" AND categories IN [fabric, "forge"]"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let FilterExpr::And(expressions) = expression else {
|
||||
panic!("expected an AND expression");
|
||||
};
|
||||
let FilterExpr::Predicate(predicate) = &expressions[0] else {
|
||||
panic!("expected a predicate");
|
||||
};
|
||||
assert!(matches!(
|
||||
&predicate.condition,
|
||||
FilterCondition::Compare {
|
||||
value: FilterLiteral::String(value),
|
||||
..
|
||||
} if value == "value with spaces"
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ use thiserror::Error;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
pub mod backend;
|
||||
pub mod filter;
|
||||
pub mod incremental;
|
||||
pub mod indexing;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user