wip: search filter AST

This commit is contained in:
aecsocket
2026-07-22 14:53:46 +01:00
parent f482e11401
commit 00b9bafe9d
11 changed files with 1117 additions and 455 deletions
+1
View File
@@ -35,6 +35,7 @@ bitflags = { workspace = true }
bytes = { workspace = true }
censor = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
chumsky = { workspace = true }
clap = { workspace = true, features = ["derive"] }
clickhouse = { workspace = true, features = ["time", "uuid"] }
color-eyre = { workspace = true }
+1
View File
@@ -167,6 +167,7 @@ vars! {
TYPESENSE_INDEX_PREFIX: String = "labrinth";
TYPESENSE_IMPORT_BATCH_SIZE: usize = 5000usize;
TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize;
TYPESENSE_USE_CACHE: bool = true;
// storage
STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;
@@ -0,0 +1,454 @@
use std::fmt::{self, Display, Formatter};
use eyre::{Result, eyre};
use crate::search::SearchField;
use crate::search::filter::{
FilterComparison, FilterCondition, FilterExpr, FilterLiteral,
FilterPredicate,
};
const MAX_DNF_CLAUSES: usize = 64;
const MAX_FILTER_DEPTH: usize = 64;
const MAX_FILTER_NODES: usize = 1024;
const MAX_SERIALIZED_FILTER_BYTES: usize = 64 * 1024;
#[derive(Clone, Copy, PartialEq, Eq)]
enum FilterScope {
Project,
Version,
Mixed,
}
enum TypesenseFilter<'a> {
And(Vec<Self>),
Or(Vec<Self>),
Predicate {
predicate: &'a FilterPredicate,
version: bool,
},
Join {
collection: &'a str,
filter: Box<Self>,
},
}
pub(super) fn serialize_filter(
filter: &FilterExpr,
versions_collection: &str,
) -> Result<String> {
let (nodes, depth) = filter_complexity(filter);
if nodes > MAX_FILTER_NODES {
return Err(eyre!("search filter has too many expressions"));
}
if depth > MAX_FILTER_DEPTH {
return Err(eyre!("search filter is nested too deeply"));
}
let filter = plan(filter, versions_collection)?;
let serialized = filter.to_string();
if serialized.len() > MAX_SERIALIZED_FILTER_BYTES {
return Err(eyre!("search filter is too large"));
}
Ok(serialized)
}
fn plan<'a>(
filter: &'a FilterExpr,
versions_collection: &'a str,
) -> Result<TypesenseFilter<'a>> {
match filter_scope(filter) {
FilterScope::Project => lower(filter, false),
FilterScope::Version => Ok(TypesenseFilter::Join {
collection: versions_collection,
filter: Box::new(lower(filter, true)?),
}),
FilterScope::Mixed => plan_mixed(filter, versions_collection),
}
}
fn plan_mixed<'a>(
filter: &'a FilterExpr,
versions_collection: &'a str,
) -> Result<TypesenseFilter<'a>> {
match filter {
FilterExpr::Or(expressions) => expressions
.iter()
.map(|expression| plan(expression, versions_collection))
.collect::<Result<Vec<_>>>()
.map(TypesenseFilter::Or),
FilterExpr::And(expressions)
if expressions.iter().all(|expression| {
filter_scope(expression) != FilterScope::Mixed
}) =>
{
plan_partitioned_and(expressions, versions_collection)
}
_ => {
let clauses = to_dnf(filter)?;
clauses
.into_iter()
.map(|clause| plan_clause(clause, versions_collection))
.collect::<Result<Vec<_>>>()
.map(TypesenseFilter::Or)
}
}
}
fn plan_partitioned_and<'a>(
expressions: &'a [FilterExpr],
versions_collection: &'a str,
) -> Result<TypesenseFilter<'a>> {
let mut project = Vec::new();
let mut version = Vec::new();
for expression in expressions {
match filter_scope(expression) {
FilterScope::Project => project.push(lower(expression, false)?),
FilterScope::Version => version.push(lower(expression, true)?),
FilterScope::Mixed => {
return Err(eyre!("could not partition mixed search filter"));
}
}
}
if let Some(filter) = and_filter(version) {
project.push(TypesenseFilter::Join {
collection: versions_collection,
filter: Box::new(filter),
});
}
and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
}
fn plan_clause<'a>(
predicates: Vec<&'a FilterPredicate>,
versions_collection: &'a str,
) -> Result<TypesenseFilter<'a>> {
let mut project = Vec::new();
let mut version = Vec::new();
for predicate in predicates {
let planned = lower_predicate(predicate)?;
if is_version_filter_field(predicate.field.as_str()) {
version.push(planned);
} else {
project.push(planned);
}
}
if let Some(filter) = and_filter(version) {
project.push(TypesenseFilter::Join {
collection: versions_collection,
filter: Box::new(filter),
});
}
and_filter(project).ok_or_else(|| eyre!("search filter is empty"))
}
fn lower(filter: &FilterExpr, version: bool) -> Result<TypesenseFilter<'_>> {
match filter {
FilterExpr::And(expressions) => expressions
.iter()
.map(|expression| lower(expression, version))
.collect::<Result<Vec<_>>>()
.map(TypesenseFilter::And),
FilterExpr::Or(expressions) => expressions
.iter()
.map(|expression| lower(expression, version))
.collect::<Result<Vec<_>>>()
.map(TypesenseFilter::Or),
FilterExpr::Predicate(predicate) => {
validate_predicate(predicate)?;
Ok(TypesenseFilter::Predicate { predicate, version })
}
}
}
fn lower_predicate(predicate: &FilterPredicate) -> Result<TypesenseFilter<'_>> {
validate_predicate(predicate)?;
Ok(TypesenseFilter::Predicate {
predicate,
version: is_version_filter_field(predicate.field.as_str()),
})
}
fn validate_predicate(predicate: &FilterPredicate) -> Result<()> {
if matches!(predicate.condition, FilterCondition::Exists { .. }) {
return Err(eyre!(
"filter field `{}` does not support `EXISTS`",
predicate.field.as_str()
));
}
Ok(())
}
fn filter_scope(filter: &FilterExpr) -> FilterScope {
match filter {
FilterExpr::Predicate(predicate) => {
if is_version_filter_field(predicate.field.as_str()) {
FilterScope::Version
} else {
FilterScope::Project
}
}
FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
let mut scopes = expressions.iter().map(filter_scope);
let Some(first) = scopes.next() else {
return FilterScope::Project;
};
if scopes.all(|scope| scope == first) {
first
} else {
FilterScope::Mixed
}
}
}
}
fn is_version_filter_field(field: &str) -> bool {
<SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
search_field.is_version_field()
&& search_field.typesense_spec().path == field
})
}
fn to_dnf(filter: &FilterExpr) -> Result<Vec<Vec<&FilterPredicate>>> {
match filter {
FilterExpr::Predicate(predicate) => Ok(vec![vec![predicate]]),
FilterExpr::Or(expressions) => {
let mut clauses = Vec::new();
for expression in expressions {
clauses.extend(to_dnf(expression)?);
if clauses.len() > MAX_DNF_CLAUSES {
return Err(eyre!(
"search filter has too many boolean clauses"
));
}
}
Ok(clauses)
}
FilterExpr::And(expressions) => {
let mut clauses = vec![Vec::new()];
for expression in expressions {
let right = to_dnf(expression)?;
if clauses.len().saturating_mul(right.len()) > MAX_DNF_CLAUSES {
return Err(eyre!(
"search filter has too many boolean clauses"
));
}
clauses = clauses
.into_iter()
.flat_map(|left| {
right.iter().map(move |right| {
let mut clause = left.clone();
clause.extend(right);
clause
})
})
.collect();
}
Ok(clauses)
}
}
}
fn and_filter(
mut expressions: Vec<TypesenseFilter<'_>>,
) -> Option<TypesenseFilter<'_>> {
match expressions.len() {
0 => None,
1 => expressions.pop(),
_ => Some(TypesenseFilter::And(expressions)),
}
}
fn filter_complexity(filter: &FilterExpr) -> (usize, usize) {
match filter {
FilterExpr::Predicate(_) => (1, 1),
FilterExpr::And(expressions) | FilterExpr::Or(expressions) => {
expressions.iter().map(filter_complexity).fold(
(1, 1),
|(nodes, depth), (child_nodes, child_depth)| {
(nodes + child_nodes, depth.max(child_depth + 1))
},
)
}
}
}
impl Display for TypesenseFilter<'_> {
fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
self.fmt_with_precedence(formatter, 0)
}
}
impl TypesenseFilter<'_> {
fn precedence(&self) -> u8 {
match self {
Self::Or(_) => 1,
Self::And(_) => 2,
Self::Predicate { .. } | Self::Join { .. } => 3,
}
}
fn fmt_with_precedence(
&self,
formatter: &mut Formatter<'_>,
parent_precedence: u8,
) -> fmt::Result {
let precedence = self.precedence();
let parenthesize = precedence < parent_precedence;
if parenthesize {
formatter.write_str("(")?;
}
match self {
Self::And(expressions) => {
format_expressions(formatter, expressions, " && ", precedence)?;
}
Self::Or(expressions) => {
format_expressions(formatter, expressions, " || ", precedence)?;
}
Self::Predicate { predicate, version } => {
format_predicate(formatter, predicate, *version)?;
}
Self::Join { collection, filter } => {
write!(formatter, "${collection}(")?;
filter.fmt_with_precedence(formatter, 0)?;
formatter.write_str(")")?;
}
}
if parenthesize {
formatter.write_str(")")?;
}
Ok(())
}
}
fn format_expressions(
formatter: &mut Formatter<'_>,
expressions: &[TypesenseFilter<'_>],
separator: &str,
precedence: u8,
) -> fmt::Result {
for (index, expression) in expressions.iter().enumerate() {
if index != 0 {
formatter.write_str(separator)?;
}
expression.fmt_with_precedence(formatter, precedence)?;
}
Ok(())
}
fn format_predicate(
formatter: &mut Formatter<'_>,
predicate: &FilterPredicate,
version: bool,
) -> fmt::Result {
formatter.write_str(predicate.field.as_str())?;
match &predicate.condition {
FilterCondition::Compare { comparison, value } => {
let operator = match comparison {
FilterComparison::Equal if version => ":",
FilterComparison::Equal => ":=",
FilterComparison::NotEqual => ":!=",
FilterComparison::GreaterThan => ":>",
FilterComparison::GreaterThanOrEqual => ":>=",
FilterComparison::LessThan => ":<",
FilterComparison::LessThanOrEqual => ":<=",
};
formatter.write_str(operator)?;
format_literal(formatter, value)
}
FilterCondition::In { values, negated } => {
formatter.write_str(if *negated { ":!=" } else { ":" })?;
formatter.write_str("[")?;
for (index, value) in values.iter().enumerate() {
if index != 0 {
formatter.write_str(",")?;
}
format_literal(formatter, value)?;
}
formatter.write_str("]")
}
FilterCondition::Exists { .. } => unreachable!(
"unsupported predicates are rejected before serialization"
),
}
}
fn format_literal(
formatter: &mut Formatter<'_>,
literal: &FilterLiteral,
) -> fmt::Result {
match literal {
FilterLiteral::String(value) => {
formatter.write_str("`")?;
for character in value.chars() {
if character == '`' {
formatter.write_str("\\")?;
}
write!(formatter, "{character}")?;
}
formatter.write_str("`")
}
FilterLiteral::Number(value) => formatter.write_str(value),
FilterLiteral::Bool(value) => Display::fmt(value, formatter),
}
}
#[cfg(test)]
mod tests {
use super::serialize_filter;
use crate::search::filter::{normalize, parse_expression};
fn serialize(input: &str) -> String {
let filter = normalize(parse_expression(input).unwrap());
serialize_filter(&filter, "versions").unwrap()
}
#[test]
fn project_filters_do_not_join_versions() {
assert_eq!(serialize("license = MIT"), "license:=`MIT`");
}
#[test]
fn correlated_version_filters_share_one_join() {
assert_eq!(
serialize("categories = fabric AND game_versions = 1.21"),
"$versions(categories:`fabric` && game_versions:1.21)"
);
}
#[test]
fn mixed_boolean_filters_preserve_version_correlation() {
let filter = serialize(
"(license = MIT OR categories = fabric) AND game_versions = 1.21",
);
assert_eq!(filter.matches("$versions(").count(), 2);
assert!(filter.contains("categories:`fabric` && game_versions:1.21"));
assert!(filter.contains("license:=`MIT`"));
}
#[test]
fn string_values_are_escaped() {
assert_eq!(
serialize(r#"license = "value, with (syntax) and `tick`""#),
r#"license:=`value, with (syntax) and \`tick\``"#,
);
}
#[test]
fn cartesian_version_filter_uses_one_join() {
let filter = serialize(
"(project_types = modpack AND game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
(project_types = modpack AND game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
(project_types = modpack AND game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
(project_types = modpack AND game_versions = 1.21.1 AND categories = forge AND categories = technology)",
);
assert_eq!(filter.matches("$versions(").count(), 1);
assert!(filter.contains("categories:[`fabric`,`forge`]"));
assert!(filter.contains("game_versions:[`1.20.1`,`1.21.1`]"));
assert!(filter.contains("categories:`technology`"));
}
}
@@ -1,425 +0,0 @@
use std::sync::LazyLock;
use eyre::{Result, WrapErr, eyre};
use itertools::Itertools;
use regex::Regex;
use serde_json::Value;
use crate::search::SearchField;
#[derive(Clone, Default)]
struct JoinedFilterClause {
project: Vec<String>,
version: Vec<String>,
}
pub(super) fn rewrite_filter_for_join(
filter: &str,
versions_collection: &str,
) -> Result<String> {
const MAX_CLAUSES: usize = 256;
fn parse(expression: &str) -> Result<Vec<JoinedFilterClause>> {
let expression = trim_outer_parentheses(expression.trim());
let or_parts = split_top_level(expression, "||");
if or_parts.len() > 1 {
let mut clauses = Vec::new();
for part in or_parts {
clauses.extend(parse(part)?);
if clauses.len() > MAX_CLAUSES {
return Err(eyre!(
"search filter has too many boolean clauses"
));
}
}
return Ok(clauses);
}
let and_parts = split_top_level(expression, "&&");
if and_parts.len() > 1 {
let mut clauses = vec![JoinedFilterClause::default()];
for part in and_parts {
let right = parse(part)?;
if clauses.len().saturating_mul(right.len()) > MAX_CLAUSES {
return Err(eyre!(
"search filter has too many boolean clauses"
));
}
clauses = clauses
.into_iter()
.cartesian_product(right)
.map(|(mut left, right)| {
left.project.extend(right.project);
left.version.extend(right.version);
left
})
.collect();
}
return Ok(clauses);
}
let field = filter_field(expression).ok_or_else(|| {
eyre!("could not determine filter field in `{expression}`")
})?;
let mut clause = JoinedFilterClause::default();
if is_version_filter_field(field) {
clause.version.push(version_filter_expression(expression));
} else {
clause.project.push(expression.to_string());
}
Ok(vec![clause])
}
let clauses = parse(filter)?;
Ok(clauses
.into_iter()
.map(|clause| {
let mut parts = clause.project;
if !clause.version.is_empty() {
parts.push(format!(
"${versions_collection}({})",
clause.version.join(" && ")
));
}
if parts.len() == 1 {
parts.pop().unwrap_or_default()
} else {
format!("({})", parts.join(" && "))
}
})
.join(" || "))
}
fn is_version_filter_field(field: &str) -> bool {
<SearchField as strum::IntoEnumIterator>::iter().any(|search_field| {
search_field.is_version_field()
&& search_field.typesense_spec().path == field
})
}
fn version_filter_expression(expression: &str) -> String {
let Some((field, value)) = expression.split_once(':') else {
return expression.to_string();
};
let Some(value) = value.strip_prefix('=') else {
return expression.to_string();
};
format!("{field}:{value}")
}
fn filter_field(expression: &str) -> Option<&str> {
let operator = expression.find(':')?;
let field = expression[..operator].trim();
(!field.is_empty()
&& field.chars().all(|character| {
character.is_ascii_alphanumeric() || "_.".contains(character)
}))
.then_some(field)
}
fn trim_outer_parentheses(mut expression: &str) -> &str {
while expression.starts_with('(')
&& expression.ends_with(')')
&& matching_outer_parentheses(expression)
{
expression = expression[1..expression.len() - 1].trim();
}
expression
}
fn matching_outer_parentheses(expression: &str) -> bool {
let mut depth = 0;
let mut quote = None;
let mut escaped = false;
for (index, character) in expression.char_indices() {
if escaped {
escaped = false;
continue;
}
if character == '\\' {
escaped = true;
continue;
}
if let Some(active_quote) = quote {
if character == active_quote {
quote = None;
}
continue;
}
if matches!(character, '\'' | '"' | '`') {
quote = Some(character);
continue;
}
match character {
'(' => depth += 1,
')' => {
depth -= 1;
if depth == 0 && index + character.len_utf8() < expression.len()
{
return false;
}
}
_ => {}
}
}
depth == 0
}
fn split_top_level<'a>(expression: &'a str, operator: &str) -> Vec<&'a str> {
let mut parts = Vec::new();
let mut start = 0;
let mut parentheses = 0;
let mut brackets = 0;
let mut quote = None;
let mut escaped = false;
let bytes = expression.as_bytes();
let mut index = 0;
while index < bytes.len() {
let character = expression[index..].chars().next().unwrap_or_default();
let width = character.len_utf8();
if escaped {
escaped = false;
index += width;
continue;
}
if character == '\\' {
escaped = true;
index += width;
continue;
}
if let Some(active_quote) = quote {
if character == active_quote {
quote = None;
}
index += width;
continue;
}
if matches!(character, '\'' | '"' | '`') {
quote = Some(character);
index += width;
continue;
}
match character {
'(' => parentheses += 1,
')' => parentheses -= 1,
'[' => brackets += 1,
']' => brackets -= 1,
_ => {}
}
if parentheses == 0
&& brackets == 0
&& expression[index..].starts_with(operator)
{
parts.push(expression[start..index].trim());
index += operator.len();
start = index;
continue;
}
index += width;
}
if parts.is_empty() {
vec![expression]
} else {
parts.push(expression[start..].trim());
parts
}
}
/// Translates a Meilisearch filter expression into Typesense `filter_by`
/// syntax.
///
/// Transformations (applied in order):
/// 1. `field (NOT )IN [v1, v2]` → `field:[v1, v2]` / `field:!=[v1, v2]`
/// 2. `field op value` for op ∈ {`!=`, `>=`, `<=`, `>`, `<`, `=`}
/// → `field:op value`
/// 3. `AND` / `OR` (case-insensitive) → `&&` / `||`
pub(super) fn meili_to_typesense(filter: &str) -> String {
static IN_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(
r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?IN\s*\[([^\]]*)\]",
)
.expect("valid regex")
});
static EXISTS_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?EXISTS\b")
.expect("valid regex")
});
static CMP_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"([a-zA-Z_.][a-zA-Z0-9_.]*)\s*(!=|>=|<=|>|<|=)\s*")
.expect("valid regex")
});
static AND_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\bAND\b").expect("valid regex"));
static OR_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\bOR\b").expect("valid regex"));
// Step 1 IN / NOT IN
let s = IN_RE.replace_all(filter, |caps: &regex::Captures<'_>| {
let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
let is_not = caps.get(2).is_some();
let values = caps.get(3).map(|m| m.as_str()).unwrap_or_default();
if is_not {
format!("{field}:!=[{values}]")
} else {
format!("{field}:[{values}]")
}
});
let s = EXISTS_RE.replace_all(&s, |caps: &regex::Captures<'_>| {
let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
let is_not = caps.get(2).is_some();
match field {
"minecraft_java_server.ping.data" => format!(
"minecraft_java_server.is_online:= {}",
if is_not { "false" } else { "true" }
),
_ => caps
.get(0)
.map(|m| m.as_str())
.unwrap_or_default()
.to_string(),
}
});
// Step 2 comparison operators (field op value → field:op value).
let s = CMP_RE.replace_all(&s, |caps: &regex::Captures<'_>| {
let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
let op = caps.get(2).map(|m| m.as_str()).unwrap_or_default();
format!("{field}:{op} ")
});
// Step 3 logical operators
let s = AND_RE.replace_all(&s, " && ");
let s = OR_RE.replace_all(&s, " || ");
s.into_owned()
}
/// Converts the legacy Meilisearch `facets` JSON array into a Typesense
/// `filter_by` string. The outer array items are AND-ed together; the inner
/// array items are OR-ed together.
pub(super) fn facets_to_typesense(facets_json: &str) -> Result<String> {
let facets = serde_json::from_str::<Vec<Vec<Value>>>(facets_json)
.wrap_err("failed to parse facets JSON")?;
let and_parts: Vec<String> = facets
.into_iter()
.map(|or_group| {
let or_parts: Vec<String> = or_group
.into_iter()
.map(|facet| {
let conditions: Vec<String> = if facet.is_array() {
serde_json::from_value::<Vec<String>>(facet)
.unwrap_or_default()
} else {
vec![
serde_json::from_value::<String>(facet)
.unwrap_or_default(),
]
};
let and_conds: Vec<String> = conditions
.into_iter()
.map(|condition| {
condition_to_typesense_filter(&condition)
})
.collect();
if and_conds.len() == 1 {
and_conds.into_iter().next().unwrap_or_default()
} else {
format!("({})", and_conds.join(" && "))
}
})
.collect();
if or_parts.len() == 1 {
or_parts.into_iter().next().unwrap_or_default()
} else {
format!("({})", or_parts.join(" || "))
}
})
.collect();
Ok(and_parts.join(" && "))
}
/// Converts a single facet condition such as `"categories:mods"`,
/// `"categories=mods"`, or `"downloads!=100"` into a Typesense filter clause.
fn condition_to_typesense_filter(condition: &str) -> String {
// Match multi-character operators before their single-character prefixes,
// and range/inequality operators before the plain `=` equality arm.
for operator in ["!=", ">=", "<=", ">", "<"] {
if let Some((field, value)) = condition.split_once(operator) {
return format!("{}:{} {}", field.trim(), operator, value.trim());
}
}
if let Some((field, value)) = condition.split_once(':') {
return format!("{}:= {}", field.trim(), value.trim());
}
if let Some((field, value)) = condition.split_once('=') {
return format!("{}:= {}", field.trim(), value.trim());
}
condition.to_string()
}
#[cfg(test)]
mod tests {
use super::rewrite_filter_for_join;
#[test]
fn project_filters_do_not_join_versions() {
assert_eq!(
rewrite_filter_for_join("license:= MIT", "versions").unwrap(),
"license:= MIT"
);
}
#[test]
fn correlated_version_filters_share_one_join() {
assert_eq!(
rewrite_filter_for_join(
"categories:= fabric && game_versions:= 1.21",
"versions",
)
.unwrap(),
"$versions(categories: fabric && game_versions: 1.21)"
);
}
#[test]
fn project_and_version_filters_are_partitioned() {
assert_eq!(
rewrite_filter_for_join(
"license:= MIT && categories:= fabric",
"versions",
)
.unwrap(),
"(license:= MIT && $versions(categories: fabric))"
);
}
#[test]
fn mixed_boolean_filters_preserve_version_correlation() {
assert_eq!(
rewrite_filter_for_join(
"(license:= MIT || categories:= fabric) && game_versions:= 1.21",
"versions",
)
.unwrap(),
"(license:= MIT && $versions(game_versions: 1.21)) || $versions(categories: fabric && game_versions: 1.21)"
);
}
#[test]
fn negative_categories_use_inherited_version_categories() {
assert_eq!(
rewrite_filter_for_join("categories:!= fabric", "versions")
.unwrap(),
"$versions(categories:!= fabric)"
);
}
}
@@ -103,6 +103,12 @@
//! is faster than two. It's even faster if you search for more facets like
//! category, game version, loader, and environment.
//!
//! Filters are parsed into a backend-independent AST before they reach this
//! module. Its normalization pass compacts Cartesian products such as many
//! game versions combined with many loaders into `IN` predicates. The
//! Typesense filter planner then puts maximal version-only expressions into a
//! single join, avoiding repeated joins and deeply expanded boolean trees.
//!
//! ### Whole-field tokenization
//!
//! For fields like a version's `categories`, `environment`, `game_versions`
@@ -117,6 +123,12 @@
//!
//! For this, we use the `:` operator instead of `:=` to tell
//! Typesense to treat this as an exact token match.
//!
//! ### Query caching
//!
//! Typesense caches identical search responses when `TYPESENSE_USE_CACHE` is
//! enabled. This avoids repeating joins for popular queries at the cost of
//! results remaining stale for Typesense's default 60-second cache lifetime.
use std::sync::LazyLock;
@@ -136,6 +148,9 @@ use crate::search::backend::{
SearchIndex, combined_search_filters, parse_search_index,
parse_search_request,
};
use crate::search::filter::{
FilterExpr, from_legacy_v2_facets_json, normalize, parse_expression,
};
use crate::search::indexing::index_local;
use crate::search::{
ResultSearchProject, SearchBackend, SearchField, SearchIndexUpdate,
@@ -144,11 +159,9 @@ use crate::search::{
};
use crate::util::error::Context;
use self::filter_rewrite::{
facets_to_typesense, meili_to_typesense, rewrite_filter_for_join,
};
use self::filter::serialize_filter;
mod filter_rewrite;
mod filter;
const DELETE_FILTER_ID_BATCH_SIZE: usize = 256;
@@ -161,6 +174,7 @@ pub struct TypesenseConfig {
pub index_chunk_size: i64,
pub import_batch_size: usize,
pub delete_batch_size: usize,
pub use_cache: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@@ -291,6 +305,7 @@ impl TypesenseConfig {
index_chunk_size: ENV.SEARCH_INDEX_CHUNK_SIZE,
import_batch_size: ENV.TYPESENSE_IMPORT_BATCH_SIZE,
delete_batch_size: ENV.TYPESENSE_DELETE_BATCH_SIZE,
use_cache: ENV.TYPESENSE_USE_CACHE,
}
}
@@ -905,38 +920,23 @@ impl Typesense {
versions_collection: &str,
) -> Result<Option<String>, ApiError> {
let facet_part = if let Some(facets_json) = info.facets.as_deref() {
Some(
facets_to_typesense(facets_json)
.wrap_request_err("failed to parse facets")?,
)
from_legacy_v2_facets_json(facets_json)
.wrap_request_err("failed to parse facets")?
} else {
None
};
let new_filters_part =
info.new_filters.as_deref().map(meili_to_typesense);
let filter_part = combined_search_filters(info)
.filter(|filter| !filter.trim().is_empty())
.map(|filter| parse_expression(&filter))
.transpose()
.wrap_request_err("failed to parse filters")?;
let legacy_part = if info.new_filters.is_none() {
combined_search_filters(info).map(|f| meili_to_typesense(&f))
} else {
None
};
let filter_part = new_filters_part.or(legacy_part);
let filter = match (facet_part, filter_part) {
(Some(f), Some(l)) if !f.is_empty() && !l.is_empty() => {
Some(format!("({f}) && ({l})"))
}
(Some(f), _) if !f.is_empty() => Some(f),
(_, Some(l)) if !l.is_empty() => Some(l),
_ => None,
};
filter
FilterExpr::and([facet_part, filter_part].into_iter().flatten())
.map(normalize)
.map(|filter| {
rewrite_filter_for_join(&filter, versions_collection)
.wrap_request_err("failed to rewrite search filter")
serialize_filter(&filter, versions_collection)
.wrap_request_err("failed to build search filter")
})
.transpose()
}
@@ -1114,6 +1114,7 @@ impl SearchBackend for Typesense {
let resp = self
.client
.request(Method::POST, "/multi_search")
.query(&[("use_cache", self.config.use_cache)])
.json(&json!({
"searches": [
serde_json::Map::from_iter(
+91
View File
@@ -0,0 +1,91 @@
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum FilterExpr {
And(Vec<Self>),
Or(Vec<Self>),
Predicate(FilterPredicate),
}
impl FilterExpr {
pub fn and(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
let mut expressions = expressions.into_iter().collect::<Vec<_>>();
match expressions.len() {
0 => None,
1 => expressions.pop(),
_ => Some(Self::And(expressions)),
}
}
pub fn or(expressions: impl IntoIterator<Item = Self>) -> Option<Self> {
let mut expressions = expressions.into_iter().collect::<Vec<_>>();
match expressions.len() {
0 => None,
1 => expressions.pop(),
_ => Some(Self::Or(expressions)),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct FilterPredicate {
pub field: FilterField,
pub condition: FilterCondition,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct FilterField(String);
impl FilterField {
pub fn new(field: impl Into<String>) -> Self {
Self(field.into())
}
pub fn as_str(&self) -> &str {
&self.0
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum FilterCondition {
Compare {
comparison: FilterComparison,
value: FilterLiteral,
},
In {
values: Vec<FilterLiteral>,
negated: bool,
},
Exists {
negated: bool,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum FilterComparison {
Equal,
NotEqual,
GreaterThan,
GreaterThanOrEqual,
LessThan,
LessThanOrEqual,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum FilterLiteral {
String(String),
Number(String),
Bool(bool),
}
impl FilterLiteral {
pub(super) fn from_bare(value: String) -> Self {
if value.eq_ignore_ascii_case("true") {
Self::Bool(true)
} else if value.eq_ignore_ascii_case("false") {
Self::Bool(false)
} else if value.parse::<f64>().is_ok() {
Self::Number(value)
} else {
Self::String(value)
}
}
}
@@ -0,0 +1,93 @@
use serde_json::Value;
use thiserror::Error;
use super::{FilterExpr, FilterParseError, parse_expression};
#[derive(Debug, Error)]
pub enum LegacyV2FacetsError {
#[error("invalid facets JSON")]
Json(#[from] serde_json::Error),
#[error("facet condition must be a string")]
InvalidCondition,
#[error(transparent)]
Filter(#[from] FilterParseError),
}
pub fn from_legacy_v2_facets_json(
input: &str,
) -> Result<Option<FilterExpr>, LegacyV2FacetsError> {
let facets = serde_json::from_str::<Vec<Vec<Value>>>(input)?;
let mut groups = Vec::new();
for or_group in facets {
let mut alternatives = Vec::new();
for facet in or_group {
let expression = match facet {
Value::String(condition) => Some(parse_condition(&condition)?),
Value::Array(conditions) => {
let mut predicates = Vec::new();
for condition in conditions {
let condition = condition
.as_str()
.ok_or(LegacyV2FacetsError::InvalidCondition)?;
predicates.push(parse_condition(condition)?);
}
FilterExpr::and(predicates)
}
_ => return Err(LegacyV2FacetsError::InvalidCondition),
};
if let Some(expression) = expression {
alternatives.push(expression);
}
}
if let Some(expression) = FilterExpr::or(alternatives) {
groups.push(expression);
}
}
Ok(FilterExpr::and(groups))
}
fn parse_condition(condition: &str) -> Result<FilterExpr, LegacyV2FacetsError> {
if ["!=", ">=", "<=", ">", "<", "="]
.iter()
.any(|operator| condition.contains(operator))
{
parse_expression(condition).map_err(Into::into)
} else if let Some((field, value)) = condition.split_once(':') {
parse_expression(&format!("{} = {}", field.trim(), value.trim()))
.map_err(Into::into)
} else {
parse_expression(condition).map_err(Into::into)
}
}
#[cfg(test)]
mod tests {
use super::from_legacy_v2_facets_json;
use crate::search::filter::FilterExpr;
#[test]
fn converts_v2_boolean_structure() {
let expression = from_legacy_v2_facets_json(
r#"[["categories:fabric", "categories:forge"], [["game_versions:1.21", "project_types:mod"]]]"#,
)
.unwrap()
.unwrap();
let FilterExpr::And(groups) = expression else {
panic!("expected outer facets to be joined with AND");
};
assert!(matches!(groups[0], FilterExpr::Or(_)));
assert!(matches!(groups[1], FilterExpr::And(_)));
}
#[test]
fn preserves_colons_inside_comparison_values() {
from_legacy_v2_facets_json(
r#"[["license='https://example.com/license'"]]"#,
)
.unwrap()
.unwrap();
}
}
+12
View File
@@ -0,0 +1,12 @@
mod ast;
mod legacy_v2;
mod normalize;
mod parse;
pub use ast::{
FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
FilterPredicate,
};
pub use legacy_v2::from_legacy_v2_facets_json;
pub use normalize::normalize;
pub use parse::{FilterParseError, parse_expression};
@@ -0,0 +1,226 @@
use std::collections::{BTreeMap, BTreeSet};
use super::{
FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
FilterPredicate,
};
pub fn normalize(expression: FilterExpr) -> FilterExpr {
match expression {
FilterExpr::And(expressions) => normalize_and(expressions),
FilterExpr::Or(expressions) => normalize_or(expressions),
FilterExpr::Predicate(predicate) => normalize_predicate(predicate),
}
}
fn normalize_predicate(predicate: FilterPredicate) -> FilterExpr {
if predicate.field.as_str() == "minecraft_java_server.ping.data"
&& let FilterCondition::Exists { negated } = predicate.condition
{
return FilterExpr::Predicate(FilterPredicate {
field: FilterField::new("minecraft_java_server.is_online"),
condition: FilterCondition::Compare {
comparison: FilterComparison::Equal,
value: FilterLiteral::Bool(!negated),
},
});
}
FilterExpr::Predicate(predicate)
}
fn normalize_and(expressions: Vec<FilterExpr>) -> FilterExpr {
let mut normalized = expressions
.into_iter()
.map(normalize)
.flat_map(|expression| match expression {
FilterExpr::And(children) => children,
expression => vec![expression],
})
.collect::<Vec<_>>();
normalized.sort();
normalized.dedup();
FilterExpr::and(normalized).expect("an AND expression is non-empty")
}
fn normalize_or(expressions: Vec<FilterExpr>) -> FilterExpr {
let mut normalized = expressions
.into_iter()
.map(normalize)
.flat_map(|expression| match expression {
FilterExpr::Or(children) => children,
expression => vec![expression],
})
.collect::<Vec<_>>();
normalized.sort();
normalized.dedup();
if let Some(expression) = compact_cartesian_product(&normalized) {
return expression;
}
FilterExpr::or(normalized).expect("an OR expression is non-empty")
}
fn compact_cartesian_product(expressions: &[FilterExpr]) -> Option<FilterExpr> {
let clauses = expressions
.iter()
.map(predicate_clause)
.collect::<Option<Vec<_>>>()?;
if clauses.len() < 2 {
return None;
}
let common = clauses
.iter()
.skip(1)
.fold(clauses[0].clone(), |common, clause| {
common.intersection(clause).cloned().collect()
});
let remaining = clauses
.iter()
.map(|clause| clause.difference(&common).cloned().collect::<Vec<_>>())
.collect::<Vec<_>>();
if remaining.iter().any(Vec::is_empty) {
return FilterExpr::and(common.into_iter().map(FilterExpr::Predicate));
}
let mut values_by_field =
BTreeMap::<FilterField, BTreeSet<FilterLiteral>>::new();
let expected_fields = remaining[0]
.iter()
.map(equality_parts)
.collect::<Option<Vec<_>>>()?
.into_iter()
.map(|(field, _)| field.clone())
.collect::<BTreeSet<_>>();
let mut unique_clauses = BTreeSet::new();
for clause in &remaining {
let parts = clause
.iter()
.map(equality_parts)
.collect::<Option<Vec<_>>>()?;
let fields = parts
.iter()
.map(|(field, _)| (*field).clone())
.collect::<BTreeSet<_>>();
if fields != expected_fields || fields.len() != parts.len() {
return None;
}
for (field, value) in parts {
values_by_field
.entry(field.clone())
.or_default()
.insert(value.clone());
}
unique_clauses.insert(clause.clone());
}
let combinations = values_by_field
.values()
.try_fold(1usize, |count, values| count.checked_mul(values.len()))?;
if combinations != unique_clauses.len() {
return None;
}
let compacted = values_by_field.into_iter().map(|(field, values)| {
let values = values.into_iter().collect::<Vec<_>>();
let condition = if values.len() == 1 {
FilterCondition::Compare {
comparison: FilterComparison::Equal,
value: values.into_iter().next().expect("one value exists"),
}
} else {
FilterCondition::In {
values,
negated: false,
}
};
FilterPredicate { field, condition }
});
FilterExpr::and(
common
.into_iter()
.chain(compacted)
.map(FilterExpr::Predicate),
)
}
fn predicate_clause(
expression: &FilterExpr,
) -> Option<BTreeSet<FilterPredicate>> {
match expression {
FilterExpr::Predicate(predicate) => {
Some(BTreeSet::from([predicate.clone()]))
}
FilterExpr::And(expressions) => expressions
.iter()
.map(|expression| match expression {
FilterExpr::Predicate(predicate) => Some(predicate.clone()),
_ => None,
})
.collect(),
FilterExpr::Or(_) => None,
}
}
fn equality_parts(
predicate: &FilterPredicate,
) -> Option<(&FilterField, &FilterLiteral)> {
match &predicate.condition {
FilterCondition::Compare {
comparison: FilterComparison::Equal,
value,
} => Some((&predicate.field, value)),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::normalize;
use crate::search::filter::{
FilterCondition, FilterExpr, FilterField, FilterLiteral,
FilterPredicate, parse_expression,
};
#[test]
fn compacts_cartesian_product() {
let expression = parse_expression(
"(game_versions = 1.20.1 AND categories = fabric AND categories = technology) OR \
(game_versions = 1.20.1 AND categories = forge AND categories = technology) OR \
(game_versions = 1.21.1 AND categories = fabric AND categories = technology) OR \
(game_versions = 1.21.1 AND categories = forge AND categories = technology)",
)
.unwrap();
let FilterExpr::And(predicates) = normalize(expression) else {
panic!("expected a compacted conjunction");
};
assert_eq!(predicates.len(), 3);
assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
field: FilterField::new("categories"),
condition: FilterCondition::In {
values: vec![
FilterLiteral::String("fabric".into()),
FilterLiteral::String("forge".into()),
],
negated: false,
},
})));
assert!(predicates.contains(&FilterExpr::Predicate(FilterPredicate {
field: FilterField::new("game_versions"),
condition: FilterCondition::In {
values: vec![
FilterLiteral::String("1.20.1".into()),
FilterLiteral::String("1.21.1".into()),
],
negated: false,
},
})));
}
}
+207
View File
@@ -0,0 +1,207 @@
use std::ops::Range;
use chumsky::{Parser, prelude::*};
use thiserror::Error;
use super::{
FilterComparison, FilterCondition, FilterExpr, FilterField, FilterLiteral,
FilterPredicate,
};
#[derive(Debug, Error)]
#[error("invalid filter at byte {position}: {message}")]
pub struct FilterParseError {
position: usize,
message: String,
}
fn keyword(
keyword: &'static str,
) -> BoxedParser<'static, char, (), Simple<char>> {
keyword
.chars()
.fold(empty().ignored().boxed(), |parser, character| {
parser
.then_ignore(one_of([
character.to_ascii_lowercase(),
character.to_ascii_uppercase(),
]))
.boxed()
})
}
fn quoted_literal(
quote: char,
) -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
let escaped = just('\\').ignore_then(any());
let character = escaped.or(filter(move |character| {
*character != quote && *character != '\\'
}));
character
.repeated()
.collect::<String>()
.delimited_by(just(quote), just(quote))
.map(FilterLiteral::String)
.boxed()
}
fn literal_parser() -> BoxedParser<'static, char, FilterLiteral, Simple<char>> {
let quoted = choice((
quoted_literal('\''),
quoted_literal('"'),
quoted_literal('`'),
));
let bare = filter(|character: &char| {
!character.is_whitespace() && !",[]()".contains(*character)
})
.repeated()
.at_least(1)
.collect::<String>()
.map(FilterLiteral::from_bare);
quoted.or(bare).padded().boxed()
}
fn parser() -> impl Parser<char, FilterExpr, Error = Simple<char>> {
let field = filter(|character: &char| {
character.is_ascii_alphabetic() || "_.".contains(*character)
})
.then(
filter(|character: &char| {
character.is_ascii_alphanumeric() || "_.".contains(*character)
})
.repeated(),
)
.map(|(first, rest)| {
FilterField::new(std::iter::once(first).chain(rest).collect::<String>())
})
.padded();
let literal = literal_parser();
let list = literal
.clone()
.separated_by(just(',').padded())
.at_least(1)
.allow_trailing()
.delimited_by(just('[').padded(), just(']').padded());
let comparison = choice((
just("!=").to(FilterComparison::NotEqual),
just(">=").to(FilterComparison::GreaterThanOrEqual),
just("<=").to(FilterComparison::LessThanOrEqual),
just('>').to(FilterComparison::GreaterThan),
just('<').to(FilterComparison::LessThan),
just('=').to(FilterComparison::Equal),
))
.padded()
.then(literal.clone())
.map(|(comparison, value)| FilterCondition::Compare { comparison, value });
let not_in = keyword("NOT")
.padded()
.ignore_then(keyword("IN"))
.padded()
.ignore_then(list.clone())
.map(|values| FilterCondition::In {
values,
negated: true,
});
let in_list = keyword("IN").padded().ignore_then(list).map(|values| {
FilterCondition::In {
values,
negated: false,
}
});
let not_exists = keyword("NOT")
.padded()
.ignore_then(keyword("EXISTS"))
.map(|()| FilterCondition::Exists { negated: true });
let exists =
keyword("EXISTS").map(|()| FilterCondition::Exists { negated: false });
let predicate = field
.then(choice((not_in, in_list, not_exists, exists, comparison)))
.map(|(field, condition)| {
FilterExpr::Predicate(FilterPredicate { field, condition })
});
recursive(|expression| {
let atom = predicate
.clone()
.or(expression.delimited_by(just('(').padded(), just(')').padded()))
.padded();
let and = atom
.clone()
.then(keyword("AND").padded().ignore_then(atom).repeated())
.map(|(first, rest)| {
FilterExpr::and(std::iter::once(first).chain(rest))
.expect("an expression always contains one operand")
});
and.clone()
.then(keyword("OR").padded().ignore_then(and).repeated())
.map(|(first, rest)| {
FilterExpr::or(std::iter::once(first).chain(rest))
.expect("an expression always contains one operand")
})
})
.padded()
.then_ignore(end())
}
pub fn parse_expression(input: &str) -> Result<FilterExpr, FilterParseError> {
parser().parse(input).map_err(|errors| {
let error = errors
.into_iter()
.next()
.unwrap_or_else(|| Simple::custom(0..0, "invalid filter"));
let Range { start, .. } = error.span();
FilterParseError {
position: start,
message: error.to_string(),
}
})
}
#[cfg(test)]
mod tests {
use super::parse_expression;
use crate::search::filter::{FilterCondition, FilterExpr, FilterLiteral};
#[test]
fn parses_boolean_precedence() {
let expression = parse_expression(
"license = MIT OR downloads >= 100 AND open_source = true",
)
.unwrap();
let FilterExpr::Or(expressions) = expression else {
panic!("expected an OR expression");
};
assert_eq!(expressions.len(), 2);
assert!(matches!(expressions[1], FilterExpr::And(_)));
}
#[test]
fn parses_quoted_and_list_values() {
let expression = parse_expression(
r#"name = "value with spaces" AND categories IN [fabric, "forge"]"#,
)
.unwrap();
let FilterExpr::And(expressions) = expression else {
panic!("expected an AND expression");
};
let FilterExpr::Predicate(predicate) = &expressions[0] else {
panic!("expected a predicate");
};
assert!(matches!(
&predicate.condition,
FilterCondition::Compare {
value: FilterLiteral::String(value),
..
} if value == "value with spaces"
));
}
}
+1
View File
@@ -16,6 +16,7 @@ use thiserror::Error;
use utoipa::ToSchema;
pub mod backend;
pub mod filter;
pub mod incremental;
pub mod indexing;