feat: usercheck & oauth email verification (#7291)

* feat: usercheck

* feat: make oauth users not skip email verification

* slightly improve message

_typos.toml

@@ -11,8 +11,6 @@ extend-exclude = [
   "packages/moderation/src/utils.ts",
   # contains payment card IDs like `IY1VMST1MOXS` which are flagged
   "apps/labrinth/src/queue/payouts/mod.rs",
-  # contains domain names with deliberate typos
-  "apps/labrinth/assets/disposable_email_blocklist.txt"
 ]
 
 [default.extend-words]

apps/labrinth/src/env.rs

@@ -317,8 +317,9 @@ vars! {
     NEVERBOUNCE_API_KEY: String = "";
     NEVERBOUNCE_BASE_URL: String = neverbounce::DEFAULT_API_URL;
 
-    EMAIL_DOMAIN_BLACKLIST: StringCsv = StringCsv(vec![]);
-    EMAIL_DOMAIN_WHITELIST: StringCsv = StringCsv(vec![]);
+    USERCHECK_API_KEY: String = "";
+    USERCHECK_GATE_ID: String = "";
+    USERCHECK_BASE_URL: String = crate::util::usercheck::DEFAULT_API_URL;
 
     CLICKHOUSE_REPLICATED: bool = false;
     CLICKHOUSE_URL: String = "http://localhost:8123";

apps/labrinth/src/routes/internal/flows.rs

@@ -24,7 +24,11 @@ use crate::util::error::ApiContext as _;
 use crate::util::error::Context;
 use crate::util::ext::get_image_ext;
 use crate::util::img::upload_image_optimized;
+use crate::util::ip::client_ip;
 use crate::util::neverbounce::{check_email, email_check_error_generic};
+use crate::util::usercheck::{
+    DecisionAction, check_email_gate, gate_block_error,
+};
 use crate::util::validate::validation_errors_to_string;
 use actix_http::header::LOCATION;
 use actix_web::http::StatusCode;
@@ -50,7 +54,7 @@ use sha2::Sha256;
 use std::collections::HashMap;
 use std::str::FromStr;
 use thiserror::Error;
-use tracing::{error, info};
+use tracing::{error, info, warn};
 use url::Url;
 use uuid::Uuid;
 use validator::Validate;
@@ -62,10 +66,6 @@ use webauthn_rs::prelude::{
 use xredis::RedisPool;
 use zxcvbn::Score;
 
-/// Sourced from <https://github.com/disposable-email-domains/disposable-email-domains>.
-const DISPOSABLE_EMAIL_BLOCKLIST: &str =
-    include_str!("../../../assets/disposable_email_blocklist.txt");
-
 pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
     cfg.service(
         web::scope("/auth")
@@ -247,7 +247,7 @@ impl TempUser {
             totp_secret: None,
             username,
             email: self.email.clone(),
-            email_verified: self.email.is_some(),
+            email_verified: false,
             avatar_url,
             raw_avatar_url,
             bio: self.bio,
@@ -1459,6 +1459,40 @@ fn validate_account_consent(account_consent: bool) -> Result<(), ApiError> {
     Ok(())
 }
 
+async fn send_verify_email(
+    email_queue: &EmailQueue,
+    txn: &mut PgTransaction<'_>,
+    redis: &RedisPool,
+    user_id: DBUserId,
+    email_address: String,
+) -> Result<(), ApiError> {
+    let mailbox: Mailbox = email_address
+        .parse()
+        .wrap_request_err("invalid email address!".to_string())?;
+
+    let flow = DBFlow::ConfirmEmail {
+        user_id,
+        confirm_email: email_address,
+    }
+    .insert(Duration::hours(24), redis)
+    .await
+    .wrap_internal_err("storing email-verification flow in Redis")?;
+
+    email_queue
+        .send_one(
+            txn,
+            NotificationBody::VerifyEmail { flow },
+            user_id,
+            mailbox,
+        )
+        .await
+        .wrap_api_err("sending account email")?
+        .as_user_error()
+        .wrap_api_err("validating email delivery status")?;
+
+    Ok(())
+}
+
 /// Create account with OAuth.
 #[utoipa::path(
 	context_path = "/auth",
@@ -1476,6 +1510,7 @@ pub async fn create_oauth_account(
     db: Data<PgPool>,
     file_host: Data<dyn FileHost>,
     redis: Data<RedisPool>,
+    email_queue: Data<EmailQueue>,
     web::Json(new_account): web::Json<NewOAuthAccount>,
 ) -> Result<HttpResponse, ApiError> {
     new_account
@@ -1510,8 +1545,9 @@ pub async fn create_oauth_account(
     };
 
     if let Some(email) = &user.email {
-        ensure_email_domain_is_allowed(email)
-            .wrap_api_err("validating email domain is allowed")?;
+        ensure_email_passes_gate(&req, email)
+            .await
+            .wrap_api_err("validating email passes the signup gate")?;
     }
 
     let mut txn = db
@@ -1519,6 +1555,8 @@ pub async fn create_oauth_account(
         .await
         .wrap_internal_err("failed to begin transaction")?;
 
+    let account_email = user.email.clone();
+
     let user_id = user
         .create_account(
             provider,
@@ -1532,6 +1570,23 @@ pub async fn create_oauth_account(
         .await
         .wrap_auth_err("inserting user ID into database")?;
 
+    if let Some(email_address) = account_email {
+        // The address comes from the OAuth provider, so the user cannot correct
+        // it here. A failed send shouldn't block signup: they can resend the
+        // verification email once they're signed in.
+        if let Err(error) = send_verify_email(
+            &email_queue,
+            &mut txn,
+            &redis,
+            user_id,
+            email_address,
+        )
+        .await
+        {
+            warn!(%error, "failed to send OAuth signup verification email");
+        }
+    }
+
     let session = issue_session(req, user_id, &mut txn, &redis, None)
         .await
         .wrap_auth_err("authenticating API request")?;
@@ -1847,80 +1902,27 @@ impl From<NewAccount> for AccountRegisterFlow {
     }
 }
 
-#[derive(PartialEq, Eq)]
-enum EmailDomainStatus {
-    Whitelisted,
-    Neutral,
-}
-
-/// Environment list entries are matched literally, unless they begin with `*.`,
-/// in which case they match any subdomain of the remaining suffix.
-fn matches_domain_entry(domain: &str, entry: &str) -> bool {
-    let entry = entry.trim().to_ascii_lowercase();
-
-    match entry.strip_prefix("*.") {
-        Some(suffix) => domain
-            .strip_suffix(suffix)
-            .is_some_and(|subdomain| subdomain.ends_with('.')),
-        None => entry == domain,
-    }
-}
-
-fn is_whitelisted_domain(domain: &str) -> bool {
-    let domain = domain.to_ascii_lowercase();
-
-    ENV.EMAIL_DOMAIN_WHITELIST
-        .iter()
-        .any(|entry| matches_domain_entry(&domain, entry))
-}
-
-/// The bundled disposable domain list is checked first, then the environment
-/// blacklist.
-fn is_blacklisted_domain(domain: &str) -> bool {
-    let domain = domain.to_ascii_lowercase();
-
-    if DISPOSABLE_EMAIL_BLOCKLIST.lines().any(|entry| {
-        // The upstream list expects listed domains to match subdomains too.
-        domain == entry
-            || domain
-                .strip_suffix(entry)
-                .is_some_and(|subdomain| subdomain.ends_with('.'))
-    }) {
-        return true;
-    }
-
-    ENV.EMAIL_DOMAIN_BLACKLIST
-        .iter()
-        .any(|entry| matches_domain_entry(&domain, entry))
-}
-
-fn ensure_email_domain_is_allowed(
+/// Runs the UserCheck gate, which covers both password and OAuth signups.
+async fn ensure_email_passes_gate(
+    req: &HttpRequest,
     email: &str,
-) -> Result<EmailDomainStatus, ApiError> {
-    let Some((_, domain)) = email.rsplit_once('@') else {
-        return Err(ApiError::Request(email_check_error_generic()));
-    };
+) -> Result<(), ApiError> {
+    let action = check_email_gate(email, client_ip(req).as_deref())
+        .await
+        .wrap_request_err("checking email address")?;
 
-    if is_whitelisted_domain(domain) {
-        info!(email.domain = domain, "whitelisted email domain, allowing");
-        return Ok(EmailDomainStatus::Whitelisted);
+    if action == DecisionAction::Block {
+        return Err(ApiError::Request(gate_block_error()));
     }
 
-    if is_blacklisted_domain(domain) {
-        info!(email.domain = domain, "blacklisted email domain, denying");
-        return Err(ApiError::Request(email_check_error_generic()));
-    }
-
-    Ok(EmailDomainStatus::Neutral)
+    Ok(())
 }
 
-async fn ensure_email_is_usable(email: &str) -> Result<(), ApiError> {
-    let status = ensure_email_domain_is_allowed(email)
-        .wrap_api_err("validating email domain is allowed")?;
-
-    if status == EmailDomainStatus::Whitelisted {
-        return Ok(());
-    }
+async fn ensure_email_is_usable(
+    req: &HttpRequest,
+    email: &str,
+) -> Result<(), ApiError> {
+    ensure_email_passes_gate(req, email).await?;
 
     let result = check_email(email)
         .await
@@ -2075,30 +2077,14 @@ impl ReadyAccountRegisterFlow {
             .wrap_auth_err("authenticating API request")?;
         let res = crate::models::sessions::Session::from(session, true, None);
 
-        let mailbox: Mailbox = register_flow
-            .email
-            .parse()
-            .wrap_request_err("invalid email address!".to_string())?;
-
-        let flow = DBFlow::ConfirmEmail {
+        send_verify_email(
+            email_queue,
+            transaction,
+            redis,
             user_id,
-            confirm_email: register_flow.email.clone(),
-        }
-        .insert(Duration::hours(24), redis)
-        .await
-        .wrap_internal_err("storing email-verification flow in Redis")?;
-
-        email_queue
-            .send_one(
-                transaction,
-                NotificationBody::VerifyEmail { flow },
-                user_id,
-                mailbox,
-            )
-            .await
-            .wrap_api_err("sending account email")?
-            .as_user_error()
-            .wrap_api_err("validating email delivery status")?;
+            register_flow.email,
+        )
+        .await?;
 
         Ok(res)
     }
@@ -2165,7 +2151,7 @@ pub async fn create_account_with_password(
         )));
     }
 
-    ensure_email_is_usable(&new_account.email)
+    ensure_email_is_usable(&req, &new_account.email)
         .await
         .wrap_api_err("validating email is usable")?;
 
@@ -3150,7 +3136,7 @@ pub async fn set_email(
         )));
     }
 
-    ensure_email_is_usable(&email_address.email)
+    ensure_email_is_usable(&req, &email_address.email)
         .await
         .wrap_api_err("validating email is usable")?;
 

apps/labrinth/src/util/ip.rs

@@ -1,5 +1,9 @@
 use std::net::{AddrParseError, IpAddr, Ipv6Addr};
 
+use actix_web::HttpRequest;
+
+use crate::env::ENV;
+
 pub fn convert_to_ip_v6(src: &str) -> Result<Ipv6Addr, AddrParseError> {
     let ip_addr: IpAddr = src.parse()?;
 
@@ -23,3 +27,13 @@ pub fn strip_ip(ip: Ipv6Addr) -> u64 {
         ])
     }
 }
+
+pub fn client_ip(req: &HttpRequest) -> Option<String> {
+    if ENV.CLOUDFLARE_INTEGRATION
+        && let Some(header) = req.headers().get("CF-Connecting-IP")
+    {
+        return header.to_str().ok().map(str::to_owned);
+    }
+
+    req.connection_info().peer_addr().map(str::to_owned)
+}

apps/labrinth/src/util/mod.rs

@@ -20,5 +20,6 @@ pub mod routes;
 pub mod sentry;
 pub mod tags;
 pub mod tiltify;
+pub mod usercheck;
 pub mod validate;
 pub mod webhook;

apps/labrinth/src/util/usercheck.rs

@@ -0,0 +1,216 @@
+use std::time::{Duration, Instant};
+
+use eyre::{WrapErr, eyre};
+use reqwest::StatusCode;
+use serde::{Deserialize, Deserializer, Serialize};
+use tracing::{debug, error, warn};
+
+use crate::env::ENV;
+use crate::util::http::HTTP_CLIENT;
+use crate::util::neverbounce::email_check_error_generic;
+
+pub const DEFAULT_API_URL: &str = "https://api.usercheck.com";
+const TIMEOUT: Duration = Duration::from_secs(5);
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum DecisionAction {
+    Allow,
+    Block,
+    Challenge,
+    Unrecognized(String),
+}
+
+impl DecisionAction {
+    fn as_str(&self) -> &str {
+        match self {
+            DecisionAction::Allow => "allow",
+            DecisionAction::Block => "block",
+            DecisionAction::Challenge => "challenge",
+            DecisionAction::Unrecognized(other) => other,
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for DecisionAction {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Ok(match String::deserialize(deserializer)?.as_str() {
+            "allow" => Self::Allow,
+            "block" => Self::Block,
+            "challenge" => Self::Challenge,
+            value => Self::Unrecognized(value.to_owned()),
+        })
+    }
+}
+
+#[derive(Deserialize)]
+struct DecisionResponse {
+    decision: Decision,
+    #[serde(default)]
+    meta: Option<ResponseMeta>,
+}
+
+#[derive(Deserialize)]
+struct Decision {
+    action: DecisionAction,
+    #[serde(default)]
+    matched_rule: Option<MatchedRule>,
+}
+
+#[derive(Deserialize)]
+struct MatchedRule {
+    id: String,
+    name: String,
+    #[serde(default)]
+    message: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct ResponseMeta {
+    #[serde(default)]
+    request_id: Option<String>,
+}
+
+#[derive(Serialize)]
+struct DecisionRequest<'a> {
+    email: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    ip: Option<&'a str>,
+}
+
+/// Asks the configured UserCheck gate whether a signup should proceed.
+///
+/// Failure handling mirrors [`crate::util::neverbounce::check_email`]: a
+/// transient failure resolves to `Allow` so an outage cannot block every
+/// signup, while anything else is an error that rejects the signup.
+/// `Challenge` resolves to `Allow` because these flows have no step-up
+/// mechanism past the captcha that already ran.
+pub async fn check_email_gate(
+    email: &str,
+    ip: Option<&str>,
+) -> eyre::Result<DecisionAction> {
+    if ENV.USERCHECK_API_KEY.is_empty() || ENV.USERCHECK_GATE_ID.is_empty() {
+        debug!(
+            action = "allow",
+            "UserCheck gate skipped because the API key or gate ID is not set",
+        );
+        return Ok(DecisionAction::Allow);
+    }
+
+    let decision_time_start = Instant::now();
+    let response = request_decision(email, ip).await;
+    let decision_time = decision_time_start.elapsed();
+
+    let response = match response {
+        Ok(response) => response,
+        Err(source) => {
+            let is_transient = is_transient(&source);
+
+            error!(
+                action = if is_transient { "allow" } else { "block" },
+                request.transient = is_transient,
+                request.time_ms = decision_time.as_millis(),
+                error = ?source,
+                "UserCheck gate decision failed",
+            );
+
+            if is_transient {
+                return Ok(DecisionAction::Allow);
+            }
+
+            return Err(eyre!(source)).wrap_err("failed to check email");
+        }
+    };
+
+    let DecisionResponse { decision, meta } = response;
+    let Decision {
+        action,
+        matched_rule,
+    } = decision;
+
+    let rule_id = matched_rule.as_ref().map(|rule| rule.id.as_str());
+    let rule_name = matched_rule.as_ref().map(|rule| rule.name.as_str());
+    let rule_message = matched_rule
+        .as_ref()
+        .and_then(|rule| rule.message.as_deref());
+    let request_id = meta.and_then(|meta| meta.request_id);
+    let time_ms = decision_time.as_millis();
+
+    match action {
+        DecisionAction::Unrecognized(ref value) => {
+            error!(
+                action = value.as_str(),
+                rule.id = rule_id,
+                rule.name = rule_name,
+                rule.message = rule_message,
+                request.id = request_id,
+                request.time_ms = time_ms,
+                "UserCheck gate returned an unrecognized action",
+            );
+            return Err(email_check_error_generic());
+        }
+        DecisionAction::Challenge => warn!(
+            action = action.as_str(),
+            rule.id = rule_id,
+            rule.name = rule_name,
+            rule.message = rule_message,
+            request.id = request_id,
+            request.time_ms = time_ms,
+            "UserCheck gate returned a challenge, allowing",
+        ),
+        _ => debug!(
+            action = action.as_str(),
+            rule.id = rule_id,
+            rule.name = rule_name,
+            rule.message = rule_message,
+            request.id = request_id,
+            request.time_ms = time_ms,
+            "UserCheck gate decision succeeded",
+        ),
+    }
+
+    Ok(action)
+}
+
+pub fn gate_block_error() -> eyre::Error {
+    eyre!(
+        "Please try a different email address, or turn off any VPN or proxy services!"
+    )
+}
+
+async fn request_decision(
+    email: &str,
+    ip: Option<&str>,
+) -> reqwest::Result<DecisionResponse> {
+    HTTP_CLIENT
+        .post(format!(
+            "{}/v0/gates/{}/decisions",
+            ENV.USERCHECK_BASE_URL.trim_end_matches('/'),
+            ENV.USERCHECK_GATE_ID,
+        ))
+        .bearer_auth(&ENV.USERCHECK_API_KEY)
+        .timeout(TIMEOUT)
+        .json(&DecisionRequest { email, ip })
+        .send()
+        .await?
+        .error_for_status()?
+        .json()
+        .await
+}
+
+fn is_transient(error: &reqwest::Error) -> bool {
+    if let Some(status) = error.status() {
+        return status.is_server_error()
+            || matches!(
+                status,
+                StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS
+            );
+    }
+
+    error.is_timeout()
+        || error.is_connect()
+        || error.is_request()
+        || error.is_body()
+}