Improve error handling in account balance notifications (#6478)

* Improve error handling in account balance notifications

* fix compile

apps/labrinth/src/queue/payouts/mod.rs

@@ -481,8 +481,7 @@ impl PayoutsQueue {
         Ok(options.options)
     }
 
-    pub async fn get_brex_balance() -> Result<Option<AccountBalance>, ApiError>
-    {
+    pub async fn get_brex_balance() -> eyre::Result<Option<AccountBalance>> {
         #[derive(Deserialize)]
         struct BrexBalance {
             pub amount: i64,
@@ -505,9 +504,11 @@ impl PayoutsQueue {
             .get(format!("{}accounts/cash", ENV.BREX_API_URL))
             .bearer_auth(&ENV.BREX_API_KEY)
             .send()
-            .await?
+            .await
+            .wrap_request_err("sending `accounts/cash` request")?
             .json::<BrexResponse>()
-            .await?;
+            .await
+            .wrap_request_err("reading `accounts/cash` request")?;
 
         Ok(Some(AccountBalance {
             available: Decimal::from(
@@ -527,8 +528,7 @@ impl PayoutsQueue {
         }))
     }
 
-    pub async fn get_paypal_balance() -> Result<Option<AccountBalance>, ApiError>
-    {
+    pub async fn get_paypal_balance() -> eyre::Result<Option<AccountBalance>> {
         let api_username = &ENV.PAYPAL_NVP_USERNAME;
         let api_password = &ENV.PAYPAL_NVP_PASSWORD;
         let api_signature = &ENV.PAYPAL_NVP_SIGNATURE;
@@ -544,9 +544,17 @@ impl PayoutsQueue {
         let endpoint = "https://api-3t.paypal.com/nvp";
 
         let client = reqwest::Client::new();
-        let response = client.post(endpoint).form(&params).send().await?;
+        let response = client
+            .post(endpoint)
+            .form(&params)
+            .send()
+            .await
+            .wrap_err("requesting `nvp` endpoint")?;
 
-        let text = response.text().await?;
+        let text = response
+            .text()
+            .await
+            .wrap_err("reading response text from `nvp`")?;
         let body = urlencoding::decode(&text).unwrap_or_default();
 
         let mut key_value_map = HashMap::new();
@@ -573,7 +581,7 @@ impl PayoutsQueue {
 
     pub async fn get_tremendous_balance(
         &self,
-    ) -> Result<Option<AccountBalance>, ApiError> {
+    ) -> eyre::Result<Option<AccountBalance>> {
         #[derive(Deserialize)]
         struct FundingSourceMeta {
             available_cents: Option<u64>,
@@ -597,7 +605,8 @@ impl PayoutsQueue {
                 "funding_sources",
                 None,
             )
-            .await?;
+            .await
+            .wrap_request_err("fetching funding sources")?;
 
         Ok(val
             .funding_sources
@@ -1257,7 +1266,7 @@ pub async fn index_payouts_notifications(
 pub async fn insert_bank_balances_and_webhook(
     payouts: &PayoutsQueue,
     pool: &PgPool,
-) -> Result<(), ApiError> {
+) -> eyre::Result<()> {
     let mut transaction = pool.begin().await?;
 
     let paypal_result = PayoutsQueue::get_paypal_balance().await;
@@ -1320,25 +1329,29 @@ pub async fn insert_bank_balances_and_webhook(
             ENV.PAYPAL_BALANCE_ALERT_THRESHOLD,
             paypal_result,
         )
-        .await?;
+        .await
+        .wrap_err("checking PayPal balance")?;
         check_balance_with_webhook(
             "brex",
             ENV.BREX_BALANCE_ALERT_THRESHOLD,
             brex_result,
         )
-        .await?;
+        .await
+        .wrap_err("checking Brex balance")?;
         check_balance_with_webhook(
             "tremendous",
             ENV.TREMENDOUS_BALANCE_ALERT_THRESHOLD,
             tremendous_result,
         )
-        .await?;
+        .await
+        .wrap_err("checking Tremendous balance")?;
         check_balance_with_webhook(
             "mural",
             ENV.MURAL_BALANCE_ALERT_THRESHOLD,
             mural_result,
         )
-        .await?;
+        .await
+        .wrap_err("checking Mural balance")?;
     }
 
     transaction.commit().await?;
@@ -1349,8 +1362,8 @@ pub async fn insert_bank_balances_and_webhook(
 async fn check_balance_with_webhook(
     source: &str,
     threshold: u64,
-    result: Result<Option<AccountBalance>, ApiError>,
-) -> Result<Option<AccountBalance>, ApiError> {
+    result: eyre::Result<Option<AccountBalance>>,
+) -> eyre::Result<Option<AccountBalance>> {
     let maybe_threshold = if threshold > 0 { Some(threshold) } else { None };
     let payout_alert_webhook = &ENV.PAYOUT_ALERT_SLACK_WEBHOOK;
 
@@ -1374,13 +1387,18 @@ async fn check_balance_with_webhook(
         }
 
         Err(error) => {
-            error!(%error, "Failure getting balance for payout source '{source}'");
+            // use compact single-line error repr here
+            error!(
+                error = format!("{error:#?}"),
+                "Failure getting balance for payout source '{source}'"
+            );
 
             if maybe_threshold.is_some() {
+                // use expanded multi-line error repr here
                 send_slack_payout_source_alert_webhook(
                     PayoutSourceAlertType::CheckFailure {
                         source: source.to_owned(),
-                        display_error: error.to_string(),
+                        display_error: format!("{error:?}"),
                     },
                     payout_alert_webhook,
                 )

apps/labrinth/src/queue/payouts/mural.rs

@@ -106,20 +106,20 @@ impl PayoutsQueue {
 
     pub async fn get_mural_balance(
         &self,
-    ) -> Result<Option<AccountBalance>, ApiError> {
+    ) -> eyre::Result<Option<AccountBalance>> {
         let muralpay = self.muralpay.load();
         let muralpay = muralpay
             .as_ref()
-            .wrap_internal_err("Mural Pay client not available")?;
+            .wrap_err("Mural Pay client not available")?;
 
         let account = muralpay
             .client
             .get_account(muralpay.source_account_id)
             .await
-            .wrap_internal_err("failed to get source account")?;
+            .wrap_err("failed to get source account")?;
         let details = account
             .account_details
-            .wrap_internal_err("source account does not have details")?;
+            .wrap_err("source account does not have details")?;
         let available = details
             .balances
             .iter()

apps/labrinth/src/util/tiltify.rs

@@ -70,7 +70,11 @@ impl TiltifyClient {
         }
 
         if state.rate_limited_until > Instant::now() {
-            return Err(eyre!("waiting for rate limit to reset"));
+            return Err(eyre!(
+                "waiting for rate limit to reset at {:.0?} (backoff {:.0?})",
+                state.rate_limited_until,
+                state.rate_limit_backoff
+            ));
         }
 
         if ENV.TILTIFY_CLIENT_ID.is_empty()

apps/labrinth/src/util/webhook.rs

@@ -198,13 +198,22 @@ impl PayoutSourceAlertType {
                 threshold,
                 current_balance,
             } => format!(
-                "\u{1f6a8} *Payout Source Alert*\n\nPayout source '{source}' has an available balance under the ${threshold} threshold.\nBalance: ${current_balance}."
+"\u{1f6a8} *Payout Source Alert*
+
+Payout source '{source}' has an available balance under the ${threshold} threshold.
+Balance: ${current_balance}."
             ),
             PayoutSourceAlertType::CheckFailure {
                 source,
                 display_error,
             } => format!(
-                "\u{1f6a8} *Payout Source Alert*\n\nFAILED TO CHECK payout source '{source}' balance.\nError: {display_error}"
+"\u{1f6a8} *Payout Source Alert*
+
+Failed to check payout source '{source}' balance.
+Error:
+```
+{display_error}
+```"
             ),
         }
     }