Improve timeout handling and error recovery

.env-sample

@@ -61,6 +61,12 @@ LOG_LEVEL='debug'
 # Max routing fee that we want to pay to the network, 0.001 = 0.1%
 MAX_ROUTING_FEE=0.001
 
+# Lightning Network pathfinding timeout in milliseconds (default: 60000 = 60 seconds)
+LN_PATHFINDING_TIMEOUT=60000
+
+# Telegram bot handler timeout in milliseconds (default: 60000 = 60 seconds)
+BOT_HANDLER_TIMEOUT=60000
+
 # Attempts to pay the invoice again when the payment failed
 PAYMENT_ATTEMPTS=2
 

CLAUDE.md

@@ -0,0 +1,96 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+LNp2pBot is a Telegram bot that facilitates peer-to-peer Lightning Network trading. Users can create buy/sell orders for Bitcoin using Lightning Network hold invoices to ensure trustless transactions.
+
+## Development Commands
+
+### Build and Run
+```bash
+npm start          # Compile TypeScript and start bot
+npm run dev        # Compile and start with nodemon for development
+npm run prestart   # Compile TypeScript only
+npm run predev     # Compile TypeScript only
+```
+
+### Code Quality
+```bash
+npm run lint       # ESLint code validation
+npm run format     # Format code with Prettier
+```
+
+### Testing
+```bash
+npm test           # Run all tests with Mocha
+npm run pretest    # Compile tests only
+```
+
+## Architecture
+
+### Core Structure
+- **app.ts**: Main entry point, handles MongoDB connection and bot initialization
+- **bot/**: Bot logic and command handlers
+  - **start.ts**: Bot initialization, command registration, and scheduled jobs
+  - **modules/**: Feature-specific modules (community, orders, dispute, etc.)
+  - **middleware/**: Authentication, validation, and context enhancement
+- **models/**: MongoDB schemas (User, Order, Community, Dispute, etc.)
+- **ln/**: Lightning Network integration (hold invoices, payments, subscriptions)
+- **jobs/**: Scheduled background tasks for order management and payments
+- **util/**: Shared utilities and helpers
+
+### Key Patterns
+
+#### Context Enhancement
+The bot uses custom context types that extend Telegraf's base context:
+- `MainContext`: Adds i18n, user, and admin properties
+- `CommunityContext`: Extends MainContext for community-specific features
+
+#### Module Structure
+Each feature module follows this pattern:
+- `commands.ts`: Command handlers
+- `actions.ts`: Action button handlers  
+- `messages.ts`: Message templates
+- `scenes.ts`: Multi-step conversation flows
+- `index.ts`: Module configuration and exports
+
+#### Hold Invoice Pattern
+The bot uses Lightning Network hold invoices for trustless escrow:
+1. Seller creates order
+2. Buyer takes order, seller pays hold invoice
+3. Funds are held until both parties confirm fiat exchange
+4. Hold invoice is settled or canceled based on dispute resolution
+
+#### Job Scheduling
+Background jobs handle critical functions:
+- Pending payment retries
+- Order expiration and cleanup
+- Community earnings calculation
+- Node health monitoring
+
+### Database Models
+- **User**: Telegram users with trading history and preferences
+- **Order**: Trading orders with status tracking and dispute handling
+- **Community**: Telegram groups with custom fee structures
+- **Dispute**: Conflict resolution with solver assignment
+- **PendingPayment**: Failed payment retry queue
+
+### Environment Setup
+Copy `.env-sample` to `.env` and configure:
+- `BOT_TOKEN`: Telegram bot API token
+- `LND_*`: Lightning Network node connection details
+- `DB_*` or `MONGO_URI`: MongoDB connection
+- `CHANNEL`: Public order announcement channel
+- `ADMIN_CHANNEL`: Admin notifications
+
+### Testing
+Tests are in TypeScript and use Mocha with Chai assertions. Test compilation uses a separate tsconfig.test.json that includes the tests directory.
+
+### Key Dependencies
+- **telegraf**: Telegram bot framework
+- **mongoose**: MongoDB ODM
+- **lightning**: LND node integration
+- **node-schedule**: Cron job scheduling
+- **@grammyjs/i18n**: Internationalization

app.ts

@@ -25,7 +25,9 @@ import { CommunityContext } from "./bot/modules/community/communityContext";
   mongoose.connection
     .once('open', async () => {
       logger.info('Connected to Mongo instance.');
-      let options: Partial<Telegraf.Options<CommunityContext>> = { handlerTimeout: 60000 };
+      // Use configurable bot handler timeout, default to 60 seconds
+      const handlerTimeout = parseInt(process.env.BOT_HANDLER_TIMEOUT || '60000');
+      let options: Partial<Telegraf.Options<CommunityContext>> = { handlerTimeout };
       if (process.env.SOCKS_PROXY_HOST) {
         const agent = new SocksProxyAgent(process.env.SOCKS_PROXY_HOST);
         options = {

jobs/pending_payments.ts

@@ -14,12 +14,21 @@ export const attemptPendingPayments = async (bot: Telegraf<CommunityContext>): P
     attempts: { $lt: process.env.PAYMENT_ATTEMPTS },
     is_invoice_expired: false,
     community_id: null,
+    next_retry: { $lte: new Date() },
   });
   for (const pending of pendingPayments) {
     const order = await Order.findOne({ _id: pending.order_id });
     try {
       if (order === null) throw Error("Order was not found in DB");
       pending.attempts++;
+
+      // Calculate exponential backoff delay
+      const baseDelay = 5 * 60 * 1000; // 5 minutes
+      const exponentialDelay = baseDelay * Math.pow(2, pending.attempts - 1);
+      const maxDelay = 60 * 60 * 1000; // 1 hour max
+      const nextRetryDelay = Math.min(exponentialDelay, maxDelay);
+      pending.next_retry = new Date(Date.now() + nextRetryDelay);
+
       if (order.status === 'SUCCESS') {
         pending.paid = true;
         await pending.save();
@@ -85,9 +94,25 @@ export const attemptPendingPayments = async (bot: Telegraf<CommunityContext>): P
         );
         await messages.rateUserMessage(bot, buyerUser, order, i18nCtx);
       } else {
+        // Enhanced error handling for different payment failure types
+        if (payment && typeof payment === 'object' && 'error' in payment) {
+          pending.last_error = payment.error as string;
+
+          if (payment.error === 'TIMEOUT') {
+            logger.warn(`Payment timeout for order ${order._id}, attempt ${pending.attempts}`);
+          } else if (payment.error === 'ROUTING_FAILED') {
+            logger.warn(`Routing failed for order ${order._id}, attempt ${pending.attempts}`);
+          } else {
+            logger.error(`Payment failed for order ${order._id}, attempt ${pending.attempts}, error: ${payment.error}`);
+          }
+        } else {
+          pending.last_error = 'PAYMENT_FAILED';
+          logger.error(`Payment failed for order ${order._id}, attempt ${pending.attempts}`);
+        }
+
         if (
           process.env.PAYMENT_ATTEMPTS !== undefined &&
-          pending.attempts === parseInt(process.env.PAYMENT_ATTEMPTS)
+          pending.attempts >= parseInt(process.env.PAYMENT_ATTEMPTS)
         ) {
           order.paid_hold_buyer_invoice_updated = false;
           await messages.toBuyerPendingPaymentFailedMessage(
@@ -124,11 +149,19 @@ export const attemptCommunitiesPendingPayments = async (bot: Telegraf<CommunityC
     attempts: { $lt: process.env.PAYMENT_ATTEMPTS },
     is_invoice_expired: false,
     community_id: { $ne: null },
+    next_retry: { $lte: new Date() },
   });
 
   for (const pending of pendingPayments) {
     try {
       pending.attempts++;
+
+      // Calculate exponential backoff delay for community payments
+      const baseDelay = 5 * 60 * 1000; // 5 minutes
+      const exponentialDelay = baseDelay * Math.pow(2, pending.attempts - 1);
+      const maxDelay = 60 * 60 * 1000; // 1 hour max
+      const nextRetryDelay = Math.min(exponentialDelay, maxDelay);
+      pending.next_retry = new Date(Date.now() + nextRetryDelay);
 
       // We check if this new payment is on flight
       const isPending: boolean = await isPendingPayment(pending.payment_request);
@@ -174,9 +207,22 @@ export const attemptCommunitiesPendingPayments = async (bot: Telegraf<CommunityC
           })
         );
       } else {
+        // Enhanced error handling for community payments
+        if (payment && typeof payment === 'object' && 'error' in payment) {
+          pending.last_error = payment.error as string;
+          logger.error(
+            `Community ${community.id}: Withdraw failed after ${pending.attempts} attempts, amount ${pending.amount} sats, error: ${payment.error}`
+          );
+        } else {
+          pending.last_error = 'PAYMENT_FAILED';
+          logger.error(
+            `Community ${community.id}: Withdraw failed after ${pending.attempts} attempts, amount ${pending.amount} sats`
+          );
+        }
+
         if (
           process.env.PAYMENT_ATTEMPTS !== undefined &&
-          pending.attempts === parseInt(process.env.PAYMENT_ATTEMPTS)
+          pending.attempts >= parseInt(process.env.PAYMENT_ATTEMPTS)
         ) {
           await bot.telegram.sendMessage(
             user.tg_id,
@@ -185,9 +231,6 @@ export const attemptCommunitiesPendingPayments = async (bot: Telegraf<CommunityC
             })
           );
         }
-        logger.error(
-          `Community ${community.id}: Withdraw failed after ${pending.attempts} attempts, amount ${pending.amount} sats`
-        );
       }
     } catch (error) {
       logger.error(`attemptCommunitiesPendingPayments catch error: ${error}`);

ln/pay_request.ts

@@ -8,7 +8,7 @@ import { User, PendingPayment } from '../models';
 import lnd from './connect';
 import { handleReputationItems, getUserI18nContext } from '../util';
 import * as messages from '../bot/messages';
-import { logger } from '../logger';
+import { logger, logTimeout, logOperationDuration } from '../logger';
 import * as OrderEvents from '../bot/modules/events/orders';
 import { IOrder } from '../models/order';
 import { HasTelegram } from '../bot/start';
@@ -24,6 +24,11 @@ interface PayViaPaymentRequestParams {
 }
 
 const payRequest = async ({ request, amount }: { request: string, amount: number }) => {
+  const startTime = Date.now();
+  const operationName = 'payRequest';
+  // Use configurable pathfinding timeout, default to 60 seconds
+  const pathfindingTimeout = parseInt(process.env.LN_PATHFINDING_TIMEOUT || '60000');
+
   try {
     const invoice = parsePaymentRequest({ request });
     if (!invoice) return false;
@@ -35,11 +40,10 @@ const payRequest = async ({ request, amount }: { request: string, amount: number
       throw new Error("Environment variable MAX_ROUTING_FEE is not defined");
     // We need to set a max fee amount
     const maxFee = amount * parseFloat(maxRoutingFee);
-
     const params : PayViaPaymentRequestParams = {
       lnd,
       request,
-      pathfinding_timeout: 60000,
+      pathfinding_timeout: pathfindingTimeout,
     };
     // If the invoice doesn't have amount we add it to the params
     if (!invoice.tokens) params.tokens = amount;
@@ -51,12 +55,35 @@ const payRequest = async ({ request, amount }: { request: string, amount: number
     // Delete all routing reputations to clear pathfinding memory
     await deleteForwardingReputations({ lnd });
 
+    logger.info(`Starting payment for ${amount} sats with ${pathfindingTimeout}ms timeout`);
     const payment = await payViaPaymentRequest(params);
-
+
+    logOperationDuration(operationName, startTime, true);
     return payment;
-  } catch (error) {
-    logger.error(`payRequest: ${error}`);
-    return false;
+  } catch (error: any) {
+    const errorMessage = error.toString();
+
+    logOperationDuration(operationName, startTime, false);
+
+    // Enhanced error handling for different timeout scenarios
+    if (errorMessage.includes('TimeoutError') || errorMessage.includes('timed out')) {
+      logTimeout('payRequest', pathfindingTimeout, error);
+      logger.error(`payRequest timeout after ${pathfindingTimeout}ms: ${errorMessage}`);
+      return { error: 'TIMEOUT', message: errorMessage };
+    }
+
+    if (errorMessage.includes('UnknownPaymentHash') || errorMessage.includes('PaymentPathfindingFailedToFindPossibleRoute')) {
+      logger.error(`payRequest routing failed: ${errorMessage}`);
+      return { error: 'ROUTING_FAILED', message: errorMessage };
+    }
+
+    if (errorMessage.includes('InsufficientBalance')) {
+      logger.error(`payRequest insufficient balance: ${errorMessage}`);
+      return { error: 'INSUFFICIENT_BALANCE', message: errorMessage };
+    }
+
+    logger.error(`payRequest unexpected error: ${errorMessage}`);
+    return { error: 'UNKNOWN', message: errorMessage };
   }
 };
 
@@ -104,14 +131,34 @@ const payToBuyer = async (bot: HasTelegram, order: IOrder) => {
       );
       await messages.rateUserMessage(bot, buyerUser, order, i18nCtx);
     } else {
-      await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+      // Handle different types of payment failures
+      if (payment && typeof payment === 'object' && 'error' in payment) {
+        const errorType = payment.error as string;
+
+        if (errorType === 'TIMEOUT') {
+          logger.warn(`Payment timeout for order ${order._id}, will retry later`);
+          await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+        } else if (errorType === 'ROUTING_FAILED') {
+          logger.warn(`Routing failed for order ${order._id}, will retry with cleared reputation`);
+          await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+        } else {
+          logger.error(`Payment failed for order ${order._id} with error: ${errorType}`);
+          await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+        }
+      } else {
+        await messages.invoicePaymentFailedMessage(bot, buyerUser, i18nCtx);
+      }
+
+      // Create pending payment for retry
       const pp = new PendingPayment({
         amount: order.amount,
         payment_request: order.buyer_invoice,
         user_id: buyerUser._id,
         description: order.description,
         hash: order.hash,
         order_id: order._id,
+        attempts: 1,
+        last_error: payment && typeof payment === 'object' && 'error' in payment ? payment.error as string : 'UNKNOWN',
       });
       await pp.save();
     }

logger.ts

@@ -23,5 +23,31 @@ const logger = winston.createLogger({
   exitOnError: false,
 });
 
+// Enhanced timeout monitoring utilities
+export const logTimeout = (operation: string, timeout: number, error?: any) => {
+  const logData = {
+    operation,
+    timeout_ms: timeout,
+    timestamp: new Date().toISOString(),
+    error: error?.toString() || 'Unknown timeout error'
+  };
+  logger.error(`TIMEOUT_MONITOR: ${JSON.stringify(logData)}`);
+};
+
+export const logOperationDuration = (operation: string, startTime: number, success: boolean = true) => {
+  const duration = Date.now() - startTime;
+  const logData = {
+    operation,
+    duration_ms: duration,
+    success,
+    timestamp: new Date().toISOString()
+  };
+
+  if (duration > 30000) { // Log slow operations (>30s)
+    logger.warn(`SLOW_OPERATION: ${JSON.stringify(logData)}`);
+  } else {
+    logger.info(`OPERATION_DURATION: ${JSON.stringify(logData)}`);
+  }
+};
 
 export { logger };

models/pending_payment.ts

@@ -13,6 +13,8 @@ export interface IPendingPayment extends Document {
     user_id: string;
     order_id: string;
     community_id: string;
+    last_error: string;
+    next_retry: Date;
 }
 
 
@@ -37,6 +39,8 @@ const PendingPaymentSchema = new Schema<IPendingPayment>({
     user_id: { type: String },
     order_id: { type: String },
     community_id: { type: String },
+    last_error: { type: String, default: '' },
+    next_retry: { type: Date, default: () => new Date(Date.now() + 5 * 60 * 1000) },
 });