Add retry + idempotency on trading when try + add more tts

2025-09-20 02:28:16 +07:00
parent cb1252214a
commit d58672f879
15 changed files with 637 additions and 78 deletions
--- a/src/Managing.Web3Proxy/README-REDIS.md
+++ b/src/Managing.Web3Proxy/README-REDIS.md
@@ -0,0 +1,58 @@
+# Web3Proxy Redis Configuration
+
+## Environment Variables
+
+The Web3Proxy service now uses Redis for distributed idempotency storage across multiple instances.
+
+### Required Environment Variables
+
+- `REDIS_URL`: Redis connection string (default: `redis://localhost:6379`)
+- `REDIS_PASSWORD`: Redis password (optional, for authenticated Redis instances)
+- `LOG_LEVEL`: Logging level (default: `info`)
+
+### Docker Configuration
+
+When running in Docker, set the Redis URL to:
+```
+REDIS_URL=redis://redis:6379
+```
+
+For password-protected Redis instances:
+```
+REDIS_URL=redis://redis:6379
+REDIS_PASSWORD=your_redis_password
+```
+
+### Production Configuration
+
+For production deployments with password-protected Redis:
+
+1. Set environment variables:
+   ```bash
+   export REDIS_URL=redis://your-redis-host:6379
+   export REDIS_PASSWORD=your_secure_password
+   ```
+
+2. Or use a connection string with embedded password:
+   ```
+   REDIS_URL=redis://:your_password@your-redis-host:6379
+   ```
+
+### Fallback Behavior
+
+If Redis is not available, the service will automatically fall back to in-memory storage with a warning message. This ensures the service continues to work even without Redis, but idempotency will only work within a single instance.
+
+### Production Deployment
+
+For production deployments with multiple Web3Proxy instances:
+
+1. Ensure Redis is running and accessible
+2. Set the `REDIS_URL` environment variable
+3. Monitor Redis connection status in logs
+4. Consider Redis clustering for high availability
+
+### Idempotency Key Format
+
+Idempotency keys are stored in Redis with the prefix `idempotency:` and have a TTL of 5 minutes.
+
+Example Redis key: `idempotency:123e4567-e89b-12d3-a456-426614174000`
--- a/src/Managing.Web3Proxy/package-lock.json
+++ b/src/Managing.Web3Proxy/package-lock.json
@@ -40,12 +40,14 @@
        "mysql2": "^3.11.3",
        "postgrator": "^8.0.0",
        "query-string": "^9.1.1",
+        "redis": "^5.8.2",
        "viem": "2.37.1",
        "vitest": "^3.0.8",
        "zod": "^3.24.2"
      },
      "devDependencies": {
        "@types/node": "^22.5.5",
+        "@types/redis": "^4.0.10",
        "c8": "^10.1.3",
        "eslint": "^9.11.0",
        "fastify-tsconfig": "^3.0.0",
@@ -1521,6 +1523,66 @@
        }
      }
    },
+    "node_modules/@redis/bloom": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.8.2.tgz",
+      "integrity": "sha512-855DR0ChetZLarblio5eM0yLwxA9Dqq50t8StXKp5bAtLT0G+rZ+eRzzqxl37sPqQKjUudSYypz55o6nNhbz0A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@redis/client": "^5.8.2"
+      }
+    },
+    "node_modules/@redis/client": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.8.2.tgz",
+      "integrity": "sha512-WtMScno3+eBpTac1Uav2zugXEoXqaU23YznwvFgkPwBQVwEHTDgOG7uEAObtZ/Nyn8SmAMbqkEubJaMOvnqdsQ==",
+      "license": "MIT",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@redis/json": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-5.8.2.tgz",
+      "integrity": "sha512-uxpVfas3I0LccBX9rIfDgJ0dBrUa3+0Gc8sEwmQQH0vHi7C1Rx1Qn8Nv1QWz5bohoeIXMICFZRcyDONvum2l/w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@redis/client": "^5.8.2"
+      }
+    },
+    "node_modules/@redis/search": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-5.8.2.tgz",
+      "integrity": "sha512-cNv7HlgayavCBXqPXgaS97DRPVWFznuzsAmmuemi2TMCx5scwLiP50TeZvUS06h/MG96YNPe6A0Zt57yayfxwA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@redis/client": "^5.8.2"
+      }
+    },
+    "node_modules/@redis/time-series": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-5.8.2.tgz",
+      "integrity": "sha512-g2NlHM07fK8H4k+613NBsk3y70R2JIM2dPMSkhIjl2Z17SYvaYKdusz85d7VYOrZBWtDrHV/WD2E3vGu+zni8A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@redis/client": "^5.8.2"
+      }
+    },
    "node_modules/@rollup/rollup-darwin-arm64": {
      "version": "4.35.0",
      "cpu": [
@@ -1769,6 +1831,16 @@
        "@types/pg": "*"
      }
    },
+    "node_modules/@types/redis": {
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/@types/redis/-/redis-4.0.10.tgz",
+      "integrity": "sha512-7CLy5b5fzzEGVcOccgZjoMlNpPhX6d10jEeRy2YWbFuaMNrSPc9ExRsMYsd+0VxvEHucf4EWx24Ja7cSU1FGUA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "redis": "*"
+      }
+    },
    "node_modules/@types/shimmer": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.2.0.tgz",
@@ -2728,6 +2800,15 @@
      "version": "2.2.0",
      "license": "MIT"
    },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/color-convert": {
      "version": "2.0.1",
      "license": "MIT",
@@ -6550,6 +6631,22 @@
      "version": "0.5.1",
      "license": "Apache-2.0"
    },
+    "node_modules/redis": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-5.8.2.tgz",
+      "integrity": "sha512-31vunZj07++Y1vcFGcnNWEf5jPoTkGARgfWI4+Tk55vdwHxhAvug8VEtW7Cx+/h47NuJTEg/JL77zAwC6E0OeA==",
+      "license": "MIT",
+      "dependencies": {
+        "@redis/bloom": "5.8.2",
+        "@redis/client": "5.8.2",
+        "@redis/json": "5.8.2",
+        "@redis/search": "5.8.2",
+        "@redis/time-series": "5.8.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
    "node_modules/reflect.getprototypeof": {
      "version": "1.0.10",
      "dev": true,
--- a/src/Managing.Web3Proxy/package.json
+++ b/src/Managing.Web3Proxy/package.json
@@ -59,12 +59,14 @@
    "mysql2": "^3.11.3",
    "postgrator": "^8.0.0",
    "query-string": "^9.1.1",
+    "redis": "^5.8.2",
    "viem": "2.37.1",
    "vitest": "^3.0.8",
    "zod": "^3.24.2"
  },
  "devDependencies": {
    "@types/node": "^22.5.5",
+    "@types/redis": "^4.0.10",
    "c8": "^10.1.3",
    "eslint": "^9.11.0",
    "fastify-tsconfig": "^3.0.0",
--- a/src/Managing.Web3Proxy/src/routes/home.ts
+++ b/src/Managing.Web3Proxy/src/routes/home.ts
@@ -2,6 +2,7 @@ import {FastifyPluginAsyncTypebox, Type} from '@fastify/type-provider-typebox'
 import {handleError} from '../utils/errorHandler.js'
 import {getClientForAddress} from '../plugins/custom/gmx.js'
 import {getPrivyClient} from '../plugins/custom/privy.js'
+import {createClient} from 'redis'

 const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
  fastify.get(
@@ -48,6 +49,11 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
              status: Type.String(),
              message: Type.String(),
              data: Type.Optional(Type.Any())
+            }),
+            redis: Type.Object({
+              status: Type.String(),
+              message: Type.String(),
+              data: Type.Optional(Type.Any())
            })
          })
        }),
@@ -59,9 +65,11 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
    }
  }, async function (request, reply) {
    try {
+      console.log('Checking health...')
      const checks = {
        privy: await checkPrivy(fastify),
-        gmx: await checkGmx()
+        gmx: await checkGmx(),
+        redis: await checkRedis()
      }

      // If any check failed, set status to degraded
@@ -150,6 +158,96 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
      };
    }
  }
+
+  // Helper function to check Redis connectivity for idempotency
+  async function checkRedis() {
+    let redisClient = null;
+    try {
+      const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
+      const redisPassword = process.env.REDIS_PASSWORD;
+
+      console.log('Redis URL:', redisUrl)
+      console.log('Redis Password:', redisPassword)
+      
+      // Create Redis client configuration
+      const redisConfig: any = { url: redisUrl };
+      // if (redisPassword) {
+      //   redisConfig.password = redisPassword;
+      // }
+      
+      redisClient = createClient(redisConfig);
+      
+      // Set up error handling
+      redisClient.on('error', (err) => {
+        console.error('Redis health check error:', err);
+      });
+      
+      // Connect to Redis
+      const startTime = Date.now();
+      await redisClient.connect();
+      const connectTime = Date.now() - startTime;
+      
+      // Test basic operations
+      const testKey = 'health-check-test';
+      const testValue = JSON.stringify({ timestamp: Date.now(), test: true });
+      
+      // Test SET operation
+      await redisClient.set(testKey, testValue, { EX: 10 }); // 10 second expiry
+      
+      // Test GET operation
+      const retrievedValue = await redisClient.get(testKey);
+      const getTime = Date.now() - startTime;
+      
+      // Test JSON parsing
+      const parsedValue = JSON.parse(retrievedValue as string);
+      
+      // Clean up test key
+      await redisClient.del(testKey);
+      
+      // Get Redis info
+      const info = await redisClient.info('server');
+      const serverInfo = info.split('\r\n').reduce((acc, line) => {
+        const [key, value] = line.split(':');
+        if (key && value) {
+          acc[key] = value;
+        }
+        return acc;
+      }, {} as Record<string, string>);
+      
+      return {
+        status: 'healthy',
+        message: 'Redis connection successful',
+        data: {
+          connectTimeMs: connectTime,
+          getTimeMs: getTime,
+          redisVersion: serverInfo.redis_version,
+          uptimeSeconds: serverInfo.uptime_in_seconds,
+          connectedClients: serverInfo.connected_clients,
+          usedMemory: serverInfo.used_memory_human,
+          hasPassword: !!redisPassword
+        }
+      };
+    } catch (error) {
+      return {
+        status: 'unhealthy',
+        message: `Redis connection failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        data: {
+          errorType: error instanceof Error ? error.constructor.name : 'Unknown',
+          redisUrl: process.env.REDIS_URL || 'redis://localhost:6379',
+          hasPassword: !!process.env.REDIS_PASSWORD
+        }
+      };
+    } finally {
+      // Always close the Redis connection
+      if (redisClient && redisClient.isOpen) {
+        try {
+          await redisClient.quit();
+        } catch (closeError) {
+          console.error('Error closing Redis connection in health check:', closeError);
+        }
+      }
+    }
+  }
 }

 export default plugin
--- a/src/Managing.Web3Proxy/src/server.ts
+++ b/src/Managing.Web3Proxy/src/server.ts
@@ -7,6 +7,7 @@

 import Fastify from 'fastify'
 import fp from 'fastify-plugin'
+import {createClient, RedisClientType} from 'redis'

 // Import library to exit fastify process, gracefully (if possible)
 import closeWithGrace from 'close-with-grace'
@@ -14,6 +15,94 @@ import closeWithGrace from 'close-with-grace'
 // Import your application as a normal plugin.
 import serviceApp from './app.js'

+// Idempotency storage using Redis
+interface IdempotencyEntry {
+  requestId: string
+  response: any
+  statusCode: number
+  timestamp: number
+  ttl: number // Time to live in milliseconds
+}
+
+let redisClient: RedisClientType | null = null
+const IDEMPOTENCY_TTL = 5 * 60 // 5 minutes TTL in seconds (Redis uses seconds)
+
+// Initialize Redis connection
+async function initializeRedis() {
+  try {
+    const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379'
+    const redisPassword = process.env.REDIS_PASSWORD
+
+    console.log('Redis URL:', redisUrl)
+    console.log('Redis Password:', redisPassword)
+    
+    // Create Redis client with password support
+    const redisConfig: any = { url: redisUrl }
+    
+    if (redisPassword) {
+      redisConfig.password = redisPassword
+    }
+    
+    redisClient = createClient(redisConfig)
+    
+    redisClient.on('error', (err) => {
+      console.error('Redis Client Error:', err)
+    })
+    
+    redisClient.on('connect', () => {
+      console.log('Connected to Redis for idempotency')
+    })
+    
+    redisClient.on('ready', () => {
+      console.log('Redis client ready for idempotency operations')
+    })
+    
+    await redisClient.connect()
+  } catch (error) {
+    console.error('Failed to connect to Redis:', error)
+    // Fallback to in-memory storage if Redis is not available
+    console.warn('Falling back to in-memory idempotency storage')
+  }
+}
+
+// Fallback in-memory storage for when Redis is not available
+const fallbackStore = new Map<string, IdempotencyEntry>()
+
+// Helper function to get idempotency entry
+async function getIdempotencyEntry(requestId: string): Promise<IdempotencyEntry | null> {
+  if (redisClient && redisClient.isOpen) {
+    try {
+      const data = await redisClient.get(`idempotency:${requestId}`)
+      if (data && typeof data === 'string') {
+        return JSON.parse(data)
+      }
+      return null
+    } catch (error) {
+      console.error('Redis get error:', error)
+      return null
+    }
+  } else {
+    // Fallback to in-memory storage
+    return fallbackStore.get(requestId) || null
+  }
+}
+
+// Helper function to set idempotency entry
+async function setIdempotencyEntry(requestId: string, entry: IdempotencyEntry): Promise<void> {
+  if (redisClient && redisClient.isOpen) {
+    try {
+      await redisClient.setEx(`idempotency:${requestId}`, IDEMPOTENCY_TTL, JSON.stringify(entry))
+    } catch (error) {
+      console.error('Redis set error:', error)
+      // Fallback to in-memory storage
+      fallbackStore.set(requestId, entry)
+    }
+  } else {
+    // Fallback to in-memory storage
+    fallbackStore.set(requestId, entry)
+  }
+}
+
 /**
 * Do not use NODE_ENV to determine what logger (or any env related feature) to use
 * @see {@link https://www.youtube.com/watch?v=HMM7GJC5E2o}
@@ -33,7 +122,7 @@ function getLoggerOptions () {
    }
  }

-  return { level: process.env.LOG_LEVEL ?? 'silent' }
+  return { level: process.env.LOG_LEVEL ?? 'info' } // Changed from 'silent' to 'info' for better debugging
 }

 const app = Fastify({
@@ -43,22 +132,108 @@ const app = Fastify({
      coerceTypes: 'array', // change type of data to match type keyword
      removeAdditional: 'all' // Remove additional body properties
    }
-  }
+  },
+  // Add connection and timeout settings for better resilience
+  connectionTimeout: 30000, // 30 seconds
+  keepAliveTimeout: 5000,   // 5 seconds
+  bodyLimit: 1048576,       // 1MB
+  maxParamLength: 200,      // 200 characters
+  // Add request timeout
+  requestTimeout: 30000,    // 30 seconds
 })

 async function init () {
+  // Initialize Redis connection
+  await initializeRedis()
+
+  // Add idempotency pre-handler hook
+  app.addHook('preHandler', async (request, reply) => {
+    // Only apply idempotency to POST requests (trading operations)
+    if (request.method !== 'POST') {
+      return
+    }
+
+    const requestId = request.headers['idempotency-key'] || request.headers['x-request-id']
+    
+    if (!requestId) {
+      // No idempotency key provided, continue normally
+      return
+    }
+
+    // Check if we've seen this request before
+    const existingEntry = await getIdempotencyEntry(requestId as string)
+    
+    if (existingEntry) {
+      // Check if entry is still valid (Redis TTL handles this, but double-check for fallback)
+      const now = Date.now()
+      if (now - existingEntry.timestamp <= existingEntry.ttl) {
+        app.log.info(`Idempotency: Returning cached response for request ${requestId}`)
+        
+        // Return the cached response
+        reply.code(existingEntry.statusCode)
+        return existingEntry.response
+      } else {
+        // Entry expired, remove it (only needed for fallback storage)
+        if (!redisClient || !redisClient.isOpen) {
+          fallbackStore.delete(requestId as string)
+        }
+      }
+    }
+
+    // Store the request ID for later use
+    request.idempotencyKey = requestId as string
+  })
+
+  // Add post-handler hook to store successful responses
+  app.addHook('onSend', async (request, reply, payload) => {
+    if (request.idempotencyKey && request.method === 'POST') {
+      const requestId = request.idempotencyKey
+      
+      // Only store successful responses (2xx status codes)
+      if (reply.statusCode >= 200 && reply.statusCode < 300) {
+        try {
+          const responseData = typeof payload === 'string' ? JSON.parse(payload) : payload
+          
+          const entry: IdempotencyEntry = {
+            requestId,
+            response: responseData,
+            statusCode: reply.statusCode,
+            timestamp: Date.now(),
+            ttl: IDEMPOTENCY_TTL * 1000 // Convert to milliseconds for consistency
+          }
+          
+          await setIdempotencyEntry(requestId, entry)
+          
+          app.log.info(`Idempotency: Stored response for request ${requestId}`)
+        } catch (error) {
+          app.log.error(`Idempotency: Failed to store response for request ${requestId}:`, error)
+        }
+      }
+    }
+  })
+
  // Register your application as a normal plugin.
  // fp must be used to override default error handler
  app.register(fp(serviceApp))

  // Delay is the number of milliseconds for the graceful close to finish
  closeWithGrace(
-    { delay: process.env.FASTIFY_CLOSE_GRACE_DELAY ?? 500 },
+    { delay: process.env.FASTIFY_CLOSE_GRACE_DELAY ?? 5000 }, // Increased from 500ms to 5s
    async ({ err }) => {
      if (err != null) {
        app.log.error(err)
      }

+      // Close Redis connection gracefully
+      if (redisClient && redisClient.isOpen) {
+        try {
+          await redisClient.quit()
+          console.log('Redis connection closed gracefully')
+        } catch (error) {
+          console.error('Error closing Redis connection:', error)
+        }
+      }
+
      await app.close()
    }
  )
@@ -66,8 +241,14 @@ async function init () {
  await app.ready()

  try {
-    // Start listening.
-    await app.listen({ port: 4111 })
+    // Start listening with better configuration
+    await app.listen({ 
+      port: 4111,
+      host: '0.0.0.0', // Listen on all interfaces
+      backlog: 511,    // Increase backlog for better connection handling
+    })
+    
+    app.log.info('Web3Proxy server started successfully on port 4111')
  } catch (err) {
    app.log.error(err)
    process.exit(1)
--- a/src/Managing.Web3Proxy/src/types/fastify.d.ts
+++ b/src/Managing.Web3Proxy/src/types/fastify.d.ts
@@ -0,0 +1,5 @@
+declare module 'fastify' {
+  interface FastifyRequest {
+    idempotencyKey?: string
+  }
+}