From 349a4c3696e0410a9784d97b45d8f91cee8c0269 Mon Sep 17 00:00:00 2001 From: cryptooda Date: Mon, 6 Oct 2025 10:42:40 +0700 Subject: [PATCH] Update redis healthcheck --- src/Managing.Web3Proxy/README-REDIS.md | 54 +++++ src/Managing.Web3Proxy/package.json | 3 +- .../scripts/test-redis-captain.sh | 121 +++++++++++ .../scripts/test-redis-connection.ts | 189 ++++++++++++++++++ src/Managing.Web3Proxy/src/routes/home.ts | 2 - 5 files changed, 366 insertions(+), 3 deletions(-) create mode 100755 src/Managing.Web3Proxy/scripts/test-redis-captain.sh create mode 100644 src/Managing.Web3Proxy/scripts/test-redis-connection.ts diff --git a/src/Managing.Web3Proxy/README-REDIS.md b/src/Managing.Web3Proxy/README-REDIS.md index c6c241ee..59a84db7 100644 --- a/src/Managing.Web3Proxy/README-REDIS.md +++ b/src/Managing.Web3Proxy/README-REDIS.md @@ -56,3 +56,57 @@ For production deployments with multiple Web3Proxy instances: Idempotency keys are stored in Redis with the prefix `idempotency:` and have a TTL of 5 minutes. Example Redis key: `idempotency:123e4567-e89b-12d3-a456-426614174000` + +## Troubleshooting Redis Connection + +If you see "Redis unavailable, using in-memory fallback" in health checks: + +### Test Connectivity Locally + +Run the diagnostic tool locally: +```bash +npm run test:redis +``` + +### Test Connectivity on Captain + +1. SSH into your Captain server or use Captain's app console +2. Run the connectivity test script: +```bash +./scripts/test-redis-captain.sh +``` + +### Common Issues + +1. **DNS Resolution Failed** + - Check if the Redis hostname is correct + - Verify DNS is configured properly + - Try using IP address instead of hostname + +2. **Connection Timeout** + - Redis may not be running at the specified host + - Firewall may be blocking port 6379 + - Check network connectivity between services + +3. **Authentication Failed** + - Verify `REDIS_PASSWORD` matches Redis configuration + - For Redis URLs with embedded passwords, use format: `redis://:password@host:port` + - Check Redis logs for authentication failures + +4. **Redis Not Accepting Remote Connections** + - Redis `bind` directive must include `0.0.0.0` or the specific interface IP + - Edit `/etc/redis/redis.conf` and set: `bind 0.0.0.0 ::1` + - Restart Redis after config changes + +5. **Captain Service Discovery** + - For Redis deployed on Captain, use internal service name + - Format: `redis://srv-captain--redis:6379` + - Ensure both services are on the same Docker network + +### Health Check Status + +- `healthy`: Redis is connected and working properly +- `degraded`: Redis unavailable, using in-memory fallback (service still works) +- `unhealthy`: Service cannot function properly + +The service will automatically fall back to in-memory storage if Redis is unavailable. This is safe for single-instance deployments. diff --git a/src/Managing.Web3Proxy/package.json b/src/Managing.Web3Proxy/package.json index b3edc78e..e6466446 100644 --- a/src/Managing.Web3Proxy/package.json +++ b/src/Managing.Web3Proxy/package.json @@ -22,7 +22,8 @@ "db:create": "tsx --env-file=.env ./scripts/create-database.ts", "db:drop": "tsx --env-file=.env ./scripts/drop-database.ts", "db:migrate": "tsx --env-file=.env ./scripts/migrate.ts", - "db:seed": "tsx --env-file=.env ./scripts/seed-database.ts" + "db:seed": "tsx --env-file=.env ./scripts/seed-database.ts", + "test:redis": "tsx --env-file=.env ./scripts/test-redis-connection.ts" }, "keywords": [], "author": "Oda", diff --git a/src/Managing.Web3Proxy/scripts/test-redis-captain.sh b/src/Managing.Web3Proxy/scripts/test-redis-captain.sh new file mode 100755 index 00000000..8e6aa780 --- /dev/null +++ b/src/Managing.Web3Proxy/scripts/test-redis-captain.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Redis connectivity test for Captain/production environments +# This script tests basic network connectivity to Redis + +echo "🔍 Redis Connectivity Test for Captain" +echo "======================================" +echo "" + +# Extract values from REDIS_URL if set +if [ -n "$REDIS_URL" ]; then + echo "REDIS_URL is set: $REDIS_URL" + + # Parse the URL to extract host and port + # Format: redis://[username]:[password]@[host]:[port] + REDIS_HOST=$(echo $REDIS_URL | sed -n 's/.*@\([^:]*\).*/\1/p') + REDIS_PORT=$(echo $REDIS_URL | sed -n 's/.*:\([0-9]*\)$/\1/p') + + if [ -z "$REDIS_PORT" ]; then + REDIS_PORT="6379" + fi + + echo "Parsed host: $REDIS_HOST" + echo "Parsed port: $REDIS_PORT" +else + echo "❌ REDIS_URL environment variable not set" + echo "Please set REDIS_URL before running this test" + exit 1 +fi + +echo "" +echo "📋 Running connectivity tests..." +echo "" + +# Test 1: DNS resolution +echo "⏳ Test 1: DNS Resolution" +if host $REDIS_HOST > /dev/null 2>&1; then + IP=$(host $REDIS_HOST | awk '/has address/ { print $4 }' | head -n 1) + echo "✅ DNS resolved: $REDIS_HOST -> $IP" +else + echo "❌ DNS resolution failed for $REDIS_HOST" + echo " This host cannot be found. Check:" + echo " - Is the hostname correct?" + echo " - Is DNS configured properly?" + exit 1 +fi + +echo "" + +# Test 2: Ping (if available) +echo "⏳ Test 2: Ping Test" +if ping -c 3 -W 2 $REDIS_HOST > /dev/null 2>&1; then + echo "✅ Host is reachable via ping" +else + echo "⚠️ Ping failed or ICMP blocked (this is often normal)" + echo " Continuing with port tests..." +fi + +echo "" + +# Test 3: Port connectivity +echo "⏳ Test 3: TCP Port Connectivity" +if command -v nc > /dev/null 2>&1; then + if nc -zv -w 3 $REDIS_HOST $REDIS_PORT 2>&1 | grep -q succeeded; then + echo "✅ Port $REDIS_PORT is open and accepting connections" + else + echo "❌ Cannot connect to port $REDIS_PORT on $REDIS_HOST" + echo " Possible issues:" + echo " - Redis is not running" + echo " - Firewall blocking port $REDIS_PORT" + echo " - Redis not configured to accept remote connections" + echo " - Wrong host or port in REDIS_URL" + exit 1 + fi +elif command -v timeout > /dev/null 2>&1; then + if timeout 3 bash -c "cat < /dev/null > /dev/tcp/$REDIS_HOST/$REDIS_PORT" 2>/dev/null; then + echo "✅ Port $REDIS_PORT is open and accepting connections" + else + echo "❌ Cannot connect to port $REDIS_PORT on $REDIS_HOST" + exit 1 + fi +else + echo "⚠️ netcat (nc) not available, cannot test port connectivity" +fi + +echo "" + +# Test 4: Redis PING using redis-cli if available +echo "⏳ Test 4: Redis Protocol Test" +if command -v redis-cli > /dev/null 2>&1; then + if [ -n "$REDIS_PASSWORD" ]; then + if redis-cli -h $REDIS_HOST -p $REDIS_PORT -a "$REDIS_PASSWORD" --no-auth-warning ping 2>&1 | grep -q PONG; then + echo "✅ Redis PING successful (with authentication)" + else + echo "❌ Redis PING failed with authentication" + echo " Check if password is correct" + exit 1 + fi + else + if redis-cli -h $REDIS_HOST -p $REDIS_PORT ping 2>&1 | grep -q PONG; then + echo "✅ Redis PING successful (no authentication)" + else + echo "❌ Redis PING failed" + exit 1 + fi + fi +else + echo "⚠️ redis-cli not available, skipping protocol test" + echo " Install redis-tools to run this test: apt-get install redis-tools" +fi + +echo "" +echo "======================================" +echo "✅ All available tests passed!" +echo "" +echo "💡 If your app still cannot connect:" +echo "1. Check that Redis bind address is 0.0.0.0 (not 127.0.0.1)" +echo "2. Verify password in REDIS_PASSWORD matches Redis config" +echo "3. Check Redis logs for authentication failures" +echo "4. Ensure firewall allows connections from Captain's IP" +echo "" + diff --git a/src/Managing.Web3Proxy/scripts/test-redis-connection.ts b/src/Managing.Web3Proxy/scripts/test-redis-connection.ts new file mode 100644 index 00000000..95d78d8c --- /dev/null +++ b/src/Managing.Web3Proxy/scripts/test-redis-connection.ts @@ -0,0 +1,189 @@ +#!/usr/bin/env tsx +/** + * Redis Connection Diagnostic Tool + * Tests Redis connectivity with detailed error reporting + */ + +import {createClient} from 'redis'; + +async function testRedisConnection() { + const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379'; + const redisPassword = process.env.REDIS_PASSWORD; + + console.log('🔍 Redis Connection Diagnostics'); + console.log('================================'); + console.log('Redis URL:', redisUrl); + console.log('Password set:', redisPassword ? 'Yes (****)' : 'No'); + console.log(''); + + // Parse URL to extract host and port + try { + const url = new URL(redisUrl); + console.log('📋 Connection Details:'); + console.log(' Protocol:', url.protocol); + console.log(' Host:', url.hostname); + console.log(' Port:', url.port || '6379'); + console.log(' Username:', url.username || '(none)'); + console.log(''); + } catch (e) { + console.error('❌ Invalid Redis URL format'); + process.exit(1); + } + + let redisClient = null; + let testsPassed = 0; + let testsFailed = 0; + + try { + console.log('⏳ Test 1: Creating Redis client...'); + const redisConfig: any = { + url: redisUrl, + socket: { + connectTimeout: 5000, + reconnectStrategy: false + } + }; + + if (redisPassword) { + redisConfig.password = redisPassword; + } + + redisClient = createClient(redisConfig); + console.log('✅ Redis client created successfully'); + testsPassed++; + console.log(''); + + // Set up event listeners for diagnostics + redisClient.on('error', (err) => { + console.error('❌ Redis client error:', err.message); + }); + + redisClient.on('connect', () => { + console.log('🔌 Connected to Redis'); + }); + + redisClient.on('ready', () => { + console.log('✅ Redis client ready'); + }); + + console.log('⏳ Test 2: Connecting to Redis...'); + const connectStart = Date.now(); + await redisClient.connect(); + const connectTime = Date.now() - connectStart; + console.log(`✅ Connected successfully in ${connectTime}ms`); + testsPassed++; + console.log(''); + + console.log('⏳ Test 3: Testing PING command...'); + const pingStart = Date.now(); + const pong = await redisClient.ping(); + const pingTime = Date.now() - pingStart; + console.log(`✅ PING successful: ${pong} (${pingTime}ms)`); + testsPassed++; + console.log(''); + + console.log('⏳ Test 4: Testing SET command...'); + const testKey = 'diagnostic-test-key'; + const testValue = JSON.stringify({ + timestamp: Date.now(), + test: true, + message: 'Diagnostic test from Web3Proxy' + }); + await redisClient.set(testKey, testValue, { EX: 10 }); + console.log('✅ SET command successful'); + testsPassed++; + console.log(''); + + console.log('⏳ Test 5: Testing GET command...'); + const retrievedValue = await redisClient.get(testKey); + if (retrievedValue) { + const parsed = JSON.parse(retrievedValue); + console.log('✅ GET command successful'); + console.log(' Retrieved value:', parsed); + testsPassed++; + } else { + console.error('❌ GET command returned null'); + testsFailed++; + } + console.log(''); + + console.log('⏳ Test 6: Testing DEL command...'); + await redisClient.del(testKey); + console.log('✅ DEL command successful'); + testsPassed++; + console.log(''); + + console.log('⏳ Test 7: Getting Redis server info...'); + const info = await redisClient.info('server'); + const serverInfo = info.split('\r\n').reduce((acc, line) => { + const [key, value] = line.split(':'); + if (key && value) { + acc[key] = value; + } + return acc; + }, {} as Record); + + console.log('✅ Server info retrieved:'); + console.log(` Redis version: ${serverInfo.redis_version}`); + console.log(` Uptime: ${serverInfo.uptime_in_seconds} seconds`); + console.log(` Connected clients: ${serverInfo.connected_clients}`); + console.log(` Used memory: ${serverInfo.used_memory_human}`); + testsPassed++; + console.log(''); + + } catch (error) { + console.error(''); + console.error('❌ Test failed with error:'); + console.error(' Type:', error instanceof Error ? error.constructor.name : 'Unknown'); + console.error(' Message:', error instanceof Error ? error.message : 'Unknown error'); + + if (error instanceof Error && error.stack) { + console.error(''); + console.error('Stack trace:'); + console.error(error.stack); + } + testsFailed++; + } finally { + if (redisClient && redisClient.isOpen) { + try { + console.log('⏳ Closing connection...'); + await redisClient.quit(); + console.log('✅ Connection closed successfully'); + } catch (closeError) { + console.error('❌ Error closing connection:', closeError); + } + } + } + + console.log(''); + console.log('================================'); + console.log('📊 Test Summary:'); + console.log(` Passed: ${testsPassed}`); + console.log(` Failed: ${testsFailed}`); + console.log(''); + + if (testsFailed > 0) { + console.log('❌ Redis connection tests failed'); + console.log(''); + console.log('💡 Troubleshooting tips:'); + console.log('1. Verify Redis is running: `redis-cli ping`'); + console.log('2. Check Redis host is reachable: `ping redis.kai.managing.live`'); + console.log('3. Check port is open: `nc -zv redis.kai.managing.live 6379`'); + console.log('4. Verify Redis bind address (should be 0.0.0.0, not 127.0.0.1)'); + console.log('5. Check firewall rules allow port 6379'); + console.log('6. Verify password is correct'); + console.log('7. Check Redis logs for connection attempts'); + console.log(''); + process.exit(1); + } else { + console.log('✅ All tests passed! Redis is working correctly.'); + process.exit(0); + } +} + +// Run the diagnostic +testRedisConnection().catch((error) => { + console.error('Unexpected error:', error); + process.exit(1); +}); + diff --git a/src/Managing.Web3Proxy/src/routes/home.ts b/src/Managing.Web3Proxy/src/routes/home.ts index b8dd2bbe..92109c4e 100644 --- a/src/Managing.Web3Proxy/src/routes/home.ts +++ b/src/Managing.Web3Proxy/src/routes/home.ts @@ -243,8 +243,6 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { message: 'Redis unavailable, using in-memory fallback', data: { errorType: error instanceof Error ? error.constructor.name : 'Unknown', - redisUrl: process.env.REDIS_URL || 'redis://localhost:6379', - hasPassword: !!process.env.REDIS_PASSWORD, note: 'Service will function normally with in-memory idempotency storage' } };