From bdb254809e40077056df78805aa2a8b9161f5f78 Mon Sep 17 00:00:00 2001 From: cryptooda Date: Fri, 10 Oct 2025 22:09:30 +0700 Subject: [PATCH] Improve CandleStore grain deactivating --- .../Bots/TradingBotBase.cs | 4 +- .../Grains/CandleStoreGrain.cs | 37 ++++++++++++++++--- src/Managing.Bootstrap/ApiBootstrap.cs | 22 ++++++++++- .../test/plugins/close-position.test.ts | 6 +-- 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/src/Managing.Application/Bots/TradingBotBase.cs b/src/Managing.Application/Bots/TradingBotBase.cs index 46f9ced1..c44e486f 100644 --- a/src/Managing.Application/Bots/TradingBotBase.cs +++ b/src/Managing.Application/Bots/TradingBotBase.cs @@ -1802,7 +1802,8 @@ public class TradingBotBase : ITradingBot } catch (Exception ex) { - await LogWarning($"Failed to update position status for signal {signalIdentifier}: {ex.Message}"); + await LogWarning($"Failed to update position status for signal {signalIdentifier}: {ex.Message} {ex.StackTrace}"); + SentrySdk.CaptureException(ex); } } @@ -1911,7 +1912,6 @@ public class TradingBotBase : ITradingBot return; message = $"[{Config.Name}] {message}"; - SentrySdk.CaptureException(new Exception(message)); try { diff --git a/src/Managing.Application/Grains/CandleStoreGrain.cs b/src/Managing.Application/Grains/CandleStoreGrain.cs index 61832fff..a767ebee 100644 --- a/src/Managing.Application/Grains/CandleStoreGrain.cs +++ b/src/Managing.Application/Grains/CandleStoreGrain.cs @@ -70,20 +70,47 @@ public class CandleStoreGrain : Grain, ICandleStoreGrain, IAsyncObserver public override async Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) { - // Unsubscribe from the stream with proper error handling + var grainKey = this.GetPrimaryKeyString(); + _logger.LogInformation("CandleStoreGrain deactivating for key: {GrainKey}. Reason: {Reason}", + grainKey, reason.Description); + + // Unsubscribe from the stream with proper error handling and timeout if (_streamSubscription != null) { try { + // Use a timeout to prevent hanging during shutdown + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var combinedCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, timeoutCts.Token); + await _streamSubscription.UnsubscribeAsync(); - _logger.LogDebug("Successfully unsubscribed from stream for grain {GrainKey}", this.GetPrimaryKeyString()); + _logger.LogDebug("Successfully unsubscribed from stream for grain {GrainKey}", grainKey); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Expected during shutdown - don't log as error + _logger.LogDebug("Stream unsubscription cancelled during shutdown for grain {GrainKey}", grainKey); + } + catch (TaskCanceledException) + { + // Expected during shutdown when pub-sub rendezvous grain is deactivated + _logger.LogDebug("Stream unsubscription timed out during shutdown for grain {GrainKey}", grainKey); + } + catch (Exception ex) when ( + ex.GetType().Name == "OrleansMessageRejectionException" && + (ex.Message.Contains("Forwarding failed") || + ex.Message.Contains("Unable to create local activation"))) + { + // Expected during shutdown when Orleans infrastructure is shutting down + _logger.LogDebug("Stream unsubscription failed due to Orleans shutdown for grain {GrainKey}: {Message}", + grainKey, ex.Message); } catch (Exception ex) { - // Log the error but don't throw - this is common during shutdown when - // the pub-sub rendezvous grain may already be deactivated + // Log other unexpected errors but don't throw - this is common during shutdown _logger.LogWarning(ex, "Failed to unsubscribe from stream during deactivation for grain {GrainKey}. This is normal during shutdown.", - this.GetPrimaryKeyString()); + grainKey); } finally { diff --git a/src/Managing.Bootstrap/ApiBootstrap.cs b/src/Managing.Bootstrap/ApiBootstrap.cs index 65954629..449ad980 100644 --- a/src/Managing.Bootstrap/ApiBootstrap.cs +++ b/src/Managing.Bootstrap/ApiBootstrap.cs @@ -178,12 +178,32 @@ public static class ApiBootstrap // Configure silo address for multi-server clustering options.SiloName = $"ManagingApi-{taskSlot}-{siloRole}"; Console.WriteLine($"Configuring silo with role: {siloRole}"); + }) + .Configure(options => + { + // Increase timeout for grain deactivation during shutdown + options.ResponseTimeout = TimeSpan.FromSeconds(30); + }) + .Configure(options => + { + // Configure grain collection timeouts + options.CollectionAge = TimeSpan.FromMinutes(10); }); } else { // Fallback to localhost clustering for testing or when database is unavailable - siloBuilder.UseLocalhostClustering(siloPort, gatewayPort); + siloBuilder.UseLocalhostClustering(siloPort, gatewayPort) + .Configure(options => + { + // Increase timeout for grain deactivation during shutdown + options.ResponseTimeout = TimeSpan.FromSeconds(30); + }) + .Configure(options => + { + // Configure grain collection timeouts + options.CollectionAge = TimeSpan.FromMinutes(10); + }); } // Conditionally configure reminder service based on flag diff --git a/src/Managing.Web3Proxy/test/plugins/close-position.test.ts b/src/Managing.Web3Proxy/test/plugins/close-position.test.ts index a109d2d9..ea5c8838 100644 --- a/src/Managing.Web3Proxy/test/plugins/close-position.test.ts +++ b/src/Managing.Web3Proxy/test/plugins/close-position.test.ts @@ -5,12 +5,12 @@ import {TradeDirection} from '../../src/generated/ManagingApiTypes' test('GMX Position Closing', async (t) => { await t.test('should close position', async () => { - const sdk = await getClientForAddress('0x932167388dD9aad41149b3cA23eBD489E2E2DD78') + const sdk = await getClientForAddress('0x1aDD85ee6f327d20340A451A8210FB32c4c97504') const result = await closeGmxPositionImpl( sdk, - "ETH", - TradeDirection.Short + "XRP", + TradeDirection.Long ) console.log('Position closing result:', result) assert.ok(result, 'Position closing result should be defined')