Improve CandleStore grain deactivating

This commit is contained in:
2025-10-10 22:09:30 +07:00
parent 82f8057ed1
commit bdb254809e
4 changed files with 58 additions and 11 deletions

View File

@@ -1802,7 +1802,8 @@ public class TradingBotBase : ITradingBot
} }
catch (Exception ex) catch (Exception ex)
{ {
await LogWarning($"Failed to update position status for signal {signalIdentifier}: {ex.Message}"); await LogWarning($"Failed to update position status for signal {signalIdentifier}: {ex.Message} {ex.StackTrace}");
SentrySdk.CaptureException(ex);
} }
} }
@@ -1911,7 +1912,6 @@ public class TradingBotBase : ITradingBot
return; return;
message = $"[{Config.Name}] {message}"; message = $"[{Config.Name}] {message}";
SentrySdk.CaptureException(new Exception(message));
try try
{ {

View File

@@ -70,20 +70,47 @@ public class CandleStoreGrain : Grain, ICandleStoreGrain, IAsyncObserver<Candle>
public override async Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) public override async Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken)
{ {
// Unsubscribe from the stream with proper error handling var grainKey = this.GetPrimaryKeyString();
_logger.LogInformation("CandleStoreGrain deactivating for key: {GrainKey}. Reason: {Reason}",
grainKey, reason.Description);
// Unsubscribe from the stream with proper error handling and timeout
if (_streamSubscription != null) if (_streamSubscription != null)
{ {
try try
{ {
// Use a timeout to prevent hanging during shutdown
using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
using var combinedCts = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken, timeoutCts.Token);
await _streamSubscription.UnsubscribeAsync(); await _streamSubscription.UnsubscribeAsync();
_logger.LogDebug("Successfully unsubscribed from stream for grain {GrainKey}", this.GetPrimaryKeyString()); _logger.LogDebug("Successfully unsubscribed from stream for grain {GrainKey}", grainKey);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Expected during shutdown - don't log as error
_logger.LogDebug("Stream unsubscription cancelled during shutdown for grain {GrainKey}", grainKey);
}
catch (TaskCanceledException)
{
// Expected during shutdown when pub-sub rendezvous grain is deactivated
_logger.LogDebug("Stream unsubscription timed out during shutdown for grain {GrainKey}", grainKey);
}
catch (Exception ex) when (
ex.GetType().Name == "OrleansMessageRejectionException" &&
(ex.Message.Contains("Forwarding failed") ||
ex.Message.Contains("Unable to create local activation")))
{
// Expected during shutdown when Orleans infrastructure is shutting down
_logger.LogDebug("Stream unsubscription failed due to Orleans shutdown for grain {GrainKey}: {Message}",
grainKey, ex.Message);
} }
catch (Exception ex) catch (Exception ex)
{ {
// Log the error but don't throw - this is common during shutdown when // Log other unexpected errors but don't throw - this is common during shutdown
// the pub-sub rendezvous grain may already be deactivated
_logger.LogWarning(ex, "Failed to unsubscribe from stream during deactivation for grain {GrainKey}. This is normal during shutdown.", _logger.LogWarning(ex, "Failed to unsubscribe from stream during deactivation for grain {GrainKey}. This is normal during shutdown.",
this.GetPrimaryKeyString()); grainKey);
} }
finally finally
{ {

View File

@@ -178,12 +178,32 @@ public static class ApiBootstrap
// Configure silo address for multi-server clustering // Configure silo address for multi-server clustering
options.SiloName = $"ManagingApi-{taskSlot}-{siloRole}"; options.SiloName = $"ManagingApi-{taskSlot}-{siloRole}";
Console.WriteLine($"Configuring silo with role: {siloRole}"); Console.WriteLine($"Configuring silo with role: {siloRole}");
})
.Configure<MessagingOptions>(options =>
{
// Increase timeout for grain deactivation during shutdown
options.ResponseTimeout = TimeSpan.FromSeconds(30);
})
.Configure<GrainCollectionOptions>(options =>
{
// Configure grain collection timeouts
options.CollectionAge = TimeSpan.FromMinutes(10);
}); });
} }
else else
{ {
// Fallback to localhost clustering for testing or when database is unavailable // Fallback to localhost clustering for testing or when database is unavailable
siloBuilder.UseLocalhostClustering(siloPort, gatewayPort); siloBuilder.UseLocalhostClustering(siloPort, gatewayPort)
.Configure<MessagingOptions>(options =>
{
// Increase timeout for grain deactivation during shutdown
options.ResponseTimeout = TimeSpan.FromSeconds(30);
})
.Configure<GrainCollectionOptions>(options =>
{
// Configure grain collection timeouts
options.CollectionAge = TimeSpan.FromMinutes(10);
});
} }
// Conditionally configure reminder service based on flag // Conditionally configure reminder service based on flag

View File

@@ -5,12 +5,12 @@ import {TradeDirection} from '../../src/generated/ManagingApiTypes'
test('GMX Position Closing', async (t) => { test('GMX Position Closing', async (t) => {
await t.test('should close position', async () => { await t.test('should close position', async () => {
const sdk = await getClientForAddress('0x932167388dD9aad41149b3cA23eBD489E2E2DD78') const sdk = await getClientForAddress('0x1aDD85ee6f327d20340A451A8210FB32c4c97504')
const result = await closeGmxPositionImpl( const result = await closeGmxPositionImpl(
sdk, sdk,
"ETH", "XRP",
TradeDirection.Short TradeDirection.Long
) )
console.log('Position closing result:', result) console.log('Position closing result:', result)
assert.ok(result, 'Position closing result should be defined') assert.ok(result, 'Position closing result should be defined')