Fix bots restart/stop

This commit is contained in:
2025-08-13 22:22:22 +07:00
parent 46a6cdcd87
commit 9d0c7cf834

View File

@@ -103,7 +103,7 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
private async Task ResumeBotInternalAsync() private async Task ResumeBotInternalAsync()
{ {
// The core of this method remains idempotent thanks to the _tradingBot null check // Idempotency check
if (_tradingBot != null) if (_tradingBot != null)
{ {
return; return;
@@ -111,21 +111,28 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
try try
{ {
// Load state from persisted grain state // Create and initialize trading bot instance
_tradingBot = CreateTradingBotInstance(_state.State.Config); _tradingBot = CreateTradingBotInstance(_state.State.Config);
LoadStateIntoBase();
await _tradingBot.Start(); await _tradingBot.Start();
// Set startup time when bot actually starts running
_state.State.StartupTime = DateTime.UtcNow;
await _state.WriteStateAsync();
// Start the in-memory timer and persistent reminder // Start the in-memory timer and persistent reminder
RegisterAndStartTimer(); RegisterAndStartTimer();
await RegisterReminder(); await RegisterReminder();
// Update both database and registry status
await SaveBotAsync(BotStatus.Up); await SaveBotAsync(BotStatus.Up);
await UpdateBotRegistryStatus(BotStatus.Up);
_logger.LogInformation("LiveTradingBotGrain {GrainId} resumed successfully", this.GetPrimaryKey()); _logger.LogInformation("LiveTradingBotGrain {GrainId} resumed successfully", this.GetPrimaryKey());
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex, "Failed to resume bot {GrainId}", this.GetPrimaryKey()); _logger.LogError(ex, "Failed to resume bot {GrainId}", this.GetPrimaryKey());
// If resume fails, update the status to Down via the registry and stop _tradingBot = null; // Clean up on failure
await UpdateBotRegistryStatus(BotStatus.Down); await UpdateBotRegistryStatus(BotStatus.Down);
throw; throw;
} }
@@ -137,7 +144,7 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
var botId = this.GetPrimaryKey(); var botId = this.GetPrimaryKey();
var status = await botRegistry.GetBotStatus(botId); var status = await botRegistry.GetBotStatus(botId);
// This is the new idempotency check, using the registry as the source of truth // Check if already running
if (status == BotStatus.Up && _tradingBot != null) if (status == BotStatus.Up && _tradingBot != null)
{ {
await RegisterReminder(); await RegisterReminder();
@@ -147,17 +154,14 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
try try
{ {
// Resume the bot using the internal logic // Resume the bot - this handles registry status update internally
await ResumeBotInternalAsync(); await ResumeBotInternalAsync();
// Update registry status (if it was previously 'Down')
await UpdateBotRegistryStatus(BotStatus.Up);
_logger.LogInformation("LiveTradingBotGrain {GrainId} started successfully", this.GetPrimaryKey()); _logger.LogInformation("LiveTradingBotGrain {GrainId} started successfully", this.GetPrimaryKey());
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex, "Failed to start LiveTradingBotGrain {GrainId}", this.GetPrimaryKey()); _logger.LogError(ex, "Failed to start LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
// Ensure registry status is correct even on failure // Ensure registry status is correct on failure
await UpdateBotRegistryStatus(BotStatus.Down); await UpdateBotRegistryStatus(BotStatus.Down);
throw; throw;
} }
@@ -253,7 +257,17 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TradingBotBase>>(); var logger = scope.ServiceProvider.GetRequiredService<ILogger<TradingBotBase>>();
var tradingBot = new TradingBotBase(logger, _scopeFactory, config); var tradingBot = new TradingBotBase(logger, _scopeFactory, config);
// Restore state from grain state // Load state into the trading bot instance
LoadStateIntoTradingBot(tradingBot);
return tradingBot;
}
/// <summary>
/// Loads grain state into a trading bot instance
/// </summary>
private void LoadStateIntoTradingBot(TradingBotBase tradingBot)
{
tradingBot.Signals = _state.State.Signals; tradingBot.Signals = _state.State.Signals;
tradingBot.Positions = _state.State.Positions; tradingBot.Positions = _state.State.Positions;
tradingBot.WalletBalances = _state.State.WalletBalances; tradingBot.WalletBalances = _state.State.WalletBalances;
@@ -262,8 +276,6 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
tradingBot.Identifier = _state.State.Identifier; tradingBot.Identifier = _state.State.Identifier;
tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime; tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime;
tradingBot.LastCandle = _state.State.LastCandle; tradingBot.LastCandle = _state.State.LastCandle;
return tradingBot;
} }
@@ -340,13 +352,28 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
{ {
if (_tradingBot == null) if (_tradingBot == null)
{ {
throw new InvalidOperationException("Bot is not running"); // For non-running bots, return data from grain state only
return Task.FromResult(new LiveTradingBotModel
{
Identifier = _state.State.Identifier,
Name = _state.State.Config?.Name ?? "Unknown",
Config = _state.State.Config,
Positions = _state.State.Positions ?? new Dictionary<Guid, Position>(),
Signals = _state.State.Signals,
WalletBalances = _state.State.WalletBalances ?? new Dictionary<DateTime, decimal>(),
ProfitAndLoss = 0, // Calculate from persisted positions if needed
WinRate = 0, // Calculate from persisted positions if needed
ExecutionCount = _state.State.ExecutionCount,
StartupTime = _state.State.StartupTime,
CreateDate = _state.State.CreateDate
});
} }
// For running bots, return live data
return Task.FromResult(new LiveTradingBotModel return Task.FromResult(new LiveTradingBotModel
{ {
Identifier = _state.State.Identifier, Identifier = _state.State.Identifier,
Name = _state.State.Name, Name = _state.State.Config?.Name ?? "Unknown",
Config = _state.State.Config, Config = _state.State.Config,
Positions = _tradingBot.Positions, Positions = _tradingBot.Positions,
Signals = _tradingBot.Signals, Signals = _tradingBot.Signals,
@@ -365,22 +392,6 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
} }
} }
private void LoadStateIntoBase()
{
if (_tradingBot == null)
_tradingBot = CreateTradingBotInstance(_state.State.Config);
if (_tradingBot == null) throw new InvalidOperationException("TradingBotBase instance could not be created");
_tradingBot.Signals = _state.State.Signals;
_tradingBot.Positions = _state.State.Positions;
_tradingBot.WalletBalances = _state.State.WalletBalances;
_tradingBot.PreloadedCandlesCount = _state.State.PreloadedCandlesCount;
_tradingBot.ExecutionCount = _state.State.ExecutionCount;
_tradingBot.Identifier = _state.State.Identifier;
_tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime;
_tradingBot.LastCandle = _state.State.LastCandle;
}
private void SyncStateFromBase() private void SyncStateFromBase()
{ {
@@ -399,15 +410,23 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
public async Task<bool> UpdateConfiguration(TradingBotConfig newConfig) public async Task<bool> UpdateConfiguration(TradingBotConfig newConfig)
{ {
if (_tradingBot == null) if (_tradingBot == null)
LoadStateIntoBase(); {
// For non-running bots, just update the configuration
_state.State.Config = newConfig;
await _state.WriteStateAsync();
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
var status = await botRegistry.GetBotStatus(this.GetPrimaryKey());
await SaveBotAsync(status);
return true;
}
var result = await _tradingBot!.UpdateConfiguration(newConfig); var result = await _tradingBot.UpdateConfiguration(newConfig);
if (result) if (result)
{ {
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0); var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
var botId = this.GetPrimaryKey(); var status = await botRegistry.GetBotStatus(this.GetPrimaryKey());
var status = await botRegistry.GetBotStatus(botId);
_state.State.Config = newConfig; _state.State.Config = newConfig;
await _state.WriteStateAsync(); await _state.WriteStateAsync();
await SaveBotAsync(status); await SaveBotAsync(status);
@@ -418,6 +437,10 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
public Task<Account> GetAccount() public Task<Account> GetAccount()
{ {
if (_tradingBot == null)
{
throw new InvalidOperationException("Bot is not running - cannot get account information");
}
return Task.FromResult(_tradingBot.Account); return Task.FromResult(_tradingBot.Account);
} }
@@ -453,8 +476,29 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
public async Task RestartAsync() public async Task RestartAsync()
{ {
await StopAsync(); _logger.LogInformation("Restarting LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
await StartAsync();
try
{
await StopAsync();
// Add a small delay to ensure stop operations complete
await Task.Delay(100);
await StartAsync();
// Verify the restart was successful
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
var finalStatus = await botRegistry.GetBotStatus(this.GetPrimaryKey());
_logger.LogInformation("LiveTradingBotGrain {GrainId} restart completed with final status: {Status}",
this.GetPrimaryKey(), finalStatus);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to restart LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
throw;
}
} }
public async Task DeleteAsync() public async Task DeleteAsync()
@@ -489,21 +533,50 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
} }
/// <summary> /// <summary>
/// Updates the bot status in the central BotRegistry /// Updates the bot status in the central BotRegistry with retry logic
/// </summary> /// </summary>
private async Task UpdateBotRegistryStatus(BotStatus status) private async Task UpdateBotRegistryStatus(BotStatus status)
{ {
try const int maxRetries = 3;
var botId = this.GetPrimaryKey();
for (int attempt = 1; attempt <= maxRetries; attempt++)
{ {
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0); try
var botId = this.GetPrimaryKey(); {
await botRegistry.UpdateBotStatus(botId, status); var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
_logger.LogDebug("Bot {BotId} status updated to {Status} in BotRegistry", botId, status); await botRegistry.UpdateBotStatus(botId, status);
}
catch (Exception ex) // Verify the update was successful
{ var actualStatus = await botRegistry.GetBotStatus(botId);
_logger.LogError(ex, "Failed to update bot {BotId} status to {Status} in BotRegistry", this.GetPrimaryKey(), if (actualStatus == status)
status); {
_logger.LogDebug("Bot {BotId} status successfully updated to {Status} in BotRegistry (attempt {Attempt})",
botId, status, attempt);
return;
}
else
{
_logger.LogWarning("Bot {BotId} status update verification failed. Expected: {Expected}, Actual: {Actual} (attempt {Attempt})",
botId, status, actualStatus, attempt);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to update bot {BotId} status to {Status} in BotRegistry (attempt {Attempt})",
botId, status, attempt);
if (attempt == maxRetries)
{
throw;
}
}
// Wait before retry
if (attempt < maxRetries)
{
await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt));
}
} }
} }