Fix bots restart/stop
This commit is contained in:
@@ -103,7 +103,7 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
|
|
||||||
private async Task ResumeBotInternalAsync()
|
private async Task ResumeBotInternalAsync()
|
||||||
{
|
{
|
||||||
// The core of this method remains idempotent thanks to the _tradingBot null check
|
// Idempotency check
|
||||||
if (_tradingBot != null)
|
if (_tradingBot != null)
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
@@ -111,21 +111,28 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// Load state from persisted grain state
|
// Create and initialize trading bot instance
|
||||||
_tradingBot = CreateTradingBotInstance(_state.State.Config);
|
_tradingBot = CreateTradingBotInstance(_state.State.Config);
|
||||||
LoadStateIntoBase();
|
|
||||||
await _tradingBot.Start();
|
await _tradingBot.Start();
|
||||||
|
|
||||||
|
// Set startup time when bot actually starts running
|
||||||
|
_state.State.StartupTime = DateTime.UtcNow;
|
||||||
|
await _state.WriteStateAsync();
|
||||||
|
|
||||||
// Start the in-memory timer and persistent reminder
|
// Start the in-memory timer and persistent reminder
|
||||||
RegisterAndStartTimer();
|
RegisterAndStartTimer();
|
||||||
await RegisterReminder();
|
await RegisterReminder();
|
||||||
|
|
||||||
|
// Update both database and registry status
|
||||||
await SaveBotAsync(BotStatus.Up);
|
await SaveBotAsync(BotStatus.Up);
|
||||||
|
await UpdateBotRegistryStatus(BotStatus.Up);
|
||||||
|
|
||||||
_logger.LogInformation("LiveTradingBotGrain {GrainId} resumed successfully", this.GetPrimaryKey());
|
_logger.LogInformation("LiveTradingBotGrain {GrainId} resumed successfully", this.GetPrimaryKey());
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogError(ex, "Failed to resume bot {GrainId}", this.GetPrimaryKey());
|
_logger.LogError(ex, "Failed to resume bot {GrainId}", this.GetPrimaryKey());
|
||||||
// If resume fails, update the status to Down via the registry and stop
|
_tradingBot = null; // Clean up on failure
|
||||||
await UpdateBotRegistryStatus(BotStatus.Down);
|
await UpdateBotRegistryStatus(BotStatus.Down);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
@@ -137,7 +144,7 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
var botId = this.GetPrimaryKey();
|
var botId = this.GetPrimaryKey();
|
||||||
var status = await botRegistry.GetBotStatus(botId);
|
var status = await botRegistry.GetBotStatus(botId);
|
||||||
|
|
||||||
// This is the new idempotency check, using the registry as the source of truth
|
// Check if already running
|
||||||
if (status == BotStatus.Up && _tradingBot != null)
|
if (status == BotStatus.Up && _tradingBot != null)
|
||||||
{
|
{
|
||||||
await RegisterReminder();
|
await RegisterReminder();
|
||||||
@@ -147,17 +154,14 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// Resume the bot using the internal logic
|
// Resume the bot - this handles registry status update internally
|
||||||
await ResumeBotInternalAsync();
|
await ResumeBotInternalAsync();
|
||||||
|
|
||||||
// Update registry status (if it was previously 'Down')
|
|
||||||
await UpdateBotRegistryStatus(BotStatus.Up);
|
|
||||||
_logger.LogInformation("LiveTradingBotGrain {GrainId} started successfully", this.GetPrimaryKey());
|
_logger.LogInformation("LiveTradingBotGrain {GrainId} started successfully", this.GetPrimaryKey());
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogError(ex, "Failed to start LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
|
_logger.LogError(ex, "Failed to start LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
|
||||||
// Ensure registry status is correct even on failure
|
// Ensure registry status is correct on failure
|
||||||
await UpdateBotRegistryStatus(BotStatus.Down);
|
await UpdateBotRegistryStatus(BotStatus.Down);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
@@ -253,7 +257,17 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TradingBotBase>>();
|
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TradingBotBase>>();
|
||||||
var tradingBot = new TradingBotBase(logger, _scopeFactory, config);
|
var tradingBot = new TradingBotBase(logger, _scopeFactory, config);
|
||||||
|
|
||||||
// Restore state from grain state
|
// Load state into the trading bot instance
|
||||||
|
LoadStateIntoTradingBot(tradingBot);
|
||||||
|
|
||||||
|
return tradingBot;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Loads grain state into a trading bot instance
|
||||||
|
/// </summary>
|
||||||
|
private void LoadStateIntoTradingBot(TradingBotBase tradingBot)
|
||||||
|
{
|
||||||
tradingBot.Signals = _state.State.Signals;
|
tradingBot.Signals = _state.State.Signals;
|
||||||
tradingBot.Positions = _state.State.Positions;
|
tradingBot.Positions = _state.State.Positions;
|
||||||
tradingBot.WalletBalances = _state.State.WalletBalances;
|
tradingBot.WalletBalances = _state.State.WalletBalances;
|
||||||
@@ -262,8 +276,6 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
tradingBot.Identifier = _state.State.Identifier;
|
tradingBot.Identifier = _state.State.Identifier;
|
||||||
tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime;
|
tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime;
|
||||||
tradingBot.LastCandle = _state.State.LastCandle;
|
tradingBot.LastCandle = _state.State.LastCandle;
|
||||||
|
|
||||||
return tradingBot;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -340,13 +352,28 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
{
|
{
|
||||||
if (_tradingBot == null)
|
if (_tradingBot == null)
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException("Bot is not running");
|
// For non-running bots, return data from grain state only
|
||||||
|
return Task.FromResult(new LiveTradingBotModel
|
||||||
|
{
|
||||||
|
Identifier = _state.State.Identifier,
|
||||||
|
Name = _state.State.Config?.Name ?? "Unknown",
|
||||||
|
Config = _state.State.Config,
|
||||||
|
Positions = _state.State.Positions ?? new Dictionary<Guid, Position>(),
|
||||||
|
Signals = _state.State.Signals,
|
||||||
|
WalletBalances = _state.State.WalletBalances ?? new Dictionary<DateTime, decimal>(),
|
||||||
|
ProfitAndLoss = 0, // Calculate from persisted positions if needed
|
||||||
|
WinRate = 0, // Calculate from persisted positions if needed
|
||||||
|
ExecutionCount = _state.State.ExecutionCount,
|
||||||
|
StartupTime = _state.State.StartupTime,
|
||||||
|
CreateDate = _state.State.CreateDate
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For running bots, return live data
|
||||||
return Task.FromResult(new LiveTradingBotModel
|
return Task.FromResult(new LiveTradingBotModel
|
||||||
{
|
{
|
||||||
Identifier = _state.State.Identifier,
|
Identifier = _state.State.Identifier,
|
||||||
Name = _state.State.Name,
|
Name = _state.State.Config?.Name ?? "Unknown",
|
||||||
Config = _state.State.Config,
|
Config = _state.State.Config,
|
||||||
Positions = _tradingBot.Positions,
|
Positions = _tradingBot.Positions,
|
||||||
Signals = _tradingBot.Signals,
|
Signals = _tradingBot.Signals,
|
||||||
@@ -365,22 +392,6 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void LoadStateIntoBase()
|
|
||||||
{
|
|
||||||
if (_tradingBot == null)
|
|
||||||
_tradingBot = CreateTradingBotInstance(_state.State.Config);
|
|
||||||
|
|
||||||
if (_tradingBot == null) throw new InvalidOperationException("TradingBotBase instance could not be created");
|
|
||||||
|
|
||||||
_tradingBot.Signals = _state.State.Signals;
|
|
||||||
_tradingBot.Positions = _state.State.Positions;
|
|
||||||
_tradingBot.WalletBalances = _state.State.WalletBalances;
|
|
||||||
_tradingBot.PreloadedCandlesCount = _state.State.PreloadedCandlesCount;
|
|
||||||
_tradingBot.ExecutionCount = _state.State.ExecutionCount;
|
|
||||||
_tradingBot.Identifier = _state.State.Identifier;
|
|
||||||
_tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime;
|
|
||||||
_tradingBot.LastCandle = _state.State.LastCandle;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SyncStateFromBase()
|
private void SyncStateFromBase()
|
||||||
{
|
{
|
||||||
@@ -399,15 +410,23 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
public async Task<bool> UpdateConfiguration(TradingBotConfig newConfig)
|
public async Task<bool> UpdateConfiguration(TradingBotConfig newConfig)
|
||||||
{
|
{
|
||||||
if (_tradingBot == null)
|
if (_tradingBot == null)
|
||||||
LoadStateIntoBase();
|
{
|
||||||
|
// For non-running bots, just update the configuration
|
||||||
|
_state.State.Config = newConfig;
|
||||||
|
await _state.WriteStateAsync();
|
||||||
|
|
||||||
|
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
|
||||||
|
var status = await botRegistry.GetBotStatus(this.GetPrimaryKey());
|
||||||
|
await SaveBotAsync(status);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
var result = await _tradingBot!.UpdateConfiguration(newConfig);
|
var result = await _tradingBot.UpdateConfiguration(newConfig);
|
||||||
|
|
||||||
if (result)
|
if (result)
|
||||||
{
|
{
|
||||||
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
|
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
|
||||||
var botId = this.GetPrimaryKey();
|
var status = await botRegistry.GetBotStatus(this.GetPrimaryKey());
|
||||||
var status = await botRegistry.GetBotStatus(botId);
|
|
||||||
_state.State.Config = newConfig;
|
_state.State.Config = newConfig;
|
||||||
await _state.WriteStateAsync();
|
await _state.WriteStateAsync();
|
||||||
await SaveBotAsync(status);
|
await SaveBotAsync(status);
|
||||||
@@ -418,6 +437,10 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
|
|
||||||
public Task<Account> GetAccount()
|
public Task<Account> GetAccount()
|
||||||
{
|
{
|
||||||
|
if (_tradingBot == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Bot is not running - cannot get account information");
|
||||||
|
}
|
||||||
return Task.FromResult(_tradingBot.Account);
|
return Task.FromResult(_tradingBot.Account);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -453,8 +476,29 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
|
|
||||||
public async Task RestartAsync()
|
public async Task RestartAsync()
|
||||||
{
|
{
|
||||||
await StopAsync();
|
_logger.LogInformation("Restarting LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
|
||||||
await StartAsync();
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await StopAsync();
|
||||||
|
|
||||||
|
// Add a small delay to ensure stop operations complete
|
||||||
|
await Task.Delay(100);
|
||||||
|
|
||||||
|
await StartAsync();
|
||||||
|
|
||||||
|
// Verify the restart was successful
|
||||||
|
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
|
||||||
|
var finalStatus = await botRegistry.GetBotStatus(this.GetPrimaryKey());
|
||||||
|
|
||||||
|
_logger.LogInformation("LiveTradingBotGrain {GrainId} restart completed with final status: {Status}",
|
||||||
|
this.GetPrimaryKey(), finalStatus);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Failed to restart LiveTradingBotGrain {GrainId}", this.GetPrimaryKey());
|
||||||
|
throw;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task DeleteAsync()
|
public async Task DeleteAsync()
|
||||||
@@ -489,21 +533,50 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Updates the bot status in the central BotRegistry
|
/// Updates the bot status in the central BotRegistry with retry logic
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private async Task UpdateBotRegistryStatus(BotStatus status)
|
private async Task UpdateBotRegistryStatus(BotStatus status)
|
||||||
{
|
{
|
||||||
try
|
const int maxRetries = 3;
|
||||||
|
var botId = this.GetPrimaryKey();
|
||||||
|
|
||||||
|
for (int attempt = 1; attempt <= maxRetries; attempt++)
|
||||||
{
|
{
|
||||||
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
|
try
|
||||||
var botId = this.GetPrimaryKey();
|
{
|
||||||
await botRegistry.UpdateBotStatus(botId, status);
|
var botRegistry = GrainFactory.GetGrain<ILiveBotRegistryGrain>(0);
|
||||||
_logger.LogDebug("Bot {BotId} status updated to {Status} in BotRegistry", botId, status);
|
await botRegistry.UpdateBotStatus(botId, status);
|
||||||
}
|
|
||||||
catch (Exception ex)
|
// Verify the update was successful
|
||||||
{
|
var actualStatus = await botRegistry.GetBotStatus(botId);
|
||||||
_logger.LogError(ex, "Failed to update bot {BotId} status to {Status} in BotRegistry", this.GetPrimaryKey(),
|
if (actualStatus == status)
|
||||||
status);
|
{
|
||||||
|
_logger.LogDebug("Bot {BotId} status successfully updated to {Status} in BotRegistry (attempt {Attempt})",
|
||||||
|
botId, status, attempt);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogWarning("Bot {BotId} status update verification failed. Expected: {Expected}, Actual: {Actual} (attempt {Attempt})",
|
||||||
|
botId, status, actualStatus, attempt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Failed to update bot {BotId} status to {Status} in BotRegistry (attempt {Attempt})",
|
||||||
|
botId, status, attempt);
|
||||||
|
|
||||||
|
if (attempt == maxRetries)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait before retry
|
||||||
|
if (attempt < maxRetries)
|
||||||
|
{
|
||||||
|
await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user