Oda/managing-apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add yield for orleans + 1min to 2h timeout for grain message + more exception send to sentry

Browse Source
This commit is contained in:
cryptooda
2025-11-07 12:40:24 +07:00
parent bc4c4c7684
commit 2dc34f07d8
3 changed files with 54 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/Managing.Application/Bots/Grains/BacktestTradingBotGrain.cs
View File

@@ -82,6 +82,10 @@ public class BacktestTradingBotGrain : Grain, IBacktestTradingBotGrain
var initialBalance = config.BotTradingBalance;
var fixedCandles = new HashSet<Candle>();
var lastYieldTime = DateTime.UtcNow;
const int yieldIntervalMs = 5000; // Yield control every 5 seconds to prevent timeout
const int candlesPerBatch = 100; // Process in batches to allow Orleans to check for cancellation
// Process all candles following the exact pattern from GetBacktestingResult
foreach (var candle in candles)
{
@@ -94,6 +98,16 @@ public class BacktestTradingBotGrain : Grain, IBacktestTradingBotGrain
currentCandle++;
// Yield control periodically to prevent Orleans from thinking the grain is stuck
// This helps prevent timeout issues during long-running backtests
var timeSinceLastYield = (DateTime.UtcNow - lastYieldTime).TotalMilliseconds;
if (timeSinceLastYield >= yieldIntervalMs || currentCandle % candlesPerBatch == 0)
{
// Yield control back to Orleans scheduler
await Task.Yield();
lastYieldTime = DateTime.UtcNow;
}
// Log progress every 10%
var currentPercentage = (currentCandle * 100) / totalCandles;
if (currentPercentage >= lastLoggedPercentage + 10)

36
src/Managing.Application/Grains/BundleBacktestGrain.cs
View File

@@ -126,6 +126,7 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
catch (Exception ex)
{
_logger.LogError(ex, "Error processing bundle backtest request {RequestId}", bundleRequest.RequestId);
SentrySdk.CaptureException(ex);
await HandleBundleRequestError(bundleRequest, backtester, ex);
}
}
@@ -133,19 +134,24 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
/// <summary>
/// Generates individual backtest requests from variant configuration
/// </summary>
private async Task<List<RunBacktestRequest>> GenerateBacktestRequestsFromVariants(BundleBacktestRequest bundleRequest)
private async Task<List<RunBacktestRequest>> GenerateBacktestRequestsFromVariants(
BundleBacktestRequest bundleRequest)
{
try
{
// Deserialize the variant configurations
var universalConfig = JsonSerializer.Deserialize<BundleBacktestUniversalConfig>(bundleRequest.UniversalConfigJson);
var universalConfig =
JsonSerializer.Deserialize<BundleBacktestUniversalConfig>(bundleRequest.UniversalConfigJson);
var dateTimeRanges = JsonSerializer.Deserialize<List<DateTimeRange>>(bundleRequest.DateTimeRangesJson);
var moneyManagementVariants = JsonSerializer.Deserialize<List<MoneyManagementVariant>>(bundleRequest.MoneyManagementVariantsJson);
var moneyManagementVariants =
JsonSerializer.Deserialize<List<MoneyManagementVariant>>(bundleRequest.MoneyManagementVariantsJson);
var tickerVariants = JsonSerializer.Deserialize<List<Ticker>>(bundleRequest.TickerVariantsJson);
if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null || tickerVariants == null)
if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null ||
tickerVariants == null)
{
_logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}", bundleRequest.RequestId);
_logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}",
bundleRequest.RequestId);
return new List<RunBacktestRequest>();
}
@@ -154,13 +160,15 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
_scopeFactory,
async service =>
{
var accounts = await service.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false);
var accounts =
await service.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false);
return accounts.FirstOrDefault();
});
if (firstAccount == null)
{
_logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}", bundleRequest.User.Id, bundleRequest.RequestId);
_logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}",
bundleRequest.User.Id, bundleRequest.RequestId);
return new List<RunBacktestRequest>();
}
@@ -179,7 +187,8 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
Timeframe = universalConfig.Timeframe,
IsForWatchingOnly = universalConfig.IsForWatchingOnly,
BotTradingBalance = universalConfig.BotTradingBalance,
Name = $"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}",
Name =
$"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}",
FlipPosition = universalConfig.FlipPosition,
CooldownPeriod = universalConfig.CooldownPeriod,
MaxLossStreak = universalConfig.MaxLossStreak,
@@ -204,15 +213,16 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
WatchOnly = universalConfig.WatchOnly,
Save = universalConfig.Save,
WithCandles = false, // Bundle backtests never return candles
MoneyManagement = mmVariant.MoneyManagement != null ?
new MoneyManagement
MoneyManagement = mmVariant.MoneyManagement != null
? new MoneyManagement
{
Name = mmVariant.MoneyManagement.Name,
Timeframe = mmVariant.MoneyManagement.Timeframe,
StopLoss = mmVariant.MoneyManagement.StopLoss,
TakeProfit = mmVariant.MoneyManagement.TakeProfit,
Leverage = mmVariant.MoneyManagement.Leverage
} : null
}
: null
};
backtestRequests.Add(backtestRequest);
@@ -224,7 +234,8 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
}
catch (Exception ex)
{
_logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}", bundleRequest.RequestId);
_logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}",
bundleRequest.RequestId);
return new List<RunBacktestRequest>();
}
}
@@ -267,6 +278,7 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
index + 1, bundleRequest.RequestId);
bundleRequest.FailedBacktests++;
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
SentrySdk.CaptureException(ex);
}
}

18
src/Managing.Bootstrap/ApiBootstrap.cs
View File

@@ -238,7 +238,8 @@ public static class ApiBootstrap
.Configure<MessagingOptions>(options =>
{
// Configure messaging for better reliability with increased timeouts
options.ResponseTimeout = TimeSpan.FromSeconds(60);
// Set to 2 hours to support long-running backtests that can take 47+ minutes
options.ResponseTimeout = TimeSpan.FromHours(2);
options.DropExpiredMessages = true;
})
.Configure<ClusterMembershipOptions>(options =>
@@ -265,7 +266,9 @@ public static class ApiBootstrap
siloBuilder.Configure<GrainCollectionOptions>(options =>
{
// Enable grain collection for active grains
options.CollectionAge = TimeSpan.FromMinutes(10);
// Set to 2.5 hours to allow long-running backtests (up to 2 hours) to complete
// without being collected prematurely
options.CollectionAge = TimeSpan.FromMinutes(150); // 2.5 hours
options.CollectionQuantum = TimeSpan.FromMinutes(1);
});
}
@@ -281,7 +284,16 @@ public static class ApiBootstrap
}
siloBuilder
.ConfigureLogging(logging => logging.SetMinimumLevel(LogLevel.Information));
.ConfigureLogging(logging =>
{
logging.SetMinimumLevel(LogLevel.Information);
// Reduce verbosity of Orleans status update messages for long-running operations
// These are informational and can be very verbose during long backtests
logging.AddFilter("Orleans.Runtime.Messaging.IncomingMessageAcceptor", LogLevel.Warning);
logging.AddFilter("Orleans.Runtime.Messaging.MessageCenter", LogLevel.Warning);
// Keep important Orleans logs but reduce status update noise
logging.AddFilter("Microsoft.Orleans.Runtime.Messaging", LogLevel.Warning);
});
// Only enable dashboard in development to avoid shutdown issues
if (!isProduction)