Add monitoring on queries with sentry alert + Fix check position list in db for backtest
This commit is contained in:
319
src/Managing.Api/Controllers/SqlMonitoringController.cs
Normal file
319
src/Managing.Api/Controllers/SqlMonitoringController.cs
Normal file
@@ -0,0 +1,319 @@
|
||||
using Managing.Application.Abstractions.Services;
|
||||
using Managing.Application.Shared;
|
||||
using Managing.Infrastructure.Databases.PostgreSql;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace Managing.Api.Controllers;
|
||||
|
||||
/// <summary>
|
||||
/// Controller for monitoring SQL query performance and detecting potential loops
|
||||
/// Provides endpoints to view query statistics and clear tracking data
|
||||
/// Requires admin authorization for access
|
||||
/// </summary>
|
||||
[ApiController]
|
||||
[Authorize]
|
||||
[Route("api/[controller]")]
|
||||
public class SqlMonitoringController : BaseController
|
||||
{
|
||||
private readonly SentrySqlMonitoringService _sentryMonitoringService;
|
||||
private readonly ManagingDbContext _context;
|
||||
private readonly ILogger<SqlMonitoringController> _logger;
|
||||
private readonly IAdminConfigurationService _adminService;
|
||||
|
||||
public SqlMonitoringController(
|
||||
SentrySqlMonitoringService sentryMonitoringService,
|
||||
ManagingDbContext context,
|
||||
ILogger<SqlMonitoringController> logger,
|
||||
IUserService userService,
|
||||
IAdminConfigurationService adminService) : base(userService)
|
||||
{
|
||||
_sentryMonitoringService = sentryMonitoringService;
|
||||
_context = context;
|
||||
_logger = logger;
|
||||
_adminService = adminService;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the current user is an admin
|
||||
/// </summary>
|
||||
/// <returns>True if the user is admin, False otherwise</returns>
|
||||
private async Task<bool> IsUserAdmin()
|
||||
{
|
||||
try
|
||||
{
|
||||
var user = await GetUser();
|
||||
if (user == null)
|
||||
return false;
|
||||
|
||||
return _adminService.IsUserAdmin(user.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error checking if user is admin");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets current SQL query execution statistics
|
||||
/// </summary>
|
||||
/// <returns>Query execution statistics</returns>
|
||||
[HttpGet("statistics")]
|
||||
public async Task<ActionResult<object>> GetQueryStatistics()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Check if user is admin
|
||||
if (!await IsUserAdmin())
|
||||
{
|
||||
return Forbid("Only administrators can access SQL monitoring statistics");
|
||||
}
|
||||
|
||||
var loopDetectionStats = _sentryMonitoringService.GetQueryStatistics();
|
||||
var contextStats = _context.GetQueryExecutionCounts();
|
||||
|
||||
var result = new
|
||||
{
|
||||
LoopDetectionStats = loopDetectionStats,
|
||||
ContextStats = contextStats,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
TotalTrackedQueries = loopDetectionStats.Count,
|
||||
ActiveQueries = loopDetectionStats.Count(kvp => kvp.Value.IsActive)
|
||||
};
|
||||
|
||||
_logger.LogInformation("[SQL-MONITORING] Query statistics retrieved: {Count} tracked queries", loopDetectionStats.Count);
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[SQL-MONITORING] Error retrieving query statistics");
|
||||
return StatusCode(500, "Error retrieving query statistics");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets potential loop alerts and performance issues
|
||||
/// </summary>
|
||||
/// <returns>List of potential issues</returns>
|
||||
[HttpGet("alerts")]
|
||||
public async Task<ActionResult<object>> GetAlerts()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Check if user is admin
|
||||
if (!await IsUserAdmin())
|
||||
{
|
||||
return Forbid("Only administrators can access SQL monitoring alerts");
|
||||
}
|
||||
|
||||
var stats = _sentryMonitoringService.GetQueryStatistics();
|
||||
var alerts = new List<object>();
|
||||
|
||||
foreach (var kvp in stats)
|
||||
{
|
||||
var stat = kvp.Value;
|
||||
var issues = new List<string>();
|
||||
|
||||
// Check for high execution frequency
|
||||
if (stat.ExecutionsPerMinute > 20)
|
||||
{
|
||||
issues.Add($"High frequency: {stat.ExecutionsPerMinute:F1} executions/minute");
|
||||
}
|
||||
|
||||
// Check for slow queries
|
||||
if (stat.AverageExecutionTime.TotalMilliseconds > 1000)
|
||||
{
|
||||
issues.Add($"Slow query: {stat.AverageExecutionTime.TotalMilliseconds:F0}ms average");
|
||||
}
|
||||
|
||||
// Check for many executions
|
||||
if (stat.ExecutionCount > 50)
|
||||
{
|
||||
issues.Add($"High count: {stat.ExecutionCount} total executions");
|
||||
}
|
||||
|
||||
if (issues.Any())
|
||||
{
|
||||
alerts.Add(new
|
||||
{
|
||||
Repository = stat.RepositoryName,
|
||||
Method = stat.MethodName,
|
||||
QueryPattern = stat.QueryPattern,
|
||||
Issues = issues,
|
||||
ExecutionCount = stat.ExecutionCount,
|
||||
ExecutionsPerMinute = stat.ExecutionsPerMinute,
|
||||
AverageExecutionTime = stat.AverageExecutionTime.TotalMilliseconds,
|
||||
LastExecution = stat.LastExecution,
|
||||
IsActive = stat.IsActive
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var result = new
|
||||
{
|
||||
Alerts = alerts,
|
||||
AlertCount = alerts.Count,
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
if (alerts.Any())
|
||||
{
|
||||
_logger.LogWarning("[SQL-MONITORING] {Count} potential issues detected", alerts.Count);
|
||||
}
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[SQL-MONITORING] Error retrieving alerts");
|
||||
return StatusCode(500, "Error retrieving alerts");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clears all SQL query tracking data
|
||||
/// </summary>
|
||||
/// <returns>Success status</returns>
|
||||
[HttpPost("clear-tracking")]
|
||||
public async Task<ActionResult> ClearTracking()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Check if user is admin
|
||||
if (!await IsUserAdmin())
|
||||
{
|
||||
return Forbid("Only administrators can clear SQL monitoring data");
|
||||
}
|
||||
|
||||
_sentryMonitoringService.ClearAllTracking();
|
||||
_context.ClearQueryTracking();
|
||||
|
||||
_logger.LogInformation("[SQL-MONITORING] All tracking data cleared");
|
||||
|
||||
return Ok(new { Message = "All tracking data cleared successfully", Timestamp = DateTime.UtcNow });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[SQL-MONITORING] Error clearing tracking data");
|
||||
return StatusCode(500, "Error clearing tracking data");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets detailed information about a specific query pattern
|
||||
/// </summary>
|
||||
/// <param name="repositoryName">Repository name</param>
|
||||
/// <param name="methodName">Method name</param>
|
||||
/// <returns>Detailed query information</returns>
|
||||
[HttpGet("query-details/{repositoryName}/{methodName}")]
|
||||
public async Task<ActionResult<object>> GetQueryDetails(string repositoryName, string methodName)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Check if user is admin
|
||||
if (!await IsUserAdmin())
|
||||
{
|
||||
return Forbid("Only administrators can access SQL query details");
|
||||
}
|
||||
|
||||
var stats = _sentryMonitoringService.GetQueryStatistics();
|
||||
var matchingQueries = stats.Where(kvp =>
|
||||
kvp.Value.RepositoryName.Equals(repositoryName, StringComparison.OrdinalIgnoreCase) &&
|
||||
kvp.Value.MethodName.Equals(methodName, StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
if (!matchingQueries.Any())
|
||||
{
|
||||
return NotFound(new { Message = $"No queries found for {repositoryName}.{methodName}" });
|
||||
}
|
||||
|
||||
var result = new
|
||||
{
|
||||
RepositoryName = repositoryName,
|
||||
MethodName = methodName,
|
||||
Queries = matchingQueries.Select(kvp => new
|
||||
{
|
||||
QueryPattern = kvp.Value.QueryPattern,
|
||||
ExecutionCount = kvp.Value.ExecutionCount,
|
||||
ExecutionsPerMinute = kvp.Value.ExecutionsPerMinute,
|
||||
AverageExecutionTime = kvp.Value.AverageExecutionTime.TotalMilliseconds,
|
||||
MinExecutionTime = kvp.Value.MinExecutionTime.TotalMilliseconds,
|
||||
MaxExecutionTime = kvp.Value.MaxExecutionTime.TotalMilliseconds,
|
||||
FirstExecution = kvp.Value.FirstExecution,
|
||||
LastExecution = kvp.Value.LastExecution,
|
||||
IsActive = kvp.Value.IsActive
|
||||
}),
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[SQL-MONITORING] Error retrieving query details for {Repository}.{Method}", repositoryName, methodName);
|
||||
return StatusCode(500, "Error retrieving query details");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a summary of SQL monitoring health
|
||||
/// </summary>
|
||||
/// <returns>Monitoring health summary</returns>
|
||||
[HttpGet("health")]
|
||||
public async Task<ActionResult<object>> GetMonitoringHealth()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Check if user is admin
|
||||
if (!await IsUserAdmin())
|
||||
{
|
||||
return Forbid("Only administrators can access SQL monitoring health");
|
||||
}
|
||||
|
||||
var stats = _sentryMonitoringService.GetQueryStatistics();
|
||||
var contextStats = _context.GetQueryExecutionCounts();
|
||||
|
||||
var activeQueries = stats.Count(kvp => kvp.Value.IsActive);
|
||||
var slowQueries = stats.Count(kvp => kvp.Value.AverageExecutionTime.TotalMilliseconds > 1000);
|
||||
var highFrequencyQueries = stats.Count(kvp => kvp.Value.ExecutionsPerMinute > 20);
|
||||
|
||||
var healthStatus = "Healthy";
|
||||
if (highFrequencyQueries > 0 || slowQueries > 5)
|
||||
{
|
||||
healthStatus = "Warning";
|
||||
}
|
||||
if (highFrequencyQueries > 2 || slowQueries > 10)
|
||||
{
|
||||
healthStatus = "Critical";
|
||||
}
|
||||
|
||||
var result = new
|
||||
{
|
||||
Status = healthStatus,
|
||||
TotalTrackedQueries = stats.Count,
|
||||
ActiveQueries = activeQueries,
|
||||
SlowQueries = slowQueries,
|
||||
HighFrequencyQueries = highFrequencyQueries,
|
||||
ContextQueryCount = contextStats.Count,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
// Add configuration status
|
||||
isEnabled = _sentryMonitoringService.IsMonitoringEnabled(),
|
||||
loggingEnabled = _sentryMonitoringService.IsLoggingEnabled(),
|
||||
sentryEnabled = _sentryMonitoringService.IsSentryEnabled(),
|
||||
loopDetectionEnabled = _sentryMonitoringService.IsLoopDetectionEnabled(),
|
||||
performanceMonitoringEnabled = _sentryMonitoringService.IsPerformanceMonitoringEnabled(),
|
||||
lastHealthCheck = DateTime.UtcNow.ToString("O"),
|
||||
totalAlerts = 0 // TODO: Implement alert counting
|
||||
};
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[SQL-MONITORING] Error retrieving monitoring health");
|
||||
return StatusCode(500, "Error retrieving monitoring health");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -89,8 +89,15 @@ builder.Services.AddHttpClient("GmxHealthCheck")
|
||||
builder.Services.AddSingleton<Web3ProxyHealthCheck>(sp =>
|
||||
new Web3ProxyHealthCheck(sp.GetRequiredService<IHttpClientFactory>(), web3ProxyUrl));
|
||||
|
||||
// Add SQL Loop Detection Service with Sentry integration
|
||||
// Configure SQL monitoring settings
|
||||
builder.Services.Configure<SqlMonitoringSettings>(builder.Configuration.GetSection("SqlMonitoring"));
|
||||
|
||||
// Register SQL monitoring services
|
||||
builder.Services.AddSingleton<SentrySqlMonitoringService>();
|
||||
|
||||
// Add PostgreSQL DbContext with improved concurrency and connection management
|
||||
builder.Services.AddDbContext<ManagingDbContext>(options =>
|
||||
builder.Services.AddDbContext<ManagingDbContext>((serviceProvider, options) =>
|
||||
{
|
||||
options.UseNpgsql(postgreSqlConnectionString, npgsqlOptions =>
|
||||
{
|
||||
@@ -114,8 +121,22 @@ builder.Services.AddDbContext<ManagingDbContext>(options =>
|
||||
// Enable service provider caching for better performance
|
||||
options.EnableServiceProviderCaching();
|
||||
|
||||
// Enable connection resiliency for backtest and high-load scenarios
|
||||
options.LogTo(msg => Console.WriteLine(msg), LogLevel.Warning); // Log warnings for connection issues
|
||||
// Enable comprehensive SQL query logging for monitoring and debugging
|
||||
var logger = serviceProvider.GetRequiredService<ILogger<ManagingDbContext>>();
|
||||
var sentryMonitoringService = serviceProvider.GetRequiredService<SentrySqlMonitoringService>();
|
||||
|
||||
options.LogTo(msg =>
|
||||
{
|
||||
// Log SQL queries with enhanced formatting
|
||||
if (msg.Contains("Executed DbCommand") || msg.Contains("Executing DbCommand"))
|
||||
{
|
||||
Console.WriteLine($"[EF-SQL] {msg}");
|
||||
}
|
||||
else if (msg.Contains("Warning") || msg.Contains("Error"))
|
||||
{
|
||||
Console.WriteLine($"[EF-WARNING] {msg}");
|
||||
}
|
||||
}, LogLevel.Information); // Log all SQL operations for monitoring
|
||||
}, ServiceLifetime.Scoped); // Explicitly specify scoped lifetime for proper request isolation
|
||||
|
||||
// Add specific health checks for databases and other services
|
||||
|
||||
@@ -44,5 +44,12 @@
|
||||
"BaseUrl": "https://api.kaigen.managing.live",
|
||||
"DebitEndpoint": "/api/credits/debit",
|
||||
"RefundEndpoint": "/api/credits/refund"
|
||||
},
|
||||
"SqlMonitoring": {
|
||||
"Enabled": true,
|
||||
"LoggingEnabled": false,
|
||||
"SentryEnabled": true,
|
||||
"LoopDetectionEnabled": true,
|
||||
"LogErrorsOnly": true
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,13 @@
|
||||
"ElasticConfiguration": {
|
||||
"Uri": "http://elasticsearch:9200"
|
||||
},
|
||||
"SqlMonitoring": {
|
||||
"Enabled": true,
|
||||
"LoggingEnabled": true,
|
||||
"SentryEnabled": true,
|
||||
"LoopDetectionEnabled": true,
|
||||
"LogSlowQueriesOnly": false
|
||||
},
|
||||
"RunOrleansGrains": true,
|
||||
"AllowedHosts": "*"
|
||||
}
|
||||
@@ -84,6 +84,20 @@
|
||||
"WorkerBundleBacktest": false,
|
||||
"WorkerBalancesTracking": false,
|
||||
"WorkerNotifyBundleBacktest": false,
|
||||
"AdminUsers": "",
|
||||
"AllowedHosts": "*"
|
||||
"SqlMonitoring": {
|
||||
"Enabled": true,
|
||||
"LoggingEnabled": true,
|
||||
"SentryEnabled": true,
|
||||
"LoopDetectionEnabled": true,
|
||||
"PerformanceMonitoringEnabled": true,
|
||||
"LoopDetectionWindowSeconds": 60,
|
||||
"MaxQueryExecutionsPesrWindow": 100,
|
||||
"MaxMethodExecutionsPerWindow": 50,
|
||||
"LongRunningQueryThresholdMs": 1000,
|
||||
"SentryAlertThreshold": 5,
|
||||
"SlowQueryThresholdMs": 2000,
|
||||
"LogSlowQueriesOnly": false,
|
||||
"LogErrorsOnly": false,
|
||||
"DataRetentionMinutes": 30
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user