Enhance TradingBotBase with recovery logic for recently canceled positions and improved error handling for Web3Proxy. Updated CheckPositionInExchangeHistory to return error status, ensuring robust position verification and cancellation processes.

src/Managing.Application/Bots/TradingBotBase.cs

@@ -400,6 +400,9 @@ public class TradingBotBase : ITradingBot
         if (!hasOpenPositions && !hasWaitingSignals)
             return;
 
+        // Recovery Logic: Check for recently canceled positions that might need recovery
+        await RecoverRecentlyCanceledPositions();
+
         // First, process all existing positions that are not finished
         // Optimized: Inline the filter to avoid LINQ overhead
         foreach (var position in Positions.Values)
@@ -598,9 +601,20 @@ public class TradingBotBase : ITradingBot
                             $"Checking position history before marking as closed...");
 
                         // Verify in exchange history before assuming it's closed
-                        bool existsInHistory = await CheckPositionInExchangeHistory(positionForSignal);
+                        var (existsInHistory, hadWeb3ProxyError) = await CheckPositionInExchangeHistory(positionForSignal);
 
-                        if (existsInHistory)
+                        if (hadWeb3ProxyError)
+                        {
+                            // Web3Proxy error - don't assume position is closed, wait for next cycle
+                            await LogWarning(
+                                $"⏳ Web3Proxy Error During Position Verification\n" +
+                                $"Position: `{positionForSignal.Identifier}`\n" +
+                                $"Cannot verify if position is closed\n" +
+                                $"Will retry on next execution cycle");
+                            // Don't change position status, wait for next cycle
+                            return;
+                        }
+                        else if (existsInHistory)
                         {
                             // Position was actually filled and closed by the exchange
                             Logger.LogInformation(
@@ -767,9 +781,20 @@ public class TradingBotBase : ITradingBot
 
                     // Position might be canceled by the broker
                     // Check if position exists in exchange history with PnL before canceling
-                    bool positionFoundInHistory = await CheckPositionInExchangeHistory(positionForSignal);
+                    var (positionFoundInHistory, hadWeb3ProxyError) = await CheckPositionInExchangeHistory(positionForSignal);
 
-                    if (positionFoundInHistory)
+                    if (hadWeb3ProxyError)
+                    {
+                        // Web3Proxy error occurred - don't mark as cancelled, wait for next cycle
+                        await LogWarning(
+                            $"⏳ Web3Proxy Error - Skipping Position Cancellation\n" +
+                            $"Position: `{positionForSignal.Identifier}`\n" +
+                            $"Status remains: `{positionForSignal.Status}`\n" +
+                            $"Will retry position verification on next execution cycle");
+                        // Don't change signal status to Expired, let it continue
+                        return;
+                    }
+                    else if (positionFoundInHistory)
                     {
                         // Position was actually filled and closed, process it properly
                         await HandleClosedPosition(positionForSignal);
@@ -2824,12 +2849,12 @@ public class TradingBotBase : ITradingBot
     /// </summary>
     /// <param name="position">The position to check</param>
     /// <returns>True if position found in exchange history with PnL, false otherwise</returns>
-    private async Task<bool> CheckPositionInExchangeHistory(Position position)
+    private async Task<(bool found, bool hadError)> CheckPositionInExchangeHistory(Position position)
     {
         if (Config.IsForBacktest)
         {
             // For backtests, we don't have exchange history, so return false
-            return false;
+            return (false, false);
         }
 
         try
@@ -2864,7 +2889,7 @@ public class TradingBotBase : ITradingBot
                         $"Direction: `{position.OriginDirection}` (Matched: ✅)\n" +
                         $"Exchange PnL: `${recentPosition.ProfitAndLoss.Realized:F2}`\n" +
                         $"Position was actually filled and closed");
-                    return true;
+                    return (true, false);
                 }
                 else
                 {
@@ -2880,11 +2905,192 @@ public class TradingBotBase : ITradingBot
 
             await LogDebug(
                 $"❌ No Position Found in Exchange History\nPosition: `{position.Identifier}`\nPosition was never filled");
-            return false;
+            return (false, false);
         }
         catch (Exception ex)
         {
             Logger.LogError(ex, "Error checking position history for position {PositionId}", position.Identifier);
+            await LogWarning(
+                $"⚠️ Web3Proxy Error During Position History Check\n" +
+                $"Position: `{position.Identifier}`\n" +
+                $"Error: {ex.Message}\n" +
+                $"Will retry on next execution cycle");
+            return (false, true); // found=false, hadError=true
+        }
+    }
+
+    private async Task RecoverRecentlyCanceledPositions()
+    {
+        if (Config.IsForBacktest)
+        {
+            // For backtests, we don't have broker positions, so skip recovery
+            return;
+        }
+
+        try
+        {
+            // Get the last (most recent) position from all positions
+            var lastPosition = Positions.Values.LastOrDefault();
+            if (lastPosition == null)
+            {
+                return; // No positions at all
+            }
+
+            // Only attempt recovery if the last position is cancelled
+            if (lastPosition.Status != PositionStatus.Canceled)
+            {
+                await LogDebug(
+                    $"❌ Recovery Skipped\nLast position `{lastPosition.Identifier}` is not cancelled (Status: {lastPosition.Status})\nNo recovery needed");
+                return;
+            }
+
+            // Also get count of cancelled positions for logging
+            var canceledPositionsCount = Positions.Values.Count(p => p.Status == PositionStatus.Canceled);
+
+            await LogDebug(
+                $"🔄 Position Recovery Check\nFound `{canceledPositionsCount}` canceled positions\nLast position `{lastPosition.Identifier}` is cancelled\nAttempting recovery from broker...");
+
+            // Get the signal for the last position
+            if (!Signals.TryGetValue(lastPosition.SignalIdentifier, out var signal))
+            {
+                await LogWarning(
+                    $"⚠️ Signal Not Found for Recovery\nPosition: `{lastPosition.Identifier}`\nSignal: `{lastPosition.SignalIdentifier}`\nCannot recover without signal");
+                return;
+            }
+
+            // Attempt recovery for the last position only
+            bool recovered = await RecoverOpenPositionFromBroker(signal, lastPosition);
+            if (recovered)
+            {
+                await LogInformation(
+                    $"🎉 Position Recovery Successful\nPosition `{lastPosition.Identifier}` recovered from broker\nStatus restored to Filled\nWill continue normal processing");
+            }
+            else
+            {
+                await LogDebug(
+                    $"❌ Recovery Not Needed\nPosition `{lastPosition.Identifier}` confirmed canceled\nNo open position found on broker");
+            }
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Error during recently canceled positions recovery");
+            await LogWarning($"Position recovery check failed due to exception: {ex.Message}");
+        }
+    }
+
+    private async Task<bool> RecoverOpenPositionFromBroker(LightSignal signal, Position positionForSignal)
+    {
+        if (Config.IsForBacktest)
+        {
+            // For backtests, we don't have broker positions, so return false
+            return false;
+        }
+
+        try
+        {
+            await LogWarning(
+                $"🔄 Attempting Position Recovery\n" +
+                $"Signal: `{signal.Identifier}`\n" +
+                $"Position: `{positionForSignal.Identifier}`\n" +
+                $"Direction: `{positionForSignal.OriginDirection}`\n" +
+                $"Ticker: `{Config.Ticker}`\n" +
+                $"Checking broker for open position...");
+
+            Position brokerPosition = null;
+            await ServiceScopeHelpers.WithScopedService<IExchangeService>(_scopeFactory,
+                async exchangeService =>
+                {
+                    var brokerPositions = await exchangeService.GetBrokerPositions(Account);
+                    brokerPosition = brokerPositions.FirstOrDefault(p => p.Ticker == Config.Ticker);
+                });
+
+            if (brokerPosition != null)
+            {
+                // Check if the broker position matches our expected direction
+                if (brokerPosition.OriginDirection == positionForSignal.OriginDirection)
+                {
+                    await LogInformation(
+                        $"✅ Position Recovered from Broker\n" +
+                        $"Position: `{positionForSignal.Identifier}`\n" +
+                        $"Direction: `{positionForSignal.OriginDirection}` (Matched: ✅)\n" +
+                        $"Broker Position Size: `{brokerPosition.Open?.Quantity ?? 0}`\n" +
+                        $"Broker Position Price: `${brokerPosition.Open?.Price ?? 0:F2}`\n" +
+                        $"Restoring position status to Filled");
+
+                    // Update position status back to Filled (from Canceled)
+                    positionForSignal.Status = PositionStatus.Filled;
+                    await SetPositionStatus(signal.Identifier, PositionStatus.Filled);
+
+                    // Update signal status back to PositionOpen since position is recovered
+                    SetSignalStatus(signal.Identifier, SignalStatus.PositionOpen);
+
+                    // Update PnL from broker position
+                    var brokerNetPnL = brokerPosition.GetPnLBeforeFees();
+                    UpdatePositionPnl(positionForSignal.Identifier, brokerNetPnL);
+
+                    // Update trade details if available
+                    if (positionForSignal.Open != null && brokerPosition.Open != null)
+                    {
+                        positionForSignal.Open.SetStatus(TradeStatus.Filled);
+                        if (brokerPosition.Open.ExchangeOrderId != null)
+                        {
+                            positionForSignal.Open.SetExchangeOrderId(brokerPosition.Open.ExchangeOrderId);
+                        }
+                    }
+
+                    // Update stop loss and take profit trades if available
+                    if (positionForSignal.StopLoss != null && brokerPosition.StopLoss != null)
+                    {
+                        positionForSignal.StopLoss.SetExchangeOrderId(brokerPosition.StopLoss.ExchangeOrderId);
+                    }
+
+                    if (positionForSignal.TakeProfit1 != null && brokerPosition.TakeProfit1 != null)
+                    {
+                        positionForSignal.TakeProfit1.SetExchangeOrderId(brokerPosition.TakeProfit1.ExchangeOrderId);
+                    }
+
+                    if (positionForSignal.TakeProfit2 != null && brokerPosition.TakeProfit2 != null)
+                    {
+                        positionForSignal.TakeProfit2.SetExchangeOrderId(brokerPosition.TakeProfit2.ExchangeOrderId);
+                    }
+
+                    // Update database
+                    await UpdatePositionDatabase(positionForSignal);
+
+                    // Notify about position recovery
+                    await NotifyAgentAndPlatformGrainAsync(NotificationEventType.PositionUpdated, positionForSignal);
+
+                    await LogInformation(
+                        $"🎉 Position Recovery Complete\n" +
+                        $"Position `{positionForSignal.Identifier}` successfully recovered\n" +
+                        $"Status restored to Filled\n" +
+                        $"Database and internal state updated");
+
+                    return true;
+                }
+                else
+                {
+                    await LogWarning(
+                        $"⚠️ Direction Mismatch During Recovery\n" +
+                        $"Expected: `{positionForSignal.OriginDirection}`\n" +
+                        $"Broker Position: `{brokerPosition.OriginDirection}`\n" +
+                        $"Cannot recover - directions don't match");
+                }
+            }
+            else
+            {
+                await LogDebug(
+                    $"❌ No Open Position Found on Broker\n" +
+                    $"Ticker: `{Config.Ticker}`\n" +
+                    $"Position recovery not possible");
+            }
+
+            return false;
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Error during position recovery for position {PositionId}", positionForSignal.Identifier);
+            await LogWarning($"Position recovery failed due to exception: {ex.Message}");
             return false;
         }
     }