228 lines
12 KiB
Bash
Executable File
228 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# Benchmark Backtest Performance Script
|
||
# This script runs backtest performance tests and records results in CSV
|
||
|
||
set -e # Exit on any error
|
||
|
||
echo "🚀 Running backtest performance benchmark..."
|
||
|
||
# Colors for output
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# Function to extract value from test output using regex
|
||
extract_value() {
|
||
local pattern="$1"
|
||
local text="$2"
|
||
echo "$text" | grep -o "$pattern" | head -1 | sed 's/.*: //' | sed 's/[^0-9.]*$//' | tr -d ','
|
||
}
|
||
|
||
# Get current timestamp
|
||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
# Get git information
|
||
COMMIT_HASH=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||
BRANCH_NAME=$(git branch --show-current 2>/dev/null || echo "unknown")
|
||
ENVIRONMENT="development"
|
||
|
||
# Run the main performance test and capture output
|
||
echo "📊 Running main performance test..."
|
||
TEST_OUTPUT=$(dotnet test src/Managing.Workers.Tests/Managing.Workers.Tests.csproj \
|
||
--filter "FullyQualifiedName~Telemetry_ETH_RSI&FullyQualifiedName!~EMACROSS" \
|
||
--verbosity minimal \
|
||
--logger "console;verbosity=detailed" 2>&1)
|
||
|
||
# Check if test passed
|
||
if echo "$TEST_OUTPUT" | grep -q "Passed: 1"; then
|
||
echo -e "${GREEN}✅ Performance test passed!${NC}"
|
||
else
|
||
echo -e "${RED}❌ Performance test failed!${NC}"
|
||
echo "$TEST_OUTPUT" | tail -30
|
||
exit 1
|
||
fi
|
||
|
||
# Run business logic validation tests
|
||
echo "📊 Running business logic validation tests..."
|
||
VALIDATION_OUTPUT=$(dotnet test src/Managing.Workers.Tests/Managing.Workers.Tests.csproj \
|
||
--filter "ExecuteBacktest_With_ETH_FifteenMinutes_Data_Should_Return_LightBacktest|LongBacktest_ETH_RSI" \
|
||
--verbosity minimal \
|
||
--logger "console;verbosity=detailed" 2>&1)
|
||
|
||
# Check if validation tests passed
|
||
if echo "$VALIDATION_OUTPUT" | grep -q "Passed: 2"; then
|
||
echo -e "${GREEN}✅ Business logic validation tests passed!${NC}"
|
||
else
|
||
echo -e "${RED}❌ Business logic validation tests failed!${NC}"
|
||
echo "$VALIDATION_OUTPUT" | tail -30
|
||
exit 1
|
||
fi
|
||
|
||
# Extract performance metrics from the output - use more robust parsing
|
||
CANDLES_COUNT=$(echo "$TEST_OUTPUT" | grep "📈 Total Candles Processed:" | sed 's/.*: //' | sed 's/[^0-9]//g' | xargs)
|
||
EXECUTION_TIME=$(echo "$TEST_OUTPUT" | grep "⏱️ Total Execution Time:" | sed 's/.*: //' | sed 's/s//' | sed 's/,/./g' | awk '{print $NF}' | xargs | awk -F' ' '{if (NF==2) print ($1+$2)/2; else print $1}')
|
||
PROCESSING_RATE=$(echo "$TEST_OUTPUT" | grep "🚀 Processing Rate:" | sed 's/.*: //' | sed 's/ candles\/sec//' | sed 's/,/./g' | xargs)
|
||
|
||
# Extract memory metrics
|
||
MEMORY_LINE=$(echo "$TEST_OUTPUT" | grep "💾 Memory Usage:")
|
||
MEMORY_START=$(echo "$MEMORY_LINE" | sed 's/.*Start=//' | sed 's/MB.*//' | xargs)
|
||
MEMORY_END=$(echo "$MEMORY_LINE" | sed 's/.*End=//' | sed 's/MB.*//' | xargs)
|
||
MEMORY_PEAK=$(echo "$MEMORY_LINE" | sed 's/.*Peak=//' | sed 's/MB.*//' | xargs)
|
||
|
||
# Extract signal update metrics
|
||
SIGNAL_LINE=$(echo "$TEST_OUTPUT" | grep "• Signal Updates:")
|
||
SIGNAL_UPDATES=$(echo "$SIGNAL_LINE" | sed 's/.*Signal Updates: //' | sed 's/ms.*//' | sed 's/,/./g' | xargs)
|
||
SIGNAL_SKIPPED=$(echo "$SIGNAL_LINE" | grep -o "[0-9,]* skipped" | sed 's/ skipped//' | tr -d ',' | xargs)
|
||
SIGNAL_EFFICIENCY=$(echo "$SIGNAL_LINE" | grep -o "[0-9.]*% efficiency" | sed 's/% efficiency//' | xargs)
|
||
|
||
# Extract backtest steps
|
||
BACKTEST_LINE=$(echo "$TEST_OUTPUT" | grep "• Backtest Steps:")
|
||
BACKTEST_STEPS=$(echo "$BACKTEST_LINE" | sed 's/.*Backtest Steps: //' | sed 's/ms.*//' | sed 's/,/./g' | xargs)
|
||
|
||
# Extract timing metrics
|
||
AVG_SIGNAL_UPDATE=$(echo "$TEST_OUTPUT" | grep "• Average Signal Update:" | sed 's/.*Average Signal Update: //' | sed 's/ms.*//' | sed 's/,/./g' | xargs)
|
||
AVG_BACKTEST_STEP=$(echo "$TEST_OUTPUT" | grep "• Average Backtest Step:" | sed 's/.*Average Backtest Step: //' | sed 's/ms.*//' | sed 's/,/./g' | xargs)
|
||
|
||
# Extract trading results
|
||
FINAL_PNL=$(echo "$TEST_OUTPUT" | grep "• Final PnL:" | sed 's/.*Final PnL: //' | sed 's/,/./g' | xargs)
|
||
WIN_RATE=$(echo "$TEST_OUTPUT" | grep "• Win Rate:" | sed 's/.*Win Rate: //' | sed 's/%//' | xargs)
|
||
GROWTH_PERCENTAGE=$(echo "$TEST_OUTPUT" | grep "• Growth:" | sed 's/.*Growth: //' | sed 's/%//' | sed 's/,/./g' | xargs)
|
||
SCORE=$(echo "$TEST_OUTPUT" | grep "• Score:" | sed 's/.*Score: //' | sed 's/[^0-9.-]//g' | xargs)
|
||
|
||
# Set defaults for missing or malformed values
|
||
CANDLES_COUNT=${CANDLES_COUNT:-0}
|
||
EXECUTION_TIME=${EXECUTION_TIME:-0.0}
|
||
PROCESSING_RATE=${PROCESSING_RATE:-0.0}
|
||
MEMORY_START=${MEMORY_START:-0.0}
|
||
MEMORY_END=${MEMORY_END:-0.0}
|
||
MEMORY_PEAK=${MEMORY_PEAK:-0.0}
|
||
SIGNAL_UPDATES=${SIGNAL_UPDATES:-0.0}
|
||
SIGNAL_SKIPPED=${SIGNAL_SKIPPED:-0}
|
||
SIGNAL_EFFICIENCY=${SIGNAL_EFFICIENCY:-0.0}
|
||
BACKTEST_STEPS=${BACKTEST_STEPS:-0.0}
|
||
AVG_SIGNAL_UPDATE=${AVG_SIGNAL_UPDATE:-0.0}
|
||
AVG_BACKTEST_STEP=${AVG_BACKTEST_STEP:-0.0}
|
||
FINAL_PNL=${FINAL_PNL:-0.00}
|
||
WIN_RATE=${WIN_RATE:-0}
|
||
GROWTH_PERCENTAGE=${GROWTH_PERCENTAGE:-0.00}
|
||
SCORE=${SCORE:-0.00}
|
||
|
||
# Fix malformed values
|
||
SCORE=$(echo "$SCORE" | sed 's/^0*$/0.00/' | xargs)
|
||
|
||
# Business Logic Validation: Check Final PnL against first run baseline
|
||
FIRST_RUN_FINAL_PNL=$(head -2 src/Managing.Workers.Tests/performance-benchmarks.csv 2>/dev/null | tail -1 | cut -d',' -f15 | xargs)
|
||
|
||
if [ -n "$FIRST_RUN_FINAL_PNL" ] && [ "$FIRST_RUN_FINAL_PNL" != "FinalPnL" ]; then
|
||
# Compare against the first run in the file (the baseline)
|
||
DIFF=$(echo "scale=2; $FINAL_PNL - $FIRST_RUN_FINAL_PNL" | bc -l 2>/dev/null || echo "0")
|
||
ABS_DIFF=$(echo "scale=2; if ($DIFF < 0) -$DIFF else $DIFF" | bc -l 2>/dev/null || echo "0")
|
||
|
||
if (( $(echo "$ABS_DIFF > 0.01" | bc -l 2>/dev/null || echo "0") )); then
|
||
echo -e "${RED}❌ BUSINESS LOGIC WARNING: Final PnL differs from baseline!${NC}"
|
||
echo " Baseline (first run): $FIRST_RUN_FINAL_PNL"
|
||
echo " Current: $FINAL_PNL"
|
||
echo " Difference: $DIFF"
|
||
echo -e "${YELLOW}⚠️ This may indicate that changes broke business logic!${NC}"
|
||
echo -e "${YELLOW} Please verify that optimizations didn't change backtest behavior.${NC}"
|
||
else
|
||
echo -e "${GREEN}✅ Business Logic OK: Final PnL matches baseline (±$ABS_DIFF)${NC}"
|
||
fi
|
||
else
|
||
# If no baseline exists, establish one
|
||
echo -e "${BLUE}ℹ️ Establishing new baseline - this is the first run${NC}"
|
||
echo -e "${GREEN}✅ First run completed successfully${NC}"
|
||
fi
|
||
|
||
# Create CSV row
|
||
CSV_ROW="$TIMESTAMP,Telemetry_ETH_RSI,$CANDLES_COUNT,$EXECUTION_TIME,$PROCESSING_RATE,$MEMORY_START,$MEMORY_END,$MEMORY_PEAK,$SIGNAL_UPDATES,$SIGNAL_SKIPPED,$SIGNAL_EFFICIENCY,$BACKTEST_STEPS,$AVG_SIGNAL_UPDATE,$AVG_BACKTEST_STEP,$FINAL_PNL,$WIN_RATE,$GROWTH_PERCENTAGE,$SCORE,$COMMIT_HASH,$BRANCH_NAME,$ENVIRONMENT"
|
||
|
||
# Append to CSV file
|
||
echo "$CSV_ROW" >> "src/Managing.Workers.Tests/performance-benchmarks.csv"
|
||
|
||
# Now run the two-scenarios test
|
||
echo "📊 Running two-scenarios performance test..."
|
||
TWO_SCENARIOS_OUTPUT=$(dotnet test src/Managing.Workers.Tests/Managing.Workers.Tests.csproj \
|
||
--filter "Telemetry_ETH_RSI_EMACROSS" \
|
||
--verbosity minimal \
|
||
--logger "console;verbosity=detailed" 2>&1)
|
||
|
||
# Check if two-scenarios test passed
|
||
if echo "$TWO_SCENARIOS_OUTPUT" | grep -q "Passed: 1"; then
|
||
echo -e "${GREEN}✅ Two-scenarios performance test passed!${NC}"
|
||
else
|
||
echo -e "${RED}❌ Two-scenarios performance test failed!${NC}"
|
||
echo "$TWO_SCENARIOS_OUTPUT" | tail -30
|
||
exit 1
|
||
fi
|
||
|
||
# Extract performance metrics from the two-scenarios test output
|
||
TWO_SCENARIOS_CANDLES_COUNT=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "📈 Candles Processed:" | head -1 | sed 's/.*Processed: //' | sed 's/ (.*//' | sed 's/[^0-9]//g' | xargs)
|
||
TWO_SCENARIOS_EXECUTION_TIME=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "⏱️ Total Execution Time:" | head -1 | sed 's/.*: //' | sed 's/s//' | sed 's/,/./g' | awk '{print $1}' | xargs)
|
||
TWO_SCENARIOS_PROCESSING_RATE=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "📈 Candles Processed:" | head -1 | sed 's/.*Processed: [0-9]* (//' | sed 's/ candles\/sec).*//' | sed 's/,/./g' | sed 's/[^0-9.]//g' | xargs)
|
||
|
||
# Extract memory metrics from backtest executor output (same format as main test)
|
||
TWO_SCENARIOS_MEMORY_LINE=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "💾 Memory Usage:")
|
||
TWO_SCENARIOS_MEMORY_START=$(echo "$TWO_SCENARIOS_MEMORY_LINE" | sed 's/.*Start=//' | sed 's/MB.*//' | xargs)
|
||
TWO_SCENARIOS_MEMORY_END=$(echo "$TWO_SCENARIOS_MEMORY_LINE" | sed 's/.*End=//' | sed 's/MB.*//' | xargs)
|
||
TWO_SCENARIOS_MEMORY_PEAK=$(echo "$TWO_SCENARIOS_MEMORY_LINE" | sed 's/.*Peak=//' | sed 's/MB.*//' | xargs)
|
||
|
||
# Set defaults for missing memory values
|
||
TWO_SCENARIOS_MEMORY_START=${TWO_SCENARIOS_MEMORY_START:-0.0}
|
||
TWO_SCENARIOS_MEMORY_END=${TWO_SCENARIOS_MEMORY_END:-0.0}
|
||
TWO_SCENARIOS_MEMORY_PEAK=${TWO_SCENARIOS_MEMORY_PEAK:-0.0}
|
||
|
||
# Extract signal update metrics (use defaults since two-scenarios test doesn't track these)
|
||
TWO_SCENARIOS_SIGNAL_UPDATES=0.0
|
||
TWO_SCENARIOS_SIGNAL_SKIPPED=0
|
||
TWO_SCENARIOS_SIGNAL_EFFICIENCY=0.0
|
||
|
||
# Extract backtest steps (use defaults)
|
||
TWO_SCENARIOS_BACKTEST_STEPS=0.0
|
||
TWO_SCENARIOS_AVG_SIGNAL_UPDATE=0.0
|
||
TWO_SCENARIOS_AVG_BACKTEST_STEP=0.0
|
||
|
||
# Extract trading results - remove "(Expected: ...)" text and clean values to pure numbers
|
||
TWO_SCENARIOS_FINAL_PNL=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "🎯 Final PnL:" | head -1 | sed 's/.*Final PnL: //' | sed 's/ (Expected:.*//' | sed 's/,/./g' | sed 's/[^0-9.-]//g' | xargs)
|
||
TWO_SCENARIOS_WIN_RATE=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "📈 Win Rate:" | head -1 | sed 's/.*Win Rate: //' | sed 's/ (Expected:.*//' | sed 's/%//' | sed 's/[^0-9]//g' | xargs)
|
||
TWO_SCENARIOS_GROWTH_PERCENTAGE=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "📈 Growth:" | head -1 | sed 's/.*Growth: //' | sed 's/ (Expected:.*//' | sed 's/%//' | sed 's/,/./g' | sed 's/[^0-9.-]//g' | xargs)
|
||
TWO_SCENARIOS_SCORE=$(echo "$TWO_SCENARIOS_OUTPUT" | grep "📊 Score:" | head -1 | sed 's/.*Score: //' | sed 's/ (Expected:.*//' | sed 's/,/./g' | sed 's/[^0-9.-]//g' | xargs)
|
||
|
||
# Set defaults for missing values and ensure clean numeric format
|
||
TWO_SCENARIOS_CANDLES_COUNT=${TWO_SCENARIOS_CANDLES_COUNT:-0}
|
||
TWO_SCENARIOS_EXECUTION_TIME=${TWO_SCENARIOS_EXECUTION_TIME:-0.0}
|
||
TWO_SCENARIOS_PROCESSING_RATE=${TWO_SCENARIOS_PROCESSING_RATE:-0.0}
|
||
TWO_SCENARIOS_FINAL_PNL=${TWO_SCENARIOS_FINAL_PNL:-0.00}
|
||
TWO_SCENARIOS_WIN_RATE=${TWO_SCENARIOS_WIN_RATE:-0}
|
||
TWO_SCENARIOS_GROWTH_PERCENTAGE=${TWO_SCENARIOS_GROWTH_PERCENTAGE:-0.00}
|
||
TWO_SCENARIOS_SCORE=${TWO_SCENARIOS_SCORE:-0.00}
|
||
|
||
# Ensure all values are clean numbers (remove any remaining non-numeric characters except decimal point and minus)
|
||
TWO_SCENARIOS_CANDLES_COUNT=$(echo "$TWO_SCENARIOS_CANDLES_COUNT" | sed 's/[^0-9]//g')
|
||
TWO_SCENARIOS_EXECUTION_TIME=$(echo "$TWO_SCENARIOS_EXECUTION_TIME" | sed 's/[^0-9.]//g')
|
||
TWO_SCENARIOS_PROCESSING_RATE=$(echo "$TWO_SCENARIOS_PROCESSING_RATE" | sed 's/[^0-9.]//g')
|
||
TWO_SCENARIOS_FINAL_PNL=$(echo "$TWO_SCENARIOS_FINAL_PNL" | sed 's/[^0-9.-]//g')
|
||
TWO_SCENARIOS_WIN_RATE=$(echo "$TWO_SCENARIOS_WIN_RATE" | sed 's/[^0-9]//g')
|
||
TWO_SCENARIOS_GROWTH_PERCENTAGE=$(echo "$TWO_SCENARIOS_GROWTH_PERCENTAGE" | sed 's/[^0-9.-]//g')
|
||
TWO_SCENARIOS_SCORE=$(echo "$TWO_SCENARIOS_SCORE" | sed 's/[^0-9.-]//g' | sed 's/^$/0.00/')
|
||
|
||
# Create CSV row for two-scenarios test
|
||
TWO_SCENARIOS_CSV_ROW="$TIMESTAMP,Telemetry_ETH_RSI_EMACROSS,$TWO_SCENARIOS_CANDLES_COUNT,$TWO_SCENARIOS_EXECUTION_TIME,$TWO_SCENARIOS_PROCESSING_RATE,$TWO_SCENARIOS_MEMORY_START,$TWO_SCENARIOS_MEMORY_END,$TWO_SCENARIOS_MEMORY_PEAK,$TWO_SCENARIOS_SIGNAL_UPDATES,$TWO_SCENARIOS_SIGNAL_SKIPPED,$TWO_SCENARIOS_SIGNAL_EFFICIENCY,$TWO_SCENARIOS_BACKTEST_STEPS,$TWO_SCENARIOS_AVG_SIGNAL_UPDATE,$TWO_SCENARIOS_AVG_BACKTEST_STEP,$TWO_SCENARIOS_FINAL_PNL,$TWO_SCENARIOS_WIN_RATE,$TWO_SCENARIOS_GROWTH_PERCENTAGE,$TWO_SCENARIOS_SCORE,$COMMIT_HASH,$BRANCH_NAME,$ENVIRONMENT"
|
||
|
||
# Append to two-scenarios CSV file
|
||
echo "$TWO_SCENARIOS_CSV_ROW" >> "src/Managing.Workers.Tests/performance-benchmarks-two-scenarios.csv"
|
||
|
||
# Display results
|
||
echo -e "${BLUE}📊 Benchmark Results:${NC}"
|
||
echo " • Processing Rate: $PROCESSING_RATE candles/sec"
|
||
echo " • Execution Time: $EXECUTION_TIME seconds"
|
||
echo " • Memory Peak: $MEMORY_PEAK MB"
|
||
echo " • Signal Efficiency: $SIGNAL_EFFICIENCY%"
|
||
echo " • Candles Processed: $CANDLES_COUNT"
|
||
echo " • Score: $SCORE"
|
||
|
||
echo -e "${GREEN}✅ Benchmark data recorded successfully!${NC}"
|