Oda/managing-apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trading bot grain (#33)

Browse Source 
* Trading bot Grain

* Fix a bit more of the trading bot

* Advance on the tradingbot grain

* Fix build

* Fix db script

* Fix user login

* Fix a bit backtest

* Fix cooldown and backtest

* start fixing bot start

* Fix startup

* Setup local db

* Fix build and update candles and scenario

* Add bot registry

* Add reminder

* Updateing the grains

* fix bootstraping

* Save stats on tick

* Save bot data every tick

* Fix serialization

* fix save bot stats

* Fix get candles

* use dict instead of list for position

* Switch hashset to dict

* Fix a bit

* Fix bot launch and bot view

* add migrations

* Remove the tolist

* Add agent grain

* Save agent summary

* clean

* Add save bot

* Update get bots

* Add get bots

* Fix stop/restart

* fix Update config

* Update scanner table on new backtest saved

* Fix backtestRowDetails.tsx

* Fix agentIndex

* Update agentIndex

* Fix more things

* Update user cache

* Fix

* Fix account load/start/restart/run
This commit is contained in:
Oda
2025-08-04 23:07:06 +02:00
committed by GitHub
parent cd378587aa
commit 082ae8714b
215 changed files with 9562 additions and 14028 deletions
Download Patch File Download Diff File Expand all files Collapse all files

315
scripts/safe-migrate.sh
View File

@@ -18,7 +18,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
LOGS_DIR="$SCRIPT_DIR/$LOGS_DIR_NAME"
mkdir -p "$LOGS_DIR" || { echo "Failed to create logs directory: $LOGS_DIR"; exit 1; }
LOG_FILE="./logs/migration_${ENVIRONMENT}_${TIMESTAMP}.log"
LOG_FILE="$SCRIPT_DIR/logs/migration_${ENVIRONMENT}_${TIMESTAMP}.log"
# Colors for output
RED='\033[0;31m'
@@ -155,6 +155,12 @@ extract_connection_details() {
log "📋 Extracted connection details: $DB_HOST:$DB_PORT/$DB_NAME (user: $DB_USER, password: $DB_PASSWORD)"
}
# Helper function to get the first migration name
get_first_migration() {
local first_migration=$(cd "$DB_PROJECT_PATH" && dotnet ef migrations list --no-build --startup-project "$API_PROJECT_PATH" | head -1 | awk '{print $1}')
echo "$first_migration"
}
# Helper function to test PostgreSQL connectivity
test_postgres_connectivity() {
if ! command -v psql >/dev/null 2>&1; then
@@ -243,13 +249,6 @@ else
error "❌ Failed to build Managing.Infrastructure.Database project"
fi
log "🔧 Building Managing.Api project..."
if (cd "$API_PROJECT_PATH" && dotnet build); then
log "✅ Managing.Api project built successfully"
else
error "❌ Failed to build Managing.Api project"
fi
# Step 1: Check Database Connection and Create if Needed
log "🔧 Step 1: Checking database connection and creating database if needed..."
@@ -417,13 +416,23 @@ else
error " This is critical. Please review the previous error messages and your connection string for '$ENVIRONMENT'."
fi
# Step 2: Create Backup
log "📦 Step 2: Creating database backup using pg_dump..."
# Step 2: Create database backup (only if database exists)
log "📦 Step 2: Checking if database backup is needed..."
# Define the actual backup file path (absolute)
BACKUP_FILE="$BACKUP_DIR/managing_${ENVIRONMENT}_backup_${TIMESTAMP}.sql"
# Backup file display path (relative to script execution)
BACKUP_FILE_DISPLAY="$BACKUP_DIR_NAME/$ENVIRONMENT/managing_${ENVIRONMENT}_backup_${TIMESTAMP}.sql"
# Check if the target database exists
DB_EXISTS=false
if PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "postgres" -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME';" 2>/dev/null | grep -q "1 row"; then
DB_EXISTS=true
log "✅ Target database '$DB_NAME' exists - proceeding with backup"
else
log "ℹ️ Target database '$DB_NAME' does not exist - skipping backup"
fi
if [ "$DB_EXISTS" = "true" ]; then
# Define the actual backup file path (absolute)
BACKUP_FILE="$BACKUP_DIR/managing_${ENVIRONMENT}_backup_${TIMESTAMP}.sql"
# Backup file display path (relative to script execution)
BACKUP_FILE_DISPLAY="$BACKUP_DIR_NAME/$ENVIRONMENT/managing_${ENVIRONMENT}_backup_${TIMESTAMP}.sql"
# Create backup with retry logic
BACKUP_SUCCESS=false
@@ -439,11 +448,84 @@ for attempt in 1 2 3; do
else
# If pg_dump fails, fall back to EF Core migration script
warn "⚠️ pg_dump failed, falling back to EF Core migration script..."
# Get the first migration name to generate complete script
FIRST_MIGRATION=$(get_first_migration)
if [ -n "$FIRST_MIGRATION" ]; then
log "📋 Generating complete backup script from initial migration: $FIRST_MIGRATION"
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --from "$FIRST_MIGRATION" --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE"); then
log "✅ Complete EF Core Migration SQL Script generated: $BACKUP_FILE_DISPLAY"
BACKUP_SUCCESS=true
break
else
# Try fallback without specifying from migration
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE") 2>&1 || true)
if [ $attempt -lt 3 ]; then
warn "⚠️ Backup attempt $attempt failed. Retrying in 5 seconds..."
warn " EF CLI Output: $ERROR_OUTPUT"
sleep 5
else
error "❌ Database backup failed after 3 attempts."
error " EF CLI Output: $ERROR_OUTPUT"
error " Migration aborted for safety reasons."
fi
fi
else
# Fallback: generate script without specifying from migration
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE"); then
log "✅ EF Core Migration SQL Script generated: $BACKUP_FILE_DISPLAY"
BACKUP_SUCCESS=true
break
else
# Try fallback without specifying from migration
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE") 2>&1 || true)
if [ $attempt -lt 3 ]; then
warn "⚠️ Backup attempt $attempt failed. Retrying in 5 seconds..."
warn " EF CLI Output: $ERROR_OUTPUT"
sleep 5
else
error "❌ Database backup failed after 3 attempts."
error " EF CLI Output: $ERROR_OUTPUT"
error " Migration aborted for safety reasons."
fi
fi
fi
fi
else
# If pg_dump is not available, use EF Core migration script
warn "⚠️ pg_dump not available, using EF Core migration script for backup..."
# Get the first migration name to generate complete script
FIRST_MIGRATION=$(get_first_migration)
if [ -n "$FIRST_MIGRATION" ]; then
log "📋 Generating complete backup script from initial migration: $FIRST_MIGRATION"
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --from "$FIRST_MIGRATION" --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE"); then
log "✅ Complete EF Core Migration SQL Script generated: $BACKUP_FILE_DISPLAY"
BACKUP_SUCCESS=true
break
else
# Try fallback without specifying from migration
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE") 2>&1 || true)
if [ $attempt -lt 3 ]; then
warn "⚠️ Backup attempt $attempt failed. Retrying in 5 seconds..."
warn " EF CLI Output: $ERROR_OUTPUT"
sleep 5
else
error "❌ Database backup failed after 3 attempts."
error " EF CLI Output: $ERROR_OUTPUT"
error " Migration aborted for safety reasons."
fi
fi
else
# Fallback: generate script without specifying from migration
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE"); then
log "✅ EF Core Migration SQL Script generated: $BACKUP_FILE_DISPLAY"
BACKUP_SUCCESS=true
break
else
# Try fallback without specifying from migration
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE") 2>&1 || true)
if [ $attempt -lt 3 ]; then
warn "⚠️ Backup attempt $attempt failed. Retrying in 5 seconds..."
@@ -456,33 +538,69 @@ for attempt in 1 2 3; do
fi
fi
fi
else
# If pg_dump is not available, use EF Core migration script
warn "⚠️ pg_dump not available, using EF Core migration script for backup..."
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE"); then
log "✅ EF Core Migration SQL Script generated: $BACKUP_FILE_DISPLAY"
BACKUP_SUCCESS=true
break
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$BACKUP_FILE") 2>&1 || true)
if [ $attempt -lt 3 ]; then
warn "⚠️ Backup attempt $attempt failed. Retrying in 5 seconds..."
warn " EF CLI Output: $ERROR_OUTPUT"
sleep 5
else
error "❌ Database backup failed after 3 attempts."
error " EF CLI Output: $ERROR_OUTPUT"
error " Migration aborted for safety reasons."
fi
fi
fi
done
# Check if backup was successful before proceeding
if [ "$BACKUP_SUCCESS" != "true" ]; then
error "❌ Database backup failed. Migration aborted for safety."
error " Cannot proceed with migration without a valid backup."
error " Please resolve backup issues and try again."
# Check if backup was successful before proceeding
if [ "$BACKUP_SUCCESS" != "true" ]; then
error "❌ Database backup failed. Migration aborted for safety."
error " Cannot proceed with migration without a valid backup."
error " Please resolve backup issues and try again."
fi
fi
# Step 2.5: Check for pending model changes and create migrations if needed
log "🔍 Step 2.5: Checking for pending model changes..."
# Check if there are any pending model changes that need migrations
PENDING_CHANGES_OUTPUT=$( (cd "$DB_PROJECT_PATH" && dotnet ef migrations add --dry-run --startup-project "$API_PROJECT_PATH" --name "PendingChanges_${TIMESTAMP}") 2>&1 || true )
if echo "$PENDING_CHANGES_OUTPUT" | grep -q "No pending model changes"; then
log "✅ No pending model changes detected - existing migrations are up to date"
else
log "⚠️ Pending model changes detected that require new migrations"
echo ""
echo "=========================================="
echo "📋 PENDING MODEL CHANGES DETECTED"
echo "=========================================="
echo "The following changes require new migrations:"
echo "$PENDING_CHANGES_OUTPUT"
echo ""
echo "Would you like to create a new migration now?"
echo "=========================================="
echo ""
read -p "🔧 Create new migration? (y/n): " create_migration
if [[ "$create_migration" =~ ^[Yy]$ ]]; then
log "📝 Creating new migration..."
# Get migration name from user
read -p "📝 Enter migration name (or press Enter for auto-generated name): " migration_name
if [ -z "$migration_name" ]; then
migration_name="Migration_${TIMESTAMP}"
fi
# Create the migration
if (cd "$DB_PROJECT_PATH" && dotnet ef migrations add "$migration_name" --startup-project "$API_PROJECT_PATH"); then
log "✅ Migration '$migration_name' created successfully"
# Show the created migration file
LATEST_MIGRATION=$(find "$DB_PROJECT_PATH/Migrations" -name "*${migration_name}.cs" | head -1)
if [ -n "$LATEST_MIGRATION" ]; then
log "📄 Migration file created: $(basename "$LATEST_MIGRATION")"
log " Location: $LATEST_MIGRATION"
fi
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && dotnet ef migrations add "$migration_name" --startup-project "$API_PROJECT_PATH") 2>&1 || true )
error "❌ Failed to create migration '$migration_name'"
error " EF CLI Output: $ERROR_OUTPUT"
error " Please resolve the model issues and try again."
fi
else
log "⚠️ Skipping migration creation. Proceeding with existing migrations only."
log " Note: If there are pending changes, the migration may fail."
fi
fi
# Step 3: Run Migration (This effectively is a retry if previous "update" failed, or a final apply)
@@ -507,8 +625,88 @@ fi
# Generate migration script first (Microsoft recommended approach)
MIGRATION_SCRIPT="$BACKUP_DIR/migration_${ENVIRONMENT}_${TIMESTAMP}.sql"
log "📝 Step 3b: Generating migration script for pending migrations..."
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT"); then
log "✅ Migration script generated: $(basename "$MIGRATION_SCRIPT")"
# Check if database is empty (no tables) to determine the best approach
log "🔍 Checking if database has existing tables..."
DB_HAS_TABLES=false
if command -v psql >/dev/null 2>&1; then
TABLE_COUNT=$(PGPASSWORD="$DB_PASSWORD" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null | tr -d ' ' || echo "0")
if [ "$TABLE_COUNT" -gt 0 ]; then
DB_HAS_TABLES=true
log "✅ Database has $TABLE_COUNT existing tables - using idempotent script generation"
else
log "⚠️ Database appears to be empty - using full migration script generation"
fi
else
log "⚠️ psql not available - assuming database has tables and using idempotent script generation"
DB_HAS_TABLES=true
fi
# Generate migration script based on database state
if [ "$DB_HAS_TABLES" = "true" ]; then
# For databases with existing tables, we need to generate a complete script
# that includes all migrations from the beginning
log "📝 Generating complete migration script from initial migration..."
# Get the first migration name to generate script from the beginning
FIRST_MIGRATION=$(get_first_migration)
if [ -n "$FIRST_MIGRATION" ]; then
log "📋 Generating complete script for all migrations (idempotent)..."
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT"); then
log "✅ Complete migration script generated (all migrations, idempotent): $(basename "$MIGRATION_SCRIPT")"
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT") 2>&1 || true )
error "❌ Failed to generate complete migration script."
error " EF CLI Output: $ERROR_OUTPUT"
error " Check the .NET project logs for detailed errors."
error " Backup script available at: $BACKUP_FILE_DISPLAY"
fi
else
# Fallback: generate script without specifying from migration
log "📝 Fallback: Generating migration script without specifying from migration..."
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT"); then
log "✅ Migration script generated (idempotent): $(basename "$MIGRATION_SCRIPT")"
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT") 2>&1 || true )
error "❌ Failed to generate idempotent migration script."
error " EF CLI Output: $ERROR_OUTPUT"
error " Check the .NET project logs for detailed errors."
error " Backup script available at: $BACKUP_FILE_DISPLAY"
fi
fi
else
# Use full script generation for empty databases (generate script from the very beginning)
log "📝 Generating full migration script for empty database..."
# Get the first migration name to generate script from the beginning
FIRST_MIGRATION=$(get_first_migration)
if [ -n "$FIRST_MIGRATION" ]; then
log "📋 Generating complete script for all migrations..."
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT"); then
log "✅ Complete migration script generated (all migrations): $(basename "$MIGRATION_SCRIPT")"
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT") 2>&1 || true )
error "❌ Failed to generate complete migration script."
error " EF CLI Output: $ERROR_OUTPUT"
error " Check the .NET project logs for detailed errors."
error " Backup script available at: $BACKUP_FILE_DISPLAY"
fi
else
# Fallback: generate script without specifying from migration
log "📝 Fallback: Generating migration script without specifying from migration..."
if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT"); then
log "✅ Migration script generated (fallback): $(basename "$MIGRATION_SCRIPT")"
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT") 2>&1 || true )
error "❌ Failed to generate fallback migration script."
error " EF CLI Output: $ERROR_OUTPUT"
error " Check the .NET project logs for detailed errors."
error " Backup script available at: $BACKUP_FILE_DISPLAY"
fi
fi
fi
# Show the migration script path to the user for review
echo ""
@@ -519,8 +717,26 @@ if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef mig
echo "Environment: $ENVIRONMENT"
echo "Database: $DB_HOST:$DB_PORT/$DB_NAME"
echo ""
# Show a preview of the migration script content
if [ -f "$MIGRATION_SCRIPT" ]; then
SCRIPT_SIZE=$(wc -l < "$MIGRATION_SCRIPT")
echo "📄 Migration script contains $SCRIPT_SIZE lines"
# Show first 20 lines as preview
echo ""
echo "📋 PREVIEW (first 20 lines):"
echo "----------------------------------------"
head -20 "$MIGRATION_SCRIPT" | sed 's/^/ /'
if [ "$SCRIPT_SIZE" -gt 20 ]; then
echo " ... (showing first 20 lines of $SCRIPT_SIZE total)"
fi
echo "----------------------------------------"
echo ""
fi
echo "⚠️ IMPORTANT: Please review the migration script before proceeding!"
echo " You can examine the script with: cat $MIGRATION_SCRIPT"
echo " You can examine the full script with: cat $MIGRATION_SCRIPT"
echo " Or open it in your editor to review the changes."
echo ""
echo "=========================================="
@@ -564,16 +780,15 @@ if (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef mig
fi
fi
# Clean up migration script after successful application
rm -f "$MIGRATION_SCRIPT"
# Save a copy of the migration script for reference before cleaning up
MIGRATION_SCRIPT_COPY="$BACKUP_DIR/migration_${ENVIRONMENT}_${TIMESTAMP}_applied.sql"
if [ -f "$MIGRATION_SCRIPT" ]; then
cp "$MIGRATION_SCRIPT" "$MIGRATION_SCRIPT_COPY"
log "📝 Migration script saved for reference: $(basename "$MIGRATION_SCRIPT_COPY")"
fi
else
ERROR_OUTPUT=$( (cd "$DB_PROJECT_PATH" && ASPNETCORE_ENVIRONMENT="$ENVIRONMENT" dotnet ef migrations script --idempotent --no-build --startup-project "$API_PROJECT_PATH" --output "$MIGRATION_SCRIPT") 2>&1 || true )
error "❌ Failed to generate migration script."
error " EF CLI Output: $ERROR_OUTPUT"
error " Check the .NET project logs for detailed errors."
error " Backup script available at: $BACKUP_FILE_DISPLAY"
fi
# Clean up temporary migration script after successful application
rm -f "$MIGRATION_SCRIPT"
# Step 4: Verify Migration
log "🔍 Step 4: Verifying migration status..."