diff --git a/.gitignore b/.gitignore index 234a5ef3..14cb5dfd 100644 --- a/.gitignore +++ b/.gitignore @@ -380,3 +380,6 @@ src/Managing.Infrastructure.Tests/PrivateKeys.cs # Node.js Tools for Visual Studio node_modules/ +# InfluxDB exports and backups +scripts/influxdb/exports/ + diff --git a/scripts/influxdb/README.md b/scripts/influxdb/README.md new file mode 100644 index 00000000..f2423188 --- /dev/null +++ b/scripts/influxdb/README.md @@ -0,0 +1,345 @@ +# InfluxDB Export and Import Scripts + +This directory contains scripts for exporting and importing InfluxDB data for the Managing Apps project using query-based methods that work with standard read/write tokens. + +## Prerequisites + +1. **InfluxDB CLI** - Required for export/import operations + ```bash + brew install influxdb-cli + ``` + +2. **jq** - JSON parser for reading configuration files + ```bash + brew install jq + ``` + +## Available Scripts + +### 1. `export-prices-bucket.sh` +Exports OHLCV candle/price data from the `prices-bucket`. + +**What it exports:** +- All candle data (open, high, low, close, volume) +- Multiple exchanges, tickers, and timeframes +- Configurable time ranges + +**Usage:** +```bash +./export-prices-bucket.sh +``` + +**Interactive Prompts:** +- Select environment (SandboxLocal or ProductionLocal) +- Select time range (7 days, 30 days, 90 days, 1 year, all data, or custom) + +**Output:** +- CSV export: `./exports///prices-bucket_data.csv` +- Metadata file with export details + +**Advantages:** +- ✅ Works with regular read tokens (no admin required) +- ✅ Flexible time range selection +- ✅ Exports in standard CSV format +- ✅ Can be imported to any InfluxDB instance + +--- + +### 2. `import-csv-data.sh` +Imports prices-bucket CSV export data into any InfluxDB environment. + +**What it imports:** +- Prices-bucket data only +- Supports large files (1.6M+ data points) +- Automatically creates bucket if needed + +**Usage:** +```bash +./import-csv-data.sh +``` + +**Interactive Prompts:** +1. Select source environment (which export to import from) +2. Select export timestamp +3. Select target environment (where to import to) +4. Confirm the import operation + +**Features:** +- ✅ Imports CSV exports to any environment +- ✅ Works with regular read/write tokens +- ✅ Batch processing for large files (5000 points per batch) +- ✅ Automatic bucket creation if needed +- ✅ Progress tracking for large imports + +**⚠️ Note:** Import adds data to the bucket. Existing data with the same timestamps will be overwritten. + +--- + +## Configuration + +The scripts automatically read InfluxDB connection settings from: +- `src/Managing.Api/appsettings.SandboxLocal.json` +- `src/Managing.Api/appsettings.ProductionLocal.json` + +**Required settings in appsettings files:** +```json +{ + "InfluxDb": { + "Url": "https://influx-db.apps.managing.live", + "Organization": "managing-org", + "Token": "your-token-here" + } +} +``` + +## Export/Import Structure + +``` +exports/ +├── SandboxLocal/ +│ └── 20241028_143022/ +│ ├── prices-bucket_data.csv +│ └── export-metadata.txt +└── ProductionLocal/ + └── 20241028_160000/ + ├── prices-bucket_data.csv + └── export-metadata.txt +``` + +## Data Structure + +### prices-bucket (Managed by these scripts) +- **Measurement**: `price` +- **Contains**: OHLCV candle data +- **Tags**: + - `exchange` (e.g., Evm, Binance) + - `ticker` (e.g., BTC, ETH, AAVE) + - `timeframe` (e.g., FifteenMinutes, OneHour, OneDay) +- **Fields**: + - `open`, `high`, `low`, `close` (price values) + - `baseVolume`, `quoteVolume` (volume data) + - `TradeCount` (number of trades) + - `takerBuyBaseVolume`, `takerBuyQuoteVolume` (taker buy volumes) + +### agent-balances-bucket (Not included in export/import scripts) +- **Measurement**: `agent_balance` +- **Contains**: User balance history over time +- **Note**: This bucket is not managed by these scripts. Balance data is derived from operational data and should be regenerated rather than migrated. + +## Common Workflows + +### Quick Export +```bash +cd scripts/influxdb +./export-prices-bucket.sh +# Select: 1 (SandboxLocal) +# Select: 5 (All data) +``` + +### Export Specific Time Range +```bash +cd scripts/influxdb +./export-prices-bucket.sh +# Select: 1 (SandboxLocal) +# Select: 3 (Last 90 days) +``` + +### Migrate Sandbox to Production +```bash +cd scripts/influxdb + +# Step 1: Export from sandbox +./export-prices-bucket.sh +# Select: 1 (SandboxLocal) +# Select: 5 (All data) + +# Step 2: Import to production +./import-csv-data.sh +# Select source: 1 (SandboxLocal) +# Select: Latest export timestamp +# Select target: 2 (ProductionLocal) +# Confirm: yes +``` + +### Backup Before Major Changes +```bash +cd scripts/influxdb + +# Export current production data +./export-prices-bucket.sh +# Select: 2 (ProductionLocal) +# Select: 5 (All data) + +# If something goes wrong, restore it: +./import-csv-data.sh +# Select the backup you just created +``` + +### Clone Environment +```bash +# Export from source +./export-prices-bucket.sh +# Select source environment + +# Import to target +./import-csv-data.sh +# Select target environment +``` + +## Token Permissions + +### Read Token (Export) +Required for `export-prices-bucket.sh`: +- ✅ Read access to buckets +- ✅ This is what you typically have in production + +### Write Token (Import) +Required for `import-csv-data.sh`: +- ✅ Read/Write access to target bucket +- ✅ Ability to create buckets (optional, for auto-creation) + +### How to Check Your Token Permissions +```bash +influx auth list --host --token +``` + +## Data Retention + +- Exports are stored indefinitely by default +- Manual cleanup recommended: + ```bash + # Remove exports older than 90 days + find ./exports -type d -mtime +90 -exec rm -rf {} + + ``` + +## Troubleshooting + +### "influx command not found" +Install InfluxDB CLI: +```bash +brew install influxdb-cli +``` + +### "jq command not found" +Install jq: +```bash +brew install jq +``` + +### "Failed to parse configuration" +Ensure the appsettings JSON file exists and is valid JSON. + +### "Connection refused" +- Check that InfluxDB URL is accessible +- Verify network connectivity to the server +- Check firewall rules + +### "401 Unauthorized" +- Verify the InfluxDB token in appsettings is correct +- For exports: ensure token has read permissions for the bucket +- For imports: ensure token has write permissions for the bucket + +### "Bucket not found" +The import script will automatically create the bucket if you have permissions. + +Or create it manually: +```bash +influx bucket create \ + --name prices-bucket \ + --org managing-org \ + --retention 0 \ + --host https://influx-db.apps.managing.live \ + --token YOUR_TOKEN +``` + +### Import is slow +- This is normal for large files (240MB+ with 1.6M+ data points) +- Expected time: 5-15 minutes depending on network speed +- The script processes data in batches of 5000 points +- Progress is shown during import + +### Duplicate data after import +- Imports overwrite data with the same timestamp +- To avoid duplicates, don't import the same data twice +- To replace all data: delete the bucket first, then import + +## Performance Tips + +### For Large Exports +- Export specific time ranges instead of all data when possible +- Exports are faster than full database dumps +- CSV files compress well (use `gzip` for storage) + +### For Large Imports +- Import during low-traffic periods +- Monitor InfluxDB memory usage during import +- Consider splitting very large imports into time ranges + +## Verify Data After Import + +```bash +# Check recent data +influx query 'from(bucket:"prices-bucket") |> range(start:-7d) |> limit(n:10)' \ + --host https://influx-db.apps.managing.live \ + --org managing-org \ + --token YOUR_TOKEN + +# Count total records +influx query 'from(bucket:"prices-bucket") |> range(start:2020-01-01T00:00:00Z) |> count()' \ + --host https://influx-db.apps.managing.live \ + --org managing-org \ + --token YOUR_TOKEN + +# Check specific ticker +influx query 'from(bucket:"prices-bucket") |> range(start:-30d) |> filter(fn: (r) => r.ticker == "BTC")' \ + --host https://influx-db.apps.managing.live \ + --org managing-org \ + --token YOUR_TOKEN +``` + +## Manual Export/Import Commands + +If you need to run commands manually: + +**Export:** +```bash +influx query 'from(bucket: "prices-bucket") |> range(start: -30d)' \ + --host https://influx-db.apps.managing.live \ + --org managing-org \ + --token YOUR_TOKEN \ + --raw > export.csv +``` + +**Import:** +```bash +influx write \ + --host https://influx-db.apps.managing.live \ + --org managing-org \ + --token YOUR_TOKEN \ + --bucket prices-bucket \ + --format csv \ + --file export.csv +``` + +## Best Practices + +1. **Regular Exports**: Schedule regular exports of production data +2. **Test Imports**: Test imports on sandbox before production +3. **Verify After Import**: Always verify data integrity after import +4. **Document Changes**: Keep notes of what data was imported when +5. **Backup Before Major Changes**: Export before major data operations + +## Support + +For issues or questions, refer to: +- [InfluxDB Query Documentation](https://docs.influxdata.com/influxdb/v2.0/query-data/) +- [InfluxDB Write Documentation](https://docs.influxdata.com/influxdb/v2.0/write-data/) +- Project documentation in `/docs` + +## Script Locations + +All scripts are located in: `/Users/oda/Desktop/Projects/managing-apps/scripts/influxdb/` + +Configuration files: +- Sandbox: `/Users/oda/Desktop/Projects/managing-apps/src/Managing.Api/appsettings.SandboxLocal.json` +- Production: `/Users/oda/Desktop/Projects/managing-apps/src/Managing.Api/appsettings.ProductionLocal.json` diff --git a/scripts/influxdb/export-prices-bucket.sh b/scripts/influxdb/export-prices-bucket.sh new file mode 100755 index 00000000..f071655d --- /dev/null +++ b/scripts/influxdb/export-prices-bucket.sh @@ -0,0 +1,265 @@ +#!/bin/bash + +# InfluxDB Prices Bucket Data Export Script (No Admin Required) +# Usage: ./export-prices-bucket.sh + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Get the directory where the script is located +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" +SRC_DIR="$PROJECT_ROOT/src" + +# Logging functions +log() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +warn() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}" +} + +error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}" + exit 1 +} + +info() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] INFO: $1${NC}" +} + +# Check if influx CLI is installed +command -v influx >/dev/null 2>&1 || error "InfluxDB CLI is not installed. Please install it first: brew install influxdb-cli" + +# Check if jq is installed for JSON parsing +if ! command -v jq >/dev/null 2>&1; then + warn "jq is not installed. Installing it for JSON parsing..." + if command -v brew >/dev/null 2>&1; then + brew install jq || error "Failed to install jq. Please install it manually: brew install jq" + else + error "jq is not installed and brew is not available. Please install jq manually." + fi +fi + +# Prompt for environment +echo "" +echo "======================================" +echo " InfluxDB Prices Data Export" +echo "======================================" +echo "" +echo "Select environment:" +echo "1) SandboxLocal" +echo "2) ProductionLocal" +echo "" +read -p "Enter your choice (1 or 2): " ENV_CHOICE + +case $ENV_CHOICE in + 1) + ENVIRONMENT="SandboxLocal" + APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.SandboxLocal.json" + ;; + 2) + ENVIRONMENT="ProductionLocal" + APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.ProductionLocal.json" + ;; + *) + error "Invalid choice. Please run the script again and select 1 or 2." + ;; +esac + +log "Selected environment: $ENVIRONMENT" + +# Check if appsettings file exists +if [ ! -f "$APPSETTINGS_FILE" ]; then + error "Configuration file not found: $APPSETTINGS_FILE" +fi + +log "Reading configuration from: $APPSETTINGS_FILE" + +# Parse InfluxDB settings from JSON +INFLUX_URL=$(jq -r '.InfluxDb.Url' "$APPSETTINGS_FILE") +INFLUX_ORG=$(jq -r '.InfluxDb.Organization' "$APPSETTINGS_FILE") +INFLUX_TOKEN=$(jq -r '.InfluxDb.Token' "$APPSETTINGS_FILE") + +# Validate parsed values +if [ "$INFLUX_URL" = "null" ] || [ -z "$INFLUX_URL" ]; then + error "Failed to parse InfluxDb.Url from configuration file" +fi + +if [ "$INFLUX_ORG" = "null" ] || [ -z "$INFLUX_ORG" ]; then + error "Failed to parse InfluxDb.Organization from configuration file" +fi + +if [ "$INFLUX_TOKEN" = "null" ] || [ -z "$INFLUX_TOKEN" ]; then + error "Failed to parse InfluxDb.Token from configuration file" +fi + +info "InfluxDB URL: $INFLUX_URL" +info "Organization: $INFLUX_ORG" +info "Token: ${INFLUX_TOKEN:0:20}..." # Only show first 20 chars for security + +# Prompt for time range +echo "" +info "Select time range for export:" +echo "1) Last 7 days" +echo "2) Last 30 days" +echo "3) Last 90 days" +echo "4) Last 1 year" +echo "5) All data (from 2020-01-01)" +echo "6) Custom range" +echo "" +read -p "Enter your choice (1-6): " TIME_CHOICE + +case $TIME_CHOICE in + 1) + START_TIME="-7d" + TIME_DESC="Last 7 days" + ;; + 2) + START_TIME="-30d" + TIME_DESC="Last 30 days" + ;; + 3) + START_TIME="-90d" + TIME_DESC="Last 90 days" + ;; + 4) + START_TIME="-1y" + TIME_DESC="Last 1 year" + ;; + 5) + START_TIME="2020-01-01T00:00:00Z" + TIME_DESC="All data" + ;; + 6) + read -p "Enter start date (YYYY-MM-DD): " START_DATE + START_TIME="${START_DATE}T00:00:00Z" + TIME_DESC="From $START_DATE" + ;; + *) + error "Invalid choice" + ;; +esac + +log "Time range: $TIME_DESC" + +# Create export directory with timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +EXPORT_BASE_DIR="$SCRIPT_DIR/exports" +EXPORT_DIR="$EXPORT_BASE_DIR/$ENVIRONMENT/$TIMESTAMP" + +log "Creating export directory: $EXPORT_DIR" +mkdir -p "$EXPORT_DIR" || error "Failed to create export directory" + +# Bucket name +BUCKET_NAME="prices-bucket" + +log "Starting export of '$BUCKET_NAME' bucket..." +info "This may take a while depending on the data size..." +echo "" + +# Export data using CSV format (more reliable than backup for non-admin tokens) +EXPORT_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.csv" + +info "Exporting data to CSV..." + +# Build the Flux query +FLUX_QUERY="from(bucket: \"$BUCKET_NAME\") + |> range(start: $START_TIME) + |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")" + +# Export to CSV +if influx query "$FLUX_QUERY" \ + --host "$INFLUX_URL" \ + --org "$INFLUX_ORG" \ + --token "$INFLUX_TOKEN" \ + --raw > "$EXPORT_FILE" 2>&1; then + + log "✅ Export completed successfully!" + + # Get export size + EXPORT_SIZE=$(du -sh "$EXPORT_FILE" | cut -f1) + info "Export location: $EXPORT_FILE" + info "Export size: $EXPORT_SIZE" + + # Count lines (data points) + LINE_COUNT=$(wc -l < "$EXPORT_FILE" | xargs) + DATA_POINTS=$((LINE_COUNT - 1)) # Subtract header + info "Data points exported: $DATA_POINTS" + + # Save export metadata + METADATA_FILE="$EXPORT_DIR/export-metadata.txt" + cat > "$METADATA_FILE" << EOF +InfluxDB Export Metadata +======================== +Date: $(date) +Environment: $ENVIRONMENT +Bucket: $BUCKET_NAME +Time Range: $TIME_DESC +Start Time: $START_TIME +InfluxDB URL: $INFLUX_URL +Organization: $INFLUX_ORG +Export File: $EXPORT_FILE +Export Size: $EXPORT_SIZE +Data Points: $DATA_POINTS +Configuration File: $APPSETTINGS_FILE + +Flux Query Used: +---------------- +$FLUX_QUERY +EOF + + log "Metadata saved to: $METADATA_FILE" + + # Also save as line protocol for easier restore + info "Converting to line protocol format..." + LP_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.lp" + + # Use influx query with --raw format for line protocol + FLUX_QUERY_LP="from(bucket: \"$BUCKET_NAME\") + |> range(start: $START_TIME)" + + if influx query "$FLUX_QUERY_LP" \ + --host "$INFLUX_URL" \ + --org "$INFLUX_ORG" \ + --token "$INFLUX_TOKEN" \ + --raw > "$LP_FILE.tmp" 2>&1; then + + # Clean up the output (remove annotations) + grep -v "^#" "$LP_FILE.tmp" > "$LP_FILE" 2>/dev/null || true + rm -f "$LP_FILE.tmp" + + LP_SIZE=$(du -sh "$LP_FILE" | cut -f1 2>/dev/null || echo "0") + if [ -s "$LP_FILE" ]; then + info "Line protocol export: $LP_FILE ($LP_SIZE)" + else + warn "Line protocol export is empty, using CSV only" + rm -f "$LP_FILE" + fi + fi + + echo "" + log "🎉 Export process completed successfully!" + echo "" + info "Export files:" + ls -lh "$EXPORT_DIR" + echo "" + info "To restore this data, you can use:" + echo " 1. CSV import via InfluxDB UI" + echo " 2. Or use the import-prices-data.sh script (coming soon)" + if [ -f "$LP_FILE" ]; then + echo " 3. Line protocol: influx write --bucket $BUCKET_NAME --file \"$LP_FILE\"" + fi + echo "" + +else + error "Export failed! Check the error messages above." +fi + diff --git a/scripts/influxdb/import-csv-data.sh b/scripts/influxdb/import-csv-data.sh new file mode 100755 index 00000000..c404ee19 --- /dev/null +++ b/scripts/influxdb/import-csv-data.sh @@ -0,0 +1,378 @@ +#!/bin/bash + +# InfluxDB CSV Data Import Script +# Usage: ./import-csv-data.sh + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Get the directory where the script is located +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" +SRC_DIR="$PROJECT_ROOT/src" +EXPORTS_BASE_DIR="$SCRIPT_DIR/exports" + +# Logging functions +log() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +warn() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}" +} + +error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}" + exit 1 +} + +info() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] INFO: $1${NC}" +} + +# Check if influx CLI is installed +command -v influx >/dev/null 2>&1 || error "InfluxDB CLI is not installed. Please install it first: brew install influxdb-cli" + +# Check if jq is installed for JSON parsing +if ! command -v jq >/dev/null 2>&1; then + warn "jq is not installed. Installing it for JSON parsing..." + if command -v brew >/dev/null 2>&1; then + brew install jq || error "Failed to install jq. Please install it manually: brew install jq" + else + error "jq is not installed and brew is not available. Please install jq manually." + fi +fi + +echo "" +echo "============================================" +echo " InfluxDB CSV Data Import" +echo "============================================" +echo "" + +# Check if exports directory exists +if [ ! -d "$EXPORTS_BASE_DIR" ]; then + error "Exports directory not found: $EXPORTS_BASE_DIR" +fi + +# List available source environments +echo "Available export source environments:" +ENVIRONMENTS=($(ls -d "$EXPORTS_BASE_DIR"/*/ 2>/dev/null | xargs -n 1 basename)) + +if [ ${#ENVIRONMENTS[@]} -eq 0 ]; then + error "No export environments found in: $EXPORTS_BASE_DIR" +fi + +for i in "${!ENVIRONMENTS[@]}"; do + echo "$((i+1))) ${ENVIRONMENTS[$i]}" +done +echo "" +read -p "Select source environment (1-${#ENVIRONMENTS[@]}): " ENV_CHOICE + +if [ "$ENV_CHOICE" -lt 1 ] || [ "$ENV_CHOICE" -gt ${#ENVIRONMENTS[@]} ]; then + error "Invalid choice" +fi + +SOURCE_ENV="${ENVIRONMENTS[$((ENV_CHOICE-1))]}" +ENV_EXPORT_DIR="$EXPORTS_BASE_DIR/$SOURCE_ENV" + +log "Selected source environment: $SOURCE_ENV" + +# List available export timestamps +echo "" +echo "Available exports for $SOURCE_ENV:" +EXPORTS=($(ls -d "$ENV_EXPORT_DIR"/*/ 2>/dev/null | xargs -n 1 basename | sort -r)) + +if [ ${#EXPORTS[@]} -eq 0 ]; then + error "No exports found for environment: $SOURCE_ENV" +fi + +for i in "${!EXPORTS[@]}"; do + EXPORT_PATH="$ENV_EXPORT_DIR/${EXPORTS[$i]}" + METADATA_FILE="$EXPORT_PATH/export-metadata.txt" + + if [ -f "$METADATA_FILE" ]; then + EXPORT_SIZE=$(grep "Export Size:" "$METADATA_FILE" | cut -d: -f2 | xargs) + DATA_POINTS=$(grep "Data Points:" "$METADATA_FILE" | cut -d: -f2 | xargs) + EXPORT_DATE=$(grep "Date:" "$METADATA_FILE" | cut -d: -f2- | xargs) + echo "$((i+1))) ${EXPORTS[$i]} - $EXPORT_DATE ($EXPORT_SIZE, $DATA_POINTS points)" + else + echo "$((i+1))) ${EXPORTS[$i]}" + fi +done +echo "" +read -p "Select export to import (1-${#EXPORTS[@]}): " EXPORT_CHOICE + +if [ "$EXPORT_CHOICE" -lt 1 ] || [ "$EXPORT_CHOICE" -gt ${#EXPORTS[@]} ]; then + error "Invalid choice" +fi + +SELECTED_EXPORT="${EXPORTS[$((EXPORT_CHOICE-1))]}" +IMPORT_FROM_DIR="$ENV_EXPORT_DIR/$SELECTED_EXPORT" + +log "Selected export: $SELECTED_EXPORT" +info "Export location: $IMPORT_FROM_DIR" + +# Find CSV file +CSV_FILE=$(find "$IMPORT_FROM_DIR" -name "*.csv" | head -1) + +if [ ! -f "$CSV_FILE" ]; then + error "No CSV file found in: $IMPORT_FROM_DIR" +fi + +CSV_SIZE=$(du -sh "$CSV_FILE" | cut -f1) +info "CSV file: $(basename "$CSV_FILE") ($CSV_SIZE)" + +# Select target environment for import +echo "" +echo "Select TARGET environment for import:" +echo "1) SandboxLocal" +echo "2) ProductionLocal" +echo "" +read -p "Enter your choice (1 or 2): " TARGET_ENV_CHOICE + +case $TARGET_ENV_CHOICE in + 1) + TARGET_ENVIRONMENT="SandboxLocal" + APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.SandboxLocal.json" + ;; + 2) + TARGET_ENVIRONMENT="ProductionLocal" + APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.ProductionLocal.json" + ;; + *) + error "Invalid choice. Please run the script again and select 1 or 2." + ;; +esac + +log "Target environment: $TARGET_ENVIRONMENT" + +# Check if appsettings file exists +if [ ! -f "$APPSETTINGS_FILE" ]; then + error "Configuration file not found: $APPSETTINGS_FILE" +fi + +log "Reading configuration from: $APPSETTINGS_FILE" + +# Parse InfluxDB settings from JSON +INFLUX_URL=$(jq -r '.InfluxDb.Url' "$APPSETTINGS_FILE") +INFLUX_ORG=$(jq -r '.InfluxDb.Organization' "$APPSETTINGS_FILE") +INFLUX_TOKEN=$(jq -r '.InfluxDb.Token' "$APPSETTINGS_FILE") + +# Validate parsed values +if [ "$INFLUX_URL" = "null" ] || [ -z "$INFLUX_URL" ]; then + error "Failed to parse InfluxDb.Url from configuration file" +fi + +if [ "$INFLUX_ORG" = "null" ] || [ -z "$INFLUX_ORG" ]; then + error "Failed to parse InfluxDb.Organization from configuration file" +fi + +if [ "$INFLUX_TOKEN" = "null" ] || [ -z "$INFLUX_TOKEN" ]; then + error "Failed to parse InfluxDb.Token from configuration file" +fi + +info "Target InfluxDB URL: $INFLUX_URL" +info "Organization: $INFLUX_ORG" + +# Get bucket name +BUCKET_NAME="prices-bucket" + +# Check if bucket exists +info "Checking if bucket '$BUCKET_NAME' exists..." +if influx bucket list --host "$INFLUX_URL" --org "$INFLUX_ORG" --token "$INFLUX_TOKEN" --name "$BUCKET_NAME" &>/dev/null; then + log "✅ Bucket '$BUCKET_NAME' exists" +else + warn "Bucket '$BUCKET_NAME' does not exist!" + read -p "Create the bucket now? (yes/no): " CREATE_BUCKET + if [ "$CREATE_BUCKET" = "yes" ]; then + influx bucket create \ + --name "$BUCKET_NAME" \ + --retention 0 \ + --host "$INFLUX_URL" \ + --org "$INFLUX_ORG" \ + --token "$INFLUX_TOKEN" || error "Failed to create bucket" + log "✅ Bucket created successfully" + else + error "Cannot proceed without target bucket" + fi +fi + +# Final confirmation +echo "" +warn "⚠️ IMPORTANT INFORMATION:" +echo " Source: $SOURCE_ENV/$SELECTED_EXPORT" +echo " Target: $TARGET_ENVIRONMENT ($INFLUX_URL)" +echo " Bucket: $BUCKET_NAME" +echo " Data Size: $CSV_SIZE" +warn " This will ADD data to the bucket (existing data will be preserved)" +warn " Duplicate timestamps may cause overwrites" +echo "" +read -p "Are you sure you want to continue? (yes/no): " CONFIRM + +if [ "$CONFIRM" != "yes" ]; then + log "Import cancelled by user" + exit 0 +fi + +# Perform import +echo "" +log "🚀 Starting import operation..." +log "This may take several minutes for large files..." +echo "" + +# Create a temporary file for line protocol conversion +TEMP_LP_FILE=$(mktemp) +trap "rm -f $TEMP_LP_FILE" EXIT + +info "Converting CSV to line protocol format..." + +# Convert annotated CSV to line protocol using awk +# Skip annotation lines (starting with #) and empty lines +awk -F',' ' + BEGIN {OFS=","} + # Skip annotation lines + /^#/ {next} + # Skip empty lines + /^[[:space:]]*$/ {next} + # Process header to get field positions + NR==1 { + for (i=1; i<=NF; i++) { + field[$i] = i + } + next + } + # Process data rows + { + # Extract values + time = $field["_time"] + measurement = $field["_measurement"] + exchange = $field["exchange"] + ticker = $field["ticker"] + timeframe = $field["timeframe"] + + # Skip if essential fields are missing + if (time == "" || measurement == "" || exchange == "" || ticker == "" || timeframe == "") next + + # Build line protocol + # Format: measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp + printf "%s,exchange=%s,ticker=%s,timeframe=%s ", measurement, exchange, ticker, timeframe + + # Add fields + first = 1 + for (fname in field) { + if (fname != "_time" && fname != "_start" && fname != "_stop" && fname != "_measurement" && + fname != "exchange" && fname != "ticker" && fname != "timeframe" && + fname != "result" && fname != "table" && fname != "") { + val = $field[fname] + if (val != "" && val != "NaN") { + if (!first) printf "," + # Check if value is numeric + if (val ~ /^[0-9]+$/) { + printf "%s=%si", fname, val + } else { + printf "%s=%s", fname, val + } + first = 0 + } + } + } + + # Add timestamp (convert RFC3339 to nanoseconds if needed) + printf " %s\n", time + } +' "$CSV_FILE" > "$TEMP_LP_FILE" 2>/dev/null || { + warn "CSV parsing method 1 failed, trying direct import..." + + # Alternative: Use influx write with CSV format directly + info "Attempting direct CSV import..." + + if influx write \ + --host "$INFLUX_URL" \ + --org "$INFLUX_ORG" \ + --token "$INFLUX_TOKEN" \ + --bucket "$BUCKET_NAME" \ + --format csv \ + --file "$CSV_FILE" 2>&1; then + + log "✅ Import completed successfully using direct CSV method!" + + echo "" + log "📊 Import Summary" + echo "============================================" + info "Source: $SOURCE_ENV/$SELECTED_EXPORT" + info "Target: $TARGET_ENVIRONMENT" + info "Bucket: $BUCKET_NAME" + log "Status: Success" + echo "============================================" + echo "" + exit 0 + else + error "Both import methods failed. Please check the error messages above." + fi +} + +# If line protocol was generated, import it +if [ -s "$TEMP_LP_FILE" ]; then + LP_LINES=$(wc -l < "$TEMP_LP_FILE" | xargs) + info "Generated $LP_LINES lines of line protocol" + + # Import in batches to avoid timeouts + BATCH_SIZE=5000 + TOTAL_LINES=$LP_LINES + CURRENT_LINE=0 + + info "Importing in batches of $BATCH_SIZE lines..." + + while [ $CURRENT_LINE -lt $TOTAL_LINES ]; do + END_LINE=$((CURRENT_LINE + BATCH_SIZE)) + BATCH_NUM=$((CURRENT_LINE / BATCH_SIZE + 1)) + PROGRESS=$((CURRENT_LINE * 100 / TOTAL_LINES)) + + info "Processing batch $BATCH_NUM (Progress: ${PROGRESS}%)..." + + # Extract batch and import + sed -n "$((CURRENT_LINE + 1)),${END_LINE}p" "$TEMP_LP_FILE" | \ + influx write \ + --host "$INFLUX_URL" \ + --org "$INFLUX_ORG" \ + --token "$INFLUX_TOKEN" \ + --bucket "$BUCKET_NAME" \ + --precision s 2>&1 || { + warn "Batch $BATCH_NUM had errors, continuing..." + } + + CURRENT_LINE=$END_LINE + done + + log "✅ Import completed successfully!" +else + error "Failed to generate line protocol data" +fi + +# Final summary +echo "" +echo "============================================" +log "📊 Import Summary" +echo "============================================" +info "Source: $SOURCE_ENV/$SELECTED_EXPORT" +info "Target: $TARGET_ENVIRONMENT" +info "Bucket: $BUCKET_NAME" +info "File: $(basename "$CSV_FILE")" +info "Size: $CSV_SIZE" +log "Status: Complete" +echo "============================================" +echo "" + +log "🎉 Data successfully imported to $TARGET_ENVIRONMENT!" +echo "" +info "Verify the import with:" +echo " influx query 'from(bucket:\"$BUCKET_NAME\") |> range(start:-1d) |> limit(n:10)' \\" +echo " --host \"$INFLUX_URL\" --org \"$INFLUX_ORG\" --token \"$INFLUX_TOKEN\"" +echo "" + diff --git a/src/Managing.Api/appsettings.ProductionLocal.json b/src/Managing.Api/appsettings.ProductionLocal.json index c5e6cac4..6e3e6504 100644 --- a/src/Managing.Api/appsettings.ProductionLocal.json +++ b/src/Managing.Api/appsettings.ProductionLocal.json @@ -4,9 +4,9 @@ "Orleans": "Host=kaigen-db.kaigen.managing.live;Port=5433;Database=orleans;Username=postgres;Password=2ab5423dcca4aa2d" }, "InfluxDb": { - "Url": "https://influx-db.apps.managing.live", + "Url": "https://influx-db.kaigen.managing.live", "Organization": "managing-org", - "Token": "eOuXcXhH7CS13Iw4CTiDDpRjIjQtEVPOloD82pLPOejI4n0BsEj1YzUw0g3Cs1mdDG5m-RaxCavCMsVTtS5wIQ==" + "Token": "ul0pkKXfHutyQgNvJIYaowsCP0cJT_APW2L427sa5DX5ekxlpw43FA_a4_uIc9IG4W99XruRPRxcj2n31XXhZA==" }, "RunOrleansGrains": true, "AllowedHosts": "*"