#!/bin/bash # InfluxDB Prices Bucket Data Export Script (No Admin Required) # Usage: ./export-prices-bucket.sh set -e # Exit on any error # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Get the directory where the script is located SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" SRC_DIR="$PROJECT_ROOT/src" # Logging functions log() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}" } warn() { echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}" } error() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}" exit 1 } info() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] INFO: $1${NC}" } # Check if influx CLI is installed command -v influx >/dev/null 2>&1 || error "InfluxDB CLI is not installed. Please install it first: brew install influxdb-cli" # Check if jq is installed for JSON parsing if ! command -v jq >/dev/null 2>&1; then warn "jq is not installed. Installing it for JSON parsing..." if command -v brew >/dev/null 2>&1; then brew install jq || error "Failed to install jq. Please install it manually: brew install jq" else error "jq is not installed and brew is not available. Please install jq manually." fi fi # Prompt for environment echo "" echo "======================================" echo " InfluxDB Prices Data Export" echo "======================================" echo "" echo "Select environment:" echo "1) SandboxLocal" echo "2) ProductionLocal" echo "" read -p "Enter your choice (1 or 2): " ENV_CHOICE case $ENV_CHOICE in 1) ENVIRONMENT="SandboxLocal" APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.SandboxLocal.json" ;; 2) ENVIRONMENT="ProductionLocal" APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.ProductionLocal.json" ;; *) error "Invalid choice. Please run the script again and select 1 or 2." ;; esac log "Selected environment: $ENVIRONMENT" # Check if appsettings file exists if [ ! -f "$APPSETTINGS_FILE" ]; then error "Configuration file not found: $APPSETTINGS_FILE" fi log "Reading configuration from: $APPSETTINGS_FILE" # Parse InfluxDB settings from JSON INFLUX_URL=$(jq -r '.InfluxDb.Url' "$APPSETTINGS_FILE") INFLUX_ORG=$(jq -r '.InfluxDb.Organization' "$APPSETTINGS_FILE") INFLUX_TOKEN=$(jq -r '.InfluxDb.Token' "$APPSETTINGS_FILE") # Validate parsed values if [ "$INFLUX_URL" = "null" ] || [ -z "$INFLUX_URL" ]; then error "Failed to parse InfluxDb.Url from configuration file" fi if [ "$INFLUX_ORG" = "null" ] || [ -z "$INFLUX_ORG" ]; then error "Failed to parse InfluxDb.Organization from configuration file" fi if [ "$INFLUX_TOKEN" = "null" ] || [ -z "$INFLUX_TOKEN" ]; then error "Failed to parse InfluxDb.Token from configuration file" fi info "InfluxDB URL: $INFLUX_URL" info "Organization: $INFLUX_ORG" info "Token: ${INFLUX_TOKEN:0:20}..." # Only show first 20 chars for security # Prompt for time range echo "" info "Select time range for export:" echo "1) Last 7 days" echo "2) Last 30 days" echo "3) Last 90 days" echo "4) Last 1 year" echo "5) All data (from 2020-01-01)" echo "6) Custom range" echo "" read -p "Enter your choice (1-6): " TIME_CHOICE case $TIME_CHOICE in 1) START_TIME="-7d" TIME_DESC="Last 7 days" ;; 2) START_TIME="-30d" TIME_DESC="Last 30 days" ;; 3) START_TIME="-90d" TIME_DESC="Last 90 days" ;; 4) START_TIME="-1y" TIME_DESC="Last 1 year" ;; 5) START_TIME="2020-01-01T00:00:00Z" TIME_DESC="All data" ;; 6) read -p "Enter start date (YYYY-MM-DD): " START_DATE START_TIME="${START_DATE}T00:00:00Z" TIME_DESC="From $START_DATE" ;; *) error "Invalid choice" ;; esac log "Time range: $TIME_DESC" # Create export directory with timestamp TIMESTAMP=$(date +%Y%m%d_%H%M%S) EXPORT_BASE_DIR="$SCRIPT_DIR/exports" EXPORT_DIR="$EXPORT_BASE_DIR/$ENVIRONMENT/$TIMESTAMP" log "Creating export directory: $EXPORT_DIR" mkdir -p "$EXPORT_DIR" || error "Failed to create export directory" # Bucket name BUCKET_NAME="prices-bucket" log "Starting export of '$BUCKET_NAME' bucket..." info "This may take a while depending on the data size..." echo "" # Export data using CSV format (more reliable than backup for non-admin tokens) EXPORT_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.csv" info "Exporting data to CSV..." # Build the Flux query FLUX_QUERY="from(bucket: \"$BUCKET_NAME\") |> range(start: $START_TIME) |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")" # Export to CSV if influx query "$FLUX_QUERY" \ --host "$INFLUX_URL" \ --org "$INFLUX_ORG" \ --token "$INFLUX_TOKEN" \ --raw > "$EXPORT_FILE" 2>&1; then log "✅ Export completed successfully!" # Get export size EXPORT_SIZE=$(du -sh "$EXPORT_FILE" | cut -f1) info "Export location: $EXPORT_FILE" info "Export size: $EXPORT_SIZE" # Count lines (data points) LINE_COUNT=$(wc -l < "$EXPORT_FILE" | xargs) DATA_POINTS=$((LINE_COUNT - 1)) # Subtract header info "Data points exported: $DATA_POINTS" # Save export metadata METADATA_FILE="$EXPORT_DIR/export-metadata.txt" cat > "$METADATA_FILE" << EOF InfluxDB Export Metadata ======================== Date: $(date) Environment: $ENVIRONMENT Bucket: $BUCKET_NAME Time Range: $TIME_DESC Start Time: $START_TIME InfluxDB URL: $INFLUX_URL Organization: $INFLUX_ORG Export File: $EXPORT_FILE Export Size: $EXPORT_SIZE Data Points: $DATA_POINTS Configuration File: $APPSETTINGS_FILE Flux Query Used: ---------------- $FLUX_QUERY EOF log "Metadata saved to: $METADATA_FILE" # Also save as line protocol for easier restore info "Converting to line protocol format..." LP_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.lp" # Use influx query with --raw format for line protocol FLUX_QUERY_LP="from(bucket: \"$BUCKET_NAME\") |> range(start: $START_TIME)" if influx query "$FLUX_QUERY_LP" \ --host "$INFLUX_URL" \ --org "$INFLUX_ORG" \ --token "$INFLUX_TOKEN" \ --raw > "$LP_FILE.tmp" 2>&1; then # Clean up the output (remove annotations) grep -v "^#" "$LP_FILE.tmp" > "$LP_FILE" 2>/dev/null || true rm -f "$LP_FILE.tmp" LP_SIZE=$(du -sh "$LP_FILE" | cut -f1 2>/dev/null || echo "0") if [ -s "$LP_FILE" ]; then info "Line protocol export: $LP_FILE ($LP_SIZE)" else warn "Line protocol export is empty, using CSV only" rm -f "$LP_FILE" fi fi echo "" log "🎉 Export process completed successfully!" echo "" info "Export files:" ls -lh "$EXPORT_DIR" echo "" info "To restore this data, you can use:" echo " 1. CSV import via InfluxDB UI" echo " 2. Or use the import-prices-data.sh script (coming soon)" if [ -f "$LP_FILE" ]; then echo " 3. Line protocol: influx write --bucket $BUCKET_NAME --file \"$LP_FILE\"" fi echo "" else error "Export failed! Check the error messages above." fi