266 lines
7.2 KiB
Bash
Executable File
266 lines
7.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# InfluxDB Prices Bucket Data Export Script (No Admin Required)
|
|
# Usage: ./export-prices-bucket.sh
|
|
|
|
set -e # Exit on any error
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Get the directory where the script is located
|
|
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
|
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
|
|
SRC_DIR="$PROJECT_ROOT/src"
|
|
|
|
# Logging functions
|
|
log() {
|
|
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
|
|
}
|
|
|
|
warn() {
|
|
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}"
|
|
}
|
|
|
|
error() {
|
|
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}"
|
|
exit 1
|
|
}
|
|
|
|
info() {
|
|
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] INFO: $1${NC}"
|
|
}
|
|
|
|
# Check if influx CLI is installed
|
|
command -v influx >/dev/null 2>&1 || error "InfluxDB CLI is not installed. Please install it first: brew install influxdb-cli"
|
|
|
|
# Check if jq is installed for JSON parsing
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
warn "jq is not installed. Installing it for JSON parsing..."
|
|
if command -v brew >/dev/null 2>&1; then
|
|
brew install jq || error "Failed to install jq. Please install it manually: brew install jq"
|
|
else
|
|
error "jq is not installed and brew is not available. Please install jq manually."
|
|
fi
|
|
fi
|
|
|
|
# Prompt for environment
|
|
echo ""
|
|
echo "======================================"
|
|
echo " InfluxDB Prices Data Export"
|
|
echo "======================================"
|
|
echo ""
|
|
echo "Select environment:"
|
|
echo "1) SandboxRemote"
|
|
echo "2) ProductionRemote"
|
|
echo ""
|
|
read -p "Enter your choice (1 or 2): " ENV_CHOICE
|
|
|
|
case $ENV_CHOICE in
|
|
1)
|
|
ENVIRONMENT="SandboxRemote"
|
|
APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.SandboxRemote.json"
|
|
;;
|
|
2)
|
|
ENVIRONMENT="ProductionRemote"
|
|
APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.ProductionRemote.json"
|
|
;;
|
|
*)
|
|
error "Invalid choice. Please run the script again and select 1 or 2."
|
|
;;
|
|
esac
|
|
|
|
log "Selected environment: $ENVIRONMENT"
|
|
|
|
# Check if appsettings file exists
|
|
if [ ! -f "$APPSETTINGS_FILE" ]; then
|
|
error "Configuration file not found: $APPSETTINGS_FILE"
|
|
fi
|
|
|
|
log "Reading configuration from: $APPSETTINGS_FILE"
|
|
|
|
# Parse InfluxDB settings from JSON
|
|
INFLUX_URL=$(jq -r '.InfluxDb.Url' "$APPSETTINGS_FILE")
|
|
INFLUX_ORG=$(jq -r '.InfluxDb.Organization' "$APPSETTINGS_FILE")
|
|
INFLUX_TOKEN=$(jq -r '.InfluxDb.Token' "$APPSETTINGS_FILE")
|
|
|
|
# Validate parsed values
|
|
if [ "$INFLUX_URL" = "null" ] || [ -z "$INFLUX_URL" ]; then
|
|
error "Failed to parse InfluxDb.Url from configuration file"
|
|
fi
|
|
|
|
if [ "$INFLUX_ORG" = "null" ] || [ -z "$INFLUX_ORG" ]; then
|
|
error "Failed to parse InfluxDb.Organization from configuration file"
|
|
fi
|
|
|
|
if [ "$INFLUX_TOKEN" = "null" ] || [ -z "$INFLUX_TOKEN" ]; then
|
|
error "Failed to parse InfluxDb.Token from configuration file"
|
|
fi
|
|
|
|
info "InfluxDB URL: $INFLUX_URL"
|
|
info "Organization: $INFLUX_ORG"
|
|
info "Token: ${INFLUX_TOKEN:0:20}..." # Only show first 20 chars for security
|
|
|
|
# Prompt for time range
|
|
echo ""
|
|
info "Select time range for export:"
|
|
echo "1) Last 7 days"
|
|
echo "2) Last 30 days"
|
|
echo "3) Last 90 days"
|
|
echo "4) Last 1 year"
|
|
echo "5) All data (from 2020-01-01)"
|
|
echo "6) Custom range"
|
|
echo ""
|
|
read -p "Enter your choice (1-6): " TIME_CHOICE
|
|
|
|
case $TIME_CHOICE in
|
|
1)
|
|
START_TIME="-7d"
|
|
TIME_DESC="Last 7 days"
|
|
;;
|
|
2)
|
|
START_TIME="-30d"
|
|
TIME_DESC="Last 30 days"
|
|
;;
|
|
3)
|
|
START_TIME="-90d"
|
|
TIME_DESC="Last 90 days"
|
|
;;
|
|
4)
|
|
START_TIME="-1y"
|
|
TIME_DESC="Last 1 year"
|
|
;;
|
|
5)
|
|
START_TIME="2020-01-01T00:00:00Z"
|
|
TIME_DESC="All data"
|
|
;;
|
|
6)
|
|
read -p "Enter start date (YYYY-MM-DD): " START_DATE
|
|
START_TIME="${START_DATE}T00:00:00Z"
|
|
TIME_DESC="From $START_DATE"
|
|
;;
|
|
*)
|
|
error "Invalid choice"
|
|
;;
|
|
esac
|
|
|
|
log "Time range: $TIME_DESC"
|
|
|
|
# Create export directory with timestamp
|
|
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
EXPORT_BASE_DIR="$SCRIPT_DIR/exports"
|
|
EXPORT_DIR="$EXPORT_BASE_DIR/$ENVIRONMENT/$TIMESTAMP"
|
|
|
|
log "Creating export directory: $EXPORT_DIR"
|
|
mkdir -p "$EXPORT_DIR" || error "Failed to create export directory"
|
|
|
|
# Bucket name
|
|
BUCKET_NAME="prices-bucket"
|
|
|
|
log "Starting export of '$BUCKET_NAME' bucket..."
|
|
info "This may take a while depending on the data size..."
|
|
echo ""
|
|
|
|
# Export data using CSV format (more reliable than backup for non-admin tokens)
|
|
EXPORT_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.csv"
|
|
|
|
info "Exporting data to CSV..."
|
|
|
|
# Build the Flux query
|
|
FLUX_QUERY="from(bucket: \"$BUCKET_NAME\")
|
|
|> range(start: $START_TIME)
|
|
|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")"
|
|
|
|
# Export to CSV
|
|
if influx query "$FLUX_QUERY" \
|
|
--host "$INFLUX_URL" \
|
|
--org "$INFLUX_ORG" \
|
|
--token "$INFLUX_TOKEN" \
|
|
--raw > "$EXPORT_FILE" 2>&1; then
|
|
|
|
log "✅ Export completed successfully!"
|
|
|
|
# Get export size
|
|
EXPORT_SIZE=$(du -sh "$EXPORT_FILE" | cut -f1)
|
|
info "Export location: $EXPORT_FILE"
|
|
info "Export size: $EXPORT_SIZE"
|
|
|
|
# Count lines (data points)
|
|
LINE_COUNT=$(wc -l < "$EXPORT_FILE" | xargs)
|
|
DATA_POINTS=$((LINE_COUNT - 1)) # Subtract header
|
|
info "Data points exported: $DATA_POINTS"
|
|
|
|
# Save export metadata
|
|
METADATA_FILE="$EXPORT_DIR/export-metadata.txt"
|
|
cat > "$METADATA_FILE" << EOF
|
|
InfluxDB Export Metadata
|
|
========================
|
|
Date: $(date)
|
|
Environment: $ENVIRONMENT
|
|
Bucket: $BUCKET_NAME
|
|
Time Range: $TIME_DESC
|
|
Start Time: $START_TIME
|
|
InfluxDB URL: $INFLUX_URL
|
|
Organization: $INFLUX_ORG
|
|
Export File: $EXPORT_FILE
|
|
Export Size: $EXPORT_SIZE
|
|
Data Points: $DATA_POINTS
|
|
Configuration File: $APPSETTINGS_FILE
|
|
|
|
Flux Query Used:
|
|
----------------
|
|
$FLUX_QUERY
|
|
EOF
|
|
|
|
log "Metadata saved to: $METADATA_FILE"
|
|
|
|
# Also save as line protocol for easier restore
|
|
info "Converting to line protocol format..."
|
|
LP_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.lp"
|
|
|
|
# Use influx query with --raw format for line protocol
|
|
FLUX_QUERY_LP="from(bucket: \"$BUCKET_NAME\")
|
|
|> range(start: $START_TIME)"
|
|
|
|
if influx query "$FLUX_QUERY_LP" \
|
|
--host "$INFLUX_URL" \
|
|
--org "$INFLUX_ORG" \
|
|
--token "$INFLUX_TOKEN" \
|
|
--raw > "$LP_FILE.tmp" 2>&1; then
|
|
|
|
# Clean up the output (remove annotations)
|
|
grep -v "^#" "$LP_FILE.tmp" > "$LP_FILE" 2>/dev/null || true
|
|
rm -f "$LP_FILE.tmp"
|
|
|
|
LP_SIZE=$(du -sh "$LP_FILE" | cut -f1 2>/dev/null || echo "0")
|
|
if [ -s "$LP_FILE" ]; then
|
|
info "Line protocol export: $LP_FILE ($LP_SIZE)"
|
|
else
|
|
warn "Line protocol export is empty, using CSV only"
|
|
rm -f "$LP_FILE"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
log "🎉 Export process completed successfully!"
|
|
echo ""
|
|
info "Export files:"
|
|
ls -lh "$EXPORT_DIR"
|
|
echo ""
|
|
info "To restore this data, you can use:"
|
|
echo " 1. CSV import via InfluxDB UI"
|
|
echo " 2. Or use the import-prices-data.sh script (coming soon)"
|
|
if [ -f "$LP_FILE" ]; then
|
|
echo " 3. Line protocol: influx write --bucket $BUCKET_NAME --file \"$LP_FILE\""
|
|
fi
|
|
echo ""
|
|
|
|
else
|
|
error "Export failed! Check the error messages above."
|
|
fi
|
|
|