Add influxdb export/import + add new influxdb instance for prod
This commit is contained in:
265
scripts/influxdb/export-prices-bucket.sh
Executable file
265
scripts/influxdb/export-prices-bucket.sh
Executable file
@@ -0,0 +1,265 @@
|
||||
#!/bin/bash
|
||||
|
||||
# InfluxDB Prices Bucket Data Export Script (No Admin Required)
|
||||
# Usage: ./export-prices-bucket.sh
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Get the directory where the script is located
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
|
||||
SRC_DIR="$PROJECT_ROOT/src"
|
||||
|
||||
# Logging functions
|
||||
log() {
|
||||
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
info() {
|
||||
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] INFO: $1${NC}"
|
||||
}
|
||||
|
||||
# Check if influx CLI is installed
|
||||
command -v influx >/dev/null 2>&1 || error "InfluxDB CLI is not installed. Please install it first: brew install influxdb-cli"
|
||||
|
||||
# Check if jq is installed for JSON parsing
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
warn "jq is not installed. Installing it for JSON parsing..."
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
brew install jq || error "Failed to install jq. Please install it manually: brew install jq"
|
||||
else
|
||||
error "jq is not installed and brew is not available. Please install jq manually."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Prompt for environment
|
||||
echo ""
|
||||
echo "======================================"
|
||||
echo " InfluxDB Prices Data Export"
|
||||
echo "======================================"
|
||||
echo ""
|
||||
echo "Select environment:"
|
||||
echo "1) SandboxLocal"
|
||||
echo "2) ProductionLocal"
|
||||
echo ""
|
||||
read -p "Enter your choice (1 or 2): " ENV_CHOICE
|
||||
|
||||
case $ENV_CHOICE in
|
||||
1)
|
||||
ENVIRONMENT="SandboxLocal"
|
||||
APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.SandboxLocal.json"
|
||||
;;
|
||||
2)
|
||||
ENVIRONMENT="ProductionLocal"
|
||||
APPSETTINGS_FILE="$SRC_DIR/Managing.Api/appsettings.ProductionLocal.json"
|
||||
;;
|
||||
*)
|
||||
error "Invalid choice. Please run the script again and select 1 or 2."
|
||||
;;
|
||||
esac
|
||||
|
||||
log "Selected environment: $ENVIRONMENT"
|
||||
|
||||
# Check if appsettings file exists
|
||||
if [ ! -f "$APPSETTINGS_FILE" ]; then
|
||||
error "Configuration file not found: $APPSETTINGS_FILE"
|
||||
fi
|
||||
|
||||
log "Reading configuration from: $APPSETTINGS_FILE"
|
||||
|
||||
# Parse InfluxDB settings from JSON
|
||||
INFLUX_URL=$(jq -r '.InfluxDb.Url' "$APPSETTINGS_FILE")
|
||||
INFLUX_ORG=$(jq -r '.InfluxDb.Organization' "$APPSETTINGS_FILE")
|
||||
INFLUX_TOKEN=$(jq -r '.InfluxDb.Token' "$APPSETTINGS_FILE")
|
||||
|
||||
# Validate parsed values
|
||||
if [ "$INFLUX_URL" = "null" ] || [ -z "$INFLUX_URL" ]; then
|
||||
error "Failed to parse InfluxDb.Url from configuration file"
|
||||
fi
|
||||
|
||||
if [ "$INFLUX_ORG" = "null" ] || [ -z "$INFLUX_ORG" ]; then
|
||||
error "Failed to parse InfluxDb.Organization from configuration file"
|
||||
fi
|
||||
|
||||
if [ "$INFLUX_TOKEN" = "null" ] || [ -z "$INFLUX_TOKEN" ]; then
|
||||
error "Failed to parse InfluxDb.Token from configuration file"
|
||||
fi
|
||||
|
||||
info "InfluxDB URL: $INFLUX_URL"
|
||||
info "Organization: $INFLUX_ORG"
|
||||
info "Token: ${INFLUX_TOKEN:0:20}..." # Only show first 20 chars for security
|
||||
|
||||
# Prompt for time range
|
||||
echo ""
|
||||
info "Select time range for export:"
|
||||
echo "1) Last 7 days"
|
||||
echo "2) Last 30 days"
|
||||
echo "3) Last 90 days"
|
||||
echo "4) Last 1 year"
|
||||
echo "5) All data (from 2020-01-01)"
|
||||
echo "6) Custom range"
|
||||
echo ""
|
||||
read -p "Enter your choice (1-6): " TIME_CHOICE
|
||||
|
||||
case $TIME_CHOICE in
|
||||
1)
|
||||
START_TIME="-7d"
|
||||
TIME_DESC="Last 7 days"
|
||||
;;
|
||||
2)
|
||||
START_TIME="-30d"
|
||||
TIME_DESC="Last 30 days"
|
||||
;;
|
||||
3)
|
||||
START_TIME="-90d"
|
||||
TIME_DESC="Last 90 days"
|
||||
;;
|
||||
4)
|
||||
START_TIME="-1y"
|
||||
TIME_DESC="Last 1 year"
|
||||
;;
|
||||
5)
|
||||
START_TIME="2020-01-01T00:00:00Z"
|
||||
TIME_DESC="All data"
|
||||
;;
|
||||
6)
|
||||
read -p "Enter start date (YYYY-MM-DD): " START_DATE
|
||||
START_TIME="${START_DATE}T00:00:00Z"
|
||||
TIME_DESC="From $START_DATE"
|
||||
;;
|
||||
*)
|
||||
error "Invalid choice"
|
||||
;;
|
||||
esac
|
||||
|
||||
log "Time range: $TIME_DESC"
|
||||
|
||||
# Create export directory with timestamp
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
EXPORT_BASE_DIR="$SCRIPT_DIR/exports"
|
||||
EXPORT_DIR="$EXPORT_BASE_DIR/$ENVIRONMENT/$TIMESTAMP"
|
||||
|
||||
log "Creating export directory: $EXPORT_DIR"
|
||||
mkdir -p "$EXPORT_DIR" || error "Failed to create export directory"
|
||||
|
||||
# Bucket name
|
||||
BUCKET_NAME="prices-bucket"
|
||||
|
||||
log "Starting export of '$BUCKET_NAME' bucket..."
|
||||
info "This may take a while depending on the data size..."
|
||||
echo ""
|
||||
|
||||
# Export data using CSV format (more reliable than backup for non-admin tokens)
|
||||
EXPORT_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.csv"
|
||||
|
||||
info "Exporting data to CSV..."
|
||||
|
||||
# Build the Flux query
|
||||
FLUX_QUERY="from(bucket: \"$BUCKET_NAME\")
|
||||
|> range(start: $START_TIME)
|
||||
|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")"
|
||||
|
||||
# Export to CSV
|
||||
if influx query "$FLUX_QUERY" \
|
||||
--host "$INFLUX_URL" \
|
||||
--org "$INFLUX_ORG" \
|
||||
--token "$INFLUX_TOKEN" \
|
||||
--raw > "$EXPORT_FILE" 2>&1; then
|
||||
|
||||
log "✅ Export completed successfully!"
|
||||
|
||||
# Get export size
|
||||
EXPORT_SIZE=$(du -sh "$EXPORT_FILE" | cut -f1)
|
||||
info "Export location: $EXPORT_FILE"
|
||||
info "Export size: $EXPORT_SIZE"
|
||||
|
||||
# Count lines (data points)
|
||||
LINE_COUNT=$(wc -l < "$EXPORT_FILE" | xargs)
|
||||
DATA_POINTS=$((LINE_COUNT - 1)) # Subtract header
|
||||
info "Data points exported: $DATA_POINTS"
|
||||
|
||||
# Save export metadata
|
||||
METADATA_FILE="$EXPORT_DIR/export-metadata.txt"
|
||||
cat > "$METADATA_FILE" << EOF
|
||||
InfluxDB Export Metadata
|
||||
========================
|
||||
Date: $(date)
|
||||
Environment: $ENVIRONMENT
|
||||
Bucket: $BUCKET_NAME
|
||||
Time Range: $TIME_DESC
|
||||
Start Time: $START_TIME
|
||||
InfluxDB URL: $INFLUX_URL
|
||||
Organization: $INFLUX_ORG
|
||||
Export File: $EXPORT_FILE
|
||||
Export Size: $EXPORT_SIZE
|
||||
Data Points: $DATA_POINTS
|
||||
Configuration File: $APPSETTINGS_FILE
|
||||
|
||||
Flux Query Used:
|
||||
----------------
|
||||
$FLUX_QUERY
|
||||
EOF
|
||||
|
||||
log "Metadata saved to: $METADATA_FILE"
|
||||
|
||||
# Also save as line protocol for easier restore
|
||||
info "Converting to line protocol format..."
|
||||
LP_FILE="$EXPORT_DIR/${BUCKET_NAME}_data.lp"
|
||||
|
||||
# Use influx query with --raw format for line protocol
|
||||
FLUX_QUERY_LP="from(bucket: \"$BUCKET_NAME\")
|
||||
|> range(start: $START_TIME)"
|
||||
|
||||
if influx query "$FLUX_QUERY_LP" \
|
||||
--host "$INFLUX_URL" \
|
||||
--org "$INFLUX_ORG" \
|
||||
--token "$INFLUX_TOKEN" \
|
||||
--raw > "$LP_FILE.tmp" 2>&1; then
|
||||
|
||||
# Clean up the output (remove annotations)
|
||||
grep -v "^#" "$LP_FILE.tmp" > "$LP_FILE" 2>/dev/null || true
|
||||
rm -f "$LP_FILE.tmp"
|
||||
|
||||
LP_SIZE=$(du -sh "$LP_FILE" | cut -f1 2>/dev/null || echo "0")
|
||||
if [ -s "$LP_FILE" ]; then
|
||||
info "Line protocol export: $LP_FILE ($LP_SIZE)"
|
||||
else
|
||||
warn "Line protocol export is empty, using CSV only"
|
||||
rm -f "$LP_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log "🎉 Export process completed successfully!"
|
||||
echo ""
|
||||
info "Export files:"
|
||||
ls -lh "$EXPORT_DIR"
|
||||
echo ""
|
||||
info "To restore this data, you can use:"
|
||||
echo " 1. CSV import via InfluxDB UI"
|
||||
echo " 2. Or use the import-prices-data.sh script (coming soon)"
|
||||
if [ -f "$LP_FILE" ]; then
|
||||
echo " 3. Line protocol: influx write --bucket $BUCKET_NAME --file \"$LP_FILE\""
|
||||
fi
|
||||
echo ""
|
||||
|
||||
else
|
||||
error "Export failed! Check the error messages above."
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user