GT AI OS Community v2.0.33 - Add NVIDIA NIM and Nemotron agents
- Updated python_coding_microproject.csv to use NVIDIA NIM Kimi K2 - Updated kali_linux_shell_simulator.csv to use NVIDIA NIM Kimi K2 - Made more general-purpose (flexible targets, expanded tools) - Added nemotron-mini-agent.csv for fast local inference via Ollama - Added nemotron-agent.csv for advanced reasoning via Ollama - Added wiki page: Projects for NVIDIA NIMs and Nemotron
This commit is contained in:
HackWeasel
61
scripts/lib/common.sh
Executable file
61
scripts/lib/common.sh
Executable file
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
# GT 2.0 Common Library Functions
|
||||
# Shared utilities for deployment scripts
|
||||
|
||||
# Color codes for output formatting
|
||||
export RED='\033[0;31m'
|
||||
export GREEN='\033[0;32m'
|
||||
export YELLOW='\033[1;33m'
|
||||
export BLUE='\033[0;34m'
|
||||
export NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions with timestamps
|
||||
log_info() {
|
||||
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] ℹ️ $*${NC}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] ✅ $*${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] ⚠️ $*${NC}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ❌ $*${NC}"
|
||||
}
|
||||
|
||||
log_header() {
|
||||
echo ""
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE}$*${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Check if running from GT-2.0 root directory
|
||||
check_root_directory() {
|
||||
if [ ! -f "docker-compose.yml" ]; then
|
||||
log_error "docker-compose.yml not found"
|
||||
echo "Please run this script from the GT-2.0 root directory"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Prompt for user confirmation
|
||||
confirm() {
|
||||
local message="$1"
|
||||
read -p "$(echo -e "${YELLOW}${message} (y/N) ${NC}")" -n 1 -r
|
||||
echo
|
||||
[[ $REPLY =~ ^[Yy]$ ]]
|
||||
}
|
||||
|
||||
# Check if deployment is running
|
||||
check_deployment_running() {
|
||||
if ! docker ps --filter "name=gentwo-" --format "{{.Names}}" | grep -q "gentwo-"; then
|
||||
log_warning "No running deployment found"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
330
scripts/lib/docker.sh
Executable file
330
scripts/lib/docker.sh
Executable file
@@ -0,0 +1,330 @@
|
||||
#!/bin/bash
|
||||
# GT 2.0 Docker Compose Wrapper Functions
|
||||
# Unified interface for platform-specific compose operations
|
||||
|
||||
# ==============================================
|
||||
# VOLUME MIGRATION (DEPRECATED - Removed Dec 2025)
|
||||
# This function has been removed because:
|
||||
# 1. It could overwrite good data with stale data from old volumes
|
||||
# 2. Docker Compose handles volumes naturally - let it manage them
|
||||
# 3. Manual migration is safer for deployments with custom volume names
|
||||
#
|
||||
# For manual migration (if needed):
|
||||
# 1. docker compose down
|
||||
# 2. docker run --rm -v old_vol:/src -v new_vol:/dst alpine cp -a /src/. /dst/
|
||||
# 3. docker compose up -d
|
||||
# ==============================================
|
||||
|
||||
# migrate_volumes_if_needed() - REMOVED
|
||||
# Function body removed to prevent accidental data loss
|
||||
|
||||
# ==============================================
|
||||
# PROJECT MIGRATION (DEPRECATED - Removed Dec 2025)
|
||||
# This function has been removed because:
|
||||
# 1. It aggressively stops/removes all containers
|
||||
# 2. Different project names don't cause issues if volumes persist
|
||||
# 3. Docker Compose derives project name from directory naturally
|
||||
#
|
||||
# Containers from different project names can coexist. If you need
|
||||
# to clean up old containers manually:
|
||||
# docker ps -a --format '{{.Names}}' | grep gentwo- | xargs docker rm -f
|
||||
# ==============================================
|
||||
|
||||
# migrate_project_if_needed() - REMOVED
|
||||
# Function body removed to prevent accidental container/data loss
|
||||
|
||||
# ==============================================
|
||||
# CONTAINER CLEANUP
|
||||
# Removes existing containers to prevent name conflicts during restart
|
||||
# ==============================================
|
||||
|
||||
remove_existing_container() {
|
||||
local service="$1"
|
||||
|
||||
# Get container name from compose config for this service
|
||||
local container_name=$(dc config --format json 2>/dev/null | jq -r ".services[\"$service\"].container_name // empty" 2>/dev/null)
|
||||
|
||||
if [ -n "$container_name" ]; then
|
||||
# Check if container exists (running or stopped)
|
||||
if docker ps -a --format '{{.Names}}' | grep -q "^${container_name}$"; then
|
||||
log_info "Removing existing container $container_name..."
|
||||
docker rm -f "$container_name" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Remove ALL gentwo-* containers to handle project name conflicts
|
||||
# This is needed when switching between project names (gt2 vs gt-20)
|
||||
cleanup_conflicting_containers() {
|
||||
# Skip in dry-run mode
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo "[DRY RUN] Would remove all gentwo-* containers"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Checking for conflicting containers..."
|
||||
|
||||
local containers=$(docker ps -a --format '{{.Names}}' | grep "^gentwo-" || true)
|
||||
|
||||
if [ -n "$containers" ]; then
|
||||
log_info "Removing existing gentwo-* containers to prevent conflicts..."
|
||||
for container in $containers; do
|
||||
docker rm -f "$container" 2>/dev/null || true
|
||||
done
|
||||
log_success "Removed conflicting containers"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================
|
||||
# DOCKER COMPOSE WRAPPER
|
||||
# ==============================================
|
||||
|
||||
# Execute docker compose with platform-specific files
|
||||
# No explicit project name - Docker Compose derives it from directory name
|
||||
# This ensures existing volumes (gt-20_*, gt2_*, etc.) continue to be used
|
||||
dc() {
|
||||
local platform="${PLATFORM:-$(detect_platform)}"
|
||||
local compose_files=$(get_compose_file "$platform" "$DEV_MODE")
|
||||
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo "[DRY RUN] docker compose -f $compose_files $*"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Pipe 'n' to auto-answer "no" to volume recreation prompts
|
||||
# This handles cases where bind mount paths don't match existing volumes
|
||||
yes n 2>/dev/null | docker compose -f $compose_files "$@"
|
||||
}
|
||||
|
||||
# Detect IMAGE_TAG from current git branch if not already set
|
||||
detect_image_tag() {
|
||||
# If IMAGE_TAG is already set, use it
|
||||
if [ -n "$IMAGE_TAG" ]; then
|
||||
log_info "Using IMAGE_TAG=$IMAGE_TAG (from environment)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Detect current git branch
|
||||
local branch=$(git branch --show-current 2>/dev/null)
|
||||
|
||||
case "$branch" in
|
||||
main|master)
|
||||
IMAGE_TAG="latest"
|
||||
;;
|
||||
dev|develop)
|
||||
IMAGE_TAG="dev"
|
||||
;;
|
||||
*)
|
||||
# Feature branches: sanitize branch name for Docker tag
|
||||
# Docker tags only allow [a-zA-Z0-9_.-], so replace / with -
|
||||
IMAGE_TAG="${branch//\//-}"
|
||||
;;
|
||||
esac
|
||||
|
||||
export IMAGE_TAG
|
||||
log_info "Auto-detected IMAGE_TAG=$IMAGE_TAG (branch: $branch)"
|
||||
}
|
||||
|
||||
# Try to authenticate Docker with GHCR using gh CLI (optional, for private repos)
|
||||
# Returns 0 if auth succeeds, 1 if not available or fails
|
||||
try_ghcr_auth() {
|
||||
# Skip in dry-run mode
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo "[DRY RUN] Try GHCR authentication"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if gh CLI is available
|
||||
if ! command -v gh &>/dev/null; then
|
||||
log_info "gh CLI not installed - skipping GHCR auth"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if gh is authenticated
|
||||
if ! gh auth status &>/dev/null 2>&1; then
|
||||
log_info "gh CLI not authenticated - skipping GHCR auth"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get GitHub username
|
||||
local gh_user=$(gh api user --jq '.login' 2>/dev/null)
|
||||
if [ -z "$gh_user" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get token and authenticate Docker
|
||||
local gh_token=$(gh auth token 2>/dev/null)
|
||||
if [ -z "$gh_token" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if echo "$gh_token" | docker login ghcr.io -u "$gh_user" --password-stdin &>/dev/null; then
|
||||
log_success "Authenticated with GHCR as $gh_user"
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Pull images with simplified auth flow
|
||||
# 1. Try pull without auth (works for public repos)
|
||||
# 2. If auth error, try gh CLI auth and retry
|
||||
# 3. If still fails, fall back to local build
|
||||
pull_images() {
|
||||
# Auto-detect image tag from git branch
|
||||
detect_image_tag
|
||||
|
||||
log_info "Pulling Docker images (tag: $IMAGE_TAG)..."
|
||||
|
||||
# Skip in dry-run mode
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo "[DRY RUN] docker compose pull"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# First attempt: try pull without auth (works for public repos)
|
||||
local pull_output
|
||||
pull_output=$(dc pull 2>&1) && {
|
||||
log_success "Successfully pulled images"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Check if it's an auth error (private repo)
|
||||
if echo "$pull_output" | grep -qi "unauthorized\|denied\|authentication required\|403"; then
|
||||
log_info "Registry requires authentication, attempting GHCR login..."
|
||||
|
||||
# Try to authenticate with gh CLI
|
||||
if try_ghcr_auth; then
|
||||
# Retry pull after auth
|
||||
if dc pull 2>&1; then
|
||||
log_success "Successfully pulled images after authentication"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
log_warning "Could not pull from registry - will build images locally"
|
||||
log_info "For faster deploys, install gh CLI and run: gh auth login"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check for rate limiting
|
||||
if echo "$pull_output" | grep -qi "rate limit\|too many requests"; then
|
||||
log_warning "Rate limited - continuing with existing images"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Other error - log and continue
|
||||
log_warning "Pull failed: ${pull_output:0:200}"
|
||||
log_info "Continuing with existing or locally built images"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Restart application service (uses pulled images by default, --build in dev mode)
|
||||
restart_app_service() {
|
||||
local service="$1"
|
||||
local build_flag=""
|
||||
|
||||
# Only use --build in dev mode (to apply local code changes)
|
||||
# In production mode, use pre-pulled GHCR images
|
||||
if [ "$DEV_MODE" = "true" ]; then
|
||||
build_flag="--build"
|
||||
log_info "Rebuilding and restarting $service (dev mode)..."
|
||||
else
|
||||
log_info "Restarting $service with pulled image..."
|
||||
fi
|
||||
|
||||
# In dry-run mode, just show the command that would be executed
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
dc up -d $build_flag "$service"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Remove existing container to prevent name conflicts
|
||||
remove_existing_container "$service"
|
||||
|
||||
# Start/restart service regardless of current state
|
||||
# dc up -d handles both starting new and restarting existing containers
|
||||
# Use --force-recreate to ensure container uses new image
|
||||
dc up -d --force-recreate $build_flag "$service" || {
|
||||
log_warning "Service $service may not be defined in compose files, skipping"
|
||||
return 0
|
||||
}
|
||||
sleep 2
|
||||
return 0
|
||||
}
|
||||
|
||||
# Legacy alias for backward compatibility
|
||||
rebuild_service() {
|
||||
restart_app_service "$@"
|
||||
}
|
||||
|
||||
# Restart service without rebuild
|
||||
restart_service() {
|
||||
local service="$1"
|
||||
|
||||
log_info "Restarting $service..."
|
||||
|
||||
# In dry-run mode, just show the command
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
dc up -d "$service"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Remove existing container to prevent name conflicts
|
||||
remove_existing_container "$service"
|
||||
|
||||
# Use dc up -d which handles both starting and restarting
|
||||
# Use --force-recreate to ensure container is recreated cleanly
|
||||
dc up -d --force-recreate "$service" || {
|
||||
log_warning "Service $service may not be defined in compose files, skipping"
|
||||
return 0
|
||||
}
|
||||
sleep 2
|
||||
return 0
|
||||
}
|
||||
|
||||
# Check service health
|
||||
check_service_health() {
|
||||
log_info "Checking service health..."
|
||||
|
||||
local unhealthy=$(dc ps --format json | jq -r 'select(.Health == "unhealthy") | .Service' 2>/dev/null || true)
|
||||
|
||||
if [ -n "$unhealthy" ]; then
|
||||
log_error "Unhealthy services detected: $unhealthy"
|
||||
echo "Check logs with: docker compose logs $unhealthy"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "All services healthy"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Display service status
|
||||
show_service_status() {
|
||||
log_info "Service Status:"
|
||||
dc ps --format "table {{.Service}}\t{{.Status}}"
|
||||
}
|
||||
|
||||
# Clean up unused Docker resources after deployment
|
||||
cleanup_docker_resources() {
|
||||
log_info "Cleaning up unused Docker resources..."
|
||||
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo "[DRY RUN] docker image prune -f"
|
||||
echo "[DRY RUN] docker builder prune -f"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# NOTE: Volume prune removed - too risky, can delete important data
|
||||
# if containers were stopped earlier in the deployment process
|
||||
|
||||
# Remove dangling images (untagged, not used by any container)
|
||||
local images_removed=$(docker image prune -f 2>/dev/null | grep "Total reclaimed space" || echo "0B")
|
||||
|
||||
# Remove build cache
|
||||
local cache_removed=$(docker builder prune -f 2>/dev/null | grep "Total reclaimed space" || echo "0B")
|
||||
|
||||
log_success "Cleanup complete"
|
||||
log_info " Images: $images_removed"
|
||||
log_info " Build cache: $cache_removed"
|
||||
}
|
||||
73
scripts/lib/health.sh
Executable file
73
scripts/lib/health.sh
Executable file
@@ -0,0 +1,73 @@
|
||||
#!/bin/bash
|
||||
# GT 2.0 Health Check and Service Status Functions
|
||||
# Verify service availability and display access points
|
||||
|
||||
# Wait for services to stabilize
|
||||
wait_for_stability() {
|
||||
local wait_time="${1:-10}"
|
||||
log_info "Waiting for services to stabilize..."
|
||||
sleep "$wait_time"
|
||||
}
|
||||
|
||||
# Check if all services are healthy
|
||||
check_all_services_healthy() {
|
||||
check_service_health
|
||||
}
|
||||
|
||||
# Display access points
|
||||
show_access_points() {
|
||||
echo ""
|
||||
log_success "Deployment Complete!"
|
||||
echo ""
|
||||
echo "🌐 Access Points:"
|
||||
echo " • Control Panel: http://localhost:3001"
|
||||
echo " • Tenant App: http://localhost:3002"
|
||||
echo ""
|
||||
echo "📊 Service Status:"
|
||||
show_service_status
|
||||
echo ""
|
||||
echo "📊 View Logs: docker compose logs -f"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Comprehensive health check with detailed output
|
||||
health_check_detailed() {
|
||||
log_header "Health Check"
|
||||
|
||||
# Check PostgreSQL databases
|
||||
log_info "Checking PostgreSQL databases..."
|
||||
if docker exec gentwo-controlpanel-postgres pg_isready -U postgres -d gt2_admin &>/dev/null; then
|
||||
log_success "Admin database: healthy"
|
||||
else
|
||||
log_error "Admin database: unhealthy"
|
||||
fi
|
||||
|
||||
if docker exec gentwo-tenant-postgres-primary pg_isready -U postgres -d gt2_tenants &>/dev/null; then
|
||||
log_success "Tenant database: healthy"
|
||||
else
|
||||
log_error "Tenant database: unhealthy"
|
||||
fi
|
||||
|
||||
# Check backend services
|
||||
log_info "Checking backend services..."
|
||||
if curl -sf http://localhost:8001/health &>/dev/null; then
|
||||
log_success "Control Panel backend: healthy"
|
||||
else
|
||||
log_warning "Control Panel backend: not responding"
|
||||
fi
|
||||
|
||||
if curl -sf http://localhost:8002/health &>/dev/null; then
|
||||
log_success "Tenant backend: healthy"
|
||||
else
|
||||
log_warning "Tenant backend: not responding"
|
||||
fi
|
||||
|
||||
if curl -sf http://localhost:8004/health &>/dev/null; then
|
||||
log_success "Resource cluster: healthy"
|
||||
else
|
||||
log_warning "Resource cluster: not responding"
|
||||
fi
|
||||
|
||||
# Check overall container health
|
||||
check_all_services_healthy
|
||||
}
|
||||
473
scripts/lib/migrations.sh
Executable file
473
scripts/lib/migrations.sh
Executable file
@@ -0,0 +1,473 @@
|
||||
#!/bin/bash
|
||||
# GT 2.0 Database Migration Functions
|
||||
# Idempotent migration checks and execution for admin and tenant databases
|
||||
|
||||
# Check if admin postgres container is running
|
||||
check_admin_db_running() {
|
||||
docker ps --filter "name=gentwo-controlpanel-postgres" --filter "status=running" --format "{{.Names}}" | grep -q "gentwo-controlpanel-postgres"
|
||||
}
|
||||
|
||||
# Check if tenant postgres container is running
|
||||
check_tenant_db_running() {
|
||||
docker ps --filter "name=gentwo-tenant-postgres-primary" --filter "status=running" --format "{{.Names}}" | grep -q "gentwo-tenant-postgres-primary"
|
||||
}
|
||||
|
||||
# Wait for a container to be healthy (up to 60 seconds)
|
||||
wait_for_container_healthy() {
|
||||
local container="$1"
|
||||
local max_wait=60
|
||||
local waited=0
|
||||
|
||||
log_info "Waiting for $container to be healthy..."
|
||||
while [ $waited -lt $max_wait ]; do
|
||||
local status=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "none")
|
||||
if [ "$status" = "healthy" ]; then
|
||||
log_success "$container is healthy"
|
||||
return 0
|
||||
fi
|
||||
# Also accept running containers without healthcheck
|
||||
local running=$(docker inspect --format='{{.State.Running}}' "$container" 2>/dev/null || echo "false")
|
||||
if [ "$running" = "true" ] && [ "$status" = "none" ]; then
|
||||
sleep 5 # Give it a few seconds to initialize
|
||||
log_success "$container is running"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
waited=$((waited + 2))
|
||||
done
|
||||
log_error "$container failed to become healthy after ${max_wait}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Ensure admin database is running
|
||||
ensure_admin_db_running() {
|
||||
if check_admin_db_running; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Starting admin database containers..."
|
||||
dc up -d postgres 2>/dev/null || {
|
||||
log_error "Failed to start admin database"
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_container_healthy "gentwo-controlpanel-postgres" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
# Ensure tenant database is running
|
||||
ensure_tenant_db_running() {
|
||||
if check_tenant_db_running; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Starting tenant database containers..."
|
||||
dc up -d tenant-postgres-primary 2>/dev/null || {
|
||||
log_error "Failed to start tenant database"
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_container_healthy "gentwo-tenant-postgres-primary" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
# Run admin database migration
|
||||
run_admin_migration() {
|
||||
local migration_num="$1"
|
||||
local migration_file="$2"
|
||||
local check_func="$3"
|
||||
|
||||
# Run check function if provided
|
||||
if [ -n "$check_func" ] && type "$check_func" &>/dev/null; then
|
||||
if ! $check_func; then
|
||||
return 0 # Migration already applied
|
||||
fi
|
||||
fi
|
||||
|
||||
log_info "Applying migration $migration_num..."
|
||||
|
||||
if [ ! -f "$migration_file" ]; then
|
||||
log_error "Migration script not found: $migration_file"
|
||||
echo "Run: git pull"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if docker exec -i gentwo-controlpanel-postgres psql -U postgres -d gt2_admin < "$migration_file"; then
|
||||
log_success "Migration $migration_num applied successfully"
|
||||
return 0
|
||||
else
|
||||
log_error "Migration $migration_num failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run tenant database migration
|
||||
run_tenant_migration() {
|
||||
local migration_num="$1"
|
||||
local migration_file="$2"
|
||||
local check_func="$3"
|
||||
|
||||
# Run check function if provided
|
||||
if [ -n "$check_func" ] && type "$check_func" &>/dev/null; then
|
||||
if ! $check_func; then
|
||||
return 0 # Migration already applied
|
||||
fi
|
||||
fi
|
||||
|
||||
log_info "Applying migration $migration_num..."
|
||||
|
||||
if [ ! -f "$migration_file" ]; then
|
||||
log_error "Migration script not found: $migration_file"
|
||||
echo "Run: git pull"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if docker exec -i gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants < "$migration_file"; then
|
||||
log_success "Migration $migration_num applied successfully"
|
||||
return 0
|
||||
else
|
||||
log_error "Migration $migration_num failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Admin migration checks
|
||||
check_migration_006() {
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='tenants' AND column_name='frontend_url');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_008() {
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='password_reset_rate_limits' AND column_name='ip_address');" 2>/dev/null || echo "false")
|
||||
[ "$exists" = "t" ]
|
||||
}
|
||||
|
||||
check_migration_009() {
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='users' AND column_name='tfa_enabled');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_010() {
|
||||
local count=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM model_configs WHERE (context_window IS NULL OR max_tokens IS NULL) AND provider = 'groq';" 2>/dev/null || echo "error")
|
||||
[ "$count" != "0" ] && [ "$count" != "error" ] && [ -n "$count" ]
|
||||
}
|
||||
|
||||
check_migration_011() {
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='public' AND table_name='system_versions');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_012() {
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='tenants' AND column_name='optics_enabled');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_013() {
|
||||
# Returns true (needs migration) if old column exists
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='model_configs' AND column_name='cost_per_1k_input');" 2>/dev/null || echo "false")
|
||||
[ "$exists" = "t" ]
|
||||
}
|
||||
|
||||
check_migration_014() {
|
||||
# Returns true (needs migration) if any Groq model has NULL or 0 pricing
|
||||
local count=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM model_configs WHERE provider = 'groq' AND (cost_per_million_input IS NULL OR cost_per_million_input = 0 OR cost_per_million_output IS NULL OR cost_per_million_output = 0);" 2>/dev/null || echo "0")
|
||||
[ "$count" != "0" ] && [ -n "$count" ]
|
||||
}
|
||||
|
||||
check_migration_015() {
|
||||
# Returns true (needs migration) if pricing is outdated
|
||||
# Check if gpt-oss-120b has old pricing ($1.20) instead of new ($0.15)
|
||||
local price=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT cost_per_million_input FROM model_configs WHERE model_id LIKE '%gpt-oss-120b%' LIMIT 1;" 2>/dev/null || echo "0")
|
||||
# Needs migration if price is > 1.0 (old pricing was $1.20)
|
||||
[ "$(echo "$price > 1.0" | bc -l 2>/dev/null || echo "0")" = "1" ]
|
||||
}
|
||||
|
||||
check_migration_016() {
|
||||
# Returns true (needs migration) if is_compound column doesn't exist
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='model_configs' AND column_name='is_compound');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_017() {
|
||||
# Returns true (needs migration) if compound pricing is incorrect (> $0.50 input means old pricing)
|
||||
local price=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT cost_per_million_input FROM model_configs WHERE model_id LIKE '%compound%' AND model_id NOT LIKE '%mini%' LIMIT 1;" 2>/dev/null || echo "0")
|
||||
[ "$(echo "$price > 0.50" | bc -l 2>/dev/null || echo "0")" = "1" ]
|
||||
}
|
||||
|
||||
check_migration_018() {
|
||||
# Returns true (needs migration) if monthly_budget_cents column doesn't exist
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='tenants' AND column_name='monthly_budget_cents');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_019() {
|
||||
# Returns true (needs migration) if embedding_usage_logs table doesn't exist
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='public' AND table_name='embedding_usage_logs');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_020() {
|
||||
# Returns true (needs migration) if:
|
||||
# 1. GROQ_API_KEY env var exists and is not a placeholder
|
||||
# 2. AND test-company tenant exists
|
||||
# 3. AND groq key is NOT already in database for test-company
|
||||
|
||||
# Check if GROQ_API_KEY env var exists
|
||||
local groq_key="${GROQ_API_KEY:-}"
|
||||
if [ -z "$groq_key" ] || [ "$groq_key" = "gsk_your_actual_groq_api_key_here" ] || [ "$groq_key" = "gsk_placeholder" ]; then
|
||||
# No valid env key to migrate
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if test-company tenant exists and has groq key already
|
||||
local has_key=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT 1 FROM tenants WHERE domain = 'test-company' AND api_keys IS NOT NULL AND api_keys->>'groq' IS NOT NULL AND api_keys->'groq'->>'key' IS NOT NULL);" 2>/dev/null || echo "false")
|
||||
|
||||
# If tenant already has key, no migration needed
|
||||
[ "$has_key" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_021() {
|
||||
# Returns true (needs migration) if NVIDIA models don't exist in model_configs
|
||||
local count=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM model_configs WHERE provider = 'nvidia';" 2>/dev/null || echo "0")
|
||||
[ "$count" = "0" ] || [ -z "$count" ]
|
||||
}
|
||||
|
||||
check_migration_022() {
|
||||
# Returns true (needs migration) if sessions table doesn't exist
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='public' AND table_name='sessions');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_023() {
|
||||
# Returns true (needs migration) if model_configs.id UUID column doesn't exist
|
||||
# This migration adds proper UUID primary key instead of using model_id string
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='public' AND table_name='model_configs' AND column_name='id' AND data_type='uuid');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_024() {
|
||||
# Returns true (needs migration) if model_configs still has unique constraint on model_id alone
|
||||
# (should be unique on model_id + provider instead)
|
||||
local exists=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.table_constraints WHERE constraint_name='model_configs_model_id_unique' AND table_name='model_configs' AND table_schema='public');" 2>/dev/null || echo "false")
|
||||
[ "$exists" = "t" ]
|
||||
}
|
||||
|
||||
check_migration_025() {
|
||||
# Returns true (needs migration) if old nvidia model format exists (nvidia/meta-* prefix)
|
||||
local count=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM model_configs WHERE provider = 'nvidia' AND model_id LIKE 'nvidia/meta-%';" 2>/dev/null || echo "0")
|
||||
[ "$count" != "0" ] && [ -n "$count" ]
|
||||
}
|
||||
|
||||
check_migration_026() {
|
||||
# Returns true (needs migration) if old format exists (moonshot-ai with hyphen instead of moonshotai)
|
||||
local count=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM model_configs WHERE provider = 'nvidia' AND model_id LIKE 'moonshot-ai/%';" 2>/dev/null || echo "0")
|
||||
[ "$count" != "0" ] && [ -n "$count" ]
|
||||
}
|
||||
|
||||
check_migration_027() {
|
||||
# Returns true (needs migration) if any tenant is missing NVIDIA model assignments
|
||||
# Counts tenants that don't have ALL active nvidia models assigned
|
||||
local nvidia_count=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM model_configs WHERE provider = 'nvidia' AND is_active = true;" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$nvidia_count" = "0" ] || [ -z "$nvidia_count" ]; then
|
||||
return 1 # No nvidia models, nothing to assign
|
||||
fi
|
||||
|
||||
# Check if any tenant is missing nvidia assignments
|
||||
local missing=$(docker exec gentwo-controlpanel-postgres psql -U postgres -d gt2_admin -tAc \
|
||||
"SELECT COUNT(*) FROM tenants t WHERE NOT EXISTS (
|
||||
SELECT 1 FROM tenant_model_configs tmc
|
||||
JOIN model_configs mc ON mc.id = tmc.model_config_id
|
||||
WHERE tmc.tenant_id = t.id AND mc.provider = 'nvidia'
|
||||
);" 2>/dev/null || echo "0")
|
||||
|
||||
[ "$missing" != "0" ] && [ -n "$missing" ]
|
||||
}
|
||||
|
||||
# Tenant migration checks
|
||||
check_migration_T001() {
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='tenant_test_company' AND table_name='tenants');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_T002() {
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='tenant_test_company' AND table_name='team_memberships');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_T002B() {
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema='tenant_test_company' AND table_name='team_memberships' AND column_name='status');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_T003() {
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='tenant_test_company' AND table_name='team_resource_shares');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_T005() {
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SET search_path TO tenant_test_company; SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'team_memberships'::regclass AND conname = 'check_team_permission' AND pg_get_constraintdef(oid) LIKE '%manager%');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_T006() {
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='tenant_test_company' AND table_name='auth_logs');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
check_migration_T009() {
|
||||
# Returns true (needs migration) if categories table doesn't exist
|
||||
local exists=$(docker exec gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants -tAc \
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='tenant_test_company' AND table_name='categories');" 2>/dev/null || echo "false")
|
||||
[ "$exists" != "t" ]
|
||||
}
|
||||
|
||||
# Run all admin migrations
|
||||
run_admin_migrations() {
|
||||
log_header "Admin Database Migrations"
|
||||
|
||||
# Ensure admin database is running (start if needed)
|
||||
if ! ensure_admin_db_running; then
|
||||
log_error "Could not start admin database, skipping admin migrations"
|
||||
return 1
|
||||
fi
|
||||
|
||||
run_admin_migration "006" "scripts/migrations/006_add_tenant_frontend_url.sql" "check_migration_006" || return 1
|
||||
run_admin_migration "008" "scripts/migrations/008_remove_ip_address_from_rate_limits.sql" "check_migration_008" || return 1
|
||||
run_admin_migration "009" "scripts/migrations/009_add_tfa_schema.sql" "check_migration_009" || return 1
|
||||
run_admin_migration "010" "scripts/migrations/010_update_model_context_windows.sql" "check_migration_010" || return 1
|
||||
run_admin_migration "011" "scripts/migrations/011_add_system_management_tables.sql" "check_migration_011" || return 1
|
||||
run_admin_migration "012" "scripts/migrations/012_add_optics_enabled.sql" "check_migration_012" || return 1
|
||||
run_admin_migration "013" "scripts/migrations/013_rename_cost_columns.sql" "check_migration_013" || return 1
|
||||
run_admin_migration "014" "scripts/migrations/014_backfill_groq_pricing.sql" "check_migration_014" || return 1
|
||||
run_admin_migration "015" "scripts/migrations/015_update_groq_pricing_dec_2025.sql" "check_migration_015" || return 1
|
||||
run_admin_migration "016" "scripts/migrations/016_add_is_compound_column.sql" "check_migration_016" || return 1
|
||||
run_admin_migration "017" "scripts/migrations/017_fix_compound_pricing.sql" "check_migration_017" || return 1
|
||||
run_admin_migration "018" "scripts/migrations/018_add_budget_storage_pricing.sql" "check_migration_018" || return 1
|
||||
run_admin_migration "019" "scripts/migrations/019_add_embedding_usage.sql" "check_migration_019" || return 1
|
||||
|
||||
# Migration 020: Import GROQ_API_KEY from environment to database (Python script)
|
||||
# This is a one-time migration for existing installations
|
||||
if check_migration_020 2>/dev/null; then
|
||||
log_info "Applying migration 020 (API key migration)..."
|
||||
if [ -f "scripts/migrations/020_migrate_env_api_keys.py" ]; then
|
||||
# Run the Python migration script
|
||||
if python3 scripts/migrations/020_migrate_env_api_keys.py; then
|
||||
log_success "Migration 020 applied successfully"
|
||||
else
|
||||
log_warning "Migration 020 skipped or failed (this is OK for fresh installs)"
|
||||
fi
|
||||
else
|
||||
log_warning "Migration 020 script not found, skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Migration 021: Add NVIDIA NIM models to model_configs (Issue #266)
|
||||
run_admin_migration "021" "scripts/migrations/021_add_nvidia_models.sql" "check_migration_021" || return 1
|
||||
|
||||
# Migration 022: Add sessions table for OWASP/NIST compliant session management (Issue #264)
|
||||
run_admin_migration "022" "scripts/migrations/022_add_session_management.sql" "check_migration_022" || return 1
|
||||
|
||||
# Migration 023: Add UUID primary key to model_configs (fix using model_id string as PK)
|
||||
run_admin_migration "023" "scripts/migrations/023_add_uuid_primary_key_to_model_configs.sql" "check_migration_023" || return 1
|
||||
|
||||
# Migration 024: Allow same model_id with different providers
|
||||
run_admin_migration "024" "scripts/migrations/024_allow_same_model_id_different_providers.sql" "check_migration_024" || return 1
|
||||
|
||||
# Migration 025: Fix NVIDIA model names to match API format
|
||||
run_admin_migration "025" "scripts/migrations/025_fix_nvidia_model_names.sql" "check_migration_025" || return 1
|
||||
|
||||
# Migration 026: Fix NVIDIA model_ids to exact API format
|
||||
run_admin_migration "026" "scripts/migrations/026_fix_nvidia_model_ids_api_format.sql" "check_migration_026" || return 1
|
||||
|
||||
# Migration 027: Ensure NVIDIA models are assigned to all tenants
|
||||
# This fixes partial 021 migrations where models were added but not assigned
|
||||
run_admin_migration "027" "scripts/migrations/027_assign_nvidia_models_to_tenants.sql" "check_migration_027" || return 1
|
||||
|
||||
log_success "All admin migrations complete"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Run all tenant migrations
|
||||
run_tenant_migrations() {
|
||||
log_header "Tenant Database Migrations"
|
||||
|
||||
# Ensure tenant database is running (start if needed)
|
||||
if ! ensure_tenant_db_running; then
|
||||
log_error "Could not start tenant database, skipping tenant migrations"
|
||||
return 1
|
||||
fi
|
||||
|
||||
run_tenant_migration "T001" "scripts/postgresql/migrations/T001_rename_teams_to_tenants.sql" "check_migration_T001" || return 1
|
||||
run_tenant_migration "T002" "scripts/postgresql/migrations/T002_create_collaboration_teams.sql" "check_migration_T002" || return 1
|
||||
run_tenant_migration "T002B" "scripts/postgresql/migrations/T002B_add_invitation_status.sql" "check_migration_T002B" || return 1
|
||||
run_tenant_migration "T003" "scripts/postgresql/migrations/T003_team_resource_shares.sql" "check_migration_T003" || return 1
|
||||
|
||||
# T004 is always run (idempotent - updates trigger function)
|
||||
log_info "Applying migration T004 (update validate_resource_share)..."
|
||||
if [ -f "scripts/postgresql/migrations/T004_update_validate_resource_share.sql" ]; then
|
||||
docker exec -i gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants \
|
||||
< scripts/postgresql/migrations/T004_update_validate_resource_share.sql || return 1
|
||||
log_success "Migration T004 applied successfully"
|
||||
fi
|
||||
|
||||
run_tenant_migration "T005" "scripts/postgresql/migrations/T005_team_observability.sql" "check_migration_T005" || return 1
|
||||
run_tenant_migration "T006" "scripts/postgresql/migrations/T006_auth_logs.sql" "check_migration_T006" || return 1
|
||||
|
||||
# T007 is always run (idempotent - creates indexes if not exists)
|
||||
log_info "Applying migration T007 (query optimization indexes)..."
|
||||
if [ -f "scripts/postgresql/migrations/T007_optimize_queries.sql" ]; then
|
||||
docker exec -i gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants \
|
||||
< scripts/postgresql/migrations/T007_optimize_queries.sql || return 1
|
||||
log_success "Migration T007 applied successfully"
|
||||
fi
|
||||
|
||||
# T008 is always run (idempotent - creates indexes if not exists)
|
||||
# Fixes GitHub Issue #173 - Database Optimizations
|
||||
log_info "Applying migration T008 (performance indexes for agents/datasets/teams)..."
|
||||
if [ -f "scripts/postgresql/migrations/T008_add_performance_indexes.sql" ]; then
|
||||
docker exec -i gentwo-tenant-postgres-primary psql -U postgres -d gt2_tenants \
|
||||
< scripts/postgresql/migrations/T008_add_performance_indexes.sql || return 1
|
||||
log_success "Migration T008 applied successfully"
|
||||
fi
|
||||
|
||||
# T009 - Tenant-scoped agent categories (Issue #215)
|
||||
run_tenant_migration "T009" "scripts/postgresql/migrations/T009_tenant_scoped_categories.sql" "check_migration_T009" || return 1
|
||||
|
||||
log_success "All tenant migrations complete"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Run all migrations
|
||||
run_all_migrations() {
|
||||
run_admin_migrations || return 1
|
||||
run_tenant_migrations || return 1
|
||||
return 0
|
||||
}
|
||||
141
scripts/lib/platform.sh
Executable file
141
scripts/lib/platform.sh
Executable file
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
# GT 2.0 Platform Detection and Compose File Selection
|
||||
# Handles ARM64, x86_64, and DGX platform differences
|
||||
|
||||
# Detect NVIDIA GPU and Container Toolkit availability
|
||||
detect_nvidia_gpu() {
|
||||
# Check for nvidia-smi command (indicates NVIDIA drivers installed)
|
||||
if ! command -v nvidia-smi &> /dev/null; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Verify GPU is accessible
|
||||
if ! nvidia-smi &> /dev/null; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check NVIDIA Container Toolkit is configured in Docker
|
||||
if ! docker info 2>/dev/null | grep -qi "nvidia"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Detect platform architecture
|
||||
detect_platform() {
|
||||
local arch=$(uname -m)
|
||||
local os=$(uname -s)
|
||||
|
||||
# Check for DGX specific environment
|
||||
if [ -f "/etc/dgx-release" ] || [ -n "${DGX_PLATFORM}" ]; then
|
||||
echo "dgx"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Detect architecture
|
||||
case "$arch" in
|
||||
aarch64|arm64)
|
||||
echo "arm64"
|
||||
;;
|
||||
x86_64|amd64)
|
||||
echo "x86"
|
||||
;;
|
||||
*)
|
||||
log_error "Unsupported architecture: $arch"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Get compose file for platform
|
||||
get_compose_file() {
|
||||
local platform="${1:-$(detect_platform)}"
|
||||
local dev_mode="${2:-false}"
|
||||
local files=""
|
||||
|
||||
case "$platform" in
|
||||
arm64)
|
||||
files="docker-compose.yml -f docker-compose.arm64.yml"
|
||||
;;
|
||||
x86)
|
||||
files="docker-compose.yml -f docker-compose.x86.yml"
|
||||
# Add GPU overlay if NVIDIA GPU detected
|
||||
if detect_nvidia_gpu; then
|
||||
files="$files -f docker-compose.x86-gpu.yml"
|
||||
fi
|
||||
;;
|
||||
dgx)
|
||||
files="docker-compose.yml -f docker-compose.dgx.yml"
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown platform: $platform"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Add dev overlay if requested
|
||||
if [ "$dev_mode" = "true" ]; then
|
||||
files="$files -f docker-compose.dev.yml"
|
||||
fi
|
||||
|
||||
echo "$files"
|
||||
}
|
||||
|
||||
# Get platform-specific settings
|
||||
get_platform_info() {
|
||||
local platform="${1:-$(detect_platform)}"
|
||||
|
||||
case "$platform" in
|
||||
arm64)
|
||||
echo "Platform: Apple Silicon (ARM64)"
|
||||
echo "Compose: docker-compose.yml + docker-compose.arm64.yml"
|
||||
echo "PgBouncer: pgbouncer/pgbouncer:latest"
|
||||
;;
|
||||
x86)
|
||||
echo "Platform: x86_64 Linux"
|
||||
if detect_nvidia_gpu; then
|
||||
echo "Compose: docker-compose.yml + docker-compose.x86.yml + docker-compose.x86-gpu.yml"
|
||||
echo "GPU: NVIDIA (accelerated embeddings)"
|
||||
else
|
||||
echo "Compose: docker-compose.yml + docker-compose.x86.yml"
|
||||
echo "GPU: None (CPU mode)"
|
||||
fi
|
||||
echo "PgBouncer: pgbouncer/pgbouncer:latest"
|
||||
;;
|
||||
dgx)
|
||||
echo "Platform: NVIDIA DGX (ARM64 Grace + Blackwell GPU)"
|
||||
echo "Compose: docker-compose.yml + docker-compose.dgx.yml"
|
||||
echo "PgBouncer: bitnamilegacy/pgbouncer:latest"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Check platform-specific prerequisites
|
||||
check_platform_prerequisites() {
|
||||
local platform="${1:-$(detect_platform)}"
|
||||
|
||||
case "$platform" in
|
||||
x86|dgx)
|
||||
# Check if user is in docker group
|
||||
if ! groups | grep -q '\bdocker\b'; then
|
||||
log_error "User $USER is not in the docker group"
|
||||
log_warning "Docker group membership is required on Linux"
|
||||
echo ""
|
||||
echo "Please run the following command:"
|
||||
echo -e "${BLUE} sudo usermod -aG docker $USER${NC}"
|
||||
echo ""
|
||||
echo "Then either:"
|
||||
echo " 1. Log out and log back in (recommended)"
|
||||
echo " 2. Run: newgrp docker (temporary for this session)"
|
||||
return 1
|
||||
fi
|
||||
log_success "Docker group membership confirmed"
|
||||
;;
|
||||
arm64)
|
||||
# macOS - no docker group check needed
|
||||
log_success "Platform prerequisites OK (macOS)"
|
||||
;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
273
scripts/lib/secrets.sh
Executable file
273
scripts/lib/secrets.sh
Executable file
@@ -0,0 +1,273 @@
|
||||
#!/bin/bash
|
||||
# GT AI OS Secret Generation Library
|
||||
# Centralized, idempotent secret generation for deployment scripts
|
||||
#
|
||||
# Usage: source scripts/lib/secrets.sh
|
||||
# generate_all_secrets # Populates .env with missing secrets only
|
||||
|
||||
set -e
|
||||
|
||||
# Source common functions if available
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
if [ -f "$SCRIPT_DIR/common.sh" ]; then
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# SECRET GENERATION FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
# Generate a random hex string (for JWT secrets, encryption keys)
|
||||
# Usage: generate_secret_hex [length]
|
||||
# Default length: 64 characters (32 bytes)
|
||||
generate_secret_hex() {
|
||||
local length=${1:-64}
|
||||
openssl rand -hex $((length / 2))
|
||||
}
|
||||
|
||||
# Generate a Fernet key (for TFA encryption, API key encryption)
|
||||
# Fernet requires base64-encoded 32-byte key
|
||||
generate_fernet_key() {
|
||||
python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" 2>/dev/null || \
|
||||
openssl rand -base64 32
|
||||
}
|
||||
|
||||
# Generate a secure password (for database passwords)
|
||||
# Usage: generate_password [length]
|
||||
# Default length: 32 characters
|
||||
generate_password() {
|
||||
local length=${1:-32}
|
||||
# Use alphanumeric + special chars, avoiding problematic shell chars
|
||||
openssl rand -base64 48 | tr -dc 'a-zA-Z0-9!@#$%^&*()_+-=' | head -c "$length"
|
||||
}
|
||||
|
||||
# Generate a simple alphanumeric password (for services that don't handle special chars well)
|
||||
# Usage: generate_simple_password [length]
|
||||
generate_simple_password() {
|
||||
local length=${1:-32}
|
||||
openssl rand -base64 48 | tr -dc 'a-zA-Z0-9' | head -c "$length"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# ENV FILE MANAGEMENT
|
||||
# =============================================================================
|
||||
|
||||
# Get value from .env file
|
||||
# Usage: get_env_value "KEY_NAME" ".env"
|
||||
get_env_value() {
|
||||
local key="$1"
|
||||
local env_file="${2:-.env}"
|
||||
|
||||
if [ -f "$env_file" ]; then
|
||||
grep "^${key}=" "$env_file" 2>/dev/null | cut -d'=' -f2- | head -1
|
||||
fi
|
||||
}
|
||||
|
||||
# Set value in .env file (preserves existing, only sets if missing or empty)
|
||||
# Usage: set_env_value "KEY_NAME" "value" ".env"
|
||||
set_env_value() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
local env_file="${3:-.env}"
|
||||
|
||||
# Create file if it doesn't exist
|
||||
touch "$env_file"
|
||||
|
||||
local existing=$(get_env_value "$key" "$env_file")
|
||||
|
||||
if [ -z "$existing" ]; then
|
||||
# Key doesn't exist or is empty, add/update it
|
||||
if grep -q "^${key}=" "$env_file" 2>/dev/null; then
|
||||
# Key exists but is empty, update it
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
sed -i '' "s|^${key}=.*|${key}=${value}|" "$env_file"
|
||||
else
|
||||
sed -i "s|^${key}=.*|${key}=${value}|" "$env_file"
|
||||
fi
|
||||
else
|
||||
# Key doesn't exist, append it
|
||||
echo "${key}=${value}" >> "$env_file"
|
||||
fi
|
||||
return 0 # Secret was generated
|
||||
fi
|
||||
return 1 # Secret already exists
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# MAIN SECRET GENERATION
|
||||
# =============================================================================
|
||||
|
||||
# Generate all required secrets for GT AI OS
|
||||
# This function is IDEMPOTENT - it only generates missing secrets
|
||||
# Usage: generate_all_secrets [env_file]
|
||||
generate_all_secrets() {
|
||||
local env_file="${1:-.env}"
|
||||
local generated_count=0
|
||||
|
||||
echo "Checking and generating missing secrets..."
|
||||
|
||||
# JWT and Authentication Secrets
|
||||
if set_env_value "JWT_SECRET" "$(generate_secret_hex 64)" "$env_file"; then
|
||||
echo " Generated: JWT_SECRET"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
if set_env_value "CONTROL_PANEL_JWT_SECRET" "$(generate_secret_hex 64)" "$env_file"; then
|
||||
echo " Generated: CONTROL_PANEL_JWT_SECRET"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
if set_env_value "RESOURCE_CLUSTER_SECRET_KEY" "$(generate_secret_hex 64)" "$env_file"; then
|
||||
echo " Generated: RESOURCE_CLUSTER_SECRET_KEY"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
# Encryption Keys
|
||||
if set_env_value "TFA_ENCRYPTION_KEY" "$(generate_fernet_key)" "$env_file"; then
|
||||
echo " Generated: TFA_ENCRYPTION_KEY"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
if set_env_value "API_KEY_ENCRYPTION_KEY" "$(generate_fernet_key)" "$env_file"; then
|
||||
echo " Generated: API_KEY_ENCRYPTION_KEY"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
# Database Passwords (use simple passwords for PostgreSQL compatibility)
|
||||
if set_env_value "ADMIN_POSTGRES_PASSWORD" "$(generate_simple_password 32)" "$env_file"; then
|
||||
echo " Generated: ADMIN_POSTGRES_PASSWORD"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
if set_env_value "TENANT_POSTGRES_PASSWORD" "$(generate_simple_password 32)" "$env_file"; then
|
||||
echo " Generated: TENANT_POSTGRES_PASSWORD"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
# Sync TENANT_USER_PASSWORD with TENANT_POSTGRES_PASSWORD
|
||||
local tenant_pass=$(get_env_value "TENANT_POSTGRES_PASSWORD" "$env_file")
|
||||
if set_env_value "TENANT_USER_PASSWORD" "$tenant_pass" "$env_file"; then
|
||||
echo " Set: TENANT_USER_PASSWORD (synced with TENANT_POSTGRES_PASSWORD)"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
if set_env_value "TENANT_REPLICATOR_PASSWORD" "$(generate_simple_password 32)" "$env_file"; then
|
||||
echo " Generated: TENANT_REPLICATOR_PASSWORD"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
# Other Service Passwords
|
||||
if set_env_value "RABBITMQ_PASSWORD" "$(generate_simple_password 24)" "$env_file"; then
|
||||
echo " Generated: RABBITMQ_PASSWORD"
|
||||
((++generated_count))
|
||||
fi
|
||||
|
||||
if [ $generated_count -eq 0 ]; then
|
||||
echo " All secrets already present (no changes needed)"
|
||||
else
|
||||
echo " Generated $generated_count new secret(s)"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Validate that all required secrets are present (non-empty)
|
||||
# Usage: validate_secrets [env_file]
|
||||
validate_secrets() {
|
||||
local env_file="${1:-.env}"
|
||||
local missing=0
|
||||
|
||||
local required_secrets=(
|
||||
"JWT_SECRET"
|
||||
"CONTROL_PANEL_JWT_SECRET"
|
||||
"RESOURCE_CLUSTER_SECRET_KEY"
|
||||
"TFA_ENCRYPTION_KEY"
|
||||
"API_KEY_ENCRYPTION_KEY"
|
||||
"ADMIN_POSTGRES_PASSWORD"
|
||||
"TENANT_POSTGRES_PASSWORD"
|
||||
"TENANT_USER_PASSWORD"
|
||||
"RABBITMQ_PASSWORD"
|
||||
)
|
||||
|
||||
echo "Validating required secrets..."
|
||||
|
||||
for secret in "${required_secrets[@]}"; do
|
||||
local value=$(get_env_value "$secret" "$env_file")
|
||||
if [ -z "$value" ]; then
|
||||
echo " MISSING: $secret"
|
||||
((missing++))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $missing -gt 0 ]; then
|
||||
echo " $missing required secret(s) missing!"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo " All required secrets present"
|
||||
return 0
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# TEMPLATE CREATION
|
||||
# =============================================================================
|
||||
|
||||
# Create a .env.template file with placeholder values
|
||||
# Usage: create_env_template [output_file]
|
||||
create_env_template() {
|
||||
local output_file="${1:-.env.template}"
|
||||
|
||||
cat > "$output_file" << 'EOF'
|
||||
# GT AI OS Environment Configuration
|
||||
# Copy this file to .env and customize values
|
||||
# Secrets are auto-generated on first install if not provided
|
||||
|
||||
# =============================================================================
|
||||
# AUTHENTICATION (Auto-generated if empty)
|
||||
# =============================================================================
|
||||
JWT_SECRET=
|
||||
CONTROL_PANEL_JWT_SECRET=
|
||||
RESOURCE_CLUSTER_SECRET_KEY=
|
||||
|
||||
# =============================================================================
|
||||
# ENCRYPTION KEYS (Auto-generated if empty)
|
||||
# =============================================================================
|
||||
PASSWORD_RESET_ENCRYPTION_KEY=
|
||||
TFA_ENCRYPTION_KEY=
|
||||
API_KEY_ENCRYPTION_KEY=
|
||||
|
||||
# =============================================================================
|
||||
# DATABASE PASSWORDS (Auto-generated if empty)
|
||||
# =============================================================================
|
||||
ADMIN_POSTGRES_PASSWORD=
|
||||
TENANT_POSTGRES_PASSWORD=
|
||||
TENANT_USER_PASSWORD=
|
||||
TENANT_REPLICATOR_PASSWORD=
|
||||
RABBITMQ_PASSWORD=
|
||||
|
||||
# =============================================================================
|
||||
# API KEYS (Configure via Control Panel UI after installation)
|
||||
# =============================================================================
|
||||
# Note: LLM API keys (Groq, OpenAI, Anthropic) are configured through
|
||||
# the Control Panel UI, not environment variables.
|
||||
|
||||
# =============================================================================
|
||||
# SMTP (Enterprise Edition Only - Password Reset)
|
||||
# =============================================================================
|
||||
# Set via environment variables or configure below
|
||||
# SMTP_HOST=smtp-relay.brevo.com
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USERNAME=
|
||||
# SMTP_PASSWORD=
|
||||
# SMTP_FROM_EMAIL=noreply@yourdomain.com
|
||||
# SMTP_FROM_NAME=GT AI OS
|
||||
|
||||
# =============================================================================
|
||||
# DEPLOYMENT
|
||||
# =============================================================================
|
||||
COMPOSE_PROJECT_NAME=gentwo
|
||||
ENVIRONMENT=production
|
||||
EOF
|
||||
|
||||
echo "Created $output_file"
|
||||
}
|
||||
Reference in New Issue
Block a user