#!/bin/bash
set -eo pipefail

# ============================================================================
# QC Orchestrator — Contingency tables + filtering + Manhattan plots
# ============================================================================

# --- ANSI Color Codes ---
C_RESET="\033[0m"
C_BOLD="\033[1m"
C_GREEN="\033[32m"
C_YELLOW="\033[33m"
C_RED="\033[31m"
C_BLUE="\033[34m"
C_CYAN="\033[36m"
C_DIM="\033[2m"
C_BGREEN="\033[1;32m"
C_BYELLOW="\033[1;33m"
C_BRED="\033[1;31m"
C_BCYAN="\033[1;36m"

# --- Defaults ---
POOL_SIZE=25
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROGRESS_FILE="$SCRIPT_DIR/qc_progress.txt"
LOG_DIR="$SCRIPT_DIR/logs_qc"
ORCH_LOG="$SCRIPT_DIR/qc_orchestrator.log"
LOCKFILE="$SCRIPT_DIR/.qc_progress.lock"
SUMMARY_FILE="${SUMMARY_FILE:-$SCRIPT_DIR/gwas_qc_summary.tsv}"
FILTERED_DIR="$SCRIPT_DIR/filtered"
QC_DIR="$SCRIPT_DIR/qc_test"
POLL_INTERVAL=30
SKIP_CONTINGENCY=false
SKIP_FILTERING=false
SKIP_PLOTTING=false
CLEAR_CACHE=false

TOOLS_DX="/results/tools"
TOOLS_MNT="/mnt/project${TOOLS_DX}"

# --- Parse Arguments ---
usage() {
    echo "Usage: $0 [--pool-size <n>] [--skip-contingency] [--skip-filtering] [--skip-plotting]"
    echo ""
    echo "Options:"
    echo "  --pool-size         Max concurrent DNAnexus jobs (default: 25)"
    echo "  --skip-contingency  Skip contingency computation (use existing TSVs)"
    echo "  --skip-filtering    Skip filtered TSV generation"
    echo "  --skip-plotting     Skip Manhattan plot generation"
    echo "  --clear-cache       Delete cached phenotype extracts on DNAnexus before running"
    echo "  --help              Show this help"
    exit 0
}

while [[ $# -gt 0 ]]; do
    case $1 in
        --pool-size) POOL_SIZE="$2"; shift 2 ;;
        --skip-contingency) SKIP_CONTINGENCY=true; shift ;;
        --skip-filtering) SKIP_FILTERING=true; shift ;;
        --skip-plotting) SKIP_PLOTTING=true; shift ;;
        --clear-cache) CLEAR_CACHE=true; shift ;;
        --help) usage ;;
        *) echo "Unknown option: $1"; usage ;;
    esac
done

# --- Data Structures ---
declare -A JOB_IDS           # phenotype -> DNAnexus job ID
declare -A JOB_START         # phenotype -> start epoch
declare -A PHENO_STATUS      # phenotype -> status
declare -A PHENO_TYPE        # phenotype -> binary|continuous
declare -A PHENO_ANALYSIS    # phenotype -> analysis_id
declare -A PHENO_GWAS_FILE   # phenotype -> path to GWAS results TSV
declare -a ALL_PHENOS        # ordered list of unique phenotypes
declare -a RECENT_EVENTS     # ring buffer for dashboard
ORCHESTRATOR_START=$(date +%s)
TOTAL_DONE_COUNT=0
TOTAL_PHENOS=0

# --- Logging ---
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$ORCH_LOG"
}

add_recent_event() {
    local type="$1" pheno="$2" detail="$3"
    local ts
    ts=$(date '+%H:%M:%S')
    local icon
    case "$type" in
        contingency_done) icon="📦" ;;
        filtered)  icon="🔍" ;;
        plotted)   icon="📊" ;;
        done)      icon="✅" ;;
        failed)    icon="❌" ;;
        *)         icon="ℹ️" ;;
    esac
    RECENT_EVENTS+=("$ts $icon $pheno $detail")
    # Keep last 10
    if [[ ${#RECENT_EVENTS[@]} -gt 10 ]]; then
        RECENT_EVENTS=("${RECENT_EVENTS[@]:1}")
    fi
}

# --- Progress File Locking ---
update_progress_entry() {
    local pheno="$1" status="$2" start_time="${3:-}" end_time="${4:-}" analysis_id="${5:-}" error_msg="${6:-}"

    (
        flock -w 5 200 || { log "WARN: Could not acquire lock for $pheno"; return; }

        local tmpfile="${PROGRESS_FILE}.tmp"
        local ptype="${PHENO_TYPE[$pheno]:-unknown}"

        if [[ -f "$PROGRESS_FILE" ]]; then
            awk -F'\t' -v OFS='\t' \
                -v p="$pheno" -v s="$status" -v t="$ptype" \
                -v st="$start_time" -v et="$end_time" -v aid="$analysis_id" -v em="$error_msg" \
            '
            NR==1 { print; next }
            $1 == p {
                print p, s, t, (st != "" ? st : $4), (et != "" ? et : $5), (aid != "" ? aid : $6), (em != "" ? em : $7)
                found=1
                next
            }
            { print }
            END { if (!found) print p, s, t, st, et, aid, em }
            ' "$PROGRESS_FILE" > "$tmpfile"
        else
            printf 'phenotype\tstatus\tpheno_type\tstart_time\tend_time\tanalysis_id\terror_msg\n' > "$tmpfile"
            printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
                "$pheno" "$status" "$ptype" \
                "$start_time" "$end_time" "$analysis_id" "$error_msg" >> "$tmpfile"
        fi

        mv "$tmpfile" "$PROGRESS_FILE"

    ) 200>"$LOCKFILE"

    PHENO_STATUS[$pheno]="$status"
}

# --- Load Phenotypes from gwas_analysis_summary.tsv ---
load_phenotypes() {
    log "Loading phenotypes from $SUMMARY_FILE..."

    # Extract unique phenotypes with their latest marpiech_ analysis_id
    while IFS=$'\t' read -r pheno analysis_id; do
        [[ -z "$pheno" ]] && continue
        ALL_PHENOS+=("$pheno")
        PHENO_ANALYSIS[$pheno]="$analysis_id"

        # Find GWAS results file
        local gwas_file
        gwas_file=$(ls "$SCRIPT_DIR/result/${analysis_id}/"*-results.tsv 2>/dev/null | head -1)
        PHENO_GWAS_FILE[$pheno]="$gwas_file"

        # Determine type from filename
        if [[ "$gwas_file" == *"_binary_"* ]]; then
            PHENO_TYPE[$pheno]="binary"
        elif [[ "$gwas_file" == *"_continuous_"* ]]; then
            PHENO_TYPE[$pheno]="continuous"
        else
            PHENO_TYPE[$pheno]="binary"  # default
        fi
    done < <(awk -F'\t' 'NR>1 && $1 ~ /^marpiech_/ {
        latest[$2]=$1
        if (!seen[$2]++) order[++n]=$2
    } END {
        for (i=1; i<=n; i++) print order[i] "\t" latest[order[i]]
    }' "$SUMMARY_FILE")

    TOTAL_PHENOS=${#ALL_PHENOS[@]}
    log "Loaded $TOTAL_PHENOS unique phenotypes"
}

# --- Upload Scripts to DNAnexus (once) ---
upload_scripts() {
    log "Uploading contingency scripts to DNAnexus..."
    dx mkdir -p "$TOOLS_DX/" 2>/dev/null || true

    for script in extract_phenotype.py contingency_all.py contingency_runner.sh; do
        for fid in $(dx ls "${TOOLS_DX}/${script}" --brief 2>/dev/null); do
            dx rm "$fid" 2>/dev/null || true
        done
        dx upload "${SCRIPT_DIR}/${script}" --path "${TOOLS_DX}/" --brief
    done
    log "Scripts uploaded successfully"

    if [[ "$CLEAR_CACHE" == "true" ]]; then
        log "Clearing cached phenotype extracts..."
        local n_deleted=0
        for fid in $(dx ls "${TOOLS_DX}/pheno_extracts/" --brief 2>/dev/null); do
            dx rm "$fid" 2>/dev/null && n_deleted=$((n_deleted + 1))
        done
        log "Deleted $n_deleted cached phenotype extracts"
    fi
}

# --- Resume Logic ---
resume_from_progress() {
    if [[ ! -f "$PROGRESS_FILE" ]]; then
        log "No progress file, starting fresh"
        printf 'phenotype\tstatus\tpheno_type\tstart_time\tend_time\tanalysis_id\terror_msg\n' > "$PROGRESS_FILE"
        for pheno in "${ALL_PHENOS[@]}"; do
            PHENO_STATUS[$pheno]="queued"
            printf '%s\t%s\t%s\t\t\t%s\t\n' "$pheno" "queued" "${PHENO_TYPE[$pheno]}" "${PHENO_ANALYSIS[$pheno]}" >> "$PROGRESS_FILE"
        done
        return
    fi

    log "Resuming from $PROGRESS_FILE"
    local -A progress_seen

    while IFS=$'\t' read -r pheno status ptype start_time end_time analysis_id error_msg; do
        [[ "$pheno" == "phenotype" ]] && continue
        [[ -z "$pheno" ]] && continue
        progress_seen[$pheno]=1

        case $status in
            done)
                PHENO_STATUS[$pheno]="done"
                TOTAL_DONE_COUNT=$((TOTAL_DONE_COUNT + 1))
                ;;
            contingency_done|filtered|plotting)
                # Check what stage we can resume from
                if [[ -f "$FILTERED_DIR/${pheno}.filtered.tsv" && -f "$QC_DIR/${pheno}_manhattan.png" ]]; then
                    PHENO_STATUS[$pheno]="done"
                    update_progress_entry "$pheno" "done" "" "$(date '+%Y-%m-%d %H:%M:%S')"
                    TOTAL_DONE_COUNT=$((TOTAL_DONE_COUNT + 1))
                elif [[ -f "$FILTERED_DIR/${pheno}.filtered.tsv" ]]; then
                    PHENO_STATUS[$pheno]="filtered"
                elif [[ -f "$SCRIPT_DIR/contingency_${pheno}.tsv" ]]; then
                    PHENO_STATUS[$pheno]="contingency_done"
                else
                    PHENO_STATUS[$pheno]="queued"
                fi
                ;;
            contingency_running)
                # Check if result downloaded while we were away
                if [[ -f "$SCRIPT_DIR/contingency_${pheno}.tsv" ]]; then
                    PHENO_STATUS[$pheno]="contingency_done"
                else
                    PHENO_STATUS[$pheno]="queued"
                fi
                ;;
            failed)
                PHENO_STATUS[$pheno]="failed"
                ;;
            *)
                PHENO_STATUS[$pheno]="queued"
                ;;
        esac
    done < "$PROGRESS_FILE"

    # Add new phenotypes not in progress file
    for pheno in "${ALL_PHENOS[@]}"; do
        if [[ -z "${progress_seen[$pheno]+x}" ]]; then
            # Check if contingency already exists (from a previous manual run)
            if [[ -f "$SCRIPT_DIR/contingency_${pheno}.tsv" ]]; then
                PHENO_STATUS[$pheno]="contingency_done"
            else
                PHENO_STATUS[$pheno]="queued"
            fi
            update_progress_entry "$pheno" "${PHENO_STATUS[$pheno]}" "" "" "${PHENO_ANALYSIS[$pheno]}"
            log "New phenotype: $pheno (${PHENO_STATUS[$pheno]})"
        fi
    done
}

# --- Submit DNAnexus Contingency Job ---
submit_contingency_job() {
    local pheno="$1"
    local pheno_type="${PHENO_TYPE[$pheno]}"

    local dest="/results/tools/contingency_results/${pheno}"
    dx mkdir -p "${dest}/" 2>/dev/null || true

    local job_id
    job_id=$(dx run app-swiss-army-knife \
        --instance-type mem3_ssd1_v2_x48 \
        -icmd="bash ${TOOLS_MNT}/contingency_runner.sh '${pheno}' 'snv,hla,cypmicro,cypdosage,lof,mpc,cnv' 36 '${pheno_type}' # v$(date +%s)" \
        --destination "${dest}/" \
        --brief -y 2>/dev/null)

    JOB_IDS[$pheno]="$job_id"
    JOB_START[$pheno]=$(date +%s)
    update_progress_entry "$pheno" "contingency_running" "$(date '+%Y-%m-%d %H:%M:%S')" "" "${PHENO_ANALYSIS[$pheno]}"
    log "Submitted contingency for $pheno: $job_id (type=$pheno_type)"
}

# --- Check Completed DNAnexus Jobs ---
check_completed_jobs() {
    local to_remove=()

    for pheno in "${!JOB_IDS[@]}"; do
        local job_id="${JOB_IDS[$pheno]}"

        local state
        state=$(dx describe "$job_id" --json 2>/dev/null | \
            python3 -c "import sys,json; print(json.load(sys.stdin).get('state','unknown'))" 2>/dev/null || echo "unknown")

        case "$state" in
            done)
                download_contingency "$pheno" "$job_id"
                to_remove+=("$pheno")
                ;;
            failed|terminated)
                local error_msg="DNAnexus job $job_id: $state"
                update_progress_entry "$pheno" "failed" "" "$(date '+%Y-%m-%d %H:%M:%S')" "" "$error_msg"
                add_recent_event "failed" "$pheno" "$error_msg"
                log "FAILED: $pheno ($error_msg)"
                to_remove+=("$pheno")
                ;;
            # running|idle|waiting -> keep polling
        esac
    done

    for pheno in "${to_remove[@]}"; do
        unset "JOB_IDS[$pheno]"
        unset "JOB_START[$pheno]"
    done
}

# --- Download Contingency Result ---
download_contingency() {
    local pheno="$1" job_id="$2"
    local result_dx="/results/tools/contingency_results/${pheno}/contingency_${pheno}.tsv"
    local output="$SCRIPT_DIR/contingency_${pheno}.tsv"

    # Try latest copy
    local fid
    fid=$(dx ls "$result_dx" --brief 2>/dev/null | tail -1)
    if [[ -n "$fid" ]] && dx download "$fid" -o "$output" -f 2>/dev/null; then
        local n_variants
        n_variants=$(tail -n +2 "$output" | wc -l)
        update_progress_entry "$pheno" "contingency_done" "" "$(date '+%Y-%m-%d %H:%M:%S')"
        add_recent_event "contingency_done" "$pheno" "${n_variants} variants"
        log "Downloaded contingency for $pheno: $n_variants variants"
    else
        update_progress_entry "$pheno" "failed" "" "$(date '+%Y-%m-%d %H:%M:%S')" "" "Download failed"
        add_recent_event "failed" "$pheno" "download failed"
        log "Failed to download contingency for $pheno"
    fi
}

# --- Run Filtering ---
run_filtering() {
    local pheno="$1"
    local pheno_type="${PHENO_TYPE[$pheno]}"
    local gwas_file="${PHENO_GWAS_FILE[$pheno]}"
    local cont_file="$SCRIPT_DIR/contingency_${pheno}.tsv"
    local out_file="$FILTERED_DIR/${pheno}.filtered.tsv"

    if [[ -z "$gwas_file" || ! -f "$gwas_file" ]]; then
        update_progress_entry "$pheno" "failed" "" "$(date '+%Y-%m-%d %H:%M:%S')" "" "No GWAS results"
        return 1
    fi

    local threshold
    if [[ "$pheno_type" == "continuous" ]]; then
        threshold=15
    else
        threshold=4
    fi

    update_progress_entry "$pheno" "filtering"

    local filter_output
    filter_output=$(python3 -c "
import sys
threshold = int(sys.argv[1])
gwas_file = sys.argv[2]
cont_file = sys.argv[3]
out_file = sys.argv[4]

mc = {}
with open(cont_file) as f:
    next(f)
    for line in f:
        parts = line.split('\t')
        mc[parts[0]] = int(parts[11])

with open(gwas_file) as fin, open(out_file, 'w') as fout:
    header = fin.readline()
    fout.write(header)
    kept = removed = 0
    for line in fin:
        vid = line.split()[2]
        if vid in mc and mc[vid] <= threshold:
            removed += 1
        else:
            fout.write(line)
            kept += 1
print(f'{kept}\t{removed}')
" "$threshold" "$gwas_file" "$cont_file" "$out_file" 2>/dev/null)

    if [[ $? -eq 0 && -n "$filter_output" ]]; then
        local kept removed
        kept=$(echo "$filter_output" | cut -f1)
        removed=$(echo "$filter_output" | cut -f2)
        update_progress_entry "$pheno" "filtered"
        add_recent_event "filtered" "$pheno" "kept=$kept removed=$removed"
        log "Filtered $pheno: kept=$kept removed=$removed (threshold=$threshold)"
        return 0
    else
        update_progress_entry "$pheno" "failed" "" "$(date '+%Y-%m-%d %H:%M:%S')" "" "Filtering failed"
        return 1
    fi
}

# --- Run Manhattan Plot ---
run_plotting() {
    local pheno="$1"
    local pheno_type="${PHENO_TYPE[$pheno]}"
    local gwas_file="$FILTERED_DIR/${pheno}.filtered.tsv"
    local cont_file="$SCRIPT_DIR/contingency_${pheno}.tsv"

    if [[ ! -f "$gwas_file" ]]; then
        gwas_file="${PHENO_GWAS_FILE[$pheno]}"
    fi

    update_progress_entry "$pheno" "plotting"

    Rscript "$SCRIPT_DIR/gwas_viz_combined.R" \
        --input "$gwas_file" \
        --contingency "$cont_file" \
        --output "$QC_DIR" \
        --prefix "$pheno" \
        --phenotype-type "$pheno_type" \
        > "$LOG_DIR/${pheno}_plot.log" 2>&1

    if [[ -f "$QC_DIR/${pheno}_manhattan.png" ]]; then
        add_recent_event "plotted" "$pheno" ""
        log "Plotted $pheno"
        return 0
    else
        log "Plot failed for $pheno"
        return 1
    fi
}

# --- Append Filtering Stats to Summary TSV ---
append_to_summary() {
    local pheno="$1" kept="$2" removed="$3"
    local filtered_path="filtered/${pheno}.filtered.tsv"

    # Update all rows for this phenotype in the summary
    # Add columns if they don't exist yet
    # For simplicity, we update the summary at the end in a batch
    log "Filtering stats for $pheno: kept=$kept removed=$removed"
}

# --- Dashboard ---
draw_dashboard() {
    local n_queued=0 n_running=0 n_cont_done=0 n_filtered=0 n_done=0 n_failed=0

    for pheno in "${ALL_PHENOS[@]}"; do
        case "${PHENO_STATUS[$pheno]}" in
            queued)             n_queued=$((n_queued+1)) ;;
            contingency_running) n_running=$((n_running+1)) ;;
            contingency_done)  n_cont_done=$((n_cont_done+1)) ;;
            filtered|plotting) n_filtered=$((n_filtered+1)) ;;
            done)              n_done=$((n_done+1)) ;;
            failed)            n_failed=$((n_failed+1)) ;;
        esac
    done

    local elapsed=$(( $(date +%s) - ORCHESTRATOR_START ))
    local elapsed_h=$((elapsed / 3600))
    local elapsed_m=$(( (elapsed % 3600) / 60 ))

    clear
    printf "${C_BCYAN}╔══════════════════════════════════════════════════════════════╗${C_RESET}\n"
    printf "${C_BCYAN}║${C_BOLD}           QC ORCHESTRATOR — CONTINGENCY + FILTER            ${C_BCYAN}║${C_RESET}\n"
    printf "${C_BCYAN}╚══════════════════════════════════════════════════════════════╝${C_RESET}\n"
    printf "\n"
    printf "  Total: ${C_BOLD}%d${C_RESET}  |  Elapsed: ${elapsed_h}h${elapsed_m}m  |  Pool: %d/%d active\n" \
        "$TOTAL_PHENOS" "${#JOB_IDS[@]}" "$POOL_SIZE"
    printf "\n"
    printf "  ${C_BGREEN}Done:${C_RESET} %-6d  ${C_BCYAN}Filtered:${C_RESET} %-6d  ${C_BYELLOW}Cont.Done:${C_RESET} %-6d\n" \
        "$n_done" "$n_filtered" "$n_cont_done"
    printf "  ${C_BLUE}Running:${C_RESET} %-6d  ${C_DIM}Queued:${C_RESET} %-6d  ${C_BRED}Failed:${C_RESET} %-6d\n" \
        "$n_running" "$n_queued" "$n_failed"
    printf "\n"

    # Progress bar
    local pct=0
    [[ $TOTAL_PHENOS -gt 0 ]] && pct=$(( (n_done * 100) / TOTAL_PHENOS ))
    local bar_len=50
    local filled=$(( pct * bar_len / 100 ))
    local empty=$(( bar_len - filled ))
    printf "  [${C_GREEN}"
    printf '%0.s█' $(seq 1 $filled 2>/dev/null) || true
    printf "${C_DIM}"
    printf '%0.s░' $(seq 1 $empty 2>/dev/null) || true
    printf "${C_RESET}] %d%%\n" "$pct"
    printf "\n"

    # Active DNAnexus jobs
    if [[ ${#JOB_IDS[@]} -gt 0 ]]; then
        printf "  ${C_BOLD}Active Jobs:${C_RESET}\n"
        local count=0
        for pheno in "${!JOB_IDS[@]}"; do
            local runtime=$(( $(date +%s) - ${JOB_START[$pheno]} ))
            local rm=$((runtime / 60))
            printf "    %-45s %s  %dm\n" "$pheno" "${JOB_IDS[$pheno]}" "$rm"
            count=$((count+1))
            [[ $count -ge 10 ]] && { printf "    ... and %d more\n" $(( ${#JOB_IDS[@]} - 10 )); break; }
        done
        printf "\n"
    fi

    # Recent events
    if [[ ${#RECENT_EVENTS[@]} -gt 0 ]]; then
        printf "  ${C_BOLD}Recent:${C_RESET}\n"
        for evt in "${RECENT_EVENTS[@]}"; do
            printf "    %s\n" "$evt"
        done
        printf "\n"
    fi
}

# --- Signal Handler ---
cleanup() {
    log "Shutting down QC orchestrator (${#JOB_IDS[@]} active DNAnexus jobs)"
    log "DNAnexus jobs left running. Progress saved for resume."
    printf '\033[?25h' 2>/dev/null
    tput cnorm 2>/dev/null || true
    exit 0
}
trap cleanup SIGINT SIGTERM

# ============================================================================
# MAIN
# ============================================================================
main() {
    mkdir -p "$LOG_DIR" "$FILTERED_DIR" "$QC_DIR"

    log "=== QC Orchestrator started ==="
    log "Pool size: $POOL_SIZE"

    # Step 1: Load phenotypes
    load_phenotypes

    # Step 2: Upload scripts once
    if [[ "$SKIP_CONTINGENCY" != "true" ]]; then
        upload_scripts
    fi

    # Step 3: Resume
    resume_from_progress

    local n_already_done=$TOTAL_DONE_COUNT
    log "Resumed: $n_already_done already done, $(( TOTAL_PHENOS - n_already_done )) to process"

    # Step 4: Process phenotypes with contingency_done that need filtering/plotting
    for pheno in "${ALL_PHENOS[@]}"; do
        if [[ "${PHENO_STATUS[$pheno]}" == "contingency_done" && "$SKIP_FILTERING" != "true" ]]; then
            run_filtering "$pheno"
        fi
        if [[ "${PHENO_STATUS[$pheno]}" == "filtered" && "$SKIP_PLOTTING" != "true" ]]; then
            if run_plotting "$pheno"; then
                update_progress_entry "$pheno" "done" "" "$(date '+%Y-%m-%d %H:%M:%S')"
                TOTAL_DONE_COUNT=$((TOTAL_DONE_COUNT + 1))
                add_recent_event "done" "$pheno" ""
            fi
        fi
        if [[ "${PHENO_STATUS[$pheno]}" == "filtered" && "$SKIP_PLOTTING" == "true" ]]; then
            update_progress_entry "$pheno" "done" "" "$(date '+%Y-%m-%d %H:%M:%S')"
            TOTAL_DONE_COUNT=$((TOTAL_DONE_COUNT + 1))
        fi
    done

    # Step 5: Main loop — submit contingency jobs and process completions
    local last_poll=0
    local last_dashboard=0

    while true; do
        local now_epoch
        now_epoch=$(date +%s)

        # Poll completed jobs
        if (( now_epoch - last_poll >= POLL_INTERVAL )) && [[ ${#JOB_IDS[@]} -gt 0 ]]; then
            check_completed_jobs
            last_poll=$now_epoch

            # Process newly completed contingencies
            for pheno in "${ALL_PHENOS[@]}"; do
                if [[ "${PHENO_STATUS[$pheno]}" == "contingency_done" && "$SKIP_FILTERING" != "true" ]]; then
                    run_filtering "$pheno"
                fi
                if [[ "${PHENO_STATUS[$pheno]}" == "filtered" && "$SKIP_PLOTTING" != "true" ]]; then
                    if run_plotting "$pheno"; then
                        update_progress_entry "$pheno" "done" "" "$(date '+%Y-%m-%d %H:%M:%S')"
                        TOTAL_DONE_COUNT=$((TOTAL_DONE_COUNT + 1))
                        add_recent_event "done" "$pheno" ""
                    fi
                fi
                if [[ "${PHENO_STATUS[$pheno]}" == "filtered" && "$SKIP_PLOTTING" == "true" ]]; then
                    update_progress_entry "$pheno" "done" "" "$(date '+%Y-%m-%d %H:%M:%S')"
                    TOTAL_DONE_COUNT=$((TOTAL_DONE_COUNT + 1))
                fi
            done
        fi

        # Submit new jobs if pool has room
        if [[ "$SKIP_CONTINGENCY" != "true" ]]; then
            while [[ ${#JOB_IDS[@]} -lt $POOL_SIZE ]]; do
                local next_pheno=""
                for pheno in "${ALL_PHENOS[@]}"; do
                    if [[ "${PHENO_STATUS[$pheno]}" == "queued" ]]; then
                        next_pheno="$pheno"
                        break
                    fi
                done
                [[ -z "$next_pheno" ]] && break
                submit_contingency_job "$next_pheno"
                sleep 2  # Avoid API rate limiting
            done
        fi

        # Check if all done
        local any_pending=false
        for pheno in "${ALL_PHENOS[@]}"; do
            case "${PHENO_STATUS[$pheno]}" in
                queued|contingency_running|contingency_done|filtering|filtered|plotting)
                    any_pending=true
                    break ;;
            esac
        done
        if [[ "$any_pending" == false && ${#JOB_IDS[@]} -eq 0 ]]; then
            break
        fi

        # Dashboard
        if (( now_epoch - last_dashboard >= 10 )); then
            draw_dashboard
            last_dashboard=$now_epoch
        fi

        sleep 5
    done

    # Final dashboard
    draw_dashboard

    log "=== QC Orchestrator complete ==="
    log "Done: $TOTAL_DONE_COUNT / $TOTAL_PHENOS"

    # Count stats
    local n_failed=0
    for pheno in "${ALL_PHENOS[@]}"; do
        [[ "${PHENO_STATUS[$pheno]}" == "failed" ]] && n_failed=$((n_failed+1))
    done

    echo ""
    echo "=== COMPLETE ==="
    echo "Done: $TOTAL_DONE_COUNT / $TOTAL_PHENOS"
    echo "Failed: $n_failed"
    echo "Progress: $PROGRESS_FILE"
    echo "Filtered: $FILTERED_DIR/"
    echo "Plots: $QC_DIR/"
}

main "$@"