Universal Pipeline

// DON'T DO THIS — brittle, doesn't scale
if (provider == 'gcp') {
    withCredentials([file(credentialsId: 'gcp-key', variable: 'GCP_KEY')]) {
        sh "gcloud auth activate-service-account --key-file=$GCP_KEY"
        sh "fluid apply contract.yaml --provider gcp --project $GCP_PROJECT"
    }
} else if (provider == 'aws') {
    withCredentials([usernamePassword(credentialsId: 'aws-creds', ...)]) {
        sh "fluid apply contract.yaml --provider aws"
    }
} else if (provider == 'snowflake') {
    // ... more branching
}

#!/usr/bin/env groovy
/**
 * FLUID Pipeline v0.7.1 — Provider-Agnostic
 *
 * This pipeline runs ANY FLUID data product on ANY provider without modification.
 * Zero provider-specific logic lives here — the contract is the single source of truth.
 *
 * Credentials convention:
 *   Create a Jenkins "Secret File" credential containing shell-exportable env vars:
 *
 *   GCP credentials file:          AWS credentials file:         Snowflake credentials file:
 *     (raw SA JSON key works too)   AWS_ACCESS_KEY_ID=AKIAxxx    SNOWFLAKE_ACCOUNT=xxx
 *     GCP_PROJECT=my-project        AWS_SECRET_ACCESS_KEY=xxx    SNOWFLAKE_USER=xxx
 *                                   AWS_REGION=eu-central-1      SNOWFLAKE_PASSWORD=xxx
 *                                   S3_BUCKET=my-bucket          SNOWFLAKE_WAREHOUSE=COMPUTE_WH
 *                                                                SNOWFLAKE_ROLE=SYSADMIN
 *
 *   For GCP, the Secret File can be the raw service-account JSON key —
 *   the pipeline auto-detects JSON vs env-file format.
 *
 * Adding a new provider:
 *   1. Add a provider to the FLUID CLI (fluid_build/providers/)
 *   2. Set binding.platform in your contract
 *   3. Create a credentials file with the env vars your provider needs
 *   4. That's it — this Jenkinsfile doesn't change
 */

pipeline {
    agent {
        docker {
            image "${params.FLUID_IMAGE}"
            alwaysPull true
            args '-v /var/run/docker.sock:/var/run/docker.sock --entrypoint='
        }
    }

    environment {
        HOME = "${WORKSPACE}"
        ENV  = "${BRANCH_NAME == 'main' ? 'prod' : BRANCH_NAME == 'develop' ? 'staging' : 'dev'}"
    }

    parameters {
        string(name: 'CONTRACT_FILE',  defaultValue: 'contract.fluid.yaml',
               description: 'FLUID contract file')
        string(name: 'FLUID_IMAGE',    defaultValue: 'localhost:5000/fluid-forge-cli:beta-latest',
               description: 'FLUID CLI Docker image')
        string(name: 'CREDENTIALS_ID', defaultValue: '',
               description: 'Jenkins Secret File credential ID (leave empty for .env fallback)')
        booleanParam(name: 'RUN_EXECUTION', defaultValue: true,
               description: 'Execute builds after infrastructure apply')
        booleanParam(name: 'ENFORCE_IAM',   defaultValue: true,
               description: 'Enforce IAM/RBAC policies from contract')
    }

    options {
        timeout(time: 30, unit: 'MINUTES')
        buildDiscarder(logRotator(numToKeepStr: '20'))
        disableConcurrentBuilds()
    }

    stages {
        // ── Setup: load credentials, detect provider ─────────
        stage('Setup') {
            steps {
                script {
                    if (!fileExists(params.CONTRACT_FILE)) {
                        error "Contract not found: ${params.CONTRACT_FILE}"
                    }
                    if (params.CREDENTIALS_ID) {
                        withCredentials([file(credentialsId: params.CREDENTIALS_ID,
                                              variable: 'CREDS_FILE')]) {
                            sh "cp \$CREDS_FILE ${WORKSPACE}/.fluid-creds"
                        }
                    } else if (fileExists('.env')) {
                        sh "cp .env ${WORKSPACE}/.fluid-creds"
                    }
                }
                // Auto-detect: JSON → GCP key; otherwise → env vars
                sh '''
                    if [ -f .fluid-creds ]; then
                        if python3 -c "import json; d=json.load(open('.fluid-creds')); \
                           assert d.get('type')=='service_account'" 2>/dev/null; then
                            cp .fluid-creds .gcp-key.json
                            PROJECT=$(python3 -c "import json; \
                              print(json.load(open('.gcp-key.json')).get('project_id',''))")
                            printf "GOOGLE_APPLICATION_CREDENTIALS=%s/.gcp-key.json\n\
                              GCP_PROJECT=%s\n" "$WORKSPACE" "$PROJECT" > .fluid-env
                            echo "Loaded GCP service account (project: ${PROJECT})"
                        else
                            grep -v '^#' .fluid-creds | grep -v '^$' \
                              | sed 's/^export //' > .fluid-env
                            echo "Loaded credentials from env file"
                        fi
                    else
                        touch .fluid-env
                        echo "No credentials file — using inherited environment"
                    fi
                '''
                sh '''
                    set -a; . .fluid-env; set +a
                    echo "=================================================="
                    echo "  FLUID Pipeline"
                    echo "=================================================="
                    echo "Contract : ${CONTRACT_FILE}"
                    echo "Env      : ${ENV}"
                    fluid --version 2>/dev/null || echo "CLI version unavailable"
                    python3 --version
                '''
            }
        }

        // ── Validate → Export → Compile → Plan → Test ────────
        stage('Validate Contract') {
            steps {
                sh '''
                    set -a; . .fluid-env; set +a
                    mkdir -p reports
                    fluid validate ${CONTRACT_FILE} --verbose \
                      2>&1 | tee reports/validation.log
                '''
            }
        }

        stage('Export Standards') {
            steps {
                sh '''
                    mkdir -p standards
                    fluid odps export ${CONTRACT_FILE} \
                      --output standards/product.odps.json || true
                    fluid odcs export ${CONTRACT_FILE} \
                      --output standards/product.odcs.yaml || true
                '''
            }
        }

        stage('Compile IAM Policies') {
            steps {
                sh '''
                    set -a; . .fluid-env; set +a
                    mkdir -p runtime/policy
                    fluid policy-compile ${CONTRACT_FILE} \
                        --env ${ENV} \
                        --out runtime/policy/bindings.json || {
                        echo '{"bindings":[],"warnings":["policy-compile unavailable"]}' \
                          > runtime/policy/bindings.json
                    }
                '''
            }
        }

        stage('Generate Plan') {
            steps {
                sh '''
                    set -a; . .fluid-env; set +a
                    mkdir -p plans
                    fluid plan ${CONTRACT_FILE} \
                      --env ${ENV} --out plans/plan-${ENV}.json || true
                '''
            }
        }

        stage('Run Tests') {
            steps {
                sh '''
                    set -a; . .fluid-env; set +a
                    fluid contract-tests ${CONTRACT_FILE} || true
                '''
            }
        }

        // ── Apply infrastructure + IAM ───────────────────────
        stage('Apply Infrastructure') {
            steps {
                sh '''
                    set -a; . .fluid-env; set +a
                    mkdir -p runtime
                    fluid apply ${CONTRACT_FILE} \
                        --env ${ENV} --yes \
                        --report runtime/apply-report-${ENV}.html
                '''
            }
        }

        stage('Apply IAM Policies') {
            steps {
                sh """
                    set -a; . .fluid-env; set +a
                    if [ -f runtime/policy/bindings.json ]; then
                        COUNT=\$(python3 -c "import json; \
                          print(len(json.load(open('runtime/policy/bindings.json')) \
                          .get('bindings',[])))" 2>/dev/null || echo 0)
                        if [ "\$COUNT" != "0" ]; then
                            echo "Applying \$COUNT IAM binding(s)..."
                            fluid policy-apply runtime/policy/bindings.json \
                                --mode ${params.ENFORCE_IAM ? 'enforce' : 'check'} \
                              || true
                        else
                            echo "No IAM bindings to apply"
                        fi
                    fi
                """
            }
        }

        // ── Execute builds + generate Airflow DAG ────────────
        stage('Execute Builds') {
            when { expression { params.RUN_EXECUTION } }
            steps {
                sh '''
                    set -a; . .fluid-env; set +a
                    [ -f requirements.txt ] && pip3 install --quiet -r requirements.txt
                    fluid apply ${CONTRACT_FILE} --mode amend-and-build
                '''
            }
        }

        stage('Generate Airflow DAG') {
            steps {
                sh '''
                    mkdir -p airflow-dags
                    CONTRACT_ID=$(grep -m1 "^id:" ${CONTRACT_FILE} \
                      | cut -d" " -f2 | tr -d "\"" | tr "." "_")
                    fluid generate-airflow ${CONTRACT_FILE} \
                        --output airflow-dags/data_product_${ENV}.py \
                        --dag-id "${CONTRACT_ID}_${ENV}" || true
                    [ -f airflow-dags/data_product_${ENV}.py ] \
                      && python3 -m py_compile airflow-dags/data_product_${ENV}.py \
                      && echo "DAG valid"
                '''
            }
        }

        // ── Summary ──────────────────────────────────────────
        stage('Summary') {
            steps {
                sh '''
                    echo "=================================================="
                    echo "  FLUID Pipeline Complete"
                    echo "=================================================="
                    grep -m1 "^id:" ${CONTRACT_FILE}   | cut -d" " -f2
                    grep -m1 "^name:" ${CONTRACT_FILE}  | cut -d" " -f2-
                    echo "Env: ${ENV}"
                    echo ""
                    echo "Artifacts:"
                    ls -lh reports/*.log 2>/dev/null                    || true
                    ls -lh runtime/apply-report-${ENV}.html 2>/dev/null || true
                    ls -lh runtime/policy/bindings.json 2>/dev/null     || true
                    ls -lh airflow-dags/*.py 2>/dev/null                || true
                '''
            }
        }
    }

    post {
        always {
            sh 'rm -f .fluid-creds .fluid-env .gcp-key.json 2>/dev/null || true'
            archiveArtifacts artifacts: '**/*.log, **/*.json, **/*.html, **/*.py',
                             allowEmptyArchive: true
            cleanWs()
        }
    }
}