Jenkins CI/CD for FLUID Data Products

stage('Deploy') {
    sh 'bq mk dataset'
    sh 'bq load ...'
    sh 'dbt run'
    sh 'python update_metadata.py'
}

stage('Deploy') {
    sh 'fluid validate contract.fluid.yaml'
    sh 'fluid plan contract.fluid.yaml'
    sh 'fluid apply contract.fluid.yaml'
    sh 'fluid verify contract.fluid.yaml'
}

// In Jenkins System Configuration

// 1. Add GCP credentials
Credentials → Add Credentials
  Kind: Google Service Account from private key
  Project Name: dust-labs-485011
  ID: gcp-data-product-deployer

// 2. Configure tools
Global Tool Configuration
  Python: Python 3.9+
  Git: Latest

// In Jenkins Job Configuration
Pipeline → Definition → Pipeline script from SCM
  SCM: Git
  Repository URL: https://github.com/your-org/fluid-mono.git
  Script Path: forge_docs/examples/bitcoin-tracker/Jenkinsfile

// Copy Jenkinsfile content directly into Jenkins job

environment {
    GCP_PROJECT_ID = 'dust-labs-485011'
    GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-data-product-deployer')
}

stage('Setup Environment') {
    steps {
        sh '''
            # Install Python packages
            pip install dbt-core dbt-bigquery google-cloud-bigquery
            
            # Verify contract file exists
            test -f contract.fluid.yaml
            
            # Test GCP access
            bq ls --project_id=${GCP_PROJECT_ID}
        '''
    }
}

stage('Validate Contract') {
    steps {
        sh '''
            python3 -m fluid_build.cli validate contract.fluid.yaml
        '''
    }
}

stage('Static Analysis') {
    steps {
        sh '''
            # Check governance metadata
            python3 check_governance.py
            
            # Verify dbt models exist
            test -f dbt/models/daily_price_summary.sql
        '''
    }
}

stage('Plan Deployment') {
    steps {
        sh '''
            python3 -m fluid_build.cli plan contract.fluid.yaml
        '''
    }
}

stage('Run Tests') {
    when {
        expression { !params.SKIP_TESTS }
    }
    steps {
        sh '''
            # Run dbt tests
            cd dbt && dbt test
            
            # Custom data quality checks
            python3 data_quality_checks.py
        '''
    }
}

stage('Deploy') {
    when {
        expression { !params.DRY_RUN }
    }
    steps {
        // Production requires manual approval
        input message: 'Deploy to Production?'
        
        sh '''
            # Run dbt models
            cd dbt && dbt run
            
            # Apply labels
            bq update --set_label environment:${ENVIRONMENT} \\
                      --set_label cost-center:engineering \\
                      ${GCP_PROJECT_ID}:crypto_data.bitcoin_prices
            
            # Load fresh data
            python3 load_bitcoin_price_batch.py
        '''
    }
}

// Require approval for production
if (params.ENVIRONMENT == 'production') {
    input message: 'Deploy to Production?', 
          ok: 'Deploy',
          submitter: 'admin,release-manager'
}

stage('Verify Deployment') {
    steps {
        sh '''
            # Run FLUID verify
            python3 -m fluid_build.cli verify contract.fluid.yaml
            
            # Check labels applied
            bq show --format=json bitcoin_prices | jq '.labels'
            
            # Query sample data
            bq query "SELECT * FROM daily_price_summary LIMIT 1"
        '''
    }
}

parameters {
    choice(
        name: 'ENVIRONMENT',
        choices: ['staging', 'production'],
        description: 'Deployment environment'
    )
    booleanParam(
        name: 'DRY_RUN',
        defaultValue: false,
        description: 'Plan only, do not apply'
    )
    booleanParam(
        name: 'SKIP_TESTS',
        defaultValue: false,
        description: 'Skip test execution'
    )
}

// ❌ Never hardcode credentials
environment {
    GCP_KEY = 'AKIA...'  // BAD!
}

// ✅ Use Jenkins credentials
environment {
    GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-service-account')
}

// Require approval for sensitive operations
if (params.ENVIRONMENT == 'production') {
    input message: 'Proceed with production deployment?',
          submitter: 'release-managers'
}

post {
    always {
        // Log all deployments
        sh '''
            echo "$(date): Deployment to ${ENVIRONMENT} by ${BUILD_USER}" \
                >> /var/log/fluid-deployments.log
        '''
    }
}

pipeline {
    stages {
        stage('Deploy to Staging') {
            environment {
                GCP_PROJECT_ID = 'project-staging'
            }
            steps {
                sh 'fluid apply contract.fluid.yaml'
            }
        }
        
        stage('Smoke Test Staging') {
            steps {
                sh 'python3 smoke_tests.py --env=staging'
            }
        }
        
        stage('Deploy to Production') {
            when {
                branch 'main'
            }
            environment {
                GCP_PROJECT_ID = 'project-production'
            }
            steps {
                input 'Deploy to production?'
                sh 'fluid apply contract.fluid.yaml'
            }
        }
    }
}

stage('Tests') {
    parallel {
        stage('Unit Tests') {
            steps {
                sh 'pytest tests/unit'
            }
        }
        stage('dbt Tests') {
            steps {
                sh 'dbt test'
            }
        }
        stage('Data Quality') {
            steps {
                sh 'python3 dq_checks.py'
            }
        }
    }
}

stage('Deploy') {
    steps {
        script {
            try {
                sh 'fluid apply contract.fluid.yaml'
            } catch (Exception e) {
                echo "Deployment failed, rolling back..."
                sh 'git checkout HEAD~1 contract.fluid.yaml'
                sh 'fluid apply contract.fluid.yaml'
                throw e
            }
        }
    }
}