# Compile and test Concrete-cuda on an AWS instance
name: Concrete Cuda tests

on:
  workflow_call:
  workflow_dispatch:

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Clang
        run: |
          sudo apt-get install clang-format-11
      - name: Format Concrete-cuda
        run: |
          cd backends/concrete-cuda/implementation
          ./clang-format.sh
      - name: Check diff
        run: git diff --exit-code

  start-runner:
    name: Start EC2 runner
    runs-on: ubuntu-20.04
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_IAM_ID }}
          aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }}
          aws-region: us-east-1
      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2.2.0
        with:
          mode: start
          github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
          ec2-image-id: ami-0c4d39cb3dba0fcff
          ec2-instance-type: p3.2xlarge
          subnet-id: subnet-8123c9e7
          security-group-id: sg-0466d33ced960ba35
          
  run-cuda-tests-linux:
    concurrency:
      group: concrete_cuda_test-${{ github.ref }}
      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
    name: Test code in EC2
    needs: start-runner
    runs-on: ${{ needs.start-runner.outputs.label }}
    strategy:
      fail-fast: false
      # explicit include-based build matrix, of known valid options
      matrix:
        include:
          # 20.04 supports CUDA 11.0+ (gcc >= 5 && gcc <= 10). SM < 52 deprecated since 11.0
          - os: ubuntu-20.04
            cuda: "11.8"
            old_cuda: "11.1"
            cuda_arch: "70"
            gcc: 8
    env:
      CUDA_PATH: /usr/local/cuda-${{ matrix.cuda }}
      OLD_CUDA_PATH: /usr/local/cuda-{{ matrix.old_cuda }}

    steps:
    - uses: actions/checkout@v2
    - name: Export CUDA variables
      if: ${{ !cancelled() }}
      run: |
        echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}"
        echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}"
        echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}"
        echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> "${GITHUB_ENV}"
    # Specify the correct host compilers
    - name: Export gcc and g++ variables
      if: ${{ !cancelled() }}
      run: |
        echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> "${GITHUB_ENV}"
        echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> "${GITHUB_ENV}"
        echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> "${GITHUB_ENV}"
        echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
    - name: Build concrete-cuda
      if: ${{ !cancelled() }}
      run: |
        cd backends/concrete-cuda/implementation
        mkdir build
        cd build
        cmake ..
        make -j8
    #- name: Test concrete-cuda with Cuda 11.8
    #  if: ${{ !cancelled() }}
    #  run: |
    #    cd backends/concrete-cuda/implementation/build
    #    ./test/test_concrete_cuda

    #- name: Export variables for CUDA 11.1
    #  run: |
    #    echo "CUDA_PATH=$OLD_CUDA_PATH" >> "${GITHUB_ENV}"
    #    echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}"
    #    echo "LD_LIBRARY_PATH=$OLD_CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}"
    #    echo "CUDACXX=$OLD_CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}"
    #- name: Build concrete-cuda with Cuda 11.1
    #  if: ${{ !cancelled() }}
    #  run: |
    #    cd backends/concrete-cuda/implementation
    #    mkdir build-old-cuda
    #    cd build-old-cuda
    #    cmake ..
    #    make -j8
    #- name: Test concrete-cuda with Cuda 11.1
    #  if: ${{ !cancelled() }}
    #  run: |
    #    cd backends/concrete-cuda/implementation/build-old-cuda
    #    ./test/test_concrete_cuda

    - name: Slack Notification
      if: ${{ always() }}
      continue-on-error: true
      uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
      env:
        SLACK_COLOR: ${{ job.status }}
        SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
        SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
        SLACK_MESSAGE: "Cuda AWS tests finished with status ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
        SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

  stop-runner:
    name: Stop EC2 runner
    needs:
      - start-runner
      - run-cuda-tests-linux
    runs-on: ubuntu-20.04
    if: ${{ always() && (needs.start-runner.result != 'skipped') }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_IAM_ID }}
          aws-secret-access-key: ${{ secrets.AWS_IAM_KEY }}
          aws-region: us-east-1
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2.2.0
        with:
          mode: stop
          github-token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}