From 0415ea138f4c7175e68fd1060be9625f35015be6 Mon Sep 17 00:00:00 2001 From: Giuseppe Carboni <giuseppecarboni89@live.com> Date: Fri, 4 Apr 2025 12:51:52 +0200 Subject: [PATCH] Continuous integration (#919) * Test for caching the virtual machine * Update for triggering the workflow * Fixed cache files path * Updated download-vm workflow * Updated deploy-discos workflow * Removed download workflow. It will go on master. * Updated deployment workflow * Updated variable for branch name * Shortened the job name * Added push as CI event --- .github/utils/download_from_gdrive.py | 53 ------------------ .github/utils/gdrive_requirements.txt | 4 -- .../{workflow.yml => deploy-discos.yml} | 54 ++++++++++++------- 3 files changed, 35 insertions(+), 76 deletions(-) delete mode 100755 .github/utils/download_from_gdrive.py delete mode 100644 .github/utils/gdrive_requirements.txt rename .github/workflows/{workflow.yml => deploy-discos.yml} (71%) diff --git a/.github/utils/download_from_gdrive.py b/.github/utils/download_from_gdrive.py deleted file mode 100755 index 1a01b42db..000000000 --- a/.github/utils/download_from_gdrive.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -from google.oauth2.credentials import Credentials -from google.auth.transport.requests import Request -from googleapiclient.discovery import build -from googleapiclient.http import MediaIoBaseDownload -import os -import io - -TOKEN_FILE = 'token.json' -VM_FILE_PATH = '/home/runner/discos_manager.ova' -ARCHIVE_FILE_PATH = '/home/runner/vagrant.tar.gz' -SCOPES = ['https://www.googleapis.com/auth/drive.readonly'] - -# Create the token file from the GH Secret -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(os.environ.get('GOOGLE_DRIVE_TOKEN')) - -# Authenticate with the token and eventually update it -creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) -if creds.expired and creds.refresh_token: - creds.refresh(Request()) - -# Download the VM -creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) -if creds.expired and creds.refresh_token: - creds.refresh(Request()) - -service = build('drive', 'v3', credentials=creds) - -downloader = MediaIoBaseDownload( - io.FileIO(ARCHIVE_FILE_PATH, 'wb'), - service.files().get_media( - fileId=os.environ.get('PROVISIONED_ARCHIVE_GDRIVE_ID') - ) -) -done = False -while not done: - _, done = downloader.next_chunk() - -downloader = MediaIoBaseDownload( - io.FileIO(VM_FILE_PATH, 'wb'), - service.files().get_media( - fileId=os.environ.get('PROVISIONED_VM_GDRIVE_ID'), - ), - chunksize=5*1024*1024 -) -done = False -while not done: - _, done = downloader.next_chunk() - -# Finally update the token file -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(creds.to_json()) diff --git a/.github/utils/gdrive_requirements.txt b/.github/utils/gdrive_requirements.txt deleted file mode 100644 index 7c6866175..000000000 --- a/.github/utils/gdrive_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google-auth -google-auth-oauthlib -google-auth-httplib2 -google-api-python-client diff --git a/.github/workflows/workflow.yml b/.github/workflows/deploy-discos.yml similarity index 71% rename from .github/workflows/workflow.yml rename to .github/workflows/deploy-discos.yml index f63b387d7..86de67074 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/deploy-discos.yml @@ -2,10 +2,22 @@ name: DISCOS deployment and build on: push: - workflow_dispatch: + pull_request: jobs: + wait-for-download: + name: Wait for any pending download + runs-on: ubuntu-latest + concurrency: + group: download + cancel-in-progress: false + steps: + - name: Go on with deployment + run: true deploy-discos: + name: Deploy DISCOS + runs-on: ubuntu-latest + needs: wait-for-download env: REPOSITORY_TOKEN: "${{ secrets.DEPENDENCIES_TOKEN }}" GH_TOKEN: "${{ secrets.GH_WORKFLOWS_TOKEN }}" @@ -16,32 +28,37 @@ jobs: fail-fast: false matrix: station: ['SRT', 'Medicina', 'Noto'] - runs-on: ubuntu-latest steps: - name: Free up space uses: jlumbroso/free-disk-space@main with: tool-cache: true - - name: Install Vagrant + - name: Install Vagrant and VirtualBox run: | wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list sudo apt update && sudo apt install vagrant - - name: Install VirtualBox - run: | sudo apt install virtualbox - - name: Clone the repository - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3' check-latest: true - - name: Download the provisioned virtual machine from Google Drive + - name: Clone the deployment repository + uses: actions/checkout@v4 + with: + repository: 'discos/deployment' + - name: Install deployment package and dependencies run: | - pip install -r .github/utils/gdrive_requirements.txt - python .github/utils/download_from_gdrive.py - gh secret set GOOGLE_DRIVE_TOKEN --org discos --visibility selected --repos discos,deployment < token.json + python -m pip install -r requirements.txt + pip install . + - name: Retrieve the VM from cache + uses: actions/cache/restore@v4 + with: + key: discos-manager-vm + path: | + /home/runner/discos_manager.ova + /home/runner/vagrant.tar.gz - name: Install the virtual machine run: | vboxmanage import discos_manager.ova --vsys 0 --options keepallmacs @@ -53,17 +70,16 @@ jobs: sed -i "s/$(cat id)/$(vboxmanage list vms | grep 'discos_manager' | grep -oP '(?<=\{)[0-9a-fA-F-]+(?=\})')/" action_provision sed -i "s/$(cat id)/$(vboxmanage list vms | grep 'discos_manager' | grep -oP '(?<=\{)[0-9a-fA-F-]+(?=\})')/" id working-directory: /home/runner/.deployment/.vagrant/machines/manager/virtualbox/ - - name: Clone the deployment repository - uses: actions/checkout@v4 - with: - repository: 'discos/deployment' - - name: Install deployment package and dependencies + - name: Set the branch name run: | - python -m pip install -r requirements.txt - pip install . + if [ "${{ github.event_name }}" == "push" ]; then + echo "BRANCH_NAME=${{ github.ref_name }}" >> $GITHUB_ENV + elif [ "${{ github.event_name }}" == "pull_request" ]; then + echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV + fi - name: Deploy DISCOS run: | - discos-deploy manager:development --deploy-only -s ${{ matrix.station }} -b ${{ github.ref_name }} + discos-deploy manager:development --deploy-only -s ${{ matrix.station }} -b ${{ env.BRANCH_NAME }} - name: Shutdown the virtual machine run: | discos-vms stop -- GitLab