diff --git a/.github/actions/download_artifacts/action.yaml b/.github/actions/download_artifacts/action.yaml new file mode 100644 index 0000000000..bee94b0556 --- /dev/null +++ b/.github/actions/download_artifacts/action.yaml @@ -0,0 +1,50 @@ +name: Download Artifacts +description: Download artifacts preserving file permissions +inputs: + keys: + description: The artifact keys + required: true +runs: + using: composite + steps: + - name: Download artifacts + shell: python3 {0} + run: | + import sys + import os + sys.path.append(os.path.abspath('.github/workflows')) + + from configure_logger import configure_logger + from github_api_request import json_github_api_request, download_github_api_request + + from logging import getLogger + from os import environ + from zipfile import ZipFile + import tarfile + + logger = getLogger(__name__) + configure_logger(logger) + + input_keys = """${{ inputs.keys }}""" + logger.debug(f'Input keys: {input_keys}') + artifact_keys = filter(None, [x.strip() for x in input_keys.split('\n')]) + logger.debug(f'Parsed keys: {artifact_keys}') + + api_prefix = 'actions' + artifacts_info = json_github_api_request(f'{api_prefix}/runs/{environ["GITHUB_RUN_ID"]}/artifacts') + + for key in artifact_keys: + artifact_id = [x['id'] for x in artifacts_info['artifacts'] if x['name'] == key][0] + logger.debug(f'Artifact id: {key}: {artifact_id}') + zip_file = f'{key}.zip' + download_github_api_request(zip_file, f'{api_prefix}/artifacts/{artifact_id}/zip') + logger.debug(f'Unzipping: {zip_file}') + with ZipFile(zip_file) as archive: + archive.extractall() + os.remove(zip_file) + tar_file = f'{key}.tar' + logger.debug(f'Extracting: {tar_file}') + with tarfile.open(tar_file, 'r') as tar: + tar.extractall() + os.remove(tar_file) + diff --git a/.github/actions/job_wrapper/action.yaml b/.github/actions/job_wrapper/action.yaml new file mode 100644 index 0000000000..b1670e3a3b --- /dev/null +++ b/.github/actions/job_wrapper/action.yaml @@ -0,0 +1,26 @@ +name: Job Wrapper +description: Setup and cleanup for build jobs +inputs: + artifacts: + description: Required artifacts + required: false + default: '' + command: + description: The build command + required: true +runs: + using: composite + steps: + - name: Get artifacts + uses: ./.github/actions/download_artifacts + with: + keys: | + JUCE-utils + ${{ inputs.artifacts }} + - run: ${{ inputs.command }} + shell: ${{ runner.os == 'Windows' && 'powershell' || 'bash' }} + - name: Handle job failure + if: failure() + run: python3 JUCE-utils/.github/workflows/post_job.py + shell: ${{ runner.os == 'Windows' && 'powershell' || 'bash' }} + diff --git a/.github/actions/upload_artifact/action.yaml b/.github/actions/upload_artifact/action.yaml new file mode 100644 index 0000000000..3c5c95a5e4 --- /dev/null +++ b/.github/actions/upload_artifact/action.yaml @@ -0,0 +1,49 @@ +name: Upload Artifact +description: Upload an artifact preserving file permissions +inputs: + key: + description: The artifact key + required: true + paths: + description: The artifact paths + required: true +runs: + using: composite + steps: + - name: Create tarball + shell: python3 {0} + run: | + import sys + import os + sys.path.append(os.path.abspath('.github/workflows')) + + from configure_logger import configure_logger + + from logging import getLogger + from os import mkdir + import tarfile + + logger = getLogger(__name__) + configure_logger(logger) + + input_paths = """${{ inputs.paths }}""" + logger.debug(f'Input paths: {input_paths}') + paths = filter(None, [x.strip() for x in input_paths.split('\n')]) + logger.debug(f'Parsed paths: {paths}') + mkdir('tmp_artifact_upload') + archive_path = 'tmp_artifact_upload/${{ inputs.key }}.tar' + logger.debug(f'Creating archive: {archive_path}') + with tarfile.open('tmp_artifact_upload/${{ inputs.key }}.tar', 'w') as tar: + for path in paths: + logger.debug(f'Adding path to archive archive: {path}') + tar.add(path) + - uses: actions/upload-artifact@v4.6.0 + with: + name: ${{ inputs.key }} + path: tmp_artifact_upload + - name: Clean up + shell: python3 {0} + run: | + from shutil import rmtree + rmtree('tmp_artifact_upload') + diff --git a/.github/workflows/check-cla.yml b/.github/workflows/check-cla.yml index c657a4cff2..b42fcc40e0 100644 --- a/.github/workflows/check-cla.yml +++ b/.github/workflows/check-cla.yml @@ -7,6 +7,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} steps: - name: check-CLA + if: github.repository == 'juce-framework/JUCE' run: | import urllib.request import json diff --git a/.github/workflows/configure_logger.py b/.github/workflows/configure_logger.py new file mode 100644 index 0000000000..4e06cd8c94 --- /dev/null +++ b/.github/workflows/configure_logger.py @@ -0,0 +1,13 @@ +import logging +from os import getenv +from sys import stdout + +def configure_logger(logger): + handler = logging.StreamHandler(stdout) + formatter = logging.Formatter('[%(name)s] %(message)s') + handler.setFormatter(formatter) + level = logging.DEBUG if (getenv('RUNNER_DEBUG', '0').lower() not in ('0', 'f', 'false')) else logging.WARNING + logger.setLevel(level) + handler.setLevel(level) + logger.addHandler(handler) + diff --git a/.github/workflows/github_api_request.py b/.github/workflows/github_api_request.py new file mode 100644 index 0000000000..4a196ed5be --- /dev/null +++ b/.github/workflows/github_api_request.py @@ -0,0 +1,55 @@ +from configure_logger import configure_logger + +from logging import getLogger +from urllib.request import Request, urlopen +from urllib.error import HTTPError +from json import dumps, loads +from os import environ +from shutil import copyfileobj +from time import sleep + +logger = getLogger(__name__) +configure_logger(logger) + +def github_api_request(path, method='GET', data=None): + url = f'https://api.github.com/repos/{environ["GITHUB_REPOSITORY"]}/{path}' + logger.debug(f'Requesting GitHub API: {url}') + serialised_data = dumps(data).encode('utf-8') if data else None + if serialised_data: + logger.debug(f'Data: {serialised_data}') + req = Request( + url=url, + method=method, + headers={ + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' + }, + data=serialised_data + ) + req.add_unredirected_header('Authorization', f'Bearer {environ["GITHUB_API_TOKEN"]}') + num_attempts = 0 + while True: + response = None + try: + response = urlopen(req) + return response + except HTTPError as e: + num_attempts += 1 + if num_attempts == 3: + logger.warning(f'GitHub API access failed\n{e.headers}\n{e.fp.read()}') + raise e + logger.debug(f'Request attempt {num_attempts} failed, retrying') + sleep(5) + +def json_github_api_request(path, method='GET', data=None): + with github_api_request(path, method, data) as response: + result = loads(response.read().decode('utf-8')) + logger.debug(f'GitHub API result: {result}') + return result + +def download_github_api_request(filename, path, method='GET', data=None): + with github_api_request(path, method, data) as response: + with open(filename, 'wb') as f: + copyfileobj(response, f) + logger.debug(f'Downloaded to: {filename}') + diff --git a/.github/workflows/juce_private_build.yml b/.github/workflows/juce_private_build.yml new file mode 100644 index 0000000000..356266738d --- /dev/null +++ b/.github/workflows/juce_private_build.yml @@ -0,0 +1,57 @@ +name: JUCE Private Build + +on: + workflow_dispatch: + inputs: + triggerer: + required: false + type: string + default: '' + description: The GitHub ID to receive email notifications (leave blank) + nightly-targets: + required: false + type: string + default: "[]" + description: A list of nightly build targets in JSON format + cpp-std: + required: false + type: string + default: "" + description: The C++ standard to use (optional [20, 23]) + +run-name: "[${{ inputs.triggerer && inputs.triggerer || github.event.sender.login }}] ${{ github.sha }}" + +jobs: + build: + name: . + # Not having the ability to do a dynamic 'uses' call is a real pain. To + # test some new CI configuration you must set the branch in both places + # below. + uses: juce-framework/JUCE-utils/.github/workflows/main.yml@master + with: + juce-utils-branch: master + nightly-targets: ${{ inputs.nightly-targets }} + triggerer: ${{ inputs.triggerer && inputs.triggerer || github.event.sender.login }} + cpp-std: ${{ inputs.cpp-std }} + secrets: inherit + deploy: + if: ${{ contains(fromJSON('["master", "develop"]'), github.ref_name) && inputs.nightly-targets == '[]' }} + needs: [build] + name: Deploy + uses: juce-framework/JUCE-utils/.github/workflows/deploy.yml@master + secrets: inherit + docs: + if: ${{ contains(fromJSON('["master", "develop"]'), github.ref_name) && inputs.nightly-targets == '[]' }} + needs: [deploy] + name: Docs + uses: juce-framework/JUCE-utils/.github/workflows/docs.yml@master + secrets: inherit + notify: + if: ${{ inputs.nightly-targets == '[]' }} + needs: [docs] + name: Notify + uses: juce-framework/JUCE-utils/.github/workflows/notify.yml@master + with: + triggerer: ${{ inputs.triggerer && inputs.triggerer || github.event.sender.login }} + secrets: inherit + diff --git a/.github/workflows/juce_private_nightly_trigger.yml b/.github/workflows/juce_private_nightly_trigger.yml new file mode 100644 index 0000000000..4ea2a027f4 --- /dev/null +++ b/.github/workflows/juce_private_nightly_trigger.yml @@ -0,0 +1,31 @@ +name: JUCE Private Nightly Trigger +on: + schedule: + - cron: '0 3 * * *' +env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TRIGGER_WORKFLOW_REF: develop +jobs: + juce-private-nightly-trigger: + if: github.repository == 'juce-framework/JUCE-dev' + name: JUCE Push Trigger + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + with: + sparse-checkout: ./.github/workflows + - env: + TRIGGER_WORKFLOW_INPUTS: | + {"triggerer":"Nightly Build","nightly-targets":${{ vars.NIGHTLY_BUILD_TARGETS }}} + run: python3 ./.github/workflows/trigger_workflow.py + - if: ${{ contains(fromJSON(vars.NIGHTLY_BUILD_TARGETS), 'cpp20') }} + env: + TRIGGER_WORKFLOW_INPUTS: | + {"triggerer":"Nightly Build C++20","cpp-std":"20"} + run: python3 ./.github/workflows/trigger_workflow.py + - if: ${{ contains(fromJSON(vars.NIGHTLY_BUILD_TARGETS), 'cpp23') }} + env: + TRIGGER_WORKFLOW_INPUTS: | + {"triggerer":"Nightly Build C++23","cpp-std":"23"} + run: python3 ./.github/workflows/trigger_workflow.py + diff --git a/.github/workflows/juce_private_push_trigger.yml b/.github/workflows/juce_private_push_trigger.yml new file mode 100644 index 0000000000..3c0208062a --- /dev/null +++ b/.github/workflows/juce_private_push_trigger.yml @@ -0,0 +1,24 @@ +name: JUCE Private Push Trigger +on: + push: + branches: + - master + - develop + - bugfix/** + - feature/** +jobs: + juce-private-push-trigger: + if: github.repository == 'juce-framework/JUCE-dev' + name: JUCE Push Trigger + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + with: + sparse-checkout: ./.github/workflows + - name: Trigger a private build using the GitHub API + env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TRIGGER_WORKFLOW_INPUTS: | + {"triggerer":"${{ github.actor }}"} + run: python3 ./.github/workflows/trigger_workflow.py + diff --git a/.github/workflows/trigger_workflow.py b/.github/workflows/trigger_workflow.py new file mode 100644 index 0000000000..f282eadc8a --- /dev/null +++ b/.github/workflows/trigger_workflow.py @@ -0,0 +1,35 @@ +from configure_logger import configure_logger +from github_api_request import github_api_request, json_github_api_request + +from logging import getLogger +from os import getenv +from json import loads, dumps + +logger = getLogger(__name__) +configure_logger(logger) + +input_string = getenv('TRIGGER_WORKFLOW_INPUTS', '{}') +logger.debug(f'Input variable: {input_string}') +input_json = loads(input_string) +for key, value in input_json.items(): + if not isinstance(value, str): + input_json[key] = dumps(value) +logger.debug(f'Stringified input: {input_json}') + +api_path_prefix = 'actions/workflows' + +workflows = json_github_api_request(api_path_prefix) +workflow_path = getenv('TRIGGER_WORKFLOW_PATH', + '.github/workflows/juce_private_build.yml') +workflow = [x for x in workflows['workflows'] if x['path'] == workflow_path][0] +logger.debug(f'Workflow: {workflow}') + +trigger_data = { + 'ref': getenv('TRIGGER_WORKFLOW_REF', getenv('GITHUB_REF_NAME')), + 'inputs': input_json +} +logger.debug(f'Trigger_data: {trigger_data}') +github_api_request(f'{api_path_prefix}/{workflow["id"]}/dispatches', + method='POST', + data=trigger_data) + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 4962bc719e..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,9 +0,0 @@ -variables: - REF: &REF master - -include: - - project: juce-repos/JUCE-utils - file: /CI/gitlab-ci.yml - ref: *REF - inputs: - ref: *REF