mirror of
https://github.com/zama-ai/concrete.git
synced 2026-02-09 03:55:04 -05:00
464 lines
18 KiB
YAML
464 lines
18 KiB
YAML
name: concretefhe CI Pipeline
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches:
|
|
- main
|
|
tags:
|
|
- "v*"
|
|
|
|
# Allows you to run this workflow manually from the Actions tab
|
|
workflow_dispatch:
|
|
|
|
# Allows external webhook trigger
|
|
repository_dispatch:
|
|
types:
|
|
- rebuild-env-docker
|
|
|
|
schedule:
|
|
# * is a special character in YAML so you have to quote this string
|
|
# At 22:00 on Sunday
|
|
# Timezone is UTC, so Paris time is +2 during the summer and +1 during winter
|
|
- cron: '0 22 * * 0'
|
|
|
|
env:
|
|
FORCE_REBUILD_DOCKER: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'repository_dispatch' && github.event.action == 'rebuild-env-docker') }}
|
|
ENV_DOCKERFILE: docker/Dockerfile.concretefhe-env
|
|
PREFLIGHT_IMAGE_BASE: ghcr.io/zama-ai/concretefhe-env:preflight
|
|
LATEST_IMAGE: ghcr.io/zama-ai/concretefhe-env:latest
|
|
BASE_IMAGE: ghcr.io/zama-ai/concretefhe-env
|
|
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
|
|
jobs:
|
|
build-preflight-docker:
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
name: Build & Push the concretefhe-env preflight Docker Image
|
|
runs-on: ubuntu-20.04
|
|
outputs:
|
|
image: ${{ steps.set_image.outputs.image || env.LATEST_IMAGE }}
|
|
needs-push: ${{ env.BUILD_DOCKER }}
|
|
force-rebuild-docker: ${{ env.FORCE_REBUILD_DOCKER }}
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
|
|
- name: Get changed files
|
|
if: ${{ (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) || github.event_name == 'pull_request' }}
|
|
uses: Ana06/get-changed-files@ea75ed777daf24d6e95f43957bd26b1eab20806c
|
|
id: files
|
|
with:
|
|
format: 'space-delimited'
|
|
- name: Should rebuild docker check
|
|
run : |
|
|
set +e
|
|
echo "${{ steps.files.outputs.all }}" | grep "${ENV_DOCKERFILE}"
|
|
DOCKERFILE_CHANGED=$?
|
|
if [[ "${DOCKERFILE_CHANGED}" == "0" || "${FORCE_REBUILD_DOCKER}" == "true" ]]; then
|
|
echo "Should rebuild docker image!"
|
|
echo "BUILD_DOCKER=true" >> "$GITHUB_ENV"
|
|
else
|
|
echo "Docker image up to date."
|
|
echo "BUILD_DOCKER=false" >> "$GITHUB_ENV"
|
|
fi
|
|
- name: Set prefligh Docker image
|
|
id: set_image
|
|
if: ${{ fromJSON(env.BUILD_DOCKER) }}
|
|
run: |
|
|
PREFLIGHT_IMAGE_TAG=$(echo ${{ github.ref }} | sed -e 's/\//-/g')
|
|
PREFLIGHT_IMAGE="${PREFLIGHT_IMAGE_BASE}-${PREFLIGHT_IMAGE_TAG}"
|
|
echo "::set-output name=image::${PREFLIGHT_IMAGE}"
|
|
echo "PREFLIGHT_IMAGE=${PREFLIGHT_IMAGE}" >> "$GITHUB_ENV"
|
|
- name: Set up Docker Buildx
|
|
if: ${{ fromJSON(env.BUILD_DOCKER) }}
|
|
id: buildx
|
|
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
|
|
- name: Login to GitHub Container Registry
|
|
if: ${{ fromJSON(env.BUILD_DOCKER) }}
|
|
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
- name: Build concretefhe-env Image
|
|
if: ${{ success() && !cancelled() && fromJSON(env.BUILD_DOCKER) }}
|
|
uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
|
|
with:
|
|
context: .
|
|
builder: ${{ steps.buildx.outputs.name }}
|
|
file: docker/Dockerfile.concretefhe-env
|
|
push: true
|
|
tags: "${{ env.PREFLIGHT_IMAGE }}"
|
|
no-cache: true
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Docker image preflight build ${{ env.PREFLIGHT_IMAGE }} finished with \
|
|
status ${{ job.status }}. Rebuilt image: ${{ env.BUILD_DOCKER || 'false' }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
build:
|
|
needs: [build-preflight-docker]
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: ${{ needs.build-preflight-docker.outputs.image }}
|
|
credentials:
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
strategy:
|
|
matrix:
|
|
python-version: [3.8]
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@dc73133d4da04e56a135ae2246682783cc7c7cb6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- name: Cache Installation Files
|
|
uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
|
|
with:
|
|
# Paths are Unix specific for now
|
|
path: |
|
|
~/.cache/pip
|
|
~/.cache/pypoetry
|
|
# Ignore line break in the evaluated double quoted string
|
|
key: "${{ runner.os }}-build-${{ matrix.python-version }}-\
|
|
${{ hashFiles('poetry.lock') }}"
|
|
restore-keys: |
|
|
${{ runner.os }}-build-${{ matrix.python-version }}-
|
|
${{ runner.os }}-build-
|
|
${{ runner.os }}-
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install poetry
|
|
make setup_env
|
|
- name: Conformance and Docs build
|
|
id: conformance
|
|
if: ${{ success() && !cancelled() }}
|
|
env:
|
|
# TODO: remove this when JIT doesn't need this
|
|
# Required to be sure that docs reads all files with MLIR imports properly
|
|
LD_PRELOAD: /compiler/build/lib/Runtime/libZamalangRuntime.so
|
|
# pcc launches an internal target with proper flags
|
|
# docs is run here too as it can fail and we catch errors during the build
|
|
run: |
|
|
make --keep-going pcc docs
|
|
- name: Archive docs artifacts
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
|
|
with:
|
|
name: html-docs
|
|
path: docs/_build/html
|
|
- name: PyTest Source Code
|
|
id: pytest
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
env:
|
|
# TODO: remove this when JIT doesn't need this
|
|
LD_PRELOAD: /compiler/build/lib/Runtime/libZamalangRuntime.so
|
|
run: |
|
|
make pytest
|
|
- name: Test CodeBlocks
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
env:
|
|
# TODO: remove this when JIT doesn't need this
|
|
LD_PRELOAD: /compiler/build/lib/Runtime/libZamalangRuntime.so
|
|
run: |
|
|
make test_codeblocks
|
|
- name: PyTest Notebooks
|
|
if: ${{ github.event_name == 'schedule' && steps.conformance.outcome == 'success' && !cancelled() }}
|
|
env:
|
|
# TODO: remove this when JIT doesn't need this
|
|
LD_PRELOAD: /compiler/build/lib/Runtime/libZamalangRuntime.so
|
|
run: |
|
|
make pytest_nb
|
|
- name: Test coverage
|
|
id: coverage
|
|
if: ${{ always() && steps.pytest.outcome != 'skipped' && !cancelled() }}
|
|
run: |
|
|
./script/actions_utils/coverage.sh ${{ github.base_ref }}
|
|
- name: Archive test coverage
|
|
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
|
|
if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
|
|
with:
|
|
name: coverage
|
|
path: coverage.html
|
|
- name: Comment with coverage
|
|
uses: marocchino/sticky-pull-request-comment@82e7a0d3c51217201b3fedc4ddde6632e969a477
|
|
if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
|
|
with:
|
|
path: diff-coverage.txt
|
|
recreate: true
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Build finished with status ${{ job.status }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
publish-docs:
|
|
needs: [build]
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
runs-on: ubuntu-20.04
|
|
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
|
|
|
steps:
|
|
- name: Download Documentation
|
|
id: download
|
|
uses: actions/download-artifact@3be87be14a055c47b01d3bd88f8fe02320a9bb60
|
|
with:
|
|
name: html-docs
|
|
|
|
- name: Publish Documentation to S3
|
|
id: publish
|
|
if: ${{ steps.download.outcome == 'success' && !cancelled() }}
|
|
uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83
|
|
with:
|
|
args: --delete --acl public-read
|
|
env:
|
|
AWS_S3_BUCKET: ${{ secrets.AWS_REPO_DOCUMENTATION_BUCKET_NAME }}
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_REGION: ${{ secrets.AWS_REGION }}
|
|
SOURCE_DIR: '.'
|
|
DEST_DIR: 'concretefhe'
|
|
|
|
- name: Invalidate CloudFront Cache
|
|
if: ${{ steps.publish.outcome == 'success' }}
|
|
uses: awact/cloudfront-action@8bcfabc7b4bbc0cb8e55e48527f0e3a6d681627c
|
|
env:
|
|
SOURCE_PATH: '/concretefhe/*'
|
|
AWS_REGION: ${{ secrets.AWS_REGION }}
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
DISTRIBUTION_ID: ${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }}
|
|
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Publishing documentation finished with status ${{ job.status }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
push-docker-image:
|
|
needs: [build-preflight-docker, build]
|
|
if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main' && fromJSON(needs.build-preflight-docker.outputs.needs-push)) || fromJSON(needs.build-preflight-docker.outputs.force-rebuild-docker) }}
|
|
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
name: Push env docker image
|
|
runs-on: ubuntu-20.04
|
|
env:
|
|
PREFLIGHT_IMAGE: ${{ needs.build-preflight-docker.outputs.image }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
|
|
- name: Login to GitHub Container Registry
|
|
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
- name: Pull preflight image
|
|
run: |
|
|
docker pull "${PREFLIGHT_IMAGE}"
|
|
- name: Retag to latest and epoch-sha1 and push
|
|
run: |
|
|
EPOCH=$(date +%s)
|
|
SHA1=$(git rev-parse HEAD)
|
|
TAGGED_IMAGE="${BASE_IMAGE}:${EPOCH}-${SHA1}"
|
|
docker tag "${PREFLIGHT_IMAGE}" "${LATEST_IMAGE}"
|
|
docker tag "${PREFLIGHT_IMAGE}" "${TAGGED_IMAGE}"
|
|
docker push "${LATEST_IMAGE}"
|
|
docker push "${TAGGED_IMAGE}"
|
|
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Pushing docker image ${{ env.BASE_IMAGE }} finished with status \
|
|
${{ job.status }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
package-release:
|
|
needs: [build]
|
|
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
|
|
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
name: Prepare docker image release
|
|
runs-on: ubuntu-20.04
|
|
|
|
env:
|
|
RELEASE_IMAGE_BASE: ghcr.io/zama-ai/concretefhe
|
|
|
|
steps:
|
|
- uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
|
|
- name: Set tag in env
|
|
run: |
|
|
GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
|
|
RELEASE_IMG_TAG="${RELEASE_IMAGE_BASE}:${GIT_TAG}"
|
|
echo "RELEASE_IMG_TAG=${RELEASE_IMG_TAG}" >> "$GITHUB_ENV"
|
|
- name: Set up Docker Buildx
|
|
id: buildx
|
|
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
|
|
- name: Login to GitHub Container Registry
|
|
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
- name: Build concretefhe Image
|
|
if: ${{ success() && !cancelled() }}
|
|
uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
|
|
with:
|
|
context: .
|
|
builder: ${{ steps.buildx.outputs.name }}
|
|
file: docker/Dockerfile.release
|
|
load: true
|
|
push: false
|
|
tags: "${{ env.RELEASE_IMG_TAG }}"
|
|
no-cache: true
|
|
- name: Release image sanity check and push
|
|
if: ${{ success() && !cancelled() }}
|
|
run: |
|
|
echo "Running sanity check for ${RELEASE_IMG_TAG}"
|
|
docker run --rm --env LD_PRELOAD=/compiler/build/lib/Runtime/libZamalangRuntime.so \
|
|
-v "$(pwd)"/docker/release_resources:/data \
|
|
"${RELEASE_IMG_TAG}" /bin/bash -c "python ./sanity_check.py"
|
|
docker push "${RELEASE_IMG_TAG}"
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Pushing docker image ${{ env.RELEASE_IMG_TAG }} finished with status \
|
|
${{ job.status }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
send-report:
|
|
if: ${{ always() }}
|
|
needs: [
|
|
build-preflight-docker,
|
|
build,
|
|
publish-docs,
|
|
push-docker-image,
|
|
package-release,
|
|
]
|
|
|
|
name: Send Slack notification
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- name: Slack Notification
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "Full run here: ${{ env.ACTION_RUN_URL }}\n\
|
|
- build-preflight-docker: ${{ needs.build-preflight-docker.outputs.report || 'Did not run.' }}\n\n\
|
|
- build: ${{ needs.build.outputs.report || 'Did not run.' }}\n\n\
|
|
- publish-docs: ${{ needs.publish-docs.outputs.report || 'Did not run.' }}\n\n\
|
|
- push-docker-image: ${{ needs.push-docker-image.outputs.report || 'Did not run.' }}\n\n\
|
|
- package-release: ${{ needs.package-release.outputs.report || 'Did not run.' }}"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|