[ci] Add benchmarks ci for master branch
This commit is contained in:
Zheng, Qi
volcano
parent
2041c7531c
commit
15ef7c1600
136
.github/workflows/benchmarks.yml
vendored
136
.github/workflows/benchmarks.yml
vendored
@ -2,6 +2,8 @@ name: Benchmarks Test
|
||||
# Currently this workflow is only used for running benchmarks test on schedule on branch 1.0.0-preview
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: labeled
|
||||
schedule:
|
||||
# Schedule to run on Wed,Sat at 10PM UTC (6AM CST)
|
||||
- cron: '0 22 * * 3,6'
|
||||
@ -9,6 +11,7 @@ on:
|
||||
jobs:
|
||||
Sysbench_Test:
|
||||
timeout-minutes: 40
|
||||
if: github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test')
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
@ -22,10 +25,19 @@ jobs:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- name: Checkout code from fork
|
||||
# This step is only needed when the pull request is labeled.
|
||||
if: contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test')
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# For pull request, we need to merge the commit from fork to the base
|
||||
ref: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
submodules: true
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
@ -48,7 +60,7 @@ jobs:
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
benchmark-data-dir-path: 'stable/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
@ -62,6 +74,7 @@ jobs:
|
||||
|
||||
Iperf3_Test:
|
||||
timeout-minutes: 40
|
||||
if: github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test')
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
@ -75,10 +88,19 @@ jobs:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- name: Checkout code from fork
|
||||
# This step is only needed when the pull request is labeled.
|
||||
if: contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test')
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# For pull request, we need to merge the commit from fork to the base
|
||||
ref: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
submodules: true
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
@ -101,113 +123,7 @@ jobs:
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
alert-threshold: '200%'
|
||||
comment-on-alert: true
|
||||
# Workflow will fail when an alert happens
|
||||
fail-on-alert: true
|
||||
- name: Clean the environment
|
||||
if: ${{ always() }}
|
||||
run: docker stop ${{ env.CONTAINER_NAME }}
|
||||
|
||||
SEFS_FIO_Test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
self_runner: [[self-hosted, SGX2-HW, benchmark]]
|
||||
|
||||
steps:
|
||||
- name: Clean before running
|
||||
run: |
|
||||
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
|
||||
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
container-name: ${{ github.job }}
|
||||
build-envs: 'OCCLUM_RELEASE_BUILD=1'
|
||||
|
||||
- name: Run fio download and build
|
||||
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /root/fio-microbench"
|
||||
|
||||
- name: Copy result
|
||||
run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json .
|
||||
|
||||
# Run `github-action-benchmark` action
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: FIO Benchmark on SEFS
|
||||
# What benchmark tool the output.txt came from
|
||||
tool: 'customBiggerIsBetter'
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
alert-threshold: '200%'
|
||||
comment-on-alert: true
|
||||
# Workflow will fail when an alert happens
|
||||
fail-on-alert: true
|
||||
- name: Clean the environment
|
||||
if: ${{ always() }}
|
||||
run: docker stop ${{ env.CONTAINER_NAME }}
|
||||
|
||||
AsyncSFS_FIO_Test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
self_runner: [[self-hosted, SGX2-HW, benchmark]]
|
||||
|
||||
steps:
|
||||
- name: Clean before running
|
||||
run: |
|
||||
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
|
||||
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
container-name: ${{ github.job }}
|
||||
build-envs: 'OCCLUM_RELEASE_BUILD=1'
|
||||
|
||||
- name: Run fio download and build
|
||||
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /sfs/fio-microbench"
|
||||
|
||||
- name: Copy result
|
||||
run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json .
|
||||
|
||||
# Run `github-action-benchmark` action
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: FIO Benchmark on AsyncSFS and JinDisk
|
||||
# What benchmark tool the output.txt came from
|
||||
tool: 'customBiggerIsBetter'
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
benchmark-data-dir-path: 'stable/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
|
||||
220
.github/workflows/benchmarks_dev.yml
vendored
Normal file
220
.github/workflows/benchmarks_dev.yml
vendored
Normal file
@ -0,0 +1,220 @@
|
||||
name: Benchmarks Test for dev branch
|
||||
# Currently this workflow is only used for running benchmarks test on schedule on branch 1.0.0-preview
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Schedule to run on Tue,Fri at 10PM UTC (6AM CST)
|
||||
- cron: '0 22 * * 2,5'
|
||||
|
||||
jobs:
|
||||
Sysbench_Test:
|
||||
timeout-minutes: 40
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
self_runner: [[self-hosted, SGX2-HW, benchmark]]
|
||||
|
||||
steps:
|
||||
- name: Clean before running
|
||||
run: |
|
||||
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
|
||||
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
container-name: ${{ github.job }}
|
||||
build-envs: 'OCCLUM_RELEASE_BUILD=1'
|
||||
|
||||
- name: Run sysbench download and build
|
||||
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/sysbench && ./sysbench.sh 120"
|
||||
|
||||
- name: Copy result
|
||||
run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/sysbench/result.json .
|
||||
|
||||
# Run `github-action-benchmark` action
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: Sysbench Benchmark
|
||||
# What benchmark tool the output.txt came from
|
||||
tool: 'customSmallerIsBetter'
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
alert-threshold: '200%'
|
||||
comment-on-alert: true
|
||||
# Workflow will fail when an alert happens
|
||||
fail-on-alert: true
|
||||
- name: Clean the environment
|
||||
if: ${{ always() }}
|
||||
run: docker stop ${{ env.CONTAINER_NAME }}
|
||||
|
||||
Iperf3_Test:
|
||||
timeout-minutes: 40
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
self_runner: [[self-hosted, SGX2-HW, benchmark]]
|
||||
|
||||
steps:
|
||||
- name: Clean before running
|
||||
run: |
|
||||
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
|
||||
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
container-name: ${{ github.job }}
|
||||
build-envs: 'OCCLUM_RELEASE_BUILD=1'
|
||||
|
||||
- name: Run iperf3 download and build
|
||||
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/iperf3 && ./iperf3.sh 120"
|
||||
|
||||
- name: Copy result
|
||||
run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/iperf3/result.json .
|
||||
|
||||
# Run `github-action-benchmark` action
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: Iperf3 Benchmark
|
||||
# What benchmark tool the output.txt came from
|
||||
tool: 'customBiggerIsBetter '
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
alert-threshold: '200%'
|
||||
comment-on-alert: true
|
||||
# Workflow will fail when an alert happens
|
||||
fail-on-alert: true
|
||||
- name: Clean the environment
|
||||
if: ${{ always() }}
|
||||
run: docker stop ${{ env.CONTAINER_NAME }}
|
||||
|
||||
SEFS_FIO_Test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
self_runner: [[self-hosted, SGX2-HW, benchmark]]
|
||||
|
||||
steps:
|
||||
- name: Clean before running
|
||||
run: |
|
||||
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
|
||||
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
container-name: ${{ github.job }}
|
||||
build-envs: 'OCCLUM_RELEASE_BUILD=1'
|
||||
|
||||
- name: Run fio download and build
|
||||
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /root/fio-microbench"
|
||||
|
||||
- name: Copy result
|
||||
run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json .
|
||||
|
||||
# Run `github-action-benchmark` action
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: FIO Benchmark on SEFS
|
||||
# What benchmark tool the output.txt came from
|
||||
tool: 'customBiggerIsBetter'
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
alert-threshold: '200%'
|
||||
comment-on-alert: true
|
||||
# Workflow will fail when an alert happens
|
||||
fail-on-alert: true
|
||||
- name: Clean the environment
|
||||
if: ${{ always() }}
|
||||
run: docker stop ${{ env.CONTAINER_NAME }}
|
||||
|
||||
AsyncSFS_FIO_Test:
|
||||
timeout-minutes: 60
|
||||
runs-on: ${{ matrix.self_runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
self_runner: [[self-hosted, SGX2-HW, benchmark]]
|
||||
|
||||
steps:
|
||||
- name: Clean before running
|
||||
run: |
|
||||
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
|
||||
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
ref: 1.0.0-preview
|
||||
|
||||
- uses: ./.github/workflows/composite_action/hw
|
||||
with:
|
||||
container-name: ${{ github.job }}
|
||||
build-envs: 'OCCLUM_RELEASE_BUILD=1'
|
||||
|
||||
- name: Run fio download and build
|
||||
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /sfs/fio-microbench"
|
||||
|
||||
- name: Copy result
|
||||
run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json .
|
||||
|
||||
# Run `github-action-benchmark` action
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@v1
|
||||
with:
|
||||
name: FIO Benchmark on AsyncSFS and JinDisk
|
||||
# What benchmark tool the output.txt came from
|
||||
tool: 'customBiggerIsBetter'
|
||||
# Where the output from the benchmark tool is stored
|
||||
output-file-path: result.json
|
||||
# Path to directory which contains benchmark files on GitHub pages branch
|
||||
benchmark-data-dir-path: 'dev/benchmarks'
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
# Show alert with commit comment on detecting possible performance regression
|
||||
alert-threshold: '200%'
|
||||
comment-on-alert: true
|
||||
# Workflow will fail when an alert happens
|
||||
fail-on-alert: true
|
||||
- name: Clean the environment
|
||||
if: ${{ always() }}
|
||||
run: docker stop ${{ env.CONTAINER_NAME }}
|
||||
@ -7,3 +7,16 @@ This set of demos shows how commonly used benchmarking tools can be run inside S
|
||||
* [iperf3](iperf3/): A demo of [Iperf3](https://github.com/esnet/iperf), a tool for measuring Internet bandwidth performance.
|
||||
* [sysbench](sysbench/): A demo of [Sysbench](https://github.com/akopytov/sysbench), a scriptable multi-threaded benchmark tool for Linux.
|
||||
|
||||
## Benchmarks Data
|
||||
|
||||
There are two enabled **benchmarks CI**(https://github.com/occlum/occlum/blob/1.0.0-preview/.github/workflows/benchmarks.yml) for continuous benchmarking. It utilizes the [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) to provide a chart view for visualized historical benchmarks data on the GitHub pages.
|
||||
|
||||
### CI and Data for Master branch
|
||||
|
||||
[**benchmarks CI**](https://github.com/occlum/occlum/blob/master/.github/workflows/benchmarks.yml)
|
||||
[**History Data**](https://occlum.io/occlum/stable/benchmarks/)
|
||||
|
||||
### CI and Data for Dev branch
|
||||
|
||||
[**benchmarks CI**](https://github.com/occlum/occlum/blob/master/.github/workflows/benchmarks_dev.yml)
|
||||
[**History Data**](https://occlum.io/occlum/dev/benchmarks/)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user