diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f646e759..326bc62d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -2,6 +2,8 @@ name: Benchmarks Test # Currently this workflow is only used for running benchmarks test on schedule on branch 1.0.0-preview on: + pull_request_target: + types: labeled schedule: # Schedule to run on Wed,Sat at 10PM UTC (6AM CST) - cron: '0 22 * * 3,6' @@ -9,6 +11,7 @@ on: jobs: Sysbench_Test: timeout-minutes: 40 + if: github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test') runs-on: ${{ matrix.self_runner }} strategy: matrix: @@ -22,10 +25,19 @@ jobs: - uses: AutoModality/action-clean@v1 - name: Checkout code + if: github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: true - ref: 1.0.0-preview + + - name: Checkout code from fork + # This step is only needed when the pull request is labeled. + if: contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test') + uses: actions/checkout@v2 + with: + # For pull request, we need to merge the commit from fork to the base + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: true - uses: ./.github/workflows/composite_action/hw with: @@ -48,7 +60,7 @@ jobs: # Where the output from the benchmark tool is stored output-file-path: result.json # Path to directory which contains benchmark files on GitHub pages branch - benchmark-data-dir-path: 'dev/benchmarks' + benchmark-data-dir-path: 'stable/benchmarks' github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # Show alert with commit comment on detecting possible performance regression @@ -62,6 +74,7 @@ jobs: Iperf3_Test: timeout-minutes: 40 + if: github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test') runs-on: ${{ matrix.self_runner }} strategy: matrix: @@ -75,10 +88,19 @@ jobs: - uses: AutoModality/action-clean@v1 - name: Checkout code + if: github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: true - ref: 1.0.0-preview + + - name: Checkout code from fork + # This step is only needed when the pull request is labeled. + if: contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') || contains(github.event.pull_request.labels.*.name, 'Pre-release-test') + uses: actions/checkout@v2 + with: + # For pull request, we need to merge the commit from fork to the base + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: true - uses: ./.github/workflows/composite_action/hw with: @@ -101,113 +123,7 @@ jobs: # Where the output from the benchmark tool is stored output-file-path: result.json # Path to directory which contains benchmark files on GitHub pages branch - benchmark-data-dir-path: 'dev/benchmarks' - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - # Workflow will fail when an alert happens - fail-on-alert: true - - name: Clean the environment - if: ${{ always() }} - run: docker stop ${{ env.CONTAINER_NAME }} - - SEFS_FIO_Test: - timeout-minutes: 60 - runs-on: ${{ matrix.self_runner }} - strategy: - matrix: - self_runner: [[self-hosted, SGX2-HW, benchmark]] - - steps: - - name: Clean before running - run: | - sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}" - - - uses: AutoModality/action-clean@v1 - - - name: Checkout code - uses: actions/checkout@v2 - with: - submodules: true - ref: 1.0.0-preview - - - uses: ./.github/workflows/composite_action/hw - with: - container-name: ${{ github.job }} - build-envs: 'OCCLUM_RELEASE_BUILD=1' - - - name: Run fio download and build - run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /root/fio-microbench" - - - name: Copy result - run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json . - - # Run `github-action-benchmark` action - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: FIO Benchmark on SEFS - # What benchmark tool the output.txt came from - tool: 'customBiggerIsBetter' - # Where the output from the benchmark tool is stored - output-file-path: result.json - # Path to directory which contains benchmark files on GitHub pages branch - benchmark-data-dir-path: 'dev/benchmarks' - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - # Workflow will fail when an alert happens - fail-on-alert: true - - name: Clean the environment - if: ${{ always() }} - run: docker stop ${{ env.CONTAINER_NAME }} - - AsyncSFS_FIO_Test: - timeout-minutes: 60 - runs-on: ${{ matrix.self_runner }} - strategy: - matrix: - self_runner: [[self-hosted, SGX2-HW, benchmark]] - - steps: - - name: Clean before running - run: | - sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}" - - - uses: AutoModality/action-clean@v1 - - - name: Checkout code - uses: actions/checkout@v2 - with: - submodules: true - ref: 1.0.0-preview - - - uses: ./.github/workflows/composite_action/hw - with: - container-name: ${{ github.job }} - build-envs: 'OCCLUM_RELEASE_BUILD=1' - - - name: Run fio download and build - run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /sfs/fio-microbench" - - - name: Copy result - run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json . - - # Run `github-action-benchmark` action - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: FIO Benchmark on AsyncSFS and JinDisk - # What benchmark tool the output.txt came from - tool: 'customBiggerIsBetter' - # Where the output from the benchmark tool is stored - output-file-path: result.json - # Path to directory which contains benchmark files on GitHub pages branch - benchmark-data-dir-path: 'dev/benchmarks' + benchmark-data-dir-path: 'stable/benchmarks' github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # Show alert with commit comment on detecting possible performance regression diff --git a/.github/workflows/benchmarks_dev.yml b/.github/workflows/benchmarks_dev.yml new file mode 100644 index 00000000..e54b94b4 --- /dev/null +++ b/.github/workflows/benchmarks_dev.yml @@ -0,0 +1,220 @@ +name: Benchmarks Test for dev branch +# Currently this workflow is only used for running benchmarks test on schedule on branch 1.0.0-preview + +on: + schedule: + # Schedule to run on Tue,Fri at 10PM UTC (6AM CST) + - cron: '0 22 * * 2,5' + +jobs: + Sysbench_Test: + timeout-minutes: 40 + runs-on: ${{ matrix.self_runner }} + strategy: + matrix: + self_runner: [[self-hosted, SGX2-HW, benchmark]] + + steps: + - name: Clean before running + run: | + sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}" + + - uses: AutoModality/action-clean@v1 + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + ref: 1.0.0-preview + + - uses: ./.github/workflows/composite_action/hw + with: + container-name: ${{ github.job }} + build-envs: 'OCCLUM_RELEASE_BUILD=1' + + - name: Run sysbench download and build + run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/sysbench && ./sysbench.sh 120" + + - name: Copy result + run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/sysbench/result.json . + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Sysbench Benchmark + # What benchmark tool the output.txt came from + tool: 'customSmallerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: result.json + # Path to directory which contains benchmark files on GitHub pages branch + benchmark-data-dir-path: 'dev/benchmarks' + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + # Workflow will fail when an alert happens + fail-on-alert: true + - name: Clean the environment + if: ${{ always() }} + run: docker stop ${{ env.CONTAINER_NAME }} + + Iperf3_Test: + timeout-minutes: 40 + runs-on: ${{ matrix.self_runner }} + strategy: + matrix: + self_runner: [[self-hosted, SGX2-HW, benchmark]] + + steps: + - name: Clean before running + run: | + sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}" + + - uses: AutoModality/action-clean@v1 + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + ref: 1.0.0-preview + + - uses: ./.github/workflows/composite_action/hw + with: + container-name: ${{ github.job }} + build-envs: 'OCCLUM_RELEASE_BUILD=1' + + - name: Run iperf3 download and build + run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/iperf3 && ./iperf3.sh 120" + + - name: Copy result + run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/iperf3/result.json . + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Iperf3 Benchmark + # What benchmark tool the output.txt came from + tool: 'customBiggerIsBetter ' + # Where the output from the benchmark tool is stored + output-file-path: result.json + # Path to directory which contains benchmark files on GitHub pages branch + benchmark-data-dir-path: 'dev/benchmarks' + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + # Workflow will fail when an alert happens + fail-on-alert: true + - name: Clean the environment + if: ${{ always() }} + run: docker stop ${{ env.CONTAINER_NAME }} + + SEFS_FIO_Test: + timeout-minutes: 60 + runs-on: ${{ matrix.self_runner }} + strategy: + matrix: + self_runner: [[self-hosted, SGX2-HW, benchmark]] + + steps: + - name: Clean before running + run: | + sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}" + + - uses: AutoModality/action-clean@v1 + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + ref: 1.0.0-preview + + - uses: ./.github/workflows/composite_action/hw + with: + container-name: ${{ github.job }} + build-envs: 'OCCLUM_RELEASE_BUILD=1' + + - name: Run fio download and build + run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /root/fio-microbench" + + - name: Copy result + run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json . + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: FIO Benchmark on SEFS + # What benchmark tool the output.txt came from + tool: 'customBiggerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: result.json + # Path to directory which contains benchmark files on GitHub pages branch + benchmark-data-dir-path: 'dev/benchmarks' + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + # Workflow will fail when an alert happens + fail-on-alert: true + - name: Clean the environment + if: ${{ always() }} + run: docker stop ${{ env.CONTAINER_NAME }} + + AsyncSFS_FIO_Test: + timeout-minutes: 60 + runs-on: ${{ matrix.self_runner }} + strategy: + matrix: + self_runner: [[self-hosted, SGX2-HW, benchmark]] + + steps: + - name: Clean before running + run: | + sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}" + + - uses: AutoModality/action-clean@v1 + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + ref: 1.0.0-preview + + - uses: ./.github/workflows/composite_action/hw + with: + container-name: ${{ github.job }} + build-envs: 'OCCLUM_RELEASE_BUILD=1' + + - name: Run fio download and build + run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/benchmarks/fio && ./fio_microbench.sh /sfs/fio-microbench" + + - name: Copy result + run: docker cp ${{ env.CONTAINER_NAME }}:/root/occlum/demos/benchmarks/fio/result.json . + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: FIO Benchmark on AsyncSFS and JinDisk + # What benchmark tool the output.txt came from + tool: 'customBiggerIsBetter' + # Where the output from the benchmark tool is stored + output-file-path: result.json + # Path to directory which contains benchmark files on GitHub pages branch + benchmark-data-dir-path: 'dev/benchmarks' + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + # Workflow will fail when an alert happens + fail-on-alert: true + - name: Clean the environment + if: ${{ always() }} + run: docker stop ${{ env.CONTAINER_NAME }} diff --git a/demos/benchmarks/README.md b/demos/benchmarks/README.md index 3e664bf7..025d2db6 100644 --- a/demos/benchmarks/README.md +++ b/demos/benchmarks/README.md @@ -7,3 +7,16 @@ This set of demos shows how commonly used benchmarking tools can be run inside S * [iperf3](iperf3/): A demo of [Iperf3](https://github.com/esnet/iperf), a tool for measuring Internet bandwidth performance. * [sysbench](sysbench/): A demo of [Sysbench](https://github.com/akopytov/sysbench), a scriptable multi-threaded benchmark tool for Linux. +## Benchmarks Data + +There are two enabled **benchmarks CI**(https://github.com/occlum/occlum/blob/1.0.0-preview/.github/workflows/benchmarks.yml) for continuous benchmarking. It utilizes the [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) to provide a chart view for visualized historical benchmarks data on the GitHub pages. + +### CI and Data for Master branch + +[**benchmarks CI**](https://github.com/occlum/occlum/blob/master/.github/workflows/benchmarks.yml) +[**History Data**](https://occlum.io/occlum/stable/benchmarks/) + +### CI and Data for Dev branch + +[**benchmarks CI**](https://github.com/occlum/occlum/blob/master/.github/workflows/benchmarks_dev.yml) +[**History Data**](https://occlum.io/occlum/dev/benchmarks/)