Remove rpm repo configuration and update deb repo for ubuntu 20.04

This commit is contained in:
Hui, Chunyang 2022-03-08 06:45:19 +00:00 committed by Zongmin.Gu
parent d0f6c9b6b6
commit aba1c7de15
2 changed files with 180 additions and 149 deletions

@ -4,6 +4,10 @@ name: Set up Package Repository and Test (Manual Trigger)
on:
workflow_dispatch:
inputs:
update_musl:
description: 'Need build new musl pacakge? <Y/N>'
required: true
default: 'N'
only_test:
description: 'Only Test? <Y/N>'
required: true
@ -14,7 +18,7 @@ on:
jobs:
Package_repository_setup_and_test:
runs-on: ubuntu-18.04
runs-on: ubuntu-20.04
if: github.event.inputs.only_test == 'N'
env:
TOKEN: ${{ secrets.PAT_TOKEN }}
@ -54,44 +58,67 @@ jobs:
run: echo "::set-output name=version::${{ env.OCCLUM_VERSION }}"
- name: Create ubuntu container
run: docker run -itd --name=ubuntu -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04
run: docker run -itd --name=ubuntu -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu20.04
- name: Build deb packages
run: docker exec ubuntu bash -c "cd /root/workspace/occlum/tools/installer/deb; make; make musl-gcc"
run: docker exec ubuntu bash -c "cd /root/workspace/occlum/tools/installer/deb; make"
- name: Update deb repo
- name: Build musl toolchain package
if: github.event.inputs.update_musl == 'Y'
run: docker exec ubuntu bash -c "cd /root/workspace/occlum/tools/installer/deb; make musl-gcc"
- name: Prepare tools and keys # Since aptly still use gpg1 by default, we all use gpg1 as gpg tool.
run: docker exec ubuntu bash -c 'apt-get update; apt-get install -y tree apt-utils gnupg reprepro rng-tools aptly; rm -rf /root/.gnupg;
echo "${{ secrets.DEB_PRIVATE_KEY }}" > /root/deb_private_key; gpg --allow-secret-key-import --import /root/deb_private_key;
gpg --list-keys;
gpg --export -a "deb_gpg_key" > /root/public.key; aptly -distribution='bionic' -architectures=amd64 repo create occlum-deb-repo;
aptly repo add occlum-deb-repo /root/workspace/occlum/build/debs/*;
find /root/workspace/occlum-package-repos/debian -name "*.deb" | xargs -I files aptly repo add occlum-deb-repo files || true;
aptly publish repo occlum-deb-repo; tree /root/.aptly/public;
cd /root/workspace/occlum-package-repos; rm -rf debian; cp -r /root/.aptly/public /root/workspace/occlum-package-repos/debian;
echo "${{ secrets.DEB_PRIVATE_KEY }}" > /root/deb_private_key; gpg1 --allow-secret-key-import --import /root/deb_private_key;
gpg1 --export -a "deb_gpg_key" > /root/public.key;
gpg1 --import /root/public.key;
gpg1 --list-keys;
apt-key add /root/public.key; apt-key list'
- name: Inherit apt repo for Ubuntu 18.04
run: docker exec ubuntu bash -c 'aptly -architectures="amd64" -keyring=/etc/apt/trusted.gpg mirror create bionic-mirror https://occlum.io/occlum-package-repos/debian/ bionic main;
aptly -keyring=/etc/apt/trusted.gpg mirror update bionic-mirror;
aptly snapshot create bionic-main from mirror bionic-mirror;
aptly publish snapshot -distribution=bionic bionic-main'
- name: Update apt repo for Ubuntu 20.04
run: docker exec ubuntu bash -c 'aptly -architectures="amd64" -keyring=/etc/apt/trusted.gpg mirror create focal-mirror https://occlum.io/occlum-package-repos/debian/ focal main;
aptly -keyring=/etc/apt/trusted.gpg mirror update focal-mirror;
aptly snapshot create focal-old from mirror focal-mirror;
aptly -distribution='focal' -architectures=amd64 repo create deb-focal-new;
aptly repo add deb-focal-new /root/workspace/occlum/build/debs/*;
aptly snapshot create focal-new from repo deb-focal-new;
aptly publish snapshot -distribution=focal focal-new;
aptly snapshot merge focal-main focal-new focal-old;
aptly publish snapshot -distribution=focal focal-main;
tree /root/.aptly/public'
- name: Update Git repo
run: docker exec ubuntu bash -c 'cd /root/workspace/occlum-package-repos; rm -rf debian; cp -r /root/.aptly/public/ /root/workspace/occlum-package-repos/debian;
cp /root/public.key /root/workspace/occlum-package-repos/debian'
- name: Clean ubuntu container and image
run: |
docker rm -f ubuntu
docker rmi -f occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04
docker rmi -f occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu20.04
- name: Create centos container
run: docker run -itd --name=centos -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-centos8.2
# - name: Create centos container
# run: docker run -itd --name=centos -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-centos8.2
- name: Build rpm packages
run: docker exec centos bash -c "cd /root/workspace/occlum/tools/installer/rpm; make; make musl-gcc"
# - name: Build rpm packages
# run: docker exec centos bash -c "cd /root/workspace/occlum/tools/installer/rpm; make; make musl-gcc"
- name: Configure the centos container
run: docker exec centos bash -c 'yum install -y gnupg pinentry createrepo rpm-sign ca-certificates; rm -rf /root/.gnupg;
echo "${{ secrets.RPM_PRIVATE_KEY }}" > /root/rpm_private_key; gpg --allow-secret-key-import --import /root/rpm_private_key;
echo "${{ secrets.CA_PRIVATE_KEY }}" > /root/ca_private_key; gpg --allow-secret-key-import --import /root/ca_private_key;
gpg --list-keys;
echo -e "%__gpg /usr/bin/gpg\n%_source_filedigest_algorithm 8\n%_binary_filedigest_algorithm 8\n%_gpg_digest_algo SHA256\n%_gpg_path /root/.gnupg\n%_gpg_name rpm_gpg_key" > /root/.rpmmacros'
# - name: Configure the centos container
# run: docker exec centos bash -c 'yum install -y gnupg pinentry createrepo rpm-sign ca-certificates; rm -rf /root/.gnupg;
# echo "${{ secrets.RPM_PRIVATE_KEY }}" > /root/rpm_private_key; gpg --allow-secret-key-import --import /root/rpm_private_key;
# echo "${{ secrets.CA_PRIVATE_KEY }}" > /root/ca_private_key; gpg --allow-secret-key-import --import /root/ca_private_key;
# gpg --list-keys;
# echo -e "%__gpg /usr/bin/gpg\n%_source_filedigest_algorithm 8\n%_binary_filedigest_algorithm 8\n%_gpg_digest_algo SHA256\n%_gpg_path /root/.gnupg\n%_gpg_name rpm_gpg_key" > /root/.rpmmacros'
- name: Sign the package and update the rpm repo
run: docker exec centos bash -c "cp /root/workspace/occlum/build/rpms/* /root/workspace/occlum-package-repos/rpm-repo; cd /root/workspace/occlum-package-repos/rpm-repo; rm -rf repodata RPM-GPG-KEY-*;
rpmsign --resign *.rpm; createrepo --update -d -p -o . .; gpg -u rpm_gpg_key --detach-sign -a repodata/repomd.xml; gpg --export -a "rpm_gpg_key" > RPM-GPG-KEY-rpm-sign;
gpg --export -a "ca_gpg_key" > RPM-GPG-KEY-rpm-sign-ca; gpg --detach-sign -a -u ca_gpg_key RPM-GPG-KEY-rpm-sign"
# - name: Sign the package and update the rpm repo
# run: docker exec centos bash -c "cp /root/workspace/occlum/build/rpms/* /root/workspace/occlum-package-repos/rpm-repo; cd /root/workspace/occlum-package-repos/rpm-repo; rm -rf repodata RPM-GPG-KEY-*;
# rpmsign --resign *.rpm; createrepo --update -d -p -o . .; gpg -u rpm_gpg_key --detach-sign -a repodata/repomd.xml; gpg --export -a "rpm_gpg_key" > RPM-GPG-KEY-rpm-sign;
# gpg --export -a "ca_gpg_key" > RPM-GPG-KEY-rpm-sign-ca; gpg --detach-sign -a -u ca_gpg_key RPM-GPG-KEY-rpm-sign"
- name: Commit files
run: |
@ -117,11 +144,11 @@ jobs:
Test_deb_package:
if: ${{ always() }}
needs: Package_repository_setup_and_test
runs-on: ubuntu-18.04
runs-on: ubuntu-20.04
steps:
- name: Create a clean ubuntu container
run: docker run -itd --name=ubuntu ubuntu:18.04
run: docker run -itd --name=ubuntu ubuntu:20.04
- name: Get occlum version from user inputs
run: echo "OCCLUM_VERSION=${{github.event.inputs.test_version}}" >> $GITHUB_ENV
@ -135,9 +162,9 @@ jobs:
# Set the default timezone to make tzdata work
docker exec ubuntu bash -c "ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo Asia/Shanghai > /etc/timezone"
docker exec ubuntu bash -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates gnupg2 jq make gdb wget libfuse-dev libtool tzdata;
echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | tee /etc/apt/sources.list.d/intel-sgx.list;
echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list;
wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add -"
docker exec ubuntu bash -c "echo 'deb [arch=amd64] https://occlum.io/occlum-package-repos/debian bionic main' | tee /etc/apt/sources.list.d/occlum.list;
docker exec ubuntu bash -c "echo 'deb [arch=amd64] https://occlum.io/occlum-package-repos/debian focal main' | tee /etc/apt/sources.list.d/occlum.list;
wget -qO - https://occlum.io/occlum-package-repos/debian/public.key | apt-key add -;"
- name: Install sgx dependencies and occlum
@ -152,45 +179,45 @@ jobs:
run: docker exec ubuntu bash -c "cat /opt/occlum/include/occlum_version.h | grep -n5 ${{ env.OCCLUM_VERSION }}"
Test_rpm_package:
if: ${{ always() }}
needs: Package_repository_setup_and_test
runs-on: ubuntu-18.04
# Test_rpm_package:
# if: ${{ always() }}
# needs: Package_repository_setup_and_test
# runs-on: ubuntu-18.04
steps:
- name: Create clean centos container
run: docker run -itd --name=centos centos:centos8.2.2004
# steps:
# - name: Create clean centos container
# run: docker run -itd --name=centos centos:centos8.2.2004
- name: Get occlum version from user inputs
run: echo "OCCLUM_VERSION=${{github.event.inputs.test_version}}" >> $GITHUB_ENV
# - name: Get occlum version from user inputs
# run: echo "OCCLUM_VERSION=${{github.event.inputs.test_version}}" >> $GITHUB_ENV
- name: Update occlum version from previous job
if: github.event.inputs.only_test == 'N'
run: echo "OCCLUM_VERSION=${{needs.Package_repository_setup_and_test.outputs.occlum_version}}" >> $GITHUB_ENV
# - name: Update occlum version from previous job
# if: github.event.inputs.only_test == 'N'
# run: echo "OCCLUM_VERSION=${{needs.Package_repository_setup_and_test.outputs.occlum_version}}" >> $GITHUB_ENV
- name: Install sgx dependencies
run: docker exec centos bash -c "yum install -y wget yum-utils make jq gdb; cd /root && wget https://download.01.org/intel-sgx/sgx-linux/2.14/distro/centos8.2-server/sgx_rpm_local_repo.tgz;
tar -xvzf sgx_rpm_local_repo.tgz; yum-config-manager --add-repo file:///root/sgx_rpm_local_repo; yum --nogpgcheck install -y libsgx-dcap-ql libsgx-epid libsgx-urts;
yum --nogpgcheck install -y libsgx-quote-ex; rpm -i /root/sgx_rpm_local_repo/libsgx-uae-service-*.rpm || true"
# - name: Install sgx dependencies
# run: docker exec centos bash -c "yum install -y wget yum-utils make jq gdb; cd /root && wget https://download.01.org/intel-sgx/sgx-linux/2.14/distro/centos8.2-server/sgx_rpm_local_repo.tgz;
# tar -xvzf sgx_rpm_local_repo.tgz; yum-config-manager --add-repo file:///root/sgx_rpm_local_repo; yum --nogpgcheck install -y libsgx-dcap-ql libsgx-epid libsgx-urts;
# yum --nogpgcheck install -y libsgx-quote-ex; rpm -i /root/sgx_rpm_local_repo/libsgx-uae-service-*.rpm || true"
- name: Install occlum
run: |
docker exec centos bash -c "cat << EOF > /etc/yum.repos.d/occlum.repo
[occlum]
name=occlum
enabled=1
baseurl=https://occlum.io/occlum-package-repos/rpm-repo/
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign
gpgcakey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign-ca
EOF"
docker exec centos bash -c "yum --showduplicate list -y occlum | grep -n5 ${{ env.OCCLUM_VERSION }} && yum install -y occlum"
# - name: Install occlum
# run: |
# docker exec centos bash -c "cat << EOF > /etc/yum.repos.d/occlum.repo
# [occlum]
# name=occlum
# enabled=1
# baseurl=https://occlum.io/occlum-package-repos/rpm-repo/
# gpgcheck=1
# repo_gpgcheck=1
# gpgkey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign
# gpgcakey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign-ca
# EOF"
# docker exec centos bash -c "yum --showduplicate list -y occlum | grep -n5 ${{ env.OCCLUM_VERSION }} && yum install -y occlum"
- name: Hello world test
run: docker exec centos bash -c "source /etc/profile; cd /root; rm -rf hello_world*; wget https://raw.githubusercontent.com/occlum/occlum/master/demos/hello_c/hello_world.c; occlum-gcc -o hello_world hello_world.c;
occlum new occlum-instance; cp hello_world /root/occlum-instance/image/bin; cd /root/occlum-instance && SGX_MODE=SIM occlum build; occlum run /bin/hello_world"
# - name: Hello world test
# run: docker exec centos bash -c "source /etc/profile; cd /root; rm -rf hello_world*; wget https://raw.githubusercontent.com/occlum/occlum/master/demos/hello_c/hello_world.c; occlum-gcc -o hello_world hello_world.c;
# occlum new occlum-instance; cp hello_world /root/occlum-instance/image/bin; cd /root/occlum-instance && SGX_MODE=SIM occlum build; occlum run /bin/hello_world"
# If there is no match, it will return 1.
- name: Check installed version
run: docker exec centos bash -c "cat /opt/occlum/include/occlum_version.h | grep -n5 ${{ env.OCCLUM_VERSION }}"
# # If there is no match, it will return 1.
# - name: Check installed version
# run: docker exec centos bash -c "cat /opt/occlum/include/occlum_version.h | grep -n5 ${{ env.OCCLUM_VERSION }}"

@ -8,8 +8,95 @@ To evaluate Occlum in non-docker environment, installers are needed. Occlum prov
- **occlum-toolchains-\<language\>**: toolchain components for specific language
- **occlum**: complete package to support all Occlum commands. `occlum-toolchains-gcc` is also installed by default. Please install packages of other programming languages based on your need.
## DEB Installer
### How to Build
Normally, Occlum installers should be provided together with release. However, users can also build them on their own.
To build deb packages, a docker container with Occlum Ubuntu image (based on Ubuntu 20.04) is needed. Execute below commands under the occlum directory:
```
cd tools/installer/deb
make
```
and occlum deb installer can be found under `build/debs`.
If a user wants to build his application on a platform installed with Occlum installer, toolchain installers are also needed. To build language specific toolchain installer, just run the command:
```
cd tools/installer/deb
make <language option>
```
Now, only `musl-gcc` and `golang` options are supported. And the installer can be found under `build/debs`.
### How to Use
DEB installer should be found together with Occlum release package at [this page](https://github.com/occlum/occlum/releases).
To run Occlum on clean Ubuntu 18.04, please follow below steps:
**Step 1. Install Prerequisites**
```
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev libprotobuf-dev libfuse-dev autoconf automake make cmake libtool gdb python jq ca-certificates gnupg wget vim
```
**Step 2. Install Intel® SGX driver and Intel® SGX PSW**
Please follow [Intel SGX Installation Guide](https://download.01.org/intel-sgx/sgx-linux/2.15.1/docs/Intel_SGX_SW_Installation_Guide_for_Linux.pdf) to install SGX driver and SGX PSW. SGX SDK is not required. Using PSW installer is recommanded.
To install PSW, follow the guide to add Intel® SGX repository to apt source. And then run:
```
apt-get update
apt-get install -y libsgx-epid libsgx-urts libsgx-quote-ex libsgx-uae-service libsgx-dcap-ql
```
After installing PSW, please make sure that the aesm service is running:
```
service aesmd status
```
**Step 3. Install enable_RDFSBASE Kernel Module**
Please follow [this README](https://github.com/occlum/enable_rdfsbase/blob/master/README.md) to install `enable_rdfsbase` kernel module.
**Step 4. Install Occlum Installer and Toolchains Installer**
```
cd <path to installer>
apt install -y ./occlum-runtime*.deb
apt install -y ./occlum-pal*.deb
apt install -y ./occlum-sgx-tools*.deb
```
Toolchains are needed when compiling applications and also during runtime. C/C++ toolchain is a must for Occlum commands.
To install C/C++ toolchain, just run the command:
```
apt install -y ./occlum-toolchains-gcc*.deb
```
Besides, users can choose to install the toolchain installer based on the application's language. Currently, we also supports Golang. More language toolchain installers are on the way. To install Golang toolchain, run the below commands:
```
apt install -y ./occlum-toolchains-golang*.deb
```
At last, install `occlum` package to get complete support of Occlum:
```
apt install -y ./occlum_*.deb
```
To make the new installed binaries and libraries work, this command must be executed:
```
source /etc/profile
```
**Step 5. Install Debug Symbol Packages (OPTIONAL)**
If users want to debug the application running inside the libos, debug symbol packages are also needed. Just run:
```
apt install -y ./occlum-dbgsym*.ddeb ./occlum-pal-dbgsym*.ddeb ./occlum-runtime-dbgsym*.ddeb ./occlum-toolchains-gcc-dbgsym*.ddeb ./occlum-sgx-tools-dbgsym*.ddeb
```
## RPM Installer
TBD
<!---
### How to Build
Normally, Occlum installers should be provided together with release. However, users can also build them on their own.
@ -87,87 +174,4 @@ If users want to debug the application running inside the libos, debug packages
```
rpm -i occlum-debuginfo*.rpm occlum-debugsource*.rpm occlum-pal-debuginfo*.rpm occlum-runtime-debuginfo*.rpm occlum-sgx-tools-debuginfo*.rpm occlum-toolchains-gcc-debuginfo*.rpm occlum-toolchains-gcc-debugsource*.rpm
```
## DEB Installer
### How to Build
Normally, Occlum installers should be provided together with release. However, users can also build them on their own.
To build deb packages, a docker container with Occlum Ubuntu image (based on Ubuntu 18.04) is needed. Execute below commands under the occlum directory:
```
cd tools/installer/deb
make
```
and occlum deb installer can be found under `build/debs`.
If a user wants to build his application on a platform installed with Occlum installer, toolchain installers are also needed. To build language specific toolchain installer, just run the command:
```
cd tools/installer/deb
make <language option>
```
Now, only `musl-gcc` and `golang` options are supported. And the installer can be found under `build/debs`.
### How to Use
DEB installer should be found together with Occlum release package at [this page](https://github.com/occlum/occlum/releases).
To run Occlum on clean Ubuntu 18.04, please follow below steps:
**Step 1. Install Prerequisites**
```
apt-get update
apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev libprotobuf-dev libfuse-dev autoconf automake make cmake libtool gdb python jq ca-certificates gnupg wget vim
```
**Step 2. Install Intel® SGX driver and Intel® SGX PSW**
Please follow [Intel SGX Installation Guide](https://download.01.org/intel-sgx/sgx-linux/2.13/docs/Intel_SGX_Installation_Guide_Linux_2.13_Open_Source.pdf) to install SGX driver and SGX PSW. SGX SDK is not required. Using PSW installer is recommanded.
To install PSW, follow the guide to add Intel® SGX repository to apt source. And then run:
```
apt-get update
apt-get install -y libsgx-epid libsgx-urts libsgx-quote-ex libsgx-uae-service
```
After installing PSW, please make sure that the aesm service is running:
```
service aesmd status
```
**Step 3. Install enable_RDFSBASE Kernel Module**
Please follow [this README](https://github.com/occlum/enable_rdfsbase/blob/master/README.md) to install `enable_rdfsbase` kernel module.
**Step 4. Install Occlum Installer and Toolchains Installer**
```
cd <path to installer>
apt install -y ./occlum-runtime*.deb
apt install -y ./occlum-pal*.deb
apt install -y ./occlum-sgx-tools*.deb
```
Toolchains are needed when compiling applications and also during runtime. C/C++ toolchain is a must for Occlum commands.
To install C/C++ toolchain, just run the command:
```
apt install -y ./occlum-toolchains-gcc*.deb
```
Besides, users can choose to install the toolchain installer based on the application's language. Currently, we also supports Golang. More language toolchain installers are on the way. To install Golang toolchain, run the below commands:
```
apt install -y ./occlum-toolchains-golang*.deb
```
At last, install `occlum` package to get complete support of Occlum:
```
apt install -y ./occlum_*.deb
```
To make the new installed binaries and libraries work, this command must be executed:
```
source /etc/profile
```
**Step 5. Install Debug Symbol Packages (OPTIONAL)**
If users want to debug the application running inside the libos, debug symbol packages are also needed. Just run:
```
apt install -y ./occlum-dbgsym*.ddeb ./occlum-pal-dbgsym*.ddeb ./occlum-runtime-dbgsym*.ddeb ./occlum-toolchains-gcc-dbgsym*.ddeb ./occlum-sgx-tools-dbgsym*.ddeb
```
-->