From aba1c7de15639e8cd8f05bceb10292e45dab7562 Mon Sep 17 00:00:00 2001 From: "Hui, Chunyang" Date: Tue, 8 Mar 2022 06:45:19 +0000 Subject: [PATCH] Remove rpm repo configuration and update deb repo for ubuntu 20.04 --- .../workflows/package_repo_setup_and_test.yml | 157 +++++++++------- tools/installer/README.md | 172 +++++++++--------- 2 files changed, 180 insertions(+), 149 deletions(-) diff --git a/.github/workflows/package_repo_setup_and_test.yml b/.github/workflows/package_repo_setup_and_test.yml index cbb0f1bc..7fbcb1f1 100644 --- a/.github/workflows/package_repo_setup_and_test.yml +++ b/.github/workflows/package_repo_setup_and_test.yml @@ -4,6 +4,10 @@ name: Set up Package Repository and Test (Manual Trigger) on: workflow_dispatch: inputs: + update_musl: + description: 'Need build new musl pacakge? ' + required: true + default: 'N' only_test: description: 'Only Test? ' required: true @@ -14,7 +18,7 @@ on: jobs: Package_repository_setup_and_test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 if: github.event.inputs.only_test == 'N' env: TOKEN: ${{ secrets.PAT_TOKEN }} @@ -54,44 +58,67 @@ jobs: run: echo "::set-output name=version::${{ env.OCCLUM_VERSION }}" - name: Create ubuntu container - run: docker run -itd --name=ubuntu -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04 + run: docker run -itd --name=ubuntu -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu20.04 - name: Build deb packages - run: docker exec ubuntu bash -c "cd /root/workspace/occlum/tools/installer/deb; make; make musl-gcc" + run: docker exec ubuntu bash -c "cd /root/workspace/occlum/tools/installer/deb; make" - - name: Update deb repo + - name: Build musl toolchain package + if: github.event.inputs.update_musl == 'Y' + run: docker exec ubuntu bash -c "cd /root/workspace/occlum/tools/installer/deb; make musl-gcc" + + - name: Prepare tools and keys # Since aptly still use gpg1 by default, we all use gpg1 as gpg tool. run: docker exec ubuntu bash -c 'apt-get update; apt-get install -y tree apt-utils gnupg reprepro rng-tools aptly; rm -rf /root/.gnupg; - echo "${{ secrets.DEB_PRIVATE_KEY }}" > /root/deb_private_key; gpg --allow-secret-key-import --import /root/deb_private_key; - gpg --list-keys; - gpg --export -a "deb_gpg_key" > /root/public.key; aptly -distribution='bionic' -architectures=amd64 repo create occlum-deb-repo; - aptly repo add occlum-deb-repo /root/workspace/occlum/build/debs/*; - find /root/workspace/occlum-package-repos/debian -name "*.deb" | xargs -I files aptly repo add occlum-deb-repo files || true; - aptly publish repo occlum-deb-repo; tree /root/.aptly/public; - cd /root/workspace/occlum-package-repos; rm -rf debian; cp -r /root/.aptly/public /root/workspace/occlum-package-repos/debian; + echo "${{ secrets.DEB_PRIVATE_KEY }}" > /root/deb_private_key; gpg1 --allow-secret-key-import --import /root/deb_private_key; + gpg1 --export -a "deb_gpg_key" > /root/public.key; + gpg1 --import /root/public.key; + gpg1 --list-keys; + apt-key add /root/public.key; apt-key list' + + - name: Inherit apt repo for Ubuntu 18.04 + run: docker exec ubuntu bash -c 'aptly -architectures="amd64" -keyring=/etc/apt/trusted.gpg mirror create bionic-mirror https://occlum.io/occlum-package-repos/debian/ bionic main; + aptly -keyring=/etc/apt/trusted.gpg mirror update bionic-mirror; + aptly snapshot create bionic-main from mirror bionic-mirror; + aptly publish snapshot -distribution=bionic bionic-main' + + - name: Update apt repo for Ubuntu 20.04 + run: docker exec ubuntu bash -c 'aptly -architectures="amd64" -keyring=/etc/apt/trusted.gpg mirror create focal-mirror https://occlum.io/occlum-package-repos/debian/ focal main; + aptly -keyring=/etc/apt/trusted.gpg mirror update focal-mirror; + aptly snapshot create focal-old from mirror focal-mirror; + aptly -distribution='focal' -architectures=amd64 repo create deb-focal-new; + aptly repo add deb-focal-new /root/workspace/occlum/build/debs/*; + aptly snapshot create focal-new from repo deb-focal-new; + aptly publish snapshot -distribution=focal focal-new; + aptly snapshot merge focal-main focal-new focal-old; + aptly publish snapshot -distribution=focal focal-main; + tree /root/.aptly/public' + + - name: Update Git repo + run: docker exec ubuntu bash -c 'cd /root/workspace/occlum-package-repos; rm -rf debian; cp -r /root/.aptly/public/ /root/workspace/occlum-package-repos/debian; cp /root/public.key /root/workspace/occlum-package-repos/debian' - name: Clean ubuntu container and image run: | docker rm -f ubuntu - docker rmi -f occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04 + docker rmi -f occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu20.04 - - name: Create centos container - run: docker run -itd --name=centos -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-centos8.2 + # - name: Create centos container + # run: docker run -itd --name=centos -v $GITHUB_WORKSPACE:/root/workspace occlum/occlum:${{ env.OCCLUM_VERSION }}-centos8.2 - - name: Build rpm packages - run: docker exec centos bash -c "cd /root/workspace/occlum/tools/installer/rpm; make; make musl-gcc" + # - name: Build rpm packages + # run: docker exec centos bash -c "cd /root/workspace/occlum/tools/installer/rpm; make; make musl-gcc" - - name: Configure the centos container - run: docker exec centos bash -c 'yum install -y gnupg pinentry createrepo rpm-sign ca-certificates; rm -rf /root/.gnupg; - echo "${{ secrets.RPM_PRIVATE_KEY }}" > /root/rpm_private_key; gpg --allow-secret-key-import --import /root/rpm_private_key; - echo "${{ secrets.CA_PRIVATE_KEY }}" > /root/ca_private_key; gpg --allow-secret-key-import --import /root/ca_private_key; - gpg --list-keys; - echo -e "%__gpg /usr/bin/gpg\n%_source_filedigest_algorithm 8\n%_binary_filedigest_algorithm 8\n%_gpg_digest_algo SHA256\n%_gpg_path /root/.gnupg\n%_gpg_name rpm_gpg_key" > /root/.rpmmacros' + # - name: Configure the centos container + # run: docker exec centos bash -c 'yum install -y gnupg pinentry createrepo rpm-sign ca-certificates; rm -rf /root/.gnupg; + # echo "${{ secrets.RPM_PRIVATE_KEY }}" > /root/rpm_private_key; gpg --allow-secret-key-import --import /root/rpm_private_key; + # echo "${{ secrets.CA_PRIVATE_KEY }}" > /root/ca_private_key; gpg --allow-secret-key-import --import /root/ca_private_key; + # gpg --list-keys; + # echo -e "%__gpg /usr/bin/gpg\n%_source_filedigest_algorithm 8\n%_binary_filedigest_algorithm 8\n%_gpg_digest_algo SHA256\n%_gpg_path /root/.gnupg\n%_gpg_name rpm_gpg_key" > /root/.rpmmacros' - - name: Sign the package and update the rpm repo - run: docker exec centos bash -c "cp /root/workspace/occlum/build/rpms/* /root/workspace/occlum-package-repos/rpm-repo; cd /root/workspace/occlum-package-repos/rpm-repo; rm -rf repodata RPM-GPG-KEY-*; - rpmsign --resign *.rpm; createrepo --update -d -p -o . .; gpg -u rpm_gpg_key --detach-sign -a repodata/repomd.xml; gpg --export -a "rpm_gpg_key" > RPM-GPG-KEY-rpm-sign; - gpg --export -a "ca_gpg_key" > RPM-GPG-KEY-rpm-sign-ca; gpg --detach-sign -a -u ca_gpg_key RPM-GPG-KEY-rpm-sign" + # - name: Sign the package and update the rpm repo + # run: docker exec centos bash -c "cp /root/workspace/occlum/build/rpms/* /root/workspace/occlum-package-repos/rpm-repo; cd /root/workspace/occlum-package-repos/rpm-repo; rm -rf repodata RPM-GPG-KEY-*; + # rpmsign --resign *.rpm; createrepo --update -d -p -o . .; gpg -u rpm_gpg_key --detach-sign -a repodata/repomd.xml; gpg --export -a "rpm_gpg_key" > RPM-GPG-KEY-rpm-sign; + # gpg --export -a "ca_gpg_key" > RPM-GPG-KEY-rpm-sign-ca; gpg --detach-sign -a -u ca_gpg_key RPM-GPG-KEY-rpm-sign" - name: Commit files run: | @@ -117,11 +144,11 @@ jobs: Test_deb_package: if: ${{ always() }} needs: Package_repository_setup_and_test - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Create a clean ubuntu container - run: docker run -itd --name=ubuntu ubuntu:18.04 + run: docker run -itd --name=ubuntu ubuntu:20.04 - name: Get occlum version from user inputs run: echo "OCCLUM_VERSION=${{github.event.inputs.test_version}}" >> $GITHUB_ENV @@ -135,9 +162,9 @@ jobs: # Set the default timezone to make tzdata work docker exec ubuntu bash -c "ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo Asia/Shanghai > /etc/timezone" docker exec ubuntu bash -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates gnupg2 jq make gdb wget libfuse-dev libtool tzdata; - echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | tee /etc/apt/sources.list.d/intel-sgx.list; + echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list; wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add -" - docker exec ubuntu bash -c "echo 'deb [arch=amd64] https://occlum.io/occlum-package-repos/debian bionic main' | tee /etc/apt/sources.list.d/occlum.list; + docker exec ubuntu bash -c "echo 'deb [arch=amd64] https://occlum.io/occlum-package-repos/debian focal main' | tee /etc/apt/sources.list.d/occlum.list; wget -qO - https://occlum.io/occlum-package-repos/debian/public.key | apt-key add -;" - name: Install sgx dependencies and occlum @@ -152,45 +179,45 @@ jobs: run: docker exec ubuntu bash -c "cat /opt/occlum/include/occlum_version.h | grep -n5 ${{ env.OCCLUM_VERSION }}" - Test_rpm_package: - if: ${{ always() }} - needs: Package_repository_setup_and_test - runs-on: ubuntu-18.04 + # Test_rpm_package: + # if: ${{ always() }} + # needs: Package_repository_setup_and_test + # runs-on: ubuntu-18.04 - steps: - - name: Create clean centos container - run: docker run -itd --name=centos centos:centos8.2.2004 + # steps: + # - name: Create clean centos container + # run: docker run -itd --name=centos centos:centos8.2.2004 - - name: Get occlum version from user inputs - run: echo "OCCLUM_VERSION=${{github.event.inputs.test_version}}" >> $GITHUB_ENV + # - name: Get occlum version from user inputs + # run: echo "OCCLUM_VERSION=${{github.event.inputs.test_version}}" >> $GITHUB_ENV - - name: Update occlum version from previous job - if: github.event.inputs.only_test == 'N' - run: echo "OCCLUM_VERSION=${{needs.Package_repository_setup_and_test.outputs.occlum_version}}" >> $GITHUB_ENV + # - name: Update occlum version from previous job + # if: github.event.inputs.only_test == 'N' + # run: echo "OCCLUM_VERSION=${{needs.Package_repository_setup_and_test.outputs.occlum_version}}" >> $GITHUB_ENV - - name: Install sgx dependencies - run: docker exec centos bash -c "yum install -y wget yum-utils make jq gdb; cd /root && wget https://download.01.org/intel-sgx/sgx-linux/2.14/distro/centos8.2-server/sgx_rpm_local_repo.tgz; - tar -xvzf sgx_rpm_local_repo.tgz; yum-config-manager --add-repo file:///root/sgx_rpm_local_repo; yum --nogpgcheck install -y libsgx-dcap-ql libsgx-epid libsgx-urts; - yum --nogpgcheck install -y libsgx-quote-ex; rpm -i /root/sgx_rpm_local_repo/libsgx-uae-service-*.rpm || true" + # - name: Install sgx dependencies + # run: docker exec centos bash -c "yum install -y wget yum-utils make jq gdb; cd /root && wget https://download.01.org/intel-sgx/sgx-linux/2.14/distro/centos8.2-server/sgx_rpm_local_repo.tgz; + # tar -xvzf sgx_rpm_local_repo.tgz; yum-config-manager --add-repo file:///root/sgx_rpm_local_repo; yum --nogpgcheck install -y libsgx-dcap-ql libsgx-epid libsgx-urts; + # yum --nogpgcheck install -y libsgx-quote-ex; rpm -i /root/sgx_rpm_local_repo/libsgx-uae-service-*.rpm || true" - - name: Install occlum - run: | - docker exec centos bash -c "cat << EOF > /etc/yum.repos.d/occlum.repo - [occlum] - name=occlum - enabled=1 - baseurl=https://occlum.io/occlum-package-repos/rpm-repo/ - gpgcheck=1 - repo_gpgcheck=1 - gpgkey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign - gpgcakey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign-ca - EOF" - docker exec centos bash -c "yum --showduplicate list -y occlum | grep -n5 ${{ env.OCCLUM_VERSION }} && yum install -y occlum" + # - name: Install occlum + # run: | + # docker exec centos bash -c "cat << EOF > /etc/yum.repos.d/occlum.repo + # [occlum] + # name=occlum + # enabled=1 + # baseurl=https://occlum.io/occlum-package-repos/rpm-repo/ + # gpgcheck=1 + # repo_gpgcheck=1 + # gpgkey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign + # gpgcakey=https://occlum.io/occlum-package-repos/rpm-repo/RPM-GPG-KEY-rpm-sign-ca + # EOF" + # docker exec centos bash -c "yum --showduplicate list -y occlum | grep -n5 ${{ env.OCCLUM_VERSION }} && yum install -y occlum" - - name: Hello world test - run: docker exec centos bash -c "source /etc/profile; cd /root; rm -rf hello_world*; wget https://raw.githubusercontent.com/occlum/occlum/master/demos/hello_c/hello_world.c; occlum-gcc -o hello_world hello_world.c; - occlum new occlum-instance; cp hello_world /root/occlum-instance/image/bin; cd /root/occlum-instance && SGX_MODE=SIM occlum build; occlum run /bin/hello_world" + # - name: Hello world test + # run: docker exec centos bash -c "source /etc/profile; cd /root; rm -rf hello_world*; wget https://raw.githubusercontent.com/occlum/occlum/master/demos/hello_c/hello_world.c; occlum-gcc -o hello_world hello_world.c; + # occlum new occlum-instance; cp hello_world /root/occlum-instance/image/bin; cd /root/occlum-instance && SGX_MODE=SIM occlum build; occlum run /bin/hello_world" - # If there is no match, it will return 1. - - name: Check installed version - run: docker exec centos bash -c "cat /opt/occlum/include/occlum_version.h | grep -n5 ${{ env.OCCLUM_VERSION }}" + # # If there is no match, it will return 1. + # - name: Check installed version + # run: docker exec centos bash -c "cat /opt/occlum/include/occlum_version.h | grep -n5 ${{ env.OCCLUM_VERSION }}" diff --git a/tools/installer/README.md b/tools/installer/README.md index 9b8fc3b2..ae84fdee 100644 --- a/tools/installer/README.md +++ b/tools/installer/README.md @@ -8,8 +8,95 @@ To evaluate Occlum in non-docker environment, installers are needed. Occlum prov - **occlum-toolchains-\**: toolchain components for specific language - **occlum**: complete package to support all Occlum commands. `occlum-toolchains-gcc` is also installed by default. Please install packages of other programming languages based on your need. +## DEB Installer + +### How to Build + +Normally, Occlum installers should be provided together with release. However, users can also build them on their own. + +To build deb packages, a docker container with Occlum Ubuntu image (based on Ubuntu 20.04) is needed. Execute below commands under the occlum directory: +``` +cd tools/installer/deb +make +``` +and occlum deb installer can be found under `build/debs`. + +If a user wants to build his application on a platform installed with Occlum installer, toolchain installers are also needed. To build language specific toolchain installer, just run the command: +``` +cd tools/installer/deb +make +``` +Now, only `musl-gcc` and `golang` options are supported. And the installer can be found under `build/debs`. + +### How to Use + +DEB installer should be found together with Occlum release package at [this page](https://github.com/occlum/occlum/releases). +To run Occlum on clean Ubuntu 18.04, please follow below steps: + +**Step 1. Install Prerequisites** +``` +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev libprotobuf-dev libfuse-dev autoconf automake make cmake libtool gdb python jq ca-certificates gnupg wget vim +``` + +**Step 2. Install Intel® SGX driver and Intel® SGX PSW** +Please follow [Intel SGX Installation Guide](https://download.01.org/intel-sgx/sgx-linux/2.15.1/docs/Intel_SGX_SW_Installation_Guide_for_Linux.pdf) to install SGX driver and SGX PSW. SGX SDK is not required. Using PSW installer is recommanded. + +To install PSW, follow the guide to add Intel® SGX repository to apt source. And then run: +``` +apt-get update +apt-get install -y libsgx-epid libsgx-urts libsgx-quote-ex libsgx-uae-service libsgx-dcap-ql +``` + +After installing PSW, please make sure that the aesm service is running: +``` +service aesmd status +``` + +**Step 3. Install enable_RDFSBASE Kernel Module** +Please follow [this README](https://github.com/occlum/enable_rdfsbase/blob/master/README.md) to install `enable_rdfsbase` kernel module. + +**Step 4. Install Occlum Installer and Toolchains Installer** +``` +cd +apt install -y ./occlum-runtime*.deb +apt install -y ./occlum-pal*.deb +apt install -y ./occlum-sgx-tools*.deb +``` + +Toolchains are needed when compiling applications and also during runtime. C/C++ toolchain is a must for Occlum commands. +To install C/C++ toolchain, just run the command: +``` +apt install -y ./occlum-toolchains-gcc*.deb +``` + +Besides, users can choose to install the toolchain installer based on the application's language. Currently, we also supports Golang. More language toolchain installers are on the way. To install Golang toolchain, run the below commands: +``` +apt install -y ./occlum-toolchains-golang*.deb +``` + +At last, install `occlum` package to get complete support of Occlum: +``` +apt install -y ./occlum_*.deb +``` + +To make the new installed binaries and libraries work, this command must be executed: +``` +source /etc/profile +``` + +**Step 5. Install Debug Symbol Packages (OPTIONAL)** +If users want to debug the application running inside the libos, debug symbol packages are also needed. Just run: +``` +apt install -y ./occlum-dbgsym*.ddeb ./occlum-pal-dbgsym*.ddeb ./occlum-runtime-dbgsym*.ddeb ./occlum-toolchains-gcc-dbgsym*.ddeb ./occlum-sgx-tools-dbgsym*.ddeb +``` + + ## RPM Installer +TBD + +