Add the flink demo

Signed-off-by: yuanwu <yuan.wu@intel.com>

.github/workflows/demo_test.yml

@@ -416,3 +416,37 @@ jobs:
 
     - name: Run redis benchmark
       run: docker exec redis_support_test bash -c "cd /root/occlum/demos/redis; SGX_MODE=SIM ./benchmark_glibc.sh"
+
+  flink_test:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
+
+    - name: Get occlum version
+      run: echo "OCCLUM_VERSION=$(grep "Version =" src/pal/include/occlum_version.h |  awk '{print $4}')" >> $GITHUB_ENV
+
+    - name: Create container
+      run: docker run -itd --name=flink_test -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04
+
+    - name: Build dependencies
+      run: docker exec flink_test bash -c "cd /root/occlum; make submodule"
+
+    - name: Make install
+      run: docker exec flink_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"
+
+    - name: Download flink
+      run: docker exec flink_test bash -c "cd /root/occlum/demos/flink && ./download_flink.sh"
+
+    - name: Run jobmanager on host
+      run: docker exec flink_test bash -c "cd /root/occlum/demos/flink && SGX_MODE=SIM ./run_flink_jobmanager_on_host.sh"
+
+    - name: Run flink taskmanager
+      run: docker exec flink_test bash -c "cd /root/occlum/demos/flink && SGX_MODE=SIM ./run_flink_on_occlum_glibc.sh tm"
+
+    - name: Run flink task
+      run: |
+        sleep ${{ env.nap_time }};
+        docker exec flink_test bash -c "cd /root/occlum/demos/flink && SGX_MODE=SIM ./run_flink_on_occlum_glibc.sh task"
+

demos/flink/README.txt

@@ -0,0 +1,8 @@
+1. Run the flink jobmanager
+    ./run_flink_jobmanager_on_host.sh
+2. Run the taskManager
+    ./run_flink_on_occlum_glibc.sh tm
+3. Run flink jobs example
+    ./run_flink_on_occlum.sh task
+
+Note: If running the jobmanager in docker, please export the port 8081 and 6123

demos/flink/download_flink.sh

@@ -0,0 +1,4 @@
+apt-get update
+apt-get install -y openjdk-11-jdk
+wget https://archive.apache.org/dist/flink/flink-1.10.1/flink-1.10.1-bin-scala_2.11.tgz
+tar -xvzf flink-1.10.1-bin-scala_2.11.tgz

demos/flink/hosts

@@ -0,0 +1,4 @@
+127.0.0.1 occlum-node
+127.0.0.1 localhost
+::1 occlum-node
+::1 localhost

demos/flink/run_flink_jobmanager_on_host.sh

@@ -0,0 +1,3 @@
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
+./flink-1.10.1/bin/jobmanager.sh start
+echo -e "${BLUE}Flink jobmanager${NC}"

demos/flink/run_flink_on_occlum_glibc.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+set -e
+
+BLUE='\033[1;34m'
+NC='\033[0m'
+occlum_glibc=/opt/occlum/glibc/lib/
+
+init_instance() {
+    # Init Occlum instance
+    postfix=$1
+    FLINK_LOG_PREFIX="/host/flink--$postfix-${id}"
+    log="${FLINK_LOG_PREFIX}.log"
+    out="./flink--$postfix-${id}.out"
+
+    rm -rf occlum_instance_$postfix && mkdir occlum_instance_$postfix
+    cd occlum_instance_$postfix
+    occlum init
+    new_json="$(jq '.resource_limits.user_space_size = "5500MB" |
+        .resource_limits.max_num_of_threads = 64 |
+        .process.default_heap_size = "128MB" |
+        .resource_limits.kernel_space_heap_size="64MB" |
+        .process.default_mmap_size = "5000MB" |
+        .entry_points = [ "/usr/lib/jvm/java-11-openjdk-amd64/bin" ] |
+        .env.default = [ "LD_LIBRARY_PATH=/usr/lib/jvm/java-11-openjdk-amd64/lib/server:/usr/lib/jvm/java-11-openjdk-amd64/lib:/usr/lib/jvm/java-11-openjdk-amd64/../lib:/lib" ]' Occlum.json)" && \
+    echo "${new_json}" > Occlum.json
+
+}
+
+build_flink() {
+    # Copy JVM and class file into Occlum instance and build
+    mkdir -p image/usr/lib/jvm
+    cp -r /usr/lib/jvm/java-11-openjdk-amd64 image/usr/lib/jvm
+    cp /lib/x86_64-linux-gnu/libz.so.1 image/lib
+    cp $occlum_glibc/libdl.so.2 image/$occlum_glibc
+    cp $occlum_glibc/librt.so.1 image/$occlum_glibc
+    cp $occlum_glibc/libm.so.6 image/$occlum_glibc
+    cp $occlum_glibc/libnss_files.so.2 image/$occlum_glibc
+    cp -rf ../flink-1.10.1/* image/bin/
+    cp -rf ../hosts image/etc/
+    occlum build
+}
+
+run_taskmanager() {
+    init_instance taskmanager
+    build_flink
+    echo -e "${BLUE}occlum run JVM taskmanager${NC}"
+    echo -e "${BLUE}logfile=$log${NC}"
+    occlum run /usr/lib/jvm/java-11-openjdk-amd64/bin/java \
+	-Xmx800m -XX:-UseCompressedOops -XX:MaxMetaspaceSize=256m \
+	-XX:ActiveProcessorCount=2 \
+	-Dlog.file=$log \
+	-Dos.name=Linux \
+	-Dlog4j.configuration=file:/bin/conf/log4j.properties \
+	-Dlogback.configurationFile=file:/bin/conf/logback.xml \
+	-classpath /bin/lib/flink-table-blink_2.11-1.10.1.jar:/bin/lib/flink-table_2.11-1.10.1.jar:/bin/lib/log4j-1.2.17.jar:/bin/lib/slf4j-log4j12-1.7.15.jar:/bin/lib/flink-dist_2.11-1.10.1.jar org.apache.flink.runtime.taskexecutor.TaskManagerRunner \
+	--configDir /bin/conf \
+	-D taskmanager.memory.network.max=64mb \
+	-D taskmanager.memory.network.min=64mb \
+	-D taskmanager.memory.managed.size=128mb \
+	-D taskmanager.cpu.cores=1.0 \
+	-D taskmanager.memory.task.heap.size=256mb \
+    &
+}
+
+run_task() {
+    
+    export FLINK_CONF_DIR=$PWD/flink-1.10.1/conf && \
+        ./flink-1.10.1/bin/flink run ./flink-1.10.1/examples/streaming/WordCount.jar
+}
+
+id=$([ -f "$pid" ] && echo $(wc -l < "$pid") || echo "0")
+
+arg=$1
+case "$arg" in
+    tm)
+        run_taskmanager
+	cd ../
+        ;;
+    task)
+        run_task
+	cd ../
+        ;;
+esac