From a0d7b96a8fd4dab254ece2e3271f8f68277f49aa Mon Sep 17 00:00:00 2001
From: LI Qing <geding.lq@antfin.com>
Date: Wed, 3 Jun 2020 17:06:07 +0800
Subject: [PATCH] Polish the XGBoost demo

1. Remove some useless installed packages
2. Adjust max number of threads according to CPU cores
---
 demos/xgboost/Makefile                      | 40 ++++++++++-----------
 demos/xgboost/download_and_build_xgboost.sh |  7 +---
 2 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/demos/xgboost/Makefile b/demos/xgboost/Makefile
index 18bffc9d..eb72bbf8 100644
--- a/demos/xgboost/Makefile
+++ b/demos/xgboost/Makefile
@@ -2,47 +2,47 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 XGBOOST_TRACKER := $(realpath $(CUR_DIR)/xgboost_src/dmlc-core/tracker/)
 BIN := xgboost
 DATA := data
+TCS_NUM := $(shell expr $(shell nproc) \* 4)
 
 .PHONY: all test test-native test-local-cluster clean
 
 all: occlum_workspace
 
 $(BIN):
-	cp xgboost_src/$(BIN) .
+	@cp -f xgboost_src/$(BIN) .
 
 $(DATA):
-	rm -rf $(DATA) && mkdir -p $(DATA)
-	cp xgboost_src/demo/binary_classification/agaricus.txt.test $(DATA)
-	cp xgboost_src/demo/binary_classification/agaricus.txt.train $(DATA)
-	cp xgboost_src/demo/binary_classification/mushroom.conf $(DATA)
-	sed -i 's/agaricus/.\/data\/agaricus/g' $(DATA)/mushroom.conf
-	sed -i 's/num_round = 2/num_round = 10/g' $(DATA)/mushroom.conf
-	sed -i '$$a\model_out = "./host/result.model"' $(DATA)/mushroom.conf
-	sed -i '$$a\verbosity = 2' $(DATA)/mushroom.conf
-	sed -i '$$a\nthread = 2' $(DATA)/mushroom.conf
+	@rm -rf $(DATA) && mkdir -p $(DATA)
+	@cp xgboost_src/demo/binary_classification/agaricus.txt.test $(DATA)
+	@cp xgboost_src/demo/binary_classification/agaricus.txt.train $(DATA)
+	@cp xgboost_src/demo/binary_classification/mushroom.conf $(DATA)
+	@sed -i 's/agaricus/.\/data\/agaricus/g' $(DATA)/mushroom.conf
+	@sed -i 's/num_round = 2/num_round = 10/g' $(DATA)/mushroom.conf
+	@sed -i '$$a\model_out = "./host/result.model"' $(DATA)/mushroom.conf
+	@sed -i '$$a\verbosity = 2' $(DATA)/mushroom.conf
+	@sed -i '$$a\nthread = 2' $(DATA)/mushroom.conf
 
 occlum_workspace: $(BIN) $(DATA)
-	rm -rf occlum_workspace && mkdir occlum_workspace
-	cd occlum_workspace && \
+	@rm -rf occlum_workspace && mkdir occlum_workspace
+	@cd occlum_workspace && \
 	occlum init && \
-	jq '.process.default_mmap_size = "96MB"' Occlum.json > temp_Occlum.json && \
+	jq '.process.default_mmap_size = "96MB" | .resource_limits.max_num_of_threads = $(TCS_NUM)' Occlum.json > temp_Occlum.json && \
 	mv temp_Occlum.json Occlum.json && \
 	cp ../$(BIN) image/bin && \
 	cp ../xgboost_src/lib/libxgboost.so image/lib && \
-	mkdir -p image/$(DATA) && \
-	cp -r ../$(DATA)/* image/$(DATA) && \
+	cp -r ../$(DATA) image && \
 	occlum build
 
 test: occlum_workspace
 	@./run_xgboost.sh
 
 test-native: $(BIN) $(DATA)
-	rm -rf host
-	mkdir host
-	./$(BIN) ./$(DATA)/mushroom.conf
+	@rm -rf host
+	@mkdir host
+	@./$(BIN) ./$(DATA)/mushroom.conf
 
 test-local-cluster: occlum_workspace
-	@$(XGBOOST_TRACKER)/dmlc-submit --cluster=local --num-workers=2 --log-level=DEBUG ./run_xgboost.sh
+	@python3 $(XGBOOST_TRACKER)/dmlc-submit --cluster=local --num-workers=2 --log-level=DEBUG ./run_xgboost.sh
 
 clean:
-	rm -rf $(BIN) $(DATA) tmp_* host occlum_workspace
+	@rm -rf $(BIN) $(DATA) tmp_* host occlum_workspace
diff --git a/demos/xgboost/download_and_build_xgboost.sh b/demos/xgboost/download_and_build_xgboost.sh
index 9abd11a5..9616db14 100755
--- a/demos/xgboost/download_and_build_xgboost.sh
+++ b/demos/xgboost/download_and_build_xgboost.sh
@@ -6,13 +6,8 @@ export PKG_CONFIG_LIBDIR=/usr/local/occlum/x86_64-linux-musl/lib
 
 # Install the dependencies
 apt-get update
-apt-get install -y python3.5
-apt-get install -y python3-pip
-apt-get install -y python3-setuptools
-apt-get install -y python-pip
-apt-get install -y python-setuptools
+apt-get install -y python3-pip python3-setuptools
 pip3 install kubernetes
-pip install kubernetes
 
 # Download and build XGBoost
 rm -rf xgboost_src && mkdir xgboost_src