general/occlum
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Polish the XGBoost demo

Browse Source 
1. Remove some useless installed packages
2. Adjust max number of threads according to CPU cores
This commit is contained in:
LI Qing 2020-06-03 17:06:07 +08:00
parent e8e14350a5
commit a0d7b96a8f
2 changed files with 21 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
demos/xgboost/Makefile
View File

@ -2,47 +2,47 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
XGBOOST_TRACKER := $(realpath $(CUR_DIR)/xgboost_src/dmlc-core/tracker/) XGBOOST_TRACKER := $(realpath $(CUR_DIR)/xgboost_src/dmlc-core/tracker/)
BIN := xgboost BIN := xgboost
DATA := data DATA := data
TCS_NUM := $(shell expr $(shell nproc) \* 4)
.PHONY: all test test-native test-local-cluster clean .PHONY: all test test-native test-local-cluster clean
all: occlum_workspace all: occlum_workspace
$(BIN): $(BIN):
cp xgboost_src/$(BIN) . @cp -f xgboost_src/$(BIN) .
$(DATA): $(DATA):
rm -rf $(DATA) && mkdir -p $(DATA) @rm -rf $(DATA) && mkdir -p $(DATA)
cp xgboost_src/demo/binary_classification/agaricus.txt.test $(DATA) @cp xgboost_src/demo/binary_classification/agaricus.txt.test $(DATA)
cp xgboost_src/demo/binary_classification/agaricus.txt.train $(DATA) @cp xgboost_src/demo/binary_classification/agaricus.txt.train $(DATA)
cp xgboost_src/demo/binary_classification/mushroom.conf $(DATA) @cp xgboost_src/demo/binary_classification/mushroom.conf $(DATA)
sed -i 's/agaricus/.\/data\/agaricus/g' $(DATA)/mushroom.conf @sed -i 's/agaricus/.\/data\/agaricus/g' $(DATA)/mushroom.conf
sed -i 's/num_round = 2/num_round = 10/g' $(DATA)/mushroom.conf @sed -i 's/num_round = 2/num_round = 10/g' $(DATA)/mushroom.conf
sed -i '$$a\model_out = "./host/result.model"' $(DATA)/mushroom.conf @sed -i '$$a\model_out = "./host/result.model"' $(DATA)/mushroom.conf
sed -i '$$a\verbosity = 2' $(DATA)/mushroom.conf @sed -i '$$a\verbosity = 2' $(DATA)/mushroom.conf
sed -i '$$a\nthread = 2' $(DATA)/mushroom.conf @sed -i '$$a\nthread = 2' $(DATA)/mushroom.conf
occlum_workspace: $(BIN) $(DATA) occlum_workspace: $(BIN) $(DATA)
rm -rf occlum_workspace && mkdir occlum_workspace @rm -rf occlum_workspace && mkdir occlum_workspace
cd occlum_workspace && \ @cd occlum_workspace && \
occlum init && \ occlum init && \
jq '.process.default_mmap_size = "96MB"' Occlum.json > temp_Occlum.json && \ jq '.process.default_mmap_size = "96MB" | .resource_limits.max_num_of_threads = $(TCS_NUM)' Occlum.json > temp_Occlum.json && \
mv temp_Occlum.json Occlum.json && \ mv temp_Occlum.json Occlum.json && \
cp ../$(BIN) image/bin && \ cp ../$(BIN) image/bin && \
cp ../xgboost_src/lib/libxgboost.so image/lib && \ cp ../xgboost_src/lib/libxgboost.so image/lib && \
mkdir -p image/$(DATA) && \ cp -r ../$(DATA) image && \
cp -r ../$(DATA)/* image/$(DATA) && \
occlum build occlum build
test: occlum_workspace test: occlum_workspace
@./run_xgboost.sh @./run_xgboost.sh
test-native: $(BIN) $(DATA) test-native: $(BIN) $(DATA)
rm -rf host @rm -rf host
mkdir host @mkdir host
./$(BIN) ./$(DATA)/mushroom.conf @./$(BIN) ./$(DATA)/mushroom.conf
test-local-cluster: occlum_workspace test-local-cluster: occlum_workspace
@$(XGBOOST_TRACKER)/dmlc-submit --cluster=local --num-workers=2 --log-level=DEBUG ./run_xgboost.sh @python3 $(XGBOOST_TRACKER)/dmlc-submit --cluster=local --num-workers=2 --log-level=DEBUG ./run_xgboost.sh
clean: clean:
rm -rf $(BIN) $(DATA) tmp_* host occlum_workspace @rm -rf $(BIN) $(DATA) tmp_* host occlum_workspace

7
demos/xgboost/download_and_build_xgboost.sh
View File

@ -6,13 +6,8 @@ export PKG_CONFIG_LIBDIR=/usr/local/occlum/x86_64-linux-musl/lib
# Install the dependencies # Install the dependencies
apt-get update apt-get update
apt-get install -y python3.5 apt-get install -y python3-pip python3-setuptools
apt-get install -y python3-pip
apt-get install -y python3-setuptools
apt-get install -y python-pip
apt-get install -y python-setuptools
pip3 install kubernetes pip3 install kubernetes
pip install kubernetes
# Download and build XGBoost # Download and build XGBoost
rm -rf xgboost_src && mkdir xgboost_src rm -rf xgboost_src && mkdir xgboost_src