From 59017c2fa01ed4764225fdbfed7a17287f81d87a Mon Sep 17 00:00:00 2001
From: Lan Yun <yun.lan@intel.com>
Date: Thu, 28 Jul 2022 11:00:15 +0800
Subject: [PATCH] occlum-go-patch

Signed-off-by: Lan Yun <yun.lan@intel.com>
---
 src/runtime/malloc.go          |  70 ++++++------
 src/runtime/os_linux.go        |   4 +
 src/runtime/sys_linux_amd64.go |   3 +
 src/runtime/sys_linux_amd64.s  | 192 +++++++++++++++++++++++++--------
 src/runtime/textflag.h         |  18 +++-
 src/syscall/asm_linux_amd64.s  |  14 +--
 6 files changed, 212 insertions(+), 89 deletions(-)
 create mode 100644 src/runtime/sys_linux_amd64.go

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 6ed6ceade2..2a94401c0c 100644
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -645,42 +645,44 @@ func (h *mheap) sysAlloc(n uintptr) (v unsafe.Pointer, size uintptr) {
 		goto mapped
 	}
 
-	// Try to grow the heap at a hint address.
-	for h.arenaHints != nil {
-		hint := h.arenaHints
-		p := hint.addr
-		if hint.down {
-			p -= n
-		}
-		if p+n < p {
-			// We can't use this, so don't ask.
-			v = nil
-		} else if arenaIndex(p+n-1) >= 1<<arenaBits {
-			// Outside addressable heap. Can't use.
-			v = nil
-		} else {
-			v = sysReserve(unsafe.Pointer(p), n)
-		}
-		if p == uintptr(v) {
-			// Success. Update the hint.
-			if !hint.down {
-				p += n
+	if occlumentry == 0 {
+		// Try to grow the heap at a hint address.
+		for h.arenaHints != nil {
+			hint := h.arenaHints
+			p := hint.addr
+			if hint.down {
+				p -= n
 			}
-			hint.addr = p
-			size = n
-			break
-		}
-		// Failed. Discard this hint and try the next.
-		//
-		// TODO: This would be cleaner if sysReserve could be
-		// told to only return the requested address. In
-		// particular, this is already how Windows behaves, so
-		// it would simplify things there.
-		if v != nil {
-			sysFree(v, n, nil)
+			if p+n < p {
+				// We can't use this, so don't ask.
+				v = nil
+			} else if arenaIndex(p+n-1) >= 1<<arenaBits {
+				// Outside addressable heap. Can't use.
+				v = nil
+			} else {
+				v = sysReserve(unsafe.Pointer(p), n)
+			}
+			if p == uintptr(v) {
+				// Success. Update the hint.
+				if !hint.down {
+					p += n
+				}
+				hint.addr = p
+				size = n
+				break
+			}
+			// Failed. Discard this hint and try the next.
+			//
+			// TODO: This would be cleaner if sysReserve could be
+			// told to only return the requested address. In
+			// particular, this is already how Windows behaves, so
+			// it would simplify things there.
+			if v != nil {
+				sysFree(v, n, nil)
+			}
+			h.arenaHints = hint.next
+			h.arenaHintAlloc.free(unsafe.Pointer(hint))
 		}
-		h.arenaHints = hint.next
-		h.arenaHintAlloc.free(unsafe.Pointer(hint))
 	}
 
 	if size == 0 {
diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
index eb8aa076e9..b65751ce65 100644
--- a/src/runtime/os_linux.go
+++ b/src/runtime/os_linux.go
@@ -207,6 +207,7 @@ const (
 	_AT_HWCAP  = 16 // hardware capability bit vector
 	_AT_RANDOM = 25 // introduced in 2.6.29
 	_AT_HWCAP2 = 26 // hardware capability bit vector 2
+	_AT_OCCLUM = 48 // gnu syscall ABI entry address
 )
 
 var procAuxv = []byte("/proc/self/auxv\x00")
@@ -286,6 +287,9 @@ func sysauxv(auxv []uintptr) int {
 
 		case _AT_PAGESZ:
 			physPageSize = val
+
+		case _AT_OCCLUM:
+			occlumentry = val
 		}
 
 		archauxv(tag, val)
diff --git a/src/runtime/sys_linux_amd64.go b/src/runtime/sys_linux_amd64.go
new file mode 100644
index 0000000000..c9369b6661
--- /dev/null
+++ b/src/runtime/sys_linux_amd64.go
@@ -0,0 +1,3 @@
+package runtime
+
+var occlumentry uintptr = 0x0
diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s
index f0e58e11db..5cc653bf8a 100644
--- a/src/runtime/sys_linux_amd64.s
+++ b/src/runtime/sys_linux_amd64.s
@@ -57,7 +57,7 @@
 TEXT runtime·exit(SB),NOSPLIT,$0-4
 	MOVL	code+0(FP), DI
 	MOVL	$SYS_exit_group, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 // func exitThread(wait *uint32)
@@ -67,7 +67,7 @@ TEXT runtime·exitThread(SB),NOSPLIT,$0-8
 	MOVL	$0, (AX)
 	MOVL	$0, DI	// exit code
 	MOVL	$SYS_exit, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	// We may not even have a stack any more.
 	INT	$3
 	JMP	0(PC)
@@ -79,7 +79,7 @@ TEXT runtime·open(SB),NOSPLIT,$0-20
 	MOVL	mode+8(FP), DX
 	MOVL	perm+12(FP), R10
 	MOVL	$SYS_openat, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	2(PC)
 	MOVL	$-1, AX
@@ -89,7 +89,7 @@ TEXT runtime·open(SB),NOSPLIT,$0-20
 TEXT runtime·closefd(SB),NOSPLIT,$0-12
 	MOVL	fd+0(FP), DI
 	MOVL	$SYS_close, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	2(PC)
 	MOVL	$-1, AX
@@ -101,7 +101,7 @@ TEXT runtime·write1(SB),NOSPLIT,$0-28
 	MOVQ	p+8(FP), SI
 	MOVL	n+16(FP), DX
 	MOVL	$SYS_write, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -110,7 +110,7 @@ TEXT runtime·read(SB),NOSPLIT,$0-28
 	MOVQ	p+8(FP), SI
 	MOVL	n+16(FP), DX
 	MOVL	$SYS_read, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -118,7 +118,7 @@ TEXT runtime·read(SB),NOSPLIT,$0-28
 TEXT runtime·pipe(SB),NOSPLIT,$0-12
 	LEAQ	r+0(FP), DI
 	MOVL	$SYS_pipe, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, errno+8(FP)
 	RET
 
@@ -127,7 +127,7 @@ TEXT runtime·pipe2(SB),NOSPLIT,$0-20
 	LEAQ	r+8(FP), DI
 	MOVL	flags+0(FP), SI
 	MOVL	$SYS_pipe2, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, errno+16(FP)
 	RET
 
@@ -145,40 +145,40 @@ TEXT runtime·usleep(SB),NOSPLIT,$16
 	MOVQ	SP, DI
 	MOVL	$0, SI
 	MOVL	$SYS_nanosleep, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 TEXT runtime·gettid(SB),NOSPLIT,$0-4
 	MOVL	$SYS_gettid, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+0(FP)
 	RET
 
 TEXT runtime·raise(SB),NOSPLIT,$0
 	MOVL	$SYS_getpid, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, R12
 	MOVL	$SYS_gettid, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, SI	// arg 2 tid
 	MOVL	R12, DI	// arg 1 pid
 	MOVL	sig+0(FP), DX	// arg 3
 	MOVL	$SYS_tgkill, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 TEXT runtime·raiseproc(SB),NOSPLIT,$0
 	MOVL	$SYS_getpid, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, DI	// arg 1 pid
 	MOVL	sig+0(FP), SI	// arg 2
 	MOVL	$SYS_kill, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 TEXT ·getpid(SB),NOSPLIT,$0-8
 	MOVL	$SYS_getpid, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVQ	AX, ret+0(FP)
 	RET
 
@@ -187,7 +187,7 @@ TEXT ·tgkill(SB),NOSPLIT,$0
 	MOVQ	tid+8(FP), SI
 	MOVQ	sig+16(FP), DX
 	MOVL	$SYS_tgkill, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 TEXT runtime·setitimer(SB),NOSPLIT,$0-24
@@ -195,7 +195,7 @@ TEXT runtime·setitimer(SB),NOSPLIT,$0-24
 	MOVQ	new+8(FP), SI
 	MOVQ	old+16(FP), DX
 	MOVL	$SYS_setittimer, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 TEXT runtime·timer_create(SB),NOSPLIT,$0-28
@@ -203,7 +203,7 @@ TEXT runtime·timer_create(SB),NOSPLIT,$0-28
 	MOVQ	sevp+8(FP), SI
 	MOVQ	timerid+16(FP), DX
 	MOVL	$SYS_timer_create, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -213,14 +213,14 @@ TEXT runtime·timer_settime(SB),NOSPLIT,$0-28
 	MOVQ	new+8(FP), DX
 	MOVQ	old+16(FP), R10
 	MOVL	$SYS_timer_settime, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
 TEXT runtime·timer_delete(SB),NOSPLIT,$0-12
 	MOVL	timerid+0(FP), DI
 	MOVL	$SYS_timer_delete, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+8(FP)
 	RET
 
@@ -229,7 +229,7 @@ TEXT runtime·mincore(SB),NOSPLIT,$0-28
 	MOVQ	n+8(FP), SI
 	MOVQ	dst+16(FP), DX
 	MOVL	$SYS_mincore, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -296,7 +296,7 @@ ret:
 	RET
 fallback:
 	MOVQ	$SYS_clock_gettime, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	JMP	ret
 
 TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
@@ -305,7 +305,7 @@ TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
 	MOVQ	old+16(FP), DX
 	MOVL	size+24(FP), R10
 	MOVL	$SYS_rt_sigprocmask, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	2(PC)
 	MOVL	$0xf1, 0xf1  // crash
@@ -317,7 +317,7 @@ TEXT runtime·rt_sigaction(SB),NOSPLIT,$0-36
 	MOVQ	old+16(FP), DX
 	MOVQ	size+24(FP), R10
 	MOVL	$SYS_rt_sigaction, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+32(FP)
 	RET
 
@@ -464,7 +464,7 @@ sigtrampnog:
 // https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup
 TEXT runtime·sigreturn(SB),NOSPLIT,$0
 	MOVQ	$SYS_rt_sigreturn, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	INT $3	// not reached
 
 TEXT runtime·sysMmap(SB),NOSPLIT,$0
@@ -476,7 +476,7 @@ TEXT runtime·sysMmap(SB),NOSPLIT,$0
 	MOVL	off+28(FP), R9
 
 	MOVL	$SYS_mmap, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	ok
 	NOTQ	AX
@@ -511,7 +511,7 @@ TEXT runtime·sysMunmap(SB),NOSPLIT,$0
 	MOVQ	addr+0(FP), DI
 	MOVQ	n+8(FP), SI
 	MOVQ	$SYS_munmap, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	2(PC)
 	MOVL	$0xf1, 0xf1  // crash
@@ -535,7 +535,7 @@ TEXT runtime·madvise(SB),NOSPLIT,$0
 	MOVQ	n+8(FP), SI
 	MOVL	flags+16(FP), DX
 	MOVQ	$SYS_madvise, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -549,12 +549,15 @@ TEXT runtime·futex(SB),NOSPLIT,$0
 	MOVQ	addr2+24(FP), R8
 	MOVL	val3+32(FP), R9
 	MOVL	$SYS_futex, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+40(FP)
 	RET
 
 // int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void));
 TEXT runtime·clone(SB),NOSPLIT,$0
+    CMPQ runtime·occlumentry(SB), $0x0
+    JNE occlum
+
 	MOVL	flags+0(FP), DI
 	MOVQ	stk+8(FP), SI
 	MOVQ	$0, DX
@@ -618,11 +621,110 @@ nog2:
 	SYSCALL
 	JMP	-3(PC)	// keep exiting
 
+occlum:
+	MOVL	flags+0(FP), DI
+	MOVQ	stk+8(FP), SI
+	MOVQ	$0, DX
+	MOVQ	$0, R10
+	MOVQ    $0, R8
+	// Copy mp, gp, fn off parent stack for use by child.
+	// Careful: Linux system call clobbers CX and R11.
+	MOVQ	mp+16(FP), R13
+	MOVQ	gp+24(FP), R9
+	MOVQ	fn+32(FP), R12
+	CMPQ	R13, $0    // m
+	JEQ	occlum_nog1
+	CMPQ	R9, $0    // g
+	JEQ	occlum_nog1
+	LEAQ	m_tls(R13), R8
+#ifdef GOOS_android
+	// Android stores the TLS offset in runtime·tls_g.
+	SUBQ	runtime·tls_g(SB), R8
+#else
+	ADDQ	$8, R8	// ELF wants to use -8(FS)
+#endif
+	ORQ 	$0x00080000, DI //add flag CLONE_SETTLS(0x00080000) to call clone
+occlum_nog1:
+	// flags
+	MOVQ DI, -16(SI)
+	// fn
+	MOVQ R12, -24(SI)
+	// gp
+	MOVQ R9, -32(SI)
+	// m
+	MOVQ R13, -40(SI)
+	// lea 0x23(%rip),%rax
+	BYTE $0x48; BYTE $0x8d; BYTE $0x05; BYTE $0x23; BYTE $0x00; BYTE $0x00; BYTE $0x00
+	// thread entry point
+	MOVQ AX, -8(SI)
+
+	SUBQ $8, SI
+
+	MOVL	$SYS_clone, AX
+	// BYTE $0xcc
+	// lea 0xa(%rip),%rcx
+	BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0a; BYTE $0x00; BYTE $0x00; BYTE $0x00
+	MOVQ runtime·occlumentry(SB), R11
+	JMP  R11
+
+	// In parent, return.
+	MOVL	AX, ret+40(FP)
+	RET
+
+thread_entrypoint:
+	// add 8, %rsp
+	// MOVQ	SI, SP
+	ADDQ $8, SP
+
+	MOVQ SP, SI
+
+	// mov -24(%rsp), %r12
+	// fn
+	BYTE $0x4c; BYTE $0x8b; BYTE $0x64; BYTE $0x24; BYTE $0xe8
+
+	// mov -32(%rsp), %r9
+	// gp
+	BYTE $0x4c; BYTE $0x8b; BYTE $0x4c; BYTE $0x24; BYTE $0xe0
+
+	// mov -40(%rsp), %r13
+	// m
+	BYTE $0x4c; BYTE $0x8b; BYTE $0x6c; BYTE $0x24; BYTE $0xd8
+
+	// BYTE $0xcc
+
+	// If g or m are nil, skip Go-related setup.
+	CMPQ	R13, $0    // m
+	JEQ	occlum_nog2
+	CMPQ	R9, $0    // g
+	JEQ	occlum_nog2
+
+	// Initialize m->procid to Linux tid
+	MOVL	$SYS_gettid, AX
+	SYSCALL_ENHANCE
+	MOVQ	AX, m_procid(R13)
+
+	// In child, set up new stack
+	get_tls(CX)
+	MOVQ	R13, g_m(R9)
+	MOVQ	R9, g(CX)
+	MOVQ	R9, R14 // set g register
+	CALL	runtime·stackcheck(SB)
+
+occlum_nog2:
+	// Call fn
+	CALL	R12
+
+	// It shouldn't return. If it does, exit that thread.
+	MOVL	$111, DI
+	MOVL	$SYS_exit, AX
+	SYSCALL_ENHANCE
+	JMP	-3(PC)	// keep exiting
+
 TEXT runtime·sigaltstack(SB),NOSPLIT,$-8
 	MOVQ	new+0(FP), DI
 	MOVQ	old+8(FP), SI
 	MOVQ	$SYS_sigaltstack, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	2(PC)
 	MOVL	$0xf1, 0xf1  // crash
@@ -639,7 +741,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32
 	MOVQ	DI, SI
 	MOVQ	$0x1002, DI	// ARCH_SET_FS
 	MOVQ	$SYS_arch_prctl, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	2(PC)
 	MOVL	$0xf1, 0xf1  // crash
@@ -647,7 +749,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32
 
 TEXT runtime·osyield(SB),NOSPLIT,$0
 	MOVL	$SYS_sched_yield, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
@@ -655,7 +757,7 @@ TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
 	MOVQ	len+8(FP), SI
 	MOVQ	buf+16(FP), DX
 	MOVL	$SYS_sched_getaffinity, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -663,7 +765,7 @@ TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
 TEXT runtime·epollcreate(SB),NOSPLIT,$0
 	MOVL    size+0(FP), DI
 	MOVL    $SYS_epoll_create, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+8(FP)
 	RET
 
@@ -671,7 +773,7 @@ TEXT runtime·epollcreate(SB),NOSPLIT,$0
 TEXT runtime·epollcreate1(SB),NOSPLIT,$0
 	MOVL	flags+0(FP), DI
 	MOVL	$SYS_epoll_create1, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+8(FP)
 	RET
 
@@ -682,7 +784,7 @@ TEXT runtime·epollctl(SB),NOSPLIT,$0
 	MOVL	fd+8(FP), DX
 	MOVQ	ev+16(FP), R10
 	MOVL	$SYS_epoll_ctl, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -695,7 +797,7 @@ TEXT runtime·epollwait(SB),NOSPLIT,$0
 	MOVL	timeout+20(FP), R10
 	MOVQ	$0, R8
 	MOVL	$SYS_epoll_pwait, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -705,7 +807,7 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0
 	MOVQ    $2, SI  // F_SETFD
 	MOVQ    $1, DX  // FD_CLOEXEC
 	MOVL	$SYS_fcntl, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 // func runtime·setNonblock(int32 fd)
@@ -714,13 +816,13 @@ TEXT runtime·setNonblock(SB),NOSPLIT,$0-4
 	MOVQ    $3, SI  // F_GETFL
 	MOVQ    $0, DX
 	MOVL	$SYS_fcntl, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	fd+0(FP), DI // fd
 	MOVQ	$4, SI // F_SETFL
 	MOVQ	$0x800, DX // O_NONBLOCK
 	ORL	AX, DX
 	MOVL	$SYS_fcntl, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	RET
 
 // int access(const char *name, int mode)
@@ -731,7 +833,7 @@ TEXT runtime·access(SB),NOSPLIT,$0
 	MOVL	mode+8(FP), DX
 	MOVL	$0, R10
 	MOVL	$SYS_faccessat, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+16(FP)
 	RET
 
@@ -741,7 +843,7 @@ TEXT runtime·connect(SB),NOSPLIT,$0-28
 	MOVQ	addr+8(FP), SI
 	MOVL	len+16(FP), DX
 	MOVL	$SYS_connect, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+24(FP)
 	RET
 
@@ -751,7 +853,7 @@ TEXT runtime·socket(SB),NOSPLIT,$0-20
 	MOVL	typ+4(FP), SI
 	MOVL	prot+8(FP), DX
 	MOVL	$SYS_socket, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVL	AX, ret+16(FP)
 	RET
 
@@ -760,6 +862,6 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8
 	// Implemented as brk(NULL).
 	MOVQ	$0, DI
 	MOVL	$SYS_brk, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVQ	AX, ret+0(FP)
 	RET
diff --git a/src/runtime/textflag.h b/src/runtime/textflag.h
index 214075e360..4b7da24e64 100644
--- a/src/runtime/textflag.h
+++ b/src/runtime/textflag.h
@@ -32,8 +32,20 @@
 #define NOFRAME 512
 // Function can call reflect.Type.Method or reflect.Type.MethodByName.
 #define REFLECTMETHOD 1024
-// Function is the outermost frame of the call stack. Call stack unwinders
-// should stop at this function.
+// Function is the top of the call stack. Call stack unwinders should stop
+// at this function.
 #define TOPFRAME 2048
-// Function is an ABI wrapper.
 #define ABIWRAPPER 4096
+// step one: assign syscall return address to register rcx, occlum know where
+//           should return according to rcx.
+// step two: jump to syscall interface address provided by occlum when go
+//           .bin file loaded.
+// <BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0c; BYTE $0x00; BYTE $0x00; BYTE $0x00>
+// actually is the assembler instruction: lea 0xc(%rip),%rcx
+#define SYSCALL_ENHANCE  \
+    CMPQ runtime·occlumentry(SB), $0x0  \
+    JBE  10(PC)  \
+    BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0c; BYTE $0x00; BYTE $0x00; BYTE $0x00  \
+    MOVQ runtime·occlumentry(SB), R11  \
+    JMP  R11  \
+    SYSCALL
diff --git a/src/syscall/asm_linux_amd64.s b/src/syscall/asm_linux_amd64.s
index a9af68d51d..5110d78bb5 100644
--- a/src/syscall/asm_linux_amd64.s
+++ b/src/syscall/asm_linux_amd64.s
@@ -22,7 +22,7 @@ TEXT ·Syscall(SB),NOSPLIT,$0-56
 	MOVQ	a2+16(FP), SI
 	MOVQ	a3+24(FP), DX
 	MOVQ	trap+0(FP), AX	// syscall entry
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	ok
 	MOVQ	$-1, r1+32(FP)
@@ -48,7 +48,7 @@ TEXT ·Syscall6(SB),NOSPLIT,$0-80
 	MOVQ	a5+40(FP), R8
 	MOVQ	a6+48(FP), R9
 	MOVQ	trap+0(FP), AX	// syscall entry
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	ok6
 	MOVQ	$-1, r1+56(FP)
@@ -70,7 +70,7 @@ TEXT ·RawSyscall(SB),NOSPLIT,$0-56
 	MOVQ	a2+16(FP), SI
 	MOVQ	a3+24(FP), DX
 	MOVQ	trap+0(FP), AX	// syscall entry
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	ok1
 	MOVQ	$-1, r1+32(FP)
@@ -93,7 +93,7 @@ TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
 	MOVQ	a5+40(FP), R8
 	MOVQ	a6+48(FP), R9
 	MOVQ	trap+0(FP), AX	// syscall entry
-	SYSCALL
+	SYSCALL_ENHANCE
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	ok2
 	MOVQ	$-1, r1+56(FP)
@@ -117,7 +117,7 @@ TEXT ·rawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-32
 	MOVQ	$0, R9
 	MOVQ	trap+0(FP), AX	// syscall entry
 	POPQ	R12 // preserve return address
-	SYSCALL
+	SYSCALL_ENHANCE
 	PUSHQ	R12
 	CMPQ	AX, $0xfffffffffffff001
 	JLS	ok2
@@ -136,7 +136,7 @@ TEXT ·rawSyscallNoError(SB),NOSPLIT,$0-48
 	MOVQ	a2+16(FP), SI
 	MOVQ	a3+24(FP), DX
 	MOVQ	trap+0(FP), AX	// syscall entry
-	SYSCALL
+	SYSCALL_ENHANCE
 	MOVQ	AX, r1+32(FP)
 	MOVQ	DX, r2+40(FP)
 	RET
@@ -157,7 +157,7 @@ ret:
 	RET
 fallback:
 	MOVL	$SYS_gettimeofday, AX
-	SYSCALL
+	SYSCALL_ENHANCE
 	JMP ret
 ok7:
 	MOVQ	$0, err+8(FP)
-- 
2.25.1