[toolchain] Use Occlum go repo
This commit is contained in:
parent
3ef075ef8a
commit
37d1f58281
4
.github/workflows/demo_test.yml
vendored
4
.github/workflows/demo_test.yml
vendored
@ -57,10 +57,10 @@ jobs:
|
||||
SGX_MODE=SIM make test"
|
||||
|
||||
- name: Run Golang v1.16.3 sqlite test
|
||||
run: docker exec ${{ github.job }} bash -c "cd /root/occlum/tools/toolchains/golang && ./build.sh go1.16.3 && cd /root/occlum/demos/golang/go_sqlite/ && SGX_MODE=SIM ./run_go_sqlite_demo.sh"
|
||||
run: docker exec ${{ github.job }} bash -c "cd /root/occlum/tools/toolchains/golang && ./build.sh go1.16.3_for_occlum && cd /root/occlum/demos/golang/go_sqlite/ && SGX_MODE=SIM ./run_go_sqlite_demo.sh"
|
||||
|
||||
- name: Run Golang v1.18.4 sqlite test
|
||||
run: docker exec ${{ github.job }} bash -c "cd /root/occlum/tools/toolchains/golang && ./build.sh go1.18.4 && cd /root/occlum/demos/golang/go_sqlite/ && SGX_MODE=SIM ./run_go_sqlite_demo.sh"
|
||||
run: docker exec ${{ github.job }} bash -c "cd /root/occlum/tools/toolchains/golang && ./build.sh go1.18.4_for_occlum && cd /root/occlum/demos/golang/go_sqlite/ && SGX_MODE=SIM ./run_go_sqlite_demo.sh"
|
||||
|
||||
- name: Go Server set up and run
|
||||
run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/golang/web_server && occlum-go mod init web_server && occlum-go get -u -v github.com/gin-gonic/gin;
|
||||
|
@ -1,696 +0,0 @@
|
||||
From c1d15cf5649843a46e2563982ed981dbda9e7219 Mon Sep 17 00:00:00 2001
|
||||
From: jeffery.wsj <jeffery.wsj@alibaba-inc.com>
|
||||
Date: Fri, 9 Apr 2021 14:23:49 +0800
|
||||
Subject: [PATCH] adapt golang1.16 to occlum libos
|
||||
|
||||
---
|
||||
src/runtime/malloc.go | 70 ++++++++++++++++++++++++++++++++++++----------------------------------
|
||||
src/runtime/os_linux.go | 4 ++++
|
||||
src/runtime/sys_linux_amd64.go | 3 +++
|
||||
src/runtime/sys_linux_amd64.s | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------
|
||||
src/runtime/textflag.h | 14 ++++++++++++++
|
||||
src/syscall/asm_linux_amd64.s | 14 +++++++-------
|
||||
6 files changed, 208 insertions(+), 84 deletions(-)
|
||||
create mode 100644 src/runtime/sys_linux_amd64.go
|
||||
|
||||
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
|
||||
index f20ded5..a557638 100644
|
||||
--- a/src/runtime/malloc.go
|
||||
+++ b/src/runtime/malloc.go
|
||||
@@ -638,42 +638,44 @@ func (h *mheap) sysAlloc(n uintptr) (v unsafe.Pointer, size uintptr) {
|
||||
goto mapped
|
||||
}
|
||||
|
||||
- // Try to grow the heap at a hint address.
|
||||
- for h.arenaHints != nil {
|
||||
- hint := h.arenaHints
|
||||
- p := hint.addr
|
||||
- if hint.down {
|
||||
- p -= n
|
||||
- }
|
||||
- if p+n < p {
|
||||
- // We can't use this, so don't ask.
|
||||
- v = nil
|
||||
- } else if arenaIndex(p+n-1) >= 1<<arenaBits {
|
||||
- // Outside addressable heap. Can't use.
|
||||
- v = nil
|
||||
- } else {
|
||||
- v = sysReserve(unsafe.Pointer(p), n)
|
||||
- }
|
||||
- if p == uintptr(v) {
|
||||
- // Success. Update the hint.
|
||||
- if !hint.down {
|
||||
- p += n
|
||||
+ if occlumentry == 0 {
|
||||
+ // Try to grow the heap at a hint address.
|
||||
+ for h.arenaHints != nil {
|
||||
+ hint := h.arenaHints
|
||||
+ p := hint.addr
|
||||
+ if hint.down {
|
||||
+ p -= n
|
||||
}
|
||||
- hint.addr = p
|
||||
- size = n
|
||||
- break
|
||||
- }
|
||||
- // Failed. Discard this hint and try the next.
|
||||
- //
|
||||
- // TODO: This would be cleaner if sysReserve could be
|
||||
- // told to only return the requested address. In
|
||||
- // particular, this is already how Windows behaves, so
|
||||
- // it would simplify things there.
|
||||
- if v != nil {
|
||||
- sysFree(v, n, nil)
|
||||
+ if p+n < p {
|
||||
+ // We can't use this, so don't ask.
|
||||
+ v = nil
|
||||
+ } else if arenaIndex(p+n-1) >= 1<<arenaBits {
|
||||
+ // Outside addressable heap. Can't use.
|
||||
+ v = nil
|
||||
+ } else {
|
||||
+ v = sysReserve(unsafe.Pointer(p), n)
|
||||
+ }
|
||||
+ if p == uintptr(v) {
|
||||
+ // Success. Update the hint.
|
||||
+ if !hint.down {
|
||||
+ p += n
|
||||
+ }
|
||||
+ hint.addr = p
|
||||
+ size = n
|
||||
+ break
|
||||
+ }
|
||||
+ // Failed. Discard this hint and try the next.
|
||||
+ //
|
||||
+ // TODO: This would be cleaner if sysReserve could be
|
||||
+ // told to only return the requested address. In
|
||||
+ // particular, this is already how Windows behaves, so
|
||||
+ // it would simplify things there.
|
||||
+ if v != nil {
|
||||
+ sysFree(v, n, nil)
|
||||
+ }
|
||||
+ h.arenaHints = hint.next
|
||||
+ h.arenaHintAlloc.free(unsafe.Pointer(hint))
|
||||
}
|
||||
- h.arenaHints = hint.next
|
||||
- h.arenaHintAlloc.free(unsafe.Pointer(hint))
|
||||
}
|
||||
|
||||
if size == 0 {
|
||||
diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
|
||||
index 058c7da..add4d3a 100644
|
||||
--- a/src/runtime/os_linux.go
|
||||
+++ b/src/runtime/os_linux.go
|
||||
@@ -185,6 +185,7 @@ const (
|
||||
_AT_HWCAP = 16 // hardware capability bit vector
|
||||
_AT_RANDOM = 25 // introduced in 2.6.29
|
||||
_AT_HWCAP2 = 26 // hardware capability bit vector 2
|
||||
+ _AT_OCCLUM = 48 // gnu syscall ABI entry address
|
||||
)
|
||||
|
||||
var procAuxv = []byte("/proc/self/auxv\x00")
|
||||
@@ -264,6 +265,9 @@ func sysauxv(auxv []uintptr) int {
|
||||
|
||||
case _AT_PAGESZ:
|
||||
physPageSize = val
|
||||
+
|
||||
+ case _AT_OCCLUM:
|
||||
+ occlumentry = val
|
||||
}
|
||||
|
||||
archauxv(tag, val)
|
||||
diff --git a/src/runtime/sys_linux_amd64.go b/src/runtime/sys_linux_amd64.go
|
||||
new file mode 100644
|
||||
index 0000000..c9369b6
|
||||
--- /dev/null
|
||||
+++ b/src/runtime/sys_linux_amd64.go
|
||||
@@ -0,0 +1,3 @@
|
||||
+package runtime
|
||||
+
|
||||
+var occlumentry uintptr = 0x0
|
||||
diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s
|
||||
index 37cb8da..2b85702 100644
|
||||
--- a/src/runtime/sys_linux_amd64.s
|
||||
+++ b/src/runtime/sys_linux_amd64.s
|
||||
@@ -53,7 +53,7 @@
|
||||
TEXT runtime·exit(SB),NOSPLIT,$0-4
|
||||
MOVL code+0(FP), DI
|
||||
MOVL $SYS_exit_group, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
// func exitThread(wait *uint32)
|
||||
@@ -63,7 +63,7 @@ TEXT runtime·exitThread(SB),NOSPLIT,$0-8
|
||||
MOVL $0, (AX)
|
||||
MOVL $0, DI // exit code
|
||||
MOVL $SYS_exit, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
// We may not even have a stack any more.
|
||||
INT $3
|
||||
JMP 0(PC)
|
||||
@@ -75,7 +75,7 @@ TEXT runtime·open(SB),NOSPLIT,$0-20
|
||||
MOVL mode+8(FP), DX
|
||||
MOVL perm+12(FP), R10
|
||||
MOVL $SYS_openat, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $-1, AX
|
||||
@@ -85,7 +85,7 @@ TEXT runtime·open(SB),NOSPLIT,$0-20
|
||||
TEXT runtime·closefd(SB),NOSPLIT,$0-12
|
||||
MOVL fd+0(FP), DI
|
||||
MOVL $SYS_close, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $-1, AX
|
||||
@@ -97,7 +97,7 @@ TEXT runtime·write1(SB),NOSPLIT,$0-28
|
||||
MOVQ p+8(FP), SI
|
||||
MOVL n+16(FP), DX
|
||||
MOVL $SYS_write, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -106,7 +106,7 @@ TEXT runtime·read(SB),NOSPLIT,$0-28
|
||||
MOVQ p+8(FP), SI
|
||||
MOVL n+16(FP), DX
|
||||
MOVL $SYS_read, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -114,7 +114,7 @@ TEXT runtime·read(SB),NOSPLIT,$0-28
|
||||
TEXT runtime·pipe(SB),NOSPLIT,$0-12
|
||||
LEAQ r+0(FP), DI
|
||||
MOVL $SYS_pipe, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, errno+8(FP)
|
||||
RET
|
||||
|
||||
@@ -123,7 +123,7 @@ TEXT runtime·pipe2(SB),NOSPLIT,$0-20
|
||||
LEAQ r+8(FP), DI
|
||||
MOVL flags+0(FP), SI
|
||||
MOVL $SYS_pipe2, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, errno+16(FP)
|
||||
RET
|
||||
|
||||
@@ -141,40 +141,40 @@ TEXT runtime·usleep(SB),NOSPLIT,$16
|
||||
MOVQ SP, DI
|
||||
MOVL $0, SI
|
||||
MOVL $SYS_nanosleep, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·gettid(SB),NOSPLIT,$0-4
|
||||
MOVL $SYS_gettid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+0(FP)
|
||||
RET
|
||||
|
||||
TEXT runtime·raise(SB),NOSPLIT,$0
|
||||
MOVL $SYS_getpid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, R12
|
||||
MOVL $SYS_gettid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, SI // arg 2 tid
|
||||
MOVL R12, DI // arg 1 pid
|
||||
MOVL sig+0(FP), DX // arg 3
|
||||
MOVL $SYS_tgkill, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·raiseproc(SB),NOSPLIT,$0
|
||||
MOVL $SYS_getpid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, DI // arg 1 pid
|
||||
MOVL sig+0(FP), SI // arg 2
|
||||
MOVL $SYS_kill, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT ·getpid(SB),NOSPLIT,$0-8
|
||||
MOVL $SYS_getpid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVQ AX, ret+0(FP)
|
||||
RET
|
||||
|
||||
@@ -183,7 +183,7 @@ TEXT ·tgkill(SB),NOSPLIT,$0
|
||||
MOVQ tid+8(FP), SI
|
||||
MOVQ sig+16(FP), DX
|
||||
MOVL $SYS_tgkill, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·setitimer(SB),NOSPLIT,$0-24
|
||||
@@ -191,7 +191,7 @@ TEXT runtime·setitimer(SB),NOSPLIT,$0-24
|
||||
MOVQ new+8(FP), SI
|
||||
MOVQ old+16(FP), DX
|
||||
MOVL $SYS_setittimer, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·mincore(SB),NOSPLIT,$0-28
|
||||
@@ -199,7 +199,7 @@ TEXT runtime·mincore(SB),NOSPLIT,$0-28
|
||||
MOVQ n+8(FP), SI
|
||||
MOVQ dst+16(FP), DX
|
||||
MOVL $SYS_mincore, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -266,7 +266,7 @@ ret:
|
||||
RET
|
||||
fallback:
|
||||
MOVQ $SYS_clock_gettime, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
JMP ret
|
||||
|
||||
// func nanotime1() int64
|
||||
@@ -329,7 +329,7 @@ ret:
|
||||
RET
|
||||
fallback:
|
||||
MOVQ $SYS_clock_gettime, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
JMP ret
|
||||
|
||||
TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
|
||||
@@ -338,7 +338,7 @@ TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
|
||||
MOVQ old+16(FP), DX
|
||||
MOVL size+24(FP), R10
|
||||
MOVL $SYS_rt_sigprocmask, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -350,7 +350,7 @@ TEXT runtime·rt_sigaction(SB),NOSPLIT,$0-36
|
||||
MOVQ old+16(FP), DX
|
||||
MOVQ size+24(FP), R10
|
||||
MOVL $SYS_rt_sigaction, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+32(FP)
|
||||
RET
|
||||
|
||||
@@ -491,7 +491,7 @@ sigtrampnog:
|
||||
// Defined as ABIInternal since it does not use the stack-based Go ABI.
|
||||
TEXT runtime·sigreturn<ABIInternal>(SB),NOSPLIT,$0
|
||||
MOVQ $SYS_rt_sigreturn, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
INT $3 // not reached
|
||||
|
||||
TEXT runtime·sysMmap(SB),NOSPLIT,$0
|
||||
@@ -503,7 +503,7 @@ TEXT runtime·sysMmap(SB),NOSPLIT,$0
|
||||
MOVL off+28(FP), R9
|
||||
|
||||
MOVL $SYS_mmap, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok
|
||||
NOTQ AX
|
||||
@@ -538,7 +538,7 @@ TEXT runtime·sysMunmap(SB),NOSPLIT,$0
|
||||
MOVQ addr+0(FP), DI
|
||||
MOVQ n+8(FP), SI
|
||||
MOVQ $SYS_munmap, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -562,7 +562,7 @@ TEXT runtime·madvise(SB),NOSPLIT,$0
|
||||
MOVQ n+8(FP), SI
|
||||
MOVL flags+16(FP), DX
|
||||
MOVQ $SYS_madvise, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -576,12 +576,15 @@ TEXT runtime·futex(SB),NOSPLIT,$0
|
||||
MOVQ addr2+24(FP), R8
|
||||
MOVL val3+32(FP), R9
|
||||
MOVL $SYS_futex, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+40(FP)
|
||||
RET
|
||||
|
||||
// int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void));
|
||||
TEXT runtime·clone(SB),NOSPLIT,$0
|
||||
+ CMPQ runtime·occlumentry(SB), $0x0
|
||||
+ JNE occlum
|
||||
+
|
||||
MOVL flags+0(FP), DI
|
||||
MOVQ stk+8(FP), SI
|
||||
MOVQ $0, DX
|
||||
@@ -644,11 +647,109 @@ nog2:
|
||||
SYSCALL
|
||||
JMP -3(PC) // keep exiting
|
||||
|
||||
+occlum:
|
||||
+ MOVL flags+0(FP), DI
|
||||
+ MOVQ stk+8(FP), SI
|
||||
+ MOVQ $0, DX
|
||||
+ MOVQ $0, R10
|
||||
+ MOVQ $0, R8
|
||||
+ // Copy mp, gp, fn off parent stack for use by child.
|
||||
+ // Careful: Linux system call clobbers CX and R11.
|
||||
+ MOVQ mp+16(FP), R13
|
||||
+ MOVQ gp+24(FP), R9
|
||||
+ MOVQ fn+32(FP), R12
|
||||
+ CMPQ R13, $0 // m
|
||||
+ JEQ occlum_nog1
|
||||
+ CMPQ R9, $0 // g
|
||||
+ JEQ occlum_nog1
|
||||
+ LEAQ m_tls(R13), R8
|
||||
+#ifdef GOOS_android
|
||||
+ // Android stores the TLS offset in runtime·tls_g.
|
||||
+ SUBQ runtime·tls_g(SB), R8
|
||||
+#else
|
||||
+ ADDQ $8, R8 // ELF wants to use -8(FS)
|
||||
+#endif
|
||||
+ ORQ $0x00080000, DI //add flag CLONE_SETTLS(0x00080000) to call clone
|
||||
+occlum_nog1:
|
||||
+ // flags
|
||||
+ MOVQ DI, -16(SI)
|
||||
+ // fn
|
||||
+ MOVQ R12, -24(SI)
|
||||
+ // gp
|
||||
+ MOVQ R9, -32(SI)
|
||||
+ // m
|
||||
+ MOVQ R13, -40(SI)
|
||||
+ // lea 0x23(%rip),%rax
|
||||
+ BYTE $0x48; BYTE $0x8d; BYTE $0x05; BYTE $0x23; BYTE $0x00; BYTE $0x00; BYTE $0x00
|
||||
+ // thread entry point
|
||||
+ MOVQ AX, -8(SI)
|
||||
+
|
||||
+ SUBQ $8, SI
|
||||
+
|
||||
+ MOVL $SYS_clone, AX
|
||||
+ // BYTE $0xcc
|
||||
+ // lea 0xa(%rip),%rcx
|
||||
+ BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0a; BYTE $0x00; BYTE $0x00; BYTE $0x00
|
||||
+ MOVQ runtime·occlumentry(SB), R11
|
||||
+ JMP R11
|
||||
+
|
||||
+ // In parent, return.
|
||||
+ MOVL AX, ret+40(FP)
|
||||
+ RET
|
||||
+
|
||||
+thread_entrypoint:
|
||||
+ // add 8, %rsp
|
||||
+ // MOVQ SI, SP
|
||||
+ ADDQ $8, SP
|
||||
+
|
||||
+ MOVQ SP, SI
|
||||
+
|
||||
+ // mov -24(%rsp), %r12
|
||||
+ // fn
|
||||
+ BYTE $0x4c; BYTE $0x8b; BYTE $0x64; BYTE $0x24; BYTE $0xe8
|
||||
+
|
||||
+ // mov -32(%rsp), %r9
|
||||
+ // gp
|
||||
+ BYTE $0x4c; BYTE $0x8b; BYTE $0x4c; BYTE $0x24; BYTE $0xe0
|
||||
+
|
||||
+ // mov -40(%rsp), %r13
|
||||
+ // m
|
||||
+ BYTE $0x4c; BYTE $0x8b; BYTE $0x6c; BYTE $0x24; BYTE $0xd8
|
||||
+
|
||||
+ // BYTE $0xcc
|
||||
+
|
||||
+ // If g or m are nil, skip Go-related setup.
|
||||
+ CMPQ R13, $0 // m
|
||||
+ JEQ occlum_nog2
|
||||
+ CMPQ R9, $0 // g
|
||||
+ JEQ occlum_nog2
|
||||
+
|
||||
+ // Initialize m->procid to Linux tid
|
||||
+ MOVL $SYS_gettid, AX
|
||||
+ SYSCALL_ENHANCE
|
||||
+ MOVQ AX, m_procid(R13)
|
||||
+
|
||||
+ // In child, set up new stack
|
||||
+ get_tls(CX)
|
||||
+ MOVQ R13, g_m(R9)
|
||||
+ MOVQ R9, g(CX)
|
||||
+ CALL runtime·stackcheck(SB)
|
||||
+
|
||||
+occlum_nog2:
|
||||
+ // Call fn
|
||||
+ CALL R12
|
||||
+
|
||||
+ // It shouldn't return. If it does, exit that thread.
|
||||
+ MOVL $111, DI
|
||||
+ MOVL $SYS_exit, AX
|
||||
+ SYSCALL_ENHANCE
|
||||
+ JMP -3(PC) // keep exiting
|
||||
+
|
||||
TEXT runtime·sigaltstack(SB),NOSPLIT,$-8
|
||||
MOVQ new+0(FP), DI
|
||||
MOVQ old+8(FP), SI
|
||||
MOVQ $SYS_sigaltstack, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -665,7 +766,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32
|
||||
MOVQ DI, SI
|
||||
MOVQ $0x1002, DI // ARCH_SET_FS
|
||||
MOVQ $SYS_arch_prctl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -673,7 +774,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32
|
||||
|
||||
TEXT runtime·osyield(SB),NOSPLIT,$0
|
||||
MOVL $SYS_sched_yield, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
|
||||
@@ -681,7 +782,7 @@ TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
|
||||
MOVQ len+8(FP), SI
|
||||
MOVQ buf+16(FP), DX
|
||||
MOVL $SYS_sched_getaffinity, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -689,7 +790,7 @@ TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
|
||||
TEXT runtime·epollcreate(SB),NOSPLIT,$0
|
||||
MOVL size+0(FP), DI
|
||||
MOVL $SYS_epoll_create, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
@@ -697,7 +798,7 @@ TEXT runtime·epollcreate(SB),NOSPLIT,$0
|
||||
TEXT runtime·epollcreate1(SB),NOSPLIT,$0
|
||||
MOVL flags+0(FP), DI
|
||||
MOVL $SYS_epoll_create1, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
@@ -708,7 +809,7 @@ TEXT runtime·epollctl(SB),NOSPLIT,$0
|
||||
MOVL fd+8(FP), DX
|
||||
MOVQ ev+16(FP), R10
|
||||
MOVL $SYS_epoll_ctl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -721,7 +822,7 @@ TEXT runtime·epollwait(SB),NOSPLIT,$0
|
||||
MOVL timeout+20(FP), R10
|
||||
MOVQ $0, R8
|
||||
MOVL $SYS_epoll_pwait, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -731,7 +832,7 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0
|
||||
MOVQ $2, SI // F_SETFD
|
||||
MOVQ $1, DX // FD_CLOEXEC
|
||||
MOVL $SYS_fcntl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
// func runtime·setNonblock(int32 fd)
|
||||
@@ -740,13 +841,13 @@ TEXT runtime·setNonblock(SB),NOSPLIT,$0-4
|
||||
MOVQ $3, SI // F_GETFL
|
||||
MOVQ $0, DX
|
||||
MOVL $SYS_fcntl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL fd+0(FP), DI // fd
|
||||
MOVQ $4, SI // F_SETFL
|
||||
MOVQ $0x800, DX // O_NONBLOCK
|
||||
ORL AX, DX
|
||||
MOVL $SYS_fcntl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
// int access(const char *name, int mode)
|
||||
@@ -757,7 +858,7 @@ TEXT runtime·access(SB),NOSPLIT,$0
|
||||
MOVL mode+8(FP), DX
|
||||
MOVL $0, R10
|
||||
MOVL $SYS_faccessat, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
@@ -767,7 +868,7 @@ TEXT runtime·connect(SB),NOSPLIT,$0-28
|
||||
MOVQ addr+8(FP), SI
|
||||
MOVL len+16(FP), DX
|
||||
MOVL $SYS_connect, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -777,7 +878,7 @@ TEXT runtime·socket(SB),NOSPLIT,$0-20
|
||||
MOVL typ+4(FP), SI
|
||||
MOVL prot+8(FP), DX
|
||||
MOVL $SYS_socket, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
@@ -786,6 +887,6 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8
|
||||
// Implemented as brk(NULL).
|
||||
MOVQ $0, DI
|
||||
MOVL $SYS_brk, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVQ AX, ret+0(FP)
|
||||
RET
|
||||
diff --git a/src/runtime/textflag.h b/src/runtime/textflag.h
|
||||
index daca36d..c4056c0 100644
|
||||
--- a/src/runtime/textflag.h
|
||||
+++ b/src/runtime/textflag.h
|
||||
@@ -35,3 +35,17 @@
|
||||
// Function is the top of the call stack. Call stack unwinders should stop
|
||||
// at this function.
|
||||
#define TOPFRAME 2048
|
||||
+
|
||||
+// step one: assign syscall return address to register rcx, occlum know where
|
||||
+// should return according to rcx.
|
||||
+// step two: jump to syscall interface address provided by occlum when go
|
||||
+// .bin file loaded.
|
||||
+// <BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0c; BYTE $0x00; BYTE $0x00; BYTE $0x00>
|
||||
+// actually is the assembler instruction: lea 0xc(%rip),%rcx
|
||||
+#define SYSCALL_ENHANCE \
|
||||
+ CMPQ runtime·occlumentry(SB), $0x0 \
|
||||
+ JBE 10(PC) \
|
||||
+ BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0c; BYTE $0x00; BYTE $0x00; BYTE $0x00 \
|
||||
+ MOVQ runtime·occlumentry(SB), R11 \
|
||||
+ JMP R11 \
|
||||
+ SYSCALL
|
||||
diff --git a/src/syscall/asm_linux_amd64.s b/src/syscall/asm_linux_amd64.s
|
||||
index ba22179..18cb535 100644
|
||||
--- a/src/syscall/asm_linux_amd64.s
|
||||
+++ b/src/syscall/asm_linux_amd64.s
|
||||
@@ -22,7 +22,7 @@ TEXT ·Syscall(SB),NOSPLIT,$0-56
|
||||
MOVQ a2+16(FP), SI
|
||||
MOVQ a3+24(FP), DX
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok
|
||||
MOVQ $-1, r1+32(FP)
|
||||
@@ -48,7 +48,7 @@ TEXT ·Syscall6(SB),NOSPLIT,$0-80
|
||||
MOVQ a5+40(FP), R8
|
||||
MOVQ a6+48(FP), R9
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok6
|
||||
MOVQ $-1, r1+56(FP)
|
||||
@@ -70,7 +70,7 @@ TEXT ·RawSyscall(SB),NOSPLIT,$0-56
|
||||
MOVQ a2+16(FP), SI
|
||||
MOVQ a3+24(FP), DX
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok1
|
||||
MOVQ $-1, r1+32(FP)
|
||||
@@ -93,7 +93,7 @@ TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
|
||||
MOVQ a5+40(FP), R8
|
||||
MOVQ a6+48(FP), R9
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok2
|
||||
MOVQ $-1, r1+56(FP)
|
||||
@@ -117,7 +117,7 @@ TEXT ·rawVforkSyscall(SB),NOSPLIT,$0-32
|
||||
MOVQ $0, R9
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
POPQ R12 // preserve return address
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
PUSHQ R12
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok2
|
||||
@@ -136,7 +136,7 @@ TEXT ·rawSyscallNoError(SB),NOSPLIT,$0-48
|
||||
MOVQ a2+16(FP), SI
|
||||
MOVQ a3+24(FP), DX
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVQ AX, r1+32(FP)
|
||||
MOVQ DX, r2+40(FP)
|
||||
RET
|
||||
@@ -157,7 +157,7 @@ ret:
|
||||
RET
|
||||
fallback:
|
||||
MOVL $SYS_gettimeofday, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
JMP ret
|
||||
ok7:
|
||||
MOVQ $0, err+8(FP)
|
||||
--
|
||||
libgit2 0.23.3
|
||||
|
@ -1,721 +0,0 @@
|
||||
From 59017c2fa01ed4764225fdbfed7a17287f81d87a Mon Sep 17 00:00:00 2001
|
||||
From: Lan Yun <yun.lan@intel.com>
|
||||
Date: Thu, 28 Jul 2022 11:00:15 +0800
|
||||
Subject: [PATCH] occlum-go-patch
|
||||
|
||||
Signed-off-by: Lan Yun <yun.lan@intel.com>
|
||||
---
|
||||
src/runtime/malloc.go | 70 ++++++------
|
||||
src/runtime/os_linux.go | 4 +
|
||||
src/runtime/sys_linux_amd64.go | 3 +
|
||||
src/runtime/sys_linux_amd64.s | 192 +++++++++++++++++++++++++--------
|
||||
src/runtime/textflag.h | 18 +++-
|
||||
src/syscall/asm_linux_amd64.s | 14 +--
|
||||
6 files changed, 212 insertions(+), 89 deletions(-)
|
||||
create mode 100644 src/runtime/sys_linux_amd64.go
|
||||
|
||||
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
|
||||
index 6ed6ceade2..2a94401c0c 100644
|
||||
--- a/src/runtime/malloc.go
|
||||
+++ b/src/runtime/malloc.go
|
||||
@@ -645,42 +645,44 @@ func (h *mheap) sysAlloc(n uintptr) (v unsafe.Pointer, size uintptr) {
|
||||
goto mapped
|
||||
}
|
||||
|
||||
- // Try to grow the heap at a hint address.
|
||||
- for h.arenaHints != nil {
|
||||
- hint := h.arenaHints
|
||||
- p := hint.addr
|
||||
- if hint.down {
|
||||
- p -= n
|
||||
- }
|
||||
- if p+n < p {
|
||||
- // We can't use this, so don't ask.
|
||||
- v = nil
|
||||
- } else if arenaIndex(p+n-1) >= 1<<arenaBits {
|
||||
- // Outside addressable heap. Can't use.
|
||||
- v = nil
|
||||
- } else {
|
||||
- v = sysReserve(unsafe.Pointer(p), n)
|
||||
- }
|
||||
- if p == uintptr(v) {
|
||||
- // Success. Update the hint.
|
||||
- if !hint.down {
|
||||
- p += n
|
||||
+ if occlumentry == 0 {
|
||||
+ // Try to grow the heap at a hint address.
|
||||
+ for h.arenaHints != nil {
|
||||
+ hint := h.arenaHints
|
||||
+ p := hint.addr
|
||||
+ if hint.down {
|
||||
+ p -= n
|
||||
}
|
||||
- hint.addr = p
|
||||
- size = n
|
||||
- break
|
||||
- }
|
||||
- // Failed. Discard this hint and try the next.
|
||||
- //
|
||||
- // TODO: This would be cleaner if sysReserve could be
|
||||
- // told to only return the requested address. In
|
||||
- // particular, this is already how Windows behaves, so
|
||||
- // it would simplify things there.
|
||||
- if v != nil {
|
||||
- sysFree(v, n, nil)
|
||||
+ if p+n < p {
|
||||
+ // We can't use this, so don't ask.
|
||||
+ v = nil
|
||||
+ } else if arenaIndex(p+n-1) >= 1<<arenaBits {
|
||||
+ // Outside addressable heap. Can't use.
|
||||
+ v = nil
|
||||
+ } else {
|
||||
+ v = sysReserve(unsafe.Pointer(p), n)
|
||||
+ }
|
||||
+ if p == uintptr(v) {
|
||||
+ // Success. Update the hint.
|
||||
+ if !hint.down {
|
||||
+ p += n
|
||||
+ }
|
||||
+ hint.addr = p
|
||||
+ size = n
|
||||
+ break
|
||||
+ }
|
||||
+ // Failed. Discard this hint and try the next.
|
||||
+ //
|
||||
+ // TODO: This would be cleaner if sysReserve could be
|
||||
+ // told to only return the requested address. In
|
||||
+ // particular, this is already how Windows behaves, so
|
||||
+ // it would simplify things there.
|
||||
+ if v != nil {
|
||||
+ sysFree(v, n, nil)
|
||||
+ }
|
||||
+ h.arenaHints = hint.next
|
||||
+ h.arenaHintAlloc.free(unsafe.Pointer(hint))
|
||||
}
|
||||
- h.arenaHints = hint.next
|
||||
- h.arenaHintAlloc.free(unsafe.Pointer(hint))
|
||||
}
|
||||
|
||||
if size == 0 {
|
||||
diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
|
||||
index eb8aa076e9..b65751ce65 100644
|
||||
--- a/src/runtime/os_linux.go
|
||||
+++ b/src/runtime/os_linux.go
|
||||
@@ -207,6 +207,7 @@ const (
|
||||
_AT_HWCAP = 16 // hardware capability bit vector
|
||||
_AT_RANDOM = 25 // introduced in 2.6.29
|
||||
_AT_HWCAP2 = 26 // hardware capability bit vector 2
|
||||
+ _AT_OCCLUM = 48 // gnu syscall ABI entry address
|
||||
)
|
||||
|
||||
var procAuxv = []byte("/proc/self/auxv\x00")
|
||||
@@ -286,6 +287,9 @@ func sysauxv(auxv []uintptr) int {
|
||||
|
||||
case _AT_PAGESZ:
|
||||
physPageSize = val
|
||||
+
|
||||
+ case _AT_OCCLUM:
|
||||
+ occlumentry = val
|
||||
}
|
||||
|
||||
archauxv(tag, val)
|
||||
diff --git a/src/runtime/sys_linux_amd64.go b/src/runtime/sys_linux_amd64.go
|
||||
new file mode 100644
|
||||
index 0000000000..c9369b6661
|
||||
--- /dev/null
|
||||
+++ b/src/runtime/sys_linux_amd64.go
|
||||
@@ -0,0 +1,3 @@
|
||||
+package runtime
|
||||
+
|
||||
+var occlumentry uintptr = 0x0
|
||||
diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s
|
||||
index f0e58e11db..5cc653bf8a 100644
|
||||
--- a/src/runtime/sys_linux_amd64.s
|
||||
+++ b/src/runtime/sys_linux_amd64.s
|
||||
@@ -57,7 +57,7 @@
|
||||
TEXT runtime·exit(SB),NOSPLIT,$0-4
|
||||
MOVL code+0(FP), DI
|
||||
MOVL $SYS_exit_group, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
// func exitThread(wait *uint32)
|
||||
@@ -67,7 +67,7 @@ TEXT runtime·exitThread(SB),NOSPLIT,$0-8
|
||||
MOVL $0, (AX)
|
||||
MOVL $0, DI // exit code
|
||||
MOVL $SYS_exit, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
// We may not even have a stack any more.
|
||||
INT $3
|
||||
JMP 0(PC)
|
||||
@@ -79,7 +79,7 @@ TEXT runtime·open(SB),NOSPLIT,$0-20
|
||||
MOVL mode+8(FP), DX
|
||||
MOVL perm+12(FP), R10
|
||||
MOVL $SYS_openat, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $-1, AX
|
||||
@@ -89,7 +89,7 @@ TEXT runtime·open(SB),NOSPLIT,$0-20
|
||||
TEXT runtime·closefd(SB),NOSPLIT,$0-12
|
||||
MOVL fd+0(FP), DI
|
||||
MOVL $SYS_close, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $-1, AX
|
||||
@@ -101,7 +101,7 @@ TEXT runtime·write1(SB),NOSPLIT,$0-28
|
||||
MOVQ p+8(FP), SI
|
||||
MOVL n+16(FP), DX
|
||||
MOVL $SYS_write, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -110,7 +110,7 @@ TEXT runtime·read(SB),NOSPLIT,$0-28
|
||||
MOVQ p+8(FP), SI
|
||||
MOVL n+16(FP), DX
|
||||
MOVL $SYS_read, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -118,7 +118,7 @@ TEXT runtime·read(SB),NOSPLIT,$0-28
|
||||
TEXT runtime·pipe(SB),NOSPLIT,$0-12
|
||||
LEAQ r+0(FP), DI
|
||||
MOVL $SYS_pipe, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, errno+8(FP)
|
||||
RET
|
||||
|
||||
@@ -127,7 +127,7 @@ TEXT runtime·pipe2(SB),NOSPLIT,$0-20
|
||||
LEAQ r+8(FP), DI
|
||||
MOVL flags+0(FP), SI
|
||||
MOVL $SYS_pipe2, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, errno+16(FP)
|
||||
RET
|
||||
|
||||
@@ -145,40 +145,40 @@ TEXT runtime·usleep(SB),NOSPLIT,$16
|
||||
MOVQ SP, DI
|
||||
MOVL $0, SI
|
||||
MOVL $SYS_nanosleep, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·gettid(SB),NOSPLIT,$0-4
|
||||
MOVL $SYS_gettid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+0(FP)
|
||||
RET
|
||||
|
||||
TEXT runtime·raise(SB),NOSPLIT,$0
|
||||
MOVL $SYS_getpid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, R12
|
||||
MOVL $SYS_gettid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, SI // arg 2 tid
|
||||
MOVL R12, DI // arg 1 pid
|
||||
MOVL sig+0(FP), DX // arg 3
|
||||
MOVL $SYS_tgkill, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·raiseproc(SB),NOSPLIT,$0
|
||||
MOVL $SYS_getpid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, DI // arg 1 pid
|
||||
MOVL sig+0(FP), SI // arg 2
|
||||
MOVL $SYS_kill, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT ·getpid(SB),NOSPLIT,$0-8
|
||||
MOVL $SYS_getpid, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVQ AX, ret+0(FP)
|
||||
RET
|
||||
|
||||
@@ -187,7 +187,7 @@ TEXT ·tgkill(SB),NOSPLIT,$0
|
||||
MOVQ tid+8(FP), SI
|
||||
MOVQ sig+16(FP), DX
|
||||
MOVL $SYS_tgkill, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·setitimer(SB),NOSPLIT,$0-24
|
||||
@@ -195,7 +195,7 @@ TEXT runtime·setitimer(SB),NOSPLIT,$0-24
|
||||
MOVQ new+8(FP), SI
|
||||
MOVQ old+16(FP), DX
|
||||
MOVL $SYS_setittimer, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·timer_create(SB),NOSPLIT,$0-28
|
||||
@@ -203,7 +203,7 @@ TEXT runtime·timer_create(SB),NOSPLIT,$0-28
|
||||
MOVQ sevp+8(FP), SI
|
||||
MOVQ timerid+16(FP), DX
|
||||
MOVL $SYS_timer_create, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -213,14 +213,14 @@ TEXT runtime·timer_settime(SB),NOSPLIT,$0-28
|
||||
MOVQ new+8(FP), DX
|
||||
MOVQ old+16(FP), R10
|
||||
MOVL $SYS_timer_settime, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
TEXT runtime·timer_delete(SB),NOSPLIT,$0-12
|
||||
MOVL timerid+0(FP), DI
|
||||
MOVL $SYS_timer_delete, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
@@ -229,7 +229,7 @@ TEXT runtime·mincore(SB),NOSPLIT,$0-28
|
||||
MOVQ n+8(FP), SI
|
||||
MOVQ dst+16(FP), DX
|
||||
MOVL $SYS_mincore, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -296,7 +296,7 @@ ret:
|
||||
RET
|
||||
fallback:
|
||||
MOVQ $SYS_clock_gettime, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
JMP ret
|
||||
|
||||
TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
|
||||
@@ -305,7 +305,7 @@ TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
|
||||
MOVQ old+16(FP), DX
|
||||
MOVL size+24(FP), R10
|
||||
MOVL $SYS_rt_sigprocmask, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -317,7 +317,7 @@ TEXT runtime·rt_sigaction(SB),NOSPLIT,$0-36
|
||||
MOVQ old+16(FP), DX
|
||||
MOVQ size+24(FP), R10
|
||||
MOVL $SYS_rt_sigaction, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+32(FP)
|
||||
RET
|
||||
|
||||
@@ -464,7 +464,7 @@ sigtrampnog:
|
||||
// https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup
|
||||
TEXT runtime·sigreturn(SB),NOSPLIT,$0
|
||||
MOVQ $SYS_rt_sigreturn, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
INT $3 // not reached
|
||||
|
||||
TEXT runtime·sysMmap(SB),NOSPLIT,$0
|
||||
@@ -476,7 +476,7 @@ TEXT runtime·sysMmap(SB),NOSPLIT,$0
|
||||
MOVL off+28(FP), R9
|
||||
|
||||
MOVL $SYS_mmap, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok
|
||||
NOTQ AX
|
||||
@@ -511,7 +511,7 @@ TEXT runtime·sysMunmap(SB),NOSPLIT,$0
|
||||
MOVQ addr+0(FP), DI
|
||||
MOVQ n+8(FP), SI
|
||||
MOVQ $SYS_munmap, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -535,7 +535,7 @@ TEXT runtime·madvise(SB),NOSPLIT,$0
|
||||
MOVQ n+8(FP), SI
|
||||
MOVL flags+16(FP), DX
|
||||
MOVQ $SYS_madvise, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -549,12 +549,15 @@ TEXT runtime·futex(SB),NOSPLIT,$0
|
||||
MOVQ addr2+24(FP), R8
|
||||
MOVL val3+32(FP), R9
|
||||
MOVL $SYS_futex, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+40(FP)
|
||||
RET
|
||||
|
||||
// int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void));
|
||||
TEXT runtime·clone(SB),NOSPLIT,$0
|
||||
+ CMPQ runtime·occlumentry(SB), $0x0
|
||||
+ JNE occlum
|
||||
+
|
||||
MOVL flags+0(FP), DI
|
||||
MOVQ stk+8(FP), SI
|
||||
MOVQ $0, DX
|
||||
@@ -618,11 +621,110 @@ nog2:
|
||||
SYSCALL
|
||||
JMP -3(PC) // keep exiting
|
||||
|
||||
+occlum:
|
||||
+ MOVL flags+0(FP), DI
|
||||
+ MOVQ stk+8(FP), SI
|
||||
+ MOVQ $0, DX
|
||||
+ MOVQ $0, R10
|
||||
+ MOVQ $0, R8
|
||||
+ // Copy mp, gp, fn off parent stack for use by child.
|
||||
+ // Careful: Linux system call clobbers CX and R11.
|
||||
+ MOVQ mp+16(FP), R13
|
||||
+ MOVQ gp+24(FP), R9
|
||||
+ MOVQ fn+32(FP), R12
|
||||
+ CMPQ R13, $0 // m
|
||||
+ JEQ occlum_nog1
|
||||
+ CMPQ R9, $0 // g
|
||||
+ JEQ occlum_nog1
|
||||
+ LEAQ m_tls(R13), R8
|
||||
+#ifdef GOOS_android
|
||||
+ // Android stores the TLS offset in runtime·tls_g.
|
||||
+ SUBQ runtime·tls_g(SB), R8
|
||||
+#else
|
||||
+ ADDQ $8, R8 // ELF wants to use -8(FS)
|
||||
+#endif
|
||||
+ ORQ $0x00080000, DI //add flag CLONE_SETTLS(0x00080000) to call clone
|
||||
+occlum_nog1:
|
||||
+ // flags
|
||||
+ MOVQ DI, -16(SI)
|
||||
+ // fn
|
||||
+ MOVQ R12, -24(SI)
|
||||
+ // gp
|
||||
+ MOVQ R9, -32(SI)
|
||||
+ // m
|
||||
+ MOVQ R13, -40(SI)
|
||||
+ // lea 0x23(%rip),%rax
|
||||
+ BYTE $0x48; BYTE $0x8d; BYTE $0x05; BYTE $0x23; BYTE $0x00; BYTE $0x00; BYTE $0x00
|
||||
+ // thread entry point
|
||||
+ MOVQ AX, -8(SI)
|
||||
+
|
||||
+ SUBQ $8, SI
|
||||
+
|
||||
+ MOVL $SYS_clone, AX
|
||||
+ // BYTE $0xcc
|
||||
+ // lea 0xa(%rip),%rcx
|
||||
+ BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0a; BYTE $0x00; BYTE $0x00; BYTE $0x00
|
||||
+ MOVQ runtime·occlumentry(SB), R11
|
||||
+ JMP R11
|
||||
+
|
||||
+ // In parent, return.
|
||||
+ MOVL AX, ret+40(FP)
|
||||
+ RET
|
||||
+
|
||||
+thread_entrypoint:
|
||||
+ // add 8, %rsp
|
||||
+ // MOVQ SI, SP
|
||||
+ ADDQ $8, SP
|
||||
+
|
||||
+ MOVQ SP, SI
|
||||
+
|
||||
+ // mov -24(%rsp), %r12
|
||||
+ // fn
|
||||
+ BYTE $0x4c; BYTE $0x8b; BYTE $0x64; BYTE $0x24; BYTE $0xe8
|
||||
+
|
||||
+ // mov -32(%rsp), %r9
|
||||
+ // gp
|
||||
+ BYTE $0x4c; BYTE $0x8b; BYTE $0x4c; BYTE $0x24; BYTE $0xe0
|
||||
+
|
||||
+ // mov -40(%rsp), %r13
|
||||
+ // m
|
||||
+ BYTE $0x4c; BYTE $0x8b; BYTE $0x6c; BYTE $0x24; BYTE $0xd8
|
||||
+
|
||||
+ // BYTE $0xcc
|
||||
+
|
||||
+ // If g or m are nil, skip Go-related setup.
|
||||
+ CMPQ R13, $0 // m
|
||||
+ JEQ occlum_nog2
|
||||
+ CMPQ R9, $0 // g
|
||||
+ JEQ occlum_nog2
|
||||
+
|
||||
+ // Initialize m->procid to Linux tid
|
||||
+ MOVL $SYS_gettid, AX
|
||||
+ SYSCALL_ENHANCE
|
||||
+ MOVQ AX, m_procid(R13)
|
||||
+
|
||||
+ // In child, set up new stack
|
||||
+ get_tls(CX)
|
||||
+ MOVQ R13, g_m(R9)
|
||||
+ MOVQ R9, g(CX)
|
||||
+ MOVQ R9, R14 // set g register
|
||||
+ CALL runtime·stackcheck(SB)
|
||||
+
|
||||
+occlum_nog2:
|
||||
+ // Call fn
|
||||
+ CALL R12
|
||||
+
|
||||
+ // It shouldn't return. If it does, exit that thread.
|
||||
+ MOVL $111, DI
|
||||
+ MOVL $SYS_exit, AX
|
||||
+ SYSCALL_ENHANCE
|
||||
+ JMP -3(PC) // keep exiting
|
||||
+
|
||||
TEXT runtime·sigaltstack(SB),NOSPLIT,$-8
|
||||
MOVQ new+0(FP), DI
|
||||
MOVQ old+8(FP), SI
|
||||
MOVQ $SYS_sigaltstack, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -639,7 +741,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32
|
||||
MOVQ DI, SI
|
||||
MOVQ $0x1002, DI // ARCH_SET_FS
|
||||
MOVQ $SYS_arch_prctl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS 2(PC)
|
||||
MOVL $0xf1, 0xf1 // crash
|
||||
@@ -647,7 +749,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32
|
||||
|
||||
TEXT runtime·osyield(SB),NOSPLIT,$0
|
||||
MOVL $SYS_sched_yield, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
|
||||
@@ -655,7 +757,7 @@ TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
|
||||
MOVQ len+8(FP), SI
|
||||
MOVQ buf+16(FP), DX
|
||||
MOVL $SYS_sched_getaffinity, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -663,7 +765,7 @@ TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
|
||||
TEXT runtime·epollcreate(SB),NOSPLIT,$0
|
||||
MOVL size+0(FP), DI
|
||||
MOVL $SYS_epoll_create, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
@@ -671,7 +773,7 @@ TEXT runtime·epollcreate(SB),NOSPLIT,$0
|
||||
TEXT runtime·epollcreate1(SB),NOSPLIT,$0
|
||||
MOVL flags+0(FP), DI
|
||||
MOVL $SYS_epoll_create1, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
@@ -682,7 +784,7 @@ TEXT runtime·epollctl(SB),NOSPLIT,$0
|
||||
MOVL fd+8(FP), DX
|
||||
MOVQ ev+16(FP), R10
|
||||
MOVL $SYS_epoll_ctl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -695,7 +797,7 @@ TEXT runtime·epollwait(SB),NOSPLIT,$0
|
||||
MOVL timeout+20(FP), R10
|
||||
MOVQ $0, R8
|
||||
MOVL $SYS_epoll_pwait, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -705,7 +807,7 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0
|
||||
MOVQ $2, SI // F_SETFD
|
||||
MOVQ $1, DX // FD_CLOEXEC
|
||||
MOVL $SYS_fcntl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
// func runtime·setNonblock(int32 fd)
|
||||
@@ -714,13 +816,13 @@ TEXT runtime·setNonblock(SB),NOSPLIT,$0-4
|
||||
MOVQ $3, SI // F_GETFL
|
||||
MOVQ $0, DX
|
||||
MOVL $SYS_fcntl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL fd+0(FP), DI // fd
|
||||
MOVQ $4, SI // F_SETFL
|
||||
MOVQ $0x800, DX // O_NONBLOCK
|
||||
ORL AX, DX
|
||||
MOVL $SYS_fcntl, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
RET
|
||||
|
||||
// int access(const char *name, int mode)
|
||||
@@ -731,7 +833,7 @@ TEXT runtime·access(SB),NOSPLIT,$0
|
||||
MOVL mode+8(FP), DX
|
||||
MOVL $0, R10
|
||||
MOVL $SYS_faccessat, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
@@ -741,7 +843,7 @@ TEXT runtime·connect(SB),NOSPLIT,$0-28
|
||||
MOVQ addr+8(FP), SI
|
||||
MOVL len+16(FP), DX
|
||||
MOVL $SYS_connect, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
@@ -751,7 +853,7 @@ TEXT runtime·socket(SB),NOSPLIT,$0-20
|
||||
MOVL typ+4(FP), SI
|
||||
MOVL prot+8(FP), DX
|
||||
MOVL $SYS_socket, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVL AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
@@ -760,6 +862,6 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8
|
||||
// Implemented as brk(NULL).
|
||||
MOVQ $0, DI
|
||||
MOVL $SYS_brk, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVQ AX, ret+0(FP)
|
||||
RET
|
||||
diff --git a/src/runtime/textflag.h b/src/runtime/textflag.h
|
||||
index 214075e360..4b7da24e64 100644
|
||||
--- a/src/runtime/textflag.h
|
||||
+++ b/src/runtime/textflag.h
|
||||
@@ -32,8 +32,20 @@
|
||||
#define NOFRAME 512
|
||||
// Function can call reflect.Type.Method or reflect.Type.MethodByName.
|
||||
#define REFLECTMETHOD 1024
|
||||
-// Function is the outermost frame of the call stack. Call stack unwinders
|
||||
-// should stop at this function.
|
||||
+// Function is the top of the call stack. Call stack unwinders should stop
|
||||
+// at this function.
|
||||
#define TOPFRAME 2048
|
||||
-// Function is an ABI wrapper.
|
||||
#define ABIWRAPPER 4096
|
||||
+// step one: assign syscall return address to register rcx, occlum know where
|
||||
+// should return according to rcx.
|
||||
+// step two: jump to syscall interface address provided by occlum when go
|
||||
+// .bin file loaded.
|
||||
+// <BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0c; BYTE $0x00; BYTE $0x00; BYTE $0x00>
|
||||
+// actually is the assembler instruction: lea 0xc(%rip),%rcx
|
||||
+#define SYSCALL_ENHANCE \
|
||||
+ CMPQ runtime·occlumentry(SB), $0x0 \
|
||||
+ JBE 10(PC) \
|
||||
+ BYTE $0x48; BYTE $0x8d; BYTE $0x0d; BYTE $0x0c; BYTE $0x00; BYTE $0x00; BYTE $0x00 \
|
||||
+ MOVQ runtime·occlumentry(SB), R11 \
|
||||
+ JMP R11 \
|
||||
+ SYSCALL
|
||||
diff --git a/src/syscall/asm_linux_amd64.s b/src/syscall/asm_linux_amd64.s
|
||||
index a9af68d51d..5110d78bb5 100644
|
||||
--- a/src/syscall/asm_linux_amd64.s
|
||||
+++ b/src/syscall/asm_linux_amd64.s
|
||||
@@ -22,7 +22,7 @@ TEXT ·Syscall(SB),NOSPLIT,$0-56
|
||||
MOVQ a2+16(FP), SI
|
||||
MOVQ a3+24(FP), DX
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok
|
||||
MOVQ $-1, r1+32(FP)
|
||||
@@ -48,7 +48,7 @@ TEXT ·Syscall6(SB),NOSPLIT,$0-80
|
||||
MOVQ a5+40(FP), R8
|
||||
MOVQ a6+48(FP), R9
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok6
|
||||
MOVQ $-1, r1+56(FP)
|
||||
@@ -70,7 +70,7 @@ TEXT ·RawSyscall(SB),NOSPLIT,$0-56
|
||||
MOVQ a2+16(FP), SI
|
||||
MOVQ a3+24(FP), DX
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok1
|
||||
MOVQ $-1, r1+32(FP)
|
||||
@@ -93,7 +93,7 @@ TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
|
||||
MOVQ a5+40(FP), R8
|
||||
MOVQ a6+48(FP), R9
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok2
|
||||
MOVQ $-1, r1+56(FP)
|
||||
@@ -117,7 +117,7 @@ TEXT ·rawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-32
|
||||
MOVQ $0, R9
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
POPQ R12 // preserve return address
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
PUSHQ R12
|
||||
CMPQ AX, $0xfffffffffffff001
|
||||
JLS ok2
|
||||
@@ -136,7 +136,7 @@ TEXT ·rawSyscallNoError(SB),NOSPLIT,$0-48
|
||||
MOVQ a2+16(FP), SI
|
||||
MOVQ a3+24(FP), DX
|
||||
MOVQ trap+0(FP), AX // syscall entry
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
MOVQ AX, r1+32(FP)
|
||||
MOVQ DX, r2+40(FP)
|
||||
RET
|
||||
@@ -157,7 +157,7 @@ ret:
|
||||
RET
|
||||
fallback:
|
||||
MOVL $SYS_gettimeofday, AX
|
||||
- SYSCALL
|
||||
+ SYSCALL_ENHANCE
|
||||
JMP ret
|
||||
ok7:
|
||||
MOVQ $0, err+8(FP)
|
||||
--
|
||||
2.25.1
|
||||
|
@ -2,22 +2,9 @@
|
||||
THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
BUILD_DIR=/tmp/occlum_golang_toolchain
|
||||
INSTALL_DIR=/opt/occlum/toolchains/golang
|
||||
GO_VERSION="1.18.4"
|
||||
|
||||
if [[ $# -ge 1 ]];then
|
||||
case $1 in
|
||||
--help|help)
|
||||
echo "Usage:$0 [go1.16.3|go1.18.4]"
|
||||
exit
|
||||
;;
|
||||
"go1.16.3"|"1.16.3"|"v1.16.3")
|
||||
GO_VERSION="1.16.3"
|
||||
;;
|
||||
"go1.18.4"|"1.18.4"|"v1.18.4")
|
||||
GO_VERSION="1.18.4"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
#go1.18.4_for_occlum or go1.16.3_for_occlum
|
||||
GO_BRANCH=${1:-"go1.18.4_for_occlum"}
|
||||
|
||||
# Exit if any command fails
|
||||
set -e
|
||||
@ -31,11 +18,7 @@ mkdir -p ${BUILD_DIR}
|
||||
cd ${BUILD_DIR}
|
||||
|
||||
# Download Golang
|
||||
git clone https://github.com/golang/go .
|
||||
# Swtich to Golang 1.16.3 or Golang 1.18.4
|
||||
git checkout -b go${GO_VERSION} tags/go${GO_VERSION}
|
||||
# Apply the patch to adapt Golang to Occlum
|
||||
git apply ${THIS_DIR}/adapt-golang${GO_VERSION}-to-occlum.patch
|
||||
git clone -b ${GO_BRANCH} https://github.com/occlum/go.git .
|
||||
|
||||
# Build Golang
|
||||
cd src
|
||||
|
Loading…
Reference in New Issue
Block a user