Text file
src/runtime/asm_s390x.s
Documentation: runtime
1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10// _rt0_s390x_lib is common startup code for s390x systems when
11// using -buildmode=c-archive or -buildmode=c-shared. The linker will
12// arrange to invoke this function as a global constructor (for
13// c-archive) or when the shared library is loaded (for c-shared).
14// We expect argc and argv to be passed in the usual C ABI registers
15// R2 and R3.
16TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
38 XOR g, g
39
40 MOVD $runtime·libInit(SB), R1
41 BL R1
42
43 // Restore F8-F15 from our stack frame.
44 FMOVD 16(R15), F8
45 FMOVD 24(R15), F9
46 FMOVD 32(R15), F10
47 FMOVD 40(R15), F11
48 FMOVD 48(R15), F12
49 FMOVD 56(R15), F13
50 FMOVD 64(R15), F14
51 FMOVD 72(R15), F15
52 MOVD $80(R15), R15
53
54 // Restore R6-R15.
55 LMG 48(R15), R6, R15
56 RET
57
58// rt0_lib_go initializes the Go runtime.
59// This is started in a separate thread by _rt0_s390x_lib.
60TEXT runtime·rt0_lib_go<ABIInternal>(SB), NOSPLIT|NOFRAME, $0
61 MOVD _rt0_s390x_lib_argc<>(SB), R2
62 MOVD _rt0_s390x_lib_argv<>(SB), R3
63 MOVD $runtime·rt0_go(SB), R1
64 BR R1
65
66DATA _rt0_s390x_lib_argc<>(SB)/8, $0
67GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
68DATA _rt0_s390x_lib_argv<>(SB)/8, $0
69GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
70
71TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
72 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
73 // C TLS base pointer in AR0:AR1
74
75 // initialize essential registers
76 XOR R0, R0
77
78 SUB $24, R15
79 MOVW R2, 8(R15) // argc
80 MOVD R3, 16(R15) // argv
81
82 // create istack out of the given (operating system) stack.
83 // _cgo_init may update stackguard.
84 MOVD $runtime·g0(SB), g
85 MOVD R15, R11
86 SUB $(64*1024), R11
87 MOVD R11, g_stackguard0(g)
88 MOVD R11, g_stackguard1(g)
89 MOVD R11, (g_stack+stack_lo)(g)
90 MOVD R15, (g_stack+stack_hi)(g)
91
92 // if there is a _cgo_init, call it using the gcc ABI.
93 MOVD _cgo_init(SB), R11
94 CMPBEQ R11, $0, nocgo
95 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
96 SLD $32, R4, R4
97 MOVW AR1, R4 // arg 2: TLS base pointer
98 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
99 MOVD g, R2 // arg 0: G
100 // C functions expect 160 bytes of space on caller stack frame
101 // and an 8-byte aligned stack pointer
102 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
103 SUB $160, R15 // reserve 160 bytes
104 MOVD $~7, R6
105 AND R6, R15 // 8-byte align
106 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
107 MOVD R9, R15 // restore stack
108 XOR R0, R0 // zero R0
109
110nocgo:
111 // update stackguard after _cgo_init
112 MOVD (g_stack+stack_lo)(g), R2
113 ADD $const_stackGuard, R2
114 MOVD R2, g_stackguard0(g)
115 MOVD R2, g_stackguard1(g)
116
117 // set the per-goroutine and per-mach "registers"
118 MOVD $runtime·m0(SB), R2
119
120 // save m->g0 = g0
121 MOVD g, m_g0(R2)
122 // save m0 to g0->m
123 MOVD R2, g_m(g)
124
125 BL runtime·check(SB)
126
127 // argc/argv are already prepared on stack
128 BL runtime·args(SB)
129 BL runtime·checkS390xCPU(SB)
130 BL runtime·osinit(SB)
131 BL runtime·schedinit(SB)
132
133 // create a new goroutine to start program
134 MOVD $runtime·mainPC(SB), R2 // entry
135 SUB $16, R15
136 MOVD R2, 8(R15)
137 MOVD $0, 0(R15)
138 BL runtime·newproc(SB)
139 ADD $16, R15
140
141 // start this M
142 BL runtime·mstart(SB)
143
144 MOVD $0, 1(R0)
145 RET
146
147DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
148GLOBL runtime·mainPC(SB),RODATA,$8
149
150TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
151 BRRK
152 RET
153
154TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
155 RET
156
157TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
158 CALL runtime·mstart0(SB)
159 RET // not reached
160
161/*
162 * go-routine
163 */
164
165// void gogo(Gobuf*)
166// restore state from Gobuf; longjmp
167TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
168 MOVD buf+0(FP), R5
169 MOVD gobuf_g(R5), R6
170 MOVD 0(R6), R7 // make sure g != nil
171 BR gogo<>(SB)
172
173TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
174 MOVD R6, g
175 BL runtime·save_g(SB)
176
177 MOVD 0(g), R4
178 MOVD gobuf_sp(R5), R15
179 MOVD gobuf_lr(R5), LR
180 MOVD gobuf_ctxt(R5), R12
181 MOVD $0, gobuf_sp(R5)
182 MOVD $0, gobuf_lr(R5)
183 MOVD $0, gobuf_ctxt(R5)
184 CMP R0, R0 // set condition codes for == test, needed by stack split
185 MOVD gobuf_pc(R5), R6
186 BR (R6)
187
188// void mcall(fn func(*g))
189// Switch to m->g0's stack, call fn(g).
190// Fn must never return. It should gogo(&g->sched)
191// to keep running g.
192TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $-8-8
193 MOVD R2, R12 // context
194 // Save caller state in g->sched
195 MOVD R15, (g_sched+gobuf_sp)(g)
196 MOVD LR, (g_sched+gobuf_pc)(g)
197 MOVD $0, (g_sched+gobuf_lr)(g)
198
199 // Switch to m->g0 & its stack, call fn.
200 MOVD g, R2
201 MOVD g_m(g), R4
202 MOVD m_g0(R4), g
203 BL runtime·save_g(SB)
204 CMP g, R2
205 BNE 2(PC)
206 BR runtime·badmcall(SB)
207 MOVD 0(R12), R4 // code pointer
208 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
209 SUB $16, R15
210 MOVD R2, 8(R15)
211 MOVD $0, 0(R15)
212 BL (R4)
213 BR runtime·badmcall2(SB)
214
215// systemstack_switch is a dummy routine that systemstack leaves at the bottom
216// of the G stack. We need to distinguish the routine that
217// lives at the bottom of the G stack from the one that lives
218// at the top of the system stack because the one at the top of
219// the system stack terminates the stack walk (see topofstack()).
220TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
221 UNDEF
222 BL (LR) // make sure this function is not leaf
223 RET
224
225// func systemstack(fn func())
226TEXT runtime·systemstack(SB), NOSPLIT, $0-8
227 MOVD fn+0(FP), R3 // R3 = fn
228 MOVD R3, R12 // context
229 MOVD g_m(g), R4 // R4 = m
230
231 MOVD m_gsignal(R4), R5 // R5 = gsignal
232 CMPBEQ g, R5, noswitch
233
234 MOVD m_g0(R4), R5 // R5 = g0
235 CMPBEQ g, R5, noswitch
236
237 MOVD m_curg(R4), R6
238 CMPBEQ g, R6, switch
239
240 // Bad: g is not gsignal, not g0, not curg. What is it?
241 // Hide call from linker nosplit analysis.
242 MOVD $runtime·badsystemstack(SB), R3
243 BL (R3)
244 BL runtime·abort(SB)
245
246switch:
247 // save our state in g->sched. Pretend to
248 // be systemstack_switch if the G stack is scanned.
249 BL gosave_systemstack_switch<>(SB)
250
251 // switch to g0
252 MOVD R5, g
253 BL runtime·save_g(SB)
254 MOVD (g_sched+gobuf_sp)(g), R15
255
256 // call target function
257 MOVD 0(R12), R3 // code pointer
258 BL (R3)
259
260 // switch back to g
261 MOVD g_m(g), R3
262 MOVD m_curg(R3), g
263 BL runtime·save_g(SB)
264 MOVD (g_sched+gobuf_sp)(g), R15
265 MOVD $0, (g_sched+gobuf_sp)(g)
266 RET
267
268noswitch:
269 // already on m stack, just call directly
270 // Using a tail call here cleans up tracebacks since we won't stop
271 // at an intermediate systemstack.
272 MOVD 0(R12), R3 // code pointer
273 MOVD 0(R15), LR // restore LR
274 ADD $8, R15
275 BR (R3)
276
277// func switchToCrashStack0(fn func())
278TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
279 MOVD R2, R12 // context
280 MOVD g_m(g), R2 // curm
281
282 // set g to gcrash
283 MOVD $runtime·gcrash(SB), g // g = &gcrash
284 BL runtime·save_g(SB)
285 MOVD R2, g_m(g) // g.m = curm
286 MOVD g, m_g0(R2) // curm.g0 = g
287
288 // switch to crashstack
289 MOVD (g_stack+stack_hi)(g), R2
290 ADD $(-4*8), R2, R15
291
292 // call target function
293 MOVD 0(R12), R3 // code pointer
294 BL (R3)
295
296 // should never return
297 BL runtime·abort(SB)
298 UNDEF
299
300/*
301 * support for morestack
302 */
303
304// Called during function prolog when more stack is needed.
305// Caller has already loaded:
306// R3: framesize, R4: argsize, R5: LR
307//
308// The traceback routines see morestack on a g0 as being
309// the top of a stack (for example, morestack calling newstack
310// calling the scheduler calling newm calling gc), so we must
311// record an argument size. For that purpose, it has no arguments.
312TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
313 // Called from f.
314 // Set g->sched to context in f.
315 MOVD R15, (g_sched+gobuf_sp)(g)
316 MOVD LR, R8
317 MOVD R8, (g_sched+gobuf_pc)(g)
318 MOVD R5, (g_sched+gobuf_lr)(g)
319 MOVD R12, (g_sched+gobuf_ctxt)(g)
320
321 // Cannot grow scheduler stack (m->g0).
322 MOVD g_m(g), R7
323 MOVD m_g0(R7), R8
324 CMPBNE g, R8, 3(PC)
325 BL runtime·badmorestackg0(SB)
326 BL runtime·abort(SB)
327
328 // Cannot grow signal stack (m->gsignal).
329 MOVD m_gsignal(R7), R8
330 CMP g, R8
331 BNE 3(PC)
332 BL runtime·badmorestackgsignal(SB)
333 BL runtime·abort(SB)
334
335 // Called from f.
336 // Set m->morebuf to f's caller.
337 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
338 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
339 MOVD g, (m_morebuf+gobuf_g)(R7)
340
341 // Call newstack on m->g0's stack.
342 MOVD m_g0(R7), g
343 BL runtime·save_g(SB)
344 MOVD (g_sched+gobuf_sp)(g), R15
345 // Create a stack frame on g0 to call newstack.
346 MOVD $0, -8(R15) // Zero saved LR in frame
347 SUB $8, R15
348 BL runtime·newstack(SB)
349
350 // Not reached, but make sure the return PC from the call to newstack
351 // is still in this function, and not the beginning of the next.
352 UNDEF
353
354TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
355 // Force SPWRITE. This function doesn't actually write SP,
356 // but it is called with a special calling convention where
357 // the caller doesn't save LR on stack but passes it as a
358 // register (R5), and the unwinder currently doesn't understand.
359 // Make it SPWRITE to stop unwinding. (See issue 54332)
360 MOVD R15, R15
361
362 MOVD $0, R12
363 BR runtime·morestack(SB)
364
365// reflectcall: call a function with the given argument list
366// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
367// we don't have variable-sized frames, so we use a small number
368// of constant-sized-frame functions to encode a few bits of size in the pc.
369// Caution: ugly multiline assembly macros in your future!
370
371#define DISPATCH(NAME,MAXSIZE) \
372 MOVD $MAXSIZE, R4; \
373 CMP R3, R4; \
374 BGT 3(PC); \
375 MOVD $NAME(SB), R5; \
376 BR (R5)
377// Note: can't just "BR NAME(SB)" - bad inlining results.
378
379TEXT ·reflectcall(SB), NOSPLIT, $-8-48
380 MOVWZ frameSize+32(FP), R3
381 DISPATCH(runtime·call16, 16)
382 DISPATCH(runtime·call32, 32)
383 DISPATCH(runtime·call64, 64)
384 DISPATCH(runtime·call128, 128)
385 DISPATCH(runtime·call256, 256)
386 DISPATCH(runtime·call512, 512)
387 DISPATCH(runtime·call1024, 1024)
388 DISPATCH(runtime·call2048, 2048)
389 DISPATCH(runtime·call4096, 4096)
390 DISPATCH(runtime·call8192, 8192)
391 DISPATCH(runtime·call16384, 16384)
392 DISPATCH(runtime·call32768, 32768)
393 DISPATCH(runtime·call65536, 65536)
394 DISPATCH(runtime·call131072, 131072)
395 DISPATCH(runtime·call262144, 262144)
396 DISPATCH(runtime·call524288, 524288)
397 DISPATCH(runtime·call1048576, 1048576)
398 DISPATCH(runtime·call2097152, 2097152)
399 DISPATCH(runtime·call4194304, 4194304)
400 DISPATCH(runtime·call8388608, 8388608)
401 DISPATCH(runtime·call16777216, 16777216)
402 DISPATCH(runtime·call33554432, 33554432)
403 DISPATCH(runtime·call67108864, 67108864)
404 DISPATCH(runtime·call134217728, 134217728)
405 DISPATCH(runtime·call268435456, 268435456)
406 DISPATCH(runtime·call536870912, 536870912)
407 DISPATCH(runtime·call1073741824, 1073741824)
408 MOVD $runtime·badreflectcall(SB), R5
409 BR (R5)
410
411#define CALLFN(NAME,MAXSIZE) \
412TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
413 NO_LOCAL_POINTERS; \
414 /* copy arguments to stack */ \
415 MOVD stackArgs+16(FP), R4; \
416 MOVWZ stackArgsSize+24(FP), R5; \
417 MOVD $stack-MAXSIZE(SP), R6; \
418loopArgs: /* copy 256 bytes at a time */ \
419 CMP R5, $256; \
420 BLT tailArgs; \
421 SUB $256, R5; \
422 MVC $256, 0(R4), 0(R6); \
423 MOVD $256(R4), R4; \
424 MOVD $256(R6), R6; \
425 BR loopArgs; \
426tailArgs: /* copy remaining bytes */ \
427 CMP R5, $0; \
428 BEQ callFunction; \
429 SUB $1, R5; \
430 EXRL $callfnMVC<>(SB), R5; \
431callFunction: \
432 MOVD f+8(FP), R12; \
433 MOVD regArgs+40(FP), R10; \
434 BL ·unspillArgs(SB); \
435 MOVD (R12), R10; \
436 PCDATA $PCDATA_StackMapIndex, $0; \
437 BL (R10); \
438 /* copy return values back */ \
439 MOVD regArgs+40(FP), R10; \
440 BL ·spillArgs(SB); \
441 MOVD stackArgsType+0(FP), R7; \
442 MOVD stackArgs+16(FP), R6; \
443 MOVWZ stackArgsSize+24(FP), R5; \
444 MOVD $stack-MAXSIZE(SP), R4; \
445 MOVWZ stackRetOffset+28(FP), R1; \
446 ADD R1, R4; \
447 ADD R1, R6; \
448 SUB R1, R5; \
449 BL callRet<>(SB); \
450 RET
451
452// callRet copies return values back at the end of call*. This is a
453// separate function so it can allocate stack space for the arguments
454// to reflectcallmove. It does not follow the Go ABI; it expects its
455// arguments in registers.
456TEXT callRet<>(SB), NOSPLIT, $40-0
457 NO_LOCAL_POINTERS;
458 MOVD R7, 8(R15)
459 MOVD R6, 16(R15)
460 MOVD R4, 24(R15)
461 MOVD R5, 32(R15)
462 MOVD R10, 40(R15)
463 BL runtime·reflectcallmove(SB)
464 RET
465
466CALLFN(·call16, 16)
467CALLFN(·call32, 32)
468CALLFN(·call64, 64)
469CALLFN(·call128, 128)
470CALLFN(·call256, 256)
471CALLFN(·call512, 512)
472CALLFN(·call1024, 1024)
473CALLFN(·call2048, 2048)
474CALLFN(·call4096, 4096)
475CALLFN(·call8192, 8192)
476CALLFN(·call16384, 16384)
477CALLFN(·call32768, 32768)
478CALLFN(·call65536, 65536)
479CALLFN(·call131072, 131072)
480CALLFN(·call262144, 262144)
481CALLFN(·call524288, 524288)
482CALLFN(·call1048576, 1048576)
483CALLFN(·call2097152, 2097152)
484CALLFN(·call4194304, 4194304)
485CALLFN(·call8388608, 8388608)
486CALLFN(·call16777216, 16777216)
487CALLFN(·call33554432, 33554432)
488CALLFN(·call67108864, 67108864)
489CALLFN(·call134217728, 134217728)
490CALLFN(·call268435456, 268435456)
491CALLFN(·call536870912, 536870912)
492CALLFN(·call1073741824, 1073741824)
493
494// Not a function: target for EXRL (execute relative long) instruction.
495TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
496 MVC $1, 0(R4), 0(R6)
497
498TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
499 RET
500
501// Save state of caller into g->sched,
502// but using fake PC from systemstack_switch.
503// Must only be called from functions with no locals ($0)
504// or else unwinding from systemstack_switch is incorrect.
505// Smashes R1.
506TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
507 MOVD $runtime·systemstack_switch(SB), R1
508 ADD $16, R1 // get past prologue
509 MOVD R1, (g_sched+gobuf_pc)(g)
510 MOVD R15, (g_sched+gobuf_sp)(g)
511 MOVD $0, (g_sched+gobuf_lr)(g)
512 // Assert ctxt is zero. See func save.
513 MOVD (g_sched+gobuf_ctxt)(g), R1
514 CMPBEQ R1, $0, 2(PC)
515 BL runtime·abort(SB)
516 RET
517
518// func asmcgocall(fn, arg unsafe.Pointer) int32
519// Call fn(arg) on the scheduler stack,
520// aligned appropriately for the gcc ABI.
521// See cgocall.go for more details.
522TEXT ·asmcgocall(SB),NOSPLIT,$0-20
523 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
524 // C TLS base pointer in AR0:AR1
525 MOVD fn+0(FP), R3
526 MOVD arg+8(FP), R4
527
528 MOVD R15, R2 // save original stack pointer
529 MOVD g, R5
530
531 // Figure out if we need to switch to m->g0 stack.
532 // We get called to create new OS threads too, and those
533 // come in on the m->g0 stack already. Or we might already
534 // be on the m->gsignal stack.
535 CMPBEQ g, $0, nosave
536 MOVD g_m(g), R6
537 MOVD m_gsignal(R6), R7
538 CMPBEQ R7, g, g0
539 MOVD m_g0(R6), R7
540 CMPBEQ R7, g, g0
541 BL gosave_systemstack_switch<>(SB)
542 MOVD R7, g
543 BL runtime·save_g(SB)
544 MOVD (g_sched+gobuf_sp)(g), R15
545
546 // Now on a scheduling stack (a pthread-created stack).
547g0:
548 // Save room for two of our pointers, plus 160 bytes of callee
549 // save area that lives on the caller stack.
550 SUB $176, R15
551 MOVD $~7, R6
552 AND R6, R15 // 8-byte alignment for gcc ABI
553 MOVD R5, 168(R15) // save old g on stack
554 MOVD (g_stack+stack_hi)(R5), R5
555 SUB R2, R5
556 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
557 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
558 MOVD R4, R2 // arg in R2
559 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
560
561 XOR R0, R0 // set R0 back to 0.
562 // Restore g, stack pointer.
563 MOVD 168(R15), g
564 BL runtime·save_g(SB)
565 MOVD (g_stack+stack_hi)(g), R5
566 MOVD 160(R15), R6
567 SUB R6, R5
568 MOVD R5, R15
569
570 MOVW R2, ret+16(FP)
571 RET
572
573nosave:
574 // Running on a system stack, perhaps even without a g.
575 // Having no g can happen during thread creation or thread teardown.
576 MOVD fn+0(FP), R3
577 MOVD arg+8(FP), R4
578 MOVD R15, R2
579 SUB $176, R15
580 MOVD $~7, R6
581 AND R6, R15
582 MOVD $0, 168(R15) // Where above code stores g, in case someone looks during debugging.
583 MOVD R2, 160(R15) // Save original stack pointer.
584 MOVD $0, 0(R15) // clear back chain pointer
585 MOVD R4, R2 // arg in R2
586 BL R3
587 XOR R0, R0
588 MOVD 160(R15), R15 // Restore stack pointer.
589 MOVW R2, ret+16(FP)
590 RET
591
592// func asmcgocall_no_g(fn, arg unsafe.Pointer)
593// Call fn(arg) aligned appropriately for the gcc ABI.
594// Called on a system stack, and there may be no g yet.
595TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
596 MOVD fn+0(FP), R3
597 MOVD arg+8(FP), R4
598
599 MOVD R15, R2 // Save original stack pointer.
600
601 // Save room for the stack pointer, plus 160 bytes of callee
602 // save area that lives on the caller stack.
603 SUB $168, R15
604 MOVD $~7, R6
605 AND R6, R15
606
607 MOVD R2, 160(R15) // Save original stack pointer.
608 MOVD $0, 0(R15) // clear back chain pointer
609 MOVD R4, R2 // arg in R2
610 BL R3
611 XOR R0, R0
612 MOVD 160(R15), R15 // Restore stack pointer.
613 RET
614
615// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
616// See cgocall.go for more details.
617TEXT ·cgocallback(SB),NOSPLIT,$24-24
618 NO_LOCAL_POINTERS
619
620 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
621 // It is used to dropm while thread is exiting.
622 MOVD fn+0(FP), R1
623 CMPBNE R1, $0, loadg
624 // Restore the g from frame.
625 MOVD frame+8(FP), g
626 BR dropm
627
628loadg:
629 // Load m and g from thread-local storage.
630 MOVB runtime·iscgo(SB), R3
631 CMPBEQ R3, $0, nocgo
632 BL runtime·load_g(SB)
633
634nocgo:
635 // If g is nil, Go did not create the current thread,
636 // or if this thread never called into Go on pthread platforms.
637 // Call needm to obtain one for temporary use.
638 // In this case, we're running on the thread stack, so there's
639 // lots of space, but the linker doesn't know. Hide the call from
640 // the linker analysis by using an indirect call.
641 CMPBEQ g, $0, needm
642
643 MOVD g_m(g), R8
644 MOVD R8, savedm-8(SP)
645 BR havem
646
647needm:
648 MOVD g, savedm-8(SP) // g is zero, so is m.
649 MOVD $runtime·needAndBindM(SB), R3
650 BL (R3)
651
652 // Set m->sched.sp = SP, so that if a panic happens
653 // during the function we are about to execute, it will
654 // have a valid SP to run on the g0 stack.
655 // The next few lines (after the havem label)
656 // will save this SP onto the stack and then write
657 // the same SP back to m->sched.sp. That seems redundant,
658 // but if an unrecovered panic happens, unwindm will
659 // restore the g->sched.sp from the stack location
660 // and then systemstack will try to use it. If we don't set it here,
661 // that restored SP will be uninitialized (typically 0) and
662 // will not be usable.
663 MOVD g_m(g), R8
664 MOVD m_g0(R8), R3
665 MOVD R15, (g_sched+gobuf_sp)(R3)
666
667havem:
668 // Now there's a valid m, and we're running on its m->g0.
669 // Save current m->g0->sched.sp on stack and then set it to SP.
670 // Save current sp in m->g0->sched.sp in preparation for
671 // switch back to m->curg stack.
672 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
673 MOVD m_g0(R8), R3
674 MOVD (g_sched+gobuf_sp)(R3), R4
675 MOVD R4, savedsp-24(SP) // must match frame size
676 MOVD R15, (g_sched+gobuf_sp)(R3)
677
678 // Switch to m->curg stack and call runtime.cgocallbackg.
679 // Because we are taking over the execution of m->curg
680 // but *not* resuming what had been running, we need to
681 // save that information (m->curg->sched) so we can restore it.
682 // We can restore m->curg->sched.sp easily, because calling
683 // runtime.cgocallbackg leaves SP unchanged upon return.
684 // To save m->curg->sched.pc, we push it onto the curg stack and
685 // open a frame the same size as cgocallback's g0 frame.
686 // Once we switch to the curg stack, the pushed PC will appear
687 // to be the return PC of cgocallback, so that the traceback
688 // will seamlessly trace back into the earlier calls.
689 MOVD m_curg(R8), g
690 BL runtime·save_g(SB)
691 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
692 MOVD (g_sched+gobuf_pc)(g), R5
693 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
694 // Gather our arguments into registers.
695 MOVD fn+0(FP), R1
696 MOVD frame+8(FP), R2
697 MOVD ctxt+16(FP), R3
698 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
699 MOVD R1, 8(R15)
700 MOVD R2, 16(R15)
701 MOVD R3, 24(R15)
702 BL runtime·cgocallbackg(SB)
703
704 // Restore g->sched (== m->curg->sched) from saved values.
705 MOVD 0(R15), R5
706 MOVD R5, (g_sched+gobuf_pc)(g)
707 MOVD $(24+8)(R15), R4 // must match frame size
708 MOVD R4, (g_sched+gobuf_sp)(g)
709
710 // Switch back to m->g0's stack and restore m->g0->sched.sp.
711 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
712 // so we do not have to restore it.)
713 MOVD g_m(g), R8
714 MOVD m_g0(R8), g
715 BL runtime·save_g(SB)
716 MOVD (g_sched+gobuf_sp)(g), R15
717 MOVD savedsp-24(SP), R4 // must match frame size
718 MOVD R4, (g_sched+gobuf_sp)(g)
719
720 // If the m on entry was nil, we called needm above to borrow an m,
721 // 1. for the duration of the call on non-pthread platforms,
722 // 2. or the duration of the C thread alive on pthread platforms.
723 // If the m on entry wasn't nil,
724 // 1. the thread might be a Go thread,
725 // 2. or it wasn't the first call from a C thread on pthread platforms,
726 // since then we skip dropm to reuse the m in the first call.
727 MOVD savedm-8(SP), R6
728 CMPBNE R6, $0, droppedm
729
730 // Skip dropm to reuse it in the next call, when a pthread key has been created.
731 MOVD _cgo_pthread_key_created(SB), R6
732 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
733 CMPBEQ R6, $0, dropm
734 MOVD (R6), R6
735 CMPBNE R6, $0, droppedm
736
737dropm:
738 MOVD $runtime·dropm(SB), R3
739 BL (R3)
740droppedm:
741
742 // Done!
743 RET
744
745// void setg(G*); set g. for use by needm.
746TEXT runtime·setg(SB), NOSPLIT, $0-8
747 MOVD gg+0(FP), g
748 // This only happens if iscgo, so jump straight to save_g
749 BL runtime·save_g(SB)
750 RET
751
752// void setg_gcc(G*); set g in C TLS.
753// Must obey the gcc calling convention.
754TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
755 // The standard prologue clobbers LR (R14), which is callee-save in
756 // the C ABI, so we have to use NOFRAME and save LR ourselves.
757 MOVD LR, R1
758 // Also save g, R10, and R11 since they're callee-save in C ABI
759 MOVD R10, R3
760 MOVD g, R4
761 MOVD R11, R5
762
763 MOVD R2, g
764 BL runtime·save_g(SB)
765
766 MOVD R5, R11
767 MOVD R4, g
768 MOVD R3, R10
769 MOVD R1, LR
770 RET
771
772TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
773 MOVW (R0), R0
774 UNDEF
775
776// int64 runtime·cputicks(void)
777TEXT runtime·cputicks(SB),NOSPLIT,$0-8
778 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
779 // This means that since about 1972 the msb has been set, making the
780 // result of a call to STORE CLOCK (stck) a negative number.
781 // We clear the msb to make it positive.
782 STCK ret+0(FP) // serialises before and after call
783 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
784 SLD $1, R3
785 SRD $1, R3
786 MOVD R3, ret+0(FP)
787 RET
788
789// spillArgs stores return values from registers to a *internal/abi.RegArgs in R10.
790TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
791 MOVD R2, 0(R10)
792 MOVD R3, 8(R10)
793 MOVD R4, 16(R10)
794 MOVD R5, 24(R10)
795 MOVD R6, 32(R10)
796 MOVD R7, 40(R10)
797 MOVD R8, 48(R10)
798 MOVD R9, 56(R10)
799 FMOVD F0, 64(R10)
800 FMOVD F1, 72(R10)
801 FMOVD F2, 80(R10)
802 FMOVD F3, 88(R10)
803 FMOVD F4, 96(R10)
804 FMOVD F5, 104(R10)
805 FMOVD F6, 112(R10)
806 FMOVD F7, 120(R10)
807 FMOVD F8, 128(R10)
808 FMOVD F9, 136(R10)
809 FMOVD F10, 144(R10)
810 FMOVD F11, 152(R10)
811 FMOVD F12, 160(R10)
812 FMOVD F13, 168(R10)
813 FMOVD F14, 176(R10)
814 FMOVD F15, 184(R10)
815 RET
816
817// unspillArgs loads args into registers from a *internal/abi.RegArgs in R10.
818TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
819 MOVD 0(R10), R2
820 MOVD 8(R10), R3
821 MOVD 16(R10), R4
822 MOVD 24(R10), R5
823 MOVD 32(R10), R6
824 MOVD 40(R10), R7
825 MOVD 48(R10), R8
826 MOVD 56(R10), R9
827 FMOVD 64(R10), F0
828 FMOVD 72(R10), F1
829 FMOVD 80(R10), F2
830 FMOVD 88(R10), F3
831 FMOVD 96(R10), F4
832 FMOVD 104(R10), F5
833 FMOVD 112(R10), F6
834 FMOVD 120(R10), F7
835 FMOVD 128(R10), F8
836 FMOVD 136(R10), F9
837 FMOVD 144(R10), F10
838 FMOVD 152(R10), F11
839 FMOVD 160(R10), F12
840 FMOVD 168(R10), F13
841 FMOVD 176(R10), F14
842 FMOVD 184(R10), F15
843 RET
844
845// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
846// Must obey the gcc calling convention.
847TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
848 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
849 MOVD g, R1
850 MOVD R10, R3
851 MOVD LR, R4
852 MOVD R11, R5
853
854 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
855 MOVD g_m(g), R2
856 MOVD m_curg(R2), R2
857 MOVD (g_stack+stack_hi)(R2), R2
858
859 MOVD R1, g
860 MOVD R3, R10
861 MOVD R4, LR
862 MOVD R5, R11
863 RET
864
865// The top-most function running on a goroutine
866// returns to goexit+PCQuantum.
867TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
868 BYTE $0x07; BYTE $0x00; // 2-byte nop
869 BL runtime·goexit1(SB) // does not return
870 // traceback from goexit1 must hit code range of goexit
871 BYTE $0x07; BYTE $0x00; // 2-byte nop
872
873TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
874 // Stores are already ordered on s390x, so this is just a
875 // compile barrier.
876 RET
877
878// This is called from .init_array and follows the platform, not Go, ABI.
879// We are overly conservative. We could only save the registers we use.
880// However, since this function is only called once per loaded module
881// performance is unimportant.
882TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
883 // Save R6-R15 in the register save area of the calling function.
884 // Don't bother saving F8-F15 as we aren't doing any calls.
885 STMG R6, R15, 48(R15)
886
887 // append the argument (passed in R2, as per the ELF ABI) to the
888 // moduledata linked list.
889 MOVD runtime·lastmoduledatap(SB), R1
890 MOVD R2, moduledata_next(R1)
891 MOVD R2, runtime·lastmoduledatap(SB)
892
893 // Restore R6-R15.
894 LMG 48(R15), R6, R15
895 RET
896
897// gcWriteBarrier informs the GC about heap pointer writes.
898//
899// gcWriteBarrier does NOT follow the Go ABI. It accepts the
900// number of bytes of buffer needed in R9, and returns a pointer
901// to the buffer space in R9.
902// It clobbers R10 (the temp register) and R1 (used by PLT stub).
903// It does not clobber any other general-purpose registers,
904// but may clobber others (e.g., floating point registers).
905TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
906 // Save the registers clobbered by the fast path.
907 MOVD R4, 96(R15)
908retry:
909 MOVD g_m(g), R1
910 MOVD m_p(R1), R1
911 // Increment wbBuf.next position.
912 MOVD R9, R4
913 ADD (p_wbBuf+wbBuf_next)(R1), R4
914 // Is the buffer full?
915 MOVD (p_wbBuf+wbBuf_end)(R1), R10
916 CMPUBGT R4, R10, flush
917 // Commit to the larger buffer.
918 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
919 // Make return value (the original next position)
920 SUB R9, R4, R9
921 // Restore registers.
922 MOVD 96(R15), R4
923 RET
924
925flush:
926 // Save all general purpose registers since these could be
927 // clobbered by wbBufFlush and were not saved by the caller.
928 STMG R2, R3, 8(R15)
929 MOVD R0, 24(R15)
930 // R1 already saved.
931 // R4 already saved.
932 STMG R5, R12, 32(R15) // save R5 - R12
933 // R13 is g.
934 // R14 is LR.
935 // R15 is SP.
936
937 CALL runtime·wbBufFlush(SB)
938
939 LMG 8(R15), R2, R3 // restore R2 - R3
940 MOVD 24(R15), R0 // restore R0
941 LMG 32(R15), R5, R12 // restore R5 - R12
942 JMP retry
943
944TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
945 MOVD $8, R9
946 JMP gcWriteBarrier<>(SB)
947TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
948 MOVD $16, R9
949 JMP gcWriteBarrier<>(SB)
950TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
951 MOVD $24, R9
952 JMP gcWriteBarrier<>(SB)
953TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
954 MOVD $32, R9
955 JMP gcWriteBarrier<>(SB)
956TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
957 MOVD $40, R9
958 JMP gcWriteBarrier<>(SB)
959TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
960 MOVD $48, R9
961 JMP gcWriteBarrier<>(SB)
962TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
963 MOVD $56, R9
964 JMP gcWriteBarrier<>(SB)
965TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
966 MOVD $64, R9
967 JMP gcWriteBarrier<>(SB)
968
969TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
970 NO_LOCAL_POINTERS
971 // Save all 16 int registers that could have an index in them.
972 // They may be pointers, but if they are they are dead.
973 STMG R0, R12, 24(R15)
974 // Note that R10 @ 104 is not needed, it is an assembler temp
975 // skip R13 aka G @ 128
976 // skip R14 aka LR @ 136
977 // skip R15 aka SP @ 144
978
979 MOVD R14, R2 // PC immediately after call to panicBounds
980 ADD $24, R15, R3 // pointer to save area
981 CALL runtime·panicBounds64<ABIInternal>(SB)
982 RET
View as plain text