Text file
src/runtime/asm_s390x.s
Documentation: runtime
1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10// _rt0_s390x_lib is common startup code for s390x systems when
11// using -buildmode=c-archive or -buildmode=c-shared. The linker will
12// arrange to invoke this function as a global constructor (for
13// c-archive) or when the shared library is loaded (for c-shared).
14// We expect argc and argv to be passed in the usual C ABI registers
15// R2 and R3.
16TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74// _rt0_s390x_lib_go initializes the Go runtime.
75// This is started in a separate thread by _rt0_s390x_lib.
76TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
87TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const_stackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·checkS390xCPU(SB)
146 BL runtime·osinit(SB)
147 BL runtime·schedinit(SB)
148
149 // create a new goroutine to start program
150 MOVD $runtime·mainPC(SB), R2 // entry
151 SUB $16, R15
152 MOVD R2, 8(R15)
153 MOVD $0, 0(R15)
154 BL runtime·newproc(SB)
155 ADD $16, R15
156
157 // start this M
158 BL runtime·mstart(SB)
159
160 MOVD $0, 1(R0)
161 RET
162
163DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
164GLOBL runtime·mainPC(SB),RODATA,$8
165
166TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 BRRK
168 RET
169
170TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
174 CALL runtime·mstart0(SB)
175 RET // not reached
176
177/*
178 * go-routine
179 */
180
181// void gogo(Gobuf*)
182// restore state from Gobuf; longjmp
183TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
184 MOVD buf+0(FP), R5
185 MOVD gobuf_g(R5), R6
186 MOVD 0(R6), R7 // make sure g != nil
187 BR gogo<>(SB)
188
189TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
190 MOVD R6, g
191 BL runtime·save_g(SB)
192
193 MOVD 0(g), R4
194 MOVD gobuf_sp(R5), R15
195 MOVD gobuf_lr(R5), LR
196 MOVD gobuf_ret(R5), R3
197 MOVD gobuf_ctxt(R5), R12
198 MOVD $0, gobuf_sp(R5)
199 MOVD $0, gobuf_ret(R5)
200 MOVD $0, gobuf_lr(R5)
201 MOVD $0, gobuf_ctxt(R5)
202 CMP R0, R0 // set condition codes for == test, needed by stack split
203 MOVD gobuf_pc(R5), R6
204 BR (R6)
205
206// void mcall(fn func(*g))
207// Switch to m->g0's stack, call fn(g).
208// Fn must never return. It should gogo(&g->sched)
209// to keep running g.
210TEXT runtime·mcall(SB), NOSPLIT, $-8-8
211 // Save caller state in g->sched
212 MOVD R15, (g_sched+gobuf_sp)(g)
213 MOVD LR, (g_sched+gobuf_pc)(g)
214 MOVD $0, (g_sched+gobuf_lr)(g)
215
216 // Switch to m->g0 & its stack, call fn.
217 MOVD g, R3
218 MOVD g_m(g), R8
219 MOVD m_g0(R8), g
220 BL runtime·save_g(SB)
221 CMP g, R3
222 BNE 2(PC)
223 BR runtime·badmcall(SB)
224 MOVD fn+0(FP), R12 // context
225 MOVD 0(R12), R4 // code pointer
226 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
227 SUB $16, R15
228 MOVD R3, 8(R15)
229 MOVD $0, 0(R15)
230 BL (R4)
231 BR runtime·badmcall2(SB)
232
233// systemstack_switch is a dummy routine that systemstack leaves at the bottom
234// of the G stack. We need to distinguish the routine that
235// lives at the bottom of the G stack from the one that lives
236// at the top of the system stack because the one at the top of
237// the system stack terminates the stack walk (see topofstack()).
238TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
239 UNDEF
240 BL (LR) // make sure this function is not leaf
241 RET
242
243// func systemstack(fn func())
244TEXT runtime·systemstack(SB), NOSPLIT, $0-8
245 MOVD fn+0(FP), R3 // R3 = fn
246 MOVD R3, R12 // context
247 MOVD g_m(g), R4 // R4 = m
248
249 MOVD m_gsignal(R4), R5 // R5 = gsignal
250 CMPBEQ g, R5, noswitch
251
252 MOVD m_g0(R4), R5 // R5 = g0
253 CMPBEQ g, R5, noswitch
254
255 MOVD m_curg(R4), R6
256 CMPBEQ g, R6, switch
257
258 // Bad: g is not gsignal, not g0, not curg. What is it?
259 // Hide call from linker nosplit analysis.
260 MOVD $runtime·badsystemstack(SB), R3
261 BL (R3)
262 BL runtime·abort(SB)
263
264switch:
265 // save our state in g->sched. Pretend to
266 // be systemstack_switch if the G stack is scanned.
267 BL gosave_systemstack_switch<>(SB)
268
269 // switch to g0
270 MOVD R5, g
271 BL runtime·save_g(SB)
272 MOVD (g_sched+gobuf_sp)(g), R15
273
274 // call target function
275 MOVD 0(R12), R3 // code pointer
276 BL (R3)
277
278 // switch back to g
279 MOVD g_m(g), R3
280 MOVD m_curg(R3), g
281 BL runtime·save_g(SB)
282 MOVD (g_sched+gobuf_sp)(g), R15
283 MOVD $0, (g_sched+gobuf_sp)(g)
284 RET
285
286noswitch:
287 // already on m stack, just call directly
288 // Using a tail call here cleans up tracebacks since we won't stop
289 // at an intermediate systemstack.
290 MOVD 0(R12), R3 // code pointer
291 MOVD 0(R15), LR // restore LR
292 ADD $8, R15
293 BR (R3)
294
295/*
296 * support for morestack
297 */
298
299// Called during function prolog when more stack is needed.
300// Caller has already loaded:
301// R3: framesize, R4: argsize, R5: LR
302//
303// The traceback routines see morestack on a g0 as being
304// the top of a stack (for example, morestack calling newstack
305// calling the scheduler calling newm calling gc), so we must
306// record an argument size. For that purpose, it has no arguments.
307TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
308 // Cannot grow scheduler stack (m->g0).
309 MOVD g_m(g), R7
310 MOVD m_g0(R7), R8
311 CMPBNE g, R8, 3(PC)
312 BL runtime·badmorestackg0(SB)
313 BL runtime·abort(SB)
314
315 // Cannot grow signal stack (m->gsignal).
316 MOVD m_gsignal(R7), R8
317 CMP g, R8
318 BNE 3(PC)
319 BL runtime·badmorestackgsignal(SB)
320 BL runtime·abort(SB)
321
322 // Called from f.
323 // Set g->sched to context in f.
324 MOVD R15, (g_sched+gobuf_sp)(g)
325 MOVD LR, R8
326 MOVD R8, (g_sched+gobuf_pc)(g)
327 MOVD R5, (g_sched+gobuf_lr)(g)
328 MOVD R12, (g_sched+gobuf_ctxt)(g)
329
330 // Called from f.
331 // Set m->morebuf to f's caller.
332 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
333 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
334 MOVD g, (m_morebuf+gobuf_g)(R7)
335
336 // Call newstack on m->g0's stack.
337 MOVD m_g0(R7), g
338 BL runtime·save_g(SB)
339 MOVD (g_sched+gobuf_sp)(g), R15
340 // Create a stack frame on g0 to call newstack.
341 MOVD $0, -8(R15) // Zero saved LR in frame
342 SUB $8, R15
343 BL runtime·newstack(SB)
344
345 // Not reached, but make sure the return PC from the call to newstack
346 // is still in this function, and not the beginning of the next.
347 UNDEF
348
349TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
350 // Force SPWRITE. This function doesn't actually write SP,
351 // but it is called with a special calling convention where
352 // the caller doesn't save LR on stack but passes it as a
353 // register (R5), and the unwinder currently doesn't understand.
354 // Make it SPWRITE to stop unwinding. (See issue 54332)
355 MOVD R15, R15
356
357 MOVD $0, R12
358 BR runtime·morestack(SB)
359
360// reflectcall: call a function with the given argument list
361// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
362// we don't have variable-sized frames, so we use a small number
363// of constant-sized-frame functions to encode a few bits of size in the pc.
364// Caution: ugly multiline assembly macros in your future!
365
366#define DISPATCH(NAME,MAXSIZE) \
367 MOVD $MAXSIZE, R4; \
368 CMP R3, R4; \
369 BGT 3(PC); \
370 MOVD $NAME(SB), R5; \
371 BR (R5)
372// Note: can't just "BR NAME(SB)" - bad inlining results.
373
374TEXT ·reflectcall(SB), NOSPLIT, $-8-48
375 MOVWZ frameSize+32(FP), R3
376 DISPATCH(runtime·call16, 16)
377 DISPATCH(runtime·call32, 32)
378 DISPATCH(runtime·call64, 64)
379 DISPATCH(runtime·call128, 128)
380 DISPATCH(runtime·call256, 256)
381 DISPATCH(runtime·call512, 512)
382 DISPATCH(runtime·call1024, 1024)
383 DISPATCH(runtime·call2048, 2048)
384 DISPATCH(runtime·call4096, 4096)
385 DISPATCH(runtime·call8192, 8192)
386 DISPATCH(runtime·call16384, 16384)
387 DISPATCH(runtime·call32768, 32768)
388 DISPATCH(runtime·call65536, 65536)
389 DISPATCH(runtime·call131072, 131072)
390 DISPATCH(runtime·call262144, 262144)
391 DISPATCH(runtime·call524288, 524288)
392 DISPATCH(runtime·call1048576, 1048576)
393 DISPATCH(runtime·call2097152, 2097152)
394 DISPATCH(runtime·call4194304, 4194304)
395 DISPATCH(runtime·call8388608, 8388608)
396 DISPATCH(runtime·call16777216, 16777216)
397 DISPATCH(runtime·call33554432, 33554432)
398 DISPATCH(runtime·call67108864, 67108864)
399 DISPATCH(runtime·call134217728, 134217728)
400 DISPATCH(runtime·call268435456, 268435456)
401 DISPATCH(runtime·call536870912, 536870912)
402 DISPATCH(runtime·call1073741824, 1073741824)
403 MOVD $runtime·badreflectcall(SB), R5
404 BR (R5)
405
406#define CALLFN(NAME,MAXSIZE) \
407TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
408 NO_LOCAL_POINTERS; \
409 /* copy arguments to stack */ \
410 MOVD stackArgs+16(FP), R4; \
411 MOVWZ stackArgsSize+24(FP), R5; \
412 MOVD $stack-MAXSIZE(SP), R6; \
413loopArgs: /* copy 256 bytes at a time */ \
414 CMP R5, $256; \
415 BLT tailArgs; \
416 SUB $256, R5; \
417 MVC $256, 0(R4), 0(R6); \
418 MOVD $256(R4), R4; \
419 MOVD $256(R6), R6; \
420 BR loopArgs; \
421tailArgs: /* copy remaining bytes */ \
422 CMP R5, $0; \
423 BEQ callFunction; \
424 SUB $1, R5; \
425 EXRL $callfnMVC<>(SB), R5; \
426callFunction: \
427 MOVD f+8(FP), R12; \
428 MOVD (R12), R8; \
429 PCDATA $PCDATA_StackMapIndex, $0; \
430 BL (R8); \
431 /* copy return values back */ \
432 MOVD stackArgsType+0(FP), R7; \
433 MOVD stackArgs+16(FP), R6; \
434 MOVWZ stackArgsSize+24(FP), R5; \
435 MOVD $stack-MAXSIZE(SP), R4; \
436 MOVWZ stackRetOffset+28(FP), R1; \
437 ADD R1, R4; \
438 ADD R1, R6; \
439 SUB R1, R5; \
440 BL callRet<>(SB); \
441 RET
442
443// callRet copies return values back at the end of call*. This is a
444// separate function so it can allocate stack space for the arguments
445// to reflectcallmove. It does not follow the Go ABI; it expects its
446// arguments in registers.
447TEXT callRet<>(SB), NOSPLIT, $40-0
448 MOVD R7, 8(R15)
449 MOVD R6, 16(R15)
450 MOVD R4, 24(R15)
451 MOVD R5, 32(R15)
452 MOVD $0, 40(R15)
453 BL runtime·reflectcallmove(SB)
454 RET
455
456CALLFN(·call16, 16)
457CALLFN(·call32, 32)
458CALLFN(·call64, 64)
459CALLFN(·call128, 128)
460CALLFN(·call256, 256)
461CALLFN(·call512, 512)
462CALLFN(·call1024, 1024)
463CALLFN(·call2048, 2048)
464CALLFN(·call4096, 4096)
465CALLFN(·call8192, 8192)
466CALLFN(·call16384, 16384)
467CALLFN(·call32768, 32768)
468CALLFN(·call65536, 65536)
469CALLFN(·call131072, 131072)
470CALLFN(·call262144, 262144)
471CALLFN(·call524288, 524288)
472CALLFN(·call1048576, 1048576)
473CALLFN(·call2097152, 2097152)
474CALLFN(·call4194304, 4194304)
475CALLFN(·call8388608, 8388608)
476CALLFN(·call16777216, 16777216)
477CALLFN(·call33554432, 33554432)
478CALLFN(·call67108864, 67108864)
479CALLFN(·call134217728, 134217728)
480CALLFN(·call268435456, 268435456)
481CALLFN(·call536870912, 536870912)
482CALLFN(·call1073741824, 1073741824)
483
484// Not a function: target for EXRL (execute relative long) instruction.
485TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
486 MVC $1, 0(R4), 0(R6)
487
488TEXT runtime·procyield(SB),NOSPLIT,$0-0
489 RET
490
491// Save state of caller into g->sched,
492// but using fake PC from systemstack_switch.
493// Must only be called from functions with no locals ($0)
494// or else unwinding from systemstack_switch is incorrect.
495// Smashes R1.
496TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
497 MOVD $runtime·systemstack_switch(SB), R1
498 ADD $16, R1 // get past prologue
499 MOVD R1, (g_sched+gobuf_pc)(g)
500 MOVD R15, (g_sched+gobuf_sp)(g)
501 MOVD $0, (g_sched+gobuf_lr)(g)
502 MOVD $0, (g_sched+gobuf_ret)(g)
503 // Assert ctxt is zero. See func save.
504 MOVD (g_sched+gobuf_ctxt)(g), R1
505 CMPBEQ R1, $0, 2(PC)
506 BL runtime·abort(SB)
507 RET
508
509// func asmcgocall(fn, arg unsafe.Pointer) int32
510// Call fn(arg) on the scheduler stack,
511// aligned appropriately for the gcc ABI.
512// See cgocall.go for more details.
513TEXT ·asmcgocall(SB),NOSPLIT,$0-20
514 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
515 // C TLS base pointer in AR0:AR1
516 MOVD fn+0(FP), R3
517 MOVD arg+8(FP), R4
518
519 MOVD R15, R2 // save original stack pointer
520 MOVD g, R5
521
522 // Figure out if we need to switch to m->g0 stack.
523 // We get called to create new OS threads too, and those
524 // come in on the m->g0 stack already. Or we might already
525 // be on the m->gsignal stack.
526 MOVD g_m(g), R6
527 MOVD m_gsignal(R6), R7
528 CMPBEQ R7, g, g0
529 MOVD m_g0(R6), R7
530 CMPBEQ R7, g, g0
531 BL gosave_systemstack_switch<>(SB)
532 MOVD R7, g
533 BL runtime·save_g(SB)
534 MOVD (g_sched+gobuf_sp)(g), R15
535
536 // Now on a scheduling stack (a pthread-created stack).
537g0:
538 // Save room for two of our pointers, plus 160 bytes of callee
539 // save area that lives on the caller stack.
540 SUB $176, R15
541 MOVD $~7, R6
542 AND R6, R15 // 8-byte alignment for gcc ABI
543 MOVD R5, 168(R15) // save old g on stack
544 MOVD (g_stack+stack_hi)(R5), R5
545 SUB R2, R5
546 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
547 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
548 MOVD R4, R2 // arg in R2
549 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
550
551 XOR R0, R0 // set R0 back to 0.
552 // Restore g, stack pointer.
553 MOVD 168(R15), g
554 BL runtime·save_g(SB)
555 MOVD (g_stack+stack_hi)(g), R5
556 MOVD 160(R15), R6
557 SUB R6, R5
558 MOVD R5, R15
559
560 MOVW R2, ret+16(FP)
561 RET
562
563// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
564// See cgocall.go for more details.
565TEXT ·cgocallback(SB),NOSPLIT,$24-24
566 NO_LOCAL_POINTERS
567
568 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
569 // It is used to dropm while thread is exiting.
570 MOVD fn+0(FP), R1
571 CMPBNE R1, $0, loadg
572 // Restore the g from frame.
573 MOVD frame+8(FP), g
574 BR dropm
575
576loadg:
577 // Load m and g from thread-local storage.
578 MOVB runtime·iscgo(SB), R3
579 CMPBEQ R3, $0, nocgo
580 BL runtime·load_g(SB)
581
582nocgo:
583 // If g is nil, Go did not create the current thread,
584 // or if this thread never called into Go on pthread platforms.
585 // Call needm to obtain one for temporary use.
586 // In this case, we're running on the thread stack, so there's
587 // lots of space, but the linker doesn't know. Hide the call from
588 // the linker analysis by using an indirect call.
589 CMPBEQ g, $0, needm
590
591 MOVD g_m(g), R8
592 MOVD R8, savedm-8(SP)
593 BR havem
594
595needm:
596 MOVD g, savedm-8(SP) // g is zero, so is m.
597 MOVD $runtime·needAndBindM(SB), R3
598 BL (R3)
599
600 // Set m->sched.sp = SP, so that if a panic happens
601 // during the function we are about to execute, it will
602 // have a valid SP to run on the g0 stack.
603 // The next few lines (after the havem label)
604 // will save this SP onto the stack and then write
605 // the same SP back to m->sched.sp. That seems redundant,
606 // but if an unrecovered panic happens, unwindm will
607 // restore the g->sched.sp from the stack location
608 // and then systemstack will try to use it. If we don't set it here,
609 // that restored SP will be uninitialized (typically 0) and
610 // will not be usable.
611 MOVD g_m(g), R8
612 MOVD m_g0(R8), R3
613 MOVD R15, (g_sched+gobuf_sp)(R3)
614
615havem:
616 // Now there's a valid m, and we're running on its m->g0.
617 // Save current m->g0->sched.sp on stack and then set it to SP.
618 // Save current sp in m->g0->sched.sp in preparation for
619 // switch back to m->curg stack.
620 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
621 MOVD m_g0(R8), R3
622 MOVD (g_sched+gobuf_sp)(R3), R4
623 MOVD R4, savedsp-24(SP) // must match frame size
624 MOVD R15, (g_sched+gobuf_sp)(R3)
625
626 // Switch to m->curg stack and call runtime.cgocallbackg.
627 // Because we are taking over the execution of m->curg
628 // but *not* resuming what had been running, we need to
629 // save that information (m->curg->sched) so we can restore it.
630 // We can restore m->curg->sched.sp easily, because calling
631 // runtime.cgocallbackg leaves SP unchanged upon return.
632 // To save m->curg->sched.pc, we push it onto the curg stack and
633 // open a frame the same size as cgocallback's g0 frame.
634 // Once we switch to the curg stack, the pushed PC will appear
635 // to be the return PC of cgocallback, so that the traceback
636 // will seamlessly trace back into the earlier calls.
637 MOVD m_curg(R8), g
638 BL runtime·save_g(SB)
639 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
640 MOVD (g_sched+gobuf_pc)(g), R5
641 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
642 // Gather our arguments into registers.
643 MOVD fn+0(FP), R1
644 MOVD frame+8(FP), R2
645 MOVD ctxt+16(FP), R3
646 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
647 MOVD R1, 8(R15)
648 MOVD R2, 16(R15)
649 MOVD R3, 24(R15)
650 BL runtime·cgocallbackg(SB)
651
652 // Restore g->sched (== m->curg->sched) from saved values.
653 MOVD 0(R15), R5
654 MOVD R5, (g_sched+gobuf_pc)(g)
655 MOVD $(24+8)(R15), R4 // must match frame size
656 MOVD R4, (g_sched+gobuf_sp)(g)
657
658 // Switch back to m->g0's stack and restore m->g0->sched.sp.
659 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
660 // so we do not have to restore it.)
661 MOVD g_m(g), R8
662 MOVD m_g0(R8), g
663 BL runtime·save_g(SB)
664 MOVD (g_sched+gobuf_sp)(g), R15
665 MOVD savedsp-24(SP), R4 // must match frame size
666 MOVD R4, (g_sched+gobuf_sp)(g)
667
668 // If the m on entry was nil, we called needm above to borrow an m,
669 // 1. for the duration of the call on non-pthread platforms,
670 // 2. or the duration of the C thread alive on pthread platforms.
671 // If the m on entry wasn't nil,
672 // 1. the thread might be a Go thread,
673 // 2. or it wasn't the first call from a C thread on pthread platforms,
674 // since then we skip dropm to reuse the m in the first call.
675 MOVD savedm-8(SP), R6
676 CMPBNE R6, $0, droppedm
677
678 // Skip dropm to reuse it in the next call, when a pthread key has been created.
679 MOVD _cgo_pthread_key_created(SB), R6
680 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
681 CMPBEQ R6, $0, dropm
682 MOVD (R6), R6
683 CMPBNE R6, $0, droppedm
684
685dropm:
686 MOVD $runtime·dropm(SB), R3
687 BL (R3)
688droppedm:
689
690 // Done!
691 RET
692
693// void setg(G*); set g. for use by needm.
694TEXT runtime·setg(SB), NOSPLIT, $0-8
695 MOVD gg+0(FP), g
696 // This only happens if iscgo, so jump straight to save_g
697 BL runtime·save_g(SB)
698 RET
699
700// void setg_gcc(G*); set g in C TLS.
701// Must obey the gcc calling convention.
702TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
703 // The standard prologue clobbers LR (R14), which is callee-save in
704 // the C ABI, so we have to use NOFRAME and save LR ourselves.
705 MOVD LR, R1
706 // Also save g, R10, and R11 since they're callee-save in C ABI
707 MOVD R10, R3
708 MOVD g, R4
709 MOVD R11, R5
710
711 MOVD R2, g
712 BL runtime·save_g(SB)
713
714 MOVD R5, R11
715 MOVD R4, g
716 MOVD R3, R10
717 MOVD R1, LR
718 RET
719
720TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
721 MOVW (R0), R0
722 UNDEF
723
724// int64 runtime·cputicks(void)
725TEXT runtime·cputicks(SB),NOSPLIT,$0-8
726 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
727 // This means that since about 1972 the msb has been set, making the
728 // result of a call to STORE CLOCK (stck) a negative number.
729 // We clear the msb to make it positive.
730 STCK ret+0(FP) // serialises before and after call
731 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
732 SLD $1, R3
733 SRD $1, R3
734 MOVD R3, ret+0(FP)
735 RET
736
737// AES hashing not implemented for s390x
738TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
739 JMP runtime·memhashFallback(SB)
740TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
741 JMP runtime·strhashFallback(SB)
742TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
743 JMP runtime·memhash32Fallback(SB)
744TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
745 JMP runtime·memhash64Fallback(SB)
746
747TEXT runtime·return0(SB), NOSPLIT, $0
748 MOVW $0, R3
749 RET
750
751// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
752// Must obey the gcc calling convention.
753TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
754 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
755 MOVD g, R1
756 MOVD R10, R3
757 MOVD LR, R4
758 MOVD R11, R5
759
760 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
761 MOVD g_m(g), R2
762 MOVD m_curg(R2), R2
763 MOVD (g_stack+stack_hi)(R2), R2
764
765 MOVD R1, g
766 MOVD R3, R10
767 MOVD R4, LR
768 MOVD R5, R11
769 RET
770
771// The top-most function running on a goroutine
772// returns to goexit+PCQuantum.
773TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
774 BYTE $0x07; BYTE $0x00; // 2-byte nop
775 BL runtime·goexit1(SB) // does not return
776 // traceback from goexit1 must hit code range of goexit
777 BYTE $0x07; BYTE $0x00; // 2-byte nop
778
779TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
780 // Stores are already ordered on s390x, so this is just a
781 // compile barrier.
782 RET
783
784// This is called from .init_array and follows the platform, not Go, ABI.
785// We are overly conservative. We could only save the registers we use.
786// However, since this function is only called once per loaded module
787// performance is unimportant.
788TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
789 // Save R6-R15 in the register save area of the calling function.
790 // Don't bother saving F8-F15 as we aren't doing any calls.
791 STMG R6, R15, 48(R15)
792
793 // append the argument (passed in R2, as per the ELF ABI) to the
794 // moduledata linked list.
795 MOVD runtime·lastmoduledatap(SB), R1
796 MOVD R2, moduledata_next(R1)
797 MOVD R2, runtime·lastmoduledatap(SB)
798
799 // Restore R6-R15.
800 LMG 48(R15), R6, R15
801 RET
802
803TEXT ·checkASM(SB),NOSPLIT,$0-1
804 MOVB $1, ret+0(FP)
805 RET
806
807// gcWriteBarrier informs the GC about heap pointer writes.
808//
809// gcWriteBarrier does NOT follow the Go ABI. It accepts the
810// number of bytes of buffer needed in R9, and returns a pointer
811// to the buffer space in R9.
812// It clobbers R10 (the temp register) and R1 (used by PLT stub).
813// It does not clobber any other general-purpose registers,
814// but may clobber others (e.g., floating point registers).
815TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
816 // Save the registers clobbered by the fast path.
817 MOVD R4, 96(R15)
818retry:
819 MOVD g_m(g), R1
820 MOVD m_p(R1), R1
821 // Increment wbBuf.next position.
822 MOVD R9, R4
823 ADD (p_wbBuf+wbBuf_next)(R1), R4
824 // Is the buffer full?
825 MOVD (p_wbBuf+wbBuf_end)(R1), R10
826 CMPUBGT R4, R10, flush
827 // Commit to the larger buffer.
828 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
829 // Make return value (the original next position)
830 SUB R9, R4, R9
831 // Restore registers.
832 MOVD 96(R15), R4
833 RET
834
835flush:
836 // Save all general purpose registers since these could be
837 // clobbered by wbBufFlush and were not saved by the caller.
838 STMG R2, R3, 8(R15)
839 MOVD R0, 24(R15)
840 // R1 already saved.
841 // R4 already saved.
842 STMG R5, R12, 32(R15) // save R5 - R12
843 // R13 is g.
844 // R14 is LR.
845 // R15 is SP.
846
847 CALL runtime·wbBufFlush(SB)
848
849 LMG 8(R15), R2, R3 // restore R2 - R3
850 MOVD 24(R15), R0 // restore R0
851 LMG 32(R15), R5, R12 // restore R5 - R12
852 JMP retry
853
854TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
855 MOVD $8, R9
856 JMP gcWriteBarrier<>(SB)
857TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
858 MOVD $16, R9
859 JMP gcWriteBarrier<>(SB)
860TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
861 MOVD $24, R9
862 JMP gcWriteBarrier<>(SB)
863TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
864 MOVD $32, R9
865 JMP gcWriteBarrier<>(SB)
866TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
867 MOVD $40, R9
868 JMP gcWriteBarrier<>(SB)
869TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
870 MOVD $48, R9
871 JMP gcWriteBarrier<>(SB)
872TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
873 MOVD $56, R9
874 JMP gcWriteBarrier<>(SB)
875TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
876 MOVD $64, R9
877 JMP gcWriteBarrier<>(SB)
878
879// Note: these functions use a special calling convention to save generated code space.
880// Arguments are passed in registers, but the space for those arguments are allocated
881// in the caller's stack frame. These stubs write the args into that stack space and
882// then tail call to the corresponding runtime handler.
883// The tail call makes these stubs disappear in backtraces.
884TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
885 MOVD R0, x+0(FP)
886 MOVD R1, y+8(FP)
887 JMP runtime·goPanicIndex(SB)
888TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
889 MOVD R0, x+0(FP)
890 MOVD R1, y+8(FP)
891 JMP runtime·goPanicIndexU(SB)
892TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
893 MOVD R1, x+0(FP)
894 MOVD R2, y+8(FP)
895 JMP runtime·goPanicSliceAlen(SB)
896TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
897 MOVD R1, x+0(FP)
898 MOVD R2, y+8(FP)
899 JMP runtime·goPanicSliceAlenU(SB)
900TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
901 MOVD R1, x+0(FP)
902 MOVD R2, y+8(FP)
903 JMP runtime·goPanicSliceAcap(SB)
904TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
905 MOVD R1, x+0(FP)
906 MOVD R2, y+8(FP)
907 JMP runtime·goPanicSliceAcapU(SB)
908TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
909 MOVD R0, x+0(FP)
910 MOVD R1, y+8(FP)
911 JMP runtime·goPanicSliceB(SB)
912TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
913 MOVD R0, x+0(FP)
914 MOVD R1, y+8(FP)
915 JMP runtime·goPanicSliceBU(SB)
916TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
917 MOVD R2, x+0(FP)
918 MOVD R3, y+8(FP)
919 JMP runtime·goPanicSlice3Alen(SB)
920TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
921 MOVD R2, x+0(FP)
922 MOVD R3, y+8(FP)
923 JMP runtime·goPanicSlice3AlenU(SB)
924TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
925 MOVD R2, x+0(FP)
926 MOVD R3, y+8(FP)
927 JMP runtime·goPanicSlice3Acap(SB)
928TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
929 MOVD R2, x+0(FP)
930 MOVD R3, y+8(FP)
931 JMP runtime·goPanicSlice3AcapU(SB)
932TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
933 MOVD R1, x+0(FP)
934 MOVD R2, y+8(FP)
935 JMP runtime·goPanicSlice3B(SB)
936TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
937 MOVD R1, x+0(FP)
938 MOVD R2, y+8(FP)
939 JMP runtime·goPanicSlice3BU(SB)
940TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
941 MOVD R0, x+0(FP)
942 MOVD R1, y+8(FP)
943 JMP runtime·goPanicSlice3C(SB)
944TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
945 MOVD R0, x+0(FP)
946 MOVD R1, y+8(FP)
947 JMP runtime·goPanicSlice3CU(SB)
948TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
949 MOVD R2, x+0(FP)
950 MOVD R3, y+8(FP)
951 JMP runtime·goPanicSliceConvert(SB)
View as plain text