Text file
src/runtime/asm_ppc64x.s
Documentation: runtime
1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build ppc64 || ppc64le
6
7#include "go_asm.h"
8#include "go_tls.h"
9#include "funcdata.h"
10#include "textflag.h"
11#include "asm_ppc64x.h"
12#include "cgo/abi_ppc64x.h"
13
14// This is called using the host ABI. argc and argv arguments
15// should be in R3 and R4 respectively.
16TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0
17 // Start with standard C stack frame layout and linkage, allocate
18 // 16 bytes of argument space, save callee-save regs, and set R0 to $0.
19 // Allocate an extra 16 bytes to account for the larger fixed frame size
20 // of aix/elfv1 (48 vs 32) to ensure 16 bytes of parameter save space.
21 STACK_AND_SAVE_HOST_TO_GO_ABI(32)
22 // The above will not preserve R2 (TOC). Save it in case Go is
23 // compiled without a TOC pointer (e.g -buildmode=default).
24 MOVD R2, 24(R1)
25
26 MOVD R3, _rt0_ppc64x_lib_argc<>(SB)
27 MOVD R4, _rt0_ppc64x_lib_argv<>(SB)
28
29 // Synchronous initialization.
30 MOVD $runtime·reginit(SB), R12
31 MOVD R12, CTR
32 BL (CTR)
33
34 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
35 MOVD R0, g
36
37#ifdef GOOS_aix
38 // See runtime/cgo/gcc_aix_ppc64.c
39 MOVBZ runtime·isarchive(SB), R3 // Check buildmode = c-archive
40 CMP $0, R3
41 BEQ skipInit
42#endif
43
44 MOVD $runtime·libInit(SB), R12
45 MOVD R12, CTR
46 BL (CTR)
47
48skipInit:
49 // Restore and return to ELFv2 caller.
50 UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(32)
51 RET
52
53TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
54 MOVD _rt0_ppc64x_lib_argc<>(SB), R3
55 MOVD _rt0_ppc64x_lib_argv<>(SB), R4
56 MOVD $runtime·rt0_go(SB), R12
57 MOVD R12, CTR
58 BR (CTR)
59
60DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0
61GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8
62DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0
63GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8
64
65
66#ifdef GOOS_aix
67#define cgoCalleeStackSize 48
68#else
69#define cgoCalleeStackSize 32
70#endif
71
72TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
73 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
74
75 // initialize essential registers
76 BL runtime·reginit(SB)
77
78 SUB $(FIXED_FRAME+16), R1
79 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
80 MOVW R3, FIXED_FRAME+0(R1) // argc
81 MOVD R4, FIXED_FRAME+8(R1) // argv
82
83 // create istack out of the given (operating system) stack.
84 // _cgo_init may update stackguard.
85 MOVD $runtime·g0(SB), g
86 BL runtime·save_g(SB)
87 MOVD $(-64*1024), R31
88 ADD R31, R1, R3
89 MOVD R3, g_stackguard0(g)
90 MOVD R3, g_stackguard1(g)
91 MOVD R3, (g_stack+stack_lo)(g)
92 MOVD R1, (g_stack+stack_hi)(g)
93
94 // If there is a _cgo_init, call it using the gcc ABI.
95 MOVD _cgo_init(SB), R12
96 CMP R12, $0
97 BEQ nocgo
98
99#ifdef GO_PPC64X_HAS_FUNCDESC
100 // Load the real entry address from the first slot of the function descriptor.
101 MOVD 8(R12), R2
102 MOVD (R12), R12
103#endif
104 MOVD R12, CTR // r12 = "global function entry point"
105 MOVD R13, R5 // arg 2: TLS base pointer
106 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
107 MOVD g, R3 // arg 0: G
108 // C functions expect 32 (48 for AIX) bytes of space on caller
109 // stack frame and a 16-byte aligned R1
110 MOVD R1, R14 // save current stack
111 SUB $cgoCalleeStackSize, R1 // reserve the callee area
112 RLDCR $0, R1, $~15, R1 // 16-byte align
113 BL (CTR) // may clobber R0, R3-R12
114 MOVD R14, R1 // restore stack
115#ifndef GOOS_aix
116 MOVD 24(R1), R2
117#endif
118 XOR R0, R0 // fix R0
119
120nocgo:
121 // update stackguard after _cgo_init
122 MOVD (g_stack+stack_lo)(g), R3
123 ADD $const_stackGuard, R3
124 MOVD R3, g_stackguard0(g)
125 MOVD R3, g_stackguard1(g)
126
127 // set the per-goroutine and per-mach "registers"
128 MOVD $runtime·m0(SB), R3
129
130 // save m->g0 = g0
131 MOVD g, m_g0(R3)
132 // save m0 to g0->m
133 MOVD R3, g_m(g)
134
135 BL runtime·check(SB)
136
137 // args are already prepared
138 BL runtime·args(SB)
139 BL runtime·osinit(SB)
140 BL runtime·schedinit(SB)
141
142 // create a new goroutine to start program
143 MOVD $runtime·mainPC(SB), R3 // entry
144 MOVDU R3, -8(R1)
145 MOVDU R0, -8(R1)
146 MOVDU R0, -8(R1)
147 MOVDU R0, -8(R1)
148 MOVDU R0, -8(R1)
149 BL runtime·newproc(SB)
150 ADD $(8+FIXED_FRAME), R1
151
152 // start this M
153 BL runtime·mstart(SB)
154 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
155 // intended to be called by debuggers.
156#ifdef GOOS_linux
157 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
158 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
159#endif
160 MOVD R0, 0(R0)
161 RET
162
163DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
164GLOBL runtime·mainPC(SB),RODATA,$8
165
166TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 TW $31, R0, R0
168 RET
169
170TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173// Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall1
174TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
175 // crosscall1 and crosscall2 need to reginit, but can't
176 // get at the 'runtime.reginit' symbol.
177 BR runtime·reginit(SB)
178
179TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
180 // set R0 to zero, it's expected by the toolchain
181 XOR R0, R0
182 RET
183
184TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
185 BL runtime·mstart0(SB)
186 RET // not reached
187
188/*
189 * go-routine
190 */
191
192// void gogo(Gobuf*)
193// restore state from Gobuf; longjmp
194TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
195 MOVD buf+0(FP), R5
196 MOVD gobuf_g(R5), R6
197 MOVD 0(R6), R4 // make sure g != nil
198 BR gogo<>(SB)
199
200TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
201 MOVD R6, g
202 BL runtime·save_g(SB)
203
204 MOVD gobuf_sp(R5), R1
205 MOVD gobuf_lr(R5), R31
206#ifndef GOOS_aix
207 MOVD 24(R1), R2 // restore R2
208#endif
209 MOVD R31, LR
210 MOVD gobuf_ctxt(R5), R11
211 MOVD R0, gobuf_sp(R5)
212 MOVD R0, gobuf_lr(R5)
213 MOVD R0, gobuf_ctxt(R5)
214 CMP R0, R0 // set condition codes for == test, needed by stack split
215 MOVD gobuf_pc(R5), R12
216 MOVD R12, CTR
217 BR (CTR)
218
219// void mcall(fn func(*g))
220// Switch to m->g0's stack, call fn(g).
221// Fn must never return. It should gogo(&g->sched)
222// to keep running g.
223TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
224 // Save caller state in g->sched
225 // R11 should be safe across save_g??
226 MOVD R3, R11
227 MOVD R1, (g_sched+gobuf_sp)(g)
228 MOVD LR, R31
229 MOVD R31, (g_sched+gobuf_pc)(g)
230 MOVD R0, (g_sched+gobuf_lr)(g)
231
232 // Switch to m->g0 & its stack, call fn.
233 MOVD g, R3
234 MOVD g_m(g), R8
235 MOVD m_g0(R8), g
236 BL runtime·save_g(SB)
237 CMP g, R3
238 BNE 2(PC)
239 BR runtime·badmcall(SB)
240 MOVD 0(R11), R12 // code pointer
241 MOVD R12, CTR
242 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
243 // Don't need to do anything special for regabiargs here
244 // R3 is g; stack is set anyway
245 MOVDU R3, -8(R1)
246 MOVDU R0, -8(R1)
247 MOVDU R0, -8(R1)
248 MOVDU R0, -8(R1)
249 MOVDU R0, -8(R1)
250 BL (CTR)
251 MOVD 24(R1), R2
252 BR runtime·badmcall2(SB)
253
254// systemstack_switch is a dummy routine that systemstack leaves at the bottom
255// of the G stack. We need to distinguish the routine that
256// lives at the bottom of the G stack from the one that lives
257// at the top of the system stack because the one at the top of
258// the system stack terminates the stack walk (see topofstack()).
259TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
260 // We have several undefs here so that 16 bytes past
261 // $runtime·systemstack_switch lies within them whether or not the
262 // instructions that derive r2 from r12 are there.
263 UNDEF
264 UNDEF
265 UNDEF
266 BL (LR) // make sure this function is not leaf
267 RET
268
269// func systemstack(fn func())
270TEXT runtime·systemstack(SB), NOSPLIT, $0-8
271 MOVD fn+0(FP), R3 // R3 = fn
272 MOVD R3, R11 // context
273 MOVD g_m(g), R4 // R4 = m
274
275 MOVD m_gsignal(R4), R5 // R5 = gsignal
276 CMP g, R5
277 BEQ noswitch
278
279 MOVD m_g0(R4), R5 // R5 = g0
280 CMP g, R5
281 BEQ noswitch
282
283 MOVD m_curg(R4), R6
284 CMP g, R6
285 BEQ switch
286
287 // Bad: g is not gsignal, not g0, not curg. What is it?
288 // Hide call from linker nosplit analysis.
289 MOVD $runtime·badsystemstack(SB), R12
290 MOVD R12, CTR
291 BL (CTR)
292 BL runtime·abort(SB)
293
294switch:
295 // save our state in g->sched. Pretend to
296 // be systemstack_switch if the G stack is scanned.
297 BL gosave_systemstack_switch<>(SB)
298
299 // switch to g0
300 MOVD R5, g
301 BL runtime·save_g(SB)
302 MOVD (g_sched+gobuf_sp)(g), R1
303
304 // call target function
305 MOVD 0(R11), R12 // code pointer
306 MOVD R12, CTR
307 BL (CTR)
308
309 // restore TOC pointer. It seems unlikely that we will use systemstack
310 // to call a function defined in another module, but the results of
311 // doing so would be so confusing that it's worth doing this.
312 MOVD g_m(g), R3
313 MOVD m_curg(R3), g
314 MOVD (g_sched+gobuf_sp)(g), R3
315#ifndef GOOS_aix
316 MOVD 24(R3), R2
317#endif
318 // switch back to g
319 MOVD g_m(g), R3
320 MOVD m_curg(R3), g
321 BL runtime·save_g(SB)
322 MOVD (g_sched+gobuf_sp)(g), R1
323 MOVD R0, (g_sched+gobuf_sp)(g)
324 RET
325
326noswitch:
327 // already on m stack, just call directly
328 // On other arches we do a tail call here, but it appears to be
329 // impossible to tail call a function pointer in shared mode on
330 // ppc64 because the caller is responsible for restoring the TOC.
331 MOVD 0(R11), R12 // code pointer
332 MOVD R12, CTR
333 BL (CTR)
334#ifndef GOOS_aix
335 MOVD 24(R1), R2
336#endif
337 RET
338
339// func switchToCrashStack0(fn func())
340TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
341 MOVD R3, R11 // context register
342 MOVD g_m(g), R3 // curm
343
344 // set g to gcrash
345 MOVD $runtime·gcrash(SB), g // g = &gcrash
346 CALL runtime·save_g(SB) // clobbers R31
347 MOVD R3, g_m(g) // g.m = curm
348 MOVD g, m_g0(R3) // curm.g0 = g
349
350 // switch to crashstack
351 MOVD (g_stack+stack_hi)(g), R3
352 SUB $(4*8), R3
353 MOVD R3, R1
354
355 // call target function
356 MOVD 0(R11), R12 // code pointer
357 MOVD R12, CTR
358 BL (CTR)
359
360 // should never return
361 CALL runtime·abort(SB)
362 UNDEF
363
364/*
365 * support for morestack
366 */
367
368// Called during function prolog when more stack is needed.
369// Caller has already loaded:
370// R3: framesize, R4: argsize, R5: LR
371//
372// The traceback routines see morestack on a g0 as being
373// the top of a stack (for example, morestack calling newstack
374// calling the scheduler calling newm calling gc), so we must
375// record an argument size. For that purpose, it has no arguments.
376TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
377 // Called from f.
378 // Set g->sched to context in f.
379 MOVD R1, (g_sched+gobuf_sp)(g)
380 MOVD LR, R8
381 MOVD R8, (g_sched+gobuf_pc)(g)
382 MOVD R5, (g_sched+gobuf_lr)(g)
383 MOVD R11, (g_sched+gobuf_ctxt)(g)
384
385 // Cannot grow scheduler stack (m->g0).
386 MOVD g_m(g), R7
387 MOVD m_g0(R7), R8
388 CMP g, R8
389 BNE 3(PC)
390 BL runtime·badmorestackg0(SB)
391 BL runtime·abort(SB)
392
393 // Cannot grow signal stack (m->gsignal).
394 MOVD m_gsignal(R7), R8
395 CMP g, R8
396 BNE 3(PC)
397 BL runtime·badmorestackgsignal(SB)
398 BL runtime·abort(SB)
399
400 // Called from f.
401 // Set m->morebuf to f's caller.
402 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
403 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
404 MOVD g, (m_morebuf+gobuf_g)(R7)
405
406 // Call newstack on m->g0's stack.
407 MOVD m_g0(R7), g
408 BL runtime·save_g(SB)
409 MOVD (g_sched+gobuf_sp)(g), R1
410 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
411 BL runtime·newstack(SB)
412
413 // Not reached, but make sure the return PC from the call to newstack
414 // is still in this function, and not the beginning of the next.
415 UNDEF
416
417TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
418 // Force SPWRITE. This function doesn't actually write SP,
419 // but it is called with a special calling convention where
420 // the caller doesn't save LR on stack but passes it as a
421 // register (R5), and the unwinder currently doesn't understand.
422 // Make it SPWRITE to stop unwinding. (See issue 54332)
423 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
424 // has a special affect on Power8,9,10 by lowering the thread
425 // priority and causing a slowdown in execution time
426
427 OR R0, R1
428 MOVD R0, R11
429 BR runtime·morestack(SB)
430
431// reflectcall: call a function with the given argument list
432// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
433// we don't have variable-sized frames, so we use a small number
434// of constant-sized-frame functions to encode a few bits of size in the pc.
435// Caution: ugly multiline assembly macros in your future!
436
437#define DISPATCH(NAME,MAXSIZE) \
438 MOVD $MAXSIZE, R31; \
439 CMP R3, R31; \
440 BGT 4(PC); \
441 MOVD $NAME(SB), R12; \
442 MOVD R12, CTR; \
443 BR (CTR)
444// Note: can't just "BR NAME(SB)" - bad inlining results.
445
446TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
447 MOVWZ frameSize+32(FP), R3
448 DISPATCH(runtime·call16, 16)
449 DISPATCH(runtime·call32, 32)
450 DISPATCH(runtime·call64, 64)
451 DISPATCH(runtime·call128, 128)
452 DISPATCH(runtime·call256, 256)
453 DISPATCH(runtime·call512, 512)
454 DISPATCH(runtime·call1024, 1024)
455 DISPATCH(runtime·call2048, 2048)
456 DISPATCH(runtime·call4096, 4096)
457 DISPATCH(runtime·call8192, 8192)
458 DISPATCH(runtime·call16384, 16384)
459 DISPATCH(runtime·call32768, 32768)
460 DISPATCH(runtime·call65536, 65536)
461 DISPATCH(runtime·call131072, 131072)
462 DISPATCH(runtime·call262144, 262144)
463 DISPATCH(runtime·call524288, 524288)
464 DISPATCH(runtime·call1048576, 1048576)
465 DISPATCH(runtime·call2097152, 2097152)
466 DISPATCH(runtime·call4194304, 4194304)
467 DISPATCH(runtime·call8388608, 8388608)
468 DISPATCH(runtime·call16777216, 16777216)
469 DISPATCH(runtime·call33554432, 33554432)
470 DISPATCH(runtime·call67108864, 67108864)
471 DISPATCH(runtime·call134217728, 134217728)
472 DISPATCH(runtime·call268435456, 268435456)
473 DISPATCH(runtime·call536870912, 536870912)
474 DISPATCH(runtime·call1073741824, 1073741824)
475 MOVD $runtime·badreflectcall(SB), R12
476 MOVD R12, CTR
477 BR (CTR)
478
479#define CALLFN(NAME,MAXSIZE) \
480TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
481 NO_LOCAL_POINTERS; \
482 /* copy arguments to stack */ \
483 MOVD stackArgs+16(FP), R3; \
484 MOVWZ stackArgsSize+24(FP), R4; \
485 MOVD R1, R5; \
486 CMP R4, $8; \
487 BLT tailsetup; \
488 /* copy 8 at a time if possible */ \
489 ADD $(FIXED_FRAME-8), R5; \
490 SUB $8, R3; \
491top: \
492 MOVDU 8(R3), R7; \
493 MOVDU R7, 8(R5); \
494 SUB $8, R4; \
495 CMP R4, $8; \
496 BGE top; \
497 /* handle remaining bytes */ \
498 CMP $0, R4; \
499 BEQ callfn; \
500 ADD $7, R3; \
501 ADD $7, R5; \
502 BR tail; \
503tailsetup: \
504 CMP $0, R4; \
505 BEQ callfn; \
506 ADD $(FIXED_FRAME-1), R5; \
507 SUB $1, R3; \
508tail: \
509 MOVBU 1(R3), R6; \
510 MOVBU R6, 1(R5); \
511 SUB $1, R4; \
512 CMP $0, R4; \
513 BGT tail; \
514callfn: \
515 /* call function */ \
516 MOVD f+8(FP), R11; \
517#ifdef GOOS_aix \
518 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
519 /* So it manually triggers it */ \
520 CMP R11, $0 \
521 BNE 2(PC) \
522 MOVD R0, 0(R0) \
523#endif \
524 MOVD regArgs+40(FP), R20; \
525 BL runtime·unspillArgs(SB); \
526 MOVD (R11), R12; \
527 MOVD R12, CTR; \
528 PCDATA $PCDATA_StackMapIndex, $0; \
529 BL (CTR); \
530#ifndef GOOS_aix \
531 MOVD 24(R1), R2; \
532#endif \
533 /* copy return values back */ \
534 MOVD regArgs+40(FP), R20; \
535 BL runtime·spillArgs(SB); \
536 MOVD stackArgsType+0(FP), R7; \
537 MOVD stackArgs+16(FP), R3; \
538 MOVWZ stackArgsSize+24(FP), R4; \
539 MOVWZ stackRetOffset+28(FP), R6; \
540 ADD $FIXED_FRAME, R1, R5; \
541 ADD R6, R5; \
542 ADD R6, R3; \
543 SUB R6, R4; \
544 BL callRet<>(SB); \
545 RET
546
547// callRet copies return values back at the end of call*. This is a
548// separate function so it can allocate stack space for the arguments
549// to reflectcallmove. It does not follow the Go ABI; it expects its
550// arguments in registers.
551TEXT callRet<>(SB), NOSPLIT, $40-0
552 NO_LOCAL_POINTERS
553 MOVD R7, FIXED_FRAME+0(R1)
554 MOVD R3, FIXED_FRAME+8(R1)
555 MOVD R5, FIXED_FRAME+16(R1)
556 MOVD R4, FIXED_FRAME+24(R1)
557 MOVD R20, FIXED_FRAME+32(R1)
558 BL runtime·reflectcallmove(SB)
559 RET
560
561CALLFN(·call16, 16)
562CALLFN(·call32, 32)
563CALLFN(·call64, 64)
564CALLFN(·call128, 128)
565CALLFN(·call256, 256)
566CALLFN(·call512, 512)
567CALLFN(·call1024, 1024)
568CALLFN(·call2048, 2048)
569CALLFN(·call4096, 4096)
570CALLFN(·call8192, 8192)
571CALLFN(·call16384, 16384)
572CALLFN(·call32768, 32768)
573CALLFN(·call65536, 65536)
574CALLFN(·call131072, 131072)
575CALLFN(·call262144, 262144)
576CALLFN(·call524288, 524288)
577CALLFN(·call1048576, 1048576)
578CALLFN(·call2097152, 2097152)
579CALLFN(·call4194304, 4194304)
580CALLFN(·call8388608, 8388608)
581CALLFN(·call16777216, 16777216)
582CALLFN(·call33554432, 33554432)
583CALLFN(·call67108864, 67108864)
584CALLFN(·call134217728, 134217728)
585CALLFN(·call268435456, 268435456)
586CALLFN(·call536870912, 536870912)
587CALLFN(·call1073741824, 1073741824)
588
589TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0-4
590 MOVW cycles+0(FP), R7
591 CMP $0, R7
592 BEQ done
593 // POWER does not have a pause/yield instruction equivalent.
594 // Instead, we can lower the program priority by setting the
595 // Program Priority Register prior to the wait loop and set it
596 // back to default afterwards. On Linux, the default priority is
597 // medium-low. For details, see page 837 of the ISA 3.0.
598 OR R1, R1, R1 // Set PPR priority to low
599again:
600 SUB $1, R7
601 CMP $0, R7
602 BNE again
603 OR R6, R6, R6 // Set PPR priority back to medium-low
604done:
605 RET
606
607// Save state of caller into g->sched,
608// but using fake PC from systemstack_switch.
609// Must only be called from functions with no locals ($0)
610// or else unwinding from systemstack_switch is incorrect.
611// Smashes R31.
612TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
613 MOVD $runtime·systemstack_switch(SB), R31
614 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
615 MOVD R31, (g_sched+gobuf_pc)(g)
616 MOVD R1, (g_sched+gobuf_sp)(g)
617 MOVD R0, (g_sched+gobuf_lr)(g)
618 // Assert ctxt is zero. See func save.
619 MOVD (g_sched+gobuf_ctxt)(g), R31
620 CMP R31, $0
621 BEQ 2(PC)
622 BL runtime·abort(SB)
623 RET
624
625#ifdef GOOS_aix
626#define asmcgocallSaveOffset cgoCalleeStackSize + 8
627#else
628#define asmcgocallSaveOffset cgoCalleeStackSize
629#endif
630
631// func asmcgocall_no_g(fn, arg unsafe.Pointer)
632// Call fn(arg) aligned appropriately for the gcc ABI.
633// Called on a system stack, and there may be no g yet (during needm).
634TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
635 MOVD fn+0(FP), R3
636 MOVD arg+8(FP), R4
637
638 MOVD R1, R15
639 SUB $(asmcgocallSaveOffset+8), R1
640 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
641 MOVD R15, asmcgocallSaveOffset(R1)
642
643 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
644
645 // This is a "global call", so put the global entry point in r12
646 MOVD R3, R12
647
648#ifdef GO_PPC64X_HAS_FUNCDESC
649 // Load the real entry address from the first slot of the function descriptor.
650 MOVD 8(R12), R2
651 MOVD (R12), R12
652#endif
653 MOVD R12, CTR
654 MOVD R4, R3 // arg in r3
655 BL (CTR)
656
657 // C code can clobber R0, so set it back to 0. F27-F31 are
658 // callee save, so we don't need to recover those.
659 XOR R0, R0
660
661 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
662#ifndef GOOS_aix
663 MOVD 24(R1), R2
664#endif
665
666 RET
667
668// func asmcgocall(fn, arg unsafe.Pointer) int32
669// Call fn(arg) on the scheduler stack,
670// aligned appropriately for the gcc ABI.
671// See cgocall.go for more details.
672TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
673 // R3 = fn
674 // R4 = arg
675
676 MOVD R1, R7 // save original stack pointer
677 CMP $0, g
678 BEQ nosave
679 MOVD g, R5
680
681 // Figure out if we need to switch to m->g0 stack.
682 // We get called to create new OS threads too, and those
683 // come in on the m->g0 stack already. Or we might already
684 // be on the m->gsignal stack.
685 MOVD g_m(g), R8
686 MOVD m_gsignal(R8), R6
687 CMP R6, g
688 BEQ nosave
689 MOVD m_g0(R8), R6
690 CMP R6, g
691 BEQ nosave
692
693 BL gosave_systemstack_switch<>(SB)
694 MOVD R6, g
695 BL runtime·save_g(SB)
696 MOVD (g_sched+gobuf_sp)(g), R1
697
698 // Now on a scheduling stack (a pthread-created stack).
699#ifdef GOOS_aix
700 // Create a fake LR to improve backtrace.
701 MOVD $runtime·asmcgocall(SB), R6
702 MOVD R6, 16(R1)
703 // AIX also saves one argument on the stack.
704 SUB $8, R1
705#endif
706 // Save room for two of our pointers, plus the callee
707 // save area that lives on the caller stack.
708 // Do arithmetics in R10 to hide from the assembler
709 // counting it as SP delta, which is irrelevant as we are
710 // on the system stack.
711 SUB $(asmcgocallSaveOffset+16), R1, R10
712 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
713 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
714 MOVD (g_stack+stack_hi)(R5), R5
715 SUB R7, R5
716 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
717#ifdef GOOS_aix
718 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
719#else
720 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
721#endif
722 // This is a "global call", so put the global entry point in r12
723 MOVD R3, R12
724
725#ifdef GO_PPC64X_HAS_FUNCDESC
726 // Load the real entry address from the first slot of the function descriptor.
727 MOVD 8(R12), R2
728 MOVD (R12), R12
729#endif
730 MOVD R12, CTR
731 MOVD R4, R3 // arg in r3
732 BL (CTR)
733
734 // Reinitialise zero value register.
735 XOR R0, R0
736
737 // Restore g, stack pointer, toc pointer.
738 // R3 is errno, so don't touch it
739 MOVD (asmcgocallSaveOffset+8)(R1), g
740 MOVD (g_stack+stack_hi)(g), R5
741 MOVD asmcgocallSaveOffset(R1), R6
742 SUB R6, R5
743#ifndef GOOS_aix
744 MOVD 24(R5), R2
745#endif
746 MOVD R5, R1
747 BL runtime·save_g(SB)
748
749 // ret = R3
750 RET
751
752nosave:
753 // Running on a system stack, perhaps even without a g.
754 // Having no g can happen during thread creation or thread teardown.
755 // This code is like the above sequence but without saving/restoring g
756 // and without worrying about the stack moving out from under us
757 // (because we're on a system stack, not a goroutine stack).
758
759 SUB $(asmcgocallSaveOffset+8), R1, R10
760 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
761 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
762
763 MOVD R3, R12 // fn
764#ifdef GO_PPC64X_HAS_FUNCDESC
765 // Load the real entry address from the first slot of the function descriptor.
766 MOVD 8(R12), R2
767 MOVD (R12), R12
768#endif
769 MOVD R12, CTR
770 MOVD R4, R3 // arg
771 BL (CTR)
772
773 // Reinitialise zero value register.
774 XOR R0, R0
775
776 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
777#ifndef GOOS_aix
778 MOVD 24(R1), R2
779#endif
780 // ret = R3
781 RET
782
783// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
784// See cgocall.go for more details.
785TEXT ·cgocallback(SB),NOSPLIT,$24-24
786 NO_LOCAL_POINTERS
787
788 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
789 // It is used to dropm while thread is exiting.
790 MOVD fn+0(FP), R5
791 CMP R5, $0
792 BNE loadg
793 // Restore the g from frame.
794 MOVD frame+8(FP), g
795 BR dropm
796
797loadg:
798 // Load m and g from thread-local storage.
799#ifndef GOOS_openbsd
800 MOVBZ runtime·iscgo(SB), R3
801 CMP R3, $0
802 BEQ nocgo
803#endif
804 BL runtime·load_g(SB)
805nocgo:
806
807 // If g is nil, Go did not create the current thread,
808 // or if this thread never called into Go on pthread platforms.
809 // Call needm to obtain one for temporary use.
810 // In this case, we're running on the thread stack, so there's
811 // lots of space, but the linker doesn't know. Hide the call from
812 // the linker analysis by using an indirect call.
813 CMP g, $0
814 BEQ needm
815
816 MOVD g_m(g), R8
817 MOVD R8, savedm-8(SP)
818 BR havem
819
820needm:
821 MOVD g, savedm-8(SP) // g is zero, so is m.
822 MOVD $runtime·needAndBindM(SB), R12
823 MOVD R12, CTR
824 BL (CTR)
825
826 // Set m->sched.sp = SP, so that if a panic happens
827 // during the function we are about to execute, it will
828 // have a valid SP to run on the g0 stack.
829 // The next few lines (after the havem label)
830 // will save this SP onto the stack and then write
831 // the same SP back to m->sched.sp. That seems redundant,
832 // but if an unrecovered panic happens, unwindm will
833 // restore the g->sched.sp from the stack location
834 // and then systemstack will try to use it. If we don't set it here,
835 // that restored SP will be uninitialized (typically 0) and
836 // will not be usable.
837 MOVD g_m(g), R8
838 MOVD m_g0(R8), R3
839 MOVD R1, (g_sched+gobuf_sp)(R3)
840
841havem:
842 // Now there's a valid m, and we're running on its m->g0.
843 // Save current m->g0->sched.sp on stack and then set it to SP.
844 // Save current sp in m->g0->sched.sp in preparation for
845 // switch back to m->curg stack.
846 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
847 MOVD m_g0(R8), R3
848 MOVD (g_sched+gobuf_sp)(R3), R4
849 MOVD R4, savedsp-24(SP) // must match frame size
850 MOVD R1, (g_sched+gobuf_sp)(R3)
851
852 // Switch to m->curg stack and call runtime.cgocallbackg.
853 // Because we are taking over the execution of m->curg
854 // but *not* resuming what had been running, we need to
855 // save that information (m->curg->sched) so we can restore it.
856 // We can restore m->curg->sched.sp easily, because calling
857 // runtime.cgocallbackg leaves SP unchanged upon return.
858 // To save m->curg->sched.pc, we push it onto the curg stack and
859 // open a frame the same size as cgocallback's g0 frame.
860 // Once we switch to the curg stack, the pushed PC will appear
861 // to be the return PC of cgocallback, so that the traceback
862 // will seamlessly trace back into the earlier calls.
863 MOVD m_curg(R8), g
864 BL runtime·save_g(SB)
865 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
866 MOVD (g_sched+gobuf_pc)(g), R5
867 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
868 // Gather our arguments into registers.
869 MOVD fn+0(FP), R5
870 MOVD frame+8(FP), R6
871 MOVD ctxt+16(FP), R7
872 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
873 MOVD R5, FIXED_FRAME+0(R1)
874 MOVD R6, FIXED_FRAME+8(R1)
875 MOVD R7, FIXED_FRAME+16(R1)
876
877 MOVD $runtime·cgocallbackg(SB), R12
878 MOVD R12, CTR
879 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
880
881 // Restore g->sched (== m->curg->sched) from saved values.
882 MOVD 0(R1), R5
883 MOVD R5, (g_sched+gobuf_pc)(g)
884 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
885 MOVD R4, (g_sched+gobuf_sp)(g)
886
887 // Switch back to m->g0's stack and restore m->g0->sched.sp.
888 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
889 // so we do not have to restore it.)
890 MOVD g_m(g), R8
891 MOVD m_g0(R8), g
892 BL runtime·save_g(SB)
893 MOVD (g_sched+gobuf_sp)(g), R1
894 MOVD savedsp-24(SP), R4 // must match frame size
895 MOVD R4, (g_sched+gobuf_sp)(g)
896
897 // If the m on entry was nil, we called needm above to borrow an m,
898 // 1. for the duration of the call on non-pthread platforms,
899 // 2. or the duration of the C thread alive on pthread platforms.
900 // If the m on entry wasn't nil,
901 // 1. the thread might be a Go thread,
902 // 2. or it wasn't the first call from a C thread on pthread platforms,
903 // since then we skip dropm to reuse the m in the first call.
904 MOVD savedm-8(SP), R6
905 CMP R6, $0
906 BNE droppedm
907
908 // Skip dropm to reuse it in the next call, when a pthread key has been created.
909 MOVD _cgo_pthread_key_created(SB), R6
910 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
911 CMP R6, $0
912 BEQ dropm
913 MOVD (R6), R6
914 CMP R6, $0
915 BNE droppedm
916
917dropm:
918 MOVD $runtime·dropm(SB), R12
919 MOVD R12, CTR
920 BL (CTR)
921droppedm:
922
923 // Done!
924 RET
925
926// void setg(G*); set g. for use by needm.
927TEXT runtime·setg(SB), NOSPLIT, $0-8
928 MOVD gg+0(FP), g
929 // This only happens if iscgo, so jump straight to save_g
930 BL runtime·save_g(SB)
931 RET
932
933#ifdef GO_PPC64X_HAS_FUNCDESC
934DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
935TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
936#else
937TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
938#endif
939 // The standard prologue clobbers R31, which is callee-save in
940 // the C ABI, so we have to use $-8-0 and save LR ourselves.
941 MOVD LR, R4
942 // Also save g and R31, since they're callee-save in C ABI
943 MOVD R31, R5
944 MOVD g, R6
945
946 MOVD R3, g
947 BL runtime·save_g(SB)
948
949 MOVD R6, g
950 MOVD R5, R31
951 MOVD R4, LR
952 RET
953
954TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
955 MOVW (R0), R0
956 UNDEF
957
958#define TBR 268
959
960// int64 runtime·cputicks(void)
961TEXT runtime·cputicks(SB),NOSPLIT,$0-8
962 MOVD SPR(TBR), R3
963 MOVD R3, ret+0(FP)
964 RET
965
966// spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
967TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
968 MOVD R3, 0(R20)
969 MOVD R4, 8(R20)
970 MOVD R5, 16(R20)
971 MOVD R6, 24(R20)
972 MOVD R7, 32(R20)
973 MOVD R8, 40(R20)
974 MOVD R9, 48(R20)
975 MOVD R10, 56(R20)
976 MOVD R14, 64(R20)
977 MOVD R15, 72(R20)
978 MOVD R16, 80(R20)
979 MOVD R17, 88(R20)
980 FMOVD F1, 96(R20)
981 FMOVD F2, 104(R20)
982 FMOVD F3, 112(R20)
983 FMOVD F4, 120(R20)
984 FMOVD F5, 128(R20)
985 FMOVD F6, 136(R20)
986 FMOVD F7, 144(R20)
987 FMOVD F8, 152(R20)
988 FMOVD F9, 160(R20)
989 FMOVD F10, 168(R20)
990 FMOVD F11, 176(R20)
991 FMOVD F12, 184(R20)
992 RET
993
994// unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
995TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
996 MOVD 0(R20), R3
997 MOVD 8(R20), R4
998 MOVD 16(R20), R5
999 MOVD 24(R20), R6
1000 MOVD 32(R20), R7
1001 MOVD 40(R20), R8
1002 MOVD 48(R20), R9
1003 MOVD 56(R20), R10
1004 MOVD 64(R20), R14
1005 MOVD 72(R20), R15
1006 MOVD 80(R20), R16
1007 MOVD 88(R20), R17
1008 FMOVD 96(R20), F1
1009 FMOVD 104(R20), F2
1010 FMOVD 112(R20), F3
1011 FMOVD 120(R20), F4
1012 FMOVD 128(R20), F5
1013 FMOVD 136(R20), F6
1014 FMOVD 144(R20), F7
1015 FMOVD 152(R20), F8
1016 FMOVD 160(R20), F9
1017 FMOVD 168(R20), F10
1018 FMOVD 176(R20), F11
1019 FMOVD 184(R20), F12
1020 RET
1021
1022// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1023// Must obey the gcc calling convention.
1024#ifdef GOOS_aix
1025// On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
1026// be a longcall in order to prevent trampolines from ld.
1027TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1028#else
1029TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1030#endif
1031 // g (R30) and R31 are callee-save in the C ABI, so save them
1032 MOVD g, R4
1033 MOVD R31, R5
1034 MOVD LR, R6
1035
1036 BL runtime·load_g(SB) // clobbers g (R30), R31
1037 MOVD g_m(g), R3
1038 MOVD m_curg(R3), R3
1039 MOVD (g_stack+stack_hi)(R3), R3
1040
1041 MOVD R4, g
1042 MOVD R5, R31
1043 MOVD R6, LR
1044 RET
1045
1046// The top-most function running on a goroutine
1047// returns to goexit+PCQuantum.
1048//
1049// When dynamically linking Go, it can be returned to from a function
1050// implemented in a different module and so needs to reload the TOC pointer
1051// from the stack (although this function declares that it does not set up x-a
1052// frame, newproc1 does in fact allocate one for goexit and saves the TOC
1053// pointer in the correct place).
1054// goexit+_PCQuantum is halfway through the usual global entry point prologue
1055// that derives r2 from r12 which is a bit silly, but not harmful.
1056TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1057 MOVD 24(R1), R2
1058 BL runtime·goexit1(SB) // does not return
1059 // traceback from goexit1 must hit code range of goexit
1060 MOVD R0, R0 // NOP
1061
1062// prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1063// module containing runtime) to the frame that goexit will execute in when
1064// the goroutine exits. It's implemented in assembly mainly because that's the
1065// easiest way to get access to R2.
1066TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1067 MOVD sp+0(FP), R3
1068 MOVD R2, 24(R3)
1069 RET
1070
1071TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1072 ADD $-8, R1
1073 MOVD R31, 0(R1)
1074 MOVD runtime·lastmoduledatap(SB), R4
1075 MOVD R3, moduledata_next(R4)
1076 MOVD R3, runtime·lastmoduledatap(SB)
1077 MOVD 0(R1), R31
1078 ADD $8, R1
1079 RET
1080
1081// gcWriteBarrier informs the GC about heap pointer writes.
1082//
1083// gcWriteBarrier does NOT follow the Go ABI. It accepts the
1084// number of bytes of buffer needed in R29, and returns a pointer
1085// to the buffer space in R29.
1086// It clobbers condition codes.
1087// It does not clobber R0 through R17 (except special registers),
1088// but may clobber any other register, *including* R31.
1089TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1090 // The standard prologue clobbers R31.
1091 // We use R18, R19, and R31 as scratch registers.
1092retry:
1093 MOVD g_m(g), R18
1094 MOVD m_p(R18), R18
1095 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1096 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1097 // Increment wbBuf.next position.
1098 ADD R29, R19
1099 // Is the buffer full?
1100 CMPU R31, R19
1101 BLT flush
1102 // Commit to the larger buffer.
1103 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1104 // Make return value (the original next position)
1105 SUB R29, R19, R29
1106 RET
1107
1108flush:
1109 // Save registers R0 through R15 since these were not saved by the caller.
1110 // We don't save all registers on ppc64 because it takes too much space.
1111 MOVD R20, (FIXED_FRAME+0)(R1)
1112 MOVD R21, (FIXED_FRAME+8)(R1)
1113 // R0 is always 0, so no need to spill.
1114 // R1 is SP.
1115 // R2 is SB.
1116 MOVD R3, (FIXED_FRAME+16)(R1)
1117 MOVD R4, (FIXED_FRAME+24)(R1)
1118 MOVD R5, (FIXED_FRAME+32)(R1)
1119 MOVD R6, (FIXED_FRAME+40)(R1)
1120 MOVD R7, (FIXED_FRAME+48)(R1)
1121 MOVD R8, (FIXED_FRAME+56)(R1)
1122 MOVD R9, (FIXED_FRAME+64)(R1)
1123 MOVD R10, (FIXED_FRAME+72)(R1)
1124 // R11, R12 may be clobbered by external-linker-inserted trampoline
1125 // R13 is REGTLS
1126 MOVD R14, (FIXED_FRAME+80)(R1)
1127 MOVD R15, (FIXED_FRAME+88)(R1)
1128 MOVD R16, (FIXED_FRAME+96)(R1)
1129 MOVD R17, (FIXED_FRAME+104)(R1)
1130 MOVD R29, (FIXED_FRAME+112)(R1)
1131
1132 CALL runtime·wbBufFlush(SB)
1133
1134 MOVD (FIXED_FRAME+0)(R1), R20
1135 MOVD (FIXED_FRAME+8)(R1), R21
1136 MOVD (FIXED_FRAME+16)(R1), R3
1137 MOVD (FIXED_FRAME+24)(R1), R4
1138 MOVD (FIXED_FRAME+32)(R1), R5
1139 MOVD (FIXED_FRAME+40)(R1), R6
1140 MOVD (FIXED_FRAME+48)(R1), R7
1141 MOVD (FIXED_FRAME+56)(R1), R8
1142 MOVD (FIXED_FRAME+64)(R1), R9
1143 MOVD (FIXED_FRAME+72)(R1), R10
1144 MOVD (FIXED_FRAME+80)(R1), R14
1145 MOVD (FIXED_FRAME+88)(R1), R15
1146 MOVD (FIXED_FRAME+96)(R1), R16
1147 MOVD (FIXED_FRAME+104)(R1), R17
1148 MOVD (FIXED_FRAME+112)(R1), R29
1149 JMP retry
1150
1151TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1152 MOVD $8, R29
1153 JMP gcWriteBarrier<>(SB)
1154TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1155 MOVD $16, R29
1156 JMP gcWriteBarrier<>(SB)
1157TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1158 MOVD $24, R29
1159 JMP gcWriteBarrier<>(SB)
1160TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1161 MOVD $32, R29
1162 JMP gcWriteBarrier<>(SB)
1163TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1164 MOVD $40, R29
1165 JMP gcWriteBarrier<>(SB)
1166TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1167 MOVD $48, R29
1168 JMP gcWriteBarrier<>(SB)
1169TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1170 MOVD $56, R29
1171 JMP gcWriteBarrier<>(SB)
1172TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1173 MOVD $64, R29
1174 JMP gcWriteBarrier<>(SB)
1175
1176DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1177GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1178
1179// debugCallV2 is the entry point for debugger-injected function
1180// calls on running goroutines. It informs the runtime that a
1181// debug call has been injected and creates a call frame for the
1182// debugger to fill in.
1183//
1184// To inject a function call, a debugger should:
1185// 1. Check that the goroutine is in state _Grunning and that
1186// there are at least 320 bytes free on the stack.
1187// 2. Set SP as SP-32.
1188// 3. Store the current LR in (SP) (using the SP after step 2).
1189// 4. Store the current PC in the LR register.
1190// 5. Write the desired argument frame size at SP-32
1191// 6. Save all machine registers (including flags and floating point registers)
1192// so they can be restored later by the debugger.
1193// 7. Set the PC to debugCallV2 and resume execution.
1194//
1195// If the goroutine is in state _Grunnable, then it's not generally
1196// safe to inject a call because it may return out via other runtime
1197// operations. Instead, the debugger should unwind the stack to find
1198// the return to non-runtime code, add a temporary breakpoint there,
1199// and inject the call once that breakpoint is hit.
1200//
1201// If the goroutine is in any other state, it's not safe to inject a call.
1202//
1203// This function communicates back to the debugger by setting R20 and
1204// invoking TW to raise a breakpoint signal. Note that the signal PC of
1205// the signal triggered by the TW instruction is the PC where the signal
1206// is trapped, not the next PC, so to resume execution, the debugger needs
1207// to set the signal PC to PC+4. See the comments in the implementation for
1208// the protocol the debugger is expected to follow. InjectDebugCall in the
1209// runtime tests demonstrates this protocol.
1210// The debugger must ensure that any pointers passed to the function
1211// obey escape analysis requirements. Specifically, it must not pass
1212// a stack pointer to an escaping argument. debugCallV2 cannot check
1213// this invariant.
1214//
1215// This is ABIInternal because Go code injects its PC directly into new
1216// goroutine stacks.
1217#ifdef GOOS_linux
1218TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1219 // save scratch register R31 first
1220 MOVD R31, -184(R1)
1221 MOVD 0(R1), R31
1222 // save caller LR
1223 MOVD R31, -304(R1)
1224 MOVD -32(R1), R31
1225 // save argument frame size
1226 MOVD R31, -192(R1)
1227 MOVD LR, R31
1228 MOVD R31, -320(R1)
1229 ADD $-320, R1
1230 // save all registers that can contain pointers
1231 // and the CR register
1232 MOVW CR, R31
1233 MOVD R31, 8(R1)
1234 MOVD R2, 24(R1)
1235 MOVD R3, 56(R1)
1236 MOVD R4, 64(R1)
1237 MOVD R5, 72(R1)
1238 MOVD R6, 80(R1)
1239 MOVD R7, 88(R1)
1240 MOVD R8, 96(R1)
1241 MOVD R9, 104(R1)
1242 MOVD R10, 112(R1)
1243 MOVD R11, 120(R1)
1244 MOVD R12, 144(R1)
1245 MOVD R13, 152(R1)
1246 MOVD R14, 160(R1)
1247 MOVD R15, 168(R1)
1248 MOVD R16, 176(R1)
1249 MOVD R17, 184(R1)
1250 MOVD R18, 192(R1)
1251 MOVD R19, 200(R1)
1252 MOVD R20, 208(R1)
1253 MOVD R21, 216(R1)
1254 MOVD R22, 224(R1)
1255 MOVD R23, 232(R1)
1256 MOVD R24, 240(R1)
1257 MOVD R25, 248(R1)
1258 MOVD R26, 256(R1)
1259 MOVD R27, 264(R1)
1260 MOVD R28, 272(R1)
1261 MOVD R29, 280(R1)
1262 MOVD g, 288(R1)
1263 MOVD LR, R31
1264 MOVD R31, 32(R1)
1265 CALL runtime·debugCallCheck(SB)
1266 MOVD 40(R1), R22
1267 XOR R0, R0
1268 CMP R22, $0
1269 BEQ good
1270 MOVD 48(R1), R22
1271 MOVD $8, R20
1272 TW $31, R0, R0
1273
1274 BR restore
1275
1276good:
1277#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1278 MOVD $MAXSIZE, R23; \
1279 CMP R26, R23; \
1280 BGT 5(PC); \
1281 MOVD $NAME(SB), R26; \
1282 MOVD R26, 32(R1); \
1283 CALL runtime·debugCallWrap(SB); \
1284 BR restore
1285
1286 // the argument frame size
1287 MOVD 128(R1), R26
1288
1289 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1290 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1291 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1292 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1293 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1294 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1295 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1296 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1297 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1298 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1299 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1300 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1301 // The frame size is too large. Report the error.
1302 MOVD $debugCallFrameTooLarge<>(SB), R22
1303 MOVD R22, 32(R1)
1304 MOVD $20, R22
1305 // length of debugCallFrameTooLarge string
1306 MOVD R22, 40(R1)
1307 MOVD $8, R20
1308 TW $31, R0, R0
1309 BR restore
1310restore:
1311 MOVD $16, R20
1312 TW $31, R0, R0
1313 // restore all registers that can contain
1314 // pointers including CR
1315 MOVD 8(R1), R31
1316 MOVW R31, CR
1317 MOVD 24(R1), R2
1318 MOVD 56(R1), R3
1319 MOVD 64(R1), R4
1320 MOVD 72(R1), R5
1321 MOVD 80(R1), R6
1322 MOVD 88(R1), R7
1323 MOVD 96(R1), R8
1324 MOVD 104(R1), R9
1325 MOVD 112(R1), R10
1326 MOVD 120(R1), R11
1327 MOVD 144(R1), R12
1328 MOVD 152(R1), R13
1329 MOVD 160(R1), R14
1330 MOVD 168(R1), R15
1331 MOVD 176(R1), R16
1332 MOVD 184(R1), R17
1333 MOVD 192(R1), R18
1334 MOVD 200(R1), R19
1335 MOVD 208(R1), R20
1336 MOVD 216(R1), R21
1337 MOVD 224(R1), R22
1338 MOVD 232(R1), R23
1339 MOVD 240(R1), R24
1340 MOVD 248(R1), R25
1341 MOVD 256(R1), R26
1342 MOVD 264(R1), R27
1343 MOVD 272(R1), R28
1344 MOVD 280(R1), R29
1345 MOVD 288(R1), g
1346 MOVD 16(R1), R31
1347 // restore old LR
1348 MOVD R31, LR
1349 // restore caller PC
1350 MOVD 0(R1), CTR
1351 MOVD 136(R1), R31
1352 // Add 32 bytes more to compensate for SP change in saveSigContext
1353 ADD $352, R1
1354 JMP (CTR)
1355#endif
1356#define DEBUG_CALL_FN(NAME,MAXSIZE) \
1357TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1358 NO_LOCAL_POINTERS; \
1359 MOVD $0, R20; \
1360 TW $31, R0, R0 \
1361 MOVD $1, R20; \
1362 TW $31, R0, R0 \
1363 RET
1364DEBUG_CALL_FN(debugCall32<>, 32)
1365DEBUG_CALL_FN(debugCall64<>, 64)
1366DEBUG_CALL_FN(debugCall128<>, 128)
1367DEBUG_CALL_FN(debugCall256<>, 256)
1368DEBUG_CALL_FN(debugCall512<>, 512)
1369DEBUG_CALL_FN(debugCall1024<>, 1024)
1370DEBUG_CALL_FN(debugCall2048<>, 2048)
1371DEBUG_CALL_FN(debugCall4096<>, 4096)
1372DEBUG_CALL_FN(debugCall8192<>, 8192)
1373DEBUG_CALL_FN(debugCall16384<>, 16384)
1374DEBUG_CALL_FN(debugCall32768<>, 32768)
1375DEBUG_CALL_FN(debugCall65536<>, 65536)
1376
1377#ifdef GOOS_linux
1378// func debugCallPanicked(val interface{})
1379TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1380 // Copy the panic value to the top of stack at SP+32.
1381 MOVD val_type+0(FP), R31
1382 MOVD R31, 32(R1)
1383 MOVD val_data+8(FP), R31
1384 MOVD R31, 40(R1)
1385 MOVD $2, R20
1386 TW $31, R0, R0
1387 RET
1388#endif
1389
1390TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$88-0
1391 // Note: frame size is 16 bytes larger than necessary
1392 // in order to pacify vet. Vet doesn't understand ppc64
1393 // layout properly.
1394 NO_LOCAL_POINTERS
1395 // Save all 7 int registers that could have an index in them.
1396 // They may be pointers, but if so they are dead.
1397 // Skip R0 aka ZERO, R1 aka SP, R2 aka SB
1398 MOVD R3, 48(R1)
1399 MOVD R4, 56(R1)
1400 MOVD R5, 64(R1)
1401 MOVD R6, 72(R1)
1402 MOVD R7, 80(R1)
1403 MOVD R8, 88(R1)
1404 MOVD R9, 96(R1)
1405 // Note: we only save 7 registers to keep under nosplit stack limit
1406 // Also, R11 is clobbered in dynamic linking situations
1407
1408 MOVD LR, R3 // PC immediately after call to panicBounds
1409 ADD $48, R1, R4 // pointer to save area
1410 CALL runtime·panicBounds64<ABIInternal>(SB)
1411 RET
1412
1413// These functions are used when internal linking cgo with external
1414// objects compiled with the -Os on gcc. They reduce prologue/epilogue
1415// size by deferring preservation of callee-save registers to a shared
1416// function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1417// in ELFv1)
1418//
1419// These appear unused, but the linker will redirect calls to functions
1420// like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1421// runtime.elf_restgpr1 with an appropriate offset based on the number
1422// register operations required when linking external objects which
1423// make these calls. For GPR/FPR saves, the minimum register value is
1424// 14, for VR it is 20.
1425//
1426// These are only used when linking such cgo code internally. Note, R12
1427// and R0 may be used in different ways than regular ELF compliant
1428// functions.
1429TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1430 // R0 holds the LR of the caller's caller, R1 holds save location
1431 MOVD R14, -144(R1)
1432 MOVD R15, -136(R1)
1433 MOVD R16, -128(R1)
1434 MOVD R17, -120(R1)
1435 MOVD R18, -112(R1)
1436 MOVD R19, -104(R1)
1437 MOVD R20, -96(R1)
1438 MOVD R21, -88(R1)
1439 MOVD R22, -80(R1)
1440 MOVD R23, -72(R1)
1441 MOVD R24, -64(R1)
1442 MOVD R25, -56(R1)
1443 MOVD R26, -48(R1)
1444 MOVD R27, -40(R1)
1445 MOVD R28, -32(R1)
1446 MOVD R29, -24(R1)
1447 MOVD g, -16(R1)
1448 MOVD R31, -8(R1)
1449 MOVD R0, 16(R1)
1450 RET
1451TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1452 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1453 MOVD -144(R1), R14
1454 MOVD -136(R1), R15
1455 MOVD -128(R1), R16
1456 MOVD -120(R1), R17
1457 MOVD -112(R1), R18
1458 MOVD -104(R1), R19
1459 MOVD -96(R1), R20
1460 MOVD -88(R1), R21
1461 MOVD -80(R1), R22
1462 MOVD -72(R1), R23
1463 MOVD -64(R1), R24
1464 MOVD -56(R1), R25
1465 MOVD -48(R1), R26
1466 MOVD -40(R1), R27
1467 MOVD -32(R1), R28
1468 MOVD -24(R1), R29
1469 MOVD -16(R1), g
1470 MOVD -8(R1), R31
1471 MOVD 16(R1), R0 // Load and return to saved LR
1472 MOVD R0, LR
1473 RET
1474TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1475 // R12 holds the save location
1476 MOVD R14, -144(R12)
1477 MOVD R15, -136(R12)
1478 MOVD R16, -128(R12)
1479 MOVD R17, -120(R12)
1480 MOVD R18, -112(R12)
1481 MOVD R19, -104(R12)
1482 MOVD R20, -96(R12)
1483 MOVD R21, -88(R12)
1484 MOVD R22, -80(R12)
1485 MOVD R23, -72(R12)
1486 MOVD R24, -64(R12)
1487 MOVD R25, -56(R12)
1488 MOVD R26, -48(R12)
1489 MOVD R27, -40(R12)
1490 MOVD R28, -32(R12)
1491 MOVD R29, -24(R12)
1492 MOVD g, -16(R12)
1493 MOVD R31, -8(R12)
1494 RET
1495TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1496 // R12 holds the save location
1497 MOVD -144(R12), R14
1498 MOVD -136(R12), R15
1499 MOVD -128(R12), R16
1500 MOVD -120(R12), R17
1501 MOVD -112(R12), R18
1502 MOVD -104(R12), R19
1503 MOVD -96(R12), R20
1504 MOVD -88(R12), R21
1505 MOVD -80(R12), R22
1506 MOVD -72(R12), R23
1507 MOVD -64(R12), R24
1508 MOVD -56(R12), R25
1509 MOVD -48(R12), R26
1510 MOVD -40(R12), R27
1511 MOVD -32(R12), R28
1512 MOVD -24(R12), R29
1513 MOVD -16(R12), g
1514 MOVD -8(R12), R31
1515 RET
1516TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1517 // R0 holds the LR of the caller's caller, R1 holds save location
1518 FMOVD F14, -144(R1)
1519 FMOVD F15, -136(R1)
1520 FMOVD F16, -128(R1)
1521 FMOVD F17, -120(R1)
1522 FMOVD F18, -112(R1)
1523 FMOVD F19, -104(R1)
1524 FMOVD F20, -96(R1)
1525 FMOVD F21, -88(R1)
1526 FMOVD F22, -80(R1)
1527 FMOVD F23, -72(R1)
1528 FMOVD F24, -64(R1)
1529 FMOVD F25, -56(R1)
1530 FMOVD F26, -48(R1)
1531 FMOVD F27, -40(R1)
1532 FMOVD F28, -32(R1)
1533 FMOVD F29, -24(R1)
1534 FMOVD F30, -16(R1)
1535 FMOVD F31, -8(R1)
1536 MOVD R0, 16(R1)
1537 RET
1538TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1539 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1540 FMOVD -144(R1), F14
1541 FMOVD -136(R1), F15
1542 FMOVD -128(R1), F16
1543 FMOVD -120(R1), F17
1544 FMOVD -112(R1), F18
1545 FMOVD -104(R1), F19
1546 FMOVD -96(R1), F20
1547 FMOVD -88(R1), F21
1548 FMOVD -80(R1), F22
1549 FMOVD -72(R1), F23
1550 FMOVD -64(R1), F24
1551 FMOVD -56(R1), F25
1552 FMOVD -48(R1), F26
1553 FMOVD -40(R1), F27
1554 FMOVD -32(R1), F28
1555 FMOVD -24(R1), F29
1556 FMOVD -16(R1), F30
1557 FMOVD -8(R1), F31
1558 MOVD 16(R1), R0 // Load and return to saved LR
1559 MOVD R0, LR
1560 RET
1561TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1562 // R0 holds the save location, R12 is clobbered
1563 MOVD $-192, R12
1564 STVX V20, (R0+R12)
1565 MOVD $-176, R12
1566 STVX V21, (R0+R12)
1567 MOVD $-160, R12
1568 STVX V22, (R0+R12)
1569 MOVD $-144, R12
1570 STVX V23, (R0+R12)
1571 MOVD $-128, R12
1572 STVX V24, (R0+R12)
1573 MOVD $-112, R12
1574 STVX V25, (R0+R12)
1575 MOVD $-96, R12
1576 STVX V26, (R0+R12)
1577 MOVD $-80, R12
1578 STVX V27, (R0+R12)
1579 MOVD $-64, R12
1580 STVX V28, (R0+R12)
1581 MOVD $-48, R12
1582 STVX V29, (R0+R12)
1583 MOVD $-32, R12
1584 STVX V30, (R0+R12)
1585 MOVD $-16, R12
1586 STVX V31, (R0+R12)
1587 RET
1588TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1589 // R0 holds the save location, R12 is clobbered
1590 MOVD $-192, R12
1591 LVX (R0+R12), V20
1592 MOVD $-176, R12
1593 LVX (R0+R12), V21
1594 MOVD $-160, R12
1595 LVX (R0+R12), V22
1596 MOVD $-144, R12
1597 LVX (R0+R12), V23
1598 MOVD $-128, R12
1599 LVX (R0+R12), V24
1600 MOVD $-112, R12
1601 LVX (R0+R12), V25
1602 MOVD $-96, R12
1603 LVX (R0+R12), V26
1604 MOVD $-80, R12
1605 LVX (R0+R12), V27
1606 MOVD $-64, R12
1607 LVX (R0+R12), V28
1608 MOVD $-48, R12
1609 LVX (R0+R12), V29
1610 MOVD $-32, R12
1611 LVX (R0+R12), V30
1612 MOVD $-16, R12
1613 LVX (R0+R12), V31
1614 RET
View as plain text