Text file
src/runtime/asm_ppc64x.s
Documentation: runtime
1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build ppc64 || ppc64le
6
7#include "go_asm.h"
8#include "go_tls.h"
9#include "funcdata.h"
10#include "textflag.h"
11#include "asm_ppc64x.h"
12
13#ifdef GOOS_aix
14#define cgoCalleeStackSize 48
15#else
16#define cgoCalleeStackSize 32
17#endif
18
19TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
21
22 // initialize essential registers
23 BL runtime·reginit(SB)
24
25 SUB $(FIXED_FRAME+16), R1
26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
27 MOVW R3, FIXED_FRAME+0(R1) // argc
28 MOVD R4, FIXED_FRAME+8(R1) // argv
29
30 // create istack out of the given (operating system) stack.
31 // _cgo_init may update stackguard.
32 MOVD $runtime·g0(SB), g
33 BL runtime·save_g(SB)
34 MOVD $(-64*1024), R31
35 ADD R31, R1, R3
36 MOVD R3, g_stackguard0(g)
37 MOVD R3, g_stackguard1(g)
38 MOVD R3, (g_stack+stack_lo)(g)
39 MOVD R1, (g_stack+stack_hi)(g)
40
41 // If there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CMP R0, R12
44 BEQ nocgo
45
46#ifdef GO_PPC64X_HAS_FUNCDESC
47 // Load the real entry address from the first slot of the function descriptor.
48 MOVD 8(R12), R2
49 MOVD (R12), R12
50#endif
51 MOVD R12, CTR // r12 = "global function entry point"
52 MOVD R13, R5 // arg 2: TLS base pointer
53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
54 MOVD g, R3 // arg 0: G
55 // C functions expect 32 (48 for AIX) bytes of space on caller
56 // stack frame and a 16-byte aligned R1
57 MOVD R1, R14 // save current stack
58 SUB $cgoCalleeStackSize, R1 // reserve the callee area
59 RLDCR $0, R1, $~15, R1 // 16-byte align
60 BL (CTR) // may clobber R0, R3-R12
61 MOVD R14, R1 // restore stack
62#ifndef GOOS_aix
63 MOVD 24(R1), R2
64#endif
65 XOR R0, R0 // fix R0
66
67nocgo:
68 // update stackguard after _cgo_init
69 MOVD (g_stack+stack_lo)(g), R3
70 ADD $const_stackGuard, R3
71 MOVD R3, g_stackguard0(g)
72 MOVD R3, g_stackguard1(g)
73
74 // set the per-goroutine and per-mach "registers"
75 MOVD $runtime·m0(SB), R3
76
77 // save m->g0 = g0
78 MOVD g, m_g0(R3)
79 // save m0 to g0->m
80 MOVD R3, g_m(g)
81
82 BL runtime·check(SB)
83
84 // args are already prepared
85 BL runtime·args(SB)
86 BL runtime·osinit(SB)
87 BL runtime·schedinit(SB)
88
89 // create a new goroutine to start program
90 MOVD $runtime·mainPC(SB), R3 // entry
91 MOVDU R3, -8(R1)
92 MOVDU R0, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 BL runtime·newproc(SB)
97 ADD $(8+FIXED_FRAME), R1
98
99 // start this M
100 BL runtime·mstart(SB)
101 // Prevent dead-code elimination of debugCallV2, which is
102 // intended to be called by debuggers.
103#ifdef GOARCH_ppc64le
104 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
105#endif
106 MOVD R0, 0(R0)
107 RET
108
109DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
110GLOBL runtime·mainPC(SB),RODATA,$8
111
112TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
113 TW $31, R0, R0
114 RET
115
116TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
117 RET
118
119// Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
120TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
121 // crosscall_ppc64 and crosscall2 need to reginit, but can't
122 // get at the 'runtime.reginit' symbol.
123 BR runtime·reginit(SB)
124
125TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
126 // set R0 to zero, it's expected by the toolchain
127 XOR R0, R0
128 RET
129
130TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
131 BL runtime·mstart0(SB)
132 RET // not reached
133
134/*
135 * go-routine
136 */
137
138// void gogo(Gobuf*)
139// restore state from Gobuf; longjmp
140TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
141 MOVD buf+0(FP), R5
142 MOVD gobuf_g(R5), R6
143 MOVD 0(R6), R4 // make sure g != nil
144 BR gogo<>(SB)
145
146TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
147 MOVD R6, g
148 BL runtime·save_g(SB)
149
150 MOVD gobuf_sp(R5), R1
151 MOVD gobuf_lr(R5), R31
152#ifndef GOOS_aix
153 MOVD 24(R1), R2 // restore R2
154#endif
155 MOVD R31, LR
156 MOVD gobuf_ret(R5), R3
157 MOVD gobuf_ctxt(R5), R11
158 MOVD R0, gobuf_sp(R5)
159 MOVD R0, gobuf_ret(R5)
160 MOVD R0, gobuf_lr(R5)
161 MOVD R0, gobuf_ctxt(R5)
162 CMP R0, R0 // set condition codes for == test, needed by stack split
163 MOVD gobuf_pc(R5), R12
164 MOVD R12, CTR
165 BR (CTR)
166
167// void mcall(fn func(*g))
168// Switch to m->g0's stack, call fn(g).
169// Fn must never return. It should gogo(&g->sched)
170// to keep running g.
171TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
172 // Save caller state in g->sched
173 // R11 should be safe across save_g??
174 MOVD R3, R11
175 MOVD R1, (g_sched+gobuf_sp)(g)
176 MOVD LR, R31
177 MOVD R31, (g_sched+gobuf_pc)(g)
178 MOVD R0, (g_sched+gobuf_lr)(g)
179
180 // Switch to m->g0 & its stack, call fn.
181 MOVD g, R3
182 MOVD g_m(g), R8
183 MOVD m_g0(R8), g
184 BL runtime·save_g(SB)
185 CMP g, R3
186 BNE 2(PC)
187 BR runtime·badmcall(SB)
188 MOVD 0(R11), R12 // code pointer
189 MOVD R12, CTR
190 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
191 // Don't need to do anything special for regabiargs here
192 // R3 is g; stack is set anyway
193 MOVDU R3, -8(R1)
194 MOVDU R0, -8(R1)
195 MOVDU R0, -8(R1)
196 MOVDU R0, -8(R1)
197 MOVDU R0, -8(R1)
198 BL (CTR)
199 MOVD 24(R1), R2
200 BR runtime·badmcall2(SB)
201
202// systemstack_switch is a dummy routine that systemstack leaves at the bottom
203// of the G stack. We need to distinguish the routine that
204// lives at the bottom of the G stack from the one that lives
205// at the top of the system stack because the one at the top of
206// the system stack terminates the stack walk (see topofstack()).
207TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
208 // We have several undefs here so that 16 bytes past
209 // $runtime·systemstack_switch lies within them whether or not the
210 // instructions that derive r2 from r12 are there.
211 UNDEF
212 UNDEF
213 UNDEF
214 BL (LR) // make sure this function is not leaf
215 RET
216
217// func systemstack(fn func())
218TEXT runtime·systemstack(SB), NOSPLIT, $0-8
219 MOVD fn+0(FP), R3 // R3 = fn
220 MOVD R3, R11 // context
221 MOVD g_m(g), R4 // R4 = m
222
223 MOVD m_gsignal(R4), R5 // R5 = gsignal
224 CMP g, R5
225 BEQ noswitch
226
227 MOVD m_g0(R4), R5 // R5 = g0
228 CMP g, R5
229 BEQ noswitch
230
231 MOVD m_curg(R4), R6
232 CMP g, R6
233 BEQ switch
234
235 // Bad: g is not gsignal, not g0, not curg. What is it?
236 // Hide call from linker nosplit analysis.
237 MOVD $runtime·badsystemstack(SB), R12
238 MOVD R12, CTR
239 BL (CTR)
240 BL runtime·abort(SB)
241
242switch:
243 // save our state in g->sched. Pretend to
244 // be systemstack_switch if the G stack is scanned.
245 BL gosave_systemstack_switch<>(SB)
246
247 // switch to g0
248 MOVD R5, g
249 BL runtime·save_g(SB)
250 MOVD (g_sched+gobuf_sp)(g), R1
251
252 // call target function
253 MOVD 0(R11), R12 // code pointer
254 MOVD R12, CTR
255 BL (CTR)
256
257 // restore TOC pointer. It seems unlikely that we will use systemstack
258 // to call a function defined in another module, but the results of
259 // doing so would be so confusing that it's worth doing this.
260 MOVD g_m(g), R3
261 MOVD m_curg(R3), g
262 MOVD (g_sched+gobuf_sp)(g), R3
263#ifndef GOOS_aix
264 MOVD 24(R3), R2
265#endif
266 // switch back to g
267 MOVD g_m(g), R3
268 MOVD m_curg(R3), g
269 BL runtime·save_g(SB)
270 MOVD (g_sched+gobuf_sp)(g), R1
271 MOVD R0, (g_sched+gobuf_sp)(g)
272 RET
273
274noswitch:
275 // already on m stack, just call directly
276 // On other arches we do a tail call here, but it appears to be
277 // impossible to tail call a function pointer in shared mode on
278 // ppc64 because the caller is responsible for restoring the TOC.
279 MOVD 0(R11), R12 // code pointer
280 MOVD R12, CTR
281 BL (CTR)
282#ifndef GOOS_aix
283 MOVD 24(R1), R2
284#endif
285 RET
286
287// func switchToCrashStack0(fn func())
288TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
289 MOVD R3, R11 // context register
290 MOVD g_m(g), R3 // curm
291
292 // set g to gcrash
293 MOVD $runtime·gcrash(SB), g // g = &gcrash
294 CALL runtime·save_g(SB) // clobbers R31
295 MOVD R3, g_m(g) // g.m = curm
296 MOVD g, m_g0(R3) // curm.g0 = g
297
298 // switch to crashstack
299 MOVD (g_stack+stack_hi)(g), R3
300 SUB $(4*8), R3
301 MOVD R3, R1
302
303 // call target function
304 MOVD 0(R11), R12 // code pointer
305 MOVD R12, CTR
306 BL (CTR)
307
308 // should never return
309 CALL runtime·abort(SB)
310 UNDEF
311
312/*
313 * support for morestack
314 */
315
316// Called during function prolog when more stack is needed.
317// Caller has already loaded:
318// R3: framesize, R4: argsize, R5: LR
319//
320// The traceback routines see morestack on a g0 as being
321// the top of a stack (for example, morestack calling newstack
322// calling the scheduler calling newm calling gc), so we must
323// record an argument size. For that purpose, it has no arguments.
324TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
325 // Called from f.
326 // Set g->sched to context in f.
327 MOVD R1, (g_sched+gobuf_sp)(g)
328 MOVD LR, R8
329 MOVD R8, (g_sched+gobuf_pc)(g)
330 MOVD R5, (g_sched+gobuf_lr)(g)
331 MOVD R11, (g_sched+gobuf_ctxt)(g)
332
333 // Cannot grow scheduler stack (m->g0).
334 MOVD g_m(g), R7
335 MOVD m_g0(R7), R8
336 CMP g, R8
337 BNE 3(PC)
338 BL runtime·badmorestackg0(SB)
339 BL runtime·abort(SB)
340
341 // Cannot grow signal stack (m->gsignal).
342 MOVD m_gsignal(R7), R8
343 CMP g, R8
344 BNE 3(PC)
345 BL runtime·badmorestackgsignal(SB)
346 BL runtime·abort(SB)
347
348 // Called from f.
349 // Set m->morebuf to f's caller.
350 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
351 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
352 MOVD g, (m_morebuf+gobuf_g)(R7)
353
354 // Call newstack on m->g0's stack.
355 MOVD m_g0(R7), g
356 BL runtime·save_g(SB)
357 MOVD (g_sched+gobuf_sp)(g), R1
358 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
359 BL runtime·newstack(SB)
360
361 // Not reached, but make sure the return PC from the call to newstack
362 // is still in this function, and not the beginning of the next.
363 UNDEF
364
365TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
366 // Force SPWRITE. This function doesn't actually write SP,
367 // but it is called with a special calling convention where
368 // the caller doesn't save LR on stack but passes it as a
369 // register (R5), and the unwinder currently doesn't understand.
370 // Make it SPWRITE to stop unwinding. (See issue 54332)
371 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
372 // has a special affect on Power8,9,10 by lowering the thread
373 // priority and causing a slowdown in execution time
374
375 OR R0, R1
376 MOVD R0, R11
377 BR runtime·morestack(SB)
378
379// reflectcall: call a function with the given argument list
380// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
381// we don't have variable-sized frames, so we use a small number
382// of constant-sized-frame functions to encode a few bits of size in the pc.
383// Caution: ugly multiline assembly macros in your future!
384
385#define DISPATCH(NAME,MAXSIZE) \
386 MOVD $MAXSIZE, R31; \
387 CMP R3, R31; \
388 BGT 4(PC); \
389 MOVD $NAME(SB), R12; \
390 MOVD R12, CTR; \
391 BR (CTR)
392// Note: can't just "BR NAME(SB)" - bad inlining results.
393
394TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
395 MOVWZ frameSize+32(FP), R3
396 DISPATCH(runtime·call16, 16)
397 DISPATCH(runtime·call32, 32)
398 DISPATCH(runtime·call64, 64)
399 DISPATCH(runtime·call128, 128)
400 DISPATCH(runtime·call256, 256)
401 DISPATCH(runtime·call512, 512)
402 DISPATCH(runtime·call1024, 1024)
403 DISPATCH(runtime·call2048, 2048)
404 DISPATCH(runtime·call4096, 4096)
405 DISPATCH(runtime·call8192, 8192)
406 DISPATCH(runtime·call16384, 16384)
407 DISPATCH(runtime·call32768, 32768)
408 DISPATCH(runtime·call65536, 65536)
409 DISPATCH(runtime·call131072, 131072)
410 DISPATCH(runtime·call262144, 262144)
411 DISPATCH(runtime·call524288, 524288)
412 DISPATCH(runtime·call1048576, 1048576)
413 DISPATCH(runtime·call2097152, 2097152)
414 DISPATCH(runtime·call4194304, 4194304)
415 DISPATCH(runtime·call8388608, 8388608)
416 DISPATCH(runtime·call16777216, 16777216)
417 DISPATCH(runtime·call33554432, 33554432)
418 DISPATCH(runtime·call67108864, 67108864)
419 DISPATCH(runtime·call134217728, 134217728)
420 DISPATCH(runtime·call268435456, 268435456)
421 DISPATCH(runtime·call536870912, 536870912)
422 DISPATCH(runtime·call1073741824, 1073741824)
423 MOVD $runtime·badreflectcall(SB), R12
424 MOVD R12, CTR
425 BR (CTR)
426
427#define CALLFN(NAME,MAXSIZE) \
428TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
429 NO_LOCAL_POINTERS; \
430 /* copy arguments to stack */ \
431 MOVD stackArgs+16(FP), R3; \
432 MOVWZ stackArgsSize+24(FP), R4; \
433 MOVD R1, R5; \
434 CMP R4, $8; \
435 BLT tailsetup; \
436 /* copy 8 at a time if possible */ \
437 ADD $(FIXED_FRAME-8), R5; \
438 SUB $8, R3; \
439top: \
440 MOVDU 8(R3), R7; \
441 MOVDU R7, 8(R5); \
442 SUB $8, R4; \
443 CMP R4, $8; \
444 BGE top; \
445 /* handle remaining bytes */ \
446 CMP $0, R4; \
447 BEQ callfn; \
448 ADD $7, R3; \
449 ADD $7, R5; \
450 BR tail; \
451tailsetup: \
452 CMP $0, R4; \
453 BEQ callfn; \
454 ADD $(FIXED_FRAME-1), R5; \
455 SUB $1, R3; \
456tail: \
457 MOVBU 1(R3), R6; \
458 MOVBU R6, 1(R5); \
459 SUB $1, R4; \
460 CMP $0, R4; \
461 BGT tail; \
462callfn: \
463 /* call function */ \
464 MOVD f+8(FP), R11; \
465#ifdef GOOS_aix \
466 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
467 /* So it manually triggers it */ \
468 CMP R0, R11 \
469 BNE 2(PC) \
470 MOVD R0, 0(R0) \
471#endif \
472 MOVD regArgs+40(FP), R20; \
473 BL runtime·unspillArgs(SB); \
474 MOVD (R11), R12; \
475 MOVD R12, CTR; \
476 PCDATA $PCDATA_StackMapIndex, $0; \
477 BL (CTR); \
478#ifndef GOOS_aix \
479 MOVD 24(R1), R2; \
480#endif \
481 /* copy return values back */ \
482 MOVD regArgs+40(FP), R20; \
483 BL runtime·spillArgs(SB); \
484 MOVD stackArgsType+0(FP), R7; \
485 MOVD stackArgs+16(FP), R3; \
486 MOVWZ stackArgsSize+24(FP), R4; \
487 MOVWZ stackRetOffset+28(FP), R6; \
488 ADD $FIXED_FRAME, R1, R5; \
489 ADD R6, R5; \
490 ADD R6, R3; \
491 SUB R6, R4; \
492 BL callRet<>(SB); \
493 RET
494
495// callRet copies return values back at the end of call*. This is a
496// separate function so it can allocate stack space for the arguments
497// to reflectcallmove. It does not follow the Go ABI; it expects its
498// arguments in registers.
499TEXT callRet<>(SB), NOSPLIT, $40-0
500 NO_LOCAL_POINTERS
501 MOVD R7, FIXED_FRAME+0(R1)
502 MOVD R3, FIXED_FRAME+8(R1)
503 MOVD R5, FIXED_FRAME+16(R1)
504 MOVD R4, FIXED_FRAME+24(R1)
505 MOVD R20, FIXED_FRAME+32(R1)
506 BL runtime·reflectcallmove(SB)
507 RET
508
509CALLFN(·call16, 16)
510CALLFN(·call32, 32)
511CALLFN(·call64, 64)
512CALLFN(·call128, 128)
513CALLFN(·call256, 256)
514CALLFN(·call512, 512)
515CALLFN(·call1024, 1024)
516CALLFN(·call2048, 2048)
517CALLFN(·call4096, 4096)
518CALLFN(·call8192, 8192)
519CALLFN(·call16384, 16384)
520CALLFN(·call32768, 32768)
521CALLFN(·call65536, 65536)
522CALLFN(·call131072, 131072)
523CALLFN(·call262144, 262144)
524CALLFN(·call524288, 524288)
525CALLFN(·call1048576, 1048576)
526CALLFN(·call2097152, 2097152)
527CALLFN(·call4194304, 4194304)
528CALLFN(·call8388608, 8388608)
529CALLFN(·call16777216, 16777216)
530CALLFN(·call33554432, 33554432)
531CALLFN(·call67108864, 67108864)
532CALLFN(·call134217728, 134217728)
533CALLFN(·call268435456, 268435456)
534CALLFN(·call536870912, 536870912)
535CALLFN(·call1073741824, 1073741824)
536
537TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
538 MOVW cycles+0(FP), R7
539 // POWER does not have a pause/yield instruction equivalent.
540 // Instead, we can lower the program priority by setting the
541 // Program Priority Register prior to the wait loop and set it
542 // back to default afterwards. On Linux, the default priority is
543 // medium-low. For details, see page 837 of the ISA 3.0.
544 OR R1, R1, R1 // Set PPR priority to low
545again:
546 SUB $1, R7
547 CMP $0, R7
548 BNE again
549 OR R6, R6, R6 // Set PPR priority back to medium-low
550 RET
551
552// Save state of caller into g->sched,
553// but using fake PC from systemstack_switch.
554// Must only be called from functions with no locals ($0)
555// or else unwinding from systemstack_switch is incorrect.
556// Smashes R31.
557TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
558 MOVD $runtime·systemstack_switch(SB), R31
559 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
560 MOVD R31, (g_sched+gobuf_pc)(g)
561 MOVD R1, (g_sched+gobuf_sp)(g)
562 MOVD R0, (g_sched+gobuf_lr)(g)
563 MOVD R0, (g_sched+gobuf_ret)(g)
564 // Assert ctxt is zero. See func save.
565 MOVD (g_sched+gobuf_ctxt)(g), R31
566 CMP R0, R31
567 BEQ 2(PC)
568 BL runtime·abort(SB)
569 RET
570
571#ifdef GOOS_aix
572#define asmcgocallSaveOffset cgoCalleeStackSize + 8
573#else
574#define asmcgocallSaveOffset cgoCalleeStackSize
575#endif
576
577// func asmcgocall_no_g(fn, arg unsafe.Pointer)
578// Call fn(arg) aligned appropriately for the gcc ABI.
579// Called on a system stack, and there may be no g yet (during needm).
580TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
581 MOVD fn+0(FP), R3
582 MOVD arg+8(FP), R4
583
584 MOVD R1, R15
585 SUB $(asmcgocallSaveOffset+8), R1
586 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
587 MOVD R15, asmcgocallSaveOffset(R1)
588
589 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
590
591 // This is a "global call", so put the global entry point in r12
592 MOVD R3, R12
593
594#ifdef GO_PPC64X_HAS_FUNCDESC
595 // Load the real entry address from the first slot of the function descriptor.
596 MOVD 8(R12), R2
597 MOVD (R12), R12
598#endif
599 MOVD R12, CTR
600 MOVD R4, R3 // arg in r3
601 BL (CTR)
602
603 // C code can clobber R0, so set it back to 0. F27-F31 are
604 // callee save, so we don't need to recover those.
605 XOR R0, R0
606
607 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
608#ifndef GOOS_aix
609 MOVD 24(R1), R2
610#endif
611
612 RET
613
614// func asmcgocall(fn, arg unsafe.Pointer) int32
615// Call fn(arg) on the scheduler stack,
616// aligned appropriately for the gcc ABI.
617// See cgocall.go for more details.
618TEXT ·asmcgocall(SB),NOSPLIT,$0-20
619 MOVD fn+0(FP), R3
620 MOVD arg+8(FP), R4
621
622 MOVD R1, R7 // save original stack pointer
623 CMP $0, g
624 BEQ nosave
625 MOVD g, R5
626
627 // Figure out if we need to switch to m->g0 stack.
628 // We get called to create new OS threads too, and those
629 // come in on the m->g0 stack already. Or we might already
630 // be on the m->gsignal stack.
631 MOVD g_m(g), R8
632 MOVD m_gsignal(R8), R6
633 CMP R6, g
634 BEQ nosave
635 MOVD m_g0(R8), R6
636 CMP R6, g
637 BEQ nosave
638
639 BL gosave_systemstack_switch<>(SB)
640 MOVD R6, g
641 BL runtime·save_g(SB)
642 MOVD (g_sched+gobuf_sp)(g), R1
643
644 // Now on a scheduling stack (a pthread-created stack).
645#ifdef GOOS_aix
646 // Create a fake LR to improve backtrace.
647 MOVD $runtime·asmcgocall(SB), R6
648 MOVD R6, 16(R1)
649 // AIX also saves one argument on the stack.
650 SUB $8, R1
651#endif
652 // Save room for two of our pointers, plus the callee
653 // save area that lives on the caller stack.
654 SUB $(asmcgocallSaveOffset+16), R1
655 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
656 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
657 MOVD (g_stack+stack_hi)(R5), R5
658 SUB R7, R5
659 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
660#ifdef GOOS_aix
661 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
662#else
663 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
664#endif
665 // This is a "global call", so put the global entry point in r12
666 MOVD R3, R12
667
668#ifdef GO_PPC64X_HAS_FUNCDESC
669 // Load the real entry address from the first slot of the function descriptor.
670 MOVD 8(R12), R2
671 MOVD (R12), R12
672#endif
673 MOVD R12, CTR
674 MOVD R4, R3 // arg in r3
675 BL (CTR)
676
677 // Reinitialise zero value register.
678 XOR R0, R0
679
680 // Restore g, stack pointer, toc pointer.
681 // R3 is errno, so don't touch it
682 MOVD (asmcgocallSaveOffset+8)(R1), g
683 MOVD (g_stack+stack_hi)(g), R5
684 MOVD asmcgocallSaveOffset(R1), R6
685 SUB R6, R5
686#ifndef GOOS_aix
687 MOVD 24(R5), R2
688#endif
689 MOVD R5, R1
690 BL runtime·save_g(SB)
691
692 MOVW R3, ret+16(FP)
693 RET
694
695nosave:
696 // Running on a system stack, perhaps even without a g.
697 // Having no g can happen during thread creation or thread teardown.
698 // This code is like the above sequence but without saving/restoring g
699 // and without worrying about the stack moving out from under us
700 // (because we're on a system stack, not a goroutine stack).
701 // The above code could be used directly if already on a system stack,
702 // but then the only path through this code would be a rare case.
703 // Using this code for all "already on system stack" calls exercises it more,
704 // which should help keep it correct.
705
706 SUB $(asmcgocallSaveOffset+8), R1
707 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
708 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
709
710 MOVD R3, R12 // fn
711#ifdef GO_PPC64X_HAS_FUNCDESC
712 // Load the real entry address from the first slot of the function descriptor.
713 MOVD 8(R12), R2
714 MOVD (R12), R12
715#endif
716 MOVD R12, CTR
717 MOVD R4, R3 // arg
718 BL (CTR)
719
720 // Reinitialise zero value register.
721 XOR R0, R0
722
723 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
724#ifndef GOOS_aix
725 MOVD 24(R1), R2
726#endif
727 MOVW R3, ret+16(FP)
728 RET
729
730// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
731// See cgocall.go for more details.
732TEXT ·cgocallback(SB),NOSPLIT,$24-24
733 NO_LOCAL_POINTERS
734
735 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
736 // It is used to dropm while thread is exiting.
737 MOVD fn+0(FP), R5
738 CMP R5, $0
739 BNE loadg
740 // Restore the g from frame.
741 MOVD frame+8(FP), g
742 BR dropm
743
744loadg:
745 // Load m and g from thread-local storage.
746#ifndef GOOS_openbsd
747 MOVBZ runtime·iscgo(SB), R3
748 CMP R3, $0
749 BEQ nocgo
750#endif
751 BL runtime·load_g(SB)
752nocgo:
753
754 // If g is nil, Go did not create the current thread,
755 // or if this thread never called into Go on pthread platforms.
756 // Call needm to obtain one for temporary use.
757 // In this case, we're running on the thread stack, so there's
758 // lots of space, but the linker doesn't know. Hide the call from
759 // the linker analysis by using an indirect call.
760 CMP g, $0
761 BEQ needm
762
763 MOVD g_m(g), R8
764 MOVD R8, savedm-8(SP)
765 BR havem
766
767needm:
768 MOVD g, savedm-8(SP) // g is zero, so is m.
769 MOVD $runtime·needAndBindM(SB), R12
770 MOVD R12, CTR
771 BL (CTR)
772
773 // Set m->sched.sp = SP, so that if a panic happens
774 // during the function we are about to execute, it will
775 // have a valid SP to run on the g0 stack.
776 // The next few lines (after the havem label)
777 // will save this SP onto the stack and then write
778 // the same SP back to m->sched.sp. That seems redundant,
779 // but if an unrecovered panic happens, unwindm will
780 // restore the g->sched.sp from the stack location
781 // and then systemstack will try to use it. If we don't set it here,
782 // that restored SP will be uninitialized (typically 0) and
783 // will not be usable.
784 MOVD g_m(g), R8
785 MOVD m_g0(R8), R3
786 MOVD R1, (g_sched+gobuf_sp)(R3)
787
788havem:
789 // Now there's a valid m, and we're running on its m->g0.
790 // Save current m->g0->sched.sp on stack and then set it to SP.
791 // Save current sp in m->g0->sched.sp in preparation for
792 // switch back to m->curg stack.
793 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
794 MOVD m_g0(R8), R3
795 MOVD (g_sched+gobuf_sp)(R3), R4
796 MOVD R4, savedsp-24(SP) // must match frame size
797 MOVD R1, (g_sched+gobuf_sp)(R3)
798
799 // Switch to m->curg stack and call runtime.cgocallbackg.
800 // Because we are taking over the execution of m->curg
801 // but *not* resuming what had been running, we need to
802 // save that information (m->curg->sched) so we can restore it.
803 // We can restore m->curg->sched.sp easily, because calling
804 // runtime.cgocallbackg leaves SP unchanged upon return.
805 // To save m->curg->sched.pc, we push it onto the curg stack and
806 // open a frame the same size as cgocallback's g0 frame.
807 // Once we switch to the curg stack, the pushed PC will appear
808 // to be the return PC of cgocallback, so that the traceback
809 // will seamlessly trace back into the earlier calls.
810 MOVD m_curg(R8), g
811 BL runtime·save_g(SB)
812 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
813 MOVD (g_sched+gobuf_pc)(g), R5
814 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
815 // Gather our arguments into registers.
816 MOVD fn+0(FP), R5
817 MOVD frame+8(FP), R6
818 MOVD ctxt+16(FP), R7
819 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
820 MOVD R5, FIXED_FRAME+0(R1)
821 MOVD R6, FIXED_FRAME+8(R1)
822 MOVD R7, FIXED_FRAME+16(R1)
823
824 MOVD $runtime·cgocallbackg(SB), R12
825 MOVD R12, CTR
826 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
827
828 // Restore g->sched (== m->curg->sched) from saved values.
829 MOVD 0(R1), R5
830 MOVD R5, (g_sched+gobuf_pc)(g)
831 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
832 MOVD R4, (g_sched+gobuf_sp)(g)
833
834 // Switch back to m->g0's stack and restore m->g0->sched.sp.
835 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
836 // so we do not have to restore it.)
837 MOVD g_m(g), R8
838 MOVD m_g0(R8), g
839 BL runtime·save_g(SB)
840 MOVD (g_sched+gobuf_sp)(g), R1
841 MOVD savedsp-24(SP), R4 // must match frame size
842 MOVD R4, (g_sched+gobuf_sp)(g)
843
844 // If the m on entry was nil, we called needm above to borrow an m,
845 // 1. for the duration of the call on non-pthread platforms,
846 // 2. or the duration of the C thread alive on pthread platforms.
847 // If the m on entry wasn't nil,
848 // 1. the thread might be a Go thread,
849 // 2. or it wasn't the first call from a C thread on pthread platforms,
850 // since then we skip dropm to reuse the m in the first call.
851 MOVD savedm-8(SP), R6
852 CMP R6, $0
853 BNE droppedm
854
855 // Skip dropm to reuse it in the next call, when a pthread key has been created.
856 MOVD _cgo_pthread_key_created(SB), R6
857 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
858 CMP R6, $0
859 BEQ dropm
860 MOVD (R6), R6
861 CMP R6, $0
862 BNE droppedm
863
864dropm:
865 MOVD $runtime·dropm(SB), R12
866 MOVD R12, CTR
867 BL (CTR)
868droppedm:
869
870 // Done!
871 RET
872
873// void setg(G*); set g. for use by needm.
874TEXT runtime·setg(SB), NOSPLIT, $0-8
875 MOVD gg+0(FP), g
876 // This only happens if iscgo, so jump straight to save_g
877 BL runtime·save_g(SB)
878 RET
879
880#ifdef GO_PPC64X_HAS_FUNCDESC
881DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
882TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
883#else
884TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
885#endif
886 // The standard prologue clobbers R31, which is callee-save in
887 // the C ABI, so we have to use $-8-0 and save LR ourselves.
888 MOVD LR, R4
889 // Also save g and R31, since they're callee-save in C ABI
890 MOVD R31, R5
891 MOVD g, R6
892
893 MOVD R3, g
894 BL runtime·save_g(SB)
895
896 MOVD R6, g
897 MOVD R5, R31
898 MOVD R4, LR
899 RET
900
901TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
902 MOVW (R0), R0
903 UNDEF
904
905#define TBR 268
906
907// int64 runtime·cputicks(void)
908TEXT runtime·cputicks(SB),NOSPLIT,$0-8
909 MOVD SPR(TBR), R3
910 MOVD R3, ret+0(FP)
911 RET
912
913// spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
914TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
915 MOVD R3, 0(R20)
916 MOVD R4, 8(R20)
917 MOVD R5, 16(R20)
918 MOVD R6, 24(R20)
919 MOVD R7, 32(R20)
920 MOVD R8, 40(R20)
921 MOVD R9, 48(R20)
922 MOVD R10, 56(R20)
923 MOVD R14, 64(R20)
924 MOVD R15, 72(R20)
925 MOVD R16, 80(R20)
926 MOVD R17, 88(R20)
927 FMOVD F1, 96(R20)
928 FMOVD F2, 104(R20)
929 FMOVD F3, 112(R20)
930 FMOVD F4, 120(R20)
931 FMOVD F5, 128(R20)
932 FMOVD F6, 136(R20)
933 FMOVD F7, 144(R20)
934 FMOVD F8, 152(R20)
935 FMOVD F9, 160(R20)
936 FMOVD F10, 168(R20)
937 FMOVD F11, 176(R20)
938 FMOVD F12, 184(R20)
939 RET
940
941// unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
942TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
943 MOVD 0(R20), R3
944 MOVD 8(R20), R4
945 MOVD 16(R20), R5
946 MOVD 24(R20), R6
947 MOVD 32(R20), R7
948 MOVD 40(R20), R8
949 MOVD 48(R20), R9
950 MOVD 56(R20), R10
951 MOVD 64(R20), R14
952 MOVD 72(R20), R15
953 MOVD 80(R20), R16
954 MOVD 88(R20), R17
955 FMOVD 96(R20), F1
956 FMOVD 104(R20), F2
957 FMOVD 112(R20), F3
958 FMOVD 120(R20), F4
959 FMOVD 128(R20), F5
960 FMOVD 136(R20), F6
961 FMOVD 144(R20), F7
962 FMOVD 152(R20), F8
963 FMOVD 160(R20), F9
964 FMOVD 168(R20), F10
965 FMOVD 176(R20), F11
966 FMOVD 184(R20), F12
967 RET
968
969// AES hashing not implemented for ppc64
970TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
971 JMP runtime·memhashFallback<ABIInternal>(SB)
972TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
973 JMP runtime·strhashFallback<ABIInternal>(SB)
974TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
975 JMP runtime·memhash32Fallback<ABIInternal>(SB)
976TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
977 JMP runtime·memhash64Fallback<ABIInternal>(SB)
978
979TEXT runtime·return0(SB), NOSPLIT, $0
980 MOVW $0, R3
981 RET
982
983// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
984// Must obey the gcc calling convention.
985#ifdef GOOS_aix
986// On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
987// be a longcall in order to prevent trampolines from ld.
988TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
989#else
990TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
991#endif
992 // g (R30) and R31 are callee-save in the C ABI, so save them
993 MOVD g, R4
994 MOVD R31, R5
995 MOVD LR, R6
996
997 BL runtime·load_g(SB) // clobbers g (R30), R31
998 MOVD g_m(g), R3
999 MOVD m_curg(R3), R3
1000 MOVD (g_stack+stack_hi)(R3), R3
1001
1002 MOVD R4, g
1003 MOVD R5, R31
1004 MOVD R6, LR
1005 RET
1006
1007// The top-most function running on a goroutine
1008// returns to goexit+PCQuantum.
1009//
1010// When dynamically linking Go, it can be returned to from a function
1011// implemented in a different module and so needs to reload the TOC pointer
1012// from the stack (although this function declares that it does not set up x-a
1013// frame, newproc1 does in fact allocate one for goexit and saves the TOC
1014// pointer in the correct place).
1015// goexit+_PCQuantum is halfway through the usual global entry point prologue
1016// that derives r2 from r12 which is a bit silly, but not harmful.
1017TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1018 MOVD 24(R1), R2
1019 BL runtime·goexit1(SB) // does not return
1020 // traceback from goexit1 must hit code range of goexit
1021 MOVD R0, R0 // NOP
1022
1023// prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1024// module containing runtime) to the frame that goexit will execute in when
1025// the goroutine exits. It's implemented in assembly mainly because that's the
1026// easiest way to get access to R2.
1027TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1028 MOVD sp+0(FP), R3
1029 MOVD R2, 24(R3)
1030 RET
1031
1032TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1033 ADD $-8, R1
1034 MOVD R31, 0(R1)
1035 MOVD runtime·lastmoduledatap(SB), R4
1036 MOVD R3, moduledata_next(R4)
1037 MOVD R3, runtime·lastmoduledatap(SB)
1038 MOVD 0(R1), R31
1039 ADD $8, R1
1040 RET
1041
1042TEXT ·checkASM(SB),NOSPLIT,$0-1
1043 MOVW $1, R3
1044 MOVB R3, ret+0(FP)
1045 RET
1046
1047// gcWriteBarrier informs the GC about heap pointer writes.
1048//
1049// gcWriteBarrier does NOT follow the Go ABI. It accepts the
1050// number of bytes of buffer needed in R29, and returns a pointer
1051// to the buffer space in R29.
1052// It clobbers condition codes.
1053// It does not clobber R0 through R17 (except special registers),
1054// but may clobber any other register, *including* R31.
1055TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1056 // The standard prologue clobbers R31.
1057 // We use R18, R19, and R31 as scratch registers.
1058retry:
1059 MOVD g_m(g), R18
1060 MOVD m_p(R18), R18
1061 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1062 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1063 // Increment wbBuf.next position.
1064 ADD R29, R19
1065 // Is the buffer full?
1066 CMPU R31, R19
1067 BLT flush
1068 // Commit to the larger buffer.
1069 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1070 // Make return value (the original next position)
1071 SUB R29, R19, R29
1072 RET
1073
1074flush:
1075 // Save registers R0 through R15 since these were not saved by the caller.
1076 // We don't save all registers on ppc64 because it takes too much space.
1077 MOVD R20, (FIXED_FRAME+0)(R1)
1078 MOVD R21, (FIXED_FRAME+8)(R1)
1079 // R0 is always 0, so no need to spill.
1080 // R1 is SP.
1081 // R2 is SB.
1082 MOVD R3, (FIXED_FRAME+16)(R1)
1083 MOVD R4, (FIXED_FRAME+24)(R1)
1084 MOVD R5, (FIXED_FRAME+32)(R1)
1085 MOVD R6, (FIXED_FRAME+40)(R1)
1086 MOVD R7, (FIXED_FRAME+48)(R1)
1087 MOVD R8, (FIXED_FRAME+56)(R1)
1088 MOVD R9, (FIXED_FRAME+64)(R1)
1089 MOVD R10, (FIXED_FRAME+72)(R1)
1090 // R11, R12 may be clobbered by external-linker-inserted trampoline
1091 // R13 is REGTLS
1092 MOVD R14, (FIXED_FRAME+80)(R1)
1093 MOVD R15, (FIXED_FRAME+88)(R1)
1094 MOVD R16, (FIXED_FRAME+96)(R1)
1095 MOVD R17, (FIXED_FRAME+104)(R1)
1096 MOVD R29, (FIXED_FRAME+112)(R1)
1097
1098 CALL runtime·wbBufFlush(SB)
1099
1100 MOVD (FIXED_FRAME+0)(R1), R20
1101 MOVD (FIXED_FRAME+8)(R1), R21
1102 MOVD (FIXED_FRAME+16)(R1), R3
1103 MOVD (FIXED_FRAME+24)(R1), R4
1104 MOVD (FIXED_FRAME+32)(R1), R5
1105 MOVD (FIXED_FRAME+40)(R1), R6
1106 MOVD (FIXED_FRAME+48)(R1), R7
1107 MOVD (FIXED_FRAME+56)(R1), R8
1108 MOVD (FIXED_FRAME+64)(R1), R9
1109 MOVD (FIXED_FRAME+72)(R1), R10
1110 MOVD (FIXED_FRAME+80)(R1), R14
1111 MOVD (FIXED_FRAME+88)(R1), R15
1112 MOVD (FIXED_FRAME+96)(R1), R16
1113 MOVD (FIXED_FRAME+104)(R1), R17
1114 MOVD (FIXED_FRAME+112)(R1), R29
1115 JMP retry
1116
1117TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1118 MOVD $8, R29
1119 JMP gcWriteBarrier<>(SB)
1120TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1121 MOVD $16, R29
1122 JMP gcWriteBarrier<>(SB)
1123TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1124 MOVD $24, R29
1125 JMP gcWriteBarrier<>(SB)
1126TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1127 MOVD $32, R29
1128 JMP gcWriteBarrier<>(SB)
1129TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1130 MOVD $40, R29
1131 JMP gcWriteBarrier<>(SB)
1132TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1133 MOVD $48, R29
1134 JMP gcWriteBarrier<>(SB)
1135TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1136 MOVD $56, R29
1137 JMP gcWriteBarrier<>(SB)
1138TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1139 MOVD $64, R29
1140 JMP gcWriteBarrier<>(SB)
1141
1142DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1143GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1144
1145// debugCallV2 is the entry point for debugger-injected function
1146// calls on running goroutines. It informs the runtime that a
1147// debug call has been injected and creates a call frame for the
1148// debugger to fill in.
1149//
1150// To inject a function call, a debugger should:
1151// 1. Check that the goroutine is in state _Grunning and that
1152// there are at least 320 bytes free on the stack.
1153// 2. Set SP as SP-32.
1154// 3. Store the current LR in (SP) (using the SP after step 2).
1155// 4. Store the current PC in the LR register.
1156// 5. Write the desired argument frame size at SP-32
1157// 6. Save all machine registers (including flags and floating point registers)
1158// so they can be restored later by the debugger.
1159// 7. Set the PC to debugCallV2 and resume execution.
1160//
1161// If the goroutine is in state _Grunnable, then it's not generally
1162// safe to inject a call because it may return out via other runtime
1163// operations. Instead, the debugger should unwind the stack to find
1164// the return to non-runtime code, add a temporary breakpoint there,
1165// and inject the call once that breakpoint is hit.
1166//
1167// If the goroutine is in any other state, it's not safe to inject a call.
1168//
1169// This function communicates back to the debugger by setting R20 and
1170// invoking TW to raise a breakpoint signal. Note that the signal PC of
1171// the signal triggered by the TW instruction is the PC where the signal
1172// is trapped, not the next PC, so to resume execution, the debugger needs
1173// to set the signal PC to PC+4. See the comments in the implementation for
1174// the protocol the debugger is expected to follow. InjectDebugCall in the
1175// runtime tests demonstrates this protocol.
1176// The debugger must ensure that any pointers passed to the function
1177// obey escape analysis requirements. Specifically, it must not pass
1178// a stack pointer to an escaping argument. debugCallV2 cannot check
1179// this invariant.
1180//
1181// This is ABIInternal because Go code injects its PC directly into new
1182// goroutine stacks.
1183#ifdef GOARCH_ppc64le
1184TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1185 // save scratch register R31 first
1186 MOVD R31, -184(R1)
1187 MOVD 0(R1), R31
1188 // save caller LR
1189 MOVD R31, -304(R1)
1190 MOVD -32(R1), R31
1191 // save argument frame size
1192 MOVD R31, -192(R1)
1193 MOVD LR, R31
1194 MOVD R31, -320(R1)
1195 ADD $-320, R1
1196 // save all registers that can contain pointers
1197 // and the CR register
1198 MOVW CR, R31
1199 MOVD R31, 8(R1)
1200 MOVD R2, 24(R1)
1201 MOVD R3, 56(R1)
1202 MOVD R4, 64(R1)
1203 MOVD R5, 72(R1)
1204 MOVD R6, 80(R1)
1205 MOVD R7, 88(R1)
1206 MOVD R8, 96(R1)
1207 MOVD R9, 104(R1)
1208 MOVD R10, 112(R1)
1209 MOVD R11, 120(R1)
1210 MOVD R12, 144(R1)
1211 MOVD R13, 152(R1)
1212 MOVD R14, 160(R1)
1213 MOVD R15, 168(R1)
1214 MOVD R16, 176(R1)
1215 MOVD R17, 184(R1)
1216 MOVD R18, 192(R1)
1217 MOVD R19, 200(R1)
1218 MOVD R20, 208(R1)
1219 MOVD R21, 216(R1)
1220 MOVD R22, 224(R1)
1221 MOVD R23, 232(R1)
1222 MOVD R24, 240(R1)
1223 MOVD R25, 248(R1)
1224 MOVD R26, 256(R1)
1225 MOVD R27, 264(R1)
1226 MOVD R28, 272(R1)
1227 MOVD R29, 280(R1)
1228 MOVD g, 288(R1)
1229 MOVD LR, R31
1230 MOVD R31, 32(R1)
1231 CALL runtime·debugCallCheck(SB)
1232 MOVD 40(R1), R22
1233 XOR R0, R0
1234 CMP R22, R0
1235 BEQ good
1236 MOVD 48(R1), R22
1237 MOVD $8, R20
1238 TW $31, R0, R0
1239
1240 BR restore
1241
1242good:
1243#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1244 MOVD $MAXSIZE, R23; \
1245 CMP R26, R23; \
1246 BGT 5(PC); \
1247 MOVD $NAME(SB), R26; \
1248 MOVD R26, 32(R1); \
1249 CALL runtime·debugCallWrap(SB); \
1250 BR restore
1251
1252 // the argument frame size
1253 MOVD 128(R1), R26
1254
1255 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1256 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1257 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1258 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1259 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1260 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1261 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1262 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1263 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1264 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1265 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1266 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1267 // The frame size is too large. Report the error.
1268 MOVD $debugCallFrameTooLarge<>(SB), R22
1269 MOVD R22, 32(R1)
1270 MOVD $20, R22
1271 // length of debugCallFrameTooLarge string
1272 MOVD R22, 40(R1)
1273 MOVD $8, R20
1274 TW $31, R0, R0
1275 BR restore
1276restore:
1277 MOVD $16, R20
1278 TW $31, R0, R0
1279 // restore all registers that can contain
1280 // pointers including CR
1281 MOVD 8(R1), R31
1282 MOVW R31, CR
1283 MOVD 24(R1), R2
1284 MOVD 56(R1), R3
1285 MOVD 64(R1), R4
1286 MOVD 72(R1), R5
1287 MOVD 80(R1), R6
1288 MOVD 88(R1), R7
1289 MOVD 96(R1), R8
1290 MOVD 104(R1), R9
1291 MOVD 112(R1), R10
1292 MOVD 120(R1), R11
1293 MOVD 144(R1), R12
1294 MOVD 152(R1), R13
1295 MOVD 160(R1), R14
1296 MOVD 168(R1), R15
1297 MOVD 176(R1), R16
1298 MOVD 184(R1), R17
1299 MOVD 192(R1), R18
1300 MOVD 200(R1), R19
1301 MOVD 208(R1), R20
1302 MOVD 216(R1), R21
1303 MOVD 224(R1), R22
1304 MOVD 232(R1), R23
1305 MOVD 240(R1), R24
1306 MOVD 248(R1), R25
1307 MOVD 256(R1), R26
1308 MOVD 264(R1), R27
1309 MOVD 272(R1), R28
1310 MOVD 280(R1), R29
1311 MOVD 288(R1), g
1312 MOVD 16(R1), R31
1313 // restore old LR
1314 MOVD R31, LR
1315 // restore caller PC
1316 MOVD 0(R1), CTR
1317 MOVD 136(R1), R31
1318 // Add 32 bytes more to compensate for SP change in saveSigContext
1319 ADD $352, R1
1320 JMP (CTR)
1321#endif
1322#define DEBUG_CALL_FN(NAME,MAXSIZE) \
1323TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1324 NO_LOCAL_POINTERS; \
1325 MOVD $0, R20; \
1326 TW $31, R0, R0 \
1327 MOVD $1, R20; \
1328 TW $31, R0, R0 \
1329 RET
1330DEBUG_CALL_FN(debugCall32<>, 32)
1331DEBUG_CALL_FN(debugCall64<>, 64)
1332DEBUG_CALL_FN(debugCall128<>, 128)
1333DEBUG_CALL_FN(debugCall256<>, 256)
1334DEBUG_CALL_FN(debugCall512<>, 512)
1335DEBUG_CALL_FN(debugCall1024<>, 1024)
1336DEBUG_CALL_FN(debugCall2048<>, 2048)
1337DEBUG_CALL_FN(debugCall4096<>, 4096)
1338DEBUG_CALL_FN(debugCall8192<>, 8192)
1339DEBUG_CALL_FN(debugCall16384<>, 16384)
1340DEBUG_CALL_FN(debugCall32768<>, 32768)
1341DEBUG_CALL_FN(debugCall65536<>, 65536)
1342
1343#ifdef GOARCH_ppc64le
1344// func debugCallPanicked(val interface{})
1345TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1346 // Copy the panic value to the top of stack at SP+32.
1347 MOVD val_type+0(FP), R31
1348 MOVD R31, 32(R1)
1349 MOVD val_data+8(FP), R31
1350 MOVD R31, 40(R1)
1351 MOVD $2, R20
1352 TW $31, R0, R0
1353 RET
1354#endif
1355// Note: these functions use a special calling convention to save generated code space.
1356// Arguments are passed in registers, but the space for those arguments are allocated
1357// in the caller's stack frame. These stubs write the args into that stack space and
1358// then tail call to the corresponding runtime handler.
1359// The tail call makes these stubs disappear in backtraces.
1360TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
1361 JMP runtime·goPanicIndex<ABIInternal>(SB)
1362TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
1363 JMP runtime·goPanicIndexU<ABIInternal>(SB)
1364TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
1365 MOVD R4, R3
1366 MOVD R5, R4
1367 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
1368TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
1369 MOVD R4, R3
1370 MOVD R5, R4
1371 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
1372TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
1373 MOVD R4, R3
1374 MOVD R5, R4
1375 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
1376TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
1377 MOVD R4, R3
1378 MOVD R5, R4
1379 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
1380TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
1381 JMP runtime·goPanicSliceB<ABIInternal>(SB)
1382TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
1383 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
1384TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
1385 MOVD R5, R3
1386 MOVD R6, R4
1387 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
1388TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
1389 MOVD R5, R3
1390 MOVD R6, R4
1391 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
1392TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
1393 MOVD R5, R3
1394 MOVD R6, R4
1395 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1396TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1397 MOVD R5, R3
1398 MOVD R6, R4
1399 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1400TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1401 MOVD R4, R3
1402 MOVD R5, R4
1403 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1404TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1405 MOVD R4, R3
1406 MOVD R5, R4
1407 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1408TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1409 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1410TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1411 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1412TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1413 MOVD R5, R3
1414 MOVD R6, R4
1415 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1416
1417// These functions are used when internal linking cgo with external
1418// objects compiled with the -Os on gcc. They reduce prologue/epilogue
1419// size by deferring preservation of callee-save registers to a shared
1420// function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1421// in ELFv1)
1422//
1423// These appear unused, but the linker will redirect calls to functions
1424// like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1425// runtime.elf_restgpr1 with an appropriate offset based on the number
1426// register operations required when linking external objects which
1427// make these calls. For GPR/FPR saves, the minimum register value is
1428// 14, for VR it is 20.
1429//
1430// These are only used when linking such cgo code internally. Note, R12
1431// and R0 may be used in different ways than regular ELF compliant
1432// functions.
1433TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1434 // R0 holds the LR of the caller's caller, R1 holds save location
1435 MOVD R14, -144(R1)
1436 MOVD R15, -136(R1)
1437 MOVD R16, -128(R1)
1438 MOVD R17, -120(R1)
1439 MOVD R18, -112(R1)
1440 MOVD R19, -104(R1)
1441 MOVD R20, -96(R1)
1442 MOVD R21, -88(R1)
1443 MOVD R22, -80(R1)
1444 MOVD R23, -72(R1)
1445 MOVD R24, -64(R1)
1446 MOVD R25, -56(R1)
1447 MOVD R26, -48(R1)
1448 MOVD R27, -40(R1)
1449 MOVD R28, -32(R1)
1450 MOVD R29, -24(R1)
1451 MOVD g, -16(R1)
1452 MOVD R31, -8(R1)
1453 MOVD R0, 16(R1)
1454 RET
1455TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1456 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1457 MOVD -144(R1), R14
1458 MOVD -136(R1), R15
1459 MOVD -128(R1), R16
1460 MOVD -120(R1), R17
1461 MOVD -112(R1), R18
1462 MOVD -104(R1), R19
1463 MOVD -96(R1), R20
1464 MOVD -88(R1), R21
1465 MOVD -80(R1), R22
1466 MOVD -72(R1), R23
1467 MOVD -64(R1), R24
1468 MOVD -56(R1), R25
1469 MOVD -48(R1), R26
1470 MOVD -40(R1), R27
1471 MOVD -32(R1), R28
1472 MOVD -24(R1), R29
1473 MOVD -16(R1), g
1474 MOVD -8(R1), R31
1475 MOVD 16(R1), R0 // Load and return to saved LR
1476 MOVD R0, LR
1477 RET
1478TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1479 // R12 holds the save location
1480 MOVD R14, -144(R12)
1481 MOVD R15, -136(R12)
1482 MOVD R16, -128(R12)
1483 MOVD R17, -120(R12)
1484 MOVD R18, -112(R12)
1485 MOVD R19, -104(R12)
1486 MOVD R20, -96(R12)
1487 MOVD R21, -88(R12)
1488 MOVD R22, -80(R12)
1489 MOVD R23, -72(R12)
1490 MOVD R24, -64(R12)
1491 MOVD R25, -56(R12)
1492 MOVD R26, -48(R12)
1493 MOVD R27, -40(R12)
1494 MOVD R28, -32(R12)
1495 MOVD R29, -24(R12)
1496 MOVD g, -16(R12)
1497 MOVD R31, -8(R12)
1498 RET
1499TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1500 // R12 holds the save location
1501 MOVD -144(R12), R14
1502 MOVD -136(R12), R15
1503 MOVD -128(R12), R16
1504 MOVD -120(R12), R17
1505 MOVD -112(R12), R18
1506 MOVD -104(R12), R19
1507 MOVD -96(R12), R20
1508 MOVD -88(R12), R21
1509 MOVD -80(R12), R22
1510 MOVD -72(R12), R23
1511 MOVD -64(R12), R24
1512 MOVD -56(R12), R25
1513 MOVD -48(R12), R26
1514 MOVD -40(R12), R27
1515 MOVD -32(R12), R28
1516 MOVD -24(R12), R29
1517 MOVD -16(R12), g
1518 MOVD -8(R12), R31
1519 RET
1520TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1521 // R0 holds the LR of the caller's caller, R1 holds save location
1522 FMOVD F14, -144(R1)
1523 FMOVD F15, -136(R1)
1524 FMOVD F16, -128(R1)
1525 FMOVD F17, -120(R1)
1526 FMOVD F18, -112(R1)
1527 FMOVD F19, -104(R1)
1528 FMOVD F20, -96(R1)
1529 FMOVD F21, -88(R1)
1530 FMOVD F22, -80(R1)
1531 FMOVD F23, -72(R1)
1532 FMOVD F24, -64(R1)
1533 FMOVD F25, -56(R1)
1534 FMOVD F26, -48(R1)
1535 FMOVD F27, -40(R1)
1536 FMOVD F28, -32(R1)
1537 FMOVD F29, -24(R1)
1538 FMOVD F30, -16(R1)
1539 FMOVD F31, -8(R1)
1540 MOVD R0, 16(R1)
1541 RET
1542TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1543 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1544 FMOVD -144(R1), F14
1545 FMOVD -136(R1), F15
1546 FMOVD -128(R1), F16
1547 FMOVD -120(R1), F17
1548 FMOVD -112(R1), F18
1549 FMOVD -104(R1), F19
1550 FMOVD -96(R1), F20
1551 FMOVD -88(R1), F21
1552 FMOVD -80(R1), F22
1553 FMOVD -72(R1), F23
1554 FMOVD -64(R1), F24
1555 FMOVD -56(R1), F25
1556 FMOVD -48(R1), F26
1557 FMOVD -40(R1), F27
1558 FMOVD -32(R1), F28
1559 FMOVD -24(R1), F29
1560 FMOVD -16(R1), F30
1561 FMOVD -8(R1), F31
1562 MOVD 16(R1), R0 // Load and return to saved LR
1563 MOVD R0, LR
1564 RET
1565TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1566 // R0 holds the save location, R12 is clobbered
1567 MOVD $-192, R12
1568 STVX V20, (R0+R12)
1569 MOVD $-176, R12
1570 STVX V21, (R0+R12)
1571 MOVD $-160, R12
1572 STVX V22, (R0+R12)
1573 MOVD $-144, R12
1574 STVX V23, (R0+R12)
1575 MOVD $-128, R12
1576 STVX V24, (R0+R12)
1577 MOVD $-112, R12
1578 STVX V25, (R0+R12)
1579 MOVD $-96, R12
1580 STVX V26, (R0+R12)
1581 MOVD $-80, R12
1582 STVX V27, (R0+R12)
1583 MOVD $-64, R12
1584 STVX V28, (R0+R12)
1585 MOVD $-48, R12
1586 STVX V29, (R0+R12)
1587 MOVD $-32, R12
1588 STVX V30, (R0+R12)
1589 MOVD $-16, R12
1590 STVX V31, (R0+R12)
1591 RET
1592TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1593 // R0 holds the save location, R12 is clobbered
1594 MOVD $-192, R12
1595 LVX (R0+R12), V20
1596 MOVD $-176, R12
1597 LVX (R0+R12), V21
1598 MOVD $-160, R12
1599 LVX (R0+R12), V22
1600 MOVD $-144, R12
1601 LVX (R0+R12), V23
1602 MOVD $-128, R12
1603 LVX (R0+R12), V24
1604 MOVD $-112, R12
1605 LVX (R0+R12), V25
1606 MOVD $-96, R12
1607 LVX (R0+R12), V26
1608 MOVD $-80, R12
1609 LVX (R0+R12), V27
1610 MOVD $-64, R12
1611 LVX (R0+R12), V28
1612 MOVD $-48, R12
1613 LVX (R0+R12), V29
1614 MOVD $-32, R12
1615 LVX (R0+R12), V30
1616 MOVD $-16, R12
1617 LVX (R0+R12), V31
1618 RET
View as plain text