Text file
src/runtime/asm_loong64.s
Documentation: runtime
1// Copyright 2022 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10#define REGCTXT R29
11
12TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
13 // R3 = stack; R4 = argc; R5 = argv
14
15 ADDV $-24, R3
16 MOVW R4, 8(R3) // argc
17 MOVV R5, 16(R3) // argv
18
19 // create istack out of the given (operating system) stack.
20 // _cgo_init may update stackguard.
21 MOVV $runtime·g0(SB), g
22 MOVV $(-64*1024), R30
23 ADDV R30, R3, R19
24 MOVV R19, g_stackguard0(g)
25 MOVV R19, g_stackguard1(g)
26 MOVV R19, (g_stack+stack_lo)(g)
27 MOVV R3, (g_stack+stack_hi)(g)
28
29 // if there is a _cgo_init, call it using the gcc ABI.
30 MOVV _cgo_init(SB), R25
31 BEQ R25, nocgo
32
33 MOVV R0, R7 // arg 3: not used
34 MOVV R0, R6 // arg 2: not used
35 MOVV $setg_gcc<>(SB), R5 // arg 1: setg
36 MOVV g, R4 // arg 0: G
37 JAL (R25)
38
39nocgo:
40 // update stackguard after _cgo_init
41 MOVV (g_stack+stack_lo)(g), R19
42 ADDV $const_stackGuard, R19
43 MOVV R19, g_stackguard0(g)
44 MOVV R19, g_stackguard1(g)
45
46 // set the per-goroutine and per-mach "registers"
47 MOVV $runtime·m0(SB), R19
48
49 // save m->g0 = g0
50 MOVV g, m_g0(R19)
51 // save m0 to g0->m
52 MOVV R19, g_m(g)
53
54 JAL runtime·check(SB)
55
56 // args are already prepared
57 JAL runtime·args(SB)
58 JAL runtime·osinit(SB)
59 JAL runtime·schedinit(SB)
60
61 // create a new goroutine to start program
62 MOVV $runtime·mainPC(SB), R19 // entry
63 ADDV $-16, R3
64 MOVV R19, 8(R3)
65 MOVV R0, 0(R3)
66 JAL runtime·newproc(SB)
67 ADDV $16, R3
68
69 // start this M
70 JAL runtime·mstart(SB)
71
72 MOVV R0, 1(R0)
73 RET
74
75DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
76GLOBL runtime·mainPC(SB),RODATA,$8
77
78TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
79 BREAK
80 RET
81
82TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
83 RET
84
85TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
86 JAL runtime·mstart0(SB)
87 RET // not reached
88
89// func cputicks() int64
90TEXT runtime·cputicks(SB),NOSPLIT,$0-8
91 RDTIMED R0, R4
92 MOVV R4, ret+0(FP)
93 RET
94
95/*
96 * go-routine
97 */
98
99// void gogo(Gobuf*)
100// restore state from Gobuf; longjmp
101TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
102 MOVV buf+0(FP), R4
103 MOVV gobuf_g(R4), R5
104 MOVV 0(R5), R0 // make sure g != nil
105 JMP gogo<>(SB)
106
107TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
108 MOVV R5, g
109 JAL runtime·save_g(SB)
110
111 MOVV gobuf_sp(R4), R3
112 MOVV gobuf_lr(R4), R1
113 MOVV gobuf_ret(R4), R19
114 MOVV gobuf_ctxt(R4), REGCTXT
115 MOVV R0, gobuf_sp(R4)
116 MOVV R0, gobuf_ret(R4)
117 MOVV R0, gobuf_lr(R4)
118 MOVV R0, gobuf_ctxt(R4)
119 MOVV gobuf_pc(R4), R6
120 JMP (R6)
121
122// void mcall(fn func(*g))
123// Switch to m->g0's stack, call fn(g).
124// Fn must never return. It should gogo(&g->sched)
125// to keep running g.
126TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
127#ifdef GOEXPERIMENT_regabiargs
128 MOVV R4, REGCTXT
129#else
130 MOVV fn+0(FP), REGCTXT
131#endif
132
133 // Save caller state in g->sched
134 MOVV R3, (g_sched+gobuf_sp)(g)
135 MOVV R1, (g_sched+gobuf_pc)(g)
136 MOVV R0, (g_sched+gobuf_lr)(g)
137
138 // Switch to m->g0 & its stack, call fn.
139 MOVV g, R4 // arg = g
140 MOVV g_m(g), R20
141 MOVV m_g0(R20), g
142 JAL runtime·save_g(SB)
143 BNE g, R4, 2(PC)
144 JMP runtime·badmcall(SB)
145 MOVV 0(REGCTXT), R20 // code pointer
146 MOVV (g_sched+gobuf_sp)(g), R3 // sp = m->g0->sched.sp
147 ADDV $-16, R3
148 MOVV R4, 8(R3)
149 MOVV R0, 0(R3)
150 JAL (R20)
151 JMP runtime·badmcall2(SB)
152
153// systemstack_switch is a dummy routine that systemstack leaves at the bottom
154// of the G stack. We need to distinguish the routine that
155// lives at the bottom of the G stack from the one that lives
156// at the top of the system stack because the one at the top of
157// the system stack terminates the stack walk (see topofstack()).
158TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
159 UNDEF
160 JAL (R1) // make sure this function is not leaf
161 RET
162
163// func systemstack(fn func())
164TEXT runtime·systemstack(SB), NOSPLIT, $0-8
165 MOVV fn+0(FP), R19 // R19 = fn
166 MOVV R19, REGCTXT // context
167 MOVV g_m(g), R4 // R4 = m
168
169 MOVV m_gsignal(R4), R5 // R5 = gsignal
170 BEQ g, R5, noswitch
171
172 MOVV m_g0(R4), R5 // R5 = g0
173 BEQ g, R5, noswitch
174
175 MOVV m_curg(R4), R6
176 BEQ g, R6, switch
177
178 // Bad: g is not gsignal, not g0, not curg. What is it?
179 // Hide call from linker nosplit analysis.
180 MOVV $runtime·badsystemstack(SB), R7
181 JAL (R7)
182 JAL runtime·abort(SB)
183
184switch:
185 // save our state in g->sched. Pretend to
186 // be systemstack_switch if the G stack is scanned.
187 JAL gosave_systemstack_switch<>(SB)
188
189 // switch to g0
190 MOVV R5, g
191 JAL runtime·save_g(SB)
192 MOVV (g_sched+gobuf_sp)(g), R19
193 MOVV R19, R3
194
195 // call target function
196 MOVV 0(REGCTXT), R6 // code pointer
197 JAL (R6)
198
199 // switch back to g
200 MOVV g_m(g), R4
201 MOVV m_curg(R4), g
202 JAL runtime·save_g(SB)
203 MOVV (g_sched+gobuf_sp)(g), R3
204 MOVV R0, (g_sched+gobuf_sp)(g)
205 RET
206
207noswitch:
208 // already on m stack, just call directly
209 // Using a tail call here cleans up tracebacks since we won't stop
210 // at an intermediate systemstack.
211 MOVV 0(REGCTXT), R4 // code pointer
212 MOVV 0(R3), R1 // restore LR
213 ADDV $8, R3
214 JMP (R4)
215
216/*
217 * support for morestack
218 */
219
220// Called during function prolog when more stack is needed.
221// Caller has already loaded:
222// loong64: R31: LR
223//
224// The traceback routines see morestack on a g0 as being
225// the top of a stack (for example, morestack calling newstack
226// calling the scheduler calling newm calling gc), so we must
227// record an argument size. For that purpose, it has no arguments.
228TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
229 // Cannot grow scheduler stack (m->g0).
230 MOVV g_m(g), R7
231 MOVV m_g0(R7), R8
232 BNE g, R8, 3(PC)
233 JAL runtime·badmorestackg0(SB)
234 JAL runtime·abort(SB)
235
236 // Cannot grow signal stack (m->gsignal).
237 MOVV m_gsignal(R7), R8
238 BNE g, R8, 3(PC)
239 JAL runtime·badmorestackgsignal(SB)
240 JAL runtime·abort(SB)
241
242 // Called from f.
243 // Set g->sched to context in f.
244 MOVV R3, (g_sched+gobuf_sp)(g)
245 MOVV R1, (g_sched+gobuf_pc)(g)
246 MOVV R31, (g_sched+gobuf_lr)(g)
247 MOVV REGCTXT, (g_sched+gobuf_ctxt)(g)
248
249 // Called from f.
250 // Set m->morebuf to f's caller.
251 MOVV R31, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
252 MOVV R3, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
253 MOVV g, (m_morebuf+gobuf_g)(R7)
254
255 // Call newstack on m->g0's stack.
256 MOVV m_g0(R7), g
257 JAL runtime·save_g(SB)
258 MOVV (g_sched+gobuf_sp)(g), R3
259 // Create a stack frame on g0 to call newstack.
260 MOVV R0, -8(R3) // Zero saved LR in frame
261 ADDV $-8, R3
262 JAL runtime·newstack(SB)
263
264 // Not reached, but make sure the return PC from the call to newstack
265 // is still in this function, and not the beginning of the next.
266 UNDEF
267
268TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
269 // Force SPWRITE. This function doesn't actually write SP,
270 // but it is called with a special calling convention where
271 // the caller doesn't save LR on stack but passes it as a
272 // register (R5), and the unwinder currently doesn't understand.
273 // Make it SPWRITE to stop unwinding. (See issue 54332)
274 MOVV R3, R3
275
276 MOVV R0, REGCTXT
277 JMP runtime·morestack(SB)
278
279// reflectcall: call a function with the given argument list
280// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
281// we don't have variable-sized frames, so we use a small number
282// of constant-sized-frame functions to encode a few bits of size in the pc.
283// Caution: ugly multiline assembly macros in your future!
284
285#define DISPATCH(NAME,MAXSIZE) \
286 MOVV $MAXSIZE, R30; \
287 SGTU R19, R30, R30; \
288 BNE R30, 3(PC); \
289 MOVV $NAME(SB), R4; \
290 JMP (R4)
291// Note: can't just "BR NAME(SB)" - bad inlining results.
292
293TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
294 MOVWU frameSize+32(FP), R19
295 DISPATCH(runtime·call32, 32)
296 DISPATCH(runtime·call64, 64)
297 DISPATCH(runtime·call128, 128)
298 DISPATCH(runtime·call256, 256)
299 DISPATCH(runtime·call512, 512)
300 DISPATCH(runtime·call1024, 1024)
301 DISPATCH(runtime·call2048, 2048)
302 DISPATCH(runtime·call4096, 4096)
303 DISPATCH(runtime·call8192, 8192)
304 DISPATCH(runtime·call16384, 16384)
305 DISPATCH(runtime·call32768, 32768)
306 DISPATCH(runtime·call65536, 65536)
307 DISPATCH(runtime·call131072, 131072)
308 DISPATCH(runtime·call262144, 262144)
309 DISPATCH(runtime·call524288, 524288)
310 DISPATCH(runtime·call1048576, 1048576)
311 DISPATCH(runtime·call2097152, 2097152)
312 DISPATCH(runtime·call4194304, 4194304)
313 DISPATCH(runtime·call8388608, 8388608)
314 DISPATCH(runtime·call16777216, 16777216)
315 DISPATCH(runtime·call33554432, 33554432)
316 DISPATCH(runtime·call67108864, 67108864)
317 DISPATCH(runtime·call134217728, 134217728)
318 DISPATCH(runtime·call268435456, 268435456)
319 DISPATCH(runtime·call536870912, 536870912)
320 DISPATCH(runtime·call1073741824, 1073741824)
321 MOVV $runtime·badreflectcall(SB), R4
322 JMP (R4)
323
324#define CALLFN(NAME,MAXSIZE) \
325TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
326 NO_LOCAL_POINTERS; \
327 /* copy arguments to stack */ \
328 MOVV arg+16(FP), R4; \
329 MOVWU argsize+24(FP), R5; \
330 MOVV R3, R12; \
331 ADDV $8, R12; \
332 ADDV R12, R5; \
333 BEQ R12, R5, 6(PC); \
334 MOVBU (R4), R6; \
335 ADDV $1, R4; \
336 MOVBU R6, (R12); \
337 ADDV $1, R12; \
338 JMP -5(PC); \
339 /* set up argument registers */ \
340 MOVV regArgs+40(FP), R25; \
341 JAL ·unspillArgs(SB); \
342 /* call function */ \
343 MOVV f+8(FP), REGCTXT; \
344 MOVV (REGCTXT), R25; \
345 PCDATA $PCDATA_StackMapIndex, $0; \
346 JAL (R25); \
347 /* copy return values back */ \
348 MOVV regArgs+40(FP), R25; \
349 JAL ·spillArgs(SB); \
350 MOVV argtype+0(FP), R7; \
351 MOVV arg+16(FP), R4; \
352 MOVWU n+24(FP), R5; \
353 MOVWU retoffset+28(FP), R6; \
354 ADDV $8, R3, R12; \
355 ADDV R6, R12; \
356 ADDV R6, R4; \
357 SUBVU R6, R5; \
358 JAL callRet<>(SB); \
359 RET
360
361// callRet copies return values back at the end of call*. This is a
362// separate function so it can allocate stack space for the arguments
363// to reflectcallmove. It does not follow the Go ABI; it expects its
364// arguments in registers.
365TEXT callRet<>(SB), NOSPLIT, $40-0
366 NO_LOCAL_POINTERS
367 MOVV R7, 8(R3)
368 MOVV R4, 16(R3)
369 MOVV R12, 24(R3)
370 MOVV R5, 32(R3)
371 MOVV R25, 40(R3)
372 JAL runtime·reflectcallmove(SB)
373 RET
374
375CALLFN(·call16, 16)
376CALLFN(·call32, 32)
377CALLFN(·call64, 64)
378CALLFN(·call128, 128)
379CALLFN(·call256, 256)
380CALLFN(·call512, 512)
381CALLFN(·call1024, 1024)
382CALLFN(·call2048, 2048)
383CALLFN(·call4096, 4096)
384CALLFN(·call8192, 8192)
385CALLFN(·call16384, 16384)
386CALLFN(·call32768, 32768)
387CALLFN(·call65536, 65536)
388CALLFN(·call131072, 131072)
389CALLFN(·call262144, 262144)
390CALLFN(·call524288, 524288)
391CALLFN(·call1048576, 1048576)
392CALLFN(·call2097152, 2097152)
393CALLFN(·call4194304, 4194304)
394CALLFN(·call8388608, 8388608)
395CALLFN(·call16777216, 16777216)
396CALLFN(·call33554432, 33554432)
397CALLFN(·call67108864, 67108864)
398CALLFN(·call134217728, 134217728)
399CALLFN(·call268435456, 268435456)
400CALLFN(·call536870912, 536870912)
401CALLFN(·call1073741824, 1073741824)
402
403TEXT runtime·procyield(SB),NOSPLIT,$0-0
404 RET
405
406// Save state of caller into g->sched.
407// but using fake PC from systemstack_switch.
408// Must only be called from functions with no locals ($0)
409// or else unwinding from systemstack_switch is incorrect.
410// Smashes R19.
411TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
412 MOVV $runtime·systemstack_switch(SB), R19
413 ADDV $8, R19
414 MOVV R19, (g_sched+gobuf_pc)(g)
415 MOVV R3, (g_sched+gobuf_sp)(g)
416 MOVV R0, (g_sched+gobuf_lr)(g)
417 MOVV R0, (g_sched+gobuf_ret)(g)
418 // Assert ctxt is zero. See func save.
419 MOVV (g_sched+gobuf_ctxt)(g), R19
420 BEQ R19, 2(PC)
421 JAL runtime·abort(SB)
422 RET
423
424// func asmcgocall(fn, arg unsafe.Pointer) int32
425// Call fn(arg) on the scheduler stack,
426// aligned appropriately for the gcc ABI.
427// See cgocall.go for more details.
428TEXT ·asmcgocall(SB),NOSPLIT,$0-20
429 MOVV fn+0(FP), R25
430 MOVV arg+8(FP), R4
431
432 MOVV R3, R12 // save original stack pointer
433 MOVV g, R13
434
435 // Figure out if we need to switch to m->g0 stack.
436 // We get called to create new OS threads too, and those
437 // come in on the m->g0 stack already.
438 MOVV g_m(g), R5
439 MOVV m_gsignal(R5), R6
440 BEQ R6, g, g0
441 MOVV m_g0(R5), R6
442 BEQ R6, g, g0
443
444 JAL gosave_systemstack_switch<>(SB)
445 MOVV R6, g
446 JAL runtime·save_g(SB)
447 MOVV (g_sched+gobuf_sp)(g), R3
448
449 // Now on a scheduling stack (a pthread-created stack).
450g0:
451 // Save room for two of our pointers.
452 ADDV $-16, R3
453 MOVV R13, 0(R3) // save old g on stack
454 MOVV (g_stack+stack_hi)(R13), R13
455 SUBVU R12, R13
456 MOVV R13, 8(R3) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
457 JAL (R25)
458
459 // Restore g, stack pointer. R4 is return value.
460 MOVV 0(R3), g
461 JAL runtime·save_g(SB)
462 MOVV (g_stack+stack_hi)(g), R5
463 MOVV 8(R3), R6
464 SUBVU R6, R5
465 MOVV R5, R3
466
467 MOVW R4, ret+16(FP)
468 RET
469
470// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
471// See cgocall.go for more details.
472TEXT ·cgocallback(SB),NOSPLIT,$24-24
473 NO_LOCAL_POINTERS
474
475 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
476 // It is used to dropm while thread is exiting.
477 MOVV fn+0(FP), R5
478 BNE R5, loadg
479 // Restore the g from frame.
480 MOVV frame+8(FP), g
481 JMP dropm
482
483loadg:
484 // Load m and g from thread-local storage.
485 MOVB runtime·iscgo(SB), R19
486 BEQ R19, nocgo
487 JAL runtime·load_g(SB)
488nocgo:
489
490 // If g is nil, Go did not create the current thread,
491 // or if this thread never called into Go on pthread platforms.
492 // Call needm to obtain one for temporary use.
493 // In this case, we're running on the thread stack, so there's
494 // lots of space, but the linker doesn't know. Hide the call from
495 // the linker analysis by using an indirect call.
496 BEQ g, needm
497
498 MOVV g_m(g), R12
499 MOVV R12, savedm-8(SP)
500 JMP havem
501
502needm:
503 MOVV g, savedm-8(SP) // g is zero, so is m.
504 MOVV $runtime·needAndBindM(SB), R4
505 JAL (R4)
506
507 // Set m->sched.sp = SP, so that if a panic happens
508 // during the function we are about to execute, it will
509 // have a valid SP to run on the g0 stack.
510 // The next few lines (after the havem label)
511 // will save this SP onto the stack and then write
512 // the same SP back to m->sched.sp. That seems redundant,
513 // but if an unrecovered panic happens, unwindm will
514 // restore the g->sched.sp from the stack location
515 // and then systemstack will try to use it. If we don't set it here,
516 // that restored SP will be uninitialized (typically 0) and
517 // will not be usable.
518 MOVV g_m(g), R12
519 MOVV m_g0(R12), R19
520 MOVV R3, (g_sched+gobuf_sp)(R19)
521
522havem:
523 // Now there's a valid m, and we're running on its m->g0.
524 // Save current m->g0->sched.sp on stack and then set it to SP.
525 // Save current sp in m->g0->sched.sp in preparation for
526 // switch back to m->curg stack.
527 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R29) aka savedsp-16(SP).
528 MOVV m_g0(R12), R19
529 MOVV (g_sched+gobuf_sp)(R19), R13
530 MOVV R13, savedsp-24(SP) // must match frame size
531 MOVV R3, (g_sched+gobuf_sp)(R19)
532
533 // Switch to m->curg stack and call runtime.cgocallbackg.
534 // Because we are taking over the execution of m->curg
535 // but *not* resuming what had been running, we need to
536 // save that information (m->curg->sched) so we can restore it.
537 // We can restore m->curg->sched.sp easily, because calling
538 // runtime.cgocallbackg leaves SP unchanged upon return.
539 // To save m->curg->sched.pc, we push it onto the stack.
540 // This has the added benefit that it looks to the traceback
541 // routine like cgocallbackg is going to return to that
542 // PC (because the frame we allocate below has the same
543 // size as cgocallback_gofunc's frame declared above)
544 // so that the traceback will seamlessly trace back into
545 // the earlier calls.
546 MOVV m_curg(R12), g
547 JAL runtime·save_g(SB)
548 MOVV (g_sched+gobuf_sp)(g), R13 // prepare stack as R13
549 MOVV (g_sched+gobuf_pc)(g), R4
550 MOVV R4, -(24+8)(R13) // "saved LR"; must match frame size
551 MOVV fn+0(FP), R5
552 MOVV frame+8(FP), R6
553 MOVV ctxt+16(FP), R7
554 MOVV $-(24+8)(R13), R3
555 MOVV R5, 8(R3)
556 MOVV R6, 16(R3)
557 MOVV R7, 24(R3)
558 JAL runtime·cgocallbackg(SB)
559
560 // Restore g->sched (== m->curg->sched) from saved values.
561 MOVV 0(R3), R4
562 MOVV R4, (g_sched+gobuf_pc)(g)
563 MOVV $(24+8)(R3), R13 // must match frame size
564 MOVV R13, (g_sched+gobuf_sp)(g)
565
566 // Switch back to m->g0's stack and restore m->g0->sched.sp.
567 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
568 // so we do not have to restore it.)
569 MOVV g_m(g), R12
570 MOVV m_g0(R12), g
571 JAL runtime·save_g(SB)
572 MOVV (g_sched+gobuf_sp)(g), R3
573 MOVV savedsp-24(SP), R13 // must match frame size
574 MOVV R13, (g_sched+gobuf_sp)(g)
575
576 // If the m on entry was nil, we called needm above to borrow an m,
577 // 1. for the duration of the call on non-pthread platforms,
578 // 2. or the duration of the C thread alive on pthread platforms.
579 // If the m on entry wasn't nil,
580 // 1. the thread might be a Go thread,
581 // 2. or it wasn't the first call from a C thread on pthread platforms,
582 // since then we skip dropm to resue the m in the first call.
583 MOVV savedm-8(SP), R12
584 BNE R12, droppedm
585
586 // Skip dropm to reuse it in the next call, when a pthread key has been created.
587 MOVV _cgo_pthread_key_created(SB), R12
588 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
589 BEQ R12, dropm
590 MOVV (R12), R12
591 BNE R12, droppedm
592
593dropm:
594 MOVV $runtime·dropm(SB), R4
595 JAL (R4)
596droppedm:
597
598 // Done!
599 RET
600
601// void setg(G*); set g. for use by needm.
602TEXT runtime·setg(SB), NOSPLIT, $0-8
603 MOVV gg+0(FP), g
604 // This only happens if iscgo, so jump straight to save_g
605 JAL runtime·save_g(SB)
606 RET
607
608// void setg_gcc(G*); set g called from gcc with g in R19
609TEXT setg_gcc<>(SB),NOSPLIT,$0-0
610 MOVV R19, g
611 JAL runtime·save_g(SB)
612 RET
613
614TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
615 MOVW (R0), R0
616 UNDEF
617
618// AES hashing not implemented for loong64
619TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
620 JMP runtime·memhashFallback<ABIInternal>(SB)
621TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
622 JMP runtime·strhashFallback<ABIInternal>(SB)
623TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
624 JMP runtime·memhash32Fallback<ABIInternal>(SB)
625TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
626 JMP runtime·memhash64Fallback<ABIInternal>(SB)
627
628TEXT runtime·return0(SB), NOSPLIT, $0
629 MOVW $0, R19
630 RET
631
632// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
633// Must obey the gcc calling convention.
634TEXT _cgo_topofstack(SB),NOSPLIT,$16
635 // g (R22) and REGTMP (R30) might be clobbered by load_g. They
636 // are callee-save in the gcc calling convention, so save them.
637 MOVV R30, savedREGTMP-16(SP)
638 MOVV g, savedG-8(SP)
639
640 JAL runtime·load_g(SB)
641 MOVV g_m(g), R19
642 MOVV m_curg(R19), R19
643 MOVV (g_stack+stack_hi)(R19), R4 // return value in R4
644
645 MOVV savedG-8(SP), g
646 MOVV savedREGTMP-16(SP), R30
647 RET
648
649// The top-most function running on a goroutine
650// returns to goexit+PCQuantum.
651TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
652 NOOP
653 JAL runtime·goexit1(SB) // does not return
654 // traceback from goexit1 must hit code range of goexit
655 NOOP
656
657// This is called from .init_array and follows the platform, not Go, ABI.
658TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
659 ADDV $-0x10, R3
660 MOVV R30, 8(R3) // The access to global variables below implicitly uses R30, which is callee-save
661 MOVV runtime·lastmoduledatap(SB), R12
662 MOVV R4, moduledata_next(R12)
663 MOVV R4, runtime·lastmoduledatap(SB)
664 MOVV 8(R3), R30
665 ADDV $0x10, R3
666 RET
667
668TEXT ·checkASM(SB),NOSPLIT,$0-1
669 MOVW $1, R19
670 MOVB R19, ret+0(FP)
671 RET
672
673#ifdef GOEXPERIMENT_regabiargs
674// spillArgs stores return values from registers to a *internal/abi.RegArgs in R25.
675TEXT ·spillArgs(SB),NOSPLIT,$0-0
676 MOVV R4, (0*8)(R25)
677 MOVV R5, (1*8)(R25)
678 MOVV R6, (2*8)(R25)
679 MOVV R7, (3*8)(R25)
680 MOVV R8, (4*8)(R25)
681 MOVV R9, (5*8)(R25)
682 MOVV R10, (6*8)(R25)
683 MOVV R11, (7*8)(R25)
684 MOVV R12, (8*8)(R25)
685 MOVV R13, (9*8)(R25)
686 MOVV R14, (10*8)(R25)
687 MOVV R15, (11*8)(R25)
688 MOVV R16, (12*8)(R25)
689 MOVV R17, (13*8)(R25)
690 MOVV R18, (14*8)(R25)
691 MOVV R19, (15*8)(R25)
692 MOVD F0, (16*8)(R25)
693 MOVD F1, (17*8)(R25)
694 MOVD F2, (18*8)(R25)
695 MOVD F3, (19*8)(R25)
696 MOVD F4, (20*8)(R25)
697 MOVD F5, (21*8)(R25)
698 MOVD F6, (22*8)(R25)
699 MOVD F7, (23*8)(R25)
700 MOVD F8, (24*8)(R25)
701 MOVD F9, (25*8)(R25)
702 MOVD F10, (26*8)(R25)
703 MOVD F11, (27*8)(R25)
704 MOVD F12, (28*8)(R25)
705 MOVD F13, (29*8)(R25)
706 MOVD F14, (30*8)(R25)
707 MOVD F15, (31*8)(R25)
708 RET
709
710// unspillArgs loads args into registers from a *internal/abi.RegArgs in R25.
711TEXT ·unspillArgs(SB),NOSPLIT,$0-0
712 MOVV (0*8)(R25), R4
713 MOVV (1*8)(R25), R5
714 MOVV (2*8)(R25), R6
715 MOVV (3*8)(R25), R7
716 MOVV (4*8)(R25), R8
717 MOVV (5*8)(R25), R9
718 MOVV (6*8)(R25), R10
719 MOVV (7*8)(R25), R11
720 MOVV (8*8)(R25), R12
721 MOVV (9*8)(R25), R13
722 MOVV (10*8)(R25), R14
723 MOVV (11*8)(R25), R15
724 MOVV (12*8)(R25), R16
725 MOVV (13*8)(R25), R17
726 MOVV (14*8)(R25), R18
727 MOVV (15*8)(R25), R19
728 MOVD (16*8)(R25), F0
729 MOVD (17*8)(R25), F1
730 MOVD (18*8)(R25), F2
731 MOVD (19*8)(R25), F3
732 MOVD (20*8)(R25), F4
733 MOVD (21*8)(R25), F5
734 MOVD (22*8)(R25), F6
735 MOVD (23*8)(R25), F7
736 MOVD (24*8)(R25), F8
737 MOVD (25*8)(R25), F9
738 MOVD (26*8)(R25), F10
739 MOVD (27*8)(R25), F11
740 MOVD (28*8)(R25), F12
741 MOVD (29*8)(R25), F13
742 MOVD (30*8)(R25), F14
743 MOVD (31*8)(R25), F15
744 RET
745#else
746TEXT ·spillArgs(SB),NOSPLIT,$0-0
747 RET
748
749TEXT ·unspillArgs(SB),NOSPLIT,$0-0
750 RET
751#endif
752
753// gcWriteBarrier informs the GC about heap pointer writes.
754//
755// gcWriteBarrier does NOT follow the Go ABI. It accepts the
756// number of bytes of buffer needed in R29, and returns a pointer
757// to the buffer space in R29.
758// It clobbers R30 (the linker temp register).
759// The act of CALLing gcWriteBarrier will clobber R1 (LR).
760// It does not clobber any other general-purpose registers,
761// but may clobber others (e.g., floating point registers).
762TEXT gcWriteBarrier<>(SB),NOSPLIT,$216
763 // Save the registers clobbered by the fast path.
764 MOVV R19, 208(R3)
765 MOVV R13, 216(R3)
766retry:
767 MOVV g_m(g), R19
768 MOVV m_p(R19), R19
769 MOVV (p_wbBuf+wbBuf_next)(R19), R13
770 MOVV (p_wbBuf+wbBuf_end)(R19), R30 // R30 is linker temp register
771 // Increment wbBuf.next position.
772 ADDV R29, R13
773 // Is the buffer full?
774 BLTU R30, R13, flush
775 // Commit to the larger buffer.
776 MOVV R13, (p_wbBuf+wbBuf_next)(R19)
777 // Make return value (the original next position)
778 SUBV R29, R13, R29
779 // Restore registers.
780 MOVV 208(R3), R19
781 MOVV 216(R3), R13
782 RET
783
784flush:
785 // Save all general purpose registers since these could be
786 // clobbered by wbBufFlush and were not saved by the caller.
787 MOVV R27, 8(R3)
788 MOVV R28, 16(R3)
789 // R1 is LR, which was saved by the prologue.
790 MOVV R2, 24(R3)
791 // R3 is SP.
792 MOVV R4, 32(R3)
793 MOVV R5, 40(R3)
794 MOVV R6, 48(R3)
795 MOVV R7, 56(R3)
796 MOVV R8, 64(R3)
797 MOVV R9, 72(R3)
798 MOVV R10, 80(R3)
799 MOVV R11, 88(R3)
800 MOVV R12, 96(R3)
801 // R13 already saved
802 MOVV R14, 104(R3)
803 MOVV R15, 112(R3)
804 MOVV R16, 120(R3)
805 MOVV R17, 128(R3)
806 MOVV R18, 136(R3)
807 // R19 already saved
808 MOVV R20, 144(R3)
809 MOVV R21, 152(R3)
810 // R22 is g.
811 MOVV R23, 160(R3)
812 MOVV R24, 168(R3)
813 MOVV R25, 176(R3)
814 MOVV R26, 184(R3)
815 // R27 already saved
816 // R28 already saved.
817 MOVV R29, 192(R3)
818 // R30 is tmp register.
819 MOVV R31, 200(R3)
820
821 CALL runtime·wbBufFlush(SB)
822
823 MOVV 8(R3), R27
824 MOVV 16(R3), R28
825 MOVV 24(R3), R2
826 MOVV 32(R3), R4
827 MOVV 40(R3), R5
828 MOVV 48(R3), R6
829 MOVV 56(R3), R7
830 MOVV 64(R3), R8
831 MOVV 72(R3), R9
832 MOVV 80(R3), R10
833 MOVV 88(R3), R11
834 MOVV 96(R3), R12
835 MOVV 104(R3), R14
836 MOVV 112(R3), R15
837 MOVV 120(R3), R16
838 MOVV 128(R3), R17
839 MOVV 136(R3), R18
840 MOVV 144(R3), R20
841 MOVV 152(R3), R21
842 MOVV 160(R3), R23
843 MOVV 168(R3), R24
844 MOVV 176(R3), R25
845 MOVV 184(R3), R26
846 MOVV 192(R3), R29
847 MOVV 200(R3), R31
848 JMP retry
849
850TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
851 MOVV $8, R29
852 JMP gcWriteBarrier<>(SB)
853TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
854 MOVV $16, R29
855 JMP gcWriteBarrier<>(SB)
856TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
857 MOVV $24, R29
858 JMP gcWriteBarrier<>(SB)
859TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
860 MOVV $32, R29
861 JMP gcWriteBarrier<>(SB)
862TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
863 MOVV $40, R29
864 JMP gcWriteBarrier<>(SB)
865TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
866 MOVV $48, R29
867 JMP gcWriteBarrier<>(SB)
868TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
869 MOVV $56, R29
870 JMP gcWriteBarrier<>(SB)
871TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
872 MOVV $64, R29
873 JMP gcWriteBarrier<>(SB)
874
875// Note: these functions use a special calling convention to save generated code space.
876// Arguments are passed in registers, but the space for those arguments are allocated
877// in the caller's stack frame. These stubs write the args into that stack space and
878// then tail call to the corresponding runtime handler.
879// The tail call makes these stubs disappear in backtraces.
880TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
881#ifdef GOEXPERIMENT_regabiargs
882 MOVV R20, R4
883 MOVV R21, R5
884#else
885 MOVV R20, x+0(FP)
886 MOVV R21, y+8(FP)
887#endif
888 JMP runtime·goPanicIndex<ABIInternal>(SB)
889TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
890#ifdef GOEXPERIMENT_regabiargs
891 MOVV R20, R4
892 MOVV R21, R5
893#else
894 MOVV R20, x+0(FP)
895 MOVV R21, y+8(FP)
896#endif
897 JMP runtime·goPanicIndexU<ABIInternal>(SB)
898TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
899#ifdef GOEXPERIMENT_regabiargs
900 MOVV R21, R4
901 MOVV R23, R5
902#else
903 MOVV R21, x+0(FP)
904 MOVV R23, y+8(FP)
905#endif
906 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
907TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
908#ifdef GOEXPERIMENT_regabiargs
909 MOVV R21, R4
910 MOVV R23, R5
911#else
912 MOVV R21, x+0(FP)
913 MOVV R23, y+8(FP)
914#endif
915 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
916TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
917#ifdef GOEXPERIMENT_regabiargs
918 MOVV R21, R4
919 MOVV R23, R5
920#else
921 MOVV R21, x+0(FP)
922 MOVV R23, y+8(FP)
923#endif
924 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
925TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
926#ifdef GOEXPERIMENT_regabiargs
927 MOVV R21, R4
928 MOVV R23, R5
929#else
930 MOVV R21, x+0(FP)
931 MOVV R23, y+8(FP)
932#endif
933 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
934TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
935#ifdef GOEXPERIMENT_regabiargs
936 MOVV R20, R4
937 MOVV R21, R5
938#else
939 MOVV R20, x+0(FP)
940 MOVV R21, y+8(FP)
941#endif
942 JMP runtime·goPanicSliceB<ABIInternal>(SB)
943TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
944#ifdef GOEXPERIMENT_regabiargs
945 MOVV R20, R4
946 MOVV R21, R5
947#else
948 MOVV R20, x+0(FP)
949 MOVV R21, y+8(FP)
950#endif
951 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
952TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
953#ifdef GOEXPERIMENT_regabiargs
954 MOVV R23, R4
955 MOVV R24, R5
956#else
957 MOVV R23, x+0(FP)
958 MOVV R24, y+8(FP)
959#endif
960 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
961TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
962#ifdef GOEXPERIMENT_regabiargs
963 MOVV R23, R4
964 MOVV R24, R5
965#else
966 MOVV R23, x+0(FP)
967 MOVV R24, y+8(FP)
968#endif
969 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
970TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
971#ifdef GOEXPERIMENT_regabiargs
972 MOVV R23, R4
973 MOVV R24, R5
974#else
975 MOVV R23, x+0(FP)
976 MOVV R24, y+8(FP)
977#endif
978 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
979TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
980#ifdef GOEXPERIMENT_regabiargs
981 MOVV R23, R4
982 MOVV R24, R5
983#else
984 MOVV R23, x+0(FP)
985 MOVV R24, y+8(FP)
986#endif
987 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
988TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
989#ifdef GOEXPERIMENT_regabiargs
990 MOVV R21, R4
991 MOVV R23, R5
992#else
993 MOVV R21, x+0(FP)
994 MOVV R23, y+8(FP)
995#endif
996 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
997TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
998#ifdef GOEXPERIMENT_regabiargs
999 MOVV R21, R4
1000 MOVV R23, R5
1001#else
1002 MOVV R21, x+0(FP)
1003 MOVV R23, y+8(FP)
1004#endif
1005 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1006TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1007#ifdef GOEXPERIMENT_regabiargs
1008 MOVV R20, R4
1009 MOVV R21, R5
1010#else
1011 MOVV R20, x+0(FP)
1012 MOVV R21, y+8(FP)
1013#endif
1014 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1015TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1016#ifdef GOEXPERIMENT_regabiargs
1017 MOVV R20, R4
1018 MOVV R21, R5
1019#else
1020 MOVV R20, x+0(FP)
1021 MOVV R21, y+8(FP)
1022#endif
1023 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1024TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1025#ifdef GOEXPERIMENT_regabiargs
1026 MOVV R23, R4
1027 MOVV R24, R5
1028#else
1029 MOVV R23, x+0(FP)
1030 MOVV R24, y+8(FP)
1031#endif
1032 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
View as plain text