...
▽

Text file src/runtime/asm_arm64.s

Documentation: runtime

     1// Copyright 2015 The Go Authors. All rights reserved.
     2// Use of this source code is governed by a BSD-style
     3// license that can be found in the LICENSE file.
     4
     5#include "go_asm.h"
     6#include "go_tls.h"
     7#include "tls_arm64.h"
     8#include "funcdata.h"
     9#include "textflag.h"
    10#include "cgo/abi_arm64.h"
    11
    12// _rt0_arm64 is common startup code for most arm64 systems when using
    13// internal linking. This is the entry point for the program from the
    14// kernel for an ordinary -buildmode=exe program. The stack holds the
    15// number of arguments and the C-style argv.
    16TEXT _rt0_arm64(SB),NOSPLIT,$0
    17	MOVD	0(RSP), R0	// argc
    18	ADD	$8, RSP, R1	// argv
    19	JMP	runtime·rt0_go(SB)
    20
    21// main is common startup code for most amd64 systems when using
    22// external linking. The C startup code will call the symbol "main"
    23// passing argc and argv in the usual C ABI registers R0 and R1.
    24TEXT main(SB),NOSPLIT,$0
    25	JMP	runtime·rt0_go(SB)
    26
    27// _rt0_arm64_lib is common startup code for most arm64 systems when
    28// using -buildmode=c-archive or -buildmode=c-shared. The linker will
    29// arrange to invoke this function as a global constructor (for
    30// c-archive) or when the shared library is loaded (for c-shared).
    31// We expect argc and argv to be passed in the usual C ABI registers
    32// R0 and R1.
    33TEXT _rt0_arm64_lib(SB),NOSPLIT,$184
    34	// Preserve callee-save registers.
    35	SAVE_R19_TO_R28(24)
    36	SAVE_F8_TO_F15(104)
    37
    38	// Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go
    39	MOVD	ZR, g
    40
    41	MOVD	R0, _rt0_arm64_lib_argc<>(SB)
    42	MOVD	R1, _rt0_arm64_lib_argv<>(SB)
    43
    44	MOVD	$runtime·libInit(SB), R4
    45	BL	(R4)
    46
    47	// Restore callee-save registers.
    48	RESTORE_R19_TO_R28(24)
    49	RESTORE_F8_TO_F15(104)
    50	RET
    51
    52TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
    53	MOVD	_rt0_arm64_lib_argc<>(SB), R0
    54	MOVD	_rt0_arm64_lib_argv<>(SB), R1
    55	MOVD	$runtime·rt0_go(SB),R4
    56	B	(R4)
    57
    58DATA _rt0_arm64_lib_argc<>(SB)/8, $0
    59GLOBL _rt0_arm64_lib_argc<>(SB),NOPTR, $8
    60DATA _rt0_arm64_lib_argv<>(SB)/8, $0
    61GLOBL _rt0_arm64_lib_argv<>(SB),NOPTR, $8
    62
    63#ifdef GOARM64_LSE
    64DATA no_lse_msg<>+0x00(SB)/64, $"This program can only run on ARM64 processors with LSE support.\n"
    65GLOBL no_lse_msg<>(SB), RODATA, $64
    66#endif
    67
    68// We know for sure that Linux and FreeBSD allow to read instruction set
    69// attribute registers (while some others OSes, like OpenBSD and Darwin,
    70// are not). Let's be conservative and allow code reading such registers
    71// only when we sure this won't lead to sigill.
    72#ifdef GOOS_linux
    73#define ISA_REGS_READABLE
    74#endif
    75#ifdef GOOS_freebsd
    76#define ISA_REGS_READABLE
    77#endif
    78
    79#ifdef GOARM64_LSE
    80#ifdef ISA_REGS_READABLE
    81#define CHECK_GOARM64_LSE
    82#endif
    83#endif
    84
    85TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
    86	// SP = stack; R0 = argc; R1 = argv
    87
    88	SUB	$32, RSP
    89	MOVW	R0, 8(RSP) // argc
    90	MOVD	R1, 16(RSP) // argv
    91
    92	// This is typically the entry point for Go programs.
    93	// Call stack unwinding must not proceed past this frame.
    94	// Set the frame pointer register to 0 so that frame pointer-based unwinders
    95	// (which don't use debug info for performance reasons)
    96	// won't attempt to unwind past this function.
    97	// See go.dev/issue/63630
    98	MOVD	$0, R29
    99
   100#ifdef TLS_darwin
   101	// Initialize TLS.
   102	MOVD	ZR, g // clear g, make sure it's not junk.
   103	SUB	$32, RSP
   104	MRS_TPIDR_R0
   105	AND	$~7, R0
   106	MOVD	R0, 16(RSP)             // arg2: TLS base
   107	MOVD	$runtime·tls_g(SB), R2
   108	MOVD	R2, 8(RSP)              // arg1: &tlsg
   109	BL	·tlsinit(SB)
   110	ADD	$32, RSP
   111#endif
   112
   113	// create istack out of the given (operating system) stack.
   114	// _cgo_init may update stackguard.
   115	MOVD	$runtime·g0(SB), g
   116	MOVD	RSP, R7
   117	MOVD	$(-64*1024)(R7), R0
   118	MOVD	R0, g_stackguard0(g)
   119	MOVD	R0, g_stackguard1(g)
   120	MOVD	R0, (g_stack+stack_lo)(g)
   121	MOVD	R7, (g_stack+stack_hi)(g)
   122
   123	// if there is a _cgo_init, call it using the gcc ABI.
   124	MOVD	_cgo_init(SB), R12
   125	CBZ	R12, nocgo
   126
   127#ifdef GOOS_android
   128	MRS_TPIDR_R0			// load TLS base pointer
   129	MOVD	R0, R3			// arg 3: TLS base pointer
   130	MOVD	$runtime·tls_g(SB), R2 	// arg 2: &tls_g
   131#else
   132	MOVD	$0, R2		        // arg 2: not used when using platform's TLS
   133#endif
   134	MOVD	$setg_gcc<>(SB), R1	// arg 1: setg
   135	MOVD	g, R0			// arg 0: G
   136	SUB	$16, RSP		// reserve 16 bytes for sp-8 where fp may be saved.
   137	BL	(R12)
   138	ADD	$16, RSP
   139
   140nocgo:
   141	BL	runtime·save_g(SB)
   142	// update stackguard after _cgo_init
   143	MOVD	(g_stack+stack_lo)(g), R0
   144	ADD	$const_stackGuard, R0
   145	MOVD	R0, g_stackguard0(g)
   146	MOVD	R0, g_stackguard1(g)
   147
   148	// set the per-goroutine and per-mach "registers"
   149	MOVD	$runtime·m0(SB), R0
   150
   151	// save m->g0 = g0
   152	MOVD	g, m_g0(R0)
   153	// save m0 to g0->m
   154	MOVD	R0, g_m(g)
   155
   156	BL	runtime·check(SB)
   157
   158#ifdef GOOS_windows
   159	BL	runtime·wintls(SB)
   160#endif
   161
   162	// Check that CPU we use for execution supports instructions targeted during compile-time.
   163#ifdef CHECK_GOARM64_LSE
   164	// Read the ID_AA64ISAR0_EL1 register
   165	MRS	ID_AA64ISAR0_EL1, R0
   166
   167	// Extract the LSE field (bits [23:20])
   168	LSR	$20, R0, R0
   169	AND	$0xf, R0, R0
   170
   171	// LSE support is indicated by a non-zero value
   172	CBZ	R0, no_lse
   173#endif
   174
   175	MOVW	8(RSP), R0	// copy argc
   176	MOVW	R0, -8(RSP)
   177	MOVD	16(RSP), R0		// copy argv
   178	MOVD	R0, 0(RSP)
   179	BL	runtime·args(SB)
   180	BL	runtime·osinit(SB)
   181	BL	runtime·schedinit(SB)
   182
   183	// create a new goroutine to start program
   184	MOVD	$runtime·mainPC(SB), R0		// entry
   185	SUB	$16, RSP
   186	MOVD	R0, 8(RSP) // arg
   187	MOVD	$0, 0(RSP) // dummy LR
   188	BL	runtime·newproc(SB)
   189	ADD	$16, RSP
   190
   191	// start this M
   192	BL	runtime·mstart(SB)
   193	UNDEF
   194
   195#ifdef CHECK_GOARM64_LSE
   196no_lse:
   197	MOVD	$1, R0 // stderr
   198	MOVD	R0, 8(RSP)
   199	MOVD	$no_lse_msg<>(SB), R1 // message address
   200	MOVD	R1, 16(RSP)
   201	MOVD	$64, R2 // message length
   202	MOVD	R2, 24(RSP)
   203	CALL	runtime·write(SB)
   204	CALL	runtime·exit(SB)
   205	CALL	runtime·abort(SB)
   206	RET
   207#endif
   208
   209	// Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
   210	// intended to be called by debuggers.
   211	MOVD	$runtime·debugPinnerV1<ABIInternal>(SB), R0
   212	MOVD	$runtime·debugCallV2<ABIInternal>(SB), R0
   213
   214	MOVD	$0, R0
   215	MOVD	R0, (R0)	// boom
   216	UNDEF
   217
   218DATA	runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
   219GLOBL	runtime·mainPC(SB),RODATA,$8
   220
   221// Windows ARM64 needs an immediate 0xf000 argument.
   222// See go.dev/issues/53837.
   223#define BREAK	\
   224#ifdef GOOS_windows	\
   225	BRK	$0xf000 	\
   226#else 				\
   227	BRK 			\
   228#endif 				\
   229
   230
   231TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
   232	BREAK
   233	RET
   234
   235TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
   236	RET
   237
   238TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
   239	// This is the root frame of new Go-created OS threads.
   240	// Call stack unwinding must not proceed past this frame.
   241	// Set the frame pointer register to 0 so that frame pointer-based unwinders
   242	// (which don't use debug info for performance reasons)
   243	// won't attempt to unwind past this function.
   244	// See go.dev/issue/63630
   245	MOVD	$0, R29
   246	BL	runtime·mstart0(SB)
   247	RET // not reached
   248
   249/*
   250 *  go-routine
   251 */
   252
   253// void gogo(Gobuf*)
   254// restore state from Gobuf; longjmp
   255TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
   256	MOVD	buf+0(FP), R5
   257	MOVD	gobuf_g(R5), R6
   258	MOVD	0(R6), R4	// make sure g != nil
   259	B	gogo<>(SB)
   260
   261TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
   262	MOVD	R6, g
   263	BL	runtime·save_g(SB)
   264
   265	MOVD	gobuf_sp(R5), R0
   266	MOVD	R0, RSP
   267	MOVD	gobuf_bp(R5), R29
   268	MOVD	gobuf_lr(R5), LR
   269	MOVD	gobuf_ctxt(R5), R26
   270	MOVD	$0, gobuf_sp(R5)
   271	MOVD	$0, gobuf_bp(R5)
   272	MOVD	$0, gobuf_lr(R5)
   273	MOVD	$0, gobuf_ctxt(R5)
   274	CMP	ZR, ZR // set condition codes for == test, needed by stack split
   275	MOVD	gobuf_pc(R5), R6
   276	B	(R6)
   277
   278// void mcall(fn func(*g))
   279// Switch to m->g0's stack, call fn(g).
   280// Fn must never return. It should gogo(&g->sched)
   281// to keep running g.
   282TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
   283#ifdef GOEXPERIMENT_runtimesecret
   284	MOVW	g_secret(g), R26
   285	CBZ 	R26, nosecret
   286	// Use R26 as a secondary link register
   287	// We purposefully don't erase it in secretEraseRegistersMcall
   288	MOVD	LR, R26
   289	BL 	runtime·secretEraseRegistersMcall(SB)
   290	MOVD	R26, LR
   291
   292nosecret:
   293#endif
   294	MOVD	R0, R26				// context
   295
   296	// Save caller state in g->sched
   297	MOVD	RSP, R0
   298	MOVD	R0, (g_sched+gobuf_sp)(g)
   299	MOVD	R29, (g_sched+gobuf_bp)(g)
   300	MOVD	LR, (g_sched+gobuf_pc)(g)
   301	MOVD	$0, (g_sched+gobuf_lr)(g)
   302
   303	// Switch to m->g0 & its stack, call fn.
   304	MOVD	g, R3
   305	MOVD	g_m(g), R8
   306	MOVD	m_g0(R8), g
   307	BL	runtime·save_g(SB)
   308	CMP	g, R3
   309	BNE	2(PC)
   310	B	runtime·badmcall(SB)
   311
   312	MOVD	(g_sched+gobuf_sp)(g), R0
   313	MOVD	R0, RSP	// sp = m->g0->sched.sp
   314	MOVD	$0, R29				// clear frame pointer, as caller may execute on another M
   315	MOVD	R3, R0				// arg = g
   316	MOVD	$0, -16(RSP)			// dummy LR
   317	SUB	$16, RSP
   318	MOVD	0(R26), R4			// code pointer
   319	BL	(R4)
   320	B	runtime·badmcall2(SB)
   321
   322// systemstack_switch is a dummy routine that systemstack leaves at the bottom
   323// of the G stack. We need to distinguish the routine that
   324// lives at the bottom of the G stack from the one that lives
   325// at the top of the system stack because the one at the top of
   326// the system stack terminates the stack walk (see topofstack()).
   327TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
   328	UNDEF
   329	BL	(LR)	// make sure this function is not leaf
   330	RET
   331
   332// func systemstack(fn func())
   333TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   334#ifdef GOEXPERIMENT_runtimesecret
   335	MOVW	g_secret(g), R3
   336	CBZ		R3, nosecret
   337	BL 		·secretEraseRegisters(SB)
   338
   339nosecret:
   340#endif
   341	MOVD	fn+0(FP), R3	// R3 = fn
   342	MOVD	R3, R26		// context
   343	MOVD	g_m(g), R4	// R4 = m
   344
   345	MOVD	m_gsignal(R4), R5	// R5 = gsignal
   346	CMP	g, R5
   347	BEQ	noswitch
   348
   349	MOVD	m_g0(R4), R5	// R5 = g0
   350	CMP	g, R5
   351	BEQ	noswitch
   352
   353	MOVD	m_curg(R4), R6
   354	CMP	g, R6
   355	BEQ	switch
   356
   357	// Bad: g is not gsignal, not g0, not curg. What is it?
   358	// Hide call from linker nosplit analysis.
   359	MOVD	$runtime·badsystemstack(SB), R3
   360	BL	(R3)
   361	B	runtime·abort(SB)
   362
   363switch:
   364	// Switch stacks.
   365	// The original frame pointer is stored in R29,
   366	// which is useful for stack unwinding.
   367	// Save our state in g->sched. Pretend to
   368	// be systemstack_switch if the G stack is scanned.
   369	BL	gosave_systemstack_switch<>(SB)
   370
   371	// switch to g0
   372	MOVD	R5, g
   373	BL	runtime·save_g(SB)
   374	MOVD	(g_sched+gobuf_sp)(g), R3
   375	MOVD	R3, RSP
   376
   377	// call target function
   378	MOVD	0(R26), R3	// code pointer
   379	BL	(R3)
   380
   381	// switch back to g
   382	MOVD	g_m(g), R3
   383	MOVD	m_curg(R3), g
   384	BL	runtime·save_g(SB)
   385	MOVD	(g_sched+gobuf_sp)(g), R0
   386	MOVD	R0, RSP
   387	MOVD	(g_sched+gobuf_bp)(g), R29
   388	MOVD	$0, (g_sched+gobuf_sp)(g)
   389	MOVD	$0, (g_sched+gobuf_bp)(g)
   390	RET
   391
   392noswitch:
   393	// already on m stack, just call directly
   394	// Using a tail call here cleans up tracebacks since we won't stop
   395	// at an intermediate systemstack.
   396	MOVD	0(R26), R3	// code pointer
   397	MOVD.P	16(RSP), R30	// restore LR
   398	SUB	$8, RSP, R29	// restore FP
   399	B	(R3)
   400
   401// func switchToCrashStack0(fn func())
   402TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
   403	MOVD	R0, R26    // context register
   404	MOVD	g_m(g), R1 // curm
   405
   406	// set g to gcrash
   407	MOVD	$runtime·gcrash(SB), g // g = &gcrash
   408	BL	runtime·save_g(SB)         // clobbers R0
   409	MOVD	R1, g_m(g)             // g.m = curm
   410	MOVD	g, m_g0(R1)            // curm.g0 = g
   411
   412	// switch to crashstack
   413	MOVD	(g_stack+stack_hi)(g), R1
   414	SUB	$(4*8), R1
   415	MOVD	R1, RSP
   416
   417	// call target function
   418	MOVD	0(R26), R0
   419	CALL	(R0)
   420
   421	// should never return
   422	CALL	runtime·abort(SB)
   423	UNDEF
   424
   425/*
   426 * support for morestack
   427 */
   428
   429// Called during function prolog when more stack is needed.
   430// Caller has already loaded:
   431// R3 prolog's LR (R30)
   432//
   433// The traceback routines see morestack on a g0 as being
   434// the top of a stack (for example, morestack calling newstack
   435// calling the scheduler calling newm calling gc), so we must
   436// record an argument size. For that purpose, it has no arguments.
   437TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
   438	// Cannot grow scheduler stack (m->g0).
   439	MOVD	g_m(g), R8
   440	MOVD	m_g0(R8), R4
   441
   442	// Called from f.
   443	// Set g->sched to context in f
   444	MOVD	RSP, R0
   445	MOVD	R0, (g_sched+gobuf_sp)(g)
   446	MOVD	R29, (g_sched+gobuf_bp)(g)
   447	MOVD	LR, (g_sched+gobuf_pc)(g)
   448	MOVD	R3, (g_sched+gobuf_lr)(g)
   449	MOVD	R26, (g_sched+gobuf_ctxt)(g)
   450
   451	CMP	g, R4
   452	BNE	3(PC)
   453	BL	runtime·badmorestackg0(SB)
   454	B	runtime·abort(SB)
   455
   456	// Cannot grow signal stack (m->gsignal).
   457	MOVD	m_gsignal(R8), R4
   458	CMP	g, R4
   459	BNE	3(PC)
   460	BL	runtime·badmorestackgsignal(SB)
   461	B	runtime·abort(SB)
   462
   463	// Called from f.
   464	// Set m->morebuf to f's callers.
   465	MOVD	R3, (m_morebuf+gobuf_pc)(R8)	// f's caller's PC
   466	MOVD	RSP, R0
   467	MOVD	R0, (m_morebuf+gobuf_sp)(R8)	// f's caller's RSP
   468	MOVD	g, (m_morebuf+gobuf_g)(R8)
   469
   470	// If in secret mode, erase registers on transition
   471	// from G stack to M stack,
   472#ifdef GOEXPERIMENT_runtimesecret
   473	MOVW	g_secret(g), R4
   474	CBZ 	R4, nosecret
   475	BL	·secretEraseRegisters(SB)
   476	MOVD	g_m(g), R8
   477nosecret:
   478#endif
   479
   480	// Call newstack on m->g0's stack.
   481	MOVD	m_g0(R8), g
   482	BL	runtime·save_g(SB)
   483	MOVD	(g_sched+gobuf_sp)(g), R0
   484	MOVD	R0, RSP
   485	MOVD	$0, R29		// clear frame pointer, as caller may execute on another M
   486	MOVD.W	$0, -16(RSP)	// create a call frame on g0 (saved LR; keep 16-aligned)
   487	BL	runtime·newstack(SB)
   488
   489	// Not reached, but make sure the return PC from the call to newstack
   490	// is still in this function, and not the beginning of the next.
   491	UNDEF
   492
   493TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
   494	// Force SPWRITE. This function doesn't actually write SP,
   495	// but it is called with a special calling convention where
   496	// the caller doesn't save LR on stack but passes it as a
   497	// register (R3), and the unwinder currently doesn't understand.
   498	// Make it SPWRITE to stop unwinding. (See issue 54332)
   499	MOVD	RSP, RSP
   500
   501	MOVW	$0, R26
   502	B runtime·morestack(SB)
   503
   504// spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
   505TEXT ·spillArgs(SB),NOSPLIT,$0-0
   506	STP	(R0, R1), (0*8)(R20)
   507	STP	(R2, R3), (2*8)(R20)
   508	STP	(R4, R5), (4*8)(R20)
   509	STP	(R6, R7), (6*8)(R20)
   510	STP	(R8, R9), (8*8)(R20)
   511	STP	(R10, R11), (10*8)(R20)
   512	STP	(R12, R13), (12*8)(R20)
   513	STP	(R14, R15), (14*8)(R20)
   514	FSTPD	(F0, F1), (16*8)(R20)
   515	FSTPD	(F2, F3), (18*8)(R20)
   516	FSTPD	(F4, F5), (20*8)(R20)
   517	FSTPD	(F6, F7), (22*8)(R20)
   518	FSTPD	(F8, F9), (24*8)(R20)
   519	FSTPD	(F10, F11), (26*8)(R20)
   520	FSTPD	(F12, F13), (28*8)(R20)
   521	FSTPD	(F14, F15), (30*8)(R20)
   522	RET
   523
   524// unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
   525TEXT ·unspillArgs(SB),NOSPLIT,$0-0
   526	LDP	(0*8)(R20), (R0, R1)
   527	LDP	(2*8)(R20), (R2, R3)
   528	LDP	(4*8)(R20), (R4, R5)
   529	LDP	(6*8)(R20), (R6, R7)
   530	LDP	(8*8)(R20), (R8, R9)
   531	LDP	(10*8)(R20), (R10, R11)
   532	LDP	(12*8)(R20), (R12, R13)
   533	LDP	(14*8)(R20), (R14, R15)
   534	FLDPD	(16*8)(R20), (F0, F1)
   535	FLDPD	(18*8)(R20), (F2, F3)
   536	FLDPD	(20*8)(R20), (F4, F5)
   537	FLDPD	(22*8)(R20), (F6, F7)
   538	FLDPD	(24*8)(R20), (F8, F9)
   539	FLDPD	(26*8)(R20), (F10, F11)
   540	FLDPD	(28*8)(R20), (F12, F13)
   541	FLDPD	(30*8)(R20), (F14, F15)
   542	RET
   543
   544// reflectcall: call a function with the given argument list
   545// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   546// we don't have variable-sized frames, so we use a small number
   547// of constant-sized-frame functions to encode a few bits of size in the pc.
   548// Caution: ugly multiline assembly macros in your future!
   549
   550#define DISPATCH(NAME,MAXSIZE)		\
   551	MOVD	$MAXSIZE, R27;		\
   552	CMP	R27, R16;		\
   553	BGT	3(PC);			\
   554	MOVD	$NAME(SB), R27;	\
   555	B	(R27)
   556// Note: can't just "B NAME(SB)" - bad inlining results.
   557
   558TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
   559	MOVWU	frameSize+32(FP), R16
   560	DISPATCH(runtime·call16, 16)
   561	DISPATCH(runtime·call32, 32)
   562	DISPATCH(runtime·call64, 64)
   563	DISPATCH(runtime·call128, 128)
   564	DISPATCH(runtime·call256, 256)
   565	DISPATCH(runtime·call512, 512)
   566	DISPATCH(runtime·call1024, 1024)
   567	DISPATCH(runtime·call2048, 2048)
   568	DISPATCH(runtime·call4096, 4096)
   569	DISPATCH(runtime·call8192, 8192)
   570	DISPATCH(runtime·call16384, 16384)
   571	DISPATCH(runtime·call32768, 32768)
   572	DISPATCH(runtime·call65536, 65536)
   573	DISPATCH(runtime·call131072, 131072)
   574	DISPATCH(runtime·call262144, 262144)
   575	DISPATCH(runtime·call524288, 524288)
   576	DISPATCH(runtime·call1048576, 1048576)
   577	DISPATCH(runtime·call2097152, 2097152)
   578	DISPATCH(runtime·call4194304, 4194304)
   579	DISPATCH(runtime·call8388608, 8388608)
   580	DISPATCH(runtime·call16777216, 16777216)
   581	DISPATCH(runtime·call33554432, 33554432)
   582	DISPATCH(runtime·call67108864, 67108864)
   583	DISPATCH(runtime·call134217728, 134217728)
   584	DISPATCH(runtime·call268435456, 268435456)
   585	DISPATCH(runtime·call536870912, 536870912)
   586	DISPATCH(runtime·call1073741824, 1073741824)
   587	MOVD	$runtime·badreflectcall(SB), R0
   588	B	(R0)
   589
   590#define CALLFN(NAME,MAXSIZE)			\
   591TEXT NAME(SB), WRAPPER, $MAXSIZE-48;		\
   592	NO_LOCAL_POINTERS;			\
   593	/* copy arguments to stack */		\
   594	MOVD	stackArgs+16(FP), R3;			\
   595	MOVWU	stackArgsSize+24(FP), R4;		\
   596	ADD	$8, RSP, R5;			\
   597	BIC	$0xf, R4, R6;			\
   598	CBZ	R6, 6(PC);			\
   599	/* if R6=(argsize&~15) != 0 */		\
   600	ADD	R6, R5, R6;			\
   601	/* copy 16 bytes a time */		\
   602	LDP.P	16(R3), (R7, R8);		\
   603	STP.P	(R7, R8), 16(R5);		\
   604	CMP	R5, R6;				\
   605	BNE	-3(PC);				\
   606	AND	$0xf, R4, R6;			\
   607	CBZ	R6, 6(PC);			\
   608	/* if R6=(argsize&15) != 0 */		\
   609	ADD	R6, R5, R6;			\
   610	/* copy 1 byte a time for the rest */	\
   611	MOVBU.P	1(R3), R7;			\
   612	MOVBU.P	R7, 1(R5);			\
   613	CMP	R5, R6;				\
   614	BNE	-3(PC);				\
   615	/* set up argument registers */		\
   616	MOVD	regArgs+40(FP), R20;		\
   617	CALL	·unspillArgs(SB);		\
   618	/* call function */			\
   619	MOVD	f+8(FP), R26;			\
   620	MOVD	(R26), R20;			\
   621	PCDATA	$PCDATA_StackMapIndex, $0;	\
   622	BL	(R20);				\
   623	/* copy return values back */		\
   624	MOVD	regArgs+40(FP), R20;		\
   625	CALL	·spillArgs(SB);		\
   626	MOVD	stackArgsType+0(FP), R7;		\
   627	MOVD	stackArgs+16(FP), R3;			\
   628	MOVWU	stackArgsSize+24(FP), R4;			\
   629	MOVWU	stackRetOffset+28(FP), R6;		\
   630	ADD	$8, RSP, R5;			\
   631	ADD	R6, R5; 			\
   632	ADD	R6, R3;				\
   633	SUB	R6, R4;				\
   634	BL	callRet<>(SB);			\
   635	RET
   636
   637// callRet copies return values back at the end of call*. This is a
   638// separate function so it can allocate stack space for the arguments
   639// to reflectcallmove. It does not follow the Go ABI; it expects its
   640// arguments in registers.
   641TEXT callRet<>(SB), NOSPLIT, $48-0
   642	NO_LOCAL_POINTERS
   643	STP	(R7, R3), 8(RSP)
   644	STP	(R5, R4), 24(RSP)
   645	MOVD	R20, 40(RSP)
   646	BL	runtime·reflectcallmove(SB)
   647	RET
   648
   649CALLFN(·call16, 16)
   650CALLFN(·call32, 32)
   651CALLFN(·call64, 64)
   652CALLFN(·call128, 128)
   653CALLFN(·call256, 256)
   654CALLFN(·call512, 512)
   655CALLFN(·call1024, 1024)
   656CALLFN(·call2048, 2048)
   657CALLFN(·call4096, 4096)
   658CALLFN(·call8192, 8192)
   659CALLFN(·call16384, 16384)
   660CALLFN(·call32768, 32768)
   661CALLFN(·call65536, 65536)
   662CALLFN(·call131072, 131072)
   663CALLFN(·call262144, 262144)
   664CALLFN(·call524288, 524288)
   665CALLFN(·call1048576, 1048576)
   666CALLFN(·call2097152, 2097152)
   667CALLFN(·call4194304, 4194304)
   668CALLFN(·call8388608, 8388608)
   669CALLFN(·call16777216, 16777216)
   670CALLFN(·call33554432, 33554432)
   671CALLFN(·call67108864, 67108864)
   672CALLFN(·call134217728, 134217728)
   673CALLFN(·call268435456, 268435456)
   674CALLFN(·call536870912, 536870912)
   675CALLFN(·call1073741824, 1073741824)
   676
   677// The Arm architecture provides a user space accessible counter-timer which
   678// is incremented at a fixed but machine-specific rate. Software can (spin)
   679// wait until the counter-timer reaches some desired value.
   680//
   681// Armv8.7-A introduced the WFET (FEAT_WFxT) instruction, which allows the
   682// processor to enter a low power state for a set time, or until an event is
   683// received.
   684//
   685// However, WFET is not used here because it is only available on newer hardware,
   686// and we aim to maintain compatibility with older Armv8-A platforms that do not
   687// support this feature.
   688//
   689// As a fallback, we can instead use the ISB instruction to decrease processor
   690// activity and thus power consumption between checks of the counter-timer.
   691// Note that we do not depend on the latency of the ISB instruction which is
   692// implementation specific. Actual delay comes from comparing against a fresh
   693// read of the counter-timer value.
   694//
   695// Read more in this Arm blog post:
   696// https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/multi-threaded-applications-arm
   697
   698TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
   699	MOVWU	cycles+0(FP), R0
   700	CBZ	 R0, done
   701	//Prevent speculation of subsequent counter/timer reads and memory accesses.
   702	ISB     $15
   703	// If the delay is very short, just return.
   704	// Hardcode 18ns as the first ISB delay.
   705	CMP     $18, R0
   706	BLS     done
   707	// Adjust for overhead of initial ISB.
   708	SUB     $18, R0, R0
   709	// Convert the delay from nanoseconds to counter/timer ticks.
   710	// Read the counter/timer frequency.
   711	// delay_ticks = (delay * CNTFRQ_EL0) / 1e9
   712	// With the below simplifications and adjustments,
   713	// we are usually within 2% of the correct value:
   714	// delay_ticks = (delay + delay / 16) * CNTFRQ_EL0 >> 30
   715	MRS     CNTFRQ_EL0, R1
   716	ADD     R0>>4, R0, R0
   717	MUL     R1, R0, R0
   718	LSR     $30, R0, R0
   719	CBZ     R0, done
   720	// start = current counter/timer value
   721	MRS     CNTVCT_EL0, R2
   722delay:
   723	// Delay using ISB for all ticks.
   724	ISB     $15
   725	// Substract and compare to handle counter roll-over.
   726	// counter_read() - start < delay_ticks
   727	MRS     CNTVCT_EL0, R1
   728	SUB     R2, R1, R1
   729	CMP     R0, R1
   730	BCC     delay
   731done:
   732	RET
   733
   734// Save state of caller into g->sched,
   735// but using fake PC from systemstack_switch.
   736// Must only be called from functions with no locals ($0)
   737// or else unwinding from systemstack_switch is incorrect.
   738// Smashes R0.
   739TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
   740	MOVD	$runtime·systemstack_switch(SB), R0
   741	ADD	$8, R0	// get past prologue
   742	MOVD	R0, (g_sched+gobuf_pc)(g)
   743	MOVD	RSP, R0
   744	MOVD	R0, (g_sched+gobuf_sp)(g)
   745	MOVD	R29, (g_sched+gobuf_bp)(g)
   746	MOVD	$0, (g_sched+gobuf_lr)(g)
   747	// Assert ctxt is zero. See func save.
   748	MOVD	(g_sched+gobuf_ctxt)(g), R0
   749	CBZ	R0, 2(PC)
   750	CALL	runtime·abort(SB)
   751	RET
   752
   753// func asmcgocall_no_g(fn, arg unsafe.Pointer)
   754// Call fn(arg) aligned appropriately for the gcc ABI.
   755// Called on a system stack, and there may be no g yet (during needm).
   756TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
   757	MOVD	fn+0(FP), R1
   758	MOVD	arg+8(FP), R0
   759	SUB	$16, RSP	// skip over saved frame pointer below RSP
   760	BL	(R1)
   761	ADD	$16, RSP	// skip over saved frame pointer below RSP
   762	RET
   763
   764// func asmcgocall(fn, arg unsafe.Pointer) int32
   765// Call fn(arg) on the scheduler stack,
   766// aligned appropriately for the gcc ABI.
   767// See cgocall.go for more details.
   768TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   769	CBZ	g, nosave
   770
   771	// Figure out if we need to switch to m->g0 stack.
   772	// We get called to create new OS threads too, and those
   773	// come in on the m->g0 stack already. Or we might already
   774	// be on the m->gsignal stack.
   775	MOVD	g_m(g), R8
   776	MOVD	m_gsignal(R8), R3
   777	CMP	R3, g
   778	BEQ	nosave
   779	MOVD	m_g0(R8), R3
   780	CMP	R3, g
   781	BEQ	nosave
   782
   783	// running on a user stack. Figure out if we're running
   784	// secret code and clear our registers if so.
   785#ifdef GOEXPERIMENT_runtimesecret
   786	MOVW 	g_secret(g), R5
   787	CBZ		R5, nosecret
   788	BL 	·secretEraseRegisters(SB)
   789	// restore g0 back into R3
   790	MOVD	g_m(g), R3
   791	MOVD	m_g0(R3), R3
   792
   793nosecret:
   794#endif
   795	MOVD	fn+0(FP), R1
   796	MOVD	arg+8(FP), R0
   797	MOVD	RSP, R2
   798	MOVD	g, R4
   799
   800	// Switch to system stack.
   801	MOVD	R0, R9	// gosave_systemstack_switch<> and save_g might clobber R0
   802	BL	gosave_systemstack_switch<>(SB)
   803	MOVD	R3, g
   804	BL	runtime·save_g(SB)
   805	MOVD	(g_sched+gobuf_sp)(g), R0
   806	MOVD	R0, RSP
   807	MOVD	(g_sched+gobuf_bp)(g), R29
   808	MOVD	R9, R0
   809
   810	// Now on a scheduling stack (a pthread-created stack).
   811	// Save room for two of our pointers /*, plus 32 bytes of callee
   812	// save area that lives on the caller stack. */
   813	MOVD	RSP, R13
   814	SUB	$16, R13
   815	MOVD	R13, RSP
   816	MOVD	R4, 0(RSP)	// save old g on stack
   817	MOVD	(g_stack+stack_hi)(R4), R4
   818	SUB	R2, R4
   819	MOVD	R4, 8(RSP)	// save depth in old g stack (can't just save SP, as stack might be copied during a callback)
   820	BL	(R1)
   821	MOVD	R0, R9
   822
   823	// Restore g, stack pointer. R0 is errno, so don't touch it
   824	MOVD	0(RSP), g
   825	BL	runtime·save_g(SB)
   826	MOVD	(g_stack+stack_hi)(g), R5
   827	MOVD	8(RSP), R6
   828	SUB	R6, R5
   829	MOVD	R9, R0
   830	MOVD	R5, RSP
   831
   832	MOVW	R0, ret+16(FP)
   833	RET
   834
   835nosave:
   836	// Running on a system stack, perhaps even without a g.
   837	// Having no g can happen during thread creation or thread teardown
   838	// (see needm/dropm on Solaris, for example).
   839	// This code is like the above sequence but without saving/restoring g
   840	// and without worrying about the stack moving out from under us
   841	// (because we're on a system stack, not a goroutine stack).
   842	MOVD	fn+0(FP), R1
   843	MOVD	arg+8(FP), R0
   844	MOVD	RSP, R2
   845	MOVD 	R2, R13
   846	SUB	$16, R13
   847	MOVD	R13, RSP
   848	MOVD	$0, R4
   849	MOVD	R4, 0(RSP)	// Where above code stores g, in case someone looks during debugging.
   850	MOVD	R2, 8(RSP)	// Save original stack pointer.
   851	BL	(R1)
   852	// Restore stack pointer.
   853	MOVD	8(RSP), R2
   854	MOVD	R2, RSP
   855	MOVD	R0, ret+16(FP)
   856	RET
   857
   858// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
   859// See cgocall.go for more details.
   860TEXT ·cgocallback(SB),NOSPLIT,$24-24
   861	NO_LOCAL_POINTERS
   862
   863	// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
   864	// It is used to dropm while thread is exiting.
   865	MOVD	fn+0(FP), R1
   866	CBNZ	R1, loadg
   867	// Restore the g from frame.
   868	MOVD	frame+8(FP), g
   869	B	dropm
   870
   871loadg:
   872	// Load g from thread-local storage.
   873	BL	runtime·load_g(SB)
   874
   875	// If g is nil, Go did not create the current thread,
   876	// or if this thread never called into Go on pthread platforms.
   877	// Call needm to obtain one for temporary use.
   878	// In this case, we're running on the thread stack, so there's
   879	// lots of space, but the linker doesn't know. Hide the call from
   880	// the linker analysis by using an indirect call.
   881	CBZ	g, needm
   882
   883	MOVD	g_m(g), R8
   884	MOVD	R8, savedm-8(SP)
   885	B	havem
   886
   887needm:
   888	MOVD	g, savedm-8(SP) // g is zero, so is m.
   889	MOVD	$runtime·needAndBindM(SB), R0
   890	BL	(R0)
   891
   892	// Set m->g0->sched.sp = SP, so that if a panic happens
   893	// during the function we are about to execute, it will
   894	// have a valid SP to run on the g0 stack.
   895	// The next few lines (after the havem label)
   896	// will save this SP onto the stack and then write
   897	// the same SP back to m->sched.sp. That seems redundant,
   898	// but if an unrecovered panic happens, unwindm will
   899	// restore the g->sched.sp from the stack location
   900	// and then systemstack will try to use it. If we don't set it here,
   901	// that restored SP will be uninitialized (typically 0) and
   902	// will not be usable.
   903	MOVD	g_m(g), R8
   904	MOVD	m_g0(R8), R3
   905	MOVD	RSP, R0
   906	MOVD	R0, (g_sched+gobuf_sp)(R3)
   907	MOVD	R29, (g_sched+gobuf_bp)(R3)
   908
   909havem:
   910	// Now there's a valid m, and we're running on its m->g0.
   911	// Save current m->g0->sched.sp on stack and then set it to SP.
   912	// Save current sp in m->g0->sched.sp in preparation for
   913	// switch back to m->curg stack.
   914	// NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
   915	// Beware that the frame size is actually 32+16.
   916	MOVD	m_g0(R8), R3
   917	MOVD	(g_sched+gobuf_sp)(R3), R4
   918	MOVD	R4, savedsp-16(SP)
   919	MOVD	RSP, R0
   920	MOVD	R0, (g_sched+gobuf_sp)(R3)
   921
   922	// Switch to m->curg stack and call runtime.cgocallbackg.
   923	// Because we are taking over the execution of m->curg
   924	// but *not* resuming what had been running, we need to
   925	// save that information (m->curg->sched) so we can restore it.
   926	// We can restore m->curg->sched.sp easily, because calling
   927	// runtime.cgocallbackg leaves SP unchanged upon return.
   928	// To save m->curg->sched.pc, we push it onto the curg stack and
   929	// open a frame the same size as cgocallback's g0 frame.
   930	// Once we switch to the curg stack, the pushed PC will appear
   931	// to be the return PC of cgocallback, so that the traceback
   932	// will seamlessly trace back into the earlier calls.
   933	MOVD	m_curg(R8), g
   934	BL	runtime·save_g(SB)
   935	MOVD	(g_sched+gobuf_sp)(g), R4 // prepare stack as R4
   936	MOVD	(g_sched+gobuf_pc)(g), R5
   937	MOVD	R5, -48(R4)
   938	MOVD	(g_sched+gobuf_bp)(g), R6
   939	MOVD	R6, -56(R4)
   940
   941	// Gather our arguments into registers.
   942	MOVD	fn+0(FP), R0
   943	MOVD	frame+8(FP), R1
   944	MOVD	ctxt+16(FP), R2
   945
   946	SUB	$48, R4		// Allocate the same frame size on the g stack
   947	MOVD	R4, RSP		// switch stack
   948	MOVD	$runtime·cgocallbackg<ABIInternal>(SB), R11
   949	CALL	(R11) // indirect call to bypass nosplit check. We're on a different stack now.
   950
   951	// Restore g->sched (== m->curg->sched) from saved values.
   952	MOVD	0(RSP), R5
   953	MOVD	R5, (g_sched+gobuf_pc)(g)
   954	MOVD	-8(RSP), R6
   955	MOVD	R6, (g_sched+gobuf_bp)(g)
   956	MOVD	RSP, R4
   957	ADD	$48, R4, R4
   958	MOVD	R4, (g_sched+gobuf_sp)(g)
   959
   960	// Switch back to m->g0's stack and restore m->g0->sched.sp.
   961	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
   962	// so we do not have to restore it.)
   963	MOVD	g_m(g), R8
   964	MOVD	m_g0(R8), g
   965	BL	runtime·save_g(SB)
   966	MOVD	(g_sched+gobuf_sp)(g), R0
   967	MOVD	R0, RSP
   968	MOVD	savedsp-16(SP), R4
   969	MOVD	R4, (g_sched+gobuf_sp)(g)
   970
   971	// If the m on entry was nil, we called needm above to borrow an m,
   972	// 1. for the duration of the call on non-pthread platforms,
   973	// 2. or the duration of the C thread alive on pthread platforms.
   974	// If the m on entry wasn't nil,
   975	// 1. the thread might be a Go thread,
   976	// 2. or it wasn't the first call from a C thread on pthread platforms,
   977	//    since then we skip dropm to reuse the m in the first call.
   978	MOVD	savedm-8(SP), R6
   979	CBNZ	R6, droppedm
   980
   981	// Skip dropm to reuse it in the next call, when a pthread key has been created.
   982	MOVD	_cgo_pthread_key_created(SB), R6
   983	// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
   984	CBZ	R6, dropm
   985	MOVD	(R6), R6
   986	CBNZ	R6, droppedm
   987
   988dropm:
   989	MOVD	$runtime·dropm(SB), R0
   990	BL	(R0)
   991droppedm:
   992
   993	// Done!
   994	RET
   995
   996// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
   997// Must obey the gcc calling convention.
   998TEXT _cgo_topofstack(SB),NOSPLIT,$24
   999	// g (R28) and REGTMP (R27)  might be clobbered by load_g. They
  1000	// are callee-save in the gcc calling convention, so save them.
  1001	MOVD	R27, savedR27-8(SP)
  1002	MOVD	g, saveG-16(SP)
  1003
  1004	BL	runtime·load_g(SB)
  1005	MOVD	g_m(g), R0
  1006	MOVD	m_curg(R0), R0
  1007	MOVD	(g_stack+stack_hi)(R0), R0
  1008
  1009	MOVD	saveG-16(SP), g
  1010	MOVD	savedR28-8(SP), R27
  1011	RET
  1012
  1013// void setg(G*); set g. for use by needm.
  1014TEXT runtime·setg(SB), NOSPLIT, $0-8
  1015	MOVD	gg+0(FP), g
  1016	// This only happens if iscgo, so jump straight to save_g
  1017	BL	runtime·save_g(SB)
  1018	RET
  1019
  1020// void setg_gcc(G*); set g called from gcc
  1021TEXT setg_gcc<>(SB),NOSPLIT,$8
  1022	MOVD	R0, g
  1023	MOVD	R27, savedR27-8(SP)
  1024	BL	runtime·save_g(SB)
  1025	MOVD	savedR27-8(SP), R27
  1026	RET
  1027
  1028TEXT runtime·emptyfunc(SB),0,$0-0
  1029	RET
  1030
  1031TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
  1032	MOVD	ZR, R0
  1033	MOVD	(R0), R0
  1034	UNDEF
  1035
  1036// The top-most function running on a goroutine
  1037// returns to goexit+PCQuantum.
  1038TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
  1039	MOVD	R0, R0	// NOP
  1040	BL	runtime·goexit1(SB)	// does not return
  1041
  1042// This is called from .init_array and follows the platform, not Go, ABI.
  1043TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
  1044	SUB	$0x10, RSP
  1045	MOVD	R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
  1046	MOVD	runtime·lastmoduledatap(SB), R1
  1047	MOVD	R0, moduledata_next(R1)
  1048	MOVD	R0, runtime·lastmoduledatap(SB)
  1049	MOVD	8(RSP), R27
  1050	ADD	$0x10, RSP
  1051	RET
  1052
  1053// gcWriteBarrier informs the GC about heap pointer writes.
  1054//
  1055// gcWriteBarrier does NOT follow the Go ABI. It accepts the
  1056// number of bytes of buffer needed in R25, and returns a pointer
  1057// to the buffer space in R25.
  1058// It clobbers condition codes.
  1059// It does not clobber any general-purpose registers except R27,
  1060// but may clobber others (e.g., floating point registers)
  1061// The act of CALLing gcWriteBarrier will clobber R30 (LR).
  1062TEXT gcWriteBarrier<>(SB),NOSPLIT,$200
  1063	// Save the registers clobbered by the fast path.
  1064	STP	(R0, R1), 184(RSP)
  1065retry:
  1066	MOVD	g_m(g), R0
  1067	MOVD	m_p(R0), R0
  1068	MOVD	(p_wbBuf+wbBuf_next)(R0), R1
  1069	MOVD	(p_wbBuf+wbBuf_end)(R0), R27
  1070	// Increment wbBuf.next position.
  1071	ADD	R25, R1
  1072	// Is the buffer full?
  1073	CMP	R27, R1
  1074	BHI	flush
  1075	// Commit to the larger buffer.
  1076	MOVD	R1, (p_wbBuf+wbBuf_next)(R0)
  1077	// Make return value (the original next position)
  1078	SUB	R25, R1, R25
  1079	// Restore registers.
  1080	LDP	184(RSP), (R0, R1)
  1081	RET
  1082
  1083flush:
  1084	// Save all general purpose registers since these could be
  1085	// clobbered by wbBufFlush and were not saved by the caller.
  1086	// R0 and R1 already saved
  1087	STP	(R2, R3), 1*8(RSP)
  1088	STP	(R4, R5), 3*8(RSP)
  1089	STP	(R6, R7), 5*8(RSP)
  1090	STP	(R8, R9), 7*8(RSP)
  1091	STP	(R10, R11), 9*8(RSP)
  1092	STP	(R12, R13), 11*8(RSP)
  1093	STP	(R14, R15), 13*8(RSP)
  1094	// R16, R17 may be clobbered by linker trampoline
  1095	// R18 is unused.
  1096	STP	(R19, R20), 15*8(RSP)
  1097	STP	(R21, R22), 17*8(RSP)
  1098	STP	(R23, R24), 19*8(RSP)
  1099	STP	(R25, R26), 21*8(RSP)
  1100	// R27 is temp register.
  1101	// R28 is g.
  1102	// R29 is frame pointer (unused).
  1103	// R30 is LR, which was saved by the prologue.
  1104	// R31 is SP.
  1105
  1106	CALL	runtime·wbBufFlush(SB)
  1107	LDP	1*8(RSP), (R2, R3)
  1108	LDP	3*8(RSP), (R4, R5)
  1109	LDP	5*8(RSP), (R6, R7)
  1110	LDP	7*8(RSP), (R8, R9)
  1111	LDP	9*8(RSP), (R10, R11)
  1112	LDP	11*8(RSP), (R12, R13)
  1113	LDP	13*8(RSP), (R14, R15)
  1114	LDP	15*8(RSP), (R19, R20)
  1115	LDP	17*8(RSP), (R21, R22)
  1116	LDP	19*8(RSP), (R23, R24)
  1117	LDP	21*8(RSP), (R25, R26)
  1118	JMP	retry
  1119
  1120TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
  1121	MOVD	$8, R25
  1122	JMP	gcWriteBarrier<>(SB)
  1123TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
  1124	MOVD	$16, R25
  1125	JMP	gcWriteBarrier<>(SB)
  1126TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
  1127	MOVD	$24, R25
  1128	JMP	gcWriteBarrier<>(SB)
  1129TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
  1130	MOVD	$32, R25
  1131	JMP	gcWriteBarrier<>(SB)
  1132TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
  1133	MOVD	$40, R25
  1134	JMP	gcWriteBarrier<>(SB)
  1135TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
  1136	MOVD	$48, R25
  1137	JMP	gcWriteBarrier<>(SB)
  1138TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
  1139	MOVD	$56, R25
  1140	JMP	gcWriteBarrier<>(SB)
  1141TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
  1142	MOVD	$64, R25
  1143	JMP	gcWriteBarrier<>(SB)
  1144
  1145DATA	debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
  1146GLOBL	debugCallFrameTooLarge<>(SB), RODATA, $20	// Size duplicated below
  1147
  1148// debugCallV2 is the entry point for debugger-injected function
  1149// calls on running goroutines. It informs the runtime that a
  1150// debug call has been injected and creates a call frame for the
  1151// debugger to fill in.
  1152//
  1153// To inject a function call, a debugger should:
  1154// 1. Check that the goroutine is in state _Grunning and that
  1155//    there are at least 288 bytes free on the stack.
  1156// 2. Set SP as SP-16.
  1157// 3. Store the current LR in (SP) (using the SP after step 2).
  1158// 4. Store the current PC in the LR register.
  1159// 5. Write the desired argument frame size at SP-16
  1160// 6. Save all machine registers (including flags and fpsimd registers)
  1161//    so they can be restored later by the debugger.
  1162// 7. Set the PC to debugCallV2 and resume execution.
  1163//
  1164// If the goroutine is in state _Grunnable, then it's not generally
  1165// safe to inject a call because it may return out via other runtime
  1166// operations. Instead, the debugger should unwind the stack to find
  1167// the return to non-runtime code, add a temporary breakpoint there,
  1168// and inject the call once that breakpoint is hit.
  1169//
  1170// If the goroutine is in any other state, it's not safe to inject a call.
  1171//
  1172// This function communicates back to the debugger by setting R20 and
  1173// invoking BRK to raise a breakpoint signal. Note that the signal PC of
  1174// the signal triggered by the BRK instruction is the PC where the signal
  1175// is trapped, not the next PC, so to resume execution, the debugger needs
  1176// to set the signal PC to PC+4. See the comments in the implementation for
  1177// the protocol the debugger is expected to follow. InjectDebugCall in the
  1178// runtime tests demonstrates this protocol.
  1179//
  1180// The debugger must ensure that any pointers passed to the function
  1181// obey escape analysis requirements. Specifically, it must not pass
  1182// a stack pointer to an escaping argument. debugCallV2 cannot check
  1183// this invariant.
  1184//
  1185// This is ABIInternal because Go code injects its PC directly into new
  1186// goroutine stacks.
  1187TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
  1188	STP	(R29, R30), -280(RSP)
  1189	SUB	$272, RSP, RSP
  1190	SUB	$8, RSP, R29
  1191	// Save all registers that may contain pointers so they can be
  1192	// conservatively scanned.
  1193	//
  1194	// We can't do anything that might clobber any of these
  1195	// registers before this.
  1196	STP	(R27, g), (30*8)(RSP)
  1197	STP	(R25, R26), (28*8)(RSP)
  1198	STP	(R23, R24), (26*8)(RSP)
  1199	STP	(R21, R22), (24*8)(RSP)
  1200	STP	(R19, R20), (22*8)(RSP)
  1201	STP	(R16, R17), (20*8)(RSP)
  1202	STP	(R14, R15), (18*8)(RSP)
  1203	STP	(R12, R13), (16*8)(RSP)
  1204	STP	(R10, R11), (14*8)(RSP)
  1205	STP	(R8, R9), (12*8)(RSP)
  1206	STP	(R6, R7), (10*8)(RSP)
  1207	STP	(R4, R5), (8*8)(RSP)
  1208	STP	(R2, R3), (6*8)(RSP)
  1209	STP	(R0, R1), (4*8)(RSP)
  1210
  1211	// Perform a safe-point check.
  1212	MOVD	R30, 8(RSP) // Caller's PC
  1213	CALL	runtime·debugCallCheck(SB)
  1214	MOVD	16(RSP), R0
  1215	CBZ	R0, good
  1216
  1217	// The safety check failed. Put the reason string at the top
  1218	// of the stack.
  1219	MOVD	R0, 8(RSP)
  1220	MOVD	24(RSP), R0
  1221	MOVD	R0, 16(RSP)
  1222
  1223	// Set R20 to 8 and invoke BRK. The debugger should get the
  1224	// reason a call can't be injected from SP+8 and resume execution.
  1225	MOVD	$8, R20
  1226	BREAK
  1227	JMP	restore
  1228
  1229good:
  1230	// Registers are saved and it's safe to make a call.
  1231	// Open up a call frame, moving the stack if necessary.
  1232	//
  1233	// Once the frame is allocated, this will set R20 to 0 and
  1234	// invoke BRK. The debugger should write the argument
  1235	// frame for the call at SP+8, set up argument registers,
  1236	// set the LR as the signal PC + 4, set the PC to the function
  1237	// to call, set R26 to point to the closure (if a closure call),
  1238	// and resume execution.
  1239	//
  1240	// If the function returns, this will set R20 to 1 and invoke
  1241	// BRK. The debugger can then inspect any return value saved
  1242	// on the stack at SP+8 and in registers. To resume execution,
  1243	// the debugger should restore the LR from (SP).
  1244	//
  1245	// If the function panics, this will set R20 to 2 and invoke BRK.
  1246	// The interface{} value of the panic will be at SP+8. The debugger
  1247	// can inspect the panic value and resume execution again.
  1248#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE)	\
  1249	CMP	$MAXSIZE, R0;			\
  1250	BGT	5(PC);				\
  1251	MOVD	$NAME(SB), R0;			\
  1252	MOVD	R0, 8(RSP);			\
  1253	CALL	runtime·debugCallWrap(SB);	\
  1254	JMP	restore
  1255
  1256	MOVD	256(RSP), R0 // the argument frame size
  1257	DEBUG_CALL_DISPATCH(debugCall32<>, 32)
  1258	DEBUG_CALL_DISPATCH(debugCall64<>, 64)
  1259	DEBUG_CALL_DISPATCH(debugCall128<>, 128)
  1260	DEBUG_CALL_DISPATCH(debugCall256<>, 256)
  1261	DEBUG_CALL_DISPATCH(debugCall512<>, 512)
  1262	DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
  1263	DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
  1264	DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
  1265	DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
  1266	DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
  1267	DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
  1268	DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
  1269	// The frame size is too large. Report the error.
  1270	MOVD	$debugCallFrameTooLarge<>(SB), R0
  1271	MOVD	R0, 8(RSP)
  1272	MOVD	$20, R0
  1273	MOVD	R0, 16(RSP) // length of debugCallFrameTooLarge string
  1274	MOVD	$8, R20
  1275	BREAK
  1276	JMP	restore
  1277
  1278restore:
  1279	// Calls and failures resume here.
  1280	//
  1281	// Set R20 to 16 and invoke BRK. The debugger should restore
  1282	// all registers except for PC and RSP and resume execution.
  1283	MOVD	$16, R20
  1284	BREAK
  1285	// We must not modify flags after this point.
  1286
  1287	// Restore pointer-containing registers, which may have been
  1288	// modified from the debugger's copy by stack copying.
  1289	LDP	(30*8)(RSP), (R27, g)
  1290	LDP	(28*8)(RSP), (R25, R26)
  1291	LDP	(26*8)(RSP), (R23, R24)
  1292	LDP	(24*8)(RSP), (R21, R22)
  1293	LDP	(22*8)(RSP), (R19, R20)
  1294	LDP	(20*8)(RSP), (R16, R17)
  1295	LDP	(18*8)(RSP), (R14, R15)
  1296	LDP	(16*8)(RSP), (R12, R13)
  1297	LDP	(14*8)(RSP), (R10, R11)
  1298	LDP	(12*8)(RSP), (R8, R9)
  1299	LDP	(10*8)(RSP), (R6, R7)
  1300	LDP	(8*8)(RSP), (R4, R5)
  1301	LDP	(6*8)(RSP), (R2, R3)
  1302	LDP	(4*8)(RSP), (R0, R1)
  1303
  1304	LDP	-8(RSP), (R29, R27)
  1305	ADD	$288, RSP, RSP // Add 16 more bytes, see saveSigContext
  1306	MOVD	-16(RSP), R30 // restore old lr
  1307	JMP	(R27)
  1308
  1309// runtime.debugCallCheck assumes that functions defined with the
  1310// DEBUG_CALL_FN macro are safe points to inject calls.
  1311#define DEBUG_CALL_FN(NAME,MAXSIZE)		\
  1312TEXT NAME(SB),WRAPPER,$MAXSIZE-0;		\
  1313	NO_LOCAL_POINTERS;		\
  1314	MOVD	$0, R20;		\
  1315	BREAK;		\
  1316	MOVD	$1, R20;		\
  1317	BREAK;		\
  1318	RET
  1319DEBUG_CALL_FN(debugCall32<>, 32)
  1320DEBUG_CALL_FN(debugCall64<>, 64)
  1321DEBUG_CALL_FN(debugCall128<>, 128)
  1322DEBUG_CALL_FN(debugCall256<>, 256)
  1323DEBUG_CALL_FN(debugCall512<>, 512)
  1324DEBUG_CALL_FN(debugCall1024<>, 1024)
  1325DEBUG_CALL_FN(debugCall2048<>, 2048)
  1326DEBUG_CALL_FN(debugCall4096<>, 4096)
  1327DEBUG_CALL_FN(debugCall8192<>, 8192)
  1328DEBUG_CALL_FN(debugCall16384<>, 16384)
  1329DEBUG_CALL_FN(debugCall32768<>, 32768)
  1330DEBUG_CALL_FN(debugCall65536<>, 65536)
  1331
  1332// func debugCallPanicked(val interface{})
  1333TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
  1334	// Copy the panic value to the top of stack at SP+8.
  1335	MOVD	val_type+0(FP), R0
  1336	MOVD	R0, 8(RSP)
  1337	MOVD	val_data+8(FP), R0
  1338	MOVD	R0, 16(RSP)
  1339	MOVD	$2, R20
  1340	BREAK
  1341	RET
  1342
  1343TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
  1344	NO_LOCAL_POINTERS
  1345	// Save all 16 int registers that could have an index in them.
  1346	// They may be pointers, but if they are they are dead.
  1347	STP	(R0, R1), 24(RSP)
  1348	STP	(R2, R3), 40(RSP)
  1349	STP	(R4, R5), 56(RSP)
  1350	STP	(R6, R7), 72(RSP)
  1351	STP	(R8, R9), 88(RSP)
  1352	STP	(R10, R11), 104(RSP)
  1353	STP	(R12, R13), 120(RSP)
  1354	STP	(R14, R15), 136(RSP)
  1355	MOVD	LR, R0		// PC immediately after call to panicBounds
  1356	ADD	$24, RSP, R1	// pointer to save area
  1357	CALL	runtime·panicBounds64<ABIInternal>(SB)
  1358	RET
  1359
  1360TEXT ·getfp<ABIInternal>(SB),NOSPLIT|NOFRAME,$0
  1361	MOVD R29, R0
  1362	RET

View as plain text