Text file
src/runtime/asm_arm.s
Documentation: runtime
1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10// _rt0_arm is common startup code for most ARM systems when using
11// internal linking. This is the entry point for the program from the
12// kernel for an ordinary -buildmode=exe program. The stack holds the
13// number of arguments and the C-style argv.
14TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0
15 MOVW (R13), R0 // argc
16 MOVW $4(R13), R1 // argv
17 B runtime·rt0_go(SB)
18
19// main is common startup code for most ARM systems when using
20// external linking. The C startup code will call the symbol "main"
21// passing argc and argv in the usual C ABI registers R0 and R1.
22TEXT main(SB),NOSPLIT|NOFRAME,$0
23 B runtime·rt0_go(SB)
24
25// _rt0_arm_lib is common startup code for most ARM systems when
26// using -buildmode=c-archive or -buildmode=c-shared. The linker will
27// arrange to invoke this function as a global constructor (for
28// c-archive) or when the shared library is loaded (for c-shared).
29// We expect argc and argv to be passed in the usual C ABI registers
30// R0 and R1.
31TEXT _rt0_arm_lib(SB),NOSPLIT,$104
32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example,
33 // actually cares that R11 is preserved.
34 MOVW R4, 12(R13)
35 MOVW R5, 16(R13)
36 MOVW R6, 20(R13)
37 MOVW R7, 24(R13)
38 MOVW R8, 28(R13)
39 MOVW g, 32(R13)
40 MOVW R11, 36(R13)
41
42 // Skip floating point registers on goarmsoftfp != 0.
43 MOVB runtime·goarmsoftfp(SB), R11
44 CMP $0, R11
45 BNE skipfpsave
46 MOVD F8, (40+8*0)(R13)
47 MOVD F9, (40+8*1)(R13)
48 MOVD F10, (40+8*2)(R13)
49 MOVD F11, (40+8*3)(R13)
50 MOVD F12, (40+8*4)(R13)
51 MOVD F13, (40+8*5)(R13)
52 MOVD F14, (40+8*6)(R13)
53 MOVD F15, (40+8*7)(R13)
54skipfpsave:
55 // Save argc/argv.
56 MOVW R0, _rt0_arm_lib_argc<>(SB)
57 MOVW R1, _rt0_arm_lib_argv<>(SB)
58
59 MOVW $0, g // Initialize g.
60
61 CALL runtime·libInit(SB)
62
63 // Restore callee-save registers and return.
64 MOVB runtime·goarmsoftfp(SB), R11
65 CMP $0, R11
66 BNE skipfprest
67 MOVD (40+8*0)(R13), F8
68 MOVD (40+8*1)(R13), F9
69 MOVD (40+8*2)(R13), F10
70 MOVD (40+8*3)(R13), F11
71 MOVD (40+8*4)(R13), F12
72 MOVD (40+8*5)(R13), F13
73 MOVD (40+8*6)(R13), F14
74 MOVD (40+8*7)(R13), F15
75skipfprest:
76 MOVW 12(R13), R4
77 MOVW 16(R13), R5
78 MOVW 20(R13), R6
79 MOVW 24(R13), R7
80 MOVW 28(R13), R8
81 MOVW 32(R13), g
82 MOVW 36(R13), R11
83 RET
84
85// rt0_lib_go initializes the Go runtime.
86// This is started in a separate thread by _rt0_arm_lib.
87TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$8
88 MOVW _rt0_arm_lib_argc<>(SB), R0
89 MOVW _rt0_arm_lib_argv<>(SB), R1
90 B runtime·rt0_go(SB)
91
92DATA _rt0_arm_lib_argc<>(SB)/4,$0
93GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4
94DATA _rt0_arm_lib_argv<>(SB)/4,$0
95GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4
96
97// using NOFRAME means do not save LR on stack.
98// argc is in R0, argv is in R1.
99TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
100 MOVW $0xcafebabe, R12
101
102 // copy arguments forward on an even stack
103 // use R13 instead of SP to avoid linker rewriting the offsets
104 SUB $64, R13 // plenty of scratch
105 AND $~7, R13
106 MOVW R0, 60(R13) // save argc, argv away
107 MOVW R1, 64(R13)
108
109 // set up g register
110 // g is R10
111 MOVW $runtime·g0(SB), g
112 MOVW $runtime·m0(SB), R8
113
114 // save m->g0 = g0
115 MOVW g, m_g0(R8)
116 // save g->m = m0
117 MOVW R8, g_m(g)
118
119 // create istack out of the OS stack
120 // (1MB of system stack is available on iOS and Android)
121 MOVW $(-64*1024+104)(R13), R0
122 MOVW R0, g_stackguard0(g)
123 MOVW R0, g_stackguard1(g)
124 MOVW R0, (g_stack+stack_lo)(g)
125 MOVW R13, (g_stack+stack_hi)(g)
126
127 BL runtime·emptyfunc(SB) // fault if stack check is wrong
128
129#ifdef GOOS_openbsd
130 // Save g to TLS so that it is available from signal trampoline.
131 BL runtime·save_g(SB)
132#endif
133
134 BL runtime·_initcgo(SB) // will clobber R0-R3
135
136 // update stackguard after _cgo_init
137 MOVW (g_stack+stack_lo)(g), R0
138 ADD $const_stackGuard, R0
139 MOVW R0, g_stackguard0(g)
140 MOVW R0, g_stackguard1(g)
141
142 BL runtime·check(SB)
143
144 // saved argc, argv
145 MOVW 60(R13), R0
146 MOVW R0, 4(R13)
147 MOVW 64(R13), R1
148 MOVW R1, 8(R13)
149 BL runtime·args(SB)
150 BL runtime·checkgoarm(SB)
151 BL runtime·osinit(SB)
152 BL runtime·schedinit(SB)
153
154 // create a new goroutine to start program
155 SUB $8, R13
156 MOVW $runtime·mainPC(SB), R0
157 MOVW R0, 4(R13) // arg 1: fn
158 MOVW $0, R0
159 MOVW R0, 0(R13) // dummy LR
160 BL runtime·newproc(SB)
161 ADD $8, R13 // pop args and LR
162
163 // start this M
164 BL runtime·mstart(SB)
165
166 MOVW $1234, R0
167 MOVW $1000, R1
168 MOVW R0, (R1) // fail hard
169
170DATA runtime·mainPC+0(SB)/4,$runtime·main(SB)
171GLOBL runtime·mainPC(SB),RODATA,$4
172
173TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
174 // gdb won't skip this breakpoint instruction automatically,
175 // so you must manually "set $pc+=4" to skip it and continue.
176#ifdef GOOS_plan9
177 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9
178#else
179 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
180#endif
181 RET
182
183TEXT runtime·asminit(SB),NOSPLIT,$0-0
184 // disable runfast (flush-to-zero) mode of vfp if runtime.goarmsoftfp == 0
185 MOVB runtime·goarmsoftfp(SB), R11
186 CMP $0, R11
187 BNE 4(PC)
188 WORD $0xeef1ba10 // vmrs r11, fpscr
189 BIC $(1<<24), R11
190 WORD $0xeee1ba10 // vmsr fpscr, r11
191 RET
192
193TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
194 BL runtime·mstart0(SB)
195 RET // not reached
196
197/*
198 * go-routine
199 */
200
201// void gogo(Gobuf*)
202// restore state from Gobuf; longjmp
203TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4
204 MOVW buf+0(FP), R1
205 MOVW gobuf_g(R1), R0
206 MOVW 0(R0), R2 // make sure g != nil
207 B gogo<>(SB)
208
209TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0
210 BL setg<>(SB)
211 MOVW gobuf_sp(R1), R13 // restore SP==R13
212 MOVW gobuf_lr(R1), LR
213 MOVW gobuf_ctxt(R1), R7
214 MOVW $0, R11
215 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
216 MOVW R11, gobuf_lr(R1)
217 MOVW R11, gobuf_ctxt(R1)
218 MOVW gobuf_pc(R1), R11
219 CMP R11, R11 // set condition codes for == test, needed by stack split
220 B (R11)
221
222// func mcall(fn func(*g))
223// Switch to m->g0's stack, call fn(g).
224// Fn must never return. It should gogo(&g->sched)
225// to keep running g.
226TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4
227 // Save caller state in g->sched.
228 MOVW R13, (g_sched+gobuf_sp)(g)
229 MOVW LR, (g_sched+gobuf_pc)(g)
230 MOVW $0, R11
231 MOVW R11, (g_sched+gobuf_lr)(g)
232
233 // Switch to m->g0 & its stack, call fn.
234 MOVW g, R1
235 MOVW g_m(g), R8
236 MOVW m_g0(R8), R0
237 BL setg<>(SB)
238 CMP g, R1
239 B.NE 2(PC)
240 B runtime·badmcall(SB)
241 MOVW fn+0(FP), R0
242 MOVW (g_sched+gobuf_sp)(g), R13
243 SUB $8, R13
244 MOVW R1, 4(R13)
245 MOVW R0, R7
246 MOVW 0(R0), R0
247 BL (R0)
248 B runtime·badmcall2(SB)
249 RET
250
251// systemstack_switch is a dummy routine that systemstack leaves at the bottom
252// of the G stack. We need to distinguish the routine that
253// lives at the bottom of the G stack from the one that lives
254// at the top of the system stack because the one at the top of
255// the system stack terminates the stack walk (see topofstack()).
256TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0
257 MOVW $0, R0
258 BL (R0) // clobber lr to ensure push {lr} is kept
259 RET
260
261// func systemstack(fn func())
262TEXT runtime·systemstack(SB),NOSPLIT,$0-4
263 MOVW fn+0(FP), R0 // R0 = fn
264 MOVW g_m(g), R1 // R1 = m
265
266 MOVW m_gsignal(R1), R2 // R2 = gsignal
267 CMP g, R2
268 B.EQ noswitch
269
270 MOVW m_g0(R1), R2 // R2 = g0
271 CMP g, R2
272 B.EQ noswitch
273
274 MOVW m_curg(R1), R3
275 CMP g, R3
276 B.EQ switch
277
278 // Bad: g is not gsignal, not g0, not curg. What is it?
279 // Hide call from linker nosplit analysis.
280 MOVW $runtime·badsystemstack(SB), R0
281 BL (R0)
282 B runtime·abort(SB)
283
284switch:
285 // save our state in g->sched. Pretend to
286 // be systemstack_switch if the G stack is scanned.
287 BL gosave_systemstack_switch<>(SB)
288
289 // switch to g0
290 MOVW R0, R5
291 MOVW R2, R0
292 BL setg<>(SB)
293 MOVW R5, R0
294 MOVW (g_sched+gobuf_sp)(R2), R13
295
296 // call target function
297 MOVW R0, R7
298 MOVW 0(R0), R0
299 BL (R0)
300
301 // switch back to g
302 MOVW g_m(g), R1
303 MOVW m_curg(R1), R0
304 BL setg<>(SB)
305 MOVW (g_sched+gobuf_sp)(g), R13
306 MOVW $0, R3
307 MOVW R3, (g_sched+gobuf_sp)(g)
308 RET
309
310noswitch:
311 // Using a tail call here cleans up tracebacks since we won't stop
312 // at an intermediate systemstack.
313 MOVW R0, R7
314 MOVW 0(R0), R0
315 MOVW.P 4(R13), R14 // restore LR
316 B (R0)
317
318// func switchToCrashStack0(fn func())
319TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-4
320 MOVW fn+0(FP), R7 // context register
321 MOVW g_m(g), R1 // curm
322
323 // set g to gcrash
324 MOVW $runtime·gcrash(SB), R0
325 BL setg<>(SB) // g = &gcrash
326 MOVW R1, g_m(g) // g.m = curm
327 MOVW g, m_g0(R1) // curm.g0 = g
328
329 // switch to crashstack
330 MOVW (g_stack+stack_hi)(g), R1
331 SUB $(4*8), R1
332 MOVW R1, R13
333
334 // call target function
335 MOVW 0(R7), R0
336 BL (R0)
337
338 // should never return
339 CALL runtime·abort(SB)
340 UNDEF
341
342/*
343 * support for morestack
344 */
345
346// Called during function prolog when more stack is needed.
347// R3 prolog's LR
348// using NOFRAME means do not save LR on stack.
349//
350// The traceback routines see morestack on a g0 as being
351// the top of a stack (for example, morestack calling newstack
352// calling the scheduler calling newm calling gc), so we must
353// record an argument size. For that purpose, it has no arguments.
354TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
355 // Cannot grow scheduler stack (m->g0).
356 MOVW g_m(g), R8
357 MOVW m_g0(R8), R4
358
359 // Called from f.
360 // Set g->sched to context in f.
361 MOVW R13, (g_sched+gobuf_sp)(g)
362 MOVW LR, (g_sched+gobuf_pc)(g)
363 MOVW R3, (g_sched+gobuf_lr)(g)
364 MOVW R7, (g_sched+gobuf_ctxt)(g)
365
366 CMP g, R4
367 BNE 3(PC)
368 BL runtime·badmorestackg0(SB)
369 B runtime·abort(SB)
370
371 // Cannot grow signal stack (m->gsignal).
372 MOVW m_gsignal(R8), R4
373 CMP g, R4
374 BNE 3(PC)
375 BL runtime·badmorestackgsignal(SB)
376 B runtime·abort(SB)
377
378 // Called from f.
379 // Set m->morebuf to f's caller.
380 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
381 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
382 MOVW g, (m_morebuf+gobuf_g)(R8)
383
384 // Call newstack on m->g0's stack.
385 MOVW m_g0(R8), R0
386 BL setg<>(SB)
387 MOVW (g_sched+gobuf_sp)(g), R13
388 MOVW $0, R0
389 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR)
390 BL runtime·newstack(SB)
391
392 // Not reached, but make sure the return PC from the call to newstack
393 // is still in this function, and not the beginning of the next.
394 RET
395
396TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
397 // Force SPWRITE. This function doesn't actually write SP,
398 // but it is called with a special calling convention where
399 // the caller doesn't save LR on stack but passes it as a
400 // register (R3), and the unwinder currently doesn't understand.
401 // Make it SPWRITE to stop unwinding. (See issue 54332)
402 MOVW R13, R13
403
404 MOVW $0, R7
405 B runtime·morestack(SB)
406
407// reflectcall: call a function with the given argument list
408// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
409// we don't have variable-sized frames, so we use a small number
410// of constant-sized-frame functions to encode a few bits of size in the pc.
411// Caution: ugly multiline assembly macros in your future!
412
413#define DISPATCH(NAME,MAXSIZE) \
414 CMP $MAXSIZE, R0; \
415 B.HI 3(PC); \
416 MOVW $NAME(SB), R1; \
417 B (R1)
418
419TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
420 MOVW frameSize+20(FP), R0
421 DISPATCH(runtime·call16, 16)
422 DISPATCH(runtime·call32, 32)
423 DISPATCH(runtime·call64, 64)
424 DISPATCH(runtime·call128, 128)
425 DISPATCH(runtime·call256, 256)
426 DISPATCH(runtime·call512, 512)
427 DISPATCH(runtime·call1024, 1024)
428 DISPATCH(runtime·call2048, 2048)
429 DISPATCH(runtime·call4096, 4096)
430 DISPATCH(runtime·call8192, 8192)
431 DISPATCH(runtime·call16384, 16384)
432 DISPATCH(runtime·call32768, 32768)
433 DISPATCH(runtime·call65536, 65536)
434 DISPATCH(runtime·call131072, 131072)
435 DISPATCH(runtime·call262144, 262144)
436 DISPATCH(runtime·call524288, 524288)
437 DISPATCH(runtime·call1048576, 1048576)
438 DISPATCH(runtime·call2097152, 2097152)
439 DISPATCH(runtime·call4194304, 4194304)
440 DISPATCH(runtime·call8388608, 8388608)
441 DISPATCH(runtime·call16777216, 16777216)
442 DISPATCH(runtime·call33554432, 33554432)
443 DISPATCH(runtime·call67108864, 67108864)
444 DISPATCH(runtime·call134217728, 134217728)
445 DISPATCH(runtime·call268435456, 268435456)
446 DISPATCH(runtime·call536870912, 536870912)
447 DISPATCH(runtime·call1073741824, 1073741824)
448 MOVW $runtime·badreflectcall(SB), R1
449 B (R1)
450
451#define CALLFN(NAME,MAXSIZE) \
452TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
453 NO_LOCAL_POINTERS; \
454 /* copy arguments to stack */ \
455 MOVW stackArgs+8(FP), R0; \
456 MOVW stackArgsSize+12(FP), R2; \
457 ADD $4, R13, R1; \
458 CMP $0, R2; \
459 B.EQ 5(PC); \
460 MOVBU.P 1(R0), R5; \
461 MOVBU.P R5, 1(R1); \
462 SUB $1, R2, R2; \
463 B -5(PC); \
464 /* call function */ \
465 MOVW f+4(FP), R7; \
466 MOVW (R7), R0; \
467 PCDATA $PCDATA_StackMapIndex, $0; \
468 BL (R0); \
469 /* copy return values back */ \
470 MOVW stackArgsType+0(FP), R4; \
471 MOVW stackArgs+8(FP), R0; \
472 MOVW stackArgsSize+12(FP), R2; \
473 MOVW stackArgsRetOffset+16(FP), R3; \
474 ADD $4, R13, R1; \
475 ADD R3, R1; \
476 ADD R3, R0; \
477 SUB R3, R2; \
478 BL callRet<>(SB); \
479 RET
480
481// callRet copies return values back at the end of call*. This is a
482// separate function so it can allocate stack space for the arguments
483// to reflectcallmove. It does not follow the Go ABI; it expects its
484// arguments in registers.
485TEXT callRet<>(SB), NOSPLIT, $20-0
486 MOVW R4, 4(R13)
487 MOVW R0, 8(R13)
488 MOVW R1, 12(R13)
489 MOVW R2, 16(R13)
490 MOVW $0, R7
491 MOVW R7, 20(R13)
492 BL runtime·reflectcallmove(SB)
493 RET
494
495CALLFN(·call16, 16)
496CALLFN(·call32, 32)
497CALLFN(·call64, 64)
498CALLFN(·call128, 128)
499CALLFN(·call256, 256)
500CALLFN(·call512, 512)
501CALLFN(·call1024, 1024)
502CALLFN(·call2048, 2048)
503CALLFN(·call4096, 4096)
504CALLFN(·call8192, 8192)
505CALLFN(·call16384, 16384)
506CALLFN(·call32768, 32768)
507CALLFN(·call65536, 65536)
508CALLFN(·call131072, 131072)
509CALLFN(·call262144, 262144)
510CALLFN(·call524288, 524288)
511CALLFN(·call1048576, 1048576)
512CALLFN(·call2097152, 2097152)
513CALLFN(·call4194304, 4194304)
514CALLFN(·call8388608, 8388608)
515CALLFN(·call16777216, 16777216)
516CALLFN(·call33554432, 33554432)
517CALLFN(·call67108864, 67108864)
518CALLFN(·call134217728, 134217728)
519CALLFN(·call268435456, 268435456)
520CALLFN(·call536870912, 536870912)
521CALLFN(·call1073741824, 1073741824)
522
523// Save state of caller into g->sched,
524// but using fake PC from systemstack_switch.
525// Must only be called from functions with no locals ($0)
526// or else unwinding from systemstack_switch is incorrect.
527// Smashes R11.
528TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
529 MOVW $runtime·systemstack_switch(SB), R11
530 ADD $4, R11 // get past push {lr}
531 MOVW R11, (g_sched+gobuf_pc)(g)
532 MOVW R13, (g_sched+gobuf_sp)(g)
533 MOVW $0, R11
534 MOVW R11, (g_sched+gobuf_lr)(g)
535 // Assert ctxt is zero. See func save.
536 MOVW (g_sched+gobuf_ctxt)(g), R11
537 TST R11, R11
538 B.EQ 2(PC)
539 BL runtime·abort(SB)
540 RET
541
542// func asmcgocall_no_g(fn, arg unsafe.Pointer)
543// Call fn(arg) aligned appropriately for the gcc ABI.
544// Called on a system stack, and there may be no g yet (during needm).
545TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8
546 MOVW fn+0(FP), R1
547 MOVW arg+4(FP), R0
548 MOVW R13, R2
549 SUB $32, R13
550 BIC $0x7, R13 // alignment for gcc ABI
551 MOVW R2, 8(R13)
552 BL (R1)
553 MOVW 8(R13), R2
554 MOVW R2, R13
555 RET
556
557// func asmcgocall(fn, arg unsafe.Pointer) int32
558// Call fn(arg) on the scheduler stack,
559// aligned appropriately for the gcc ABI.
560// See cgocall.go for more details.
561TEXT ·asmcgocall(SB),NOSPLIT,$0-12
562 MOVW fn+0(FP), R1
563 MOVW arg+4(FP), R0
564
565 MOVW R13, R2
566 CMP $0, g
567 BEQ nosave
568 MOVW g, R4
569
570 // Figure out if we need to switch to m->g0 stack.
571 // We get called to create new OS threads too, and those
572 // come in on the m->g0 stack already. Or we might already
573 // be on the m->gsignal stack.
574 MOVW g_m(g), R8
575 MOVW m_gsignal(R8), R3
576 CMP R3, g
577 BEQ nosave
578 MOVW m_g0(R8), R3
579 CMP R3, g
580 BEQ nosave
581 BL gosave_systemstack_switch<>(SB)
582 MOVW R0, R5
583 MOVW R3, R0
584 BL setg<>(SB)
585 MOVW R5, R0
586 MOVW (g_sched+gobuf_sp)(g), R13
587
588 // Now on a scheduling stack (a pthread-created stack).
589 SUB $24, R13
590 BIC $0x7, R13 // alignment for gcc ABI
591 MOVW R4, 20(R13) // save old g
592 MOVW (g_stack+stack_hi)(R4), R4
593 SUB R2, R4
594 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
595 BL (R1)
596
597 // Restore registers, g, stack pointer.
598 MOVW R0, R5
599 MOVW 20(R13), R0
600 BL setg<>(SB)
601 MOVW (g_stack+stack_hi)(g), R1
602 MOVW 16(R13), R2
603 SUB R2, R1
604 MOVW R5, R0
605 MOVW R1, R13
606
607 MOVW R0, ret+8(FP)
608 RET
609
610nosave:
611 // Running on a system stack, perhaps even without a g.
612 // Having no g can happen during thread creation or thread teardown
613 // (see needm/dropm on Solaris, for example).
614 // This code is like the above sequence but without saving/restoring g
615 // and without worrying about the stack moving out from under us
616 // (because we're on a system stack, not a goroutine stack).
617 SUB $24, R13
618 BIC $0x7, R13 // alignment for gcc ABI
619 // save null g in case someone looks during debugging.
620 MOVW $0, R4
621 MOVW R4, 20(R13)
622 MOVW R2, 16(R13) // Save old stack pointer.
623 BL (R1)
624 // Restore stack pointer.
625 MOVW 16(R13), R2
626 MOVW R2, R13
627 MOVW R0, ret+8(FP)
628 RET
629
630// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
631// See cgocall.go for more details.
632TEXT ·cgocallback(SB),NOSPLIT,$12-12
633 NO_LOCAL_POINTERS
634
635 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
636 // It is used to dropm while thread is exiting.
637 MOVW fn+0(FP), R1
638 CMP $0, R1
639 B.NE loadg
640 // Restore the g from frame.
641 MOVW frame+4(FP), g
642 B dropm
643
644loadg:
645 // Load m and g from thread-local storage.
646#ifdef GOOS_openbsd
647 BL runtime·load_g(SB)
648#else
649 MOVB runtime·iscgo(SB), R0
650 CMP $0, R0
651 BL.NE runtime·load_g(SB)
652#endif
653
654 // If g is nil, Go did not create the current thread,
655 // or if this thread never called into Go on pthread platforms.
656 // Call needm to obtain one for temporary use.
657 // In this case, we're running on the thread stack, so there's
658 // lots of space, but the linker doesn't know. Hide the call from
659 // the linker analysis by using an indirect call.
660 CMP $0, g
661 B.EQ needm
662
663 MOVW g_m(g), R8
664 MOVW R8, savedm-4(SP)
665 B havem
666
667needm:
668 MOVW g, savedm-4(SP) // g is zero, so is m.
669 MOVW $runtime·needAndBindM(SB), R0
670 BL (R0)
671
672 // Set m->g0->sched.sp = SP, so that if a panic happens
673 // during the function we are about to execute, it will
674 // have a valid SP to run on the g0 stack.
675 // The next few lines (after the havem label)
676 // will save this SP onto the stack and then write
677 // the same SP back to m->sched.sp. That seems redundant,
678 // but if an unrecovered panic happens, unwindm will
679 // restore the g->sched.sp from the stack location
680 // and then systemstack will try to use it. If we don't set it here,
681 // that restored SP will be uninitialized (typically 0) and
682 // will not be usable.
683 MOVW g_m(g), R8
684 MOVW m_g0(R8), R3
685 MOVW R13, (g_sched+gobuf_sp)(R3)
686
687havem:
688 // Now there's a valid m, and we're running on its m->g0.
689 // Save current m->g0->sched.sp on stack and then set it to SP.
690 // Save current sp in m->g0->sched.sp in preparation for
691 // switch back to m->curg stack.
692 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP).
693 MOVW m_g0(R8), R3
694 MOVW (g_sched+gobuf_sp)(R3), R4
695 MOVW R4, savedsp-12(SP) // must match frame size
696 MOVW R13, (g_sched+gobuf_sp)(R3)
697
698 // Switch to m->curg stack and call runtime.cgocallbackg.
699 // Because we are taking over the execution of m->curg
700 // but *not* resuming what had been running, we need to
701 // save that information (m->curg->sched) so we can restore it.
702 // We can restore m->curg->sched.sp easily, because calling
703 // runtime.cgocallbackg leaves SP unchanged upon return.
704 // To save m->curg->sched.pc, we push it onto the curg stack and
705 // open a frame the same size as cgocallback's g0 frame.
706 // Once we switch to the curg stack, the pushed PC will appear
707 // to be the return PC of cgocallback, so that the traceback
708 // will seamlessly trace back into the earlier calls.
709 MOVW m_curg(R8), R0
710 BL setg<>(SB)
711 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
712 MOVW (g_sched+gobuf_pc)(g), R5
713 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size
714 // Gather our arguments into registers.
715 MOVW fn+0(FP), R1
716 MOVW frame+4(FP), R2
717 MOVW ctxt+8(FP), R3
718 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size
719 MOVW R1, 4(R13)
720 MOVW R2, 8(R13)
721 MOVW R3, 12(R13)
722 BL runtime·cgocallbackg(SB)
723
724 // Restore g->sched (== m->curg->sched) from saved values.
725 MOVW 0(R13), R5
726 MOVW R5, (g_sched+gobuf_pc)(g)
727 MOVW $(12+4)(R13), R4 // must match frame size
728 MOVW R4, (g_sched+gobuf_sp)(g)
729
730 // Switch back to m->g0's stack and restore m->g0->sched.sp.
731 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
732 // so we do not have to restore it.)
733 MOVW g_m(g), R8
734 MOVW m_g0(R8), R0
735 BL setg<>(SB)
736 MOVW (g_sched+gobuf_sp)(g), R13
737 MOVW savedsp-12(SP), R4 // must match frame size
738 MOVW R4, (g_sched+gobuf_sp)(g)
739
740 // If the m on entry was nil, we called needm above to borrow an m,
741 // 1. for the duration of the call on non-pthread platforms,
742 // 2. or the duration of the C thread alive on pthread platforms.
743 // If the m on entry wasn't nil,
744 // 1. the thread might be a Go thread,
745 // 2. or it wasn't the first call from a C thread on pthread platforms,
746 // since then we skip dropm to reuse the m in the first call.
747 MOVW savedm-4(SP), R6
748 CMP $0, R6
749 B.NE done
750
751 // Skip dropm to reuse it in the next call, when a pthread key has been created.
752 MOVW _cgo_pthread_key_created(SB), R6
753 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
754 CMP $0, R6
755 B.EQ dropm
756 MOVW (R6), R6
757 CMP $0, R6
758 B.NE done
759
760dropm:
761 MOVW $runtime·dropm(SB), R0
762 BL (R0)
763
764done:
765 // Done!
766 RET
767
768// void setg(G*); set g. for use by needm.
769TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4
770 MOVW gg+0(FP), R0
771 B setg<>(SB)
772
773TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0
774 MOVW R0, g
775
776 // Save g to thread-local storage.
777#ifdef GOOS_openbsd
778 B runtime·save_g(SB)
779#else
780 MOVB runtime·iscgo(SB), R0
781 CMP $0, R0
782 B.EQ 2(PC)
783 B runtime·save_g(SB)
784
785 MOVW g, R0
786 RET
787#endif
788
789TEXT runtime·emptyfunc(SB),0,$0-0
790 RET
791
792TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
793 MOVW $0, R0
794 MOVW (R0), R1
795
796// armPublicationBarrier is a native store/store barrier for ARMv7+.
797// On earlier ARM revisions, armPublicationBarrier is a no-op.
798// This will not work on SMP ARMv6 machines, if any are in use.
799// To implement publicationBarrier in sys_$GOOS_arm.s using the native
800// instructions, use:
801//
802// TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
803// B runtime·armPublicationBarrier(SB)
804//
805TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
806 MOVB runtime·goarm(SB), R11
807 CMP $7, R11
808 BLT 2(PC)
809 DMB MB_ST
810 RET
811
812TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0
813 MOVW cycles+0(FP), R1
814 MOVW $0, R0
815yieldloop:
816 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K)
817 CMP R0, R1
818 B.NE 2(PC)
819 RET
820 SUB $1, R1
821 B yieldloop
822
823// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
824// Must obey the gcc calling convention.
825TEXT _cgo_topofstack(SB),NOSPLIT,$8
826 // R11 and g register are clobbered by load_g. They are
827 // callee-save in the gcc calling convention, so save them here.
828 MOVW R11, saveR11-4(SP)
829 MOVW g, saveG-8(SP)
830
831 BL runtime·load_g(SB)
832 MOVW g_m(g), R0
833 MOVW m_curg(R0), R0
834 MOVW (g_stack+stack_hi)(R0), R0
835
836 MOVW saveG-8(SP), g
837 MOVW saveR11-4(SP), R11
838 RET
839
840// The top-most function running on a goroutine
841// returns to goexit+PCQuantum.
842TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
843 MOVW R0, R0 // NOP
844 BL runtime·goexit1(SB) // does not return
845 // traceback from goexit1 must hit code range of goexit
846 MOVW R0, R0 // NOP
847
848// x -> x/1000000, x%1000000, called from Go with args, results on stack.
849TEXT runtime·usplit(SB),NOSPLIT,$0-12
850 MOVW x+0(FP), R0
851 CALL runtime·usplitR0(SB)
852 MOVW R0, q+4(FP)
853 MOVW R1, r+8(FP)
854 RET
855
856// R0, R1 = R0/1000000, R0%1000000
857TEXT runtime·usplitR0(SB),NOSPLIT,$0
858 // magic multiply to avoid software divide without available m.
859 // see output of go tool compile -S for x/1000000.
860 MOVW R0, R3
861 MOVW $1125899907, R1
862 MULLU R1, R0, (R0, R1)
863 MOVW R0>>18, R0
864 MOVW $1000000, R1
865 MULU R0, R1
866 SUB R1, R3, R1
867 RET
868
869// This is called from .init_array and follows the platform, not Go, ABI.
870TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
871 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save
872 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI
873 MOVW runtime·lastmoduledatap(SB), R1
874 MOVW R0, moduledata_next(R1)
875 MOVW R0, runtime·lastmoduledatap(SB)
876 MOVW saver11-8(SP), R11
877 MOVW saver9-4(SP), R9
878 RET
879
880// gcWriteBarrier informs the GC about heap pointer writes.
881//
882// gcWriteBarrier does NOT follow the Go ABI. It accepts the
883// number of bytes of buffer needed in R8, and returns a pointer
884// to the buffer space in R8.
885// It clobbers condition codes.
886// It does not clobber any other general-purpose registers,
887// but may clobber others (e.g., floating point registers).
888// The act of CALLing gcWriteBarrier will clobber R14 (LR).
889TEXT gcWriteBarrier<>(SB),NOSPLIT|NOFRAME,$0
890 // Save the registers clobbered by the fast path.
891 MOVM.DB.W [R0,R1], (R13)
892retry:
893 MOVW g_m(g), R0
894 MOVW m_p(R0), R0
895 MOVW (p_wbBuf+wbBuf_next)(R0), R1
896 MOVW (p_wbBuf+wbBuf_end)(R0), R11
897 // Increment wbBuf.next position.
898 ADD R8, R1
899 // Is the buffer full?
900 CMP R11, R1
901 BHI flush
902 // Commit to the larger buffer.
903 MOVW R1, (p_wbBuf+wbBuf_next)(R0)
904 // Make return value (the original next position)
905 SUB R8, R1, R8
906 // Restore registers.
907 MOVM.IA.W (R13), [R0,R1]
908 RET
909
910flush:
911 // Save all general purpose registers since these could be
912 // clobbered by wbBufFlush and were not saved by the caller.
913 //
914 // R0 and R1 were saved at entry.
915 // R10 is g, so preserved.
916 // R11 is linker temp, so no need to save.
917 // R13 is stack pointer.
918 // R15 is PC.
919 MOVM.DB.W [R2-R9,R12], (R13)
920 // Save R14 (LR) because the fast path above doesn't save it,
921 // but needs it to RET.
922 MOVM.DB.W [R14], (R13)
923
924 CALL runtime·wbBufFlush(SB)
925
926 MOVM.IA.W (R13), [R14]
927 MOVM.IA.W (R13), [R2-R9,R12]
928 JMP retry
929
930TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
931 MOVW $4, R8
932 JMP gcWriteBarrier<>(SB)
933TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
934 MOVW $8, R8
935 JMP gcWriteBarrier<>(SB)
936TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
937 MOVW $12, R8
938 JMP gcWriteBarrier<>(SB)
939TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
940 MOVW $16, R8
941 JMP gcWriteBarrier<>(SB)
942TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
943 MOVW $20, R8
944 JMP gcWriteBarrier<>(SB)
945TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
946 MOVW $24, R8
947 JMP gcWriteBarrier<>(SB)
948TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
949 MOVW $28, R8
950 JMP gcWriteBarrier<>(SB)
951TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
952 MOVW $32, R8
953 JMP gcWriteBarrier<>(SB)
954
955TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$72-0
956 NO_LOCAL_POINTERS
957 // Save all int registers that could have an index in them.
958 // They may be pointers, but if they are they are dead.
959 MOVW R0, 12(R13)
960 MOVW R1, 16(R13)
961 MOVW R2, 20(R13)
962 MOVW R3, 24(R13)
963 MOVW R4, 28(R13)
964 MOVW R5, 32(R13)
965 MOVW R6, 36(R13)
966 MOVW R7, 40(R13)
967 MOVW R8, 44(R13)
968 MOVW R9, 48(R13)
969 // skip R10 aka G @ 52(R13)
970 // skip R11 aka tmp @ 56(R13)
971 MOVW R12, 60(R13)
972 // skip R13 aka SP @ 64(R13)
973 MOVW R14, 68(R13)
974 // skip R15 aka PC @ 72(R13)
975
976 MOVW R14, 4(R13) // PC immediately after call to panicBounds
977 ADD $12, R13, R0 // pointer to save area
978 MOVW R0, 8(R13)
979 CALL runtime·panicBounds32<ABIInternal>(SB)
980 RET
981
982TEXT runtime·panicExtend<ABIInternal>(SB),NOSPLIT,$72-0
983 NO_LOCAL_POINTERS
984 // Save all int registers that could have an index in them.
985 // They may be pointers, but if they are they are dead.
986 MOVW R0, 12(R13)
987 MOVW R1, 16(R13)
988 MOVW R2, 20(R13)
989 MOVW R3, 24(R13)
990 MOVW R4, 28(R13)
991 MOVW R5, 32(R13)
992 MOVW R6, 36(R13)
993 MOVW R7, 40(R13)
994 MOVW R8, 44(R13)
995 MOVW R9, 48(R13)
996 // skip R10 aka G @ 52(R13)
997 // skip R11 aka tmp @ 56(R13)
998 MOVW R12, 60(R13)
999 // skip R13 aka SP @ 64(R13)
1000 // skip R14 aka LR @ 68(R13)
1001 // skip R15 aka PC @ 72(R13)
1002
1003 MOVW R14, 4(R13) // PC immediately after call to panicExtend
1004 ADD $12, R13, R0 // pointer to save area
1005 MOVW R0, 8(R13)
1006 CALL runtime·panicBounds32X<ABIInternal>(SB)
1007 RET
View as plain text