Text file
src/runtime/asm_arm.s
Documentation: runtime
1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10// _rt0_arm is common startup code for most ARM systems when using
11// internal linking. This is the entry point for the program from the
12// kernel for an ordinary -buildmode=exe program. The stack holds the
13// number of arguments and the C-style argv.
14TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0
15 MOVW (R13), R0 // argc
16 MOVW $4(R13), R1 // argv
17 B runtime·rt0_go(SB)
18
19// main is common startup code for most ARM systems when using
20// external linking. The C startup code will call the symbol "main"
21// passing argc and argv in the usual C ABI registers R0 and R1.
22TEXT main(SB),NOSPLIT|NOFRAME,$0
23 B runtime·rt0_go(SB)
24
25// _rt0_arm_lib is common startup code for most ARM systems when
26// using -buildmode=c-archive or -buildmode=c-shared. The linker will
27// arrange to invoke this function as a global constructor (for
28// c-archive) or when the shared library is loaded (for c-shared).
29// We expect argc and argv to be passed in the usual C ABI registers
30// R0 and R1.
31TEXT _rt0_arm_lib(SB),NOSPLIT,$104
32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example,
33 // actually cares that R11 is preserved.
34 MOVW R4, 12(R13)
35 MOVW R5, 16(R13)
36 MOVW R6, 20(R13)
37 MOVW R7, 24(R13)
38 MOVW R8, 28(R13)
39 MOVW g, 32(R13)
40 MOVW R11, 36(R13)
41
42 // Skip floating point registers on goarmsoftfp != 0.
43 MOVB runtime·goarmsoftfp(SB), R11
44 CMP $0, R11
45 BNE skipfpsave
46 MOVD F8, (40+8*0)(R13)
47 MOVD F9, (40+8*1)(R13)
48 MOVD F10, (40+8*2)(R13)
49 MOVD F11, (40+8*3)(R13)
50 MOVD F12, (40+8*4)(R13)
51 MOVD F13, (40+8*5)(R13)
52 MOVD F14, (40+8*6)(R13)
53 MOVD F15, (40+8*7)(R13)
54skipfpsave:
55 // Save argc/argv.
56 MOVW R0, _rt0_arm_lib_argc<>(SB)
57 MOVW R1, _rt0_arm_lib_argv<>(SB)
58
59 MOVW $0, g // Initialize g.
60
61 // Synchronous initialization.
62 CALL runtime·libpreinit(SB)
63
64 // Create a new thread to do the runtime initialization.
65 MOVW _cgo_sys_thread_create(SB), R2
66 CMP $0, R2
67 BEQ nocgo
68 MOVW $_rt0_arm_lib_go<>(SB), R0
69 MOVW $0, R1
70 BL (R2)
71 B rr
72nocgo:
73 MOVW $0x800000, R0 // stacksize = 8192KB
74 MOVW $_rt0_arm_lib_go<>(SB), R1 // fn
75 MOVW R0, 4(R13)
76 MOVW R1, 8(R13)
77 BL runtime·newosproc0(SB)
78rr:
79 // Restore callee-save registers and return.
80 MOVB runtime·goarmsoftfp(SB), R11
81 CMP $0, R11
82 BNE skipfprest
83 MOVD (40+8*0)(R13), F8
84 MOVD (40+8*1)(R13), F9
85 MOVD (40+8*2)(R13), F10
86 MOVD (40+8*3)(R13), F11
87 MOVD (40+8*4)(R13), F12
88 MOVD (40+8*5)(R13), F13
89 MOVD (40+8*6)(R13), F14
90 MOVD (40+8*7)(R13), F15
91skipfprest:
92 MOVW 12(R13), R4
93 MOVW 16(R13), R5
94 MOVW 20(R13), R6
95 MOVW 24(R13), R7
96 MOVW 28(R13), R8
97 MOVW 32(R13), g
98 MOVW 36(R13), R11
99 RET
100
101// _rt0_arm_lib_go initializes the Go runtime.
102// This is started in a separate thread by _rt0_arm_lib.
103TEXT _rt0_arm_lib_go<>(SB),NOSPLIT,$8
104 MOVW _rt0_arm_lib_argc<>(SB), R0
105 MOVW _rt0_arm_lib_argv<>(SB), R1
106 B runtime·rt0_go(SB)
107
108DATA _rt0_arm_lib_argc<>(SB)/4,$0
109GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4
110DATA _rt0_arm_lib_argv<>(SB)/4,$0
111GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4
112
113// using NOFRAME means do not save LR on stack.
114// argc is in R0, argv is in R1.
115TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
116 MOVW $0xcafebabe, R12
117
118 // copy arguments forward on an even stack
119 // use R13 instead of SP to avoid linker rewriting the offsets
120 SUB $64, R13 // plenty of scratch
121 AND $~7, R13
122 MOVW R0, 60(R13) // save argc, argv away
123 MOVW R1, 64(R13)
124
125 // set up g register
126 // g is R10
127 MOVW $runtime·g0(SB), g
128 MOVW $runtime·m0(SB), R8
129
130 // save m->g0 = g0
131 MOVW g, m_g0(R8)
132 // save g->m = m0
133 MOVW R8, g_m(g)
134
135 // create istack out of the OS stack
136 // (1MB of system stack is available on iOS and Android)
137 MOVW $(-64*1024+104)(R13), R0
138 MOVW R0, g_stackguard0(g)
139 MOVW R0, g_stackguard1(g)
140 MOVW R0, (g_stack+stack_lo)(g)
141 MOVW R13, (g_stack+stack_hi)(g)
142
143 BL runtime·emptyfunc(SB) // fault if stack check is wrong
144
145#ifdef GOOS_openbsd
146 // Save g to TLS so that it is available from signal trampoline.
147 BL runtime·save_g(SB)
148#endif
149
150 BL runtime·_initcgo(SB) // will clobber R0-R3
151
152 // update stackguard after _cgo_init
153 MOVW (g_stack+stack_lo)(g), R0
154 ADD $const_stackGuard, R0
155 MOVW R0, g_stackguard0(g)
156 MOVW R0, g_stackguard1(g)
157
158 BL runtime·check(SB)
159
160 // saved argc, argv
161 MOVW 60(R13), R0
162 MOVW R0, 4(R13)
163 MOVW 64(R13), R1
164 MOVW R1, 8(R13)
165 BL runtime·args(SB)
166 BL runtime·checkgoarm(SB)
167 BL runtime·osinit(SB)
168 BL runtime·schedinit(SB)
169
170 // create a new goroutine to start program
171 SUB $8, R13
172 MOVW $runtime·mainPC(SB), R0
173 MOVW R0, 4(R13) // arg 1: fn
174 MOVW $0, R0
175 MOVW R0, 0(R13) // dummy LR
176 BL runtime·newproc(SB)
177 ADD $8, R13 // pop args and LR
178
179 // start this M
180 BL runtime·mstart(SB)
181
182 MOVW $1234, R0
183 MOVW $1000, R1
184 MOVW R0, (R1) // fail hard
185
186DATA runtime·mainPC+0(SB)/4,$runtime·main(SB)
187GLOBL runtime·mainPC(SB),RODATA,$4
188
189TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
190 // gdb won't skip this breakpoint instruction automatically,
191 // so you must manually "set $pc+=4" to skip it and continue.
192#ifdef GOOS_plan9
193 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9
194#else
195 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
196#endif
197 RET
198
199TEXT runtime·asminit(SB),NOSPLIT,$0-0
200 // disable runfast (flush-to-zero) mode of vfp if runtime.goarmsoftfp == 0
201 MOVB runtime·goarmsoftfp(SB), R11
202 CMP $0, R11
203 BNE 4(PC)
204 WORD $0xeef1ba10 // vmrs r11, fpscr
205 BIC $(1<<24), R11
206 WORD $0xeee1ba10 // vmsr fpscr, r11
207 RET
208
209TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
210 BL runtime·mstart0(SB)
211 RET // not reached
212
213/*
214 * go-routine
215 */
216
217// void gogo(Gobuf*)
218// restore state from Gobuf; longjmp
219TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4
220 MOVW buf+0(FP), R1
221 MOVW gobuf_g(R1), R0
222 MOVW 0(R0), R2 // make sure g != nil
223 B gogo<>(SB)
224
225TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0
226 BL setg<>(SB)
227 MOVW gobuf_sp(R1), R13 // restore SP==R13
228 MOVW gobuf_lr(R1), LR
229 MOVW gobuf_ret(R1), R0
230 MOVW gobuf_ctxt(R1), R7
231 MOVW $0, R11
232 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
233 MOVW R11, gobuf_ret(R1)
234 MOVW R11, gobuf_lr(R1)
235 MOVW R11, gobuf_ctxt(R1)
236 MOVW gobuf_pc(R1), R11
237 CMP R11, R11 // set condition codes for == test, needed by stack split
238 B (R11)
239
240// func mcall(fn func(*g))
241// Switch to m->g0's stack, call fn(g).
242// Fn must never return. It should gogo(&g->sched)
243// to keep running g.
244TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4
245 // Save caller state in g->sched.
246 MOVW R13, (g_sched+gobuf_sp)(g)
247 MOVW LR, (g_sched+gobuf_pc)(g)
248 MOVW $0, R11
249 MOVW R11, (g_sched+gobuf_lr)(g)
250
251 // Switch to m->g0 & its stack, call fn.
252 MOVW g, R1
253 MOVW g_m(g), R8
254 MOVW m_g0(R8), R0
255 BL setg<>(SB)
256 CMP g, R1
257 B.NE 2(PC)
258 B runtime·badmcall(SB)
259 MOVW fn+0(FP), R0
260 MOVW (g_sched+gobuf_sp)(g), R13
261 SUB $8, R13
262 MOVW R1, 4(R13)
263 MOVW R0, R7
264 MOVW 0(R0), R0
265 BL (R0)
266 B runtime·badmcall2(SB)
267 RET
268
269// systemstack_switch is a dummy routine that systemstack leaves at the bottom
270// of the G stack. We need to distinguish the routine that
271// lives at the bottom of the G stack from the one that lives
272// at the top of the system stack because the one at the top of
273// the system stack terminates the stack walk (see topofstack()).
274TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0
275 MOVW $0, R0
276 BL (R0) // clobber lr to ensure push {lr} is kept
277 RET
278
279// func systemstack(fn func())
280TEXT runtime·systemstack(SB),NOSPLIT,$0-4
281 MOVW fn+0(FP), R0 // R0 = fn
282 MOVW g_m(g), R1 // R1 = m
283
284 MOVW m_gsignal(R1), R2 // R2 = gsignal
285 CMP g, R2
286 B.EQ noswitch
287
288 MOVW m_g0(R1), R2 // R2 = g0
289 CMP g, R2
290 B.EQ noswitch
291
292 MOVW m_curg(R1), R3
293 CMP g, R3
294 B.EQ switch
295
296 // Bad: g is not gsignal, not g0, not curg. What is it?
297 // Hide call from linker nosplit analysis.
298 MOVW $runtime·badsystemstack(SB), R0
299 BL (R0)
300 B runtime·abort(SB)
301
302switch:
303 // save our state in g->sched. Pretend to
304 // be systemstack_switch if the G stack is scanned.
305 BL gosave_systemstack_switch<>(SB)
306
307 // switch to g0
308 MOVW R0, R5
309 MOVW R2, R0
310 BL setg<>(SB)
311 MOVW R5, R0
312 MOVW (g_sched+gobuf_sp)(R2), R13
313
314 // call target function
315 MOVW R0, R7
316 MOVW 0(R0), R0
317 BL (R0)
318
319 // switch back to g
320 MOVW g_m(g), R1
321 MOVW m_curg(R1), R0
322 BL setg<>(SB)
323 MOVW (g_sched+gobuf_sp)(g), R13
324 MOVW $0, R3
325 MOVW R3, (g_sched+gobuf_sp)(g)
326 RET
327
328noswitch:
329 // Using a tail call here cleans up tracebacks since we won't stop
330 // at an intermediate systemstack.
331 MOVW R0, R7
332 MOVW 0(R0), R0
333 MOVW.P 4(R13), R14 // restore LR
334 B (R0)
335
336/*
337 * support for morestack
338 */
339
340// Called during function prolog when more stack is needed.
341// R3 prolog's LR
342// using NOFRAME means do not save LR on stack.
343//
344// The traceback routines see morestack on a g0 as being
345// the top of a stack (for example, morestack calling newstack
346// calling the scheduler calling newm calling gc), so we must
347// record an argument size. For that purpose, it has no arguments.
348TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
349 // Cannot grow scheduler stack (m->g0).
350 MOVW g_m(g), R8
351 MOVW m_g0(R8), R4
352 CMP g, R4
353 BNE 3(PC)
354 BL runtime·badmorestackg0(SB)
355 B runtime·abort(SB)
356
357 // Cannot grow signal stack (m->gsignal).
358 MOVW m_gsignal(R8), R4
359 CMP g, R4
360 BNE 3(PC)
361 BL runtime·badmorestackgsignal(SB)
362 B runtime·abort(SB)
363
364 // Called from f.
365 // Set g->sched to context in f.
366 MOVW R13, (g_sched+gobuf_sp)(g)
367 MOVW LR, (g_sched+gobuf_pc)(g)
368 MOVW R3, (g_sched+gobuf_lr)(g)
369 MOVW R7, (g_sched+gobuf_ctxt)(g)
370
371 // Called from f.
372 // Set m->morebuf to f's caller.
373 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
374 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
375 MOVW g, (m_morebuf+gobuf_g)(R8)
376
377 // Call newstack on m->g0's stack.
378 MOVW m_g0(R8), R0
379 BL setg<>(SB)
380 MOVW (g_sched+gobuf_sp)(g), R13
381 MOVW $0, R0
382 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR)
383 BL runtime·newstack(SB)
384
385 // Not reached, but make sure the return PC from the call to newstack
386 // is still in this function, and not the beginning of the next.
387 RET
388
389TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
390 // Force SPWRITE. This function doesn't actually write SP,
391 // but it is called with a special calling convention where
392 // the caller doesn't save LR on stack but passes it as a
393 // register (R3), and the unwinder currently doesn't understand.
394 // Make it SPWRITE to stop unwinding. (See issue 54332)
395 MOVW R13, R13
396
397 MOVW $0, R7
398 B runtime·morestack(SB)
399
400// reflectcall: call a function with the given argument list
401// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
402// we don't have variable-sized frames, so we use a small number
403// of constant-sized-frame functions to encode a few bits of size in the pc.
404// Caution: ugly multiline assembly macros in your future!
405
406#define DISPATCH(NAME,MAXSIZE) \
407 CMP $MAXSIZE, R0; \
408 B.HI 3(PC); \
409 MOVW $NAME(SB), R1; \
410 B (R1)
411
412TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
413 MOVW frameSize+20(FP), R0
414 DISPATCH(runtime·call16, 16)
415 DISPATCH(runtime·call32, 32)
416 DISPATCH(runtime·call64, 64)
417 DISPATCH(runtime·call128, 128)
418 DISPATCH(runtime·call256, 256)
419 DISPATCH(runtime·call512, 512)
420 DISPATCH(runtime·call1024, 1024)
421 DISPATCH(runtime·call2048, 2048)
422 DISPATCH(runtime·call4096, 4096)
423 DISPATCH(runtime·call8192, 8192)
424 DISPATCH(runtime·call16384, 16384)
425 DISPATCH(runtime·call32768, 32768)
426 DISPATCH(runtime·call65536, 65536)
427 DISPATCH(runtime·call131072, 131072)
428 DISPATCH(runtime·call262144, 262144)
429 DISPATCH(runtime·call524288, 524288)
430 DISPATCH(runtime·call1048576, 1048576)
431 DISPATCH(runtime·call2097152, 2097152)
432 DISPATCH(runtime·call4194304, 4194304)
433 DISPATCH(runtime·call8388608, 8388608)
434 DISPATCH(runtime·call16777216, 16777216)
435 DISPATCH(runtime·call33554432, 33554432)
436 DISPATCH(runtime·call67108864, 67108864)
437 DISPATCH(runtime·call134217728, 134217728)
438 DISPATCH(runtime·call268435456, 268435456)
439 DISPATCH(runtime·call536870912, 536870912)
440 DISPATCH(runtime·call1073741824, 1073741824)
441 MOVW $runtime·badreflectcall(SB), R1
442 B (R1)
443
444#define CALLFN(NAME,MAXSIZE) \
445TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
446 NO_LOCAL_POINTERS; \
447 /* copy arguments to stack */ \
448 MOVW stackArgs+8(FP), R0; \
449 MOVW stackArgsSize+12(FP), R2; \
450 ADD $4, R13, R1; \
451 CMP $0, R2; \
452 B.EQ 5(PC); \
453 MOVBU.P 1(R0), R5; \
454 MOVBU.P R5, 1(R1); \
455 SUB $1, R2, R2; \
456 B -5(PC); \
457 /* call function */ \
458 MOVW f+4(FP), R7; \
459 MOVW (R7), R0; \
460 PCDATA $PCDATA_StackMapIndex, $0; \
461 BL (R0); \
462 /* copy return values back */ \
463 MOVW stackArgsType+0(FP), R4; \
464 MOVW stackArgs+8(FP), R0; \
465 MOVW stackArgsSize+12(FP), R2; \
466 MOVW stackArgsRetOffset+16(FP), R3; \
467 ADD $4, R13, R1; \
468 ADD R3, R1; \
469 ADD R3, R0; \
470 SUB R3, R2; \
471 BL callRet<>(SB); \
472 RET
473
474// callRet copies return values back at the end of call*. This is a
475// separate function so it can allocate stack space for the arguments
476// to reflectcallmove. It does not follow the Go ABI; it expects its
477// arguments in registers.
478TEXT callRet<>(SB), NOSPLIT, $20-0
479 MOVW R4, 4(R13)
480 MOVW R0, 8(R13)
481 MOVW R1, 12(R13)
482 MOVW R2, 16(R13)
483 MOVW $0, R7
484 MOVW R7, 20(R13)
485 BL runtime·reflectcallmove(SB)
486 RET
487
488CALLFN(·call16, 16)
489CALLFN(·call32, 32)
490CALLFN(·call64, 64)
491CALLFN(·call128, 128)
492CALLFN(·call256, 256)
493CALLFN(·call512, 512)
494CALLFN(·call1024, 1024)
495CALLFN(·call2048, 2048)
496CALLFN(·call4096, 4096)
497CALLFN(·call8192, 8192)
498CALLFN(·call16384, 16384)
499CALLFN(·call32768, 32768)
500CALLFN(·call65536, 65536)
501CALLFN(·call131072, 131072)
502CALLFN(·call262144, 262144)
503CALLFN(·call524288, 524288)
504CALLFN(·call1048576, 1048576)
505CALLFN(·call2097152, 2097152)
506CALLFN(·call4194304, 4194304)
507CALLFN(·call8388608, 8388608)
508CALLFN(·call16777216, 16777216)
509CALLFN(·call33554432, 33554432)
510CALLFN(·call67108864, 67108864)
511CALLFN(·call134217728, 134217728)
512CALLFN(·call268435456, 268435456)
513CALLFN(·call536870912, 536870912)
514CALLFN(·call1073741824, 1073741824)
515
516// Save state of caller into g->sched,
517// but using fake PC from systemstack_switch.
518// Must only be called from functions with no locals ($0)
519// or else unwinding from systemstack_switch is incorrect.
520// Smashes R11.
521TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
522 MOVW $runtime·systemstack_switch(SB), R11
523 ADD $4, R11 // get past push {lr}
524 MOVW R11, (g_sched+gobuf_pc)(g)
525 MOVW R13, (g_sched+gobuf_sp)(g)
526 MOVW $0, R11
527 MOVW R11, (g_sched+gobuf_lr)(g)
528 MOVW R11, (g_sched+gobuf_ret)(g)
529 // Assert ctxt is zero. See func save.
530 MOVW (g_sched+gobuf_ctxt)(g), R11
531 TST R11, R11
532 B.EQ 2(PC)
533 BL runtime·abort(SB)
534 RET
535
536// func asmcgocall_no_g(fn, arg unsafe.Pointer)
537// Call fn(arg) aligned appropriately for the gcc ABI.
538// Called on a system stack, and there may be no g yet (during needm).
539TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8
540 MOVW fn+0(FP), R1
541 MOVW arg+4(FP), R0
542 MOVW R13, R2
543 SUB $32, R13
544 BIC $0x7, R13 // alignment for gcc ABI
545 MOVW R2, 8(R13)
546 BL (R1)
547 MOVW 8(R13), R2
548 MOVW R2, R13
549 RET
550
551// func asmcgocall(fn, arg unsafe.Pointer) int32
552// Call fn(arg) on the scheduler stack,
553// aligned appropriately for the gcc ABI.
554// See cgocall.go for more details.
555TEXT ·asmcgocall(SB),NOSPLIT,$0-12
556 MOVW fn+0(FP), R1
557 MOVW arg+4(FP), R0
558
559 MOVW R13, R2
560 CMP $0, g
561 BEQ nosave
562 MOVW g, R4
563
564 // Figure out if we need to switch to m->g0 stack.
565 // We get called to create new OS threads too, and those
566 // come in on the m->g0 stack already. Or we might already
567 // be on the m->gsignal stack.
568 MOVW g_m(g), R8
569 MOVW m_gsignal(R8), R3
570 CMP R3, g
571 BEQ nosave
572 MOVW m_g0(R8), R3
573 CMP R3, g
574 BEQ nosave
575 BL gosave_systemstack_switch<>(SB)
576 MOVW R0, R5
577 MOVW R3, R0
578 BL setg<>(SB)
579 MOVW R5, R0
580 MOVW (g_sched+gobuf_sp)(g), R13
581
582 // Now on a scheduling stack (a pthread-created stack).
583 SUB $24, R13
584 BIC $0x7, R13 // alignment for gcc ABI
585 MOVW R4, 20(R13) // save old g
586 MOVW (g_stack+stack_hi)(R4), R4
587 SUB R2, R4
588 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
589 BL (R1)
590
591 // Restore registers, g, stack pointer.
592 MOVW R0, R5
593 MOVW 20(R13), R0
594 BL setg<>(SB)
595 MOVW (g_stack+stack_hi)(g), R1
596 MOVW 16(R13), R2
597 SUB R2, R1
598 MOVW R5, R0
599 MOVW R1, R13
600
601 MOVW R0, ret+8(FP)
602 RET
603
604nosave:
605 // Running on a system stack, perhaps even without a g.
606 // Having no g can happen during thread creation or thread teardown
607 // (see needm/dropm on Solaris, for example).
608 // This code is like the above sequence but without saving/restoring g
609 // and without worrying about the stack moving out from under us
610 // (because we're on a system stack, not a goroutine stack).
611 // The above code could be used directly if already on a system stack,
612 // but then the only path through this code would be a rare case on Solaris.
613 // Using this code for all "already on system stack" calls exercises it more,
614 // which should help keep it correct.
615 SUB $24, R13
616 BIC $0x7, R13 // alignment for gcc ABI
617 // save null g in case someone looks during debugging.
618 MOVW $0, R4
619 MOVW R4, 20(R13)
620 MOVW R2, 16(R13) // Save old stack pointer.
621 BL (R1)
622 // Restore stack pointer.
623 MOVW 16(R13), R2
624 MOVW R2, R13
625 MOVW R0, ret+8(FP)
626 RET
627
628// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
629// See cgocall.go for more details.
630TEXT ·cgocallback(SB),NOSPLIT,$12-12
631 NO_LOCAL_POINTERS
632
633 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
634 // It is used to dropm while thread is exiting.
635 MOVW fn+0(FP), R1
636 CMP $0, R1
637 B.NE loadg
638 // Restore the g from frame.
639 MOVW frame+4(FP), g
640 B dropm
641
642loadg:
643 // Load m and g from thread-local storage.
644#ifdef GOOS_openbsd
645 BL runtime·load_g(SB)
646#else
647 MOVB runtime·iscgo(SB), R0
648 CMP $0, R0
649 BL.NE runtime·load_g(SB)
650#endif
651
652 // If g is nil, Go did not create the current thread,
653 // or if this thread never called into Go on pthread platforms.
654 // Call needm to obtain one for temporary use.
655 // In this case, we're running on the thread stack, so there's
656 // lots of space, but the linker doesn't know. Hide the call from
657 // the linker analysis by using an indirect call.
658 CMP $0, g
659 B.EQ needm
660
661 MOVW g_m(g), R8
662 MOVW R8, savedm-4(SP)
663 B havem
664
665needm:
666 MOVW g, savedm-4(SP) // g is zero, so is m.
667 MOVW $runtime·needAndBindM(SB), R0
668 BL (R0)
669
670 // Set m->g0->sched.sp = SP, so that if a panic happens
671 // during the function we are about to execute, it will
672 // have a valid SP to run on the g0 stack.
673 // The next few lines (after the havem label)
674 // will save this SP onto the stack and then write
675 // the same SP back to m->sched.sp. That seems redundant,
676 // but if an unrecovered panic happens, unwindm will
677 // restore the g->sched.sp from the stack location
678 // and then systemstack will try to use it. If we don't set it here,
679 // that restored SP will be uninitialized (typically 0) and
680 // will not be usable.
681 MOVW g_m(g), R8
682 MOVW m_g0(R8), R3
683 MOVW R13, (g_sched+gobuf_sp)(R3)
684
685havem:
686 // Now there's a valid m, and we're running on its m->g0.
687 // Save current m->g0->sched.sp on stack and then set it to SP.
688 // Save current sp in m->g0->sched.sp in preparation for
689 // switch back to m->curg stack.
690 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP).
691 MOVW m_g0(R8), R3
692 MOVW (g_sched+gobuf_sp)(R3), R4
693 MOVW R4, savedsp-12(SP) // must match frame size
694 MOVW R13, (g_sched+gobuf_sp)(R3)
695
696 // Switch to m->curg stack and call runtime.cgocallbackg.
697 // Because we are taking over the execution of m->curg
698 // but *not* resuming what had been running, we need to
699 // save that information (m->curg->sched) so we can restore it.
700 // We can restore m->curg->sched.sp easily, because calling
701 // runtime.cgocallbackg leaves SP unchanged upon return.
702 // To save m->curg->sched.pc, we push it onto the curg stack and
703 // open a frame the same size as cgocallback's g0 frame.
704 // Once we switch to the curg stack, the pushed PC will appear
705 // to be the return PC of cgocallback, so that the traceback
706 // will seamlessly trace back into the earlier calls.
707 MOVW m_curg(R8), R0
708 BL setg<>(SB)
709 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
710 MOVW (g_sched+gobuf_pc)(g), R5
711 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size
712 // Gather our arguments into registers.
713 MOVW fn+0(FP), R1
714 MOVW frame+4(FP), R2
715 MOVW ctxt+8(FP), R3
716 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size
717 MOVW R1, 4(R13)
718 MOVW R2, 8(R13)
719 MOVW R3, 12(R13)
720 BL runtime·cgocallbackg(SB)
721
722 // Restore g->sched (== m->curg->sched) from saved values.
723 MOVW 0(R13), R5
724 MOVW R5, (g_sched+gobuf_pc)(g)
725 MOVW $(12+4)(R13), R4 // must match frame size
726 MOVW R4, (g_sched+gobuf_sp)(g)
727
728 // Switch back to m->g0's stack and restore m->g0->sched.sp.
729 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
730 // so we do not have to restore it.)
731 MOVW g_m(g), R8
732 MOVW m_g0(R8), R0
733 BL setg<>(SB)
734 MOVW (g_sched+gobuf_sp)(g), R13
735 MOVW savedsp-12(SP), R4 // must match frame size
736 MOVW R4, (g_sched+gobuf_sp)(g)
737
738 // If the m on entry was nil, we called needm above to borrow an m,
739 // 1. for the duration of the call on non-pthread platforms,
740 // 2. or the duration of the C thread alive on pthread platforms.
741 // If the m on entry wasn't nil,
742 // 1. the thread might be a Go thread,
743 // 2. or it wasn't the first call from a C thread on pthread platforms,
744 // since then we skip dropm to reuse the m in the first call.
745 MOVW savedm-4(SP), R6
746 CMP $0, R6
747 B.NE done
748
749 // Skip dropm to reuse it in the next call, when a pthread key has been created.
750 MOVW _cgo_pthread_key_created(SB), R6
751 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
752 CMP $0, R6
753 B.EQ dropm
754 MOVW (R6), R6
755 CMP $0, R6
756 B.NE done
757
758dropm:
759 MOVW $runtime·dropm(SB), R0
760 BL (R0)
761
762done:
763 // Done!
764 RET
765
766// void setg(G*); set g. for use by needm.
767TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4
768 MOVW gg+0(FP), R0
769 B setg<>(SB)
770
771TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0
772 MOVW R0, g
773
774 // Save g to thread-local storage.
775#ifdef GOOS_windows
776 B runtime·save_g(SB)
777#else
778#ifdef GOOS_openbsd
779 B runtime·save_g(SB)
780#else
781 MOVB runtime·iscgo(SB), R0
782 CMP $0, R0
783 B.EQ 2(PC)
784 B runtime·save_g(SB)
785
786 MOVW g, R0
787 RET
788#endif
789#endif
790
791TEXT runtime·emptyfunc(SB),0,$0-0
792 RET
793
794TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
795 MOVW $0, R0
796 MOVW (R0), R1
797
798// armPublicationBarrier is a native store/store barrier for ARMv7+.
799// On earlier ARM revisions, armPublicationBarrier is a no-op.
800// This will not work on SMP ARMv6 machines, if any are in use.
801// To implement publicationBarrier in sys_$GOOS_arm.s using the native
802// instructions, use:
803//
804// TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
805// B runtime·armPublicationBarrier(SB)
806//
807TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
808 MOVB runtime·goarm(SB), R11
809 CMP $7, R11
810 BLT 2(PC)
811 DMB MB_ST
812 RET
813
814// AES hashing not implemented for ARM
815TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16
816 JMP runtime·memhashFallback(SB)
817TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12
818 JMP runtime·strhashFallback(SB)
819TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12
820 JMP runtime·memhash32Fallback(SB)
821TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12
822 JMP runtime·memhash64Fallback(SB)
823
824TEXT runtime·return0(SB),NOSPLIT,$0
825 MOVW $0, R0
826 RET
827
828TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0
829 MOVW cycles+0(FP), R1
830 MOVW $0, R0
831yieldloop:
832 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K)
833 CMP R0, R1
834 B.NE 2(PC)
835 RET
836 SUB $1, R1
837 B yieldloop
838
839// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
840// Must obey the gcc calling convention.
841TEXT _cgo_topofstack(SB),NOSPLIT,$8
842 // R11 and g register are clobbered by load_g. They are
843 // callee-save in the gcc calling convention, so save them here.
844 MOVW R11, saveR11-4(SP)
845 MOVW g, saveG-8(SP)
846
847 BL runtime·load_g(SB)
848 MOVW g_m(g), R0
849 MOVW m_curg(R0), R0
850 MOVW (g_stack+stack_hi)(R0), R0
851
852 MOVW saveG-8(SP), g
853 MOVW saveR11-4(SP), R11
854 RET
855
856// The top-most function running on a goroutine
857// returns to goexit+PCQuantum.
858TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
859 MOVW R0, R0 // NOP
860 BL runtime·goexit1(SB) // does not return
861 // traceback from goexit1 must hit code range of goexit
862 MOVW R0, R0 // NOP
863
864// x -> x/1000000, x%1000000, called from Go with args, results on stack.
865TEXT runtime·usplit(SB),NOSPLIT,$0-12
866 MOVW x+0(FP), R0
867 CALL runtime·usplitR0(SB)
868 MOVW R0, q+4(FP)
869 MOVW R1, r+8(FP)
870 RET
871
872// R0, R1 = R0/1000000, R0%1000000
873TEXT runtime·usplitR0(SB),NOSPLIT,$0
874 // magic multiply to avoid software divide without available m.
875 // see output of go tool compile -S for x/1000000.
876 MOVW R0, R3
877 MOVW $1125899907, R1
878 MULLU R1, R0, (R0, R1)
879 MOVW R0>>18, R0
880 MOVW $1000000, R1
881 MULU R0, R1
882 SUB R1, R3, R1
883 RET
884
885// This is called from .init_array and follows the platform, not Go, ABI.
886TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
887 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save
888 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI
889 MOVW runtime·lastmoduledatap(SB), R1
890 MOVW R0, moduledata_next(R1)
891 MOVW R0, runtime·lastmoduledatap(SB)
892 MOVW saver11-8(SP), R11
893 MOVW saver9-4(SP), R9
894 RET
895
896TEXT ·checkASM(SB),NOSPLIT,$0-1
897 MOVW $1, R3
898 MOVB R3, ret+0(FP)
899 RET
900
901// gcWriteBarrier informs the GC about heap pointer writes.
902//
903// gcWriteBarrier does NOT follow the Go ABI. It accepts the
904// number of bytes of buffer needed in R8, and returns a pointer
905// to the buffer space in R8.
906// It clobbers condition codes.
907// It does not clobber any other general-purpose registers,
908// but may clobber others (e.g., floating point registers).
909// The act of CALLing gcWriteBarrier will clobber R14 (LR).
910TEXT gcWriteBarrier<>(SB),NOSPLIT|NOFRAME,$0
911 // Save the registers clobbered by the fast path.
912 MOVM.DB.W [R0,R1], (R13)
913retry:
914 MOVW g_m(g), R0
915 MOVW m_p(R0), R0
916 MOVW (p_wbBuf+wbBuf_next)(R0), R1
917 MOVW (p_wbBuf+wbBuf_end)(R0), R11
918 // Increment wbBuf.next position.
919 ADD R8, R1
920 // Is the buffer full?
921 CMP R11, R1
922 BHI flush
923 // Commit to the larger buffer.
924 MOVW R1, (p_wbBuf+wbBuf_next)(R0)
925 // Make return value (the original next position)
926 SUB R8, R1, R8
927 // Restore registers.
928 MOVM.IA.W (R13), [R0,R1]
929 RET
930
931flush:
932 // Save all general purpose registers since these could be
933 // clobbered by wbBufFlush and were not saved by the caller.
934 //
935 // R0 and R1 were saved at entry.
936 // R10 is g, so preserved.
937 // R11 is linker temp, so no need to save.
938 // R13 is stack pointer.
939 // R15 is PC.
940 MOVM.DB.W [R2-R9,R12], (R13)
941 // Save R14 (LR) because the fast path above doesn't save it,
942 // but needs it to RET.
943 MOVM.DB.W [R14], (R13)
944
945 CALL runtime·wbBufFlush(SB)
946
947 MOVM.IA.W (R13), [R14]
948 MOVM.IA.W (R13), [R2-R9,R12]
949 JMP retry
950
951TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
952 MOVW $4, R8
953 JMP gcWriteBarrier<>(SB)
954TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
955 MOVW $8, R8
956 JMP gcWriteBarrier<>(SB)
957TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
958 MOVW $12, R8
959 JMP gcWriteBarrier<>(SB)
960TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
961 MOVW $16, R8
962 JMP gcWriteBarrier<>(SB)
963TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
964 MOVW $20, R8
965 JMP gcWriteBarrier<>(SB)
966TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
967 MOVW $24, R8
968 JMP gcWriteBarrier<>(SB)
969TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
970 MOVW $28, R8
971 JMP gcWriteBarrier<>(SB)
972TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
973 MOVW $32, R8
974 JMP gcWriteBarrier<>(SB)
975
976// Note: these functions use a special calling convention to save generated code space.
977// Arguments are passed in registers, but the space for those arguments are allocated
978// in the caller's stack frame. These stubs write the args into that stack space and
979// then tail call to the corresponding runtime handler.
980// The tail call makes these stubs disappear in backtraces.
981TEXT runtime·panicIndex(SB),NOSPLIT,$0-8
982 MOVW R0, x+0(FP)
983 MOVW R1, y+4(FP)
984 JMP runtime·goPanicIndex(SB)
985TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8
986 MOVW R0, x+0(FP)
987 MOVW R1, y+4(FP)
988 JMP runtime·goPanicIndexU(SB)
989TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8
990 MOVW R1, x+0(FP)
991 MOVW R2, y+4(FP)
992 JMP runtime·goPanicSliceAlen(SB)
993TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8
994 MOVW R1, x+0(FP)
995 MOVW R2, y+4(FP)
996 JMP runtime·goPanicSliceAlenU(SB)
997TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8
998 MOVW R1, x+0(FP)
999 MOVW R2, y+4(FP)
1000 JMP runtime·goPanicSliceAcap(SB)
1001TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8
1002 MOVW R1, x+0(FP)
1003 MOVW R2, y+4(FP)
1004 JMP runtime·goPanicSliceAcapU(SB)
1005TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8
1006 MOVW R0, x+0(FP)
1007 MOVW R1, y+4(FP)
1008 JMP runtime·goPanicSliceB(SB)
1009TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8
1010 MOVW R0, x+0(FP)
1011 MOVW R1, y+4(FP)
1012 JMP runtime·goPanicSliceBU(SB)
1013TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8
1014 MOVW R2, x+0(FP)
1015 MOVW R3, y+4(FP)
1016 JMP runtime·goPanicSlice3Alen(SB)
1017TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8
1018 MOVW R2, x+0(FP)
1019 MOVW R3, y+4(FP)
1020 JMP runtime·goPanicSlice3AlenU(SB)
1021TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8
1022 MOVW R2, x+0(FP)
1023 MOVW R3, y+4(FP)
1024 JMP runtime·goPanicSlice3Acap(SB)
1025TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8
1026 MOVW R2, x+0(FP)
1027 MOVW R3, y+4(FP)
1028 JMP runtime·goPanicSlice3AcapU(SB)
1029TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8
1030 MOVW R1, x+0(FP)
1031 MOVW R2, y+4(FP)
1032 JMP runtime·goPanicSlice3B(SB)
1033TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8
1034 MOVW R1, x+0(FP)
1035 MOVW R2, y+4(FP)
1036 JMP runtime·goPanicSlice3BU(SB)
1037TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8
1038 MOVW R0, x+0(FP)
1039 MOVW R1, y+4(FP)
1040 JMP runtime·goPanicSlice3C(SB)
1041TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8
1042 MOVW R0, x+0(FP)
1043 MOVW R1, y+4(FP)
1044 JMP runtime·goPanicSlice3CU(SB)
1045TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8
1046 MOVW R2, x+0(FP)
1047 MOVW R3, y+4(FP)
1048 JMP runtime·goPanicSliceConvert(SB)
1049
1050// Extended versions for 64-bit indexes.
1051TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12
1052 MOVW R4, hi+0(FP)
1053 MOVW R0, lo+4(FP)
1054 MOVW R1, y+8(FP)
1055 JMP runtime·goPanicExtendIndex(SB)
1056TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12
1057 MOVW R4, hi+0(FP)
1058 MOVW R0, lo+4(FP)
1059 MOVW R1, y+8(FP)
1060 JMP runtime·goPanicExtendIndexU(SB)
1061TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12
1062 MOVW R4, hi+0(FP)
1063 MOVW R1, lo+4(FP)
1064 MOVW R2, y+8(FP)
1065 JMP runtime·goPanicExtendSliceAlen(SB)
1066TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12
1067 MOVW R4, hi+0(FP)
1068 MOVW R1, lo+4(FP)
1069 MOVW R2, y+8(FP)
1070 JMP runtime·goPanicExtendSliceAlenU(SB)
1071TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12
1072 MOVW R4, hi+0(FP)
1073 MOVW R1, lo+4(FP)
1074 MOVW R2, y+8(FP)
1075 JMP runtime·goPanicExtendSliceAcap(SB)
1076TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12
1077 MOVW R4, hi+0(FP)
1078 MOVW R1, lo+4(FP)
1079 MOVW R2, y+8(FP)
1080 JMP runtime·goPanicExtendSliceAcapU(SB)
1081TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12
1082 MOVW R4, hi+0(FP)
1083 MOVW R0, lo+4(FP)
1084 MOVW R1, y+8(FP)
1085 JMP runtime·goPanicExtendSliceB(SB)
1086TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12
1087 MOVW R4, hi+0(FP)
1088 MOVW R0, lo+4(FP)
1089 MOVW R1, y+8(FP)
1090 JMP runtime·goPanicExtendSliceBU(SB)
1091TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12
1092 MOVW R4, hi+0(FP)
1093 MOVW R2, lo+4(FP)
1094 MOVW R3, y+8(FP)
1095 JMP runtime·goPanicExtendSlice3Alen(SB)
1096TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12
1097 MOVW R4, hi+0(FP)
1098 MOVW R2, lo+4(FP)
1099 MOVW R3, y+8(FP)
1100 JMP runtime·goPanicExtendSlice3AlenU(SB)
1101TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12
1102 MOVW R4, hi+0(FP)
1103 MOVW R2, lo+4(FP)
1104 MOVW R3, y+8(FP)
1105 JMP runtime·goPanicExtendSlice3Acap(SB)
1106TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12
1107 MOVW R4, hi+0(FP)
1108 MOVW R2, lo+4(FP)
1109 MOVW R3, y+8(FP)
1110 JMP runtime·goPanicExtendSlice3AcapU(SB)
1111TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12
1112 MOVW R4, hi+0(FP)
1113 MOVW R1, lo+4(FP)
1114 MOVW R2, y+8(FP)
1115 JMP runtime·goPanicExtendSlice3B(SB)
1116TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12
1117 MOVW R4, hi+0(FP)
1118 MOVW R1, lo+4(FP)
1119 MOVW R2, y+8(FP)
1120 JMP runtime·goPanicExtendSlice3BU(SB)
1121TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12
1122 MOVW R4, hi+0(FP)
1123 MOVW R0, lo+4(FP)
1124 MOVW R1, y+8(FP)
1125 JMP runtime·goPanicExtendSlice3C(SB)
1126TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12
1127 MOVW R4, hi+0(FP)
1128 MOVW R0, lo+4(FP)
1129 MOVW R1, y+8(FP)
1130 JMP runtime·goPanicExtendSlice3CU(SB)
View as plain text