Text file
src/runtime/asm_arm.s
Documentation: runtime
1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "go_tls.h"
7#include "funcdata.h"
8#include "textflag.h"
9
10// _rt0_arm is common startup code for most ARM systems when using
11// internal linking. This is the entry point for the program from the
12// kernel for an ordinary -buildmode=exe program. The stack holds the
13// number of arguments and the C-style argv.
14TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0
15 MOVW (R13), R0 // argc
16 MOVW $4(R13), R1 // argv
17 B runtime·rt0_go(SB)
18
19// main is common startup code for most ARM systems when using
20// external linking. The C startup code will call the symbol "main"
21// passing argc and argv in the usual C ABI registers R0 and R1.
22TEXT main(SB),NOSPLIT|NOFRAME,$0
23 B runtime·rt0_go(SB)
24
25// _rt0_arm_lib is common startup code for most ARM systems when
26// using -buildmode=c-archive or -buildmode=c-shared. The linker will
27// arrange to invoke this function as a global constructor (for
28// c-archive) or when the shared library is loaded (for c-shared).
29// We expect argc and argv to be passed in the usual C ABI registers
30// R0 and R1.
31TEXT _rt0_arm_lib(SB),NOSPLIT,$104
32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example,
33 // actually cares that R11 is preserved.
34 MOVW R4, 12(R13)
35 MOVW R5, 16(R13)
36 MOVW R6, 20(R13)
37 MOVW R7, 24(R13)
38 MOVW R8, 28(R13)
39 MOVW g, 32(R13)
40 MOVW R11, 36(R13)
41
42 // Skip floating point registers on goarmsoftfp != 0.
43 MOVB runtime·goarmsoftfp(SB), R11
44 CMP $0, R11
45 BNE skipfpsave
46 MOVD F8, (40+8*0)(R13)
47 MOVD F9, (40+8*1)(R13)
48 MOVD F10, (40+8*2)(R13)
49 MOVD F11, (40+8*3)(R13)
50 MOVD F12, (40+8*4)(R13)
51 MOVD F13, (40+8*5)(R13)
52 MOVD F14, (40+8*6)(R13)
53 MOVD F15, (40+8*7)(R13)
54skipfpsave:
55 // Save argc/argv.
56 MOVW R0, _rt0_arm_lib_argc<>(SB)
57 MOVW R1, _rt0_arm_lib_argv<>(SB)
58
59 MOVW $0, g // Initialize g.
60
61 // Synchronous initialization.
62 CALL runtime·libpreinit(SB)
63
64 // Create a new thread to do the runtime initialization.
65 MOVW _cgo_sys_thread_create(SB), R2
66 CMP $0, R2
67 BEQ nocgo
68 MOVW $_rt0_arm_lib_go<>(SB), R0
69 MOVW $0, R1
70 BL (R2)
71 B rr
72nocgo:
73 MOVW $0x800000, R0 // stacksize = 8192KB
74 MOVW $_rt0_arm_lib_go<>(SB), R1 // fn
75 MOVW R0, 4(R13)
76 MOVW R1, 8(R13)
77 BL runtime·newosproc0(SB)
78rr:
79 // Restore callee-save registers and return.
80 MOVB runtime·goarmsoftfp(SB), R11
81 CMP $0, R11
82 BNE skipfprest
83 MOVD (40+8*0)(R13), F8
84 MOVD (40+8*1)(R13), F9
85 MOVD (40+8*2)(R13), F10
86 MOVD (40+8*3)(R13), F11
87 MOVD (40+8*4)(R13), F12
88 MOVD (40+8*5)(R13), F13
89 MOVD (40+8*6)(R13), F14
90 MOVD (40+8*7)(R13), F15
91skipfprest:
92 MOVW 12(R13), R4
93 MOVW 16(R13), R5
94 MOVW 20(R13), R6
95 MOVW 24(R13), R7
96 MOVW 28(R13), R8
97 MOVW 32(R13), g
98 MOVW 36(R13), R11
99 RET
100
101// _rt0_arm_lib_go initializes the Go runtime.
102// This is started in a separate thread by _rt0_arm_lib.
103TEXT _rt0_arm_lib_go<>(SB),NOSPLIT,$8
104 MOVW _rt0_arm_lib_argc<>(SB), R0
105 MOVW _rt0_arm_lib_argv<>(SB), R1
106 B runtime·rt0_go(SB)
107
108DATA _rt0_arm_lib_argc<>(SB)/4,$0
109GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4
110DATA _rt0_arm_lib_argv<>(SB)/4,$0
111GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4
112
113// using NOFRAME means do not save LR on stack.
114// argc is in R0, argv is in R1.
115TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0
116 MOVW $0xcafebabe, R12
117
118 // copy arguments forward on an even stack
119 // use R13 instead of SP to avoid linker rewriting the offsets
120 SUB $64, R13 // plenty of scratch
121 AND $~7, R13
122 MOVW R0, 60(R13) // save argc, argv away
123 MOVW R1, 64(R13)
124
125 // set up g register
126 // g is R10
127 MOVW $runtime·g0(SB), g
128 MOVW $runtime·m0(SB), R8
129
130 // save m->g0 = g0
131 MOVW g, m_g0(R8)
132 // save g->m = m0
133 MOVW R8, g_m(g)
134
135 // create istack out of the OS stack
136 // (1MB of system stack is available on iOS and Android)
137 MOVW $(-64*1024+104)(R13), R0
138 MOVW R0, g_stackguard0(g)
139 MOVW R0, g_stackguard1(g)
140 MOVW R0, (g_stack+stack_lo)(g)
141 MOVW R13, (g_stack+stack_hi)(g)
142
143 BL runtime·emptyfunc(SB) // fault if stack check is wrong
144
145#ifdef GOOS_openbsd
146 // Save g to TLS so that it is available from signal trampoline.
147 BL runtime·save_g(SB)
148#endif
149
150 BL runtime·_initcgo(SB) // will clobber R0-R3
151
152 // update stackguard after _cgo_init
153 MOVW (g_stack+stack_lo)(g), R0
154 ADD $const_stackGuard, R0
155 MOVW R0, g_stackguard0(g)
156 MOVW R0, g_stackguard1(g)
157
158 BL runtime·check(SB)
159
160 // saved argc, argv
161 MOVW 60(R13), R0
162 MOVW R0, 4(R13)
163 MOVW 64(R13), R1
164 MOVW R1, 8(R13)
165 BL runtime·args(SB)
166 BL runtime·checkgoarm(SB)
167 BL runtime·osinit(SB)
168 BL runtime·schedinit(SB)
169
170 // create a new goroutine to start program
171 SUB $8, R13
172 MOVW $runtime·mainPC(SB), R0
173 MOVW R0, 4(R13) // arg 1: fn
174 MOVW $0, R0
175 MOVW R0, 0(R13) // dummy LR
176 BL runtime·newproc(SB)
177 ADD $8, R13 // pop args and LR
178
179 // start this M
180 BL runtime·mstart(SB)
181
182 MOVW $1234, R0
183 MOVW $1000, R1
184 MOVW R0, (R1) // fail hard
185
186DATA runtime·mainPC+0(SB)/4,$runtime·main(SB)
187GLOBL runtime·mainPC(SB),RODATA,$4
188
189TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
190 // gdb won't skip this breakpoint instruction automatically,
191 // so you must manually "set $pc+=4" to skip it and continue.
192#ifdef GOOS_plan9
193 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9
194#else
195 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
196#endif
197 RET
198
199TEXT runtime·asminit(SB),NOSPLIT,$0-0
200 // disable runfast (flush-to-zero) mode of vfp if runtime.goarmsoftfp == 0
201 MOVB runtime·goarmsoftfp(SB), R11
202 CMP $0, R11
203 BNE 4(PC)
204 WORD $0xeef1ba10 // vmrs r11, fpscr
205 BIC $(1<<24), R11
206 WORD $0xeee1ba10 // vmsr fpscr, r11
207 RET
208
209TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
210 BL runtime·mstart0(SB)
211 RET // not reached
212
213/*
214 * go-routine
215 */
216
217// void gogo(Gobuf*)
218// restore state from Gobuf; longjmp
219TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4
220 MOVW buf+0(FP), R1
221 MOVW gobuf_g(R1), R0
222 MOVW 0(R0), R2 // make sure g != nil
223 B gogo<>(SB)
224
225TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0
226 BL setg<>(SB)
227 MOVW gobuf_sp(R1), R13 // restore SP==R13
228 MOVW gobuf_lr(R1), LR
229 MOVW gobuf_ctxt(R1), R7
230 MOVW $0, R11
231 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
232 MOVW R11, gobuf_lr(R1)
233 MOVW R11, gobuf_ctxt(R1)
234 MOVW gobuf_pc(R1), R11
235 CMP R11, R11 // set condition codes for == test, needed by stack split
236 B (R11)
237
238// func mcall(fn func(*g))
239// Switch to m->g0's stack, call fn(g).
240// Fn must never return. It should gogo(&g->sched)
241// to keep running g.
242TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4
243 // Save caller state in g->sched.
244 MOVW R13, (g_sched+gobuf_sp)(g)
245 MOVW LR, (g_sched+gobuf_pc)(g)
246 MOVW $0, R11
247 MOVW R11, (g_sched+gobuf_lr)(g)
248
249 // Switch to m->g0 & its stack, call fn.
250 MOVW g, R1
251 MOVW g_m(g), R8
252 MOVW m_g0(R8), R0
253 BL setg<>(SB)
254 CMP g, R1
255 B.NE 2(PC)
256 B runtime·badmcall(SB)
257 MOVW fn+0(FP), R0
258 MOVW (g_sched+gobuf_sp)(g), R13
259 SUB $8, R13
260 MOVW R1, 4(R13)
261 MOVW R0, R7
262 MOVW 0(R0), R0
263 BL (R0)
264 B runtime·badmcall2(SB)
265 RET
266
267// systemstack_switch is a dummy routine that systemstack leaves at the bottom
268// of the G stack. We need to distinguish the routine that
269// lives at the bottom of the G stack from the one that lives
270// at the top of the system stack because the one at the top of
271// the system stack terminates the stack walk (see topofstack()).
272TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0
273 MOVW $0, R0
274 BL (R0) // clobber lr to ensure push {lr} is kept
275 RET
276
277// func systemstack(fn func())
278TEXT runtime·systemstack(SB),NOSPLIT,$0-4
279 MOVW fn+0(FP), R0 // R0 = fn
280 MOVW g_m(g), R1 // R1 = m
281
282 MOVW m_gsignal(R1), R2 // R2 = gsignal
283 CMP g, R2
284 B.EQ noswitch
285
286 MOVW m_g0(R1), R2 // R2 = g0
287 CMP g, R2
288 B.EQ noswitch
289
290 MOVW m_curg(R1), R3
291 CMP g, R3
292 B.EQ switch
293
294 // Bad: g is not gsignal, not g0, not curg. What is it?
295 // Hide call from linker nosplit analysis.
296 MOVW $runtime·badsystemstack(SB), R0
297 BL (R0)
298 B runtime·abort(SB)
299
300switch:
301 // save our state in g->sched. Pretend to
302 // be systemstack_switch if the G stack is scanned.
303 BL gosave_systemstack_switch<>(SB)
304
305 // switch to g0
306 MOVW R0, R5
307 MOVW R2, R0
308 BL setg<>(SB)
309 MOVW R5, R0
310 MOVW (g_sched+gobuf_sp)(R2), R13
311
312 // call target function
313 MOVW R0, R7
314 MOVW 0(R0), R0
315 BL (R0)
316
317 // switch back to g
318 MOVW g_m(g), R1
319 MOVW m_curg(R1), R0
320 BL setg<>(SB)
321 MOVW (g_sched+gobuf_sp)(g), R13
322 MOVW $0, R3
323 MOVW R3, (g_sched+gobuf_sp)(g)
324 RET
325
326noswitch:
327 // Using a tail call here cleans up tracebacks since we won't stop
328 // at an intermediate systemstack.
329 MOVW R0, R7
330 MOVW 0(R0), R0
331 MOVW.P 4(R13), R14 // restore LR
332 B (R0)
333
334// func switchToCrashStack0(fn func())
335TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-4
336 MOVW fn+0(FP), R7 // context register
337 MOVW g_m(g), R1 // curm
338
339 // set g to gcrash
340 MOVW $runtime·gcrash(SB), R0
341 BL setg<>(SB) // g = &gcrash
342 MOVW R1, g_m(g) // g.m = curm
343 MOVW g, m_g0(R1) // curm.g0 = g
344
345 // switch to crashstack
346 MOVW (g_stack+stack_hi)(g), R1
347 SUB $(4*8), R1
348 MOVW R1, R13
349
350 // call target function
351 MOVW 0(R7), R0
352 BL (R0)
353
354 // should never return
355 CALL runtime·abort(SB)
356 UNDEF
357
358/*
359 * support for morestack
360 */
361
362// Called during function prolog when more stack is needed.
363// R3 prolog's LR
364// using NOFRAME means do not save LR on stack.
365//
366// The traceback routines see morestack on a g0 as being
367// the top of a stack (for example, morestack calling newstack
368// calling the scheduler calling newm calling gc), so we must
369// record an argument size. For that purpose, it has no arguments.
370TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
371 // Cannot grow scheduler stack (m->g0).
372 MOVW g_m(g), R8
373 MOVW m_g0(R8), R4
374
375 // Called from f.
376 // Set g->sched to context in f.
377 MOVW R13, (g_sched+gobuf_sp)(g)
378 MOVW LR, (g_sched+gobuf_pc)(g)
379 MOVW R3, (g_sched+gobuf_lr)(g)
380 MOVW R7, (g_sched+gobuf_ctxt)(g)
381
382 CMP g, R4
383 BNE 3(PC)
384 BL runtime·badmorestackg0(SB)
385 B runtime·abort(SB)
386
387 // Cannot grow signal stack (m->gsignal).
388 MOVW m_gsignal(R8), R4
389 CMP g, R4
390 BNE 3(PC)
391 BL runtime·badmorestackgsignal(SB)
392 B runtime·abort(SB)
393
394 // Called from f.
395 // Set m->morebuf to f's caller.
396 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
397 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
398 MOVW g, (m_morebuf+gobuf_g)(R8)
399
400 // Call newstack on m->g0's stack.
401 MOVW m_g0(R8), R0
402 BL setg<>(SB)
403 MOVW (g_sched+gobuf_sp)(g), R13
404 MOVW $0, R0
405 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR)
406 BL runtime·newstack(SB)
407
408 // Not reached, but make sure the return PC from the call to newstack
409 // is still in this function, and not the beginning of the next.
410 RET
411
412TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
413 // Force SPWRITE. This function doesn't actually write SP,
414 // but it is called with a special calling convention where
415 // the caller doesn't save LR on stack but passes it as a
416 // register (R3), and the unwinder currently doesn't understand.
417 // Make it SPWRITE to stop unwinding. (See issue 54332)
418 MOVW R13, R13
419
420 MOVW $0, R7
421 B runtime·morestack(SB)
422
423// reflectcall: call a function with the given argument list
424// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
425// we don't have variable-sized frames, so we use a small number
426// of constant-sized-frame functions to encode a few bits of size in the pc.
427// Caution: ugly multiline assembly macros in your future!
428
429#define DISPATCH(NAME,MAXSIZE) \
430 CMP $MAXSIZE, R0; \
431 B.HI 3(PC); \
432 MOVW $NAME(SB), R1; \
433 B (R1)
434
435TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
436 MOVW frameSize+20(FP), R0
437 DISPATCH(runtime·call16, 16)
438 DISPATCH(runtime·call32, 32)
439 DISPATCH(runtime·call64, 64)
440 DISPATCH(runtime·call128, 128)
441 DISPATCH(runtime·call256, 256)
442 DISPATCH(runtime·call512, 512)
443 DISPATCH(runtime·call1024, 1024)
444 DISPATCH(runtime·call2048, 2048)
445 DISPATCH(runtime·call4096, 4096)
446 DISPATCH(runtime·call8192, 8192)
447 DISPATCH(runtime·call16384, 16384)
448 DISPATCH(runtime·call32768, 32768)
449 DISPATCH(runtime·call65536, 65536)
450 DISPATCH(runtime·call131072, 131072)
451 DISPATCH(runtime·call262144, 262144)
452 DISPATCH(runtime·call524288, 524288)
453 DISPATCH(runtime·call1048576, 1048576)
454 DISPATCH(runtime·call2097152, 2097152)
455 DISPATCH(runtime·call4194304, 4194304)
456 DISPATCH(runtime·call8388608, 8388608)
457 DISPATCH(runtime·call16777216, 16777216)
458 DISPATCH(runtime·call33554432, 33554432)
459 DISPATCH(runtime·call67108864, 67108864)
460 DISPATCH(runtime·call134217728, 134217728)
461 DISPATCH(runtime·call268435456, 268435456)
462 DISPATCH(runtime·call536870912, 536870912)
463 DISPATCH(runtime·call1073741824, 1073741824)
464 MOVW $runtime·badreflectcall(SB), R1
465 B (R1)
466
467#define CALLFN(NAME,MAXSIZE) \
468TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
469 NO_LOCAL_POINTERS; \
470 /* copy arguments to stack */ \
471 MOVW stackArgs+8(FP), R0; \
472 MOVW stackArgsSize+12(FP), R2; \
473 ADD $4, R13, R1; \
474 CMP $0, R2; \
475 B.EQ 5(PC); \
476 MOVBU.P 1(R0), R5; \
477 MOVBU.P R5, 1(R1); \
478 SUB $1, R2, R2; \
479 B -5(PC); \
480 /* call function */ \
481 MOVW f+4(FP), R7; \
482 MOVW (R7), R0; \
483 PCDATA $PCDATA_StackMapIndex, $0; \
484 BL (R0); \
485 /* copy return values back */ \
486 MOVW stackArgsType+0(FP), R4; \
487 MOVW stackArgs+8(FP), R0; \
488 MOVW stackArgsSize+12(FP), R2; \
489 MOVW stackArgsRetOffset+16(FP), R3; \
490 ADD $4, R13, R1; \
491 ADD R3, R1; \
492 ADD R3, R0; \
493 SUB R3, R2; \
494 BL callRet<>(SB); \
495 RET
496
497// callRet copies return values back at the end of call*. This is a
498// separate function so it can allocate stack space for the arguments
499// to reflectcallmove. It does not follow the Go ABI; it expects its
500// arguments in registers.
501TEXT callRet<>(SB), NOSPLIT, $20-0
502 MOVW R4, 4(R13)
503 MOVW R0, 8(R13)
504 MOVW R1, 12(R13)
505 MOVW R2, 16(R13)
506 MOVW $0, R7
507 MOVW R7, 20(R13)
508 BL runtime·reflectcallmove(SB)
509 RET
510
511CALLFN(·call16, 16)
512CALLFN(·call32, 32)
513CALLFN(·call64, 64)
514CALLFN(·call128, 128)
515CALLFN(·call256, 256)
516CALLFN(·call512, 512)
517CALLFN(·call1024, 1024)
518CALLFN(·call2048, 2048)
519CALLFN(·call4096, 4096)
520CALLFN(·call8192, 8192)
521CALLFN(·call16384, 16384)
522CALLFN(·call32768, 32768)
523CALLFN(·call65536, 65536)
524CALLFN(·call131072, 131072)
525CALLFN(·call262144, 262144)
526CALLFN(·call524288, 524288)
527CALLFN(·call1048576, 1048576)
528CALLFN(·call2097152, 2097152)
529CALLFN(·call4194304, 4194304)
530CALLFN(·call8388608, 8388608)
531CALLFN(·call16777216, 16777216)
532CALLFN(·call33554432, 33554432)
533CALLFN(·call67108864, 67108864)
534CALLFN(·call134217728, 134217728)
535CALLFN(·call268435456, 268435456)
536CALLFN(·call536870912, 536870912)
537CALLFN(·call1073741824, 1073741824)
538
539// Save state of caller into g->sched,
540// but using fake PC from systemstack_switch.
541// Must only be called from functions with no locals ($0)
542// or else unwinding from systemstack_switch is incorrect.
543// Smashes R11.
544TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
545 MOVW $runtime·systemstack_switch(SB), R11
546 ADD $4, R11 // get past push {lr}
547 MOVW R11, (g_sched+gobuf_pc)(g)
548 MOVW R13, (g_sched+gobuf_sp)(g)
549 MOVW $0, R11
550 MOVW R11, (g_sched+gobuf_lr)(g)
551 // Assert ctxt is zero. See func save.
552 MOVW (g_sched+gobuf_ctxt)(g), R11
553 TST R11, R11
554 B.EQ 2(PC)
555 BL runtime·abort(SB)
556 RET
557
558// func asmcgocall_no_g(fn, arg unsafe.Pointer)
559// Call fn(arg) aligned appropriately for the gcc ABI.
560// Called on a system stack, and there may be no g yet (during needm).
561TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8
562 MOVW fn+0(FP), R1
563 MOVW arg+4(FP), R0
564 MOVW R13, R2
565 SUB $32, R13
566 BIC $0x7, R13 // alignment for gcc ABI
567 MOVW R2, 8(R13)
568 BL (R1)
569 MOVW 8(R13), R2
570 MOVW R2, R13
571 RET
572
573// func asmcgocall(fn, arg unsafe.Pointer) int32
574// Call fn(arg) on the scheduler stack,
575// aligned appropriately for the gcc ABI.
576// See cgocall.go for more details.
577TEXT ·asmcgocall(SB),NOSPLIT,$0-12
578 MOVW fn+0(FP), R1
579 MOVW arg+4(FP), R0
580
581 MOVW R13, R2
582 CMP $0, g
583 BEQ nosave
584 MOVW g, R4
585
586 // Figure out if we need to switch to m->g0 stack.
587 // We get called to create new OS threads too, and those
588 // come in on the m->g0 stack already. Or we might already
589 // be on the m->gsignal stack.
590 MOVW g_m(g), R8
591 MOVW m_gsignal(R8), R3
592 CMP R3, g
593 BEQ nosave
594 MOVW m_g0(R8), R3
595 CMP R3, g
596 BEQ nosave
597 BL gosave_systemstack_switch<>(SB)
598 MOVW R0, R5
599 MOVW R3, R0
600 BL setg<>(SB)
601 MOVW R5, R0
602 MOVW (g_sched+gobuf_sp)(g), R13
603
604 // Now on a scheduling stack (a pthread-created stack).
605 SUB $24, R13
606 BIC $0x7, R13 // alignment for gcc ABI
607 MOVW R4, 20(R13) // save old g
608 MOVW (g_stack+stack_hi)(R4), R4
609 SUB R2, R4
610 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
611 BL (R1)
612
613 // Restore registers, g, stack pointer.
614 MOVW R0, R5
615 MOVW 20(R13), R0
616 BL setg<>(SB)
617 MOVW (g_stack+stack_hi)(g), R1
618 MOVW 16(R13), R2
619 SUB R2, R1
620 MOVW R5, R0
621 MOVW R1, R13
622
623 MOVW R0, ret+8(FP)
624 RET
625
626nosave:
627 // Running on a system stack, perhaps even without a g.
628 // Having no g can happen during thread creation or thread teardown
629 // (see needm/dropm on Solaris, for example).
630 // This code is like the above sequence but without saving/restoring g
631 // and without worrying about the stack moving out from under us
632 // (because we're on a system stack, not a goroutine stack).
633 // The above code could be used directly if already on a system stack,
634 // but then the only path through this code would be a rare case on Solaris.
635 // Using this code for all "already on system stack" calls exercises it more,
636 // which should help keep it correct.
637 SUB $24, R13
638 BIC $0x7, R13 // alignment for gcc ABI
639 // save null g in case someone looks during debugging.
640 MOVW $0, R4
641 MOVW R4, 20(R13)
642 MOVW R2, 16(R13) // Save old stack pointer.
643 BL (R1)
644 // Restore stack pointer.
645 MOVW 16(R13), R2
646 MOVW R2, R13
647 MOVW R0, ret+8(FP)
648 RET
649
650// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
651// See cgocall.go for more details.
652TEXT ·cgocallback(SB),NOSPLIT,$12-12
653 NO_LOCAL_POINTERS
654
655 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
656 // It is used to dropm while thread is exiting.
657 MOVW fn+0(FP), R1
658 CMP $0, R1
659 B.NE loadg
660 // Restore the g from frame.
661 MOVW frame+4(FP), g
662 B dropm
663
664loadg:
665 // Load m and g from thread-local storage.
666#ifdef GOOS_openbsd
667 BL runtime·load_g(SB)
668#else
669 MOVB runtime·iscgo(SB), R0
670 CMP $0, R0
671 BL.NE runtime·load_g(SB)
672#endif
673
674 // If g is nil, Go did not create the current thread,
675 // or if this thread never called into Go on pthread platforms.
676 // Call needm to obtain one for temporary use.
677 // In this case, we're running on the thread stack, so there's
678 // lots of space, but the linker doesn't know. Hide the call from
679 // the linker analysis by using an indirect call.
680 CMP $0, g
681 B.EQ needm
682
683 MOVW g_m(g), R8
684 MOVW R8, savedm-4(SP)
685 B havem
686
687needm:
688 MOVW g, savedm-4(SP) // g is zero, so is m.
689 MOVW $runtime·needAndBindM(SB), R0
690 BL (R0)
691
692 // Set m->g0->sched.sp = SP, so that if a panic happens
693 // during the function we are about to execute, it will
694 // have a valid SP to run on the g0 stack.
695 // The next few lines (after the havem label)
696 // will save this SP onto the stack and then write
697 // the same SP back to m->sched.sp. That seems redundant,
698 // but if an unrecovered panic happens, unwindm will
699 // restore the g->sched.sp from the stack location
700 // and then systemstack will try to use it. If we don't set it here,
701 // that restored SP will be uninitialized (typically 0) and
702 // will not be usable.
703 MOVW g_m(g), R8
704 MOVW m_g0(R8), R3
705 MOVW R13, (g_sched+gobuf_sp)(R3)
706
707havem:
708 // Now there's a valid m, and we're running on its m->g0.
709 // Save current m->g0->sched.sp on stack and then set it to SP.
710 // Save current sp in m->g0->sched.sp in preparation for
711 // switch back to m->curg stack.
712 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP).
713 MOVW m_g0(R8), R3
714 MOVW (g_sched+gobuf_sp)(R3), R4
715 MOVW R4, savedsp-12(SP) // must match frame size
716 MOVW R13, (g_sched+gobuf_sp)(R3)
717
718 // Switch to m->curg stack and call runtime.cgocallbackg.
719 // Because we are taking over the execution of m->curg
720 // but *not* resuming what had been running, we need to
721 // save that information (m->curg->sched) so we can restore it.
722 // We can restore m->curg->sched.sp easily, because calling
723 // runtime.cgocallbackg leaves SP unchanged upon return.
724 // To save m->curg->sched.pc, we push it onto the curg stack and
725 // open a frame the same size as cgocallback's g0 frame.
726 // Once we switch to the curg stack, the pushed PC will appear
727 // to be the return PC of cgocallback, so that the traceback
728 // will seamlessly trace back into the earlier calls.
729 MOVW m_curg(R8), R0
730 BL setg<>(SB)
731 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
732 MOVW (g_sched+gobuf_pc)(g), R5
733 MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size
734 // Gather our arguments into registers.
735 MOVW fn+0(FP), R1
736 MOVW frame+4(FP), R2
737 MOVW ctxt+8(FP), R3
738 MOVW $-(12+4)(R4), R13 // switch stack; must match frame size
739 MOVW R1, 4(R13)
740 MOVW R2, 8(R13)
741 MOVW R3, 12(R13)
742 BL runtime·cgocallbackg(SB)
743
744 // Restore g->sched (== m->curg->sched) from saved values.
745 MOVW 0(R13), R5
746 MOVW R5, (g_sched+gobuf_pc)(g)
747 MOVW $(12+4)(R13), R4 // must match frame size
748 MOVW R4, (g_sched+gobuf_sp)(g)
749
750 // Switch back to m->g0's stack and restore m->g0->sched.sp.
751 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
752 // so we do not have to restore it.)
753 MOVW g_m(g), R8
754 MOVW m_g0(R8), R0
755 BL setg<>(SB)
756 MOVW (g_sched+gobuf_sp)(g), R13
757 MOVW savedsp-12(SP), R4 // must match frame size
758 MOVW R4, (g_sched+gobuf_sp)(g)
759
760 // If the m on entry was nil, we called needm above to borrow an m,
761 // 1. for the duration of the call on non-pthread platforms,
762 // 2. or the duration of the C thread alive on pthread platforms.
763 // If the m on entry wasn't nil,
764 // 1. the thread might be a Go thread,
765 // 2. or it wasn't the first call from a C thread on pthread platforms,
766 // since then we skip dropm to reuse the m in the first call.
767 MOVW savedm-4(SP), R6
768 CMP $0, R6
769 B.NE done
770
771 // Skip dropm to reuse it in the next call, when a pthread key has been created.
772 MOVW _cgo_pthread_key_created(SB), R6
773 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
774 CMP $0, R6
775 B.EQ dropm
776 MOVW (R6), R6
777 CMP $0, R6
778 B.NE done
779
780dropm:
781 MOVW $runtime·dropm(SB), R0
782 BL (R0)
783
784done:
785 // Done!
786 RET
787
788// void setg(G*); set g. for use by needm.
789TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4
790 MOVW gg+0(FP), R0
791 B setg<>(SB)
792
793TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0
794 MOVW R0, g
795
796 // Save g to thread-local storage.
797#ifdef GOOS_openbsd
798 B runtime·save_g(SB)
799#else
800 MOVB runtime·iscgo(SB), R0
801 CMP $0, R0
802 B.EQ 2(PC)
803 B runtime·save_g(SB)
804
805 MOVW g, R0
806 RET
807#endif
808
809TEXT runtime·emptyfunc(SB),0,$0-0
810 RET
811
812TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
813 MOVW $0, R0
814 MOVW (R0), R1
815
816// armPublicationBarrier is a native store/store barrier for ARMv7+.
817// On earlier ARM revisions, armPublicationBarrier is a no-op.
818// This will not work on SMP ARMv6 machines, if any are in use.
819// To implement publicationBarrier in sys_$GOOS_arm.s using the native
820// instructions, use:
821//
822// TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
823// B runtime·armPublicationBarrier(SB)
824//
825TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
826 MOVB runtime·goarm(SB), R11
827 CMP $7, R11
828 BLT 2(PC)
829 DMB MB_ST
830 RET
831
832// AES hashing not implemented for ARM
833TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-16
834 JMP runtime·memhashFallback(SB)
835TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-12
836 JMP runtime·strhashFallback(SB)
837TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12
838 JMP runtime·memhash32Fallback(SB)
839TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12
840 JMP runtime·memhash64Fallback(SB)
841
842TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0
843 MOVW cycles+0(FP), R1
844 MOVW $0, R0
845yieldloop:
846 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K)
847 CMP R0, R1
848 B.NE 2(PC)
849 RET
850 SUB $1, R1
851 B yieldloop
852
853// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
854// Must obey the gcc calling convention.
855TEXT _cgo_topofstack(SB),NOSPLIT,$8
856 // R11 and g register are clobbered by load_g. They are
857 // callee-save in the gcc calling convention, so save them here.
858 MOVW R11, saveR11-4(SP)
859 MOVW g, saveG-8(SP)
860
861 BL runtime·load_g(SB)
862 MOVW g_m(g), R0
863 MOVW m_curg(R0), R0
864 MOVW (g_stack+stack_hi)(R0), R0
865
866 MOVW saveG-8(SP), g
867 MOVW saveR11-4(SP), R11
868 RET
869
870// The top-most function running on a goroutine
871// returns to goexit+PCQuantum.
872TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
873 MOVW R0, R0 // NOP
874 BL runtime·goexit1(SB) // does not return
875 // traceback from goexit1 must hit code range of goexit
876 MOVW R0, R0 // NOP
877
878// x -> x/1000000, x%1000000, called from Go with args, results on stack.
879TEXT runtime·usplit(SB),NOSPLIT,$0-12
880 MOVW x+0(FP), R0
881 CALL runtime·usplitR0(SB)
882 MOVW R0, q+4(FP)
883 MOVW R1, r+8(FP)
884 RET
885
886// R0, R1 = R0/1000000, R0%1000000
887TEXT runtime·usplitR0(SB),NOSPLIT,$0
888 // magic multiply to avoid software divide without available m.
889 // see output of go tool compile -S for x/1000000.
890 MOVW R0, R3
891 MOVW $1125899907, R1
892 MULLU R1, R0, (R0, R1)
893 MOVW R0>>18, R0
894 MOVW $1000000, R1
895 MULU R0, R1
896 SUB R1, R3, R1
897 RET
898
899// This is called from .init_array and follows the platform, not Go, ABI.
900TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
901 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save
902 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI
903 MOVW runtime·lastmoduledatap(SB), R1
904 MOVW R0, moduledata_next(R1)
905 MOVW R0, runtime·lastmoduledatap(SB)
906 MOVW saver11-8(SP), R11
907 MOVW saver9-4(SP), R9
908 RET
909
910TEXT ·checkASM(SB),NOSPLIT,$0-1
911 MOVW $1, R3
912 MOVB R3, ret+0(FP)
913 RET
914
915// gcWriteBarrier informs the GC about heap pointer writes.
916//
917// gcWriteBarrier does NOT follow the Go ABI. It accepts the
918// number of bytes of buffer needed in R8, and returns a pointer
919// to the buffer space in R8.
920// It clobbers condition codes.
921// It does not clobber any other general-purpose registers,
922// but may clobber others (e.g., floating point registers).
923// The act of CALLing gcWriteBarrier will clobber R14 (LR).
924TEXT gcWriteBarrier<>(SB),NOSPLIT|NOFRAME,$0
925 // Save the registers clobbered by the fast path.
926 MOVM.DB.W [R0,R1], (R13)
927retry:
928 MOVW g_m(g), R0
929 MOVW m_p(R0), R0
930 MOVW (p_wbBuf+wbBuf_next)(R0), R1
931 MOVW (p_wbBuf+wbBuf_end)(R0), R11
932 // Increment wbBuf.next position.
933 ADD R8, R1
934 // Is the buffer full?
935 CMP R11, R1
936 BHI flush
937 // Commit to the larger buffer.
938 MOVW R1, (p_wbBuf+wbBuf_next)(R0)
939 // Make return value (the original next position)
940 SUB R8, R1, R8
941 // Restore registers.
942 MOVM.IA.W (R13), [R0,R1]
943 RET
944
945flush:
946 // Save all general purpose registers since these could be
947 // clobbered by wbBufFlush and were not saved by the caller.
948 //
949 // R0 and R1 were saved at entry.
950 // R10 is g, so preserved.
951 // R11 is linker temp, so no need to save.
952 // R13 is stack pointer.
953 // R15 is PC.
954 MOVM.DB.W [R2-R9,R12], (R13)
955 // Save R14 (LR) because the fast path above doesn't save it,
956 // but needs it to RET.
957 MOVM.DB.W [R14], (R13)
958
959 CALL runtime·wbBufFlush(SB)
960
961 MOVM.IA.W (R13), [R14]
962 MOVM.IA.W (R13), [R2-R9,R12]
963 JMP retry
964
965TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
966 MOVW $4, R8
967 JMP gcWriteBarrier<>(SB)
968TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
969 MOVW $8, R8
970 JMP gcWriteBarrier<>(SB)
971TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
972 MOVW $12, R8
973 JMP gcWriteBarrier<>(SB)
974TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
975 MOVW $16, R8
976 JMP gcWriteBarrier<>(SB)
977TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
978 MOVW $20, R8
979 JMP gcWriteBarrier<>(SB)
980TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
981 MOVW $24, R8
982 JMP gcWriteBarrier<>(SB)
983TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
984 MOVW $28, R8
985 JMP gcWriteBarrier<>(SB)
986TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
987 MOVW $32, R8
988 JMP gcWriteBarrier<>(SB)
989
990TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$72-0
991 NO_LOCAL_POINTERS
992 // Save all int registers that could have an index in them.
993 // They may be pointers, but if they are they are dead.
994 MOVW R0, 12(R13)
995 MOVW R1, 16(R13)
996 MOVW R2, 20(R13)
997 MOVW R3, 24(R13)
998 MOVW R4, 28(R13)
999 MOVW R5, 32(R13)
1000 MOVW R6, 36(R13)
1001 MOVW R7, 40(R13)
1002 MOVW R8, 44(R13)
1003 MOVW R9, 48(R13)
1004 // skip R10 aka G @ 52(R13)
1005 // skip R11 aka tmp @ 56(R13)
1006 MOVW R12, 60(R13)
1007 // skip R13 aka SP @ 64(R13)
1008 MOVW R14, 68(R13)
1009 // skip R15 aka PC @ 72(R13)
1010
1011 MOVW R14, 4(R13) // PC immediately after call to panicBounds
1012 ADD $12, R13, R0 // pointer to save area
1013 MOVW R0, 8(R13)
1014 CALL runtime·panicBounds32<ABIInternal>(SB)
1015 RET
1016
1017TEXT runtime·panicExtend<ABIInternal>(SB),NOSPLIT,$72-0
1018 NO_LOCAL_POINTERS
1019 // Save all int registers that could have an index in them.
1020 // They may be pointers, but if they are they are dead.
1021 MOVW R0, 12(R13)
1022 MOVW R1, 16(R13)
1023 MOVW R2, 20(R13)
1024 MOVW R3, 24(R13)
1025 MOVW R4, 28(R13)
1026 MOVW R5, 32(R13)
1027 MOVW R6, 36(R13)
1028 MOVW R7, 40(R13)
1029 MOVW R8, 44(R13)
1030 MOVW R9, 48(R13)
1031 // skip R10 aka G @ 52(R13)
1032 // skip R11 aka tmp @ 56(R13)
1033 MOVW R12, 60(R13)
1034 // skip R13 aka SP @ 64(R13)
1035 // skip R14 aka LR @ 68(R13)
1036 // skip R15 aka PC @ 72(R13)
1037
1038 MOVW R14, 4(R13) // PC immediately after call to panicExtend
1039 ADD $12, R13, R0 // pointer to save area
1040 MOVW R0, 8(R13)
1041 CALL runtime·panicBounds32X<ABIInternal>(SB)
1042 RET
View as plain text