1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7
8TEXT ·Casint32(SB), NOSPLIT, $0-17
9 B ·Cas(SB)
10
11TEXT ·Casint64(SB), NOSPLIT, $0-25
12 B ·Cas64(SB)
13
14TEXT ·Casuintptr(SB), NOSPLIT, $0-25
15 B ·Cas64(SB)
16
17TEXT ·CasRel(SB), NOSPLIT, $0-17
18 B ·Cas(SB)
19
20TEXT ·Loadint32(SB), NOSPLIT, $0-12
21 B ·Load(SB)
22
23TEXT ·Loadint64(SB), NOSPLIT, $0-16
24 B ·Load64(SB)
25
26TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
27 B ·Load64(SB)
28
29TEXT ·Loaduint(SB), NOSPLIT, $0-16
30 B ·Load64(SB)
31
32TEXT ·Storeint32(SB), NOSPLIT, $0-12
33 B ·Store(SB)
34
35TEXT ·Storeint64(SB), NOSPLIT, $0-16
36 B ·Store64(SB)
37
38TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
39 B ·Store64(SB)
40
41TEXT ·Xaddint32(SB), NOSPLIT, $0-20
42 B ·Xadd(SB)
43
44TEXT ·Xaddint64(SB), NOSPLIT, $0-24
45 B ·Xadd64(SB)
46
47TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
48 B ·Xadd64(SB)
49
50TEXT ·Casp1(SB), NOSPLIT, $0-25
51 B ·Cas64(SB)
52
53// uint32 ·Load(uint32 volatile* addr)
54TEXT ·Load(SB),NOSPLIT,$0-12
55 MOVD ptr+0(FP), R0
56 LDARW (R0), R0
57 MOVW R0, ret+8(FP)
58 RET
59
60// uint8 ·Load8(uint8 volatile* addr)
61TEXT ·Load8(SB),NOSPLIT,$0-9
62 MOVD ptr+0(FP), R0
63 LDARB (R0), R0
64 MOVB R0, ret+8(FP)
65 RET
66
67// uint64 ·Load64(uint64 volatile* addr)
68TEXT ·Load64(SB),NOSPLIT,$0-16
69 MOVD ptr+0(FP), R0
70 LDAR (R0), R0
71 MOVD R0, ret+8(FP)
72 RET
73
74// void *·Loadp(void *volatile *addr)
75TEXT ·Loadp(SB),NOSPLIT,$0-16
76 MOVD ptr+0(FP), R0
77 LDAR (R0), R0
78 MOVD R0, ret+8(FP)
79 RET
80
81// uint32 ·LoadAcq(uint32 volatile* addr)
82TEXT ·LoadAcq(SB),NOSPLIT,$0-12
83 B ·Load(SB)
84
85// uint64 ·LoadAcquintptr(uint64 volatile* addr)
86TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
87 B ·Load64(SB)
88
89// uintptr ·LoadAcq64(uintptr volatile* addr)
90TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
91 B ·Load64(SB)
92
93TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
94 B ·Store64(SB)
95
96TEXT ·StoreRel(SB), NOSPLIT, $0-12
97 B ·Store(SB)
98
99TEXT ·StoreRel64(SB), NOSPLIT, $0-16
100 B ·Store64(SB)
101
102TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
103 B ·Store64(SB)
104
105TEXT ·Store(SB), NOSPLIT, $0-12
106 MOVD ptr+0(FP), R0
107 MOVW val+8(FP), R1
108 STLRW R1, (R0)
109 RET
110
111TEXT ·Store8(SB), NOSPLIT, $0-9
112 MOVD ptr+0(FP), R0
113 MOVB val+8(FP), R1
114 STLRB R1, (R0)
115 RET
116
117TEXT ·Store64(SB), NOSPLIT, $0-16
118 MOVD ptr+0(FP), R0
119 MOVD val+8(FP), R1
120 STLR R1, (R0)
121 RET
122
123// uint32 Xchg(ptr *uint32, new uint32)
124// Atomically:
125// old := *ptr;
126// *ptr = new;
127// return old;
128TEXT ·Xchg(SB), NOSPLIT, $0-20
129 MOVD ptr+0(FP), R0
130 MOVW new+8(FP), R1
131#ifndef GOARM64_LSE
132 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
133 CBZ R4, load_store_loop
134#endif
135 SWPALW R1, (R0), R2
136 MOVW R2, ret+16(FP)
137 RET
138#ifndef GOARM64_LSE
139load_store_loop:
140 LDAXRW (R0), R2
141 STLXRW R1, (R0), R3
142 CBNZ R3, load_store_loop
143 MOVW R2, ret+16(FP)
144 RET
145#endif
146
147// uint64 Xchg64(ptr *uint64, new uint64)
148// Atomically:
149// old := *ptr;
150// *ptr = new;
151// return old;
152TEXT ·Xchg64(SB), NOSPLIT, $0-24
153 MOVD ptr+0(FP), R0
154 MOVD new+8(FP), R1
155#ifndef GOARM64_LSE
156 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
157 CBZ R4, load_store_loop
158#endif
159 SWPALD R1, (R0), R2
160 MOVD R2, ret+16(FP)
161 RET
162#ifndef GOARM64_LSE
163load_store_loop:
164 LDAXR (R0), R2
165 STLXR R1, (R0), R3
166 CBNZ R3, load_store_loop
167 MOVD R2, ret+16(FP)
168 RET
169#endif
170
171// bool Cas(uint32 *ptr, uint32 old, uint32 new)
172// Atomically:
173// if(*val == old){
174// *val = new;
175// return 1;
176// } else
177// return 0;
178TEXT ·Cas(SB), NOSPLIT, $0-17
179 MOVD ptr+0(FP), R0
180 MOVW old+8(FP), R1
181 MOVW new+12(FP), R2
182#ifndef GOARM64_LSE
183 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
184 CBZ R4, load_store_loop
185#endif
186 MOVD R1, R3
187 CASALW R3, (R0), R2
188 CMP R1, R3
189 CSET EQ, R0
190 MOVB R0, ret+16(FP)
191 RET
192#ifndef GOARM64_LSE
193load_store_loop:
194 LDAXRW (R0), R3
195 CMPW R1, R3
196 BNE ok
197 STLXRW R2, (R0), R3
198 CBNZ R3, load_store_loop
199ok:
200 CSET EQ, R0
201 MOVB R0, ret+16(FP)
202 RET
203#endif
204
205// bool ·Cas64(uint64 *ptr, uint64 old, uint64 new)
206// Atomically:
207// if(*val == old){
208// *val = new;
209// return 1;
210// } else {
211// return 0;
212// }
213TEXT ·Cas64(SB), NOSPLIT, $0-25
214 MOVD ptr+0(FP), R0
215 MOVD old+8(FP), R1
216 MOVD new+16(FP), R2
217#ifndef GOARM64_LSE
218 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
219 CBZ R4, load_store_loop
220#endif
221 MOVD R1, R3
222 CASALD R3, (R0), R2
223 CMP R1, R3
224 CSET EQ, R0
225 MOVB R0, ret+24(FP)
226 RET
227#ifndef GOARM64_LSE
228load_store_loop:
229 LDAXR (R0), R3
230 CMP R1, R3
231 BNE ok
232 STLXR R2, (R0), R3
233 CBNZ R3, load_store_loop
234ok:
235 CSET EQ, R0
236 MOVB R0, ret+24(FP)
237 RET
238#endif
239
240// uint32 xadd(uint32 volatile *ptr, int32 delta)
241// Atomically:
242// *val += delta;
243// return *val;
244TEXT ·Xadd(SB), NOSPLIT, $0-20
245 MOVD ptr+0(FP), R0
246 MOVW delta+8(FP), R1
247#ifndef GOARM64_LSE
248 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
249 CBZ R4, load_store_loop
250#endif
251 LDADDALW R1, (R0), R2
252 ADD R1, R2
253 MOVW R2, ret+16(FP)
254 RET
255#ifndef GOARM64_LSE
256load_store_loop:
257 LDAXRW (R0), R2
258 ADDW R2, R1, R2
259 STLXRW R2, (R0), R3
260 CBNZ R3, load_store_loop
261 MOVW R2, ret+16(FP)
262 RET
263#endif
264
265// uint64 Xadd64(uint64 volatile *ptr, int64 delta)
266// Atomically:
267// *val += delta;
268// return *val;
269TEXT ·Xadd64(SB), NOSPLIT, $0-24
270 MOVD ptr+0(FP), R0
271 MOVD delta+8(FP), R1
272#ifndef GOARM64_LSE
273 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
274 CBZ R4, load_store_loop
275#endif
276 LDADDALD R1, (R0), R2
277 ADD R1, R2
278 MOVD R2, ret+16(FP)
279 RET
280#ifndef GOARM64_LSE
281load_store_loop:
282 LDAXR (R0), R2
283 ADD R2, R1, R2
284 STLXR R2, (R0), R3
285 CBNZ R3, load_store_loop
286 MOVD R2, ret+16(FP)
287 RET
288#endif
289
290TEXT ·Xchgint32(SB), NOSPLIT, $0-20
291 B ·Xchg(SB)
292
293TEXT ·Xchgint64(SB), NOSPLIT, $0-24
294 B ·Xchg64(SB)
295
296TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
297 B ·Xchg64(SB)
298
299TEXT ·And8(SB), NOSPLIT, $0-9
300 MOVD ptr+0(FP), R0
301 MOVB val+8(FP), R1
302#ifndef GOARM64_LSE
303 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
304 CBZ R4, load_store_loop
305#endif
306 MVN R1, R2
307 LDCLRALB R2, (R0), R3
308 RET
309#ifndef GOARM64_LSE
310load_store_loop:
311 LDAXRB (R0), R2
312 AND R1, R2
313 STLXRB R2, (R0), R3
314 CBNZ R3, load_store_loop
315 RET
316#endif
317
318TEXT ·Or8(SB), NOSPLIT, $0-9
319 MOVD ptr+0(FP), R0
320 MOVB val+8(FP), R1
321#ifndef GOARM64_LSE
322 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
323 CBZ R4, load_store_loop
324#endif
325 LDORALB R1, (R0), R2
326 RET
327#ifndef GOARM64_LSE
328load_store_loop:
329 LDAXRB (R0), R2
330 ORR R1, R2
331 STLXRB R2, (R0), R3
332 CBNZ R3, load_store_loop
333 RET
334#endif
335
336// func And(addr *uint32, v uint32)
337TEXT ·And(SB), NOSPLIT, $0-12
338 MOVD ptr+0(FP), R0
339 MOVW val+8(FP), R1
340#ifndef GOARM64_LSE
341 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
342 CBZ R4, load_store_loop
343#endif
344 MVN R1, R2
345 LDCLRALW R2, (R0), R3
346 RET
347#ifndef GOARM64_LSE
348load_store_loop:
349 LDAXRW (R0), R2
350 AND R1, R2
351 STLXRW R2, (R0), R3
352 CBNZ R3, load_store_loop
353 RET
354#endif
355
356// func Or(addr *uint32, v uint32)
357TEXT ·Or(SB), NOSPLIT, $0-12
358 MOVD ptr+0(FP), R0
359 MOVW val+8(FP), R1
360#ifndef GOARM64_LSE
361 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
362 CBZ R4, load_store_loop
363#endif
364 LDORALW R1, (R0), R2
365 RET
366#ifndef GOARM64_LSE
367load_store_loop:
368 LDAXRW (R0), R2
369 ORR R1, R2
370 STLXRW R2, (R0), R3
371 CBNZ R3, load_store_loop
372 RET
373#endif
374
375// func Or32(addr *uint32, v uint32) old uint32
376TEXT ·Or32(SB), NOSPLIT, $0-20
377 MOVD ptr+0(FP), R0
378 MOVW val+8(FP), R1
379#ifndef GOARM64_LSE
380 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
381 CBZ R4, load_store_loop
382#endif
383 LDORALW R1, (R0), R2
384 MOVD R2, ret+16(FP)
385 RET
386#ifndef GOARM64_LSE
387load_store_loop:
388 LDAXRW (R0), R2
389 ORR R1, R2, R3
390 STLXRW R3, (R0), R4
391 CBNZ R4, load_store_loop
392 MOVD R2, ret+16(FP)
393 RET
394#endif
395
396// func And32(addr *uint32, v uint32) old uint32
397TEXT ·And32(SB), NOSPLIT, $0-20
398 MOVD ptr+0(FP), R0
399 MOVW val+8(FP), R1
400#ifndef GOARM64_LSE
401 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
402 CBZ R4, load_store_loop
403#endif
404 MVN R1, R2
405 LDCLRALW R2, (R0), R3
406 MOVD R3, ret+16(FP)
407 RET
408#ifndef GOARM64_LSE
409load_store_loop:
410 LDAXRW (R0), R2
411 AND R1, R2, R3
412 STLXRW R3, (R0), R4
413 CBNZ R4, load_store_loop
414 MOVD R2, ret+16(FP)
415 RET
416#endif
417
418// func Or64(addr *uint64, v uint64) old uint64
419TEXT ·Or64(SB), NOSPLIT, $0-24
420 MOVD ptr+0(FP), R0
421 MOVD val+8(FP), R1
422#ifndef GOARM64_LSE
423 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
424 CBZ R4, load_store_loop
425#endif
426 LDORALD R1, (R0), R2
427 MOVD R2, ret+16(FP)
428 RET
429#ifndef GOARM64_LSE
430load_store_loop:
431 LDAXR (R0), R2
432 ORR R1, R2, R3
433 STLXR R3, (R0), R4
434 CBNZ R4, load_store_loop
435 MOVD R2, ret+16(FP)
436 RET
437#endif
438
439// func And64(addr *uint64, v uint64) old uint64
440TEXT ·And64(SB), NOSPLIT, $0-24
441 MOVD ptr+0(FP), R0
442 MOVD val+8(FP), R1
443#ifndef GOARM64_LSE
444 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
445 CBZ R4, load_store_loop
446#endif
447 MVN R1, R2
448 LDCLRALD R2, (R0), R3
449 MOVD R3, ret+16(FP)
450 RET
451#ifndef GOARM64_LSE
452load_store_loop:
453 LDAXR (R0), R2
454 AND R1, R2, R3
455 STLXR R3, (R0), R4
456 CBNZ R4, load_store_loop
457 MOVD R2, ret+16(FP)
458 RET
459#endif
460
461// func Anduintptr(addr *uintptr, v uintptr) old uintptr
462TEXT ·Anduintptr(SB), NOSPLIT, $0-24
463 B ·And64(SB)
464
465// func Oruintptr(addr *uintptr, v uintptr) old uintptr
466TEXT ·Oruintptr(SB), NOSPLIT, $0-24
467 B ·Or64(SB)
View as plain text