1// Copyright 2022 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7
8// func Cas(ptr *int32, old, new int32) bool
9// Atomically:
10// if *ptr == old {
11// *ptr = new
12// return true
13// } else {
14// return false
15// }
16TEXT ·Cas(SB), NOSPLIT, $0-17
17 MOVV ptr+0(FP), R4
18 MOVW old+8(FP), R5
19 MOVW new+12(FP), R6
20
21 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
22 BEQ R8, ll_sc
23 MOVV R5, R7 // backup old value
24 AMCASDBW R6, (R4), R5
25 BNE R7, R5, cas_fail0
26 MOVV $1, R4
27 MOVB R4, ret+16(FP)
28 RET
29cas_fail0:
30 MOVB R0, ret+16(FP)
31 RET
32
33ll_sc:
34 // Implemented using the ll-sc instruction pair
35 DBAR $0x14 // LoadAcquire barrier
36cas_again:
37 MOVV R6, R7
38 LL (R4), R8
39 BNE R5, R8, cas_fail1
40 SC R7, (R4)
41 BEQ R7, cas_again
42 MOVV $1, R4
43 MOVB R4, ret+16(FP)
44 DBAR $0x12 // StoreRelease barrier
45 RET
46cas_fail1:
47 MOVV $0, R4
48 JMP -4(PC)
49
50// func Cas64(ptr *uint64, old, new uint64) bool
51// Atomically:
52// if *ptr == old {
53// *ptr = new
54// return true
55// } else {
56// return false
57// }
58TEXT ·Cas64(SB), NOSPLIT, $0-25
59 MOVV ptr+0(FP), R4
60 MOVV old+8(FP), R5
61 MOVV new+16(FP), R6
62
63 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
64 BEQ R8, ll_sc_64
65 MOVV R5, R7 // backup old value
66 AMCASDBV R6, (R4), R5
67 BNE R7, R5, cas64_fail0
68 MOVV $1, R4
69 MOVB R4, ret+24(FP)
70 RET
71cas64_fail0:
72 MOVB R0, ret+24(FP)
73 RET
74
75ll_sc_64:
76 // Implemented using the ll-sc instruction pair
77 DBAR $0x14
78cas64_again:
79 MOVV R6, R7
80 LLV (R4), R8
81 BNE R5, R8, cas64_fail1
82 SCV R7, (R4)
83 BEQ R7, cas64_again
84 MOVV $1, R4
85 MOVB R4, ret+24(FP)
86 DBAR $0x12
87 RET
88cas64_fail1:
89 MOVV $0, R4
90 JMP -4(PC)
91
92TEXT ·Casint32(SB),NOSPLIT,$0-17
93 JMP ·Cas(SB)
94
95TEXT ·Casint64(SB),NOSPLIT,$0-25
96 JMP ·Cas64(SB)
97
98TEXT ·Casuintptr(SB), NOSPLIT, $0-25
99 JMP ·Cas64(SB)
100
101TEXT ·CasRel(SB), NOSPLIT, $0-17
102 JMP ·Cas(SB)
103
104TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
105 JMP ·Load64(SB)
106
107TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
108 JMP ·Load64(SB)
109
110TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
111 JMP ·Store64(SB)
112
113TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
114 JMP ·Xadd64(SB)
115
116TEXT ·Loadint64(SB), NOSPLIT, $0-16
117 JMP ·Load64(SB)
118
119TEXT ·Xaddint32(SB),NOSPLIT,$0-20
120 JMP ·Xadd(SB)
121
122TEXT ·Xaddint64(SB), NOSPLIT, $0-24
123 JMP ·Xadd64(SB)
124
125// func Casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
126// Atomically:
127// if *ptr == old {
128// *ptr = new
129// return true
130// } else {
131// return false
132// }
133TEXT ·Casp1(SB), NOSPLIT, $0-25
134 JMP ·Cas64(SB)
135
136// uint32 Xadd(uint32 volatile *ptr, int32 delta)
137// Atomically:
138// *val += delta;
139// return *val;
140TEXT ·Xadd(SB), NOSPLIT, $0-20
141 MOVV ptr+0(FP), R4
142 MOVW delta+8(FP), R5
143 AMADDDBW R5, (R4), R6
144 ADDV R6, R5, R4
145 MOVW R4, ret+16(FP)
146 RET
147
148// func Xadd64(ptr *uint64, delta int64) uint64
149TEXT ·Xadd64(SB), NOSPLIT, $0-24
150 MOVV ptr+0(FP), R4
151 MOVV delta+8(FP), R5
152 AMADDDBV R5, (R4), R6
153 ADDV R6, R5, R4
154 MOVV R4, ret+16(FP)
155 RET
156
157// uint8 Xchg8(ptr *uint8, new uint8)
158// Atomically:
159// old := *ptr;
160// *ptr = new;
161// return old;
162TEXT ·Xchg8(SB), NOSPLIT, $0-17
163 MOVV ptr+0(FP), R4
164 MOVBU new+8(FP), R5
165
166 // R6 = ((ptr & 3) * 8)
167 AND $3, R4, R6
168 SLLV $3, R6
169
170 // R7 = ((0xFF) << R6) ^ (-1)
171 MOVV $0xFF, R8
172 SLLV R6, R8, R7
173 XOR $-1, R7
174
175 // R4 = ptr & (~3)
176 MOVV $~3, R8
177 AND R8, R4
178
179 // R5 = ((val) << R6)
180 SLLV R6, R5
181
182 DBAR $0x14 // LoadAcquire barrier
183_xchg8_again:
184 LL (R4), R8
185 MOVV R8, R9 // backup old val
186 AND R7, R8
187 OR R5, R8
188 SC R8, (R4)
189 BEQ R8, _xchg8_again
190 DBAR $0x12 // StoreRelease barrier
191 SRLV R6, R9, R9
192 MOVBU R9, ret+16(FP)
193 RET
194
195// func Xchg(ptr *uint32, new uint32) uint32
196TEXT ·Xchg(SB), NOSPLIT, $0-20
197 MOVV ptr+0(FP), R4
198 MOVW new+8(FP), R5
199 AMSWAPDBW R5, (R4), R6
200 MOVW R6, ret+16(FP)
201 RET
202
203// func Xchg64(ptr *uint64, new uint64) uint64
204TEXT ·Xchg64(SB), NOSPLIT, $0-24
205 MOVV ptr+0(FP), R4
206 MOVV new+8(FP), R5
207 AMSWAPDBV R5, (R4), R6
208 MOVV R6, ret+16(FP)
209 RET
210
211TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
212 JMP ·Xchg64(SB)
213
214// func Xchgint32(ptr *int32, new int32) int32
215TEXT ·Xchgint32(SB), NOSPLIT, $0-20
216 JMP ·Xchg(SB)
217
218// func Xchgint64(ptr *int64, new int64) int64
219TEXT ·Xchgint64(SB), NOSPLIT, $0-24
220 JMP ·Xchg64(SB)
221
222TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
223 JMP ·Store64(SB)
224
225TEXT ·StoreRel(SB), NOSPLIT, $0-12
226 JMP ·Store(SB)
227
228TEXT ·StoreRel64(SB), NOSPLIT, $0-16
229 JMP ·Store64(SB)
230
231TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
232 JMP ·Store64(SB)
233
234TEXT ·Store8(SB), NOSPLIT, $0-9
235 MOVV ptr+0(FP), R4
236 MOVB val+8(FP), R5
237 // StoreRelease barrier
238 DBAR $0x12
239 MOVB R5, 0(R4)
240 DBAR $0x18
241 RET
242
243TEXT ·Store(SB), NOSPLIT, $0-12
244 MOVV ptr+0(FP), R4
245 MOVW val+8(FP), R5
246 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasDBAR_HINTS(SB), R6
247 BEQ R6, _variant_
248 // StoreRelease barrier
249 DBAR $0x12
250 MOVW R5, 0(R4)
251 DBAR $0x18
252 RET
253_variant_:
254 AMSWAPDBW R5, (R4), R0
255 RET
256
257TEXT ·Store64(SB), NOSPLIT, $0-16
258 MOVV ptr+0(FP), R4
259 MOVV val+8(FP), R5
260 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasDBAR_HINTS(SB), R6
261 BEQ R6, _variant_
262 // StoreRelease barrier
263 DBAR $0x12
264 MOVV R5, 0(R4)
265 DBAR $0x18
266 RET
267_variant_:
268 AMSWAPDBV R5, (R4), R0
269 RET
270
271// void Or8(byte volatile*, byte);
272TEXT ·Or8(SB), NOSPLIT, $0-9
273 MOVV ptr+0(FP), R4
274 MOVBU val+8(FP), R5
275 // R6 = ptr & (~3)
276 MOVV $~3, R6
277 AND R4, R6
278 // R7 = ((ptr & 3) * 8)
279 AND $3, R4, R7
280 SLLV $3, R7
281 // R5 = val << R7
282 SLLV R7, R5
283 AMORDBW R5, (R6), R0
284 RET
285
286// void And8(byte volatile*, byte);
287TEXT ·And8(SB), NOSPLIT, $0-9
288 MOVV ptr+0(FP), R4
289 MOVBU val+8(FP), R5
290 // R6 = ptr & (~3)
291 MOVV $~3, R6
292 AND R4, R6
293 // R7 = ((ptr & 3) * 8)
294 AND $3, R4, R7
295 SLLV $3, R7
296 // R5 = ((val ^ 0xFF) << R7) ^ (-1)
297 XOR $255, R5
298 SLLV R7, R5
299 XOR $-1, R5
300 AMANDDBW R5, (R6), R0
301 RET
302
303// func Or(addr *uint32, v uint32)
304TEXT ·Or(SB), NOSPLIT, $0-12
305 MOVV ptr+0(FP), R4
306 MOVW val+8(FP), R5
307 AMORDBW R5, (R4), R0
308 RET
309
310// func And(addr *uint32, v uint32)
311TEXT ·And(SB), NOSPLIT, $0-12
312 MOVV ptr+0(FP), R4
313 MOVW val+8(FP), R5
314 AMANDDBW R5, (R4), R0
315 RET
316
317// func Or32(addr *uint32, v uint32) old uint32
318TEXT ·Or32(SB), NOSPLIT, $0-20
319 MOVV ptr+0(FP), R4
320 MOVW val+8(FP), R5
321 AMORDBW R5, (R4), R6
322 MOVW R6, ret+16(FP)
323 RET
324
325// func And32(addr *uint32, v uint32) old uint32
326TEXT ·And32(SB), NOSPLIT, $0-20
327 MOVV ptr+0(FP), R4
328 MOVW val+8(FP), R5
329 AMANDDBW R5, (R4), R6
330 MOVW R6, ret+16(FP)
331 RET
332
333// func Or64(addr *uint64, v uint64) old uint64
334TEXT ·Or64(SB), NOSPLIT, $0-24
335 MOVV ptr+0(FP), R4
336 MOVV val+8(FP), R5
337 AMORDBV R5, (R4), R6
338 MOVV R6, ret+16(FP)
339 RET
340
341// func And64(addr *uint64, v uint64) old uint64
342TEXT ·And64(SB), NOSPLIT, $0-24
343 MOVV ptr+0(FP), R4
344 MOVV val+8(FP), R5
345 AMANDDBV R5, (R4), R6
346 MOVV R6, ret+16(FP)
347 RET
348
349// func Anduintptr(addr *uintptr, v uintptr) old uintptr
350TEXT ·Anduintptr(SB), NOSPLIT, $0-24
351 JMP ·And64(SB)
352
353// func Oruintptr(addr *uintptr, v uintptr) old uintptr
354TEXT ·Oruintptr(SB), NOSPLIT, $0-24
355 JMP ·Or64(SB)
356
357// uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr)
358TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
359 MOVV ptr+0(FP), R19
360 MOVWU 0(R19), R19
361 DBAR $0x14 // LoadAcquire barrier
362 MOVW R19, ret+8(FP)
363 RET
364
365// uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr)
366TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
367 MOVV ptr+0(FP), R19
368 MOVBU 0(R19), R19
369 DBAR $0x14
370 MOVB R19, ret+8(FP)
371 RET
372
373// uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr)
374TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
375 MOVV ptr+0(FP), R19
376 MOVV 0(R19), R19
377 DBAR $0x14
378 MOVV R19, ret+8(FP)
379 RET
380
381// void *internal∕runtime∕atomic·Loadp(void *volatile *ptr)
382TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16
383 JMP ·Load64(SB)
384
385// uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr)
386TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
387 JMP ·Load(SB)
388
389// uint64 ·LoadAcq64(uint64 volatile* ptr)
390TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
391 JMP ·Load64(SB)
392
393// uintptr ·LoadAcquintptr(uintptr volatile* ptr)
394TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
395 JMP ·Load64(SB)
396
View as plain text