1// Copyright 2022 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7
8// bool cas(uint32 *ptr, uint32 old, uint32 new)
9// Atomically:
10// if(*ptr == old){
11// *ptr = new;
12// return 1;
13// } else
14// return 0;
15TEXT ·Cas(SB), NOSPLIT, $0-17
16 MOVV ptr+0(FP), R4
17 MOVW old+8(FP), R5
18 MOVW new+12(FP), R6
19
20 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
21 BEQ R8, cas_again
22 MOVV R5, R7 // backup old value
23 AMCASDBW R6, (R4), R5
24 BNE R7, R5, cas_fail0
25 MOVV $1, R4
26 MOVB R4, ret+16(FP)
27 RET
28cas_fail0:
29 MOVB R0, ret+16(FP)
30 RET
31
32 // Implemented using the ll-sc instruction pair
33 DBAR $0x14 // LoadAcquire barrier
34cas_again:
35 MOVV R6, R7
36 LL (R4), R8
37 BNE R5, R8, cas_fail1
38 SC R7, (R4)
39 BEQ R7, cas_again
40 MOVV $1, R4
41 MOVB R4, ret+16(FP)
42 DBAR $0x12 // StoreRelease barrier
43 RET
44cas_fail1:
45 MOVV $0, R4
46 JMP -4(PC)
47
48// bool cas64(uint64 *ptr, uint64 old, uint64 new)
49// Atomically:
50// if(*ptr == old){
51// *ptr = new;
52// return 1;
53// } else {
54// return 0;
55// }
56TEXT ·Cas64(SB), NOSPLIT, $0-25
57 MOVV ptr+0(FP), R4
58 MOVV old+8(FP), R5
59 MOVV new+16(FP), R6
60
61 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
62 BEQ R8, cas64_again
63 MOVV R5, R7 // backup old value
64 AMCASDBV R6, (R4), R5
65 BNE R7, R5, cas64_fail0
66 MOVV $1, R4
67 MOVB R4, ret+24(FP)
68 RET
69cas64_fail0:
70 MOVB R0, ret+24(FP)
71 RET
72
73 // Implemented using the ll-sc instruction pair
74 DBAR $0x14
75cas64_again:
76 MOVV R6, R7
77 LLV (R4), R8
78 BNE R5, R8, cas64_fail1
79 SCV R7, (R4)
80 BEQ R7, cas64_again
81 MOVV $1, R4
82 MOVB R4, ret+24(FP)
83 DBAR $0x12
84 RET
85cas64_fail1:
86 MOVV $0, R4
87 JMP -4(PC)
88
89TEXT ·Casint32(SB),NOSPLIT,$0-17
90 JMP ·Cas(SB)
91
92TEXT ·Casint64(SB),NOSPLIT,$0-25
93 JMP ·Cas64(SB)
94
95TEXT ·Casuintptr(SB), NOSPLIT, $0-25
96 JMP ·Cas64(SB)
97
98TEXT ·CasRel(SB), NOSPLIT, $0-17
99 JMP ·Cas(SB)
100
101TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
102 JMP ·Load64(SB)
103
104TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
105 JMP ·Load64(SB)
106
107TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
108 JMP ·Store64(SB)
109
110TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
111 JMP ·Xadd64(SB)
112
113TEXT ·Loadint64(SB), NOSPLIT, $0-16
114 JMP ·Load64(SB)
115
116TEXT ·Xaddint32(SB),NOSPLIT,$0-20
117 JMP ·Xadd(SB)
118
119TEXT ·Xaddint64(SB), NOSPLIT, $0-24
120 JMP ·Xadd64(SB)
121
122// bool casp(void **val, void *old, void *new)
123// Atomically:
124// if(*val == old){
125// *val = new;
126// return 1;
127// } else
128// return 0;
129TEXT ·Casp1(SB), NOSPLIT, $0-25
130 JMP ·Cas64(SB)
131
132// uint32 Xadd(uint32 volatile *ptr, int32 delta)
133// Atomically:
134// *val += delta;
135// return *val;
136TEXT ·Xadd(SB), NOSPLIT, $0-20
137 MOVV ptr+0(FP), R4
138 MOVW delta+8(FP), R5
139 AMADDDBW R5, (R4), R6
140 ADDV R6, R5, R4
141 MOVW R4, ret+16(FP)
142 RET
143
144// func Xadd64(ptr *uint64, delta int64) uint64
145TEXT ·Xadd64(SB), NOSPLIT, $0-24
146 MOVV ptr+0(FP), R4
147 MOVV delta+8(FP), R5
148 AMADDDBV R5, (R4), R6
149 ADDV R6, R5, R4
150 MOVV R4, ret+16(FP)
151 RET
152
153// uint8 Xchg8(ptr *uint8, new uint8)
154// Atomically:
155// old := *ptr;
156// *ptr = new;
157// return old;
158TEXT ·Xchg8(SB), NOSPLIT, $0-17
159 MOVV ptr+0(FP), R4
160 MOVBU new+8(FP), R5
161
162 // R6 = ((ptr & 3) * 8)
163 AND $3, R4, R6
164 SLLV $3, R6
165
166 // R7 = ((0xFF) << R6) ^ (-1)
167 MOVV $0xFF, R8
168 SLLV R6, R8, R7
169 XOR $-1, R7
170
171 // R4 = ptr & (~3)
172 MOVV $~3, R8
173 AND R8, R4
174
175 // R5 = ((val) << R6)
176 SLLV R6, R5
177
178 DBAR $0x14 // LoadAcquire barrier
179_xchg8_again:
180 LL (R4), R8
181 MOVV R8, R9 // backup old val
182 AND R7, R8
183 OR R5, R8
184 SC R8, (R4)
185 BEQ R8, _xchg8_again
186 DBAR $0x12 // StoreRelease barrier
187 SRLV R6, R9, R9
188 MOVBU R9, ret+16(FP)
189 RET
190
191// func Xchg(ptr *uint32, new uint32) uint32
192TEXT ·Xchg(SB), NOSPLIT, $0-20
193 MOVV ptr+0(FP), R4
194 MOVW new+8(FP), R5
195 AMSWAPDBW R5, (R4), R6
196 MOVW R6, ret+16(FP)
197 RET
198
199// func Xchg64(ptr *uint64, new uint64) uint64
200TEXT ·Xchg64(SB), NOSPLIT, $0-24
201 MOVV ptr+0(FP), R4
202 MOVV new+8(FP), R5
203 AMSWAPDBV R5, (R4), R6
204 MOVV R6, ret+16(FP)
205 RET
206
207TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
208 JMP ·Xchg64(SB)
209
210// func Xchgint32(ptr *int32, new int32) int32
211TEXT ·Xchgint32(SB), NOSPLIT, $0-20
212 JMP ·Xchg(SB)
213
214// func Xchgint64(ptr *int64, new int64) int64
215TEXT ·Xchgint64(SB), NOSPLIT, $0-24
216 JMP ·Xchg64(SB)
217
218TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
219 JMP ·Store64(SB)
220
221TEXT ·StoreRel(SB), NOSPLIT, $0-12
222 JMP ·Store(SB)
223
224TEXT ·StoreRel64(SB), NOSPLIT, $0-16
225 JMP ·Store64(SB)
226
227TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
228 JMP ·Store64(SB)
229
230TEXT ·Store(SB), NOSPLIT, $0-12
231 MOVV ptr+0(FP), R4
232 MOVW val+8(FP), R5
233 AMSWAPDBW R5, (R4), R0
234 RET
235
236TEXT ·Store8(SB), NOSPLIT, $0-9
237 MOVV ptr+0(FP), R4
238 MOVB val+8(FP), R5
239 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasLAM_BH(SB), R6
240 BEQ R6, _legacy_store8_
241 AMSWAPDBB R5, (R4), R0
242 RET
243_legacy_store8_:
244 // StoreRelease barrier
245 DBAR $0x12
246 MOVB R5, 0(R4)
247 DBAR $0x18
248 RET
249
250TEXT ·Store64(SB), NOSPLIT, $0-16
251 MOVV ptr+0(FP), R4
252 MOVV val+8(FP), R5
253 AMSWAPDBV R5, (R4), R0
254 RET
255
256// void Or8(byte volatile*, byte);
257TEXT ·Or8(SB), NOSPLIT, $0-9
258 MOVV ptr+0(FP), R4
259 MOVBU val+8(FP), R5
260 // R6 = ptr & (~3)
261 MOVV $~3, R6
262 AND R4, R6
263 // R7 = ((ptr & 3) * 8)
264 AND $3, R4, R7
265 SLLV $3, R7
266 // R5 = val << R7
267 SLLV R7, R5
268 AMORDBW R5, (R6), R0
269 RET
270
271// void And8(byte volatile*, byte);
272TEXT ·And8(SB), NOSPLIT, $0-9
273 MOVV ptr+0(FP), R4
274 MOVBU val+8(FP), R5
275 // R6 = ptr & (~3)
276 MOVV $~3, R6
277 AND R4, R6
278 // R7 = ((ptr & 3) * 8)
279 AND $3, R4, R7
280 SLLV $3, R7
281 // R5 = ((val ^ 0xFF) << R7) ^ (-1)
282 XOR $255, R5
283 SLLV R7, R5
284 XOR $-1, R5
285 AMANDDBW R5, (R6), R0
286 RET
287
288// func Or(addr *uint32, v uint32)
289TEXT ·Or(SB), NOSPLIT, $0-12
290 MOVV ptr+0(FP), R4
291 MOVW val+8(FP), R5
292 AMORDBW R5, (R4), R0
293 RET
294
295// func And(addr *uint32, v uint32)
296TEXT ·And(SB), NOSPLIT, $0-12
297 MOVV ptr+0(FP), R4
298 MOVW val+8(FP), R5
299 AMANDDBW R5, (R4), R0
300 RET
301
302// func Or32(addr *uint32, v uint32) old uint32
303TEXT ·Or32(SB), NOSPLIT, $0-20
304 MOVV ptr+0(FP), R4
305 MOVW val+8(FP), R5
306 AMORDBW R5, (R4), R6
307 MOVW R6, ret+16(FP)
308 RET
309
310// func And32(addr *uint32, v uint32) old uint32
311TEXT ·And32(SB), NOSPLIT, $0-20
312 MOVV ptr+0(FP), R4
313 MOVW val+8(FP), R5
314 AMANDDBW R5, (R4), R6
315 MOVW R6, ret+16(FP)
316 RET
317
318// func Or64(addr *uint64, v uint64) old uint64
319TEXT ·Or64(SB), NOSPLIT, $0-24
320 MOVV ptr+0(FP), R4
321 MOVV val+8(FP), R5
322 AMORDBV R5, (R4), R6
323 MOVV R6, ret+16(FP)
324 RET
325
326// func And64(addr *uint64, v uint64) old uint64
327TEXT ·And64(SB), NOSPLIT, $0-24
328 MOVV ptr+0(FP), R4
329 MOVV val+8(FP), R5
330 AMANDDBV R5, (R4), R6
331 MOVV R6, ret+16(FP)
332 RET
333
334// func Anduintptr(addr *uintptr, v uintptr) old uintptr
335TEXT ·Anduintptr(SB), NOSPLIT, $0-24
336 JMP ·And64(SB)
337
338// func Oruintptr(addr *uintptr, v uintptr) old uintptr
339TEXT ·Oruintptr(SB), NOSPLIT, $0-24
340 JMP ·Or64(SB)
341
342// uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr)
343TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
344 MOVV ptr+0(FP), R19
345 MOVWU 0(R19), R19
346 DBAR $0x14 // LoadAcquire barrier
347 MOVW R19, ret+8(FP)
348 RET
349
350// uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr)
351TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
352 MOVV ptr+0(FP), R19
353 MOVBU 0(R19), R19
354 DBAR $0x14
355 MOVB R19, ret+8(FP)
356 RET
357
358// uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr)
359TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
360 MOVV ptr+0(FP), R19
361 MOVV 0(R19), R19
362 DBAR $0x14
363 MOVV R19, ret+8(FP)
364 RET
365
366// void *internal∕runtime∕atomic·Loadp(void *volatile *ptr)
367TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16
368 JMP ·Load64(SB)
369
370// uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr)
371TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
372 JMP ·Load(SB)
373
374// uint64 ·LoadAcq64(uint64 volatile* ptr)
375TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
376 JMP ·Load64(SB)
377
378// uintptr ·LoadAcquintptr(uintptr volatile* ptr)
379TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
380 JMP ·Load64(SB)
381
View as plain text