1// Copyright 2022 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7
8// func Cas(ptr *int32, old, new int32) bool
9// Atomically:
10// if *ptr == old {
11// *ptr = new
12// return true
13// } else {
14// return false
15// }
16TEXT ·Cas(SB), NOSPLIT, $0-17
17 MOVV ptr+0(FP), R4
18 MOVW old+8(FP), R5
19 MOVW new+12(FP), R6
20
21 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
22 BEQ R8, cas_again
23 MOVV R5, R7 // backup old value
24 AMCASDBW R6, (R4), R5
25 BNE R7, R5, cas_fail0
26 MOVV $1, R4
27 MOVB R4, ret+16(FP)
28 RET
29cas_fail0:
30 MOVB R0, ret+16(FP)
31 RET
32
33 // Implemented using the ll-sc instruction pair
34 DBAR $0x14 // LoadAcquire barrier
35cas_again:
36 MOVV R6, R7
37 LL (R4), R8
38 BNE R5, R8, cas_fail1
39 SC R7, (R4)
40 BEQ R7, cas_again
41 MOVV $1, R4
42 MOVB R4, ret+16(FP)
43 DBAR $0x12 // StoreRelease barrier
44 RET
45cas_fail1:
46 MOVV $0, R4
47 JMP -4(PC)
48
49// func Cas64(ptr *uint64, old, new uint64) bool
50// Atomically:
51// if *ptr == old {
52// *ptr = new
53// return true
54// } else {
55// return false
56// }
57TEXT ·Cas64(SB), NOSPLIT, $0-25
58 MOVV ptr+0(FP), R4
59 MOVV old+8(FP), R5
60 MOVV new+16(FP), R6
61
62 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
63 BEQ R8, cas64_again
64 MOVV R5, R7 // backup old value
65 AMCASDBV R6, (R4), R5
66 BNE R7, R5, cas64_fail0
67 MOVV $1, R4
68 MOVB R4, ret+24(FP)
69 RET
70cas64_fail0:
71 MOVB R0, ret+24(FP)
72 RET
73
74 // Implemented using the ll-sc instruction pair
75 DBAR $0x14
76cas64_again:
77 MOVV R6, R7
78 LLV (R4), R8
79 BNE R5, R8, cas64_fail1
80 SCV R7, (R4)
81 BEQ R7, cas64_again
82 MOVV $1, R4
83 MOVB R4, ret+24(FP)
84 DBAR $0x12
85 RET
86cas64_fail1:
87 MOVV $0, R4
88 JMP -4(PC)
89
90TEXT ·Casint32(SB),NOSPLIT,$0-17
91 JMP ·Cas(SB)
92
93TEXT ·Casint64(SB),NOSPLIT,$0-25
94 JMP ·Cas64(SB)
95
96TEXT ·Casuintptr(SB), NOSPLIT, $0-25
97 JMP ·Cas64(SB)
98
99TEXT ·CasRel(SB), NOSPLIT, $0-17
100 JMP ·Cas(SB)
101
102TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
103 JMP ·Load64(SB)
104
105TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
106 JMP ·Load64(SB)
107
108TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
109 JMP ·Store64(SB)
110
111TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
112 JMP ·Xadd64(SB)
113
114TEXT ·Loadint64(SB), NOSPLIT, $0-16
115 JMP ·Load64(SB)
116
117TEXT ·Xaddint32(SB),NOSPLIT,$0-20
118 JMP ·Xadd(SB)
119
120TEXT ·Xaddint64(SB), NOSPLIT, $0-24
121 JMP ·Xadd64(SB)
122
123// func Casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
124// Atomically:
125// if *ptr == old {
126// *ptr = new
127// return true
128// } else {
129// return false
130// }
131TEXT ·Casp1(SB), NOSPLIT, $0-25
132 JMP ·Cas64(SB)
133
134// uint32 Xadd(uint32 volatile *ptr, int32 delta)
135// Atomically:
136// *val += delta;
137// return *val;
138TEXT ·Xadd(SB), NOSPLIT, $0-20
139 MOVV ptr+0(FP), R4
140 MOVW delta+8(FP), R5
141 AMADDDBW R5, (R4), R6
142 ADDV R6, R5, R4
143 MOVW R4, ret+16(FP)
144 RET
145
146// func Xadd64(ptr *uint64, delta int64) uint64
147TEXT ·Xadd64(SB), NOSPLIT, $0-24
148 MOVV ptr+0(FP), R4
149 MOVV delta+8(FP), R5
150 AMADDDBV R5, (R4), R6
151 ADDV R6, R5, R4
152 MOVV R4, ret+16(FP)
153 RET
154
155// uint8 Xchg8(ptr *uint8, new uint8)
156// Atomically:
157// old := *ptr;
158// *ptr = new;
159// return old;
160TEXT ·Xchg8(SB), NOSPLIT, $0-17
161 MOVV ptr+0(FP), R4
162 MOVBU new+8(FP), R5
163
164 // R6 = ((ptr & 3) * 8)
165 AND $3, R4, R6
166 SLLV $3, R6
167
168 // R7 = ((0xFF) << R6) ^ (-1)
169 MOVV $0xFF, R8
170 SLLV R6, R8, R7
171 XOR $-1, R7
172
173 // R4 = ptr & (~3)
174 MOVV $~3, R8
175 AND R8, R4
176
177 // R5 = ((val) << R6)
178 SLLV R6, R5
179
180 DBAR $0x14 // LoadAcquire barrier
181_xchg8_again:
182 LL (R4), R8
183 MOVV R8, R9 // backup old val
184 AND R7, R8
185 OR R5, R8
186 SC R8, (R4)
187 BEQ R8, _xchg8_again
188 DBAR $0x12 // StoreRelease barrier
189 SRLV R6, R9, R9
190 MOVBU R9, ret+16(FP)
191 RET
192
193// func Xchg(ptr *uint32, new uint32) uint32
194TEXT ·Xchg(SB), NOSPLIT, $0-20
195 MOVV ptr+0(FP), R4
196 MOVW new+8(FP), R5
197 AMSWAPDBW R5, (R4), R6
198 MOVW R6, ret+16(FP)
199 RET
200
201// func Xchg64(ptr *uint64, new uint64) uint64
202TEXT ·Xchg64(SB), NOSPLIT, $0-24
203 MOVV ptr+0(FP), R4
204 MOVV new+8(FP), R5
205 AMSWAPDBV R5, (R4), R6
206 MOVV R6, ret+16(FP)
207 RET
208
209TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
210 JMP ·Xchg64(SB)
211
212// func Xchgint32(ptr *int32, new int32) int32
213TEXT ·Xchgint32(SB), NOSPLIT, $0-20
214 JMP ·Xchg(SB)
215
216// func Xchgint64(ptr *int64, new int64) int64
217TEXT ·Xchgint64(SB), NOSPLIT, $0-24
218 JMP ·Xchg64(SB)
219
220TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
221 JMP ·Store64(SB)
222
223TEXT ·StoreRel(SB), NOSPLIT, $0-12
224 JMP ·Store(SB)
225
226TEXT ·StoreRel64(SB), NOSPLIT, $0-16
227 JMP ·Store64(SB)
228
229TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
230 JMP ·Store64(SB)
231
232TEXT ·Store(SB), NOSPLIT, $0-12
233 MOVV ptr+0(FP), R4
234 MOVW val+8(FP), R5
235 AMSWAPDBW R5, (R4), R0
236 RET
237
238TEXT ·Store8(SB), NOSPLIT, $0-9
239 MOVV ptr+0(FP), R4
240 MOVB val+8(FP), R5
241 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasLAM_BH(SB), R6
242 BEQ R6, _legacy_store8_
243 AMSWAPDBB R5, (R4), R0
244 RET
245_legacy_store8_:
246 // StoreRelease barrier
247 DBAR $0x12
248 MOVB R5, 0(R4)
249 DBAR $0x18
250 RET
251
252TEXT ·Store64(SB), NOSPLIT, $0-16
253 MOVV ptr+0(FP), R4
254 MOVV val+8(FP), R5
255 AMSWAPDBV R5, (R4), R0
256 RET
257
258// void Or8(byte volatile*, byte);
259TEXT ·Or8(SB), NOSPLIT, $0-9
260 MOVV ptr+0(FP), R4
261 MOVBU val+8(FP), R5
262 // R6 = ptr & (~3)
263 MOVV $~3, R6
264 AND R4, R6
265 // R7 = ((ptr & 3) * 8)
266 AND $3, R4, R7
267 SLLV $3, R7
268 // R5 = val << R7
269 SLLV R7, R5
270 AMORDBW R5, (R6), R0
271 RET
272
273// void And8(byte volatile*, byte);
274TEXT ·And8(SB), NOSPLIT, $0-9
275 MOVV ptr+0(FP), R4
276 MOVBU val+8(FP), R5
277 // R6 = ptr & (~3)
278 MOVV $~3, R6
279 AND R4, R6
280 // R7 = ((ptr & 3) * 8)
281 AND $3, R4, R7
282 SLLV $3, R7
283 // R5 = ((val ^ 0xFF) << R7) ^ (-1)
284 XOR $255, R5
285 SLLV R7, R5
286 XOR $-1, R5
287 AMANDDBW R5, (R6), R0
288 RET
289
290// func Or(addr *uint32, v uint32)
291TEXT ·Or(SB), NOSPLIT, $0-12
292 MOVV ptr+0(FP), R4
293 MOVW val+8(FP), R5
294 AMORDBW R5, (R4), R0
295 RET
296
297// func And(addr *uint32, v uint32)
298TEXT ·And(SB), NOSPLIT, $0-12
299 MOVV ptr+0(FP), R4
300 MOVW val+8(FP), R5
301 AMANDDBW R5, (R4), R0
302 RET
303
304// func Or32(addr *uint32, v uint32) old uint32
305TEXT ·Or32(SB), NOSPLIT, $0-20
306 MOVV ptr+0(FP), R4
307 MOVW val+8(FP), R5
308 AMORDBW R5, (R4), R6
309 MOVW R6, ret+16(FP)
310 RET
311
312// func And32(addr *uint32, v uint32) old uint32
313TEXT ·And32(SB), NOSPLIT, $0-20
314 MOVV ptr+0(FP), R4
315 MOVW val+8(FP), R5
316 AMANDDBW R5, (R4), R6
317 MOVW R6, ret+16(FP)
318 RET
319
320// func Or64(addr *uint64, v uint64) old uint64
321TEXT ·Or64(SB), NOSPLIT, $0-24
322 MOVV ptr+0(FP), R4
323 MOVV val+8(FP), R5
324 AMORDBV R5, (R4), R6
325 MOVV R6, ret+16(FP)
326 RET
327
328// func And64(addr *uint64, v uint64) old uint64
329TEXT ·And64(SB), NOSPLIT, $0-24
330 MOVV ptr+0(FP), R4
331 MOVV val+8(FP), R5
332 AMANDDBV R5, (R4), R6
333 MOVV R6, ret+16(FP)
334 RET
335
336// func Anduintptr(addr *uintptr, v uintptr) old uintptr
337TEXT ·Anduintptr(SB), NOSPLIT, $0-24
338 JMP ·And64(SB)
339
340// func Oruintptr(addr *uintptr, v uintptr) old uintptr
341TEXT ·Oruintptr(SB), NOSPLIT, $0-24
342 JMP ·Or64(SB)
343
344// uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr)
345TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
346 MOVV ptr+0(FP), R19
347 MOVWU 0(R19), R19
348 DBAR $0x14 // LoadAcquire barrier
349 MOVW R19, ret+8(FP)
350 RET
351
352// uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr)
353TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
354 MOVV ptr+0(FP), R19
355 MOVBU 0(R19), R19
356 DBAR $0x14
357 MOVB R19, ret+8(FP)
358 RET
359
360// uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr)
361TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
362 MOVV ptr+0(FP), R19
363 MOVV 0(R19), R19
364 DBAR $0x14
365 MOVV R19, ret+8(FP)
366 RET
367
368// void *internal∕runtime∕atomic·Loadp(void *volatile *ptr)
369TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16
370 JMP ·Load64(SB)
371
372// uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr)
373TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
374 JMP ·Load(SB)
375
376// uint64 ·LoadAcq64(uint64 volatile* ptr)
377TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
378 JMP ·Load64(SB)
379
380// uintptr ·LoadAcquintptr(uintptr volatile* ptr)
381TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
382 JMP ·Load64(SB)
383
View as plain text