1// Copyright 2026 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "textflag.h"
6
7// func memHash32AES(k uint32, h uintptr) uintptr
8TEXT ·memHash32AES<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
9 MOVD $·aeskeysched+0(SB), R3
10
11 VEOR V0.B16, V0.B16, V0.B16
12 VLD1 (R3), [V2.B16]
13 VMOV R0, V0.S[2]
14 VMOV R1, V0.D[0]
15
16 AESE V2.B16, V0.B16
17 AESMC V0.B16, V0.B16
18 AESE V2.B16, V0.B16
19 AESMC V0.B16, V0.B16
20 AESE V2.B16, V0.B16
21
22 VMOV V0.D[0], R0
23 RET
24
25// func memHash64AES(k uint64, h uintptr) uintptr
26TEXT ·memHash64AES<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
27 MOVD $·aeskeysched+0(SB), R3
28
29 VEOR V0.B16, V0.B16, V0.B16
30 VLD1 (R3), [V2.B16]
31 VMOV R0, V0.D[1]
32 VMOV R1, V0.D[0]
33
34 AESE V2.B16, V0.B16
35 AESMC V0.B16, V0.B16
36 AESE V2.B16, V0.B16
37 AESMC V0.B16, V0.B16
38 AESE V2.B16, V0.B16
39
40 VMOV V0.D[0], R0
41 RET
42
43// func memHashAES(p unsafe.Pointer, h, size uintptr) uintptr
44TEXT ·memHashAES<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
45 // R0: data
46 // R1: seed data
47 // R2: length
48 // At return, R0 = return value
49 VEOR V30.B16, V30.B16, V30.B16
50 VMOV R1, V30.D[0]
51 VMOV R2, V30.D[1] // load length into seed
52
53 MOVD $·aeskeysched+0(SB), R4
54 VLD1.P 16(R4), [V0.B16]
55 AESE V30.B16, V0.B16
56 AESMC V0.B16, V0.B16
57 CMP $16, R2
58 BLO aes0to15
59 BEQ aes16
60 CMP $32, R2
61 BLS aes17to32
62 CMP $64, R2
63 BLS aes33to64
64 CMP $128, R2
65 BLS aes65to128
66 B aes129plus
67
68aes0to15:
69 CBZ R2, aes0
70 VEOR V2.B16, V2.B16, V2.B16
71 TBZ $3, R2, less_than_8
72 VLD1.P 8(R0), V2.D[0]
73
74less_than_8:
75 TBZ $2, R2, less_than_4
76 VLD1.P 4(R0), V2.S[2]
77
78less_than_4:
79 TBZ $1, R2, less_than_2
80 VLD1.P 2(R0), V2.H[6]
81
82less_than_2:
83 TBZ $0, R2, done
84 VLD1 (R0), V2.B[14]
85done:
86 AESE V0.B16, V2.B16
87 AESMC V2.B16, V2.B16
88 AESE V0.B16, V2.B16
89 AESMC V2.B16, V2.B16
90 AESE V0.B16, V2.B16
91 AESMC V2.B16, V2.B16
92
93 VMOV V2.D[0], R0
94 RET
95
96aes0:
97 VMOV V0.D[0], R0
98 RET
99
100aes16:
101 VLD1 (R0), [V2.B16]
102 B done
103
104aes17to32:
105 // make second seed
106 VLD1 (R4), [V1.B16]
107 AESE V30.B16, V1.B16
108 AESMC V1.B16, V1.B16
109 SUB $16, R2, R10
110 VLD1.P (R0)(R10), [V2.B16]
111 VLD1 (R0), [V3.B16]
112
113 AESE V0.B16, V2.B16
114 AESMC V2.B16, V2.B16
115 AESE V1.B16, V3.B16
116 AESMC V3.B16, V3.B16
117
118 AESE V0.B16, V2.B16
119 AESMC V2.B16, V2.B16
120 AESE V1.B16, V3.B16
121 AESMC V3.B16, V3.B16
122
123 AESE V0.B16, V2.B16
124 AESE V1.B16, V3.B16
125
126 VEOR V3.B16, V2.B16, V2.B16
127
128 VMOV V2.D[0], R0
129 RET
130
131aes33to64:
132 VLD1 (R4), [V1.B16, V2.B16, V3.B16]
133 AESE V30.B16, V1.B16
134 AESMC V1.B16, V1.B16
135 AESE V30.B16, V2.B16
136 AESMC V2.B16, V2.B16
137 AESE V30.B16, V3.B16
138 AESMC V3.B16, V3.B16
139 SUB $32, R2, R10
140
141 VLD1.P (R0)(R10), [V4.B16, V5.B16]
142 VLD1 (R0), [V6.B16, V7.B16]
143
144 AESE V0.B16, V4.B16
145 AESMC V4.B16, V4.B16
146 AESE V1.B16, V5.B16
147 AESMC V5.B16, V5.B16
148 AESE V2.B16, V6.B16
149 AESMC V6.B16, V6.B16
150 AESE V3.B16, V7.B16
151 AESMC V7.B16, V7.B16
152
153 AESE V0.B16, V4.B16
154 AESMC V4.B16, V4.B16
155 AESE V1.B16, V5.B16
156 AESMC V5.B16, V5.B16
157 AESE V2.B16, V6.B16
158 AESMC V6.B16, V6.B16
159 AESE V3.B16, V7.B16
160 AESMC V7.B16, V7.B16
161
162 AESE V0.B16, V4.B16
163 AESE V1.B16, V5.B16
164 AESE V2.B16, V6.B16
165 AESE V3.B16, V7.B16
166
167 VEOR V6.B16, V4.B16, V4.B16
168 VEOR V7.B16, V5.B16, V5.B16
169 VEOR V5.B16, V4.B16, V4.B16
170
171 VMOV V4.D[0], R0
172 RET
173
174aes65to128:
175 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
176 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
177 AESE V30.B16, V1.B16
178 AESMC V1.B16, V1.B16
179 AESE V30.B16, V2.B16
180 AESMC V2.B16, V2.B16
181 AESE V30.B16, V3.B16
182 AESMC V3.B16, V3.B16
183 AESE V30.B16, V4.B16
184 AESMC V4.B16, V4.B16
185 AESE V30.B16, V5.B16
186 AESMC V5.B16, V5.B16
187 AESE V30.B16, V6.B16
188 AESMC V6.B16, V6.B16
189 AESE V30.B16, V7.B16
190 AESMC V7.B16, V7.B16
191
192 SUB $64, R2, R10
193 VLD1.P (R0)(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
194 VLD1 (R0), [V12.B16, V13.B16, V14.B16, V15.B16]
195 AESE V0.B16, V8.B16
196 AESMC V8.B16, V8.B16
197 AESE V1.B16, V9.B16
198 AESMC V9.B16, V9.B16
199 AESE V2.B16, V10.B16
200 AESMC V10.B16, V10.B16
201 AESE V3.B16, V11.B16
202 AESMC V11.B16, V11.B16
203 AESE V4.B16, V12.B16
204 AESMC V12.B16, V12.B16
205 AESE V5.B16, V13.B16
206 AESMC V13.B16, V13.B16
207 AESE V6.B16, V14.B16
208 AESMC V14.B16, V14.B16
209 AESE V7.B16, V15.B16
210 AESMC V15.B16, V15.B16
211
212 AESE V0.B16, V8.B16
213 AESMC V8.B16, V8.B16
214 AESE V1.B16, V9.B16
215 AESMC V9.B16, V9.B16
216 AESE V2.B16, V10.B16
217 AESMC V10.B16, V10.B16
218 AESE V3.B16, V11.B16
219 AESMC V11.B16, V11.B16
220 AESE V4.B16, V12.B16
221 AESMC V12.B16, V12.B16
222 AESE V5.B16, V13.B16
223 AESMC V13.B16, V13.B16
224 AESE V6.B16, V14.B16
225 AESMC V14.B16, V14.B16
226 AESE V7.B16, V15.B16
227 AESMC V15.B16, V15.B16
228
229 AESE V0.B16, V8.B16
230 AESE V1.B16, V9.B16
231 AESE V2.B16, V10.B16
232 AESE V3.B16, V11.B16
233 AESE V4.B16, V12.B16
234 AESE V5.B16, V13.B16
235 AESE V6.B16, V14.B16
236 AESE V7.B16, V15.B16
237
238 VEOR V12.B16, V8.B16, V8.B16
239 VEOR V13.B16, V9.B16, V9.B16
240 VEOR V14.B16, V10.B16, V10.B16
241 VEOR V15.B16, V11.B16, V11.B16
242 VEOR V10.B16, V8.B16, V8.B16
243 VEOR V11.B16, V9.B16, V9.B16
244 VEOR V9.B16, V8.B16, V8.B16
245
246 VMOV V8.D[0], R0
247 RET
248
249aes129plus:
250 PRFM (R0), PLDL1KEEP
251 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
252 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
253 AESE V30.B16, V1.B16
254 AESMC V1.B16, V1.B16
255 AESE V30.B16, V2.B16
256 AESMC V2.B16, V2.B16
257 AESE V30.B16, V3.B16
258 AESMC V3.B16, V3.B16
259 AESE V30.B16, V4.B16
260 AESMC V4.B16, V4.B16
261 AESE V30.B16, V5.B16
262 AESMC V5.B16, V5.B16
263 AESE V30.B16, V6.B16
264 AESMC V6.B16, V6.B16
265 AESE V30.B16, V7.B16
266 AESMC V7.B16, V7.B16
267 ADD R0, R2, R10
268 SUB $128, R10, R10
269 VLD1.P 64(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
270 VLD1 (R10), [V12.B16, V13.B16, V14.B16, V15.B16]
271 SUB $1, R2, R2
272 LSR $7, R2, R2
273
274aesloop:
275 AESE V8.B16, V0.B16
276 AESMC V0.B16, V0.B16
277 AESE V9.B16, V1.B16
278 AESMC V1.B16, V1.B16
279 AESE V10.B16, V2.B16
280 AESMC V2.B16, V2.B16
281 AESE V11.B16, V3.B16
282 AESMC V3.B16, V3.B16
283 AESE V12.B16, V4.B16
284 AESMC V4.B16, V4.B16
285 AESE V13.B16, V5.B16
286 AESMC V5.B16, V5.B16
287 AESE V14.B16, V6.B16
288 AESMC V6.B16, V6.B16
289 AESE V15.B16, V7.B16
290 AESMC V7.B16, V7.B16
291
292 VLD1.P 64(R0), [V8.B16, V9.B16, V10.B16, V11.B16]
293 AESE V8.B16, V0.B16
294 AESMC V0.B16, V0.B16
295 AESE V9.B16, V1.B16
296 AESMC V1.B16, V1.B16
297 AESE V10.B16, V2.B16
298 AESMC V2.B16, V2.B16
299 AESE V11.B16, V3.B16
300 AESMC V3.B16, V3.B16
301
302 VLD1.P 64(R0), [V12.B16, V13.B16, V14.B16, V15.B16]
303 AESE V12.B16, V4.B16
304 AESMC V4.B16, V4.B16
305 AESE V13.B16, V5.B16
306 AESMC V5.B16, V5.B16
307 AESE V14.B16, V6.B16
308 AESMC V6.B16, V6.B16
309 AESE V15.B16, V7.B16
310 AESMC V7.B16, V7.B16
311 SUB $1, R2, R2
312 CBNZ R2, aesloop
313
314 AESE V8.B16, V0.B16
315 AESMC V0.B16, V0.B16
316 AESE V9.B16, V1.B16
317 AESMC V1.B16, V1.B16
318 AESE V10.B16, V2.B16
319 AESMC V2.B16, V2.B16
320 AESE V11.B16, V3.B16
321 AESMC V3.B16, V3.B16
322 AESE V12.B16, V4.B16
323 AESMC V4.B16, V4.B16
324 AESE V13.B16, V5.B16
325 AESMC V5.B16, V5.B16
326 AESE V14.B16, V6.B16
327 AESMC V6.B16, V6.B16
328 AESE V15.B16, V7.B16
329 AESMC V7.B16, V7.B16
330
331 AESE V8.B16, V0.B16
332 AESMC V0.B16, V0.B16
333 AESE V9.B16, V1.B16
334 AESMC V1.B16, V1.B16
335 AESE V10.B16, V2.B16
336 AESMC V2.B16, V2.B16
337 AESE V11.B16, V3.B16
338 AESMC V3.B16, V3.B16
339 AESE V12.B16, V4.B16
340 AESMC V4.B16, V4.B16
341 AESE V13.B16, V5.B16
342 AESMC V5.B16, V5.B16
343 AESE V14.B16, V6.B16
344 AESMC V6.B16, V6.B16
345 AESE V15.B16, V7.B16
346 AESMC V7.B16, V7.B16
347
348 AESE V8.B16, V0.B16
349 AESE V9.B16, V1.B16
350 AESE V10.B16, V2.B16
351 AESE V11.B16, V3.B16
352 AESE V12.B16, V4.B16
353 AESE V13.B16, V5.B16
354 AESE V14.B16, V6.B16
355 AESE V15.B16, V7.B16
356
357 VEOR V0.B16, V1.B16, V0.B16
358 VEOR V2.B16, V3.B16, V2.B16
359 VEOR V4.B16, V5.B16, V4.B16
360 VEOR V6.B16, V7.B16, V6.B16
361 VEOR V0.B16, V2.B16, V0.B16
362 VEOR V4.B16, V6.B16, V4.B16
363 VEOR V4.B16, V0.B16, V0.B16
364
365 VMOV V0.D[0], R0
366 RET
View as plain text