1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "textflag.h"
8
9 // For more details about how various memory models are
10 // enforced on POWER, the following paper provides more
11 // details about how they enforce C/C++ like models. This
12 // gives context about why the strange looking code
13 // sequences below work.
14 //
15 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
16
17 // uint32 ·Load(uint32 volatile* ptr)
18 TEXT ·Load(SB),NOSPLIT|NOFRAME,$-8-12
19 MOVD ptr+0(FP), R3
20 SYNC
21 MOVWZ 0(R3), R3
22 CMPW R3, R3, CR7
23 BC 4, 30, 1(PC) // bne- cr7,0x4
24 ISYNC
25 MOVW R3, ret+8(FP)
26 RET
27
28 // uint8 ·Load8(uint8 volatile* ptr)
29 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$-8-9
30 MOVD ptr+0(FP), R3
31 SYNC
32 MOVBZ 0(R3), R3
33 CMP R3, R3, CR7
34 BC 4, 30, 1(PC) // bne- cr7,0x4
35 ISYNC
36 MOVB R3, ret+8(FP)
37 RET
38
39 // uint64 ·Load64(uint64 volatile* ptr)
40 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$-8-16
41 MOVD ptr+0(FP), R3
42 SYNC
43 MOVD 0(R3), R3
44 CMP R3, R3, CR7
45 BC 4, 30, 1(PC) // bne- cr7,0x4
46 ISYNC
47 MOVD R3, ret+8(FP)
48 RET
49
50 // void *·Loadp(void *volatile *ptr)
51 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$-8-16
52 MOVD ptr+0(FP), R3
53 SYNC
54 MOVD 0(R3), R3
55 CMP R3, R3, CR7
56 BC 4, 30, 1(PC) // bne- cr7,0x4
57 ISYNC
58 MOVD R3, ret+8(FP)
59 RET
60
61 // uint32 ·LoadAcq(uint32 volatile* ptr)
62 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$-8-12
63 MOVD ptr+0(FP), R3
64 MOVWZ 0(R3), R3
65 CMPW R3, R3, CR7
66 BC 4, 30, 1(PC) // bne- cr7, 0x4
67 ISYNC
68 MOVW R3, ret+8(FP)
69 RET
70
71 // uint64 ·LoadAcq64(uint64 volatile* ptr)
72 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$-8-16
73 MOVD ptr+0(FP), R3
74 MOVD 0(R3), R3
75 CMP R3, R3, CR7
76 BC 4, 30, 1(PC) // bne- cr7, 0x4
77 ISYNC
78 MOVD R3, ret+8(FP)
79 RET
80
81 // func Cas(ptr *int32, old, new int32) bool
82 // Atomically:
83 // if *ptr == old {
84 // *ptr = new
85 // return true
86 // } else {
87 // return false
88 // }
89 TEXT ·Cas(SB), NOSPLIT, $0-17
90 MOVD ptr+0(FP), R3
91 MOVWZ old+8(FP), R4
92 MOVWZ new+12(FP), R5
93 LWSYNC
94 cas_again:
95 LWAR (R3), R6
96 CMPW R6, R4
97 BNE cas_fail
98 STWCCC R5, (R3)
99 BNE cas_again
100 MOVD $1, R3
101 LWSYNC
102 MOVB R3, ret+16(FP)
103 RET
104 cas_fail:
105 LWSYNC
106 MOVB R0, ret+16(FP)
107 RET
108
109 // func Cas64(ptr *uint64, old, new uint64) bool
110 // Atomically:
111 // if *ptr == old {
112 // *ptr = new
113 // return true
114 // } else {
115 // return false
116 // }
117 TEXT ·Cas64(SB), NOSPLIT, $0-25
118 MOVD ptr+0(FP), R3
119 MOVD old+8(FP), R4
120 MOVD new+16(FP), R5
121 LWSYNC
122 cas64_again:
123 LDAR (R3), R6
124 CMP R6, R4
125 BNE cas64_fail
126 STDCCC R5, (R3)
127 BNE cas64_again
128 MOVD $1, R3
129 LWSYNC
130 MOVB R3, ret+24(FP)
131 RET
132 cas64_fail:
133 LWSYNC
134 MOVB R0, ret+24(FP)
135 RET
136
137 TEXT ·CasRel(SB), NOSPLIT, $0-17
138 MOVD ptr+0(FP), R3
139 MOVWZ old+8(FP), R4
140 MOVWZ new+12(FP), R5
141 LWSYNC
142 cas_again:
143 LWAR (R3), $0, R6 // 0 = Mutex release hint
144 CMPW R6, R4
145 BNE cas_fail
146 STWCCC R5, (R3)
147 BNE cas_again
148 MOVD $1, R3
149 MOVB R3, ret+16(FP)
150 RET
151 cas_fail:
152 MOVB R0, ret+16(FP)
153 RET
154
155 TEXT ·Casint32(SB), NOSPLIT, $0-17
156 BR ·Cas(SB)
157
158 TEXT ·Casint64(SB), NOSPLIT, $0-25
159 BR ·Cas64(SB)
160
161 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
162 BR ·Cas64(SB)
163
164 TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
165 BR ·Load64(SB)
166
167 TEXT ·LoadAcquintptr(SB), NOSPLIT|NOFRAME, $0-16
168 BR ·LoadAcq64(SB)
169
170 TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
171 BR ·Load64(SB)
172
173 TEXT ·Storeint32(SB), NOSPLIT, $0-12
174 BR ·Store(SB)
175
176 TEXT ·Storeint64(SB), NOSPLIT, $0-16
177 BR ·Store64(SB)
178
179 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
180 BR ·Store64(SB)
181
182 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
183 BR ·StoreRel64(SB)
184
185 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
186 BR ·Xadd64(SB)
187
188 TEXT ·Loadint32(SB), NOSPLIT, $0-12
189 BR ·Load(SB)
190
191 TEXT ·Loadint64(SB), NOSPLIT, $0-16
192 BR ·Load64(SB)
193
194 TEXT ·Xaddint32(SB), NOSPLIT, $0-20
195 BR ·Xadd(SB)
196
197 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
198 BR ·Xadd64(SB)
199
200 // func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
201 // Atomically:
202 // if *ptr == old {
203 // *ptr = new
204 // return true
205 // } else {
206 // return false
207 // }
208 TEXT ·Casp1(SB), NOSPLIT, $0-25
209 BR ·Cas64(SB)
210
211 // uint32 xadd(uint32 volatile *ptr, int32 delta)
212 // Atomically:
213 // *val += delta;
214 // return *val;
215 TEXT ·Xadd(SB), NOSPLIT, $0-20
216 MOVD ptr+0(FP), R4
217 MOVW delta+8(FP), R5
218 LWSYNC
219 LWAR (R4), R3
220 ADD R5, R3
221 STWCCC R3, (R4)
222 BNE -3(PC)
223 MOVW R3, ret+16(FP)
224 RET
225
226 // uint64 Xadd64(uint64 volatile *val, int64 delta)
227 // Atomically:
228 // *val += delta;
229 // return *val;
230 TEXT ·Xadd64(SB), NOSPLIT, $0-24
231 MOVD ptr+0(FP), R4
232 MOVD delta+8(FP), R5
233 LWSYNC
234 LDAR (R4), R3
235 ADD R5, R3
236 STDCCC R3, (R4)
237 BNE -3(PC)
238 MOVD R3, ret+16(FP)
239 RET
240
241 // uint8 Xchg(ptr *uint8, new uint8)
242 // Atomically:
243 // old := *ptr;
244 // *ptr = new;
245 // return old;
246 TEXT ·Xchg8(SB), NOSPLIT, $0-17
247 MOVD ptr+0(FP), R4
248 MOVB new+8(FP), R5
249 LWSYNC
250 LBAR (R4), R3
251 STBCCC R5, (R4)
252 BNE -2(PC)
253 ISYNC
254 MOVB R3, ret+16(FP)
255 RET
256
257 // uint32 Xchg(ptr *uint32, new uint32)
258 // Atomically:
259 // old := *ptr;
260 // *ptr = new;
261 // return old;
262 TEXT ·Xchg(SB), NOSPLIT, $0-20
263 MOVD ptr+0(FP), R4
264 MOVW new+8(FP), R5
265 LWSYNC
266 LWAR (R4), R3
267 STWCCC R5, (R4)
268 BNE -2(PC)
269 ISYNC
270 MOVW R3, ret+16(FP)
271 RET
272
273 // uint64 Xchg64(ptr *uint64, new uint64)
274 // Atomically:
275 // old := *ptr;
276 // *ptr = new;
277 // return old;
278 TEXT ·Xchg64(SB), NOSPLIT, $0-24
279 MOVD ptr+0(FP), R4
280 MOVD new+8(FP), R5
281 LWSYNC
282 LDAR (R4), R3
283 STDCCC R5, (R4)
284 BNE -2(PC)
285 ISYNC
286 MOVD R3, ret+16(FP)
287 RET
288
289 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
290 BR ·Xchg(SB)
291
292 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
293 BR ·Xchg64(SB)
294
295 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
296 BR ·Xchg64(SB)
297
298 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
299 BR ·Store64(SB)
300
301 TEXT ·Store(SB), NOSPLIT, $0-12
302 MOVD ptr+0(FP), R3
303 MOVW val+8(FP), R4
304 SYNC
305 MOVW R4, 0(R3)
306 RET
307
308 TEXT ·Store8(SB), NOSPLIT, $0-9
309 MOVD ptr+0(FP), R3
310 MOVB val+8(FP), R4
311 SYNC
312 MOVB R4, 0(R3)
313 RET
314
315 TEXT ·Store64(SB), NOSPLIT, $0-16
316 MOVD ptr+0(FP), R3
317 MOVD val+8(FP), R4
318 SYNC
319 MOVD R4, 0(R3)
320 RET
321
322 TEXT ·StoreRel(SB), NOSPLIT, $0-12
323 MOVD ptr+0(FP), R3
324 MOVW val+8(FP), R4
325 LWSYNC
326 MOVW R4, 0(R3)
327 RET
328
329 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
330 MOVD ptr+0(FP), R3
331 MOVD val+8(FP), R4
332 LWSYNC
333 MOVD R4, 0(R3)
334 RET
335
336 // void ·Or8(byte volatile*, byte);
337 TEXT ·Or8(SB), NOSPLIT, $0-9
338 MOVD ptr+0(FP), R3
339 MOVBZ val+8(FP), R4
340 LWSYNC
341 again:
342 LBAR (R3), R6
343 OR R4, R6
344 STBCCC R6, (R3)
345 BNE again
346 RET
347
348 // void ·And8(byte volatile*, byte);
349 TEXT ·And8(SB), NOSPLIT, $0-9
350 MOVD ptr+0(FP), R3
351 MOVBZ val+8(FP), R4
352 LWSYNC
353 again:
354 LBAR (R3), R6
355 AND R4, R6
356 STBCCC R6, (R3)
357 BNE again
358 RET
359
360 // func Or(addr *uint32, v uint32)
361 TEXT ·Or(SB), NOSPLIT, $0-12
362 MOVD ptr+0(FP), R3
363 MOVW val+8(FP), R4
364 LWSYNC
365 again:
366 LWAR (R3), R6
367 OR R4, R6
368 STWCCC R6, (R3)
369 BNE again
370 RET
371
372 // func And(addr *uint32, v uint32)
373 TEXT ·And(SB), NOSPLIT, $0-12
374 MOVD ptr+0(FP), R3
375 MOVW val+8(FP), R4
376 LWSYNC
377 again:
378 LWAR (R3),R6
379 AND R4, R6
380 STWCCC R6, (R3)
381 BNE again
382 RET
383
384 // func Or32(addr *uint32, v uint32) old uint32
385 TEXT ·Or32(SB), NOSPLIT, $0-20
386 MOVD ptr+0(FP), R3
387 MOVW val+8(FP), R4
388 LWSYNC
389 again:
390 LWAR (R3), R6
391 OR R4, R6, R7
392 STWCCC R7, (R3)
393 BNE again
394 MOVW R6, ret+16(FP)
395 RET
396
397 // func And32(addr *uint32, v uint32) old uint32
398 TEXT ·And32(SB), NOSPLIT, $0-20
399 MOVD ptr+0(FP), R3
400 MOVW val+8(FP), R4
401 LWSYNC
402 again:
403 LWAR (R3),R6
404 AND R4, R6, R7
405 STWCCC R7, (R3)
406 BNE again
407 MOVW R6, ret+16(FP)
408 RET
409
410 // func Or64(addr *uint64, v uint64) old uint64
411 TEXT ·Or64(SB), NOSPLIT, $0-24
412 MOVD ptr+0(FP), R3
413 MOVD val+8(FP), R4
414 LWSYNC
415 again:
416 LDAR (R3), R6
417 OR R4, R6, R7
418 STDCCC R7, (R3)
419 BNE again
420 MOVD R6, ret+16(FP)
421 RET
422
423 // func And64(addr *uint64, v uint64) old uint64
424 TEXT ·And64(SB), NOSPLIT, $0-24
425 MOVD ptr+0(FP), R3
426 MOVD val+8(FP), R4
427 LWSYNC
428 again:
429 LDAR (R3),R6
430 AND R4, R6, R7
431 STDCCC R7, (R3)
432 BNE again
433 MOVD R6, ret+16(FP)
434 RET
435
436 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
437 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
438 JMP ·And64(SB)
439
440 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
441 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
442 JMP ·Or64(SB)
443
View as plain text