1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #ifndef _SYS_ATOMIC_H
28 #define _SYS_ATOMIC_H
29
30 #pragma ident "%Z%%M% %I% %E% SMI"
31
32 #include <sys/types.h>
33 #include <sys/inttypes.h>
34
35 #ifdef __cplusplus
36 extern "C" {
37 #endif
38
39 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
40 (defined(__i386) || defined(__amd64))
41 #include <asm/atomic.h>
42 #endif
43
44 #if defined(_KERNEL) || defined(__STDC__)
45 /*
46 * Increment target.
47 */
48 extern void atomic_inc_8(volatile uint8_t *);
49 extern void atomic_inc_uchar(volatile uchar_t *);
50 extern void atomic_inc_16(volatile uint16_t *);
51 extern void atomic_inc_ushort(volatile ushort_t *);
52 extern void atomic_inc_32(volatile uint32_t *);
53 extern void atomic_inc_uint(volatile uint_t *);
54 extern void atomic_inc_ulong(volatile ulong_t *);
55 #if defined(_KERNEL) || defined(_INT64_TYPE)
56 extern void atomic_inc_64(volatile uint64_t *);
57 #endif
58
59 /*
60 * Decrement target
61 */
62 extern void atomic_dec_8(volatile uint8_t *);
63 extern void atomic_dec_uchar(volatile uchar_t *);
64 extern void atomic_dec_16(volatile uint16_t *);
65 extern void atomic_dec_ushort(volatile ushort_t *);
66 extern void atomic_dec_32(volatile uint32_t *);
67 extern void atomic_dec_uint(volatile uint_t *);
68 extern void atomic_dec_ulong(volatile ulong_t *);
69 #if defined(_KERNEL) || defined(_INT64_TYPE)
70 extern void atomic_dec_64(volatile uint64_t *);
71 #endif
72
73 /*
74 * Add delta to target
75 */
76 extern void atomic_add_8(volatile uint8_t *, int8_t);
77 extern void atomic_add_char(volatile uchar_t *, signed char);
78 extern void atomic_add_16(volatile uint16_t *, int16_t);
79 extern void atomic_add_short(volatile ushort_t *, short);
80 extern void atomic_add_32(volatile uint32_t *, int32_t);
81 extern void atomic_add_int(volatile uint_t *, int);
82 extern void atomic_add_ptr(volatile void *, ssize_t);
83 extern void atomic_add_long(volatile ulong_t *, long);
84 #if defined(_KERNEL) || defined(_INT64_TYPE)
85 extern void atomic_add_64(volatile uint64_t *, int64_t);
86 #endif
87
88 /*
89 * logical OR bits with target
90 */
91 extern void atomic_or_8(volatile uint8_t *, uint8_t);
92 extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
93 extern void atomic_or_16(volatile uint16_t *, uint16_t);
94 extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
95 extern void atomic_or_32(volatile uint32_t *, uint32_t);
96 extern void atomic_or_uint(volatile uint_t *, uint_t);
97 extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
98 #if defined(_KERNEL) || defined(_INT64_TYPE)
99 extern void atomic_or_64(volatile uint64_t *, uint64_t);
100 #endif
101
102 /*
103 * logical AND bits with target
104 */
105 extern void atomic_and_8(volatile uint8_t *, uint8_t);
106 extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
107 extern void atomic_and_16(volatile uint16_t *, uint16_t);
108 extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
109 extern void atomic_and_32(volatile uint32_t *, uint32_t);
110 extern void atomic_and_uint(volatile uint_t *, uint_t);
111 extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
112 #if defined(_KERNEL) || defined(_INT64_TYPE)
113 extern void atomic_and_64(volatile uint64_t *, uint64_t);
114 #endif
115
116 /*
117 * As above, but return the new value. Note that these _nv() variants are
118 * substantially more expensive on some platforms than the no-return-value
119 * versions above, so don't use them unless you really need to know the
120 * new value *atomically* (e.g. when decrementing a reference count and
121 * checking whether it went to zero).
122 */
123
124 /*
125 * Increment target and return new value.
126 */
127 extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
128 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
129 extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
130 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
131 extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
132 extern uint_t atomic_inc_uint_nv(volatile uint_t *);
133 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
134 #if defined(_KERNEL) || defined(_INT64_TYPE)
135 extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
136 #endif
137
138 /*
139 * Decrement target and return new value.
140 */
141 extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
142 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
143 extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
144 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
145 extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
146 extern uint_t atomic_dec_uint_nv(volatile uint_t *);
147 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
148 #if defined(_KERNEL) || defined(_INT64_TYPE)
149 extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
150 #endif
151
152 /*
153 * Add delta to target
154 */
155 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
156 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
157 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
158 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
159 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
160 extern uint_t atomic_add_int_nv(volatile uint_t *, int);
161 extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
162 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
163 #if defined(_KERNEL) || defined(_INT64_TYPE)
164 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
165 #endif
166
167 /*
168 * logical OR bits with target and return new value.
169 */
170 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
171 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
172 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
173 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
174 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
175 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
176 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
177 #if defined(_KERNEL) || defined(_INT64_TYPE)
178 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
179 #endif
180
181 /*
182 * logical AND bits with target and return new value.
183 */
184 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
185 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
186 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
187 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
188 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
189 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
190 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
191 #if defined(_KERNEL) || defined(_INT64_TYPE)
192 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
193 #endif
194
195 /*
196 * If *arg1 == arg2, set *arg1 = arg3; return old value
197 */
198 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
199 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
200 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
201 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
202 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
203 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
204 extern void *atomic_cas_ptr(volatile void *, void *, void *);
205 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
206 #if defined(_KERNEL) || defined(_INT64_TYPE)
207 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
208 #endif
209
210 /*
211 * Swap target and return old value
212 */
213 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
214 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
215 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
216 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
217 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
218 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
219 extern void *atomic_swap_ptr(volatile void *, void *);
220 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
221 #if defined(_KERNEL) || defined(_INT64_TYPE)
222 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
223 #endif
224
225 /*
226 * Perform an exclusive atomic bit set/clear on a target.
227 * Returns 0 if bit was sucessfully set/cleared, or -1
228 * if the bit was already set/cleared.
229 */
230 extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
231 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
232
233 /*
234 * Generic memory barrier used during lock entry, placed after the
235 * memory operation that acquires the lock to guarantee that the lock
236 * protects its data. No stores from after the memory barrier will
237 * reach visibility, and no loads from after the barrier will be
238 * resolved, before the lock acquisition reaches global visibility.
239 */
240 extern void membar_enter(void);
241
242 /*
243 * Generic memory barrier used during lock exit, placed before the
244 * memory operation that releases the lock to guarantee that the lock
245 * protects its data. All loads and stores issued before the barrier
246 * will be resolved before the subsequent lock update reaches visibility.
247 */
248 extern void membar_exit(void);
249
250 /*
251 * Arrange that all stores issued before this point in the code reach
252 * global visibility before any stores that follow; useful in producer
253 * modules that update a data item, then set a flag that it is available.
254 * The memory barrier guarantees that the available flag is not visible
255 * earlier than the updated data, i.e. it imposes store ordering.
256 */
257 extern void membar_producer(void);
258
259 /*
260 * Arrange that all loads issued before this point in the code are
261 * completed before any subsequent loads; useful in consumer modules
262 * that check to see if data is available and read the data.
263 * The memory barrier guarantees that the data is not sampled until
264 * after the available flag has been seen, i.e. it imposes load ordering.
265 */
266 extern void membar_consumer(void);
267 #endif
268
269 #if !defined(_KERNEL) && !defined(__STDC__)
270 extern void atomic_inc_8();
271 extern void atomic_inc_uchar();
272 extern void atomic_inc_16();
273 extern void atomic_inc_ushort();
274 extern void atomic_inc_32();
275 extern void atomic_inc_uint();
276 extern void atomic_inc_ulong();
277 #if defined(_INT64_TYPE)
278 extern void atomic_inc_64();
279 #endif /* defined(_INT64_TYPE) */
280 extern void atomic_dec_8();
281 extern void atomic_dec_uchar();
282 extern void atomic_dec_16();
283 extern void atomic_dec_ushort();
284 extern void atomic_dec_32();
285 extern void atomic_dec_uint();
286 extern void atomic_dec_ulong();
287 #if defined(_INT64_TYPE)
288 extern void atomic_dec_64();
289 #endif /* defined(_INT64_TYPE) */
290 extern void atomic_add_8();
291 extern void atomic_add_char();
292 extern void atomic_add_16();
293 extern void atomic_add_short();
294 extern void atomic_add_32();
295 extern void atomic_add_int();
296 extern void atomic_add_ptr();
297 extern void atomic_add_long();
298 #if defined(_INT64_TYPE)
299 extern void atomic_add_64();
300 #endif /* defined(_INT64_TYPE) */
301 extern void atomic_or_8();
302 extern void atomic_or_uchar();
303 extern void atomic_or_16();
304 extern void atomic_or_ushort();
305 extern void atomic_or_32();
306 extern void atomic_or_uint();
307 extern void atomic_or_ulong();
308 #if defined(_INT64_TYPE)
309 extern void atomic_or_64();
310 #endif /* defined(_INT64_TYPE) */
311 extern void atomic_and_8();
312 extern void atomic_and_uchar();
313 extern void atomic_and_16();
314 extern void atomic_and_ushort();
315 extern void atomic_and_32();
316 extern void atomic_and_uint();
317 extern void atomic_and_ulong();
318 #if defined(_INT64_TYPE)
319 extern void atomic_and_64();
320 #endif /* defined(_INT64_TYPE) */
321 extern uint8_t atomic_inc_8_nv();
322 extern uchar_t atomic_inc_uchar_nv();
323 extern uint16_t atomic_inc_16_nv();
324 extern ushort_t atomic_inc_ushort_nv();
325 extern uint32_t atomic_inc_32_nv();
326 extern uint_t atomic_inc_uint_nv();
327 extern ulong_t atomic_inc_ulong_nv();
328 #if defined(_INT64_TYPE)
329 extern uint64_t atomic_inc_64_nv();
330 #endif /* defined(_INT64_TYPE) */
331 extern uint8_t atomic_dec_8_nv();
332 extern uchar_t atomic_dec_uchar_nv();
333 extern uint16_t atomic_dec_16_nv();
334 extern ushort_t atomic_dec_ushort_nv();
335 extern uint32_t atomic_dec_32_nv();
336 extern uint_t atomic_dec_uint_nv();
337 extern ulong_t atomic_dec_ulong_nv();
338 #if defined(_INT64_TYPE)
339 extern uint64_t atomic_dec_64_nv();
340 #endif /* defined(_INT64_TYPE) */
341 extern uint8_t atomic_add_8_nv();
342 extern uchar_t atomic_add_char_nv();
343 extern uint16_t atomic_add_16_nv();
344 extern ushort_t atomic_add_short_nv();
345 extern uint32_t atomic_add_32_nv();
346 extern uint_t atomic_add_int_nv();
347 extern void *atomic_add_ptr_nv();
348 extern ulong_t atomic_add_long_nv();
349 #if defined(_INT64_TYPE)
350 extern uint64_t atomic_add_64_nv();
351 #endif /* defined(_INT64_TYPE) */
352 extern uint8_t atomic_or_8_nv();
353 extern uchar_t atomic_or_uchar_nv();
354 extern uint16_t atomic_or_16_nv();
355 extern ushort_t atomic_or_ushort_nv();
356 extern uint32_t atomic_or_32_nv();
357 extern uint_t atomic_or_uint_nv();
358 extern ulong_t atomic_or_ulong_nv();
359 #if defined(_INT64_TYPE)
360 extern uint64_t atomic_or_64_nv();
361 #endif /* defined(_INT64_TYPE) */
362 extern uint8_t atomic_and_8_nv();
363 extern uchar_t atomic_and_uchar_nv();
364 extern uint16_t atomic_and_16_nv();
365 extern ushort_t atomic_and_ushort_nv();
366 extern uint32_t atomic_and_32_nv();
367 extern uint_t atomic_and_uint_nv();
368 extern ulong_t atomic_and_ulong_nv();
369 #if defined(_INT64_TYPE)
370 extern uint64_t atomic_and_64_nv();
371 #endif /* defined(_INT64_TYPE) */
372 extern uint8_t atomic_cas_8();
373 extern uchar_t atomic_cas_uchar();
374 extern uint16_t atomic_cas_16();
375 extern ushort_t atomic_cas_ushort();
376 extern uint32_t atomic_cas_32();
377 extern uint_t atomic_cas_uint();
378 extern void *atomic_cas_ptr();
379 extern ulong_t atomic_cas_ulong();
380 #if defined(_INT64_TYPE)
381 extern uint64_t atomic_cas_64();
382 #endif /* defined(_INT64_TYPE) */
383 extern uint8_t atomic_swap_8();
384 extern uchar_t atomic_swap_uchar();
385 extern uint16_t atomic_swap_16();
386 extern ushort_t atomic_swap_ushort();
387 extern uint32_t atomic_swap_32();
388 extern uint_t atomic_swap_uint();
389 extern void *atomic_swap_ptr();
390 extern ulong_t atomic_swap_ulong();
391 #if defined(_INT64_TYPE)
392 extern uint64_t atomic_swap_64();
393 #endif /* defined(_INT64_TYPE) */
394
395
396 extern int atomic_set_long_excl();
397 extern int atomic_clear_long_excl();
398
399 extern void membar_enter();
400 extern void membar_exit();
401 extern void membar_producer();
402 extern void membar_consumer();
403
404 #endif
405
406 #if defined(_KERNEL)
407
408 #if defined(_LP64) || defined(_ILP32)
409 #define atomic_add_ip atomic_add_long
410 #define atomic_add_ip_nv atomic_add_long_nv
411 #define casip atomic_cas_ulong
412 #endif
413
414 #if defined(__sparc)
415 extern uint8_t ldstub(uint8_t *);
416 #endif
417
418 /*
419 * Legacy kernel interfaces; they will go away (eventually).
420 */
421 extern uint8_t cas8(uint8_t *, uint8_t, uint8_t);
422 extern uint32_t cas32(uint32_t *, uint32_t, uint32_t);
423 extern uint64_t cas64(uint64_t *, uint64_t, uint64_t);
424 extern ulong_t caslong(ulong_t *, ulong_t, ulong_t);
425 extern void *casptr(void *, void *, void *);
426 extern void atomic_and_long(ulong_t *, ulong_t);
427 extern void atomic_or_long(ulong_t *, ulong_t);
428 #if defined(__sparc)
429 extern uint32_t swapl(uint32_t *, uint32_t);
430 #endif
431
432 #endif /* _KERNEL */
433
434 #ifdef __cplusplus
435 }
436 #endif
437
438 #endif /* _SYS_ATOMIC_H */