1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #pragma ident "%Z%%M% %I% %E% SMI"
27
28 #if defined(lint)
29 #include <sys/types.h>
30 #include <sys/thread.h>
31 #include <sys/cpuvar.h>
32 #else /* lint */
33 #include "assym.h"
34 #endif /* lint */
35
36 #include <sys/t_lock.h>
37 #include <sys/mutex.h>
38 #include <sys/mutex_impl.h>
39 #include <sys/rwlock_impl.h>
40 #include <sys/asm_linkage.h>
41 #include <sys/machlock.h>
42 #include <sys/machthread.h>
43 #include <sys/lockstat.h>
44
45 /* #define DEBUG */
46
47 #ifdef DEBUG
48 #include <sys/machparam.h>
49 #endif /* DEBUG */
50
51 /************************************************************************
52 * ATOMIC OPERATIONS
53 */
54
55 /*
56 * uint8_t ldstub(uint8_t *cp)
57 *
58 * Store 0xFF at the specified location, and return its previous content.
59 */
60
61 #if defined(lint)
62 uint8_t
63 ldstub(uint8_t *cp)
64 {
65 uint8_t rv;
66 rv = *cp;
67 *cp = 0xFF;
68 return rv;
69 }
70 #else /* lint */
71
72 ENTRY(ldstub)
73 retl
74 ldstub [%o0], %o0
75 SET_SIZE(ldstub)
76
77 #endif /* lint */
78
79 /************************************************************************
80 * MEMORY BARRIERS -- see atomic.h for full descriptions.
81 */
82
83 #if defined(lint)
84
85 void
86 membar_enter(void)
87 {}
88
89 void
90 membar_exit(void)
91 {}
92
93 void
94 membar_producer(void)
95 {}
96
97 void
98 membar_consumer(void)
99 {}
100
101 #else /* lint */
102
103 #ifdef SF_ERRATA_51
104 .align 32
105 ENTRY(membar_return)
106 retl
107 nop
108 SET_SIZE(membar_return)
109 #define MEMBAR_RETURN ba,pt %icc, membar_return
110 #else
111 #define MEMBAR_RETURN retl
112 #endif
113
114 ENTRY(membar_enter)
115 MEMBAR_RETURN
116 membar #StoreLoad|#StoreStore
117 SET_SIZE(membar_enter)
118
119 ENTRY(membar_exit)
120 MEMBAR_RETURN
121 membar #LoadStore|#StoreStore
122 SET_SIZE(membar_exit)
123
124 ENTRY(membar_producer)
125 MEMBAR_RETURN
126 membar #StoreStore
127 SET_SIZE(membar_producer)
128
129 ENTRY(membar_consumer)
130 MEMBAR_RETURN
131 membar #LoadLoad
132 SET_SIZE(membar_consumer)
133
134 #endif /* lint */
135
136 /************************************************************************
137 * MINIMUM LOCKS
138 */
139
140 #if defined(lint)
141
142 /*
143 * lock_try(lp), ulock_try(lp)
144 * - returns non-zero on success.
145 * - doesn't block interrupts so don't use this to spin on a lock.
146 * - uses "0xFF is busy, anything else is free" model.
147 *
148 * ulock_try() is for a lock in the user address space.
149 * For all V7/V8 sparc systems they are same since the kernel and
150 * user are mapped in a user' context.
151 * For V9 platforms the lock_try and ulock_try are different impl.
152 */
153
154 int
155 lock_try(lock_t *lp)
156 {
157 return (0xFF ^ ldstub(lp));
158 }
159
160 int
161 lock_spin_try(lock_t *lp)
162 {
163 return (0xFF ^ ldstub(lp));
164 }
165
166 void
167 lock_set(lock_t *lp)
168 {
169 extern void lock_set_spin(lock_t *);
170
171 if (!lock_try(lp))
172 lock_set_spin(lp);
173 membar_enter();
174 }
175
176 void
177 lock_clear(lock_t *lp)
178 {
179 membar_exit();
180 *lp = 0;
181 }
182
183 int
184 ulock_try(lock_t *lp)
185 {
186 return (0xFF ^ ldstub(lp));
187 }
188
189 void
190 ulock_clear(lock_t *lp)
191 {
192 membar_exit();
193 *lp = 0;
194 }
195
196 #else /* lint */
197
198 .align 32
199 ENTRY(lock_try)
200 ldstub [%o0], %o1 ! try to set lock, get value in %o1
201 brnz,pn %o1, 1f
202 membar #LoadLoad
203 .lock_try_lockstat_patch_point:
204 retl
205 or %o0, 1, %o0 ! ensure lo32 != 0
206 1:
207 retl
208 clr %o0
209 SET_SIZE(lock_try)
210
211 .align 32
212 ENTRY(lock_spin_try)
213 ldstub [%o0], %o1 ! try to set lock, get value in %o1
214 brnz,pn %o1, 1f
215 membar #LoadLoad
216 retl
217 or %o0, 1, %o0 ! ensure lo32 != 0
218 1:
219 retl
220 clr %o0
221 SET_SIZE(lock_spin_try)
222
223 .align 32
224 ENTRY(lock_set)
225 ldstub [%o0], %o1
226 brnz,pn %o1, 1f ! go to C for the hard case
227 membar #LoadLoad
228 .lock_set_lockstat_patch_point:
229 retl
230 nop
231 1:
232 sethi %hi(lock_set_spin), %o2 ! load up for jump to C
233 jmp %o2 + %lo(lock_set_spin)
234 nop ! delay: do nothing
235 SET_SIZE(lock_set)
236
237 ENTRY(lock_clear)
238 membar #LoadStore|#StoreStore
239 .lock_clear_lockstat_patch_point:
240 retl
241 clrb [%o0]
242 SET_SIZE(lock_clear)
243
244 .align 32
245 ENTRY(ulock_try)
246 ldstuba [%o0]ASI_USER, %o1 ! try to set lock, get value in %o1
247 xor %o1, 0xff, %o0 ! delay - return non-zero if success
248 retl
249 membar #LoadLoad
250 SET_SIZE(ulock_try)
251
252 ENTRY(ulock_clear)
253 membar #LoadStore|#StoreStore
254 retl
255 stba %g0, [%o0]ASI_USER ! clear lock
256 SET_SIZE(ulock_clear)
257
258 #endif /* lint */
259
260
261 /*
262 * lock_set_spl(lp, new_pil, *old_pil_addr)
263 * Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr.
264 */
265
266 #if defined(lint)
267
268 /* ARGSUSED */
269 void
270 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil_addr)
271 {
272 extern int splr(int);
273 extern void lock_set_spl_spin(lock_t *, int, u_short *, int);
274 int old_pil;
275
276 old_pil = splr(new_pil);
277 if (!lock_try(lp)) {
278 lock_set_spl_spin(lp, new_pil, old_pil_addr, old_pil);
279 } else {
280 *old_pil_addr = (u_short)old_pil;
281 membar_enter();
282 }
283 }
284
285 #else /* lint */
286
287 ENTRY(lock_set_spl)
288 rdpr %pil, %o3 ! %o3 = current pil
289 cmp %o3, %o1 ! is current pil high enough?
290 bl,a,pt %icc, 1f ! if not, write %pil in delay
291 wrpr %g0, %o1, %pil
292 1:
293 ldstub [%o0], %o4 ! try the lock
294 brnz,pn %o4, 2f ! go to C for the miss case
295 membar #LoadLoad
296 .lock_set_spl_lockstat_patch_point:
297 retl
298 sth %o3, [%o2] ! delay - save original pil
299 2:
300 sethi %hi(lock_set_spl_spin), %o5 ! load up jmp to C
301 jmp %o5 + %lo(lock_set_spl_spin) ! jmp to lock_set_spl_spin
302 nop ! delay: do nothing
303 SET_SIZE(lock_set_spl)
304
305 #endif /* lint */
306
307 /*
308 * lock_clear_splx(lp, s)
309 */
310
311 #if defined(lint)
312
313 void
314 lock_clear_splx(lock_t *lp, int s)
315 {
316 extern void splx(int);
317
318 lock_clear(lp);
319 splx(s);
320 }
321
322 #else /* lint */
323
324 ENTRY(lock_clear_splx)
325 ldn [THREAD_REG + T_CPU], %o2 ! get CPU pointer
326 membar #LoadStore|#StoreStore
327 ld [%o2 + CPU_BASE_SPL], %o2
328 clrb [%o0] ! clear lock
329 cmp %o2, %o1 ! compare new to base
330 movl %xcc, %o1, %o2 ! use new pri if base is less
331 .lock_clear_splx_lockstat_patch_point:
332 retl
333 wrpr %g0, %o2, %pil
334 SET_SIZE(lock_clear_splx)
335
336 #endif /* lint */
337
338 /*
339 * mutex_enter() and mutex_exit().
340 *
341 * These routines handle the simple cases of mutex_enter() (adaptive
342 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
343 * If anything complicated is going on we punt to mutex_vector_enter().
344 *
345 * mutex_tryenter() is similar to mutex_enter() but returns zero if
346 * the lock cannot be acquired, nonzero on success.
347 *
348 * If mutex_exit() gets preempted in the window between checking waiters
349 * and clearing the lock, we can miss wakeups. Disabling preemption
350 * in the mutex code is prohibitively expensive, so instead we detect
351 * mutex preemption by examining the trapped PC in the interrupt path.
352 * If we interrupt a thread in mutex_exit() that has not yet cleared
353 * the lock, pil_interrupt() resets its PC back to the beginning of
354 * mutex_exit() so it will check again for waiters when it resumes.
355 *
356 * The lockstat code below is activated when the lockstat driver
357 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
358 * Note that we don't need to test lockstat_event_mask here -- we won't
359 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
360 */
361
362 #if defined (lint)
363
364 /* ARGSUSED */
365 void
366 mutex_enter(kmutex_t *lp)
367 {}
368
369 /* ARGSUSED */
370 int
371 mutex_tryenter(kmutex_t *lp)
372 { return (0); }
373
374 /* ARGSUSED */
375 void
376 mutex_exit(kmutex_t *lp)
377 {}
378
379 /* ARGSUSED */
380 void *
381 mutex_owner_running(mutex_impl_t *lp)
382 { return (NULL); }
383
384 #else
385 .align 32
386 ENTRY(mutex_enter)
387 mov THREAD_REG, %o1
388 casx [%o0], %g0, %o1 ! try to acquire as adaptive
389 brnz,pn %o1, 1f ! locked or wrong type
390 membar #LoadLoad
391 .mutex_enter_lockstat_patch_point:
392 retl
393 nop
394 1:
395 sethi %hi(mutex_vector_enter), %o2 ! load up for jump to C
396 jmp %o2 + %lo(mutex_vector_enter)
397 nop
398 SET_SIZE(mutex_enter)
399
400 ENTRY(mutex_tryenter)
401 mov THREAD_REG, %o1
402 casx [%o0], %g0, %o1 ! try to acquire as adaptive
403 brnz,pn %o1, 1f ! locked or wrong type continue
404 membar #LoadLoad
405 .mutex_tryenter_lockstat_patch_point:
406 retl
407 or %o0, 1, %o0 ! ensure lo32 != 0
408 1:
409 sethi %hi(mutex_vector_tryenter), %o2 ! hi bits
410 jmp %o2 + %lo(mutex_vector_tryenter) ! go to C
411 nop
412 SET_SIZE(mutex_tryenter)
413
414 ENTRY(mutex_adaptive_tryenter)
415 mov THREAD_REG, %o1
416 casx [%o0], %g0, %o1 ! try to acquire as adaptive
417 brnz,pn %o1, 0f ! locked or wrong type
418 membar #LoadLoad
419 retl
420 or %o0, 1, %o0 ! ensure lo32 != 0
421 0:
422 retl
423 mov %g0, %o0
424 SET_SIZE(mutex_adaptive_tryenter)
425
426 ! these need to be together and cache aligned for performance.
427 .align 64
428 .global mutex_exit_critical_size
429 .global mutex_exit_critical_start
430 .global mutex_owner_running_critical_size
431 .global mutex_owner_running_critical_start
432
433 mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start
434
435 .align 32
436
437 ENTRY(mutex_exit)
438 mutex_exit_critical_start: ! If we are interrupted, restart here
439 ldn [%o0], %o1 ! get the owner field
440 membar #LoadStore|#StoreStore
441 cmp THREAD_REG, %o1 ! do we own lock with no waiters?
442 be,a,pt %ncc, 1f ! if so, drive on ...
443 stn %g0, [%o0] ! delay: clear lock if we owned it
444 .mutex_exit_critical_end: ! for pil_interrupt() hook
445 ba,a,pt %xcc, mutex_vector_exit ! go to C for the hard cases
446 1:
447 .mutex_exit_lockstat_patch_point:
448 retl
449 nop
450 SET_SIZE(mutex_exit)
451
452 mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start
453
454 .align 32
455
456 ENTRY(mutex_owner_running)
457 mutex_owner_running_critical_start: ! If interrupted restart here
458 ldn [%o0], %o1 ! get the owner field
459 and %o1, MUTEX_THREAD, %o1 ! remove the waiters bit if any
460 brz,pn %o1, 1f ! if so, drive on ...
461 nop
462 ldn [%o1+T_CPU], %o2 ! get owner->t_cpu
463 ldn [%o2+CPU_THREAD], %o3 ! get owner->t_cpu->cpu_thread
464 .mutex_owner_running_critical_end: ! for pil_interrupt() hook
465 cmp %o1, %o3 ! owner == running thread?
466 be,a,pt %xcc, 2f ! yes, go return cpu
467 nop
468 1:
469 retl
470 mov %g0, %o0 ! return 0 (owner not running)
471 2:
472 retl
473 mov %o2, %o0 ! owner running, return cpu
474 SET_SIZE(mutex_owner_running)
475
476 #endif /* lint */
477
478 /*
479 * rw_enter() and rw_exit().
480 *
481 * These routines handle the simple cases of rw_enter (write-locking an unheld
482 * lock or read-locking a lock that's neither write-locked nor write-wanted)
483 * and rw_exit (no waiters or not the last reader). If anything complicated
484 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
485 */
486 #if defined(lint)
487
488 /* ARGSUSED */
489 void
490 rw_enter(krwlock_t *lp, krw_t rw)
491 {}
492
493 /* ARGSUSED */
494 void
495 rw_exit(krwlock_t *lp)
496 {}
497
498 #else
499
500 .align 16
501 ENTRY(rw_enter)
502 cmp %o1, RW_WRITER ! entering as writer?
503 be,a,pn %icc, 2f ! if so, go do it ...
504 or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
505 ld [THREAD_REG + T_KPRI_REQ], %o3 ! begin THREAD_KPRI_REQUEST()
506 ldn [%o0], %o4 ! %o4 = old lock value
507 inc %o3 ! bump kpri
508 st %o3, [THREAD_REG + T_KPRI_REQ] ! store new kpri
509 1:
510 andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted?
511 bz,pt %xcc, 3f ! if so, prepare to block
512 add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count
513 sethi %hi(rw_enter_sleep), %o2 ! load up jump
514 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep
515 nop ! delay: do nothing
516 3:
517 casx [%o0], %o4, %o5 ! try to grab read lock
518 cmp %o4, %o5 ! did we get it?
519 #ifdef sun4v
520 be,a,pt %xcc, 0f
521 membar #LoadLoad
522 sethi %hi(rw_enter_sleep), %o2 ! load up jump
523 jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep
524 nop ! delay: do nothing
525 0:
526 #else /* sun4v */
527 bne,pn %xcc, 1b ! if not, try again
528 mov %o5, %o4 ! delay: %o4 = old lock value
529 membar #LoadLoad
530 #endif /* sun4v */
531 .rw_read_enter_lockstat_patch_point:
532 retl
533 nop
534 2:
535 casx [%o0], %g0, %o5 ! try to grab write lock
536 brz,pt %o5, 4f ! branch around if we got it
537 membar #LoadLoad ! done regardless of where we go
538 sethi %hi(rw_enter_sleep), %o2
539 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not
540 nop ! delay: do nothing
541 4:
542 .rw_write_enter_lockstat_patch_point:
543 retl
544 nop
545 SET_SIZE(rw_enter)
546
547 .align 16
548 ENTRY(rw_exit)
549 ldn [%o0], %o4 ! %o4 = old lock value
550 membar #LoadStore|#StoreStore ! membar_exit()
551 subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader
552 bnz,pn %xcc, 2f ! single reader, no waiters?
553 clr %o1
554 1:
555 ld [THREAD_REG + T_KPRI_REQ], %g1 ! begin THREAD_KPRI_RELEASE()
556 srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat)
557 casx [%o0], %o4, %o5 ! try to drop lock
558 cmp %o4, %o5 ! did we succeed?
559 bne,pn %xcc, rw_exit_wakeup ! if not, go to C
560 dec %g1 ! delay: drop kpri
561 .rw_read_exit_lockstat_patch_point:
562 retl
563 st %g1, [THREAD_REG + T_KPRI_REQ] ! delay: store new kpri
564 2:
565 andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer?
566 bnz,a,pt %xcc, 3f
567 or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
568 cmp %o5, RW_READ_LOCK ! would lock still be held?
569 bge,pt %xcc, 1b ! if so, go ahead and drop it
570 nop
571 ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters
572 nop
573 3:
574 casx [%o0], %o4, %o1 ! try to drop write lock
575 cmp %o4, %o1 ! did we succeed?
576 bne,pn %xcc, rw_exit_wakeup ! if not, go to C
577 nop
578 .rw_write_exit_lockstat_patch_point:
579 retl
580 nop
581 SET_SIZE(rw_exit)
582
583 #endif
584
585 #if defined(lint)
586
587 void
588 lockstat_hot_patch(void)
589 {}
590
591 #else
592
593 #define RETL 0x81c3e008
594 #define NOP 0x01000000
595 #define BA 0x10800000
596
597 #define DISP22 ((1 << 22) - 1)
598 #define ANNUL 0x20000000
599
600 #define HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs) \
601 ba 1f; \
602 rd %pc, %o0; \
603 save %sp, -SA(MINFRAME), %sp; \
604 set lockstat_probemap, %l1; \
605 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \
606 brz,pn %o0, 0f; \
607 ldub [THREAD_REG + T_LOCKSTAT], %l0; \
608 add %l0, 1, %l2; \
609 stub %l2, [THREAD_REG + T_LOCKSTAT]; \
610 set lockstat_probe, %g1; \
611 ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \
612 brz,a,pn %o0, 0f; \
613 stub %l0, [THREAD_REG + T_LOCKSTAT]; \
614 ldn [%g1], %g2; \
615 mov rs, %o2; \
616 jmpl %g2, %o7; \
617 mov %i0, %o1; \
618 stub %l0, [THREAD_REG + T_LOCKSTAT]; \
619 0: ret; \
620 restore %g0, 1, %o0; /* for mutex_tryenter / lock_try */ \
621 1: set addr, %o1; \
622 sub %o0, %o1, %o0; \
623 srl %o0, 2, %o0; \
624 inc %o0; \
625 set DISP22, %o1; \
626 and %o1, %o0, %o0; \
627 set BA, %o1; \
628 or %o1, %o0, %o0; \
629 sethi %hi(annul), %o2; \
630 add %o0, %o2, %o2; \
631 set addr, %o0; \
632 set normal_instr, %o1; \
633 ld [%i0 + (event * DTRACE_IDSIZE)], %o3; \
634 tst %o3; \
635 movnz %icc, %o2, %o1; \
636 call hot_patch_kernel_text; \
637 mov 4, %o2; \
638 membar #Sync
639
640 #define HOT_PATCH(addr, event, normal_instr) \
641 HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1)
642
643 #define HOT_PATCH_ARG(addr, event, normal_instr, arg) \
644 HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg)
645
646 #define HOT_PATCH_ANNULLED(addr, event, normal_instr) \
647 HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1)
648
649 ENTRY(lockstat_hot_patch)
650 save %sp, -SA(MINFRAME), %sp
651 set lockstat_probemap, %i0
652 HOT_PATCH(.mutex_enter_lockstat_patch_point,
653 LS_MUTEX_ENTER_ACQUIRE, RETL)
654 HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point,
655 LS_MUTEX_TRYENTER_ACQUIRE, RETL)
656 HOT_PATCH(.mutex_exit_lockstat_patch_point,
657 LS_MUTEX_EXIT_RELEASE, RETL)
658 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
659 LS_RW_ENTER_ACQUIRE, RETL)
660 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
661 LS_RW_ENTER_ACQUIRE, RETL)
662 HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point,
663 LS_RW_EXIT_RELEASE, RETL, RW_WRITER)
664 HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point,
665 LS_RW_EXIT_RELEASE, RETL, RW_READER)
666 HOT_PATCH(.lock_set_lockstat_patch_point,
667 LS_LOCK_SET_ACQUIRE, RETL)
668 HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point,
669 LS_LOCK_TRY_ACQUIRE, RETL)
670 HOT_PATCH(.lock_clear_lockstat_patch_point,
671 LS_LOCK_CLEAR_RELEASE, RETL)
672 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
673 LS_LOCK_SET_SPL_ACQUIRE, RETL)
674 HOT_PATCH(.lock_clear_splx_lockstat_patch_point,
675 LS_LOCK_CLEAR_SPLX_RELEASE, RETL)
676 ret
677 restore
678 SET_SIZE(lockstat_hot_patch)
679
680 #endif /* lint */
681
682 /*
683 * asm_mutex_spin_enter(mutex_t *)
684 *
685 * For use by assembly interrupt handler only.
686 * Does not change spl, since the interrupt handler is assumed to be
687 * running at high level already.
688 * Traps may be off, so cannot panic.
689 * Does not keep statistics on the lock.
690 *
691 * Entry: %l6 - points to mutex
692 * %l7 - address of call (returns to %l7+8)
693 * Uses: %l6, %l5
694 */
695 #ifndef lint
696 .align 16
697 ENTRY_NP(asm_mutex_spin_enter)
698 ldstub [%l6 + M_SPINLOCK], %l5 ! try to set lock, get value in %l5
699 1:
700 tst %l5
701 bnz 3f ! lock already held - go spin
702 nop
703 2:
704 jmp %l7 + 8 ! return
705 membar #LoadLoad
706 !
707 ! Spin on lock without using an atomic operation to prevent the caches
708 ! from unnecessarily moving ownership of the line around.
709 !
710 3:
711 ldub [%l6 + M_SPINLOCK], %l5
712 4:
713 tst %l5
714 bz,a 1b ! lock appears to be free, try again
715 ldstub [%l6 + M_SPINLOCK], %l5 ! delay slot - try to set lock
716
717 sethi %hi(panicstr) , %l5
718 ldn [%l5 + %lo(panicstr)], %l5
719 tst %l5
720 bnz 2b ! after panic, feign success
721 nop
722 b 4b
723 ldub [%l6 + M_SPINLOCK], %l5 ! delay - reload lock
724 SET_SIZE(asm_mutex_spin_enter)
725 #endif /* lint */
726
727 /*
728 * asm_mutex_spin_exit(mutex_t *)
729 *
730 * For use by assembly interrupt handler only.
731 * Does not change spl, since the interrupt handler is assumed to be
732 * running at high level already.
733 *
734 * Entry: %l6 - points to mutex
735 * %l7 - address of call (returns to %l7+8)
736 * Uses: none
737 */
738 #ifndef lint
739 ENTRY_NP(asm_mutex_spin_exit)
740 membar #LoadStore|#StoreStore
741 jmp %l7 + 8 ! return
742 clrb [%l6 + M_SPINLOCK] ! delay - clear lock
743 SET_SIZE(asm_mutex_spin_exit)
744 #endif /* lint */
745
746 /*
747 * thread_onproc()
748 * Set thread in onproc state for the specified CPU.
749 * Also set the thread lock pointer to the CPU's onproc lock.
750 * Since the new lock isn't held, the store ordering is important.
751 * If not done in assembler, the compiler could reorder the stores.
752 */
753 #if defined(lint)
754
755 void
756 thread_onproc(kthread_id_t t, cpu_t *cp)
757 {
758 t->t_state = TS_ONPROC;
759 t->t_lockp = &cp->cpu_thread_lock;
760 }
761
762 #else /* lint */
763
764 ENTRY(thread_onproc)
765 set TS_ONPROC, %o2 ! TS_ONPROC state
766 st %o2, [%o0 + T_STATE] ! store state
767 add %o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running
768 retl ! return
769 stn %o3, [%o0 + T_LOCKP] ! delay - store new lock pointer
770 SET_SIZE(thread_onproc)
771
772 #endif /* lint */
773
774 /* delay function used in some mutex code - just do 3 nop cas ops */
775 #if defined(lint)
776
777 /* ARGSUSED */
778 void
779 cas_delay(void *addr)
780 {}
781 #else /* lint */
782 ENTRY(cas_delay)
783 casx [%o0], %g0, %g0
784 casx [%o0], %g0, %g0
785 retl
786 casx [%o0], %g0, %g0
787 SET_SIZE(cas_delay)
788 #endif /* lint */
789
790 #if defined(lint)
791
792 /*
793 * alternative delay function for some niagara processors. The rd
794 * instruction uses less resources than casx on those cpus.
795 */
796 /* ARGSUSED */
797 void
798 rdccr_delay(void)
799 {}
800 #else /* lint */
801 ENTRY(rdccr_delay)
802 rd %ccr, %g0
803 rd %ccr, %g0
804 retl
805 rd %ccr, %g0
806 SET_SIZE(rdccr_delay)
807 #endif /* lint */
808
809 /*
810 * mutex_delay_default(void)
811 * Spins for approx a few hundred processor cycles and returns to caller.
812 */
813 #if defined(lint)
814
815 void
816 mutex_delay_default(void)
817 {}
818
819 #else /* lint */
820
821 ENTRY(mutex_delay_default)
822 mov 72,%o0
823 1: brgz %o0, 1b
824 dec %o0
825 retl
826 nop
827 SET_SIZE(mutex_delay_default)
828
829 #endif /* lint */