Print this page
OS-7753 THREAD_KPRI_RELEASE does nothing of the sort
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/lock_prim.s
+++ new/usr/src/uts/intel/ia32/ml/lock_prim.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 + * Copyright 2019 Joyent, Inc.
24 25 */
25 26
26 -#pragma ident "%Z%%M% %I% %E% SMI"
27 -
28 27 #if defined(lint) || defined(__lint)
29 28 #include <sys/types.h>
30 29 #include <sys/thread.h>
31 30 #include <sys/cpuvar.h>
32 31 #include <vm/page.h>
33 32 #else /* __lint */
34 33 #include "assym.h"
35 34 #endif /* __lint */
36 35
37 36 #include <sys/mutex_impl.h>
38 37 #include <sys/asm_linkage.h>
39 38 #include <sys/asm_misc.h>
40 39 #include <sys/regset.h>
41 40 #include <sys/rwlock_impl.h>
42 41 #include <sys/lockstat.h>
43 42
44 43 /*
45 44 * lock_try(lp), ulock_try(lp)
46 45 * - returns non-zero on success.
47 46 * - doesn't block interrupts so don't use this to spin on a lock.
48 47 *
49 48 * ulock_try() is for a lock in the user address space.
50 49 */
51 50
52 51 #if defined(lint) || defined(__lint)
53 52
54 53 /* ARGSUSED */
55 54 int
56 55 lock_try(lock_t *lp)
57 56 { return (0); }
58 57
59 58 /* ARGSUSED */
60 59 int
61 60 lock_spin_try(lock_t *lp)
62 61 { return (0); }
63 62
64 63 /* ARGSUSED */
65 64 int
66 65 ulock_try(lock_t *lp)
67 66 { return (0); }
68 67
69 68 #else /* __lint */
70 69 .globl kernelbase
71 70
72 71 #if defined(__amd64)
73 72
74 73 ENTRY(lock_try)
75 74 movb $-1, %dl
76 75 movzbq %dl, %rax
77 76 xchgb %dl, (%rdi)
78 77 xorb %dl, %al
79 78 .lock_try_lockstat_patch_point:
80 79 ret
81 80 testb %al, %al
82 81 jnz 0f
83 82 ret
84 83 0:
85 84 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
86 85 movq %rdi, %rsi /* rsi = lock addr */
87 86 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
88 87 jmp lockstat_wrapper
89 88 SET_SIZE(lock_try)
90 89
91 90 ENTRY(lock_spin_try)
92 91 movb $-1, %dl
93 92 movzbq %dl, %rax
94 93 xchgb %dl, (%rdi)
95 94 xorb %dl, %al
96 95 ret
97 96 SET_SIZE(lock_spin_try)
98 97
99 98 ENTRY(ulock_try)
100 99 #ifdef DEBUG
101 100 movq kernelbase(%rip), %rax
102 101 cmpq %rax, %rdi /* test uaddr < kernelbase */
103 102 jb ulock_pass /* uaddr < kernelbase, proceed */
104 103
105 104 movq %rdi, %r12 /* preserve lock ptr for debugging */
106 105 leaq .ulock_panic_msg(%rip), %rdi
107 106 pushq %rbp /* align stack properly */
108 107 movq %rsp, %rbp
109 108 xorl %eax, %eax /* clear for varargs */
110 109 call panic
111 110
112 111 #endif /* DEBUG */
113 112
114 113 ulock_pass:
115 114 movl $1, %eax
116 115 xchgb %al, (%rdi)
117 116 xorb $1, %al
118 117 ret
119 118 SET_SIZE(ulock_try)
120 119
121 120 #else
122 121
123 122 ENTRY(lock_try)
124 123 movl $1,%edx
125 124 movl 4(%esp),%ecx /* ecx = lock addr */
126 125 xorl %eax,%eax
127 126 xchgb %dl, (%ecx) /* using dl will avoid partial */
128 127 testb %dl,%dl /* stalls on P6 ? */
129 128 setz %al
130 129 .lock_try_lockstat_patch_point:
131 130 ret
132 131 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
133 132 testl %eax, %eax
134 133 jz 0f
135 134 movl $LS_LOCK_TRY_ACQUIRE, %eax
136 135 jmp lockstat_wrapper
137 136 0:
138 137 ret
139 138 SET_SIZE(lock_try)
140 139
141 140 ENTRY(lock_spin_try)
142 141 movl $-1,%edx
143 142 movl 4(%esp),%ecx /* ecx = lock addr */
144 143 xorl %eax,%eax
145 144 xchgb %dl, (%ecx) /* using dl will avoid partial */
146 145 testb %dl,%dl /* stalls on P6 ? */
147 146 setz %al
148 147 ret
149 148 SET_SIZE(lock_spin_try)
150 149
151 150 ENTRY(ulock_try)
152 151 #ifdef DEBUG
153 152 movl kernelbase, %eax
154 153 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */
155 154 jb ulock_pass /* uaddr < kernelbase, proceed */
156 155
157 156 pushl $.ulock_panic_msg
158 157 call panic
159 158
160 159 #endif /* DEBUG */
161 160
162 161 ulock_pass:
163 162 movl $1,%eax
164 163 movl 4(%esp),%ecx
165 164 xchgb %al, (%ecx)
166 165 xorb $1, %al
167 166 ret
168 167 SET_SIZE(ulock_try)
169 168
170 169 #endif /* !__amd64 */
171 170
172 171 #ifdef DEBUG
173 172 .data
174 173 .ulock_panic_msg:
175 174 .string "ulock_try: Argument is above kernelbase"
176 175 .text
177 176 #endif /* DEBUG */
178 177
179 178 #endif /* __lint */
180 179
181 180 /*
182 181 * lock_clear(lp)
183 182 * - unlock lock without changing interrupt priority level.
184 183 */
185 184
186 185 #if defined(lint) || defined(__lint)
187 186
188 187 /* ARGSUSED */
189 188 void
190 189 lock_clear(lock_t *lp)
191 190 {}
192 191
193 192 /* ARGSUSED */
194 193 void
195 194 ulock_clear(lock_t *lp)
196 195 {}
197 196
198 197 #else /* __lint */
199 198
200 199 #if defined(__amd64)
201 200
202 201 ENTRY(lock_clear)
203 202 movb $0, (%rdi)
204 203 .lock_clear_lockstat_patch_point:
205 204 ret
206 205 movq %rdi, %rsi /* rsi = lock addr */
207 206 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
208 207 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */
209 208 jmp lockstat_wrapper
210 209 SET_SIZE(lock_clear)
211 210
212 211 ENTRY(ulock_clear)
213 212 #ifdef DEBUG
214 213 movq kernelbase(%rip), %rcx
215 214 cmpq %rcx, %rdi /* test uaddr < kernelbase */
216 215 jb ulock_clr /* uaddr < kernelbase, proceed */
217 216
218 217 leaq .ulock_clear_msg(%rip), %rdi
219 218 pushq %rbp /* align stack properly */
220 219 movq %rsp, %rbp
221 220 xorl %eax, %eax /* clear for varargs */
222 221 call panic
223 222 #endif
224 223
225 224 ulock_clr:
226 225 movb $0, (%rdi)
227 226 ret
228 227 SET_SIZE(ulock_clear)
229 228
230 229 #else
231 230
232 231 ENTRY(lock_clear)
233 232 movl 4(%esp), %eax
234 233 movb $0, (%eax)
235 234 .lock_clear_lockstat_patch_point:
236 235 ret
237 236 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
238 237 movl %eax, %ecx /* ecx = lock pointer */
239 238 movl $LS_LOCK_CLEAR_RELEASE, %eax
240 239 jmp lockstat_wrapper
241 240 SET_SIZE(lock_clear)
242 241
243 242 ENTRY(ulock_clear)
244 243 #ifdef DEBUG
245 244 movl kernelbase, %ecx
246 245 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */
247 246 jb ulock_clr /* uaddr < kernelbase, proceed */
248 247
249 248 pushl $.ulock_clear_msg
250 249 call panic
251 250 #endif
252 251
253 252 ulock_clr:
254 253 movl 4(%esp),%eax
255 254 xorl %ecx,%ecx
256 255 movb %cl, (%eax)
257 256 ret
258 257 SET_SIZE(ulock_clear)
259 258
260 259 #endif /* !__amd64 */
261 260
262 261 #ifdef DEBUG
263 262 .data
264 263 .ulock_clear_msg:
265 264 .string "ulock_clear: Argument is above kernelbase"
266 265 .text
267 266 #endif /* DEBUG */
268 267
269 268
270 269 #endif /* __lint */
271 270
272 271 /*
273 272 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
274 273 * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
275 274 */
276 275
277 276 #if defined(lint) || defined(__lint)
278 277
279 278 /* ARGSUSED */
280 279 void
281 280 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
282 281 {}
283 282
284 283 #else /* __lint */
285 284
286 285 #if defined(__amd64)
287 286
288 287 ENTRY(lock_set_spl)
289 288 pushq %rbp
290 289 movq %rsp, %rbp
291 290 subq $32, %rsp
292 291 movl %esi, 8(%rsp) /* save priority level */
293 292 movq %rdx, 16(%rsp) /* save old pil ptr */
294 293 movq %rdi, 24(%rsp) /* save lock pointer */
295 294 movl %esi, %edi /* pass priority level */
296 295 call splr /* raise priority level */
297 296 movq 24(%rsp), %rdi /* rdi = lock addr */
298 297 movb $-1, %dl
299 298 xchgb %dl, (%rdi) /* try to set lock */
300 299 testb %dl, %dl /* did we get the lock? ... */
301 300 jnz .lss_miss /* ... no, go to C for the hard case */
302 301 movq 16(%rsp), %rdx /* rdx = old pil addr */
303 302 movw %ax, (%rdx) /* store old pil */
304 303 leave
305 304 .lock_set_spl_lockstat_patch_point:
306 305 ret
307 306 movq %rdi, %rsi /* rsi = lock addr */
308 307 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
309 308 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi
310 309 jmp lockstat_wrapper
311 310 .lss_miss:
312 311 movl 8(%rsp), %esi /* new_pil */
313 312 movq 16(%rsp), %rdx /* old_pil_addr */
314 313 movl %eax, %ecx /* original pil */
|
↓ open down ↓ |
277 lines elided |
↑ open up ↑ |
315 314 leave /* unwind stack */
316 315 jmp lock_set_spl_spin
317 316 SET_SIZE(lock_set_spl)
318 317
319 318 #else
320 319
321 320 ENTRY(lock_set_spl)
322 321 movl 8(%esp), %eax /* get priority level */
323 322 pushl %eax
324 323 call splr /* raise priority level */
325 - movl 8(%esp), %ecx /* ecx = lock addr */
324 + movl 8(%esp), %ecx /* ecx = lock addr */
326 325 movl $-1, %edx
327 326 addl $4, %esp
328 327 xchgb %dl, (%ecx) /* try to set lock */
329 328 testb %dl, %dl /* did we get the lock? ... */
330 329 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */
331 330 jnz .lss_miss /* ... no, go to C for the hard case */
332 331 movw %ax, (%edx) /* store old pil */
333 332 .lock_set_spl_lockstat_patch_point:
334 333 ret
335 334 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/
336 335 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax
337 336 jmp lockstat_wrapper
338 337 .lss_miss:
339 338 pushl %eax /* original pil */
340 339 pushl %edx /* old_pil addr */
341 340 pushl 16(%esp) /* new_pil */
342 341 pushl %ecx /* lock addr */
343 342 call lock_set_spl_spin
344 343 addl $16, %esp
345 344 ret
346 345 SET_SIZE(lock_set_spl)
347 346
348 347 #endif /* !__amd64 */
349 348
350 349 #endif /* __lint */
351 350
352 351 /*
353 352 * void
354 353 * lock_init(lp)
355 354 */
356 355
357 356 #if defined(__lint)
358 357
359 358 /* ARGSUSED */
360 359 void
361 360 lock_init(lock_t *lp)
362 361 {}
363 362
364 363 #else /* __lint */
365 364
366 365 #if defined(__amd64)
367 366
368 367 ENTRY(lock_init)
369 368 movb $0, (%rdi)
370 369 ret
371 370 SET_SIZE(lock_init)
372 371
373 372 #else
374 373
375 374 ENTRY(lock_init)
376 375 movl 4(%esp), %eax
377 376 movb $0, (%eax)
378 377 ret
379 378 SET_SIZE(lock_init)
380 379
381 380 #endif /* !__amd64 */
382 381
383 382 #endif /* __lint */
384 383
385 384 /*
386 385 * void
387 386 * lock_set(lp)
388 387 */
389 388
390 389 #if defined(lint) || defined(__lint)
391 390
392 391 /* ARGSUSED */
393 392 void
394 393 lock_set(lock_t *lp)
395 394 {}
396 395
397 396 #else /* __lint */
398 397
399 398 #if defined(__amd64)
400 399
401 400 ENTRY(lock_set)
402 401 movb $-1, %dl
403 402 xchgb %dl, (%rdi) /* try to set lock */
404 403 testb %dl, %dl /* did we get it? */
405 404 jnz lock_set_spin /* no, go to C for the hard case */
406 405 .lock_set_lockstat_patch_point:
407 406 ret
408 407 movq %rdi, %rsi /* rsi = lock addr */
409 408 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
410 409 movl $LS_LOCK_SET_ACQUIRE, %edi
411 410 jmp lockstat_wrapper
412 411 SET_SIZE(lock_set)
413 412
414 413 #else
415 414
416 415 ENTRY(lock_set)
417 416 movl 4(%esp), %ecx /* ecx = lock addr */
418 417 movl $-1, %edx
419 418 xchgb %dl, (%ecx) /* try to set lock */
420 419 testb %dl, %dl /* did we get it? */
421 420 jnz lock_set_spin /* no, go to C for the hard case */
422 421 .lock_set_lockstat_patch_point:
423 422 ret
424 423 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
425 424 movl $LS_LOCK_SET_ACQUIRE, %eax
426 425 jmp lockstat_wrapper
427 426 SET_SIZE(lock_set)
428 427
429 428 #endif /* !__amd64 */
430 429
431 430 #endif /* __lint */
432 431
433 432 /*
434 433 * lock_clear_splx(lp, s)
435 434 */
436 435
437 436 #if defined(lint) || defined(__lint)
438 437
439 438 /* ARGSUSED */
440 439 void
441 440 lock_clear_splx(lock_t *lp, int s)
442 441 {}
443 442
444 443 #else /* __lint */
445 444
446 445 #if defined(__amd64)
447 446
448 447 ENTRY(lock_clear_splx)
449 448 movb $0, (%rdi) /* clear lock */
450 449 .lock_clear_splx_lockstat_patch_point:
451 450 jmp 0f
452 451 0:
453 452 movl %esi, %edi /* arg for splx */
454 453 jmp splx /* let splx do its thing */
455 454 .lock_clear_splx_lockstat:
456 455 pushq %rbp /* align stack properly */
457 456 movq %rsp, %rbp
458 457 subq $16, %rsp /* space to save args across splx */
459 458 movq %rdi, 8(%rsp) /* save lock ptr across splx call */
460 459 movl %esi, %edi /* arg for splx */
461 460 call splx /* lower the priority */
462 461 movq 8(%rsp), %rsi /* rsi = lock ptr */
463 462 leave /* unwind stack */
464 463 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
465 464 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
466 465 jmp lockstat_wrapper
467 466 SET_SIZE(lock_clear_splx)
468 467
469 468 #else
470 469
471 470 ENTRY(lock_clear_splx)
472 471 movl 4(%esp), %eax /* eax = lock addr */
473 472 movb $0, (%eax) /* clear lock */
474 473 .lock_clear_splx_lockstat_patch_point:
475 474 jmp 0f
476 475 0:
477 476 movl 8(%esp), %edx /* edx = desired pil */
478 477 movl %edx, 4(%esp) /* set spl arg up for splx */
479 478 jmp splx /* let splx do it's thing */
480 479 .lock_clear_splx_lockstat:
481 480 movl 8(%esp), %edx /* edx = desired pil */
482 481 pushl %ebp /* set up stack frame */
483 482 movl %esp, %ebp
484 483 pushl %edx
485 484 call splx
486 485 leave /* unwind stack */
487 486 movl 4(%esp), %ecx /* ecx = lock pointer */
488 487 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
489 488 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax
490 489 jmp lockstat_wrapper
491 490 SET_SIZE(lock_clear_splx)
492 491
493 492 #endif /* !__amd64 */
494 493
495 494 #if defined(__GNUC_AS__)
496 495 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
497 496 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
498 497
499 498 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
500 499 (.lock_clear_splx_lockstat_patch_point + 1)
501 500 #else
502 501 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
503 502 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
504 503
505 504 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
506 505 [.lock_clear_splx_lockstat_patch_point + 1]
507 506 #endif
508 507
509 508 #endif /* __lint */
510 509
511 510 /*
512 511 * mutex_enter() and mutex_exit().
513 512 *
514 513 * These routines handle the simple cases of mutex_enter() (adaptive
515 514 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
516 515 * If anything complicated is going on we punt to mutex_vector_enter().
517 516 *
518 517 * mutex_tryenter() is similar to mutex_enter() but returns zero if
519 518 * the lock cannot be acquired, nonzero on success.
520 519 *
521 520 * If mutex_exit() gets preempted in the window between checking waiters
522 521 * and clearing the lock, we can miss wakeups. Disabling preemption
523 522 * in the mutex code is prohibitively expensive, so instead we detect
524 523 * mutex preemption by examining the trapped PC in the interrupt path.
525 524 * If we interrupt a thread in mutex_exit() that has not yet cleared
526 525 * the lock, cmnint() resets its PC back to the beginning of
527 526 * mutex_exit() so it will check again for waiters when it resumes.
528 527 *
529 528 * The lockstat code below is activated when the lockstat driver
530 529 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
531 530 * Note that we don't need to test lockstat_event_mask here -- we won't
532 531 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
533 532 */
534 533 #if defined(lint) || defined(__lint)
535 534
536 535 /* ARGSUSED */
537 536 void
538 537 mutex_enter(kmutex_t *lp)
539 538 {}
540 539
541 540 /* ARGSUSED */
542 541 int
543 542 mutex_tryenter(kmutex_t *lp)
544 543 { return (0); }
545 544
546 545 /* ARGSUSED */
547 546 int
548 547 mutex_adaptive_tryenter(mutex_impl_t *lp)
549 548 { return (0); }
550 549
551 550 /* ARGSUSED */
552 551 void
553 552 mutex_exit(kmutex_t *lp)
554 553 {}
555 554
556 555 #else
557 556
558 557 #if defined(__amd64)
559 558
560 559 ENTRY_NP(mutex_enter)
561 560 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
|
↓ open down ↓ |
226 lines elided |
↑ open up ↑ |
562 561 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
563 562 lock
564 563 cmpxchgq %rdx, (%rdi)
565 564 jnz mutex_vector_enter
566 565 .mutex_enter_lockstat_patch_point:
567 566 #if defined(OPTERON_WORKAROUND_6323525)
568 567 .mutex_enter_6323525_patch_point:
569 568 ret /* nop space for lfence */
570 569 nop
571 570 nop
572 -.mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
571 +.mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
573 572 nop
574 573 #else /* OPTERON_WORKAROUND_6323525 */
575 574 ret
576 575 #endif /* OPTERON_WORKAROUND_6323525 */
577 576 movq %rdi, %rsi
578 577 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
579 578 /*
580 579 * expects %rdx=thread, %rsi=lock, %edi=lockstat event
581 580 */
582 581 ALTENTRY(lockstat_wrapper)
583 582 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */
584 583 leaq lockstat_probemap(%rip), %rax
585 584 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
586 585 testl %eax, %eax /* check for non-zero probe */
587 586 jz 1f
588 587 pushq %rbp /* align stack properly */
589 588 movq %rsp, %rbp
590 589 movl %eax, %edi
591 590 movq lockstat_probe, %rax
592 591 INDIRECT_CALL_REG(rax)
593 592 leave /* unwind stack */
594 593 1:
595 594 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
596 595 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
597 596 movl $1, %eax /* return success if tryenter */
598 597 ret
599 598 SET_SIZE(lockstat_wrapper)
600 599 SET_SIZE(mutex_enter)
601 600
602 601 /*
603 602 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event
604 603 */
605 604 ENTRY(lockstat_wrapper_arg)
606 605 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */
607 606 leaq lockstat_probemap(%rip), %rax
608 607 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
609 608 testl %eax, %eax /* check for non-zero probe */
610 609 jz 1f
611 610 pushq %rbp /* align stack properly */
612 611 movq %rsp, %rbp
613 612 movl %eax, %edi
614 613 movq lockstat_probe, %rax
615 614 INDIRECT_CALL_REG(rax)
616 615 leave /* unwind stack */
617 616 1:
618 617 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
619 618 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
620 619 movl $1, %eax /* return success if tryenter */
621 620 ret
622 621 SET_SIZE(lockstat_wrapper_arg)
623 622
624 623
625 624 ENTRY(mutex_tryenter)
626 625 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
627 626 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
628 627 lock
629 628 cmpxchgq %rdx, (%rdi)
630 629 jnz mutex_vector_tryenter
631 630 not %eax /* return success (nonzero) */
632 631 #if defined(OPTERON_WORKAROUND_6323525)
633 632 .mutex_tryenter_lockstat_patch_point:
634 633 .mutex_tryenter_6323525_patch_point:
635 634 ret /* nop space for lfence */
636 635 nop
637 636 nop
638 637 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
639 638 nop
640 639 #else /* OPTERON_WORKAROUND_6323525 */
641 640 .mutex_tryenter_lockstat_patch_point:
642 641 ret
643 642 #endif /* OPTERON_WORKAROUND_6323525 */
644 643 movq %rdi, %rsi
645 644 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
646 645 jmp lockstat_wrapper
647 646 SET_SIZE(mutex_tryenter)
648 647
649 648 ENTRY(mutex_adaptive_tryenter)
650 649 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
651 650 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
652 651 lock
653 652 cmpxchgq %rdx, (%rdi)
654 653 jnz 0f
655 654 not %eax /* return success (nonzero) */
656 655 #if defined(OPTERON_WORKAROUND_6323525)
657 656 .mutex_atryenter_6323525_patch_point:
658 657 ret /* nop space for lfence */
659 658 nop
660 659 nop
661 660 nop
662 661 #else /* OPTERON_WORKAROUND_6323525 */
663 662 ret
664 663 #endif /* OPTERON_WORKAROUND_6323525 */
665 664 0:
666 665 xorl %eax, %eax /* return failure */
667 666 ret
668 667 SET_SIZE(mutex_adaptive_tryenter)
669 668
670 669 .globl mutex_owner_running_critical_start
671 670
672 671 ENTRY(mutex_owner_running)
673 672 mutex_owner_running_critical_start:
674 673 movq (%rdi), %r11 /* get owner field */
675 674 andq $MUTEX_THREAD, %r11 /* remove waiters bit */
676 675 cmpq $0, %r11 /* if free, skip */
677 676 je 1f /* go return 0 */
678 677 movq T_CPU(%r11), %r8 /* get owner->t_cpu */
679 678 movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */
680 679 .mutex_owner_running_critical_end:
681 680 cmpq %r11, %r9 /* owner == running thread? */
682 681 je 2f /* yes, go return cpu */
683 682 1:
684 683 xorq %rax, %rax /* return 0 */
685 684 ret
686 685 2:
687 686 movq %r8, %rax /* return cpu */
688 687 ret
689 688 SET_SIZE(mutex_owner_running)
690 689
691 690 .globl mutex_owner_running_critical_size
692 691 .type mutex_owner_running_critical_size, @object
693 692 .align CPTRSIZE
694 693 mutex_owner_running_critical_size:
695 694 .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start
696 695 SET_SIZE(mutex_owner_running_critical_size)
697 696
698 697 .globl mutex_exit_critical_start
699 698
700 699 ENTRY(mutex_exit)
701 700 mutex_exit_critical_start: /* If interrupted, restart here */
702 701 movq %gs:CPU_THREAD, %rdx
703 702 cmpq %rdx, (%rdi)
704 703 jne mutex_vector_exit /* wrong type or wrong owner */
705 704 movq $0, (%rdi) /* clear owner AND lock */
706 705 .mutex_exit_critical_end:
707 706 .mutex_exit_lockstat_patch_point:
708 707 ret
709 708 movq %rdi, %rsi
710 709 movl $LS_MUTEX_EXIT_RELEASE, %edi
711 710 jmp lockstat_wrapper
712 711 SET_SIZE(mutex_exit)
713 712
714 713 .globl mutex_exit_critical_size
715 714 .type mutex_exit_critical_size, @object
716 715 .align CPTRSIZE
717 716 mutex_exit_critical_size:
718 717 .quad .mutex_exit_critical_end - mutex_exit_critical_start
719 718 SET_SIZE(mutex_exit_critical_size)
720 719
721 720 #else
722 721
723 722 ENTRY_NP(mutex_enter)
724 723 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
725 724 movl 4(%esp), %ecx /* ecx = lock ptr */
726 725 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
727 726 lock
728 727 cmpxchgl %edx, (%ecx)
729 728 jnz mutex_vector_enter
730 729 #if defined(OPTERON_WORKAROUND_6323525)
731 730 .mutex_enter_lockstat_patch_point:
732 731 .mutex_enter_6323525_patch_point:
733 732 ret /* nop space for lfence */
734 733 nop
735 734 nop
736 735 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
737 736 nop
738 737 #else /* OPTERON_WORKAROUND_6323525 */
739 738 .mutex_enter_lockstat_patch_point:
740 739 ret
741 740 #endif /* OPTERON_WORKAROUND_6323525 */
742 741 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
743 742 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */
744 743 /* eax=lockstat event */
745 744 pushl %ebp /* buy a frame */
746 745 movl %esp, %ebp
747 746 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
748 747 pushl %edx /* save thread pointer */
749 748 movl $lockstat_probemap, %edx
750 749 movl (%edx, %eax, DTRACE_IDSIZE), %eax
751 750 testl %eax, %eax /* check for non-zero probe */
752 751 jz 1f
753 752 pushl %ecx /* push lock */
754 753 pushl %eax /* push probe ID */
755 754 call *lockstat_probe
756 755 addl $8, %esp
757 756 1:
758 757 popl %edx /* restore thread pointer */
759 758 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
760 759 movl $1, %eax /* return success if tryenter */
761 760 popl %ebp /* pop off frame */
762 761 ret
763 762 SET_SIZE(lockstat_wrapper)
764 763 SET_SIZE(mutex_enter)
765 764
766 765 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */
767 766 /* eax=lockstat event, pushed arg */
768 767 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
769 768 pushl %edx /* save thread pointer */
770 769 movl $lockstat_probemap, %edx
771 770 movl (%edx, %eax, DTRACE_IDSIZE), %eax
772 771 testl %eax, %eax /* check for non-zero probe */
773 772 jz 1f
774 773 pushl %ebp /* save %ebp */
775 774 pushl 8(%esp) /* push arg1 */
776 775 movl %ebp, 12(%esp) /* fake up the stack frame */
777 776 movl %esp, %ebp /* fake up base pointer */
778 777 addl $12, %ebp /* adjust faked base pointer */
779 778 pushl %ecx /* push lock */
780 779 pushl %eax /* push probe ID */
781 780 call *lockstat_probe
782 781 addl $12, %esp /* adjust for arguments */
783 782 popl %ebp /* pop frame */
784 783 1:
785 784 popl %edx /* restore thread pointer */
786 785 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
787 786 movl $1, %eax /* return success if tryenter */
788 787 addl $4, %esp /* pop argument */
789 788 ret
790 789 SET_SIZE(lockstat_wrapper_arg)
791 790
792 791
793 792 ENTRY(mutex_tryenter)
794 793 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
795 794 movl 4(%esp), %ecx /* ecx = lock ptr */
796 795 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
797 796 lock
798 797 cmpxchgl %edx, (%ecx)
799 798 jnz mutex_vector_tryenter
800 799 movl %ecx, %eax
801 800 #if defined(OPTERON_WORKAROUND_6323525)
802 801 .mutex_tryenter_lockstat_patch_point:
803 802 .mutex_tryenter_6323525_patch_point:
804 803 ret /* nop space for lfence */
805 804 nop
806 805 nop
807 806 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
808 807 nop
809 808 #else /* OPTERON_WORKAROUND_6323525 */
810 809 .mutex_tryenter_lockstat_patch_point:
811 810 ret
812 811 #endif /* OPTERON_WORKAROUND_6323525 */
813 812 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
814 813 jmp lockstat_wrapper
815 814 SET_SIZE(mutex_tryenter)
816 815
817 816 ENTRY(mutex_adaptive_tryenter)
818 817 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
819 818 movl 4(%esp), %ecx /* ecx = lock ptr */
820 819 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
821 820 lock
822 821 cmpxchgl %edx, (%ecx)
823 822 jnz 0f
824 823 movl %ecx, %eax
825 824 #if defined(OPTERON_WORKAROUND_6323525)
826 825 .mutex_atryenter_6323525_patch_point:
827 826 ret /* nop space for lfence */
828 827 nop
829 828 nop
830 829 nop
831 830 #else /* OPTERON_WORKAROUND_6323525 */
832 831 ret
833 832 #endif /* OPTERON_WORKAROUND_6323525 */
834 833 0:
835 834 xorl %eax, %eax
836 835 ret
837 836 SET_SIZE(mutex_adaptive_tryenter)
838 837
839 838 .globl mutex_owner_running_critical_start
840 839
841 840 ENTRY(mutex_owner_running)
842 841 mutex_owner_running_critical_start:
843 842 movl 4(%esp), %eax /* get owner field */
844 843 movl (%eax), %eax
845 844 andl $MUTEX_THREAD, %eax /* remove waiters bit */
846 845 cmpl $0, %eax /* if free, skip */
847 846 je 1f /* go return 0 */
848 847 movl T_CPU(%eax), %ecx /* get owner->t_cpu */
849 848 movl CPU_THREAD(%ecx), %edx /* get t_cpu->cpu_thread */
850 849 .mutex_owner_running_critical_end:
851 850 cmpl %eax, %edx /* owner == running thread? */
852 851 je 2f /* yes, go return cpu */
853 852 1:
854 853 xorl %eax, %eax /* return 0 */
855 854 ret
856 855 2:
857 856 movl %ecx, %eax /* return cpu */
858 857 ret
859 858
860 859 SET_SIZE(mutex_owner_running)
861 860
862 861 .globl mutex_owner_running_critical_size
863 862 .type mutex_owner_running_critical_size, @object
864 863 .align CPTRSIZE
865 864 mutex_owner_running_critical_size:
866 865 .long .mutex_owner_running_critical_end - mutex_owner_running_critical_start
867 866 SET_SIZE(mutex_owner_running_critical_size)
868 867
869 868 .globl mutex_exit_critical_start
870 869
871 870 ENTRY(mutex_exit)
872 871 mutex_exit_critical_start: /* If interrupted, restart here */
873 872 movl %gs:CPU_THREAD, %edx
874 873 movl 4(%esp), %ecx
875 874 cmpl %edx, (%ecx)
876 875 jne mutex_vector_exit /* wrong type or wrong owner */
877 876 movl $0, (%ecx) /* clear owner AND lock */
878 877 .mutex_exit_critical_end:
879 878 .mutex_exit_lockstat_patch_point:
880 879 ret
881 880 movl $LS_MUTEX_EXIT_RELEASE, %eax
882 881 jmp lockstat_wrapper
883 882 SET_SIZE(mutex_exit)
884 883
885 884 .globl mutex_exit_critical_size
886 885 .type mutex_exit_critical_size, @object
887 886 .align CPTRSIZE
888 887 mutex_exit_critical_size:
889 888 .long .mutex_exit_critical_end - mutex_exit_critical_start
890 889 SET_SIZE(mutex_exit_critical_size)
891 890
892 891 #endif /* !__amd64 */
893 892
894 893 #endif /* __lint */
895 894
896 895 /*
897 896 * rw_enter() and rw_exit().
898 897 *
899 898 * These routines handle the simple cases of rw_enter (write-locking an unheld
900 899 * lock or read-locking a lock that's neither write-locked nor write-wanted)
901 900 * and rw_exit (no waiters or not the last reader). If anything complicated
902 901 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
903 902 */
904 903 #if defined(lint) || defined(__lint)
905 904
906 905 /* ARGSUSED */
907 906 void
908 907 rw_enter(krwlock_t *lp, krw_t rw)
909 908 {}
910 909
|
↓ open down ↓ |
328 lines elided |
↑ open up ↑ |
911 910 /* ARGSUSED */
912 911 void
913 912 rw_exit(krwlock_t *lp)
914 913 {}
915 914
916 915 #else /* __lint */
917 916
918 917 #if defined(__amd64)
919 918
920 919 ENTRY(rw_enter)
921 - movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
922 920 cmpl $RW_WRITER, %esi
923 921 je .rw_write_enter
924 - incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */
925 922 movq (%rdi), %rax /* rax = old rw_wwwh value */
926 923 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
927 924 jnz rw_enter_sleep
928 925 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
929 926 lock
930 927 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
931 928 jnz rw_enter_sleep
932 929 .rw_read_enter_lockstat_patch_point:
933 930 ret
934 931 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
935 932 movq %rdi, %rsi /* rsi = lock ptr */
936 933 movl $LS_RW_ENTER_ACQUIRE, %edi
937 934 movl $RW_READER, %edx
938 935 jmp lockstat_wrapper_arg
939 936 .rw_write_enter:
937 + movq %gs:CPU_THREAD, %rdx
940 938 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */
941 939 xorl %eax, %eax /* rax = unheld value */
942 940 lock
943 941 cmpxchgq %rdx, (%rdi) /* try to grab write lock */
944 942 jnz rw_enter_sleep
945 943
946 944 #if defined(OPTERON_WORKAROUND_6323525)
947 945 .rw_write_enter_lockstat_patch_point:
948 946 .rw_write_enter_6323525_patch_point:
949 947 ret
950 948 nop
951 949 nop
952 950 .rw_write_enter_lockstat_6323525_patch_point:
953 951 nop
954 952 #else /* OPTERON_WORKAROUND_6323525 */
955 953 .rw_write_enter_lockstat_patch_point:
956 954 ret
957 955 #endif /* OPTERON_WORKAROUND_6323525 */
958 956
959 957 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
960 958 movq %rdi, %rsi /* rsi = lock ptr */
961 959 movl $LS_RW_ENTER_ACQUIRE, %edi
962 960 movl $RW_WRITER, %edx
963 961 jmp lockstat_wrapper_arg
964 962 SET_SIZE(rw_enter)
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
965 963
966 964 ENTRY(rw_exit)
967 965 movq (%rdi), %rax /* rax = old rw_wwwh value */
968 966 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
969 967 jne .rw_not_single_reader
970 968 xorl %edx, %edx /* rdx = new value (unheld) */
971 969 .rw_read_exit:
972 970 lock
973 971 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
974 972 jnz rw_exit_wakeup
975 - movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
976 - decl T_KPRI_REQ(%rcx) /* THREAD_KPRI_RELEASE() */
977 973 .rw_read_exit_lockstat_patch_point:
978 974 ret
975 + movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
979 976 movq %rdi, %rsi /* rsi = lock ptr */
980 977 movl $LS_RW_EXIT_RELEASE, %edi
981 978 movl $RW_READER, %edx
982 979 jmp lockstat_wrapper_arg
983 980 .rw_not_single_reader:
984 981 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
985 982 jnz .rw_write_exit
986 983 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
987 984 cmpl $RW_READ_LOCK, %edx
988 985 jge .rw_read_exit /* not last reader, safe to drop */
989 986 jmp rw_exit_wakeup /* last reader with waiters */
990 987 .rw_write_exit:
991 988 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
992 989 xorl %edx, %edx /* rdx = new value (unheld) */
993 990 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
994 991 lock
995 992 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
996 993 jnz rw_exit_wakeup
997 994 .rw_write_exit_lockstat_patch_point:
998 995 ret
|
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
999 996 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
1000 997 movq %rdi, %rsi /* rsi - lock ptr */
1001 998 movl $LS_RW_EXIT_RELEASE, %edi
1002 999 movl $RW_WRITER, %edx
1003 1000 jmp lockstat_wrapper_arg
1004 1001 SET_SIZE(rw_exit)
1005 1002
1006 1003 #else
1007 1004
1008 1005 ENTRY(rw_enter)
1009 - movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1010 1006 movl 4(%esp), %ecx /* ecx = lock ptr */
1011 1007 cmpl $RW_WRITER, 8(%esp)
1012 1008 je .rw_write_enter
1013 - incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */
1014 1009 movl (%ecx), %eax /* eax = old rw_wwwh value */
1015 1010 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1016 1011 jnz rw_enter_sleep
1017 1012 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */
1018 1013 lock
1019 1014 cmpxchgl %edx, (%ecx) /* try to grab read lock */
1020 1015 jnz rw_enter_sleep
1021 1016 .rw_read_enter_lockstat_patch_point:
1022 1017 ret
1023 1018 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1024 1019 movl $LS_RW_ENTER_ACQUIRE, %eax
1025 1020 pushl $RW_READER
1026 1021 jmp lockstat_wrapper_arg
1027 1022 .rw_write_enter:
1023 + movl %gs:CPU_THREAD, %edx
1028 1024 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */
1029 1025 xorl %eax, %eax /* eax = unheld value */
1030 1026 lock
1031 1027 cmpxchgl %edx, (%ecx) /* try to grab write lock */
1032 1028 jnz rw_enter_sleep
1033 1029
1034 1030 #if defined(OPTERON_WORKAROUND_6323525)
1035 1031 .rw_write_enter_lockstat_patch_point:
1036 1032 .rw_write_enter_6323525_patch_point:
1037 1033 ret
1038 1034 nop
1039 1035 nop
1040 1036 .rw_write_enter_lockstat_6323525_patch_point:
1041 1037 nop
1042 1038 #else /* OPTERON_WORKAROUND_6323525 */
1043 1039 .rw_write_enter_lockstat_patch_point:
1044 1040 ret
1045 1041 #endif /* OPTERON_WORKAROUND_6323525 */
1046 1042
1047 1043 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1048 1044 movl $LS_RW_ENTER_ACQUIRE, %eax
1049 1045 pushl $RW_WRITER
1050 1046 jmp lockstat_wrapper_arg
1051 1047 SET_SIZE(rw_enter)
1052 1048
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
1053 1049 ENTRY(rw_exit)
1054 1050 movl 4(%esp), %ecx /* ecx = lock ptr */
1055 1051 movl (%ecx), %eax /* eax = old rw_wwwh value */
1056 1052 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
1057 1053 jne .rw_not_single_reader
1058 1054 xorl %edx, %edx /* edx = new value (unheld) */
1059 1055 .rw_read_exit:
1060 1056 lock
1061 1057 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1062 1058 jnz rw_exit_wakeup
1063 - movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1064 - decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */
1065 1059 .rw_read_exit_lockstat_patch_point:
1066 1060 ret
1067 1061 movl $LS_RW_EXIT_RELEASE, %eax
1068 1062 pushl $RW_READER
1069 1063 jmp lockstat_wrapper_arg
1070 1064 .rw_not_single_reader:
1071 1065 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
1072 1066 jnz .rw_write_exit
1073 1067 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */
1074 1068 cmpl $RW_READ_LOCK, %edx
1075 1069 jge .rw_read_exit /* not last reader, safe to drop */
1076 1070 jmp rw_exit_wakeup /* last reader with waiters */
1077 1071 .rw_write_exit:
1078 1072 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */
1079 1073 xorl %edx, %edx /* edx = new value (unheld) */
1080 1074 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */
1081 1075 lock
1082 1076 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1083 1077 jnz rw_exit_wakeup
1084 1078 .rw_write_exit_lockstat_patch_point:
1085 1079 ret
1086 1080 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1087 1081 movl $LS_RW_EXIT_RELEASE, %eax
1088 1082 pushl $RW_WRITER
1089 1083 jmp lockstat_wrapper_arg
1090 1084 SET_SIZE(rw_exit)
1091 1085
1092 1086 #endif /* !__amd64 */
1093 1087
1094 1088 #endif /* __lint */
1095 1089
1096 1090 #if defined(OPTERON_WORKAROUND_6323525)
1097 1091 #if defined(lint) || defined(__lint)
1098 1092
1099 1093 int workaround_6323525_patched;
1100 1094
1101 1095 void
1102 1096 patch_workaround_6323525(void)
1103 1097 {}
1104 1098
1105 1099 #else /* lint */
1106 1100
1107 1101 /*
1108 1102 * If it is necessary to patch the lock enter routines with the lfence
1109 1103 * workaround, workaround_6323525_patched is set to a non-zero value so that
1110 1104 * the lockstat_hat_patch routine can patch to the new location of the 'ret'
1111 1105 * instruction.
1112 1106 */
1113 1107 DGDEF3(workaround_6323525_patched, 4, 4)
1114 1108 .long 0
1115 1109
1116 1110 #if defined(__amd64)
1117 1111
1118 1112 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1119 1113 movq $size, %rbx; \
1120 1114 movq $dstaddr, %r13; \
1121 1115 addq %rbx, %r13; \
1122 1116 movq $srcaddr, %r12; \
1123 1117 addq %rbx, %r12; \
1124 1118 0: \
1125 1119 decq %r13; \
1126 1120 decq %r12; \
1127 1121 movzbl (%r12), %esi; \
1128 1122 movq $1, %rdx; \
1129 1123 movq %r13, %rdi; \
1130 1124 call hot_patch_kernel_text; \
1131 1125 decq %rbx; \
1132 1126 testq %rbx, %rbx; \
1133 1127 jg 0b;
1134 1128
1135 1129 /*
1136 1130 * patch_workaround_6323525: provide workaround for 6323525
1137 1131 *
1138 1132 * The workaround is to place a fencing instruction (lfence) between the
1139 1133 * mutex operation and the subsequent read-modify-write instruction.
1140 1134 *
1141 1135 * This routine hot patches the lfence instruction on top of the space
1142 1136 * reserved by nops in the lock enter routines.
1143 1137 */
1144 1138 ENTRY_NP(patch_workaround_6323525)
1145 1139 pushq %rbp
1146 1140 movq %rsp, %rbp
1147 1141 pushq %r12
1148 1142 pushq %r13
1149 1143 pushq %rbx
1150 1144
1151 1145 /*
1152 1146 * lockstat_hot_patch() to use the alternate lockstat workaround
1153 1147 * 6323525 patch points (points past the lfence instruction to the
1154 1148 * new ret) when workaround_6323525_patched is set.
1155 1149 */
1156 1150 movl $1, workaround_6323525_patched
1157 1151
1158 1152 /*
1159 1153 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter
1160 1154 * routines. The 4 bytes are patched in reverse order so that the
1161 1155 * the existing ret is overwritten last. This provides lock enter
1162 1156 * sanity during the intermediate patching stages.
1163 1157 */
1164 1158 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1165 1159 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1166 1160 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1167 1161 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1168 1162
1169 1163 popq %rbx
1170 1164 popq %r13
1171 1165 popq %r12
1172 1166 movq %rbp, %rsp
1173 1167 popq %rbp
1174 1168 ret
1175 1169 _lfence_insn:
1176 1170 lfence
1177 1171 ret
1178 1172 SET_SIZE(patch_workaround_6323525)
|
↓ open down ↓ |
104 lines elided |
↑ open up ↑ |
1179 1173
1180 1174
1181 1175 #else /* __amd64 */
1182 1176
1183 1177 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1184 1178 movl $size, %ebx; \
1185 1179 movl $srcaddr, %esi; \
1186 1180 addl %ebx, %esi; \
1187 1181 movl $dstaddr, %edi; \
1188 1182 addl %ebx, %edi; \
1189 -0: \
1183 +0: \
1190 1184 decl %esi; \
1191 1185 decl %edi; \
1192 1186 pushl $1; \
1193 1187 movzbl (%esi), %eax; \
1194 1188 pushl %eax; \
1195 1189 pushl %edi; \
1196 1190 call hot_patch_kernel_text; \
1197 1191 addl $12, %esp; \
1198 1192 decl %ebx; \
1199 1193 testl %ebx, %ebx; \
1200 1194 jg 0b;
1201 1195
1202 1196
1203 1197 /* see comments above */
1204 1198 ENTRY_NP(patch_workaround_6323525)
1205 1199 pushl %ebp
1206 1200 movl %esp, %ebp
1207 1201 pushl %ebx
1208 1202 pushl %esi
1209 1203 pushl %edi
1210 1204
1211 1205 movl $1, workaround_6323525_patched
1212 1206
1213 1207 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1214 1208 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1215 1209 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1216 1210 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1217 1211
1218 1212 popl %edi
1219 1213 popl %esi
1220 1214 popl %ebx
1221 1215 movl %ebp, %esp
1222 1216 popl %ebp
1223 1217 ret
1224 1218 _lfence_insn:
1225 1219 .byte 0xf, 0xae, 0xe8 / [lfence instruction]
1226 1220 ret
1227 1221 SET_SIZE(patch_workaround_6323525)
1228 1222
1229 1223 #endif /* !__amd64 */
1230 1224 #endif /* !lint */
1231 1225 #endif /* OPTERON_WORKAROUND_6323525 */
1232 1226
1233 1227
1234 1228 #if defined(lint) || defined(__lint)
1235 1229
1236 1230 void
1237 1231 lockstat_hot_patch(void)
|
↓ open down ↓ |
38 lines elided |
↑ open up ↑ |
1238 1232 {}
1239 1233
1240 1234 #else
1241 1235
1242 1236 #if defined(__amd64)
1243 1237
1244 1238 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1245 1239 movq $normal_instr, %rsi; \
1246 1240 movq $active_instr, %rdi; \
1247 1241 leaq lockstat_probemap(%rip), %rax; \
1248 - movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1242 + movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1249 1243 testl %eax, %eax; \
1250 1244 jz 9f; \
1251 1245 movq %rdi, %rsi; \
1252 1246 9: \
1253 1247 movq $len, %rdx; \
1254 1248 movq $addr, %rdi; \
1255 1249 call hot_patch_kernel_text
1256 1250
1257 1251 #else
1258 1252
1259 1253 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1260 1254 movl $normal_instr, %ecx; \
1261 1255 movl $active_instr, %edx; \
1262 1256 movl $lockstat_probemap, %eax; \
1263 1257 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \
1264 1258 testl %eax, %eax; \
1265 1259 jz . + 4; \
1266 1260 movl %edx, %ecx; \
1267 1261 pushl $len; \
1268 1262 pushl %ecx; \
1269 1263 pushl $addr; \
1270 1264 call hot_patch_kernel_text; \
1271 1265 addl $12, %esp;
1272 1266
1273 1267 #endif /* !__amd64 */
1274 1268
1275 1269 ENTRY(lockstat_hot_patch)
1276 1270 #if defined(__amd64)
1277 1271 pushq %rbp /* align stack properly */
1278 1272 movq %rsp, %rbp
1279 1273 #endif /* __amd64 */
1280 1274
1281 1275 #if defined(OPTERON_WORKAROUND_6323525)
1282 1276 cmpl $0, workaround_6323525_patched
1283 1277 je 1f
1284 1278 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
1285 1279 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1286 1280 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
1287 1281 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1288 1282 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
1289 1283 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1290 1284 jmp 2f
1291 1285 1:
1292 1286 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1293 1287 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1294 1288 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1295 1289 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1296 1290 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1297 1291 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1298 1292 2:
1299 1293 #else /* OPTERON_WORKAROUND_6323525 */
1300 1294 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1301 1295 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1302 1296 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1303 1297 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1304 1298 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1305 1299 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1306 1300 #endif /* !OPTERON_WORKAROUND_6323525 */
1307 1301 HOT_PATCH(.mutex_exit_lockstat_patch_point,
1308 1302 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1309 1303 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
1310 1304 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1311 1305 HOT_PATCH(.rw_write_exit_lockstat_patch_point,
1312 1306 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1313 1307 HOT_PATCH(.rw_read_exit_lockstat_patch_point,
1314 1308 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1315 1309 HOT_PATCH(.lock_set_lockstat_patch_point,
1316 1310 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1317 1311 HOT_PATCH(.lock_try_lockstat_patch_point,
1318 1312 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1319 1313 HOT_PATCH(.lock_clear_lockstat_patch_point,
1320 1314 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
1321 1315 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
1322 1316 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1323 1317
1324 1318 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
1325 1319 LS_LOCK_CLEAR_SPLX_RELEASE,
1326 1320 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
1327 1321 #if defined(__amd64)
1328 1322 leave /* unwind stack */
1329 1323 #endif /* __amd64 */
1330 1324 ret
1331 1325 SET_SIZE(lockstat_hot_patch)
1332 1326
1333 1327 #endif /* __lint */
1334 1328
1335 1329 #if defined(lint) || defined(__lint)
1336 1330
1337 1331 /* XX64 membar_*() should be inlines */
1338 1332
1339 1333 void
1340 1334 membar_sync(void)
1341 1335 {}
1342 1336
1343 1337 void
1344 1338 membar_enter(void)
1345 1339 {}
1346 1340
1347 1341 void
1348 1342 membar_exit(void)
1349 1343 {}
1350 1344
1351 1345 void
1352 1346 membar_producer(void)
1353 1347 {}
1354 1348
1355 1349 void
1356 1350 membar_consumer(void)
1357 1351 {}
1358 1352
1359 1353 #else /* __lint */
1360 1354
1361 1355 #if defined(__amd64)
1362 1356
1363 1357 ENTRY(membar_enter)
1364 1358 ALTENTRY(membar_exit)
1365 1359 ALTENTRY(membar_sync)
1366 1360 mfence /* lighter weight than lock; xorq $0,(%rsp) */
1367 1361 ret
1368 1362 SET_SIZE(membar_sync)
1369 1363 SET_SIZE(membar_exit)
1370 1364 SET_SIZE(membar_enter)
1371 1365
1372 1366 ENTRY(membar_producer)
1373 1367 sfence
1374 1368 ret
1375 1369 SET_SIZE(membar_producer)
1376 1370
1377 1371 ENTRY(membar_consumer)
1378 1372 lfence
1379 1373 ret
1380 1374 SET_SIZE(membar_consumer)
1381 1375
1382 1376 #else
1383 1377
1384 1378 ENTRY(membar_enter)
1385 1379 ALTENTRY(membar_exit)
1386 1380 ALTENTRY(membar_sync)
1387 1381 lock
1388 1382 xorl $0, (%esp)
1389 1383 ret
1390 1384 SET_SIZE(membar_sync)
1391 1385 SET_SIZE(membar_exit)
1392 1386 SET_SIZE(membar_enter)
1393 1387
1394 1388 /*
1395 1389 * On machines that support sfence and lfence, these
1396 1390 * memory barriers can be more precisely implemented
1397 1391 * without causing the whole world to stop
1398 1392 */
1399 1393 ENTRY(membar_producer)
1400 1394 .globl _patch_sfence_ret
1401 1395 _patch_sfence_ret: /* c.f. membar #StoreStore */
1402 1396 lock
1403 1397 xorl $0, (%esp)
1404 1398 ret
1405 1399 SET_SIZE(membar_producer)
1406 1400
1407 1401 ENTRY(membar_consumer)
1408 1402 .globl _patch_lfence_ret
1409 1403 _patch_lfence_ret: /* c.f. membar #LoadLoad */
1410 1404 lock
1411 1405 xorl $0, (%esp)
1412 1406 ret
1413 1407 SET_SIZE(membar_consumer)
1414 1408
1415 1409 #endif /* !__amd64 */
1416 1410
1417 1411 #endif /* __lint */
1418 1412
1419 1413 /*
1420 1414 * thread_onproc()
1421 1415 * Set thread in onproc state for the specified CPU.
1422 1416 * Also set the thread lock pointer to the CPU's onproc lock.
1423 1417 * Since the new lock isn't held, the store ordering is important.
1424 1418 * If not done in assembler, the compiler could reorder the stores.
1425 1419 */
1426 1420 #if defined(lint) || defined(__lint)
1427 1421
1428 1422 void
1429 1423 thread_onproc(kthread_id_t t, cpu_t *cp)
1430 1424 {
1431 1425 t->t_state = TS_ONPROC;
1432 1426 t->t_lockp = &cp->cpu_thread_lock;
1433 1427 }
1434 1428
1435 1429 #else /* __lint */
1436 1430
1437 1431 #if defined(__amd64)
1438 1432
1439 1433 ENTRY(thread_onproc)
1440 1434 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */
1441 1435 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */
1442 1436 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */
1443 1437 ret
1444 1438 SET_SIZE(thread_onproc)
1445 1439
1446 1440 #else
1447 1441
1448 1442 ENTRY(thread_onproc)
1449 1443 movl 4(%esp), %eax
1450 1444 movl 8(%esp), %ecx
1451 1445 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */
1452 1446 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */
1453 1447 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */
1454 1448 ret
1455 1449 SET_SIZE(thread_onproc)
1456 1450
1457 1451 #endif /* !__amd64 */
1458 1452
1459 1453 #endif /* __lint */
1460 1454
1461 1455 /*
1462 1456 * mutex_delay_default(void)
1463 1457 * Spins for approx a few hundred processor cycles and returns to caller.
1464 1458 */
1465 1459
1466 1460 #if defined(lint) || defined(__lint)
1467 1461
1468 1462 void
1469 1463 mutex_delay_default(void)
1470 1464 {}
1471 1465
1472 1466 #else /* __lint */
1473 1467
1474 1468 #if defined(__amd64)
1475 1469
1476 1470 ENTRY(mutex_delay_default)
1477 1471 movq $92,%r11
1478 1472 0: decq %r11
1479 1473 jg 0b
1480 1474 ret
1481 1475 SET_SIZE(mutex_delay_default)
1482 1476
1483 1477 #else
1484 1478
1485 1479 ENTRY(mutex_delay_default)
1486 1480 push %ebp
1487 1481 movl %esp,%ebp
1488 1482 andl $-16,%esp
1489 1483 push %ebx
1490 1484 movl $93,%ebx
1491 1485 0: decl %ebx
1492 1486 jg 0b
1493 1487 pop %ebx
1494 1488 leave
1495 1489 ret
1496 1490 SET_SIZE(mutex_delay_default)
1497 1491
1498 1492 #endif /* !__amd64 */
1499 1493 #endif /* __lint */
|
↓ open down ↓ |
241 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX