Print this page
11787 Kernel needs to be built with retpolines
11788 Kernel needs to generally use RSB stuffing
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: John Levon <john.levon@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/lock_prim.s
+++ new/usr/src/uts/intel/ia32/ml/lock_prim.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #pragma ident "%Z%%M% %I% %E% SMI"
27 27
28 28 #if defined(lint) || defined(__lint)
29 29 #include <sys/types.h>
30 30 #include <sys/thread.h>
31 31 #include <sys/cpuvar.h>
32 32 #include <vm/page.h>
33 33 #else /* __lint */
34 34 #include "assym.h"
35 35 #endif /* __lint */
36 36
37 37 #include <sys/mutex_impl.h>
38 38 #include <sys/asm_linkage.h>
39 39 #include <sys/asm_misc.h>
40 40 #include <sys/regset.h>
41 41 #include <sys/rwlock_impl.h>
42 42 #include <sys/lockstat.h>
43 43
44 44 /*
45 45 * lock_try(lp), ulock_try(lp)
46 46 * - returns non-zero on success.
47 47 * - doesn't block interrupts so don't use this to spin on a lock.
48 48 *
49 49 * ulock_try() is for a lock in the user address space.
50 50 */
51 51
52 52 #if defined(lint) || defined(__lint)
53 53
54 54 /* ARGSUSED */
55 55 int
56 56 lock_try(lock_t *lp)
57 57 { return (0); }
58 58
59 59 /* ARGSUSED */
60 60 int
61 61 lock_spin_try(lock_t *lp)
62 62 { return (0); }
63 63
64 64 /* ARGSUSED */
65 65 int
66 66 ulock_try(lock_t *lp)
67 67 { return (0); }
68 68
69 69 #else /* __lint */
70 70 .globl kernelbase
71 71
72 72 #if defined(__amd64)
73 73
74 74 ENTRY(lock_try)
75 75 movb $-1, %dl
76 76 movzbq %dl, %rax
77 77 xchgb %dl, (%rdi)
78 78 xorb %dl, %al
79 79 .lock_try_lockstat_patch_point:
80 80 ret
81 81 testb %al, %al
82 82 jnz 0f
83 83 ret
84 84 0:
85 85 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
86 86 movq %rdi, %rsi /* rsi = lock addr */
87 87 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
88 88 jmp lockstat_wrapper
89 89 SET_SIZE(lock_try)
90 90
91 91 ENTRY(lock_spin_try)
92 92 movb $-1, %dl
93 93 movzbq %dl, %rax
94 94 xchgb %dl, (%rdi)
95 95 xorb %dl, %al
96 96 ret
97 97 SET_SIZE(lock_spin_try)
98 98
99 99 ENTRY(ulock_try)
100 100 #ifdef DEBUG
101 101 movq kernelbase(%rip), %rax
102 102 cmpq %rax, %rdi /* test uaddr < kernelbase */
103 103 jb ulock_pass /* uaddr < kernelbase, proceed */
104 104
105 105 movq %rdi, %r12 /* preserve lock ptr for debugging */
106 106 leaq .ulock_panic_msg(%rip), %rdi
107 107 pushq %rbp /* align stack properly */
108 108 movq %rsp, %rbp
109 109 xorl %eax, %eax /* clear for varargs */
110 110 call panic
111 111
112 112 #endif /* DEBUG */
113 113
114 114 ulock_pass:
115 115 movl $1, %eax
116 116 xchgb %al, (%rdi)
117 117 xorb $1, %al
118 118 ret
119 119 SET_SIZE(ulock_try)
120 120
121 121 #else
122 122
123 123 ENTRY(lock_try)
124 124 movl $1,%edx
125 125 movl 4(%esp),%ecx /* ecx = lock addr */
126 126 xorl %eax,%eax
127 127 xchgb %dl, (%ecx) /* using dl will avoid partial */
128 128 testb %dl,%dl /* stalls on P6 ? */
129 129 setz %al
130 130 .lock_try_lockstat_patch_point:
131 131 ret
132 132 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
133 133 testl %eax, %eax
134 134 jz 0f
135 135 movl $LS_LOCK_TRY_ACQUIRE, %eax
136 136 jmp lockstat_wrapper
137 137 0:
138 138 ret
139 139 SET_SIZE(lock_try)
140 140
141 141 ENTRY(lock_spin_try)
142 142 movl $-1,%edx
143 143 movl 4(%esp),%ecx /* ecx = lock addr */
144 144 xorl %eax,%eax
145 145 xchgb %dl, (%ecx) /* using dl will avoid partial */
146 146 testb %dl,%dl /* stalls on P6 ? */
147 147 setz %al
148 148 ret
149 149 SET_SIZE(lock_spin_try)
150 150
151 151 ENTRY(ulock_try)
152 152 #ifdef DEBUG
153 153 movl kernelbase, %eax
154 154 cmpl %eax, 4(%esp) /* test uaddr < kernelbase */
155 155 jb ulock_pass /* uaddr < kernelbase, proceed */
156 156
157 157 pushl $.ulock_panic_msg
158 158 call panic
159 159
160 160 #endif /* DEBUG */
161 161
162 162 ulock_pass:
163 163 movl $1,%eax
164 164 movl 4(%esp),%ecx
165 165 xchgb %al, (%ecx)
166 166 xorb $1, %al
167 167 ret
168 168 SET_SIZE(ulock_try)
169 169
170 170 #endif /* !__amd64 */
171 171
172 172 #ifdef DEBUG
173 173 .data
174 174 .ulock_panic_msg:
175 175 .string "ulock_try: Argument is above kernelbase"
176 176 .text
177 177 #endif /* DEBUG */
178 178
179 179 #endif /* __lint */
180 180
181 181 /*
182 182 * lock_clear(lp)
183 183 * - unlock lock without changing interrupt priority level.
184 184 */
185 185
186 186 #if defined(lint) || defined(__lint)
187 187
188 188 /* ARGSUSED */
189 189 void
190 190 lock_clear(lock_t *lp)
191 191 {}
192 192
193 193 /* ARGSUSED */
194 194 void
195 195 ulock_clear(lock_t *lp)
196 196 {}
197 197
198 198 #else /* __lint */
199 199
200 200 #if defined(__amd64)
201 201
202 202 ENTRY(lock_clear)
203 203 movb $0, (%rdi)
204 204 .lock_clear_lockstat_patch_point:
205 205 ret
206 206 movq %rdi, %rsi /* rsi = lock addr */
207 207 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
208 208 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */
209 209 jmp lockstat_wrapper
210 210 SET_SIZE(lock_clear)
211 211
212 212 ENTRY(ulock_clear)
213 213 #ifdef DEBUG
214 214 movq kernelbase(%rip), %rcx
215 215 cmpq %rcx, %rdi /* test uaddr < kernelbase */
216 216 jb ulock_clr /* uaddr < kernelbase, proceed */
217 217
218 218 leaq .ulock_clear_msg(%rip), %rdi
219 219 pushq %rbp /* align stack properly */
220 220 movq %rsp, %rbp
221 221 xorl %eax, %eax /* clear for varargs */
222 222 call panic
223 223 #endif
224 224
225 225 ulock_clr:
226 226 movb $0, (%rdi)
227 227 ret
228 228 SET_SIZE(ulock_clear)
229 229
230 230 #else
231 231
232 232 ENTRY(lock_clear)
233 233 movl 4(%esp), %eax
234 234 movb $0, (%eax)
235 235 .lock_clear_lockstat_patch_point:
236 236 ret
237 237 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
238 238 movl %eax, %ecx /* ecx = lock pointer */
239 239 movl $LS_LOCK_CLEAR_RELEASE, %eax
240 240 jmp lockstat_wrapper
241 241 SET_SIZE(lock_clear)
242 242
243 243 ENTRY(ulock_clear)
244 244 #ifdef DEBUG
245 245 movl kernelbase, %ecx
246 246 cmpl %ecx, 4(%esp) /* test uaddr < kernelbase */
247 247 jb ulock_clr /* uaddr < kernelbase, proceed */
248 248
249 249 pushl $.ulock_clear_msg
250 250 call panic
251 251 #endif
252 252
253 253 ulock_clr:
254 254 movl 4(%esp),%eax
255 255 xorl %ecx,%ecx
256 256 movb %cl, (%eax)
257 257 ret
258 258 SET_SIZE(ulock_clear)
259 259
260 260 #endif /* !__amd64 */
261 261
262 262 #ifdef DEBUG
263 263 .data
264 264 .ulock_clear_msg:
265 265 .string "ulock_clear: Argument is above kernelbase"
266 266 .text
267 267 #endif /* DEBUG */
268 268
269 269
270 270 #endif /* __lint */
271 271
272 272 /*
273 273 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
274 274 * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
275 275 */
276 276
277 277 #if defined(lint) || defined(__lint)
278 278
279 279 /* ARGSUSED */
280 280 void
281 281 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
282 282 {}
283 283
284 284 #else /* __lint */
285 285
286 286 #if defined(__amd64)
287 287
288 288 ENTRY(lock_set_spl)
289 289 pushq %rbp
290 290 movq %rsp, %rbp
291 291 subq $32, %rsp
292 292 movl %esi, 8(%rsp) /* save priority level */
293 293 movq %rdx, 16(%rsp) /* save old pil ptr */
294 294 movq %rdi, 24(%rsp) /* save lock pointer */
295 295 movl %esi, %edi /* pass priority level */
296 296 call splr /* raise priority level */
297 297 movq 24(%rsp), %rdi /* rdi = lock addr */
298 298 movb $-1, %dl
299 299 xchgb %dl, (%rdi) /* try to set lock */
300 300 testb %dl, %dl /* did we get the lock? ... */
301 301 jnz .lss_miss /* ... no, go to C for the hard case */
302 302 movq 16(%rsp), %rdx /* rdx = old pil addr */
303 303 movw %ax, (%rdx) /* store old pil */
304 304 leave
305 305 .lock_set_spl_lockstat_patch_point:
306 306 ret
307 307 movq %rdi, %rsi /* rsi = lock addr */
308 308 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
309 309 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi
310 310 jmp lockstat_wrapper
311 311 .lss_miss:
312 312 movl 8(%rsp), %esi /* new_pil */
313 313 movq 16(%rsp), %rdx /* old_pil_addr */
314 314 movl %eax, %ecx /* original pil */
315 315 leave /* unwind stack */
316 316 jmp lock_set_spl_spin
317 317 SET_SIZE(lock_set_spl)
318 318
319 319 #else
320 320
321 321 ENTRY(lock_set_spl)
322 322 movl 8(%esp), %eax /* get priority level */
323 323 pushl %eax
324 324 call splr /* raise priority level */
325 325 movl 8(%esp), %ecx /* ecx = lock addr */
326 326 movl $-1, %edx
327 327 addl $4, %esp
328 328 xchgb %dl, (%ecx) /* try to set lock */
329 329 testb %dl, %dl /* did we get the lock? ... */
330 330 movl 12(%esp), %edx /* edx = olp pil addr (ZF unaffected) */
331 331 jnz .lss_miss /* ... no, go to C for the hard case */
332 332 movw %ax, (%edx) /* store old pil */
333 333 .lock_set_spl_lockstat_patch_point:
334 334 ret
335 335 movl %gs:CPU_THREAD, %edx /* edx = thread addr*/
336 336 movl $LS_LOCK_SET_SPL_ACQUIRE, %eax
337 337 jmp lockstat_wrapper
338 338 .lss_miss:
339 339 pushl %eax /* original pil */
340 340 pushl %edx /* old_pil addr */
341 341 pushl 16(%esp) /* new_pil */
342 342 pushl %ecx /* lock addr */
343 343 call lock_set_spl_spin
344 344 addl $16, %esp
345 345 ret
346 346 SET_SIZE(lock_set_spl)
347 347
348 348 #endif /* !__amd64 */
349 349
350 350 #endif /* __lint */
351 351
352 352 /*
353 353 * void
354 354 * lock_init(lp)
355 355 */
356 356
357 357 #if defined(__lint)
358 358
359 359 /* ARGSUSED */
360 360 void
361 361 lock_init(lock_t *lp)
362 362 {}
363 363
364 364 #else /* __lint */
365 365
366 366 #if defined(__amd64)
367 367
368 368 ENTRY(lock_init)
369 369 movb $0, (%rdi)
370 370 ret
371 371 SET_SIZE(lock_init)
372 372
373 373 #else
374 374
375 375 ENTRY(lock_init)
376 376 movl 4(%esp), %eax
377 377 movb $0, (%eax)
378 378 ret
379 379 SET_SIZE(lock_init)
380 380
381 381 #endif /* !__amd64 */
382 382
383 383 #endif /* __lint */
384 384
385 385 /*
386 386 * void
387 387 * lock_set(lp)
388 388 */
389 389
390 390 #if defined(lint) || defined(__lint)
391 391
392 392 /* ARGSUSED */
393 393 void
394 394 lock_set(lock_t *lp)
395 395 {}
396 396
397 397 #else /* __lint */
398 398
399 399 #if defined(__amd64)
400 400
401 401 ENTRY(lock_set)
402 402 movb $-1, %dl
403 403 xchgb %dl, (%rdi) /* try to set lock */
404 404 testb %dl, %dl /* did we get it? */
405 405 jnz lock_set_spin /* no, go to C for the hard case */
406 406 .lock_set_lockstat_patch_point:
407 407 ret
408 408 movq %rdi, %rsi /* rsi = lock addr */
409 409 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
410 410 movl $LS_LOCK_SET_ACQUIRE, %edi
411 411 jmp lockstat_wrapper
412 412 SET_SIZE(lock_set)
413 413
414 414 #else
415 415
416 416 ENTRY(lock_set)
417 417 movl 4(%esp), %ecx /* ecx = lock addr */
418 418 movl $-1, %edx
419 419 xchgb %dl, (%ecx) /* try to set lock */
420 420 testb %dl, %dl /* did we get it? */
421 421 jnz lock_set_spin /* no, go to C for the hard case */
422 422 .lock_set_lockstat_patch_point:
423 423 ret
424 424 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
425 425 movl $LS_LOCK_SET_ACQUIRE, %eax
426 426 jmp lockstat_wrapper
427 427 SET_SIZE(lock_set)
428 428
429 429 #endif /* !__amd64 */
430 430
431 431 #endif /* __lint */
432 432
433 433 /*
434 434 * lock_clear_splx(lp, s)
435 435 */
436 436
437 437 #if defined(lint) || defined(__lint)
438 438
439 439 /* ARGSUSED */
440 440 void
441 441 lock_clear_splx(lock_t *lp, int s)
442 442 {}
443 443
444 444 #else /* __lint */
445 445
446 446 #if defined(__amd64)
447 447
448 448 ENTRY(lock_clear_splx)
449 449 movb $0, (%rdi) /* clear lock */
450 450 .lock_clear_splx_lockstat_patch_point:
451 451 jmp 0f
452 452 0:
453 453 movl %esi, %edi /* arg for splx */
454 454 jmp splx /* let splx do its thing */
455 455 .lock_clear_splx_lockstat:
456 456 pushq %rbp /* align stack properly */
457 457 movq %rsp, %rbp
458 458 subq $16, %rsp /* space to save args across splx */
459 459 movq %rdi, 8(%rsp) /* save lock ptr across splx call */
460 460 movl %esi, %edi /* arg for splx */
461 461 call splx /* lower the priority */
462 462 movq 8(%rsp), %rsi /* rsi = lock ptr */
463 463 leave /* unwind stack */
464 464 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
465 465 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
466 466 jmp lockstat_wrapper
467 467 SET_SIZE(lock_clear_splx)
468 468
469 469 #else
470 470
471 471 ENTRY(lock_clear_splx)
472 472 movl 4(%esp), %eax /* eax = lock addr */
473 473 movb $0, (%eax) /* clear lock */
474 474 .lock_clear_splx_lockstat_patch_point:
475 475 jmp 0f
476 476 0:
477 477 movl 8(%esp), %edx /* edx = desired pil */
478 478 movl %edx, 4(%esp) /* set spl arg up for splx */
479 479 jmp splx /* let splx do it's thing */
480 480 .lock_clear_splx_lockstat:
481 481 movl 8(%esp), %edx /* edx = desired pil */
482 482 pushl %ebp /* set up stack frame */
483 483 movl %esp, %ebp
484 484 pushl %edx
485 485 call splx
486 486 leave /* unwind stack */
487 487 movl 4(%esp), %ecx /* ecx = lock pointer */
488 488 movl %gs:CPU_THREAD, %edx /* edx = thread addr */
489 489 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %eax
490 490 jmp lockstat_wrapper
491 491 SET_SIZE(lock_clear_splx)
492 492
493 493 #endif /* !__amd64 */
494 494
495 495 #if defined(__GNUC_AS__)
496 496 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
497 497 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
498 498
499 499 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
500 500 (.lock_clear_splx_lockstat_patch_point + 1)
501 501 #else
502 502 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
503 503 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
504 504
505 505 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
506 506 [.lock_clear_splx_lockstat_patch_point + 1]
507 507 #endif
508 508
509 509 #endif /* __lint */
510 510
511 511 /*
512 512 * mutex_enter() and mutex_exit().
513 513 *
514 514 * These routines handle the simple cases of mutex_enter() (adaptive
515 515 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
516 516 * If anything complicated is going on we punt to mutex_vector_enter().
517 517 *
518 518 * mutex_tryenter() is similar to mutex_enter() but returns zero if
519 519 * the lock cannot be acquired, nonzero on success.
520 520 *
521 521 * If mutex_exit() gets preempted in the window between checking waiters
522 522 * and clearing the lock, we can miss wakeups. Disabling preemption
523 523 * in the mutex code is prohibitively expensive, so instead we detect
524 524 * mutex preemption by examining the trapped PC in the interrupt path.
525 525 * If we interrupt a thread in mutex_exit() that has not yet cleared
526 526 * the lock, cmnint() resets its PC back to the beginning of
527 527 * mutex_exit() so it will check again for waiters when it resumes.
528 528 *
529 529 * The lockstat code below is activated when the lockstat driver
530 530 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
531 531 * Note that we don't need to test lockstat_event_mask here -- we won't
532 532 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
533 533 */
534 534 #if defined(lint) || defined(__lint)
535 535
536 536 /* ARGSUSED */
537 537 void
538 538 mutex_enter(kmutex_t *lp)
539 539 {}
540 540
541 541 /* ARGSUSED */
542 542 int
543 543 mutex_tryenter(kmutex_t *lp)
544 544 { return (0); }
545 545
546 546 /* ARGSUSED */
547 547 int
548 548 mutex_adaptive_tryenter(mutex_impl_t *lp)
549 549 { return (0); }
550 550
551 551 /* ARGSUSED */
552 552 void
553 553 mutex_exit(kmutex_t *lp)
554 554 {}
555 555
556 556 #else
557 557
558 558 #if defined(__amd64)
559 559
560 560 ENTRY_NP(mutex_enter)
561 561 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
562 562 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
563 563 lock
564 564 cmpxchgq %rdx, (%rdi)
565 565 jnz mutex_vector_enter
566 566 .mutex_enter_lockstat_patch_point:
567 567 #if defined(OPTERON_WORKAROUND_6323525)
568 568 .mutex_enter_6323525_patch_point:
569 569 ret /* nop space for lfence */
570 570 nop
571 571 nop
572 572 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
573 573 nop
574 574 #else /* OPTERON_WORKAROUND_6323525 */
575 575 ret
576 576 #endif /* OPTERON_WORKAROUND_6323525 */
577 577 movq %rdi, %rsi
578 578 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
579 579 /*
580 580 * expects %rdx=thread, %rsi=lock, %edi=lockstat event
|
↓ open down ↓ |
580 lines elided |
↑ open up ↑ |
581 581 */
582 582 ALTENTRY(lockstat_wrapper)
583 583 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */
584 584 leaq lockstat_probemap(%rip), %rax
585 585 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
586 586 testl %eax, %eax /* check for non-zero probe */
587 587 jz 1f
588 588 pushq %rbp /* align stack properly */
589 589 movq %rsp, %rbp
590 590 movl %eax, %edi
591 - call *lockstat_probe
591 + movq lockstat_probe, %rax
592 + INDIRECT_CALL_REG(rax)
592 593 leave /* unwind stack */
593 594 1:
594 595 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
595 596 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
596 597 movl $1, %eax /* return success if tryenter */
597 598 ret
598 599 SET_SIZE(lockstat_wrapper)
599 600 SET_SIZE(mutex_enter)
600 601
601 602 /*
602 603 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event
|
↓ open down ↓ |
1 lines elided |
↑ open up ↑ |
603 604 */
604 605 ENTRY(lockstat_wrapper_arg)
605 606 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */
606 607 leaq lockstat_probemap(%rip), %rax
607 608 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
608 609 testl %eax, %eax /* check for non-zero probe */
609 610 jz 1f
610 611 pushq %rbp /* align stack properly */
611 612 movq %rsp, %rbp
612 613 movl %eax, %edi
613 - call *lockstat_probe
614 + movq lockstat_probe, %rax
615 + INDIRECT_CALL_REG(rax)
614 616 leave /* unwind stack */
615 617 1:
616 618 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
617 619 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
618 620 movl $1, %eax /* return success if tryenter */
619 621 ret
620 622 SET_SIZE(lockstat_wrapper_arg)
621 623
622 624
623 625 ENTRY(mutex_tryenter)
624 626 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
625 627 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
626 628 lock
627 629 cmpxchgq %rdx, (%rdi)
628 630 jnz mutex_vector_tryenter
629 631 not %eax /* return success (nonzero) */
630 632 #if defined(OPTERON_WORKAROUND_6323525)
631 633 .mutex_tryenter_lockstat_patch_point:
632 634 .mutex_tryenter_6323525_patch_point:
633 635 ret /* nop space for lfence */
634 636 nop
635 637 nop
636 638 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
637 639 nop
638 640 #else /* OPTERON_WORKAROUND_6323525 */
639 641 .mutex_tryenter_lockstat_patch_point:
640 642 ret
641 643 #endif /* OPTERON_WORKAROUND_6323525 */
642 644 movq %rdi, %rsi
643 645 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
644 646 jmp lockstat_wrapper
645 647 SET_SIZE(mutex_tryenter)
646 648
647 649 ENTRY(mutex_adaptive_tryenter)
648 650 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
649 651 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
650 652 lock
651 653 cmpxchgq %rdx, (%rdi)
652 654 jnz 0f
653 655 not %eax /* return success (nonzero) */
654 656 #if defined(OPTERON_WORKAROUND_6323525)
655 657 .mutex_atryenter_6323525_patch_point:
656 658 ret /* nop space for lfence */
657 659 nop
658 660 nop
659 661 nop
660 662 #else /* OPTERON_WORKAROUND_6323525 */
661 663 ret
662 664 #endif /* OPTERON_WORKAROUND_6323525 */
663 665 0:
664 666 xorl %eax, %eax /* return failure */
665 667 ret
666 668 SET_SIZE(mutex_adaptive_tryenter)
667 669
668 670 .globl mutex_owner_running_critical_start
669 671
670 672 ENTRY(mutex_owner_running)
671 673 mutex_owner_running_critical_start:
672 674 movq (%rdi), %r11 /* get owner field */
673 675 andq $MUTEX_THREAD, %r11 /* remove waiters bit */
674 676 cmpq $0, %r11 /* if free, skip */
675 677 je 1f /* go return 0 */
676 678 movq T_CPU(%r11), %r8 /* get owner->t_cpu */
677 679 movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */
678 680 .mutex_owner_running_critical_end:
679 681 cmpq %r11, %r9 /* owner == running thread? */
680 682 je 2f /* yes, go return cpu */
681 683 1:
682 684 xorq %rax, %rax /* return 0 */
683 685 ret
684 686 2:
685 687 movq %r8, %rax /* return cpu */
686 688 ret
687 689 SET_SIZE(mutex_owner_running)
688 690
689 691 .globl mutex_owner_running_critical_size
690 692 .type mutex_owner_running_critical_size, @object
691 693 .align CPTRSIZE
692 694 mutex_owner_running_critical_size:
693 695 .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start
694 696 SET_SIZE(mutex_owner_running_critical_size)
695 697
696 698 .globl mutex_exit_critical_start
697 699
698 700 ENTRY(mutex_exit)
699 701 mutex_exit_critical_start: /* If interrupted, restart here */
700 702 movq %gs:CPU_THREAD, %rdx
701 703 cmpq %rdx, (%rdi)
702 704 jne mutex_vector_exit /* wrong type or wrong owner */
703 705 movq $0, (%rdi) /* clear owner AND lock */
704 706 .mutex_exit_critical_end:
705 707 .mutex_exit_lockstat_patch_point:
706 708 ret
707 709 movq %rdi, %rsi
708 710 movl $LS_MUTEX_EXIT_RELEASE, %edi
709 711 jmp lockstat_wrapper
710 712 SET_SIZE(mutex_exit)
711 713
712 714 .globl mutex_exit_critical_size
713 715 .type mutex_exit_critical_size, @object
714 716 .align CPTRSIZE
715 717 mutex_exit_critical_size:
716 718 .quad .mutex_exit_critical_end - mutex_exit_critical_start
717 719 SET_SIZE(mutex_exit_critical_size)
718 720
719 721 #else
720 722
721 723 ENTRY_NP(mutex_enter)
722 724 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
723 725 movl 4(%esp), %ecx /* ecx = lock ptr */
724 726 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
725 727 lock
726 728 cmpxchgl %edx, (%ecx)
727 729 jnz mutex_vector_enter
728 730 #if defined(OPTERON_WORKAROUND_6323525)
729 731 .mutex_enter_lockstat_patch_point:
730 732 .mutex_enter_6323525_patch_point:
731 733 ret /* nop space for lfence */
732 734 nop
733 735 nop
734 736 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
735 737 nop
736 738 #else /* OPTERON_WORKAROUND_6323525 */
737 739 .mutex_enter_lockstat_patch_point:
738 740 ret
739 741 #endif /* OPTERON_WORKAROUND_6323525 */
740 742 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
741 743 ALTENTRY(lockstat_wrapper) /* expects edx=thread, ecx=lock, */
742 744 /* eax=lockstat event */
743 745 pushl %ebp /* buy a frame */
744 746 movl %esp, %ebp
745 747 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
746 748 pushl %edx /* save thread pointer */
747 749 movl $lockstat_probemap, %edx
748 750 movl (%edx, %eax, DTRACE_IDSIZE), %eax
749 751 testl %eax, %eax /* check for non-zero probe */
750 752 jz 1f
751 753 pushl %ecx /* push lock */
752 754 pushl %eax /* push probe ID */
753 755 call *lockstat_probe
754 756 addl $8, %esp
755 757 1:
756 758 popl %edx /* restore thread pointer */
757 759 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
758 760 movl $1, %eax /* return success if tryenter */
759 761 popl %ebp /* pop off frame */
760 762 ret
761 763 SET_SIZE(lockstat_wrapper)
762 764 SET_SIZE(mutex_enter)
763 765
764 766 ENTRY(lockstat_wrapper_arg) /* expects edx=thread, ecx=lock, */
765 767 /* eax=lockstat event, pushed arg */
766 768 incb T_LOCKSTAT(%edx) /* curthread->t_lockstat++ */
767 769 pushl %edx /* save thread pointer */
768 770 movl $lockstat_probemap, %edx
769 771 movl (%edx, %eax, DTRACE_IDSIZE), %eax
770 772 testl %eax, %eax /* check for non-zero probe */
771 773 jz 1f
772 774 pushl %ebp /* save %ebp */
773 775 pushl 8(%esp) /* push arg1 */
774 776 movl %ebp, 12(%esp) /* fake up the stack frame */
775 777 movl %esp, %ebp /* fake up base pointer */
776 778 addl $12, %ebp /* adjust faked base pointer */
777 779 pushl %ecx /* push lock */
778 780 pushl %eax /* push probe ID */
779 781 call *lockstat_probe
780 782 addl $12, %esp /* adjust for arguments */
781 783 popl %ebp /* pop frame */
782 784 1:
783 785 popl %edx /* restore thread pointer */
784 786 decb T_LOCKSTAT(%edx) /* curthread->t_lockstat-- */
785 787 movl $1, %eax /* return success if tryenter */
786 788 addl $4, %esp /* pop argument */
787 789 ret
788 790 SET_SIZE(lockstat_wrapper_arg)
789 791
790 792
791 793 ENTRY(mutex_tryenter)
792 794 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
793 795 movl 4(%esp), %ecx /* ecx = lock ptr */
794 796 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
795 797 lock
796 798 cmpxchgl %edx, (%ecx)
797 799 jnz mutex_vector_tryenter
798 800 movl %ecx, %eax
799 801 #if defined(OPTERON_WORKAROUND_6323525)
800 802 .mutex_tryenter_lockstat_patch_point:
801 803 .mutex_tryenter_6323525_patch_point:
802 804 ret /* nop space for lfence */
803 805 nop
804 806 nop
805 807 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
806 808 nop
807 809 #else /* OPTERON_WORKAROUND_6323525 */
808 810 .mutex_tryenter_lockstat_patch_point:
809 811 ret
810 812 #endif /* OPTERON_WORKAROUND_6323525 */
811 813 movl $LS_MUTEX_ENTER_ACQUIRE, %eax
812 814 jmp lockstat_wrapper
813 815 SET_SIZE(mutex_tryenter)
814 816
815 817 ENTRY(mutex_adaptive_tryenter)
816 818 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
817 819 movl 4(%esp), %ecx /* ecx = lock ptr */
818 820 xorl %eax, %eax /* eax = 0 (unheld adaptive) */
819 821 lock
820 822 cmpxchgl %edx, (%ecx)
821 823 jnz 0f
822 824 movl %ecx, %eax
823 825 #if defined(OPTERON_WORKAROUND_6323525)
824 826 .mutex_atryenter_6323525_patch_point:
825 827 ret /* nop space for lfence */
826 828 nop
827 829 nop
828 830 nop
829 831 #else /* OPTERON_WORKAROUND_6323525 */
830 832 ret
831 833 #endif /* OPTERON_WORKAROUND_6323525 */
832 834 0:
833 835 xorl %eax, %eax
834 836 ret
835 837 SET_SIZE(mutex_adaptive_tryenter)
836 838
837 839 .globl mutex_owner_running_critical_start
838 840
839 841 ENTRY(mutex_owner_running)
840 842 mutex_owner_running_critical_start:
841 843 movl 4(%esp), %eax /* get owner field */
842 844 movl (%eax), %eax
843 845 andl $MUTEX_THREAD, %eax /* remove waiters bit */
844 846 cmpl $0, %eax /* if free, skip */
845 847 je 1f /* go return 0 */
846 848 movl T_CPU(%eax), %ecx /* get owner->t_cpu */
847 849 movl CPU_THREAD(%ecx), %edx /* get t_cpu->cpu_thread */
848 850 .mutex_owner_running_critical_end:
849 851 cmpl %eax, %edx /* owner == running thread? */
850 852 je 2f /* yes, go return cpu */
851 853 1:
852 854 xorl %eax, %eax /* return 0 */
853 855 ret
854 856 2:
855 857 movl %ecx, %eax /* return cpu */
856 858 ret
857 859
858 860 SET_SIZE(mutex_owner_running)
859 861
860 862 .globl mutex_owner_running_critical_size
861 863 .type mutex_owner_running_critical_size, @object
862 864 .align CPTRSIZE
863 865 mutex_owner_running_critical_size:
864 866 .long .mutex_owner_running_critical_end - mutex_owner_running_critical_start
865 867 SET_SIZE(mutex_owner_running_critical_size)
866 868
867 869 .globl mutex_exit_critical_start
868 870
869 871 ENTRY(mutex_exit)
870 872 mutex_exit_critical_start: /* If interrupted, restart here */
871 873 movl %gs:CPU_THREAD, %edx
872 874 movl 4(%esp), %ecx
873 875 cmpl %edx, (%ecx)
874 876 jne mutex_vector_exit /* wrong type or wrong owner */
875 877 movl $0, (%ecx) /* clear owner AND lock */
876 878 .mutex_exit_critical_end:
877 879 .mutex_exit_lockstat_patch_point:
878 880 ret
879 881 movl $LS_MUTEX_EXIT_RELEASE, %eax
880 882 jmp lockstat_wrapper
881 883 SET_SIZE(mutex_exit)
882 884
883 885 .globl mutex_exit_critical_size
884 886 .type mutex_exit_critical_size, @object
885 887 .align CPTRSIZE
886 888 mutex_exit_critical_size:
887 889 .long .mutex_exit_critical_end - mutex_exit_critical_start
888 890 SET_SIZE(mutex_exit_critical_size)
889 891
890 892 #endif /* !__amd64 */
891 893
892 894 #endif /* __lint */
893 895
894 896 /*
895 897 * rw_enter() and rw_exit().
896 898 *
897 899 * These routines handle the simple cases of rw_enter (write-locking an unheld
898 900 * lock or read-locking a lock that's neither write-locked nor write-wanted)
899 901 * and rw_exit (no waiters or not the last reader). If anything complicated
900 902 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
901 903 */
902 904 #if defined(lint) || defined(__lint)
903 905
904 906 /* ARGSUSED */
905 907 void
906 908 rw_enter(krwlock_t *lp, krw_t rw)
907 909 {}
908 910
909 911 /* ARGSUSED */
910 912 void
911 913 rw_exit(krwlock_t *lp)
912 914 {}
913 915
914 916 #else /* __lint */
915 917
916 918 #if defined(__amd64)
917 919
918 920 ENTRY(rw_enter)
919 921 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
920 922 cmpl $RW_WRITER, %esi
921 923 je .rw_write_enter
922 924 incl T_KPRI_REQ(%rdx) /* THREAD_KPRI_REQUEST() */
923 925 movq (%rdi), %rax /* rax = old rw_wwwh value */
924 926 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
925 927 jnz rw_enter_sleep
926 928 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
927 929 lock
928 930 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
929 931 jnz rw_enter_sleep
930 932 .rw_read_enter_lockstat_patch_point:
931 933 ret
932 934 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
933 935 movq %rdi, %rsi /* rsi = lock ptr */
934 936 movl $LS_RW_ENTER_ACQUIRE, %edi
935 937 movl $RW_READER, %edx
936 938 jmp lockstat_wrapper_arg
937 939 .rw_write_enter:
938 940 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */
939 941 xorl %eax, %eax /* rax = unheld value */
940 942 lock
941 943 cmpxchgq %rdx, (%rdi) /* try to grab write lock */
942 944 jnz rw_enter_sleep
943 945
944 946 #if defined(OPTERON_WORKAROUND_6323525)
945 947 .rw_write_enter_lockstat_patch_point:
946 948 .rw_write_enter_6323525_patch_point:
947 949 ret
948 950 nop
949 951 nop
950 952 .rw_write_enter_lockstat_6323525_patch_point:
951 953 nop
952 954 #else /* OPTERON_WORKAROUND_6323525 */
953 955 .rw_write_enter_lockstat_patch_point:
954 956 ret
955 957 #endif /* OPTERON_WORKAROUND_6323525 */
956 958
957 959 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
958 960 movq %rdi, %rsi /* rsi = lock ptr */
959 961 movl $LS_RW_ENTER_ACQUIRE, %edi
960 962 movl $RW_WRITER, %edx
961 963 jmp lockstat_wrapper_arg
962 964 SET_SIZE(rw_enter)
963 965
964 966 ENTRY(rw_exit)
965 967 movq (%rdi), %rax /* rax = old rw_wwwh value */
966 968 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
967 969 jne .rw_not_single_reader
968 970 xorl %edx, %edx /* rdx = new value (unheld) */
969 971 .rw_read_exit:
970 972 lock
971 973 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
972 974 jnz rw_exit_wakeup
973 975 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
974 976 decl T_KPRI_REQ(%rcx) /* THREAD_KPRI_RELEASE() */
975 977 .rw_read_exit_lockstat_patch_point:
976 978 ret
977 979 movq %rdi, %rsi /* rsi = lock ptr */
978 980 movl $LS_RW_EXIT_RELEASE, %edi
979 981 movl $RW_READER, %edx
980 982 jmp lockstat_wrapper_arg
981 983 .rw_not_single_reader:
982 984 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
983 985 jnz .rw_write_exit
984 986 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
985 987 cmpl $RW_READ_LOCK, %edx
986 988 jge .rw_read_exit /* not last reader, safe to drop */
987 989 jmp rw_exit_wakeup /* last reader with waiters */
988 990 .rw_write_exit:
989 991 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
990 992 xorl %edx, %edx /* rdx = new value (unheld) */
991 993 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
992 994 lock
993 995 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
994 996 jnz rw_exit_wakeup
995 997 .rw_write_exit_lockstat_patch_point:
996 998 ret
997 999 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
998 1000 movq %rdi, %rsi /* rsi - lock ptr */
999 1001 movl $LS_RW_EXIT_RELEASE, %edi
1000 1002 movl $RW_WRITER, %edx
1001 1003 jmp lockstat_wrapper_arg
1002 1004 SET_SIZE(rw_exit)
1003 1005
1004 1006 #else
1005 1007
1006 1008 ENTRY(rw_enter)
1007 1009 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1008 1010 movl 4(%esp), %ecx /* ecx = lock ptr */
1009 1011 cmpl $RW_WRITER, 8(%esp)
1010 1012 je .rw_write_enter
1011 1013 incl T_KPRI_REQ(%edx) /* THREAD_KPRI_REQUEST() */
1012 1014 movl (%ecx), %eax /* eax = old rw_wwwh value */
1013 1015 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1014 1016 jnz rw_enter_sleep
1015 1017 leal RW_READ_LOCK(%eax), %edx /* edx = new rw_wwwh value */
1016 1018 lock
1017 1019 cmpxchgl %edx, (%ecx) /* try to grab read lock */
1018 1020 jnz rw_enter_sleep
1019 1021 .rw_read_enter_lockstat_patch_point:
1020 1022 ret
1021 1023 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1022 1024 movl $LS_RW_ENTER_ACQUIRE, %eax
1023 1025 pushl $RW_READER
1024 1026 jmp lockstat_wrapper_arg
1025 1027 .rw_write_enter:
1026 1028 orl $RW_WRITE_LOCKED, %edx /* edx = write-locked value */
1027 1029 xorl %eax, %eax /* eax = unheld value */
1028 1030 lock
1029 1031 cmpxchgl %edx, (%ecx) /* try to grab write lock */
1030 1032 jnz rw_enter_sleep
1031 1033
1032 1034 #if defined(OPTERON_WORKAROUND_6323525)
1033 1035 .rw_write_enter_lockstat_patch_point:
1034 1036 .rw_write_enter_6323525_patch_point:
1035 1037 ret
1036 1038 nop
1037 1039 nop
1038 1040 .rw_write_enter_lockstat_6323525_patch_point:
1039 1041 nop
1040 1042 #else /* OPTERON_WORKAROUND_6323525 */
1041 1043 .rw_write_enter_lockstat_patch_point:
1042 1044 ret
1043 1045 #endif /* OPTERON_WORKAROUND_6323525 */
1044 1046
1045 1047 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1046 1048 movl $LS_RW_ENTER_ACQUIRE, %eax
1047 1049 pushl $RW_WRITER
1048 1050 jmp lockstat_wrapper_arg
1049 1051 SET_SIZE(rw_enter)
1050 1052
1051 1053 ENTRY(rw_exit)
1052 1054 movl 4(%esp), %ecx /* ecx = lock ptr */
1053 1055 movl (%ecx), %eax /* eax = old rw_wwwh value */
1054 1056 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
1055 1057 jne .rw_not_single_reader
1056 1058 xorl %edx, %edx /* edx = new value (unheld) */
1057 1059 .rw_read_exit:
1058 1060 lock
1059 1061 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1060 1062 jnz rw_exit_wakeup
1061 1063 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1062 1064 decl T_KPRI_REQ(%edx) /* THREAD_KPRI_RELEASE() */
1063 1065 .rw_read_exit_lockstat_patch_point:
1064 1066 ret
1065 1067 movl $LS_RW_EXIT_RELEASE, %eax
1066 1068 pushl $RW_READER
1067 1069 jmp lockstat_wrapper_arg
1068 1070 .rw_not_single_reader:
1069 1071 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
1070 1072 jnz .rw_write_exit
1071 1073 leal -RW_READ_LOCK(%eax), %edx /* edx = new value */
1072 1074 cmpl $RW_READ_LOCK, %edx
1073 1075 jge .rw_read_exit /* not last reader, safe to drop */
1074 1076 jmp rw_exit_wakeup /* last reader with waiters */
1075 1077 .rw_write_exit:
1076 1078 movl %gs:CPU_THREAD, %eax /* eax = thread ptr */
1077 1079 xorl %edx, %edx /* edx = new value (unheld) */
1078 1080 orl $RW_WRITE_LOCKED, %eax /* eax = write-locked value */
1079 1081 lock
1080 1082 cmpxchgl %edx, (%ecx) /* try to drop read lock */
1081 1083 jnz rw_exit_wakeup
1082 1084 .rw_write_exit_lockstat_patch_point:
1083 1085 ret
1084 1086 movl %gs:CPU_THREAD, %edx /* edx = thread ptr */
1085 1087 movl $LS_RW_EXIT_RELEASE, %eax
1086 1088 pushl $RW_WRITER
1087 1089 jmp lockstat_wrapper_arg
1088 1090 SET_SIZE(rw_exit)
1089 1091
1090 1092 #endif /* !__amd64 */
1091 1093
1092 1094 #endif /* __lint */
1093 1095
1094 1096 #if defined(OPTERON_WORKAROUND_6323525)
1095 1097 #if defined(lint) || defined(__lint)
1096 1098
1097 1099 int workaround_6323525_patched;
1098 1100
1099 1101 void
1100 1102 patch_workaround_6323525(void)
1101 1103 {}
1102 1104
1103 1105 #else /* lint */
1104 1106
1105 1107 /*
1106 1108 * If it is necessary to patch the lock enter routines with the lfence
1107 1109 * workaround, workaround_6323525_patched is set to a non-zero value so that
1108 1110 * the lockstat_hat_patch routine can patch to the new location of the 'ret'
1109 1111 * instruction.
1110 1112 */
1111 1113 DGDEF3(workaround_6323525_patched, 4, 4)
1112 1114 .long 0
1113 1115
1114 1116 #if defined(__amd64)
1115 1117
1116 1118 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1117 1119 movq $size, %rbx; \
1118 1120 movq $dstaddr, %r13; \
1119 1121 addq %rbx, %r13; \
1120 1122 movq $srcaddr, %r12; \
1121 1123 addq %rbx, %r12; \
1122 1124 0: \
1123 1125 decq %r13; \
1124 1126 decq %r12; \
1125 1127 movzbl (%r12), %esi; \
1126 1128 movq $1, %rdx; \
1127 1129 movq %r13, %rdi; \
1128 1130 call hot_patch_kernel_text; \
1129 1131 decq %rbx; \
1130 1132 testq %rbx, %rbx; \
1131 1133 jg 0b;
1132 1134
1133 1135 /*
1134 1136 * patch_workaround_6323525: provide workaround for 6323525
1135 1137 *
1136 1138 * The workaround is to place a fencing instruction (lfence) between the
1137 1139 * mutex operation and the subsequent read-modify-write instruction.
1138 1140 *
1139 1141 * This routine hot patches the lfence instruction on top of the space
1140 1142 * reserved by nops in the lock enter routines.
1141 1143 */
1142 1144 ENTRY_NP(patch_workaround_6323525)
1143 1145 pushq %rbp
1144 1146 movq %rsp, %rbp
1145 1147 pushq %r12
1146 1148 pushq %r13
1147 1149 pushq %rbx
1148 1150
1149 1151 /*
1150 1152 * lockstat_hot_patch() to use the alternate lockstat workaround
1151 1153 * 6323525 patch points (points past the lfence instruction to the
1152 1154 * new ret) when workaround_6323525_patched is set.
1153 1155 */
1154 1156 movl $1, workaround_6323525_patched
1155 1157
1156 1158 /*
1157 1159 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter
1158 1160 * routines. The 4 bytes are patched in reverse order so that the
1159 1161 * the existing ret is overwritten last. This provides lock enter
1160 1162 * sanity during the intermediate patching stages.
1161 1163 */
1162 1164 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1163 1165 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1164 1166 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1165 1167 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1166 1168
1167 1169 popq %rbx
1168 1170 popq %r13
1169 1171 popq %r12
1170 1172 movq %rbp, %rsp
1171 1173 popq %rbp
1172 1174 ret
1173 1175 _lfence_insn:
1174 1176 lfence
1175 1177 ret
1176 1178 SET_SIZE(patch_workaround_6323525)
1177 1179
1178 1180
1179 1181 #else /* __amd64 */
1180 1182
1181 1183 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1182 1184 movl $size, %ebx; \
1183 1185 movl $srcaddr, %esi; \
1184 1186 addl %ebx, %esi; \
1185 1187 movl $dstaddr, %edi; \
1186 1188 addl %ebx, %edi; \
1187 1189 0: \
1188 1190 decl %esi; \
1189 1191 decl %edi; \
1190 1192 pushl $1; \
1191 1193 movzbl (%esi), %eax; \
1192 1194 pushl %eax; \
1193 1195 pushl %edi; \
1194 1196 call hot_patch_kernel_text; \
1195 1197 addl $12, %esp; \
1196 1198 decl %ebx; \
1197 1199 testl %ebx, %ebx; \
1198 1200 jg 0b;
1199 1201
1200 1202
1201 1203 /* see comments above */
1202 1204 ENTRY_NP(patch_workaround_6323525)
1203 1205 pushl %ebp
1204 1206 movl %esp, %ebp
1205 1207 pushl %ebx
1206 1208 pushl %esi
1207 1209 pushl %edi
1208 1210
1209 1211 movl $1, workaround_6323525_patched
1210 1212
1211 1213 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1212 1214 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1213 1215 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1214 1216 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1215 1217
1216 1218 popl %edi
1217 1219 popl %esi
1218 1220 popl %ebx
1219 1221 movl %ebp, %esp
1220 1222 popl %ebp
1221 1223 ret
1222 1224 _lfence_insn:
1223 1225 .byte 0xf, 0xae, 0xe8 / [lfence instruction]
1224 1226 ret
1225 1227 SET_SIZE(patch_workaround_6323525)
1226 1228
1227 1229 #endif /* !__amd64 */
1228 1230 #endif /* !lint */
1229 1231 #endif /* OPTERON_WORKAROUND_6323525 */
1230 1232
1231 1233
1232 1234 #if defined(lint) || defined(__lint)
1233 1235
1234 1236 void
1235 1237 lockstat_hot_patch(void)
1236 1238 {}
1237 1239
1238 1240 #else
1239 1241
1240 1242 #if defined(__amd64)
1241 1243
1242 1244 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1243 1245 movq $normal_instr, %rsi; \
1244 1246 movq $active_instr, %rdi; \
1245 1247 leaq lockstat_probemap(%rip), %rax; \
1246 1248 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1247 1249 testl %eax, %eax; \
1248 1250 jz 9f; \
1249 1251 movq %rdi, %rsi; \
1250 1252 9: \
1251 1253 movq $len, %rdx; \
1252 1254 movq $addr, %rdi; \
1253 1255 call hot_patch_kernel_text
1254 1256
1255 1257 #else
1256 1258
1257 1259 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1258 1260 movl $normal_instr, %ecx; \
1259 1261 movl $active_instr, %edx; \
1260 1262 movl $lockstat_probemap, %eax; \
1261 1263 movl _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \
1262 1264 testl %eax, %eax; \
1263 1265 jz . + 4; \
1264 1266 movl %edx, %ecx; \
1265 1267 pushl $len; \
1266 1268 pushl %ecx; \
1267 1269 pushl $addr; \
1268 1270 call hot_patch_kernel_text; \
1269 1271 addl $12, %esp;
1270 1272
1271 1273 #endif /* !__amd64 */
1272 1274
1273 1275 ENTRY(lockstat_hot_patch)
1274 1276 #if defined(__amd64)
1275 1277 pushq %rbp /* align stack properly */
1276 1278 movq %rsp, %rbp
1277 1279 #endif /* __amd64 */
1278 1280
1279 1281 #if defined(OPTERON_WORKAROUND_6323525)
1280 1282 cmpl $0, workaround_6323525_patched
1281 1283 je 1f
1282 1284 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
1283 1285 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1284 1286 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
1285 1287 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1286 1288 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
1287 1289 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1288 1290 jmp 2f
1289 1291 1:
1290 1292 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1291 1293 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1292 1294 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1293 1295 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1294 1296 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1295 1297 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1296 1298 2:
1297 1299 #else /* OPTERON_WORKAROUND_6323525 */
1298 1300 HOT_PATCH(.mutex_enter_lockstat_patch_point,
1299 1301 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1300 1302 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1301 1303 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1302 1304 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1303 1305 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1304 1306 #endif /* !OPTERON_WORKAROUND_6323525 */
1305 1307 HOT_PATCH(.mutex_exit_lockstat_patch_point,
1306 1308 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1307 1309 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
1308 1310 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1309 1311 HOT_PATCH(.rw_write_exit_lockstat_patch_point,
1310 1312 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1311 1313 HOT_PATCH(.rw_read_exit_lockstat_patch_point,
1312 1314 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1313 1315 HOT_PATCH(.lock_set_lockstat_patch_point,
1314 1316 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1315 1317 HOT_PATCH(.lock_try_lockstat_patch_point,
1316 1318 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1317 1319 HOT_PATCH(.lock_clear_lockstat_patch_point,
1318 1320 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
1319 1321 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
1320 1322 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1321 1323
1322 1324 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
1323 1325 LS_LOCK_CLEAR_SPLX_RELEASE,
1324 1326 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
1325 1327 #if defined(__amd64)
1326 1328 leave /* unwind stack */
1327 1329 #endif /* __amd64 */
1328 1330 ret
1329 1331 SET_SIZE(lockstat_hot_patch)
1330 1332
1331 1333 #endif /* __lint */
1332 1334
1333 1335 #if defined(lint) || defined(__lint)
1334 1336
1335 1337 /* XX64 membar_*() should be inlines */
1336 1338
1337 1339 void
1338 1340 membar_sync(void)
1339 1341 {}
1340 1342
1341 1343 void
1342 1344 membar_enter(void)
1343 1345 {}
1344 1346
1345 1347 void
1346 1348 membar_exit(void)
1347 1349 {}
1348 1350
1349 1351 void
1350 1352 membar_producer(void)
1351 1353 {}
1352 1354
1353 1355 void
1354 1356 membar_consumer(void)
1355 1357 {}
1356 1358
1357 1359 #else /* __lint */
1358 1360
1359 1361 #if defined(__amd64)
1360 1362
1361 1363 ENTRY(membar_enter)
1362 1364 ALTENTRY(membar_exit)
1363 1365 ALTENTRY(membar_sync)
1364 1366 mfence /* lighter weight than lock; xorq $0,(%rsp) */
1365 1367 ret
1366 1368 SET_SIZE(membar_sync)
1367 1369 SET_SIZE(membar_exit)
1368 1370 SET_SIZE(membar_enter)
1369 1371
1370 1372 ENTRY(membar_producer)
1371 1373 sfence
1372 1374 ret
1373 1375 SET_SIZE(membar_producer)
1374 1376
1375 1377 ENTRY(membar_consumer)
1376 1378 lfence
1377 1379 ret
1378 1380 SET_SIZE(membar_consumer)
1379 1381
1380 1382 #else
1381 1383
1382 1384 ENTRY(membar_enter)
1383 1385 ALTENTRY(membar_exit)
1384 1386 ALTENTRY(membar_sync)
1385 1387 lock
1386 1388 xorl $0, (%esp)
1387 1389 ret
1388 1390 SET_SIZE(membar_sync)
1389 1391 SET_SIZE(membar_exit)
1390 1392 SET_SIZE(membar_enter)
1391 1393
1392 1394 /*
1393 1395 * On machines that support sfence and lfence, these
1394 1396 * memory barriers can be more precisely implemented
1395 1397 * without causing the whole world to stop
1396 1398 */
1397 1399 ENTRY(membar_producer)
1398 1400 .globl _patch_sfence_ret
1399 1401 _patch_sfence_ret: /* c.f. membar #StoreStore */
1400 1402 lock
1401 1403 xorl $0, (%esp)
1402 1404 ret
1403 1405 SET_SIZE(membar_producer)
1404 1406
1405 1407 ENTRY(membar_consumer)
1406 1408 .globl _patch_lfence_ret
1407 1409 _patch_lfence_ret: /* c.f. membar #LoadLoad */
1408 1410 lock
1409 1411 xorl $0, (%esp)
1410 1412 ret
1411 1413 SET_SIZE(membar_consumer)
1412 1414
1413 1415 #endif /* !__amd64 */
1414 1416
1415 1417 #endif /* __lint */
1416 1418
1417 1419 /*
1418 1420 * thread_onproc()
1419 1421 * Set thread in onproc state for the specified CPU.
1420 1422 * Also set the thread lock pointer to the CPU's onproc lock.
1421 1423 * Since the new lock isn't held, the store ordering is important.
1422 1424 * If not done in assembler, the compiler could reorder the stores.
1423 1425 */
1424 1426 #if defined(lint) || defined(__lint)
1425 1427
1426 1428 void
1427 1429 thread_onproc(kthread_id_t t, cpu_t *cp)
1428 1430 {
1429 1431 t->t_state = TS_ONPROC;
1430 1432 t->t_lockp = &cp->cpu_thread_lock;
1431 1433 }
1432 1434
1433 1435 #else /* __lint */
1434 1436
1435 1437 #if defined(__amd64)
1436 1438
1437 1439 ENTRY(thread_onproc)
1438 1440 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */
1439 1441 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */
1440 1442 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */
1441 1443 ret
1442 1444 SET_SIZE(thread_onproc)
1443 1445
1444 1446 #else
1445 1447
1446 1448 ENTRY(thread_onproc)
1447 1449 movl 4(%esp), %eax
1448 1450 movl 8(%esp), %ecx
1449 1451 addl $CPU_THREAD_LOCK, %ecx /* pointer to disp_lock while running */
1450 1452 movl $ONPROC_THREAD, T_STATE(%eax) /* set state to TS_ONPROC */
1451 1453 movl %ecx, T_LOCKP(%eax) /* store new lock pointer */
1452 1454 ret
1453 1455 SET_SIZE(thread_onproc)
1454 1456
1455 1457 #endif /* !__amd64 */
1456 1458
1457 1459 #endif /* __lint */
1458 1460
1459 1461 /*
1460 1462 * mutex_delay_default(void)
1461 1463 * Spins for approx a few hundred processor cycles and returns to caller.
1462 1464 */
1463 1465
1464 1466 #if defined(lint) || defined(__lint)
1465 1467
1466 1468 void
1467 1469 mutex_delay_default(void)
1468 1470 {}
1469 1471
1470 1472 #else /* __lint */
1471 1473
1472 1474 #if defined(__amd64)
1473 1475
1474 1476 ENTRY(mutex_delay_default)
1475 1477 movq $92,%r11
1476 1478 0: decq %r11
1477 1479 jg 0b
1478 1480 ret
1479 1481 SET_SIZE(mutex_delay_default)
1480 1482
1481 1483 #else
1482 1484
1483 1485 ENTRY(mutex_delay_default)
1484 1486 push %ebp
1485 1487 movl %esp,%ebp
1486 1488 andl $-16,%esp
1487 1489 push %ebx
1488 1490 movl $93,%ebx
1489 1491 0: decl %ebx
1490 1492 jg 0b
1491 1493 pop %ebx
1492 1494 leave
1493 1495 ret
1494 1496 SET_SIZE(mutex_delay_default)
1495 1497
1496 1498 #endif /* !__amd64 */
1497 1499 #endif /* __lint */
|
↓ open down ↓ |
874 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX