Print this page
12771 11909 re-introduced lockstat panic
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/lock_prim.s
+++ new/usr/src/uts/intel/ia32/ml/lock_prim.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /*
27 27 * Copyright 2019 Joyent, Inc.
28 28 */
29 29
30 30 #include "assym.h"
31 31
32 32 #include <sys/mutex_impl.h>
33 33 #include <sys/asm_linkage.h>
34 34 #include <sys/asm_misc.h>
35 35 #include <sys/regset.h>
36 36 #include <sys/rwlock_impl.h>
37 37 #include <sys/lockstat.h>
38 38
39 39 /*
40 40 * lock_try(lp), ulock_try(lp)
41 41 * - returns non-zero on success.
42 42 * - doesn't block interrupts so don't use this to spin on a lock.
43 43 *
44 44 * ulock_try() is for a lock in the user address space.
45 45 */
46 46
47 47 .globl kernelbase
48 48
49 49 ENTRY(lock_try)
50 50 movb $-1, %dl
51 51 movzbq %dl, %rax
52 52 xchgb %dl, (%rdi)
53 53 xorb %dl, %al
54 54 .lock_try_lockstat_patch_point:
55 55 ret
56 56 testb %al, %al
57 57 jnz 0f
58 58 ret
59 59 0:
60 60 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
61 61 movq %rdi, %rsi /* rsi = lock addr */
62 62 movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
63 63 jmp lockstat_wrapper
64 64 SET_SIZE(lock_try)
65 65
66 66 ENTRY(lock_spin_try)
67 67 movb $-1, %dl
68 68 movzbq %dl, %rax
69 69 xchgb %dl, (%rdi)
70 70 xorb %dl, %al
71 71 ret
72 72 SET_SIZE(lock_spin_try)
73 73
74 74 ENTRY(ulock_try)
75 75 #ifdef DEBUG
76 76 movq kernelbase(%rip), %rax
77 77 cmpq %rax, %rdi /* test uaddr < kernelbase */
78 78 jb ulock_pass /* uaddr < kernelbase, proceed */
79 79
80 80 movq %rdi, %r12 /* preserve lock ptr for debugging */
81 81 leaq .ulock_panic_msg(%rip), %rdi
82 82 pushq %rbp /* align stack properly */
83 83 movq %rsp, %rbp
84 84 xorl %eax, %eax /* clear for varargs */
85 85 call panic
86 86
87 87 #endif /* DEBUG */
88 88
89 89 ulock_pass:
90 90 movl $1, %eax
91 91 xchgb %al, (%rdi)
92 92 xorb $1, %al
93 93 ret
94 94 SET_SIZE(ulock_try)
95 95
96 96 #ifdef DEBUG
97 97 .data
98 98 .ulock_panic_msg:
99 99 .string "ulock_try: Argument is above kernelbase"
100 100 .text
101 101 #endif /* DEBUG */
102 102
103 103 /*
104 104 * lock_clear(lp)
105 105 * - unlock lock without changing interrupt priority level.
106 106 */
107 107
108 108 ENTRY(lock_clear)
109 109 movb $0, (%rdi)
110 110 .lock_clear_lockstat_patch_point:
111 111 ret
112 112 movq %rdi, %rsi /* rsi = lock addr */
113 113 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
114 114 movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */
115 115 jmp lockstat_wrapper
116 116 SET_SIZE(lock_clear)
117 117
118 118 ENTRY(ulock_clear)
119 119 #ifdef DEBUG
120 120 movq kernelbase(%rip), %rcx
121 121 cmpq %rcx, %rdi /* test uaddr < kernelbase */
122 122 jb ulock_clr /* uaddr < kernelbase, proceed */
123 123
124 124 leaq .ulock_clear_msg(%rip), %rdi
125 125 pushq %rbp /* align stack properly */
126 126 movq %rsp, %rbp
127 127 xorl %eax, %eax /* clear for varargs */
128 128 call panic
129 129 #endif
130 130
131 131 ulock_clr:
132 132 movb $0, (%rdi)
133 133 ret
134 134 SET_SIZE(ulock_clear)
135 135
136 136 #ifdef DEBUG
137 137 .data
138 138 .ulock_clear_msg:
139 139 .string "ulock_clear: Argument is above kernelbase"
140 140 .text
141 141 #endif /* DEBUG */
142 142
143 143
144 144 /*
145 145 * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
146 146 * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
147 147 */
148 148
149 149 ENTRY(lock_set_spl)
150 150 pushq %rbp
151 151 movq %rsp, %rbp
152 152 subq $32, %rsp
153 153 movl %esi, 8(%rsp) /* save priority level */
154 154 movq %rdx, 16(%rsp) /* save old pil ptr */
155 155 movq %rdi, 24(%rsp) /* save lock pointer */
156 156 movl %esi, %edi /* pass priority level */
157 157 call splr /* raise priority level */
158 158 movq 24(%rsp), %rdi /* rdi = lock addr */
159 159 movb $-1, %dl
160 160 xchgb %dl, (%rdi) /* try to set lock */
161 161 testb %dl, %dl /* did we get the lock? ... */
162 162 jnz .lss_miss /* ... no, go to C for the hard case */
163 163 movq 16(%rsp), %rdx /* rdx = old pil addr */
164 164 movw %ax, (%rdx) /* store old pil */
165 165 leave
166 166 .lock_set_spl_lockstat_patch_point:
167 167 ret
168 168 movq %rdi, %rsi /* rsi = lock addr */
169 169 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
170 170 movl $LS_LOCK_SET_SPL_ACQUIRE, %edi
171 171 jmp lockstat_wrapper
172 172 .lss_miss:
173 173 movl 8(%rsp), %esi /* new_pil */
174 174 movq 16(%rsp), %rdx /* old_pil_addr */
175 175 movl %eax, %ecx /* original pil */
176 176 leave /* unwind stack */
177 177 jmp lock_set_spl_spin
178 178 SET_SIZE(lock_set_spl)
179 179
180 180 /*
181 181 * void
182 182 * lock_init(lp)
183 183 */
184 184
185 185 ENTRY(lock_init)
186 186 movb $0, (%rdi)
187 187 ret
188 188 SET_SIZE(lock_init)
189 189
190 190 /*
191 191 * void
192 192 * lock_set(lp)
193 193 */
194 194
195 195 ENTRY(lock_set)
196 196 movb $-1, %dl
197 197 xchgb %dl, (%rdi) /* try to set lock */
198 198 testb %dl, %dl /* did we get it? */
199 199 jnz lock_set_spin /* no, go to C for the hard case */
200 200 .lock_set_lockstat_patch_point:
201 201 ret
202 202 movq %rdi, %rsi /* rsi = lock addr */
203 203 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
204 204 movl $LS_LOCK_SET_ACQUIRE, %edi
205 205 jmp lockstat_wrapper
206 206 SET_SIZE(lock_set)
207 207
208 208 /*
209 209 * lock_clear_splx(lp, s)
210 210 */
211 211
212 212 ENTRY(lock_clear_splx)
213 213 movb $0, (%rdi) /* clear lock */
214 214 .lock_clear_splx_lockstat_patch_point:
215 215 jmp 0f
216 216 0:
217 217 movl %esi, %edi /* arg for splx */
218 218 jmp splx /* let splx do its thing */
219 219 .lock_clear_splx_lockstat:
220 220 pushq %rbp /* align stack properly */
221 221 movq %rsp, %rbp
222 222 subq $16, %rsp /* space to save args across splx */
223 223 movq %rdi, 8(%rsp) /* save lock ptr across splx call */
224 224 movl %esi, %edi /* arg for splx */
225 225 call splx /* lower the priority */
226 226 movq 8(%rsp), %rsi /* rsi = lock ptr */
227 227 leave /* unwind stack */
228 228 movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */
229 229 movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
230 230 jmp lockstat_wrapper
231 231 SET_SIZE(lock_clear_splx)
232 232
233 233 #if defined(__GNUC_AS__)
234 234 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
235 235 (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
236 236
237 237 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
238 238 (.lock_clear_splx_lockstat_patch_point + 1)
239 239 #else
240 240 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL \
241 241 [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
242 242
243 243 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT \
244 244 [.lock_clear_splx_lockstat_patch_point + 1]
245 245 #endif
246 246
247 247 /*
248 248 * mutex_enter() and mutex_exit().
249 249 *
250 250 * These routines handle the simple cases of mutex_enter() (adaptive
251 251 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
252 252 * If anything complicated is going on we punt to mutex_vector_enter().
253 253 *
254 254 * mutex_tryenter() is similar to mutex_enter() but returns zero if
255 255 * the lock cannot be acquired, nonzero on success.
256 256 *
257 257 * If mutex_exit() gets preempted in the window between checking waiters
258 258 * and clearing the lock, we can miss wakeups. Disabling preemption
259 259 * in the mutex code is prohibitively expensive, so instead we detect
260 260 * mutex preemption by examining the trapped PC in the interrupt path.
261 261 * If we interrupt a thread in mutex_exit() that has not yet cleared
262 262 * the lock, cmnint() resets its PC back to the beginning of
263 263 * mutex_exit() so it will check again for waiters when it resumes.
264 264 *
265 265 * The lockstat code below is activated when the lockstat driver
266 266 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
267 267 * Note that we don't need to test lockstat_event_mask here -- we won't
268 268 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
269 269 */
270 270
271 271 ENTRY_NP(mutex_enter)
272 272 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
273 273 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
274 274 lock
275 275 cmpxchgq %rdx, (%rdi)
276 276 jnz mutex_vector_enter
277 277 .mutex_enter_lockstat_patch_point:
278 278 #if defined(OPTERON_WORKAROUND_6323525)
279 279 .mutex_enter_6323525_patch_point:
280 280 ret /* nop space for lfence */
281 281 nop
282 282 nop
283 283 .mutex_enter_lockstat_6323525_patch_point: /* new patch point if lfence */
284 284 nop
285 285 #else /* OPTERON_WORKAROUND_6323525 */
286 286 ret
287 287 #endif /* OPTERON_WORKAROUND_6323525 */
288 288 movq %rdi, %rsi
289 289 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
290 290 /*
291 291 * expects %rdx=thread, %rsi=lock, %edi=lockstat event
292 292 */
293 293 ALTENTRY(lockstat_wrapper)
294 294 incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */
295 295 leaq lockstat_probemap(%rip), %rax
296 296 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
297 297 testl %eax, %eax /* check for non-zero probe */
298 298 jz 1f
299 299 pushq %rbp /* align stack properly */
300 300 movq %rsp, %rbp
301 301 movl %eax, %edi
302 302 movq lockstat_probe, %rax
303 303 INDIRECT_CALL_REG(rax)
304 304 leave /* unwind stack */
305 305 1:
306 306 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
307 307 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
308 308 movl $1, %eax /* return success if tryenter */
309 309 ret
310 310 SET_SIZE(lockstat_wrapper)
311 311 SET_SIZE(mutex_enter)
312 312
313 313 /*
314 314 * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event
315 315 */
316 316 ENTRY(lockstat_wrapper_arg)
317 317 incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */
318 318 leaq lockstat_probemap(%rip), %rax
319 319 movl (%rax, %rdi, DTRACE_IDSIZE), %eax
320 320 testl %eax, %eax /* check for non-zero probe */
321 321 jz 1f
322 322 pushq %rbp /* align stack properly */
323 323 movq %rsp, %rbp
324 324 movl %eax, %edi
325 325 movq lockstat_probe, %rax
326 326 INDIRECT_CALL_REG(rax)
327 327 leave /* unwind stack */
328 328 1:
329 329 movq %gs:CPU_THREAD, %rdx /* reload thread ptr */
330 330 decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */
331 331 movl $1, %eax /* return success if tryenter */
332 332 ret
333 333 SET_SIZE(lockstat_wrapper_arg)
334 334
335 335
336 336 ENTRY(mutex_tryenter)
337 337 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
338 338 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
339 339 lock
340 340 cmpxchgq %rdx, (%rdi)
341 341 jnz mutex_vector_tryenter
342 342 not %eax /* return success (nonzero) */
343 343 #if defined(OPTERON_WORKAROUND_6323525)
344 344 .mutex_tryenter_lockstat_patch_point:
345 345 .mutex_tryenter_6323525_patch_point:
346 346 ret /* nop space for lfence */
347 347 nop
348 348 nop
349 349 .mutex_tryenter_lockstat_6323525_patch_point: /* new patch point if lfence */
350 350 nop
351 351 #else /* OPTERON_WORKAROUND_6323525 */
352 352 .mutex_tryenter_lockstat_patch_point:
353 353 ret
354 354 #endif /* OPTERON_WORKAROUND_6323525 */
355 355 movq %rdi, %rsi
356 356 movl $LS_MUTEX_ENTER_ACQUIRE, %edi
357 357 jmp lockstat_wrapper
358 358 SET_SIZE(mutex_tryenter)
359 359
360 360 ENTRY(mutex_adaptive_tryenter)
361 361 movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */
362 362 xorl %eax, %eax /* rax = 0 (unheld adaptive) */
363 363 lock
364 364 cmpxchgq %rdx, (%rdi)
365 365 jnz 0f
366 366 not %eax /* return success (nonzero) */
367 367 #if defined(OPTERON_WORKAROUND_6323525)
368 368 .mutex_atryenter_6323525_patch_point:
369 369 ret /* nop space for lfence */
370 370 nop
371 371 nop
372 372 nop
373 373 #else /* OPTERON_WORKAROUND_6323525 */
374 374 ret
375 375 #endif /* OPTERON_WORKAROUND_6323525 */
376 376 0:
377 377 xorl %eax, %eax /* return failure */
378 378 ret
379 379 SET_SIZE(mutex_adaptive_tryenter)
380 380
381 381 .globl mutex_owner_running_critical_start
382 382
383 383 ENTRY(mutex_owner_running)
384 384 mutex_owner_running_critical_start:
385 385 movq (%rdi), %r11 /* get owner field */
386 386 andq $MUTEX_THREAD, %r11 /* remove waiters bit */
387 387 cmpq $0, %r11 /* if free, skip */
388 388 je 1f /* go return 0 */
389 389 movq T_CPU(%r11), %r8 /* get owner->t_cpu */
390 390 movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */
391 391 .mutex_owner_running_critical_end:
392 392 cmpq %r11, %r9 /* owner == running thread? */
393 393 je 2f /* yes, go return cpu */
394 394 1:
395 395 xorq %rax, %rax /* return 0 */
396 396 ret
397 397 2:
398 398 movq %r8, %rax /* return cpu */
399 399 ret
400 400 SET_SIZE(mutex_owner_running)
401 401
402 402 .globl mutex_owner_running_critical_size
403 403 .type mutex_owner_running_critical_size, @object
404 404 .align CPTRSIZE
405 405 mutex_owner_running_critical_size:
406 406 .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start
407 407 SET_SIZE(mutex_owner_running_critical_size)
408 408
409 409 .globl mutex_exit_critical_start
410 410
411 411 ENTRY(mutex_exit)
412 412 mutex_exit_critical_start: /* If interrupted, restart here */
413 413 movq %gs:CPU_THREAD, %rdx
414 414 cmpq %rdx, (%rdi)
415 415 jne mutex_vector_exit /* wrong type or wrong owner */
416 416 movq $0, (%rdi) /* clear owner AND lock */
417 417 .mutex_exit_critical_end:
418 418 .mutex_exit_lockstat_patch_point:
419 419 ret
420 420 movq %rdi, %rsi
421 421 movl $LS_MUTEX_EXIT_RELEASE, %edi
422 422 jmp lockstat_wrapper
423 423 SET_SIZE(mutex_exit)
424 424
425 425 .globl mutex_exit_critical_size
426 426 .type mutex_exit_critical_size, @object
427 427 .align CPTRSIZE
428 428 mutex_exit_critical_size:
429 429 .quad .mutex_exit_critical_end - mutex_exit_critical_start
430 430 SET_SIZE(mutex_exit_critical_size)
431 431
432 432 /*
433 433 * rw_enter() and rw_exit().
434 434 *
435 435 * These routines handle the simple cases of rw_enter (write-locking an unheld
436 436 * lock or read-locking a lock that's neither write-locked nor write-wanted)
437 437 * and rw_exit (no waiters or not the last reader). If anything complicated
438 438 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
439 439 */
440 440
441 441 ENTRY(rw_enter)
442 442 cmpl $RW_WRITER, %esi
443 443 je .rw_write_enter
444 444 movq (%rdi), %rax /* rax = old rw_wwwh value */
445 445 testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
446 446 jnz rw_enter_sleep
447 447 leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */
448 448 lock
449 449 cmpxchgq %rdx, (%rdi) /* try to grab read lock */
450 450 jnz rw_enter_sleep
451 451 .rw_read_enter_lockstat_patch_point:
452 452 ret
453 453 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
454 454 movq %rdi, %rsi /* rsi = lock ptr */
455 455 movl $LS_RW_ENTER_ACQUIRE, %edi
456 456 movl $RW_READER, %edx
457 457 jmp lockstat_wrapper_arg
458 458 .rw_write_enter:
459 459 movq %gs:CPU_THREAD, %rdx
460 460 orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */
461 461 xorl %eax, %eax /* rax = unheld value */
462 462 lock
463 463 cmpxchgq %rdx, (%rdi) /* try to grab write lock */
464 464 jnz rw_enter_sleep
465 465
466 466 #if defined(OPTERON_WORKAROUND_6323525)
467 467 .rw_write_enter_lockstat_patch_point:
468 468 .rw_write_enter_6323525_patch_point:
469 469 ret
470 470 nop
471 471 nop
472 472 .rw_write_enter_lockstat_6323525_patch_point:
473 473 nop
474 474 #else /* OPTERON_WORKAROUND_6323525 */
475 475 .rw_write_enter_lockstat_patch_point:
476 476 ret
477 477 #endif /* OPTERON_WORKAROUND_6323525 */
478 478
479 479 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
480 480 movq %rdi, %rsi /* rsi = lock ptr */
481 481 movl $LS_RW_ENTER_ACQUIRE, %edi
482 482 movl $RW_WRITER, %edx
483 483 jmp lockstat_wrapper_arg
484 484 SET_SIZE(rw_enter)
485 485
486 486 ENTRY(rw_exit)
|
↓ open down ↓ |
486 lines elided |
↑ open up ↑ |
487 487 movq (%rdi), %rax /* rax = old rw_wwwh value */
488 488 cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */
489 489 jne .rw_not_single_reader
490 490 xorl %edx, %edx /* rdx = new value (unheld) */
491 491 .rw_read_exit:
492 492 lock
493 493 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
494 494 jnz rw_exit_wakeup
495 495 .rw_read_exit_lockstat_patch_point:
496 496 ret
497 + movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
497 498 movq %rdi, %rsi /* rsi = lock ptr */
498 499 movl $LS_RW_EXIT_RELEASE, %edi
499 500 movl $RW_READER, %edx
500 501 jmp lockstat_wrapper_arg
501 502 .rw_not_single_reader:
502 503 testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */
503 504 jnz .rw_write_exit
504 505 leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */
505 506 cmpl $RW_READ_LOCK, %edx
506 507 jge .rw_read_exit /* not last reader, safe to drop */
507 508 jmp rw_exit_wakeup /* last reader with waiters */
508 509 .rw_write_exit:
509 510 movq %gs:CPU_THREAD, %rax /* rax = thread ptr */
510 511 xorl %edx, %edx /* rdx = new value (unheld) */
511 512 orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */
512 513 lock
513 514 cmpxchgq %rdx, (%rdi) /* try to drop read lock */
514 515 jnz rw_exit_wakeup
515 516 .rw_write_exit_lockstat_patch_point:
516 517 ret
517 518 movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */
518 519 movq %rdi, %rsi /* rsi - lock ptr */
519 520 movl $LS_RW_EXIT_RELEASE, %edi
520 521 movl $RW_WRITER, %edx
521 522 jmp lockstat_wrapper_arg
522 523 SET_SIZE(rw_exit)
523 524
524 525 #if defined(OPTERON_WORKAROUND_6323525)
525 526
526 527 /*
527 528 * If it is necessary to patch the lock enter routines with the lfence
528 529 * workaround, workaround_6323525_patched is set to a non-zero value so that
529 530 * the lockstat_hat_patch routine can patch to the new location of the 'ret'
530 531 * instruction.
531 532 */
532 533 DGDEF3(workaround_6323525_patched, 4, 4)
533 534 .long 0
534 535
535 536 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
536 537 movq $size, %rbx; \
537 538 movq $dstaddr, %r13; \
538 539 addq %rbx, %r13; \
539 540 movq $srcaddr, %r12; \
540 541 addq %rbx, %r12; \
541 542 0: \
542 543 decq %r13; \
543 544 decq %r12; \
544 545 movzbl (%r12), %esi; \
545 546 movq $1, %rdx; \
546 547 movq %r13, %rdi; \
547 548 call hot_patch_kernel_text; \
548 549 decq %rbx; \
549 550 testq %rbx, %rbx; \
550 551 jg 0b;
551 552
552 553 /*
553 554 * patch_workaround_6323525: provide workaround for 6323525
554 555 *
555 556 * The workaround is to place a fencing instruction (lfence) between the
556 557 * mutex operation and the subsequent read-modify-write instruction.
557 558 *
558 559 * This routine hot patches the lfence instruction on top of the space
559 560 * reserved by nops in the lock enter routines.
560 561 */
561 562 ENTRY_NP(patch_workaround_6323525)
562 563 pushq %rbp
563 564 movq %rsp, %rbp
564 565 pushq %r12
565 566 pushq %r13
566 567 pushq %rbx
567 568
568 569 /*
569 570 * lockstat_hot_patch() to use the alternate lockstat workaround
570 571 * 6323525 patch points (points past the lfence instruction to the
571 572 * new ret) when workaround_6323525_patched is set.
572 573 */
573 574 movl $1, workaround_6323525_patched
574 575
575 576 /*
576 577 * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter
577 578 * routines. The 4 bytes are patched in reverse order so that the
578 579 * the existing ret is overwritten last. This provides lock enter
579 580 * sanity during the intermediate patching stages.
580 581 */
581 582 HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
582 583 HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
583 584 HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
584 585 HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
585 586
586 587 popq %rbx
587 588 popq %r13
588 589 popq %r12
589 590 movq %rbp, %rsp
590 591 popq %rbp
591 592 ret
592 593 _lfence_insn:
593 594 lfence
594 595 ret
595 596 SET_SIZE(patch_workaround_6323525)
596 597
597 598
598 599 #endif /* OPTERON_WORKAROUND_6323525 */
599 600
600 601
601 602 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
602 603 movq $normal_instr, %rsi; \
603 604 movq $active_instr, %rdi; \
604 605 leaq lockstat_probemap(%rip), %rax; \
605 606 movl _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
606 607 testl %eax, %eax; \
607 608 jz 9f; \
608 609 movq %rdi, %rsi; \
609 610 9: \
610 611 movq $len, %rdx; \
611 612 movq $addr, %rdi; \
612 613 call hot_patch_kernel_text
613 614
614 615 ENTRY(lockstat_hot_patch)
615 616 pushq %rbp /* align stack properly */
616 617 movq %rsp, %rbp
617 618
618 619 #if defined(OPTERON_WORKAROUND_6323525)
619 620 cmpl $0, workaround_6323525_patched
620 621 je 1f
621 622 HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
622 623 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
623 624 HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
624 625 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
625 626 HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
626 627 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
627 628 jmp 2f
628 629 1:
629 630 HOT_PATCH(.mutex_enter_lockstat_patch_point,
630 631 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
631 632 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
632 633 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
633 634 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
634 635 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
635 636 2:
636 637 #else /* OPTERON_WORKAROUND_6323525 */
637 638 HOT_PATCH(.mutex_enter_lockstat_patch_point,
638 639 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
639 640 HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
640 641 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
641 642 HOT_PATCH(.rw_write_enter_lockstat_patch_point,
642 643 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
643 644 #endif /* !OPTERON_WORKAROUND_6323525 */
644 645 HOT_PATCH(.mutex_exit_lockstat_patch_point,
645 646 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
646 647 HOT_PATCH(.rw_read_enter_lockstat_patch_point,
647 648 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
648 649 HOT_PATCH(.rw_write_exit_lockstat_patch_point,
649 650 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
650 651 HOT_PATCH(.rw_read_exit_lockstat_patch_point,
651 652 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
652 653 HOT_PATCH(.lock_set_lockstat_patch_point,
653 654 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
654 655 HOT_PATCH(.lock_try_lockstat_patch_point,
655 656 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
656 657 HOT_PATCH(.lock_clear_lockstat_patch_point,
657 658 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
658 659 HOT_PATCH(.lock_set_spl_lockstat_patch_point,
659 660 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
660 661
661 662 HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
662 663 LS_LOCK_CLEAR_SPLX_RELEASE,
663 664 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
664 665 leave /* unwind stack */
665 666 ret
666 667 SET_SIZE(lockstat_hot_patch)
667 668
668 669 ENTRY(membar_enter)
669 670 ALTENTRY(membar_exit)
670 671 ALTENTRY(membar_sync)
671 672 mfence /* lighter weight than lock; xorq $0,(%rsp) */
672 673 ret
673 674 SET_SIZE(membar_sync)
674 675 SET_SIZE(membar_exit)
675 676 SET_SIZE(membar_enter)
676 677
677 678 ENTRY(membar_producer)
678 679 sfence
679 680 ret
680 681 SET_SIZE(membar_producer)
681 682
682 683 ENTRY(membar_consumer)
683 684 lfence
684 685 ret
685 686 SET_SIZE(membar_consumer)
686 687
687 688 /*
688 689 * thread_onproc()
689 690 * Set thread in onproc state for the specified CPU.
690 691 * Also set the thread lock pointer to the CPU's onproc lock.
691 692 * Since the new lock isn't held, the store ordering is important.
692 693 * If not done in assembler, the compiler could reorder the stores.
693 694 */
694 695
695 696 ENTRY(thread_onproc)
696 697 addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */
697 698 movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */
698 699 movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */
699 700 ret
700 701 SET_SIZE(thread_onproc)
701 702
702 703 /*
703 704 * mutex_delay_default(void)
704 705 * Spins for approx a few hundred processor cycles and returns to caller.
705 706 */
706 707
707 708 ENTRY(mutex_delay_default)
708 709 movq $92,%r11
709 710 0: decq %r11
710 711 jg 0b
711 712 ret
712 713 SET_SIZE(mutex_delay_default)
713 714
|
↓ open down ↓ |
207 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX