Print this page
11909 THREAD_KPRI_RELEASE does nothing of the sort
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>

Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/os/rwlock.c
          +++ new/usr/src/uts/common/os/rwlock.c
↓ open down ↓ 16 lines elided ↑ open up ↑
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  /*
  22   22   * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23   23   * Use is subject to license terms.
  24   24   */
  25   25  
  26   26  /*
  27      - * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
       27 + * Copyright 2019 Joyent, Inc.
  28   28   */
  29   29  
  30   30  #include <sys/param.h>
  31   31  #include <sys/thread.h>
  32   32  #include <sys/cmn_err.h>
  33   33  #include <sys/debug.h>
  34   34  #include <sys/cpuvar.h>
  35   35  #include <sys/sobject.h>
  36   36  #include <sys/turnstile.h>
  37   37  #include <sys/rwlock.h>
↓ open down ↓ 224 lines elided ↑ open up ↑
 262  262  
 263  263          return (0);
 264  264  }
 265  265  
 266  266  uint_t (*rw_lock_backoff)(uint_t) = NULL;
 267  267  void (*rw_lock_delay)(uint_t) = NULL;
 268  268  
 269  269  /*
 270  270   * Full-service implementation of rw_enter() to handle all the hard cases.
 271  271   * Called from the assembly version if anything complicated is going on.
 272      - * The only semantic difference between calling rw_enter() and calling
 273      - * rw_enter_sleep() directly is that we assume the caller has already done
 274      - * a THREAD_KPRI_REQUEST() in the RW_READER cases.
 275  272   */
 276  273  void
 277  274  rw_enter_sleep(rwlock_impl_t *lp, krw_t rw)
 278  275  {
 279  276          uintptr_t old, new, lock_value, lock_busy, lock_wait;
 280  277          hrtime_t sleep_time;
 281  278          turnstile_t *ts;
 282  279          uint_t  backoff = 0;
 283  280          int loop_count = 0;
 284  281  
↓ open down ↓ 50 lines elided ↑ open up ↑
 335  332  
 336  333                  if ((old & lock_busy) == 0) {
 337  334                          /*
 338  335                           * The lock appears free now; try the dance again
 339  336                           */
 340  337                          turnstile_exit(lp);
 341  338                          continue;
 342  339                  }
 343  340  
 344  341                  /*
 345      -                 * We really are going to block.  Bump the stats, and drop
 346      -                 * kpri if we're a reader.
      342 +                 * We really are going to block, so bump the stats.
 347  343                   */
 348  344                  ASSERT(lp->rw_wwwh & lock_wait);
 349  345                  ASSERT(lp->rw_wwwh & RW_LOCKED);
 350  346  
 351  347                  sleep_time = -gethrtime();
 352  348                  if (rw != RW_WRITER) {
 353      -                        THREAD_KPRI_RELEASE();
 354  349                          CPU_STATS_ADDQ(CPU, sys, rw_rdfails, 1);
 355  350                          (void) turnstile_block(ts, TS_READER_Q, lp,
 356  351                              &rw_sobj_ops, NULL, NULL);
 357  352                  } else {
 358  353                          CPU_STATS_ADDQ(CPU, sys, rw_wrfails, 1);
 359  354                          (void) turnstile_block(ts, TS_WRITER_Q, lp,
 360  355                              &rw_sobj_ops, NULL, NULL);
 361  356                  }
 362  357                  sleep_time += gethrtime();
 363  358  
 364  359                  LOCKSTAT_RECORD4(LS_RW_ENTER_BLOCK, lp, sleep_time, rw,
 365  360                      (old & RW_WRITE_LOCKED) ? 1 : 0,
 366  361                      old >> RW_HOLD_COUNT_SHIFT);
 367  362  
 368  363                  /*
 369      -                 * We wake up holding the lock (and having kpri if we're
 370      -                 * a reader) via direct handoff from the previous owner.
      364 +                 * We wake up holding the lock via direct handoff from the
      365 +                 * previous owner.
 371  366                   */
 372  367                  break;
 373  368          }
 374  369  
 375  370          ASSERT(rw_locked(lp, rw));
 376  371  
 377  372          membar_enter();
 378  373  
 379  374          LOCKSTAT_RECORD(LS_RW_ENTER_ACQUIRE, lp, rw);
 380  375  }
↓ open down ↓ 6 lines elided ↑ open up ↑
 387  382  rw_readers_to_wake(turnstile_t *ts)
 388  383  {
 389  384          kthread_t *next_writer = ts->ts_sleepq[TS_WRITER_Q].sq_first;
 390  385          kthread_t *next_reader = ts->ts_sleepq[TS_READER_Q].sq_first;
 391  386          pri_t wpri = (next_writer != NULL) ? DISP_PRIO(next_writer) : -1;
 392  387          int count = 0;
 393  388  
 394  389          while (next_reader != NULL) {
 395  390                  if (DISP_PRIO(next_reader) < wpri)
 396  391                          break;
 397      -                next_reader->t_kpri_req++;
 398  392                  next_reader = next_reader->t_link;
 399  393                  count++;
 400  394          }
 401  395          return (count);
 402  396  }
 403  397  
 404  398  /*
 405  399   * Full-service implementation of rw_exit() to handle all the hard cases.
 406  400   * Called from the assembly version if anything complicated is going on.
 407  401   * There is no semantic difference between calling rw_exit() and calling
↓ open down ↓ 108 lines elided ↑ open up ↑
 516  510                          if (next_writer->t_link)
 517  511                                  new |= RW_WRITE_WANTED;
 518  512                          lp->rw_wwwh = new;
 519  513                          membar_enter();
 520  514                          turnstile_wakeup(ts, TS_WRITER_Q, 1, next_writer);
 521  515                  }
 522  516                  break;
 523  517          }
 524  518  
 525  519          if (lock_value == RW_READ_LOCK) {
 526      -                THREAD_KPRI_RELEASE();
 527  520                  LOCKSTAT_RECORD(LS_RW_EXIT_RELEASE, lp, RW_READER);
 528  521          } else {
 529  522                  LOCKSTAT_RECORD(LS_RW_EXIT_RELEASE, lp, RW_WRITER);
 530  523          }
 531  524  }
 532  525  
 533  526  int
 534  527  rw_tryenter(krwlock_t *rwlp, krw_t rw)
 535  528  {
 536  529          rwlock_impl_t *lp = (rwlock_impl_t *)rwlp;
 537  530          uintptr_t old;
 538  531  
 539  532          if (rw != RW_WRITER) {
 540  533                  uint_t backoff = 0;
 541  534                  int loop_count = 0;
 542      -                THREAD_KPRI_REQUEST();
 543  535                  for (;;) {
 544  536                          if ((old = lp->rw_wwwh) & (rw == RW_READER ?
 545  537                              RW_WRITE_CLAIMED : RW_WRITE_LOCKED)) {
 546      -                                THREAD_KPRI_RELEASE();
 547  538                                  return (0);
 548  539                          }
 549  540                          if (casip(&lp->rw_wwwh, old, old + RW_READ_LOCK) == old)
 550  541                                  break;
 551  542                          if (rw_lock_delay != NULL) {
 552  543                                  backoff = rw_lock_backoff(backoff);
 553  544                                  rw_lock_delay(backoff);
 554  545                                  if (++loop_count == ncpus_online) {
 555  546                                          backoff = 0;
 556  547                                          loop_count = 0;
↓ open down ↓ 9 lines elided ↑ open up ↑
 566  557          ASSERT(rw_locked(lp, rw));
 567  558          membar_enter();
 568  559          return (1);
 569  560  }
 570  561  
 571  562  void
 572  563  rw_downgrade(krwlock_t *rwlp)
 573  564  {
 574  565          rwlock_impl_t *lp = (rwlock_impl_t *)rwlp;
 575  566  
 576      -        THREAD_KPRI_REQUEST();
 577  567          membar_exit();
 578  568  
 579  569          if ((lp->rw_wwwh & RW_OWNER) != (uintptr_t)curthread) {
 580  570                  rw_panic("rw_downgrade: not owner", lp);
 581  571                  return;
 582  572          }
 583  573  
 584  574          if (atomic_add_ip_nv(&lp->rw_wwwh,
 585  575              RW_READ_LOCK - RW_WRITE_LOCK(curthread)) & RW_HAS_WAITERS) {
 586  576                  turnstile_t *ts = turnstile_lookup(lp);
↓ open down ↓ 18 lines elided ↑ open up ↑
 605  595  
 606  596          ASSERT(rw_locked(lp, RW_READER));
 607  597  
 608  598          do {
 609  599                  if (((old = lp->rw_wwwh) & ~RW_HAS_WAITERS) != RW_READ_LOCK)
 610  600                          return (0);
 611  601                  new = old + RW_WRITE_LOCK(curthread) - RW_READ_LOCK;
 612  602          } while (casip(&lp->rw_wwwh, old, new) != old);
 613  603  
 614  604          membar_enter();
 615      -        THREAD_KPRI_RELEASE();
 616  605          LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, lp);
 617  606          ASSERT(rw_locked(lp, RW_WRITER));
 618  607          return (1);
 619  608  }
 620  609  
 621  610  int
 622  611  rw_read_held(krwlock_t *rwlp)
 623  612  {
 624  613          uintptr_t tmp;
 625  614  
↓ open down ↓ 43 lines elided ↑ open up ↑
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX