1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * hermon_qp.c
  28  *    Hermon Queue Pair Processing Routines
  29  *
  30  *    Implements all the routines necessary for allocating, freeing, and
  31  *    querying the Hermon queue pairs.
  32  */
  33 
  34 #include <sys/types.h>
  35 #include <sys/conf.h>
  36 #include <sys/ddi.h>
  37 #include <sys/sunddi.h>
  38 #include <sys/modctl.h>
  39 #include <sys/bitmap.h>
  40 #include <sys/sysmacros.h>
  41 
  42 #include <sys/ib/adapters/hermon/hermon.h>
  43 #include <sys/ib/ib_pkt_hdrs.h>
  44 
  45 static int hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
  46     hermon_rsrc_t *qpc);
  47 static int hermon_qpn_avl_compare(const void *q, const void *e);
  48 static int hermon_special_qp_rsrc_alloc(hermon_state_t *state,
  49     ibt_sqp_type_t type, uint_t port, hermon_rsrc_t **qp_rsrc);
  50 static int hermon_special_qp_rsrc_free(hermon_state_t *state,
  51     ibt_sqp_type_t type, uint_t port);
  52 static void hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
  53     uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
  54     uint_t *logwqesz, uint_t *max_sgl);
  55 
  56 /*
  57  * hermon_qp_alloc()
  58  *    Context: Can be called only from user or kernel context.
  59  */
  60 int
  61 hermon_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
  62     uint_t sleepflag)
  63 {
  64         hermon_rsrc_t                   *qpc, *rsrc;
  65         hermon_rsrc_type_t              rsrc_type;
  66         hermon_umap_db_entry_t          *umapdb;
  67         hermon_qphdl_t                  qp;
  68         ibt_qp_alloc_attr_t             *attr_p;
  69         ibt_qp_alloc_flags_t            alloc_flags;
  70         ibt_qp_type_t                   type;
  71         hermon_qp_wq_type_t             swq_type;
  72         ibtl_qp_hdl_t                   ibt_qphdl;
  73         ibt_chan_sizes_t                *queuesz_p;
  74         ib_qpn_t                        *qpn;
  75         hermon_qphdl_t                  *qphdl;
  76         ibt_mr_attr_t                   mr_attr;
  77         hermon_mr_options_t             mr_op;
  78         hermon_srqhdl_t                 srq;
  79         hermon_pdhdl_t                  pd;
  80         hermon_cqhdl_t                  sq_cq, rq_cq;
  81         hermon_mrhdl_t                  mr;
  82         uint64_t                        value, qp_desc_off;
  83         uint64_t                        *thewqe, thewqesz;
  84         uint32_t                        *sq_buf, *rq_buf;
  85         uint32_t                        log_qp_sq_size, log_qp_rq_size;
  86         uint32_t                        sq_size, rq_size;
  87         uint32_t                        sq_depth, rq_depth;
  88         uint32_t                        sq_wqe_size, rq_wqe_size, wqesz_shift;
  89         uint32_t                        max_sgl, max_recv_sgl, uarpg;
  90         uint_t                          qp_is_umap;
  91         uint_t                          qp_srq_en, i, j;
  92         int                             status, flag;
  93 
  94         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
  95 
  96         /*
  97          * Extract the necessary info from the hermon_qp_info_t structure
  98          */
  99         attr_p    = qpinfo->qpi_attrp;
 100         type      = qpinfo->qpi_type;
 101         ibt_qphdl = qpinfo->qpi_ibt_qphdl;
 102         queuesz_p = qpinfo->qpi_queueszp;
 103         qpn       = qpinfo->qpi_qpn;
 104         qphdl     = &qpinfo->qpi_qphdl;
 105         alloc_flags = attr_p->qp_alloc_flags;
 106 
 107         /*
 108          * Verify correctness of alloc_flags.
 109          *
 110          * 1. FEXCH and RSS are only allocated via qp_range.
 111          */
 112         if (alloc_flags & (IBT_QP_USES_FEXCH | IBT_QP_USES_RSS)) {
 113                 return (IBT_INVALID_PARAM);
 114         }
 115         rsrc_type = HERMON_QPC;
 116         qp_is_umap = 0;
 117 
 118         /* 2. Make sure only one of these flags is set. */
 119         switch (alloc_flags &
 120             (IBT_QP_USER_MAP | IBT_QP_USES_RFCI | IBT_QP_USES_FCMD)) {
 121         case IBT_QP_USER_MAP:
 122                 qp_is_umap = 1;
 123                 break;
 124         case IBT_QP_USES_RFCI:
 125                 if (type != IBT_UD_RQP)
 126                         return (IBT_INVALID_PARAM);
 127 
 128                 switch (attr_p->qp_fc.fc_hca_port) {
 129                 case 1:
 130                         rsrc_type = HERMON_QPC_RFCI_PORT1;
 131                         break;
 132                 case 2:
 133                         rsrc_type = HERMON_QPC_RFCI_PORT2;
 134                         break;
 135                 default:
 136                         return (IBT_INVALID_PARAM);
 137                 }
 138                 break;
 139         case IBT_QP_USES_FCMD:
 140                 if (type != IBT_UD_RQP)
 141                         return (IBT_INVALID_PARAM);
 142                 break;
 143         case 0:
 144                 break;
 145         default:
 146                 return (IBT_INVALID_PARAM);     /* conflicting flags set */
 147         }
 148 
 149         /*
 150          * Determine whether QP is being allocated for userland access or
 151          * whether it is being allocated for kernel access.  If the QP is
 152          * being allocated for userland access, then lookup the UAR
 153          * page number for the current process.  Note:  If this is not found
 154          * (e.g. if the process has not previously open()'d the Hermon driver),
 155          * then an error is returned.
 156          */
 157         if (qp_is_umap) {
 158                 status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
 159                     MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
 160                 if (status != DDI_SUCCESS) {
 161                         return (IBT_INVALID_PARAM);
 162                 }
 163                 uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
 164         } else {
 165                 uarpg = state->hs_kernel_uar_index;
 166         }
 167 
 168         /*
 169          * Determine whether QP is being associated with an SRQ
 170          */
 171         qp_srq_en = (alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
 172         if (qp_srq_en) {
 173                 /*
 174                  * Check for valid SRQ handle pointers
 175                  */
 176                 if (attr_p->qp_ibc_srq_hdl == NULL) {
 177                         status = IBT_SRQ_HDL_INVALID;
 178                         goto qpalloc_fail;
 179                 }
 180                 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
 181         }
 182 
 183         /*
 184          * Check for valid QP service type (only UD/RC/UC supported)
 185          */
 186         if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) &&
 187             (type != IBT_UC_RQP))) {
 188                 status = IBT_QP_SRV_TYPE_INVALID;
 189                 goto qpalloc_fail;
 190         }
 191 
 192 
 193         /*
 194          * Check for valid PD handle pointer
 195          */
 196         if (attr_p->qp_pd_hdl == NULL) {
 197                 status = IBT_PD_HDL_INVALID;
 198                 goto qpalloc_fail;
 199         }
 200         pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
 201 
 202         /*
 203          * If on an SRQ, check to make sure the PD is the same
 204          */
 205         if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
 206                 status = IBT_PD_HDL_INVALID;
 207                 goto qpalloc_fail;
 208         }
 209 
 210         /* Increment the reference count on the protection domain (PD) */
 211         hermon_pd_refcnt_inc(pd);
 212 
 213         /*
 214          * Check for valid CQ handle pointers
 215          *
 216          * FCMD QPs do not require a receive cq handle.
 217          */
 218         if (attr_p->qp_ibc_scq_hdl == NULL) {
 219                 status = IBT_CQ_HDL_INVALID;
 220                 goto qpalloc_fail1;
 221         }
 222         sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
 223         if ((attr_p->qp_ibc_rcq_hdl == NULL)) {
 224                 if ((alloc_flags & IBT_QP_USES_FCMD) == 0) {
 225                         status = IBT_CQ_HDL_INVALID;
 226                         goto qpalloc_fail1;
 227                 }
 228                 rq_cq = sq_cq;  /* just use the send cq */
 229         } else
 230                 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
 231 
 232         /*
 233          * Increment the reference count on the CQs.  One or both of these
 234          * could return error if we determine that the given CQ is already
 235          * being used with a special (SMI/GSI) QP.
 236          */
 237         status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
 238         if (status != DDI_SUCCESS) {
 239                 status = IBT_CQ_HDL_INVALID;
 240                 goto qpalloc_fail1;
 241         }
 242         status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
 243         if (status != DDI_SUCCESS) {
 244                 status = IBT_CQ_HDL_INVALID;
 245                 goto qpalloc_fail2;
 246         }
 247 
 248         /*
 249          * Allocate an QP context entry.  This will be filled in with all
 250          * the necessary parameters to define the Queue Pair.  Unlike
 251          * other Hermon hardware resources, ownership is not immediately
 252          * given to hardware in the final step here.  Instead, we must
 253          * wait until the QP is later transitioned to the "Init" state before
 254          * passing the QP to hardware.  If we fail here, we must undo all
 255          * the reference count (CQ and PD).
 256          */
 257         status = hermon_rsrc_alloc(state, rsrc_type, 1, sleepflag, &qpc);
 258         if (status != DDI_SUCCESS) {
 259                 status = IBT_INSUFF_RESOURCE;
 260                 goto qpalloc_fail3;
 261         }
 262 
 263         /*
 264          * Allocate the software structure for tracking the queue pair
 265          * (i.e. the Hermon Queue Pair handle).  If we fail here, we must
 266          * undo the reference counts and the previous resource allocation.
 267          */
 268         status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
 269         if (status != DDI_SUCCESS) {
 270                 status = IBT_INSUFF_RESOURCE;
 271                 goto qpalloc_fail4;
 272         }
 273         qp = (hermon_qphdl_t)rsrc->hr_addr;
 274         bzero(qp, sizeof (struct hermon_sw_qp_s));
 275         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
 276 
 277         qp->qp_alloc_flags = alloc_flags;
 278 
 279         /*
 280          * Calculate the QP number from QPC index.  This routine handles
 281          * all of the operations necessary to keep track of used, unused,
 282          * and released QP numbers.
 283          */
 284         if (type == IBT_UD_RQP) {
 285                 qp->qp_qpnum = qpc->hr_indx;
 286                 qp->qp_ring = qp->qp_qpnum << 8;
 287                 qp->qp_qpn_hdl = NULL;
 288         } else {
 289                 status = hermon_qp_create_qpn(state, qp, qpc);
 290                 if (status != DDI_SUCCESS) {
 291                         status = IBT_INSUFF_RESOURCE;
 292                         goto qpalloc_fail5;
 293                 }
 294         }
 295 
 296         /*
 297          * If this will be a user-mappable QP, then allocate an entry for
 298          * the "userland resources database".  This will later be added to
 299          * the database (after all further QP operations are successful).
 300          * If we fail here, we must undo the reference counts and the
 301          * previous resource allocation.
 302          */
 303         if (qp_is_umap) {
 304                 umapdb = hermon_umap_db_alloc(state->hs_instance, qp->qp_qpnum,
 305                     MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
 306                 if (umapdb == NULL) {
 307                         status = IBT_INSUFF_RESOURCE;
 308                         goto qpalloc_fail6;
 309                 }
 310         }
 311 
 312         /*
 313          * Allocate the doorbell record.  Hermon just needs one for the RQ,
 314          * if the QP is not associated with an SRQ, and use uarpg (above) as
 315          * the uar index
 316          */
 317 
 318         if (!qp_srq_en) {
 319                 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
 320                     &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
 321                 if (status != DDI_SUCCESS) {
 322                         status = IBT_INSUFF_RESOURCE;
 323                         goto qpalloc_fail6;
 324                 }
 325         }
 326 
 327         qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
 328 
 329         /*
 330          * We verify that the requested number of SGL is valid (i.e.
 331          * consistent with the device limits and/or software-configured
 332          * limits).  If not, then obviously the same cleanup needs to be done.
 333          */
 334         if (type == IBT_UD_RQP) {
 335                 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
 336                 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
 337         } else {
 338                 max_sgl = state->hs_ibtfinfo.hca_attr->hca_conn_send_sgl_sz;
 339                 swq_type = HERMON_QP_WQ_TYPE_SENDQ_CONN;
 340         }
 341         max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
 342         if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
 343             (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
 344                 status = IBT_HCA_SGL_EXCEEDED;
 345                 goto qpalloc_fail7;
 346         }
 347 
 348         /*
 349          * Determine this QP's WQE stride (for both the Send and Recv WQEs).
 350          * This will depend on the requested number of SGLs.  Note: this
 351          * has the side-effect of also calculating the real number of SGLs
 352          * (for the calculated WQE size).
 353          *
 354          * For QP's on an SRQ, we set these to 0.
 355          */
 356         if (qp_srq_en) {
 357                 qp->qp_rq_log_wqesz = 0;
 358                 qp->qp_rq_sgl = 0;
 359         } else {
 360                 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
 361                     max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
 362                     &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
 363         }
 364         hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
 365             max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
 366 
 367         sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
 368 
 369         /* NOTE: currently policy in driver, later maybe IBTF interface */
 370         qp->qp_no_prefetch = 0;
 371 
 372         /*
 373          * for prefetching, we need to add the number of wqes in
 374          * the 2k area plus one to the number requested, but
 375          * ONLY for send queue.  If no_prefetch == 1 (prefetch off)
 376          * it's exactly TWO wqes for the headroom
 377          */
 378         if (qp->qp_no_prefetch)
 379                 qp->qp_sq_headroom = 2 * sq_wqe_size;
 380         else
 381                 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
 382         /*
 383          * hdrm wqes must be integral since both sq_wqe_size &
 384          * HERMON_QP_OH_SIZE are power of 2
 385          */
 386         qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
 387 
 388 
 389         /*
 390          * Calculate the appropriate size for the work queues.
 391          * For send queue, add in the headroom wqes to the calculation.
 392          * Note:  All Hermon QP work queues must be a power-of-2 in size.  Also
 393          * they may not be any smaller than HERMON_QP_MIN_SIZE.  This step is
 394          * to round the requested size up to the next highest power-of-2
 395          */
 396         /* first, adjust to a minimum and tell the caller the change */
 397         attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
 398             HERMON_QP_MIN_SIZE);
 399         attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
 400             HERMON_QP_MIN_SIZE);
 401         /*
 402          * now, calculate the alloc size, taking into account
 403          * the headroom for the sq
 404          */
 405         log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
 406         /* if the total is a power of two, reduce it */
 407         if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes))    {
 408                 log_qp_sq_size = log_qp_sq_size - 1;
 409         }
 410 
 411         log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
 412         if (ISP2(attr_p->qp_sizes.cs_rq)) {
 413                 log_qp_rq_size = log_qp_rq_size - 1;
 414         }
 415 
 416         /*
 417          * Next we verify that the rounded-up size is valid (i.e. consistent
 418          * with the device limits and/or software-configured limits).  If not,
 419          * then obviously we have a lot of cleanup to do before returning.
 420          *
 421          * NOTE: the first condition deals with the (test) case of cs_sq
 422          * being just less than 2^32.  In this case, the headroom addition
 423          * to the requested cs_sq will pass the test when it should not.
 424          * This test no longer lets that case slip through the check.
 425          */
 426         if ((attr_p->qp_sizes.cs_sq >
 427             (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
 428             (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
 429             (!qp_srq_en && (log_qp_rq_size >
 430             state->hs_cfg_profile->cp_log_max_qp_sz))) {
 431                 status = IBT_HCA_WR_EXCEEDED;
 432                 goto qpalloc_fail7;
 433         }
 434 
 435         /*
 436          * Allocate the memory for QP work queues. Since Hermon work queues
 437          * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
 438          * the work queue memory is very important.  We used to allocate
 439          * work queues (the combined receive and send queues) so that they
 440          * would be aligned on their combined size.  That alignment guaranteed
 441          * that they would never cross the 4GB boundary (Hermon work queues
 442          * are on the order of MBs at maximum).  Now we are able to relax
 443          * this alignment constraint by ensuring that the IB address assigned
 444          * to the queue memory (as a result of the hermon_mr_register() call)
 445          * is offset from zero.
 446          * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
 447          * guarantee the alignment, but when attempting to use IOMMU bypass
 448          * mode we found that we were not allowed to specify any alignment
 449          * that was more restrictive than the system page size.
 450          * So we avoided this constraint by passing two alignment values,
 451          * one for the memory allocation itself and the other for the DMA
 452          * handle (for later bind).  This used to cause more memory than
 453          * necessary to be allocated (in order to guarantee the more
 454          * restrictive alignment contraint).  But by guaranteeing the
 455          * zero-based IB virtual address for the queue, we are able to
 456          * conserve this memory.
 457          */
 458         sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
 459         sq_depth    = 1 << log_qp_sq_size;
 460         sq_size     = sq_depth * sq_wqe_size;
 461 
 462         /* QP on SRQ sets these to 0 */
 463         if (qp_srq_en) {
 464                 rq_wqe_size = 0;
 465                 rq_size     = 0;
 466         } else {
 467                 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
 468                 rq_depth    = 1 << log_qp_rq_size;
 469                 rq_size     = rq_depth * rq_wqe_size;
 470         }
 471 
 472         qp->qp_wqinfo.qa_size = sq_size + rq_size;
 473 
 474         qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
 475         qp->qp_wqinfo.qa_bind_align  = PAGESIZE;
 476 
 477         if (qp_is_umap) {
 478                 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
 479         } else {
 480                 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
 481         }
 482         status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
 483         if (status != DDI_SUCCESS) {
 484                 status = IBT_INSUFF_RESOURCE;
 485                 goto qpalloc_fail7;
 486         }
 487 
 488         /*
 489          * Sort WQs in memory according to stride (*q_wqe_size), largest first
 490          * If they are equal, still put the SQ first
 491          */
 492         qp->qp_sq_baseaddr = 0;
 493         qp->qp_rq_baseaddr = 0;
 494         if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
 495                 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
 496 
 497                 /* if this QP is on an SRQ, set the rq_buf to NULL */
 498                 if (qp_srq_en) {
 499                         rq_buf = NULL;
 500                 } else {
 501                         rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
 502                         qp->qp_rq_baseaddr = sq_size;
 503                 }
 504         } else {
 505                 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
 506                 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
 507                 qp->qp_sq_baseaddr = rq_size;
 508         }
 509 
 510         if (qp_is_umap == 0) {
 511                 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
 512                 if (qp->qp_sq_wqhdr == NULL) {
 513                         status = IBT_INSUFF_RESOURCE;
 514                         goto qpalloc_fail8;
 515                 }
 516                 if (qp_srq_en) {
 517                         qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
 518                         qp->qp_rq_wqavl.wqa_srq_en = 1;
 519                         qp->qp_rq_wqavl.wqa_srq = srq;
 520                 } else {
 521                         qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
 522                         if (qp->qp_rq_wqhdr == NULL) {
 523                                 status = IBT_INSUFF_RESOURCE;
 524                                 goto qpalloc_fail8;
 525                         }
 526                         qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
 527                 }
 528                 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
 529                 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
 530                 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
 531                 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
 532                 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
 533         }
 534 
 535         /*
 536          * Register the memory for the QP work queues.  The memory for the
 537          * QP must be registered in the Hermon cMPT tables.  This gives us the
 538          * LKey to specify in the QP context later.  Note: The memory for
 539          * Hermon work queues (both Send and Recv) must be contiguous and
 540          * registered as a single memory region.  Note: If the QP memory is
 541          * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
 542          * meet the alignment restriction, we pass the "mro_bind_override_addr"
 543          * flag in the call to hermon_mr_register(). This guarantees that the
 544          * resulting IB vaddr will be zero-based (modulo the offset into the
 545          * first page). If we fail here, we still have the bunch of resource
 546          * and reference count cleanup to do.
 547          */
 548         flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
 549             IBT_MR_NOSLEEP;
 550         mr_attr.mr_vaddr    = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
 551         mr_attr.mr_len      = qp->qp_wqinfo.qa_size;
 552         mr_attr.mr_as       = NULL;
 553         mr_attr.mr_flags    = flag;
 554         if (qp_is_umap) {
 555                 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
 556         } else {
 557                 /* HERMON_QUEUE_LOCATION_NORMAL */
 558                 mr_op.mro_bind_type =
 559                     state->hs_cfg_profile->cp_iommu_bypass;
 560         }
 561         mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
 562         mr_op.mro_bind_override_addr = 1;
 563         status = hermon_mr_register(state, pd, &mr_attr, &mr,
 564             &mr_op, HERMON_QP_CMPT);
 565         if (status != DDI_SUCCESS) {
 566                 status = IBT_INSUFF_RESOURCE;
 567                 goto qpalloc_fail9;
 568         }
 569 
 570         /*
 571          * Calculate the offset between the kernel virtual address space
 572          * and the IB virtual address space.  This will be used when
 573          * posting work requests to properly initialize each WQE.
 574          */
 575         qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
 576             (uint64_t)mr->mr_bindinfo.bi_addr;
 577 
 578         /*
 579          * Fill in all the return arguments (if necessary).  This includes
 580          * real work queue sizes (in wqes), real SGLs, and QP number
 581          */
 582         if (queuesz_p != NULL) {
 583                 queuesz_p->cs_sq     =
 584                     (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
 585                 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
 586 
 587                 /* if this QP is on an SRQ, set these to 0 */
 588                 if (qp_srq_en) {
 589                         queuesz_p->cs_rq     = 0;
 590                         queuesz_p->cs_rq_sgl = 0;
 591                 } else {
 592                         queuesz_p->cs_rq     = (1 << log_qp_rq_size);
 593                         queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
 594                 }
 595         }
 596         if (qpn != NULL) {
 597                 *qpn = (ib_qpn_t)qp->qp_qpnum;
 598         }
 599 
 600         /*
 601          * Fill in the rest of the Hermon Queue Pair handle.
 602          */
 603         qp->qp_qpcrsrcp              = qpc;
 604         qp->qp_rsrcp         = rsrc;
 605         qp->qp_state         = HERMON_QP_RESET;
 606         HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
 607         qp->qp_pdhdl         = pd;
 608         qp->qp_mrhdl         = mr;
 609         qp->qp_sq_sigtype    = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
 610             HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
 611         qp->qp_is_special    = 0;
 612         qp->qp_uarpg         = uarpg;
 613         qp->qp_umap_dhp              = (devmap_cookie_t)NULL;
 614         qp->qp_sq_cqhdl              = sq_cq;
 615         qp->qp_sq_bufsz              = (1 << log_qp_sq_size);
 616         qp->qp_sq_logqsz     = log_qp_sq_size;
 617         qp->qp_sq_buf                = sq_buf;
 618         qp->qp_desc_off              = qp_desc_off;
 619         qp->qp_rq_cqhdl              = rq_cq;
 620         qp->qp_rq_buf                = rq_buf;
 621         qp->qp_rlky          = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
 622             0;
 623 
 624         /* if this QP is on an SRQ, set rq_bufsz to 0 */
 625         if (qp_srq_en) {
 626                 qp->qp_rq_bufsz              = 0;
 627                 qp->qp_rq_logqsz     = 0;
 628         } else {
 629                 qp->qp_rq_bufsz              = (1 << log_qp_rq_size);
 630                 qp->qp_rq_logqsz     = log_qp_rq_size;
 631         }
 632 
 633         qp->qp_forward_sqd_event  = 0;
 634         qp->qp_sqd_still_draining = 0;
 635         qp->qp_hdlrarg               = (void *)ibt_qphdl;
 636         qp->qp_mcg_refcnt    = 0;
 637 
 638         /*
 639          * If this QP is to be associated with an SRQ, set the SRQ handle
 640          */
 641         if (qp_srq_en) {
 642                 qp->qp_srqhdl = srq;
 643                 hermon_srq_refcnt_inc(qp->qp_srqhdl);
 644         } else {
 645                 qp->qp_srqhdl = NULL;
 646         }
 647 
 648         /* Determine the QP service type */
 649         qp->qp_type = type;
 650         if (type == IBT_RC_RQP) {
 651                 qp->qp_serv_type = HERMON_QP_RC;
 652         } else if (type == IBT_UD_RQP) {
 653                 if (alloc_flags & IBT_QP_USES_RFCI)
 654                         qp->qp_serv_type = HERMON_QP_RFCI;
 655                 else if (alloc_flags & IBT_QP_USES_FCMD)
 656                         qp->qp_serv_type = HERMON_QP_FCMND;
 657                 else
 658                         qp->qp_serv_type = HERMON_QP_UD;
 659         } else {
 660                 qp->qp_serv_type = HERMON_QP_UC;
 661         }
 662 
 663         /*
 664          * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
 665          */
 666 
 667         /*
 668          * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
 669          * set the quadword to all F's - high-order bit is owner (init to one)
 670          * and the rest for the headroom definition of prefetching
 671          *
 672          */
 673         wqesz_shift = qp->qp_sq_log_wqesz;
 674         thewqesz    = 1 << wqesz_shift;
 675         thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
 676         if (qp_is_umap == 0) {
 677                 for (i = 0; i < sq_depth; i++) {
 678                         /*
 679                          * for each stride, go through and every 64 bytes
 680                          * write the init value - having set the address
 681                          * once, just keep incrementing it
 682                          */
 683                         for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
 684                                 *(uint32_t *)thewqe = 0xFFFFFFFF;
 685                         }
 686                 }
 687         }
 688 
 689         /* Zero out the QP context */
 690         bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
 691 
 692         /*
 693          * Put QP handle in Hermon QPNum-to-QPHdl list.  Then fill in the
 694          * "qphdl" and return success
 695          */
 696         hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx, qp);
 697 
 698         /*
 699          * If this is a user-mappable QP, then we need to insert the previously
 700          * allocated entry into the "userland resources database".  This will
 701          * allow for later lookup during devmap() (i.e. mmap()) calls.
 702          */
 703         if (qp_is_umap) {
 704                 hermon_umap_db_add(umapdb);
 705         }
 706         mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
 707             DDI_INTR_PRI(state->hs_intrmsi_pri));
 708 
 709         *qphdl = qp;
 710 
 711         return (DDI_SUCCESS);
 712 
 713 /*
 714  * The following is cleanup for all possible failure cases in this routine
 715  */
 716 qpalloc_fail9:
 717         hermon_queue_free(&qp->qp_wqinfo);
 718 qpalloc_fail8:
 719         if (qp->qp_sq_wqhdr)
 720                 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
 721         if (qp->qp_rq_wqhdr)
 722                 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
 723 qpalloc_fail7:
 724         if (qp_is_umap) {
 725                 hermon_umap_db_free(umapdb);
 726         }
 727         if (!qp_srq_en) {
 728                 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
 729         }
 730 
 731 qpalloc_fail6:
 732         /*
 733          * Releasing the QPN will also free up the QPC context.  Update
 734          * the QPC context pointer to indicate this.
 735          */
 736         if (qp->qp_qpn_hdl) {
 737                 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
 738                     HERMON_QPN_RELEASE);
 739         } else {
 740                 hermon_rsrc_free(state, &qpc);
 741         }
 742         qpc = NULL;
 743 qpalloc_fail5:
 744         hermon_rsrc_free(state, &rsrc);
 745 qpalloc_fail4:
 746         if (qpc) {
 747                 hermon_rsrc_free(state, &qpc);
 748         }
 749 qpalloc_fail3:
 750         hermon_cq_refcnt_dec(rq_cq);
 751 qpalloc_fail2:
 752         hermon_cq_refcnt_dec(sq_cq);
 753 qpalloc_fail1:
 754         hermon_pd_refcnt_dec(pd);
 755 qpalloc_fail:
 756         return (status);
 757 }
 758 
 759 
 760 
 761 /*
 762  * hermon_special_qp_alloc()
 763  *    Context: Can be called only from user or kernel context.
 764  */
 765 int
 766 hermon_special_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
 767     uint_t sleepflag)
 768 {
 769         hermon_rsrc_t           *qpc, *rsrc;
 770         hermon_qphdl_t          qp;
 771         ibt_qp_alloc_attr_t     *attr_p;
 772         ibt_sqp_type_t          type;
 773         uint8_t                 port;
 774         ibtl_qp_hdl_t           ibt_qphdl;
 775         ibt_chan_sizes_t        *queuesz_p;
 776         hermon_qphdl_t          *qphdl;
 777         ibt_mr_attr_t           mr_attr;
 778         hermon_mr_options_t     mr_op;
 779         hermon_pdhdl_t          pd;
 780         hermon_cqhdl_t          sq_cq, rq_cq;
 781         hermon_mrhdl_t          mr;
 782         uint64_t                qp_desc_off;
 783         uint64_t                *thewqe, thewqesz;
 784         uint32_t                *sq_buf, *rq_buf;
 785         uint32_t                log_qp_sq_size, log_qp_rq_size;
 786         uint32_t                sq_size, rq_size, max_sgl;
 787         uint32_t                uarpg;
 788         uint32_t                sq_depth;
 789         uint32_t                sq_wqe_size, rq_wqe_size, wqesz_shift;
 790         int                     status, flag, i, j;
 791 
 792         /*
 793          * Extract the necessary info from the hermon_qp_info_t structure
 794          */
 795         attr_p    = qpinfo->qpi_attrp;
 796         type      = qpinfo->qpi_type;
 797         port      = qpinfo->qpi_port;
 798         ibt_qphdl = qpinfo->qpi_ibt_qphdl;
 799         queuesz_p = qpinfo->qpi_queueszp;
 800         qphdl     = &qpinfo->qpi_qphdl;
 801 
 802         /*
 803          * Check for valid special QP type (only SMI & GSI supported)
 804          */
 805         if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) {
 806                 status = IBT_QP_SPECIAL_TYPE_INVALID;
 807                 goto spec_qpalloc_fail;
 808         }
 809 
 810         /*
 811          * Check for valid port number
 812          */
 813         if (!hermon_portnum_is_valid(state, port)) {
 814                 status = IBT_HCA_PORT_INVALID;
 815                 goto spec_qpalloc_fail;
 816         }
 817         port = port - 1;
 818 
 819         /*
 820          * Check for valid PD handle pointer
 821          */
 822         if (attr_p->qp_pd_hdl == NULL) {
 823                 status = IBT_PD_HDL_INVALID;
 824                 goto spec_qpalloc_fail;
 825         }
 826         pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
 827 
 828         /* Increment the reference count on the PD */
 829         hermon_pd_refcnt_inc(pd);
 830 
 831         /*
 832          * Check for valid CQ handle pointers
 833          */
 834         if ((attr_p->qp_ibc_scq_hdl == NULL) ||
 835             (attr_p->qp_ibc_rcq_hdl == NULL)) {
 836                 status = IBT_CQ_HDL_INVALID;
 837                 goto spec_qpalloc_fail1;
 838         }
 839         sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
 840         rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
 841 
 842         /*
 843          * Increment the reference count on the CQs.  One or both of these
 844          * could return error if we determine that the given CQ is already
 845          * being used with a non-special QP (i.e. a normal QP).
 846          */
 847         status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_SPECIAL);
 848         if (status != DDI_SUCCESS) {
 849                 status = IBT_CQ_HDL_INVALID;
 850                 goto spec_qpalloc_fail1;
 851         }
 852         status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_SPECIAL);
 853         if (status != DDI_SUCCESS) {
 854                 status = IBT_CQ_HDL_INVALID;
 855                 goto spec_qpalloc_fail2;
 856         }
 857 
 858         /*
 859          * Allocate the special QP resources.  Essentially, this allocation
 860          * amounts to checking if the request special QP has already been
 861          * allocated.  If successful, the QP context return is an actual
 862          * QP context that has been "aliased" to act as a special QP of the
 863          * appropriate type (and for the appropriate port).  Just as in
 864          * hermon_qp_alloc() above, ownership for this QP context is not
 865          * immediately given to hardware in the final step here.  Instead, we
 866          * wait until the QP is later transitioned to the "Init" state before
 867          * passing the QP to hardware.  If we fail here, we must undo all
 868          * the reference count (CQ and PD).
 869          */
 870         status = hermon_special_qp_rsrc_alloc(state, type, port, &qpc);
 871         if (status != DDI_SUCCESS) {
 872                 goto spec_qpalloc_fail3;
 873         }
 874 
 875         /*
 876          * Allocate the software structure for tracking the special queue
 877          * pair (i.e. the Hermon Queue Pair handle).  If we fail here, we
 878          * must undo the reference counts and the previous resource allocation.
 879          */
 880         status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
 881         if (status != DDI_SUCCESS) {
 882                 status = IBT_INSUFF_RESOURCE;
 883                 goto spec_qpalloc_fail4;
 884         }
 885         qp = (hermon_qphdl_t)rsrc->hr_addr;
 886 
 887         bzero(qp, sizeof (struct hermon_sw_qp_s));
 888 
 889         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
 890         qp->qp_alloc_flags = attr_p->qp_alloc_flags;
 891 
 892         /*
 893          * Actual QP number is a combination of the index of the QPC and
 894          * the port number.  This is because the special QP contexts must
 895          * be allocated two-at-a-time.
 896          */
 897         qp->qp_qpnum = qpc->hr_indx + port;
 898         qp->qp_ring = qp->qp_qpnum << 8;
 899 
 900         uarpg = state->hs_kernel_uar_index; /* must be for spec qp */
 901         /*
 902          * Allocate the doorbell record.  Hermon uses only one for the RQ so
 903          * alloc a qp doorbell, using uarpg (above) as the uar index
 904          */
 905 
 906         status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
 907             &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
 908         if (status != DDI_SUCCESS) {
 909                 status = IBT_INSUFF_RESOURCE;
 910                 goto spec_qpalloc_fail5;
 911         }
 912         /*
 913          * Calculate the appropriate size for the work queues.
 914          * Note:  All Hermon QP work queues must be a power-of-2 in size.  Also
 915          * they may not be any smaller than HERMON_QP_MIN_SIZE.  This step is
 916          * to round the requested size up to the next highest power-of-2
 917          */
 918         attr_p->qp_sizes.cs_sq =
 919             max(attr_p->qp_sizes.cs_sq, HERMON_QP_MIN_SIZE);
 920         attr_p->qp_sizes.cs_rq =
 921             max(attr_p->qp_sizes.cs_rq, HERMON_QP_MIN_SIZE);
 922         log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
 923         if (ISP2(attr_p->qp_sizes.cs_sq)) {
 924                 log_qp_sq_size = log_qp_sq_size - 1;
 925         }
 926         log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
 927         if (ISP2(attr_p->qp_sizes.cs_rq)) {
 928                 log_qp_rq_size = log_qp_rq_size - 1;
 929         }
 930 
 931         /*
 932          * Next we verify that the rounded-up size is valid (i.e. consistent
 933          * with the device limits and/or software-configured limits).  If not,
 934          * then obviously we have a bit of cleanup to do before returning.
 935          */
 936         if ((log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
 937             (log_qp_rq_size > state->hs_cfg_profile->cp_log_max_qp_sz)) {
 938                 status = IBT_HCA_WR_EXCEEDED;
 939                 goto spec_qpalloc_fail5a;
 940         }
 941 
 942         /*
 943          * Next we verify that the requested number of SGL is valid (i.e.
 944          * consistent with the device limits and/or software-configured
 945          * limits).  If not, then obviously the same cleanup needs to be done.
 946          */
 947         max_sgl = state->hs_cfg_profile->cp_wqe_real_max_sgl;
 948         if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
 949             (attr_p->qp_sizes.cs_rq_sgl > max_sgl)) {
 950                 status = IBT_HCA_SGL_EXCEEDED;
 951                 goto spec_qpalloc_fail5a;
 952         }
 953 
 954         /*
 955          * Determine this QP's WQE stride (for both the Send and Recv WQEs).
 956          * This will depend on the requested number of SGLs.  Note: this
 957          * has the side-effect of also calculating the real number of SGLs
 958          * (for the calculated WQE size).
 959          */
 960         hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
 961             max_sgl, HERMON_QP_WQ_TYPE_RECVQ,
 962             &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
 963         if (type == IBT_SMI_SQP) {
 964                 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
 965                     max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP0,
 966                     &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
 967         } else {
 968                 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
 969                     max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP1,
 970                     &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
 971         }
 972 
 973         /*
 974          * Allocate the memory for QP work queues. Since Hermon work queues
 975          * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
 976          * the work queue memory is very important.  We used to allocate
 977          * work queues (the combined receive and send queues) so that they
 978          * would be aligned on their combined size.  That alignment guaranteed
 979          * that they would never cross the 4GB boundary (Hermon work queues
 980          * are on the order of MBs at maximum).  Now we are able to relax
 981          * this alignment constraint by ensuring that the IB address assigned
 982          * to the queue memory (as a result of the hermon_mr_register() call)
 983          * is offset from zero.
 984          * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
 985          * guarantee the alignment, but when attempting to use IOMMU bypass
 986          * mode we found that we were not allowed to specify any alignment
 987          * that was more restrictive than the system page size.
 988          * So we avoided this constraint by passing two alignment values,
 989          * one for the memory allocation itself and the other for the DMA
 990          * handle (for later bind).  This used to cause more memory than
 991          * necessary to be allocated (in order to guarantee the more
 992          * restrictive alignment contraint).  But by guaranteeing the
 993          * zero-based IB virtual address for the queue, we are able to
 994          * conserve this memory.
 995          */
 996         sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
 997         sq_depth    = 1 << log_qp_sq_size;
 998         sq_size     = (1 << log_qp_sq_size) * sq_wqe_size;
 999 
1000         rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1001         rq_size     = (1 << log_qp_rq_size) * rq_wqe_size;
1002 
1003         qp->qp_wqinfo.qa_size          = sq_size + rq_size;
1004 
1005         qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1006         qp->qp_wqinfo.qa_bind_align  = PAGESIZE;
1007         qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1008 
1009         status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1010         if (status != NULL) {
1011                 status = IBT_INSUFF_RESOURCE;
1012                 goto spec_qpalloc_fail5a;
1013         }
1014 
1015         /*
1016          * Sort WQs in memory according to depth, stride (*q_wqe_size),
1017          * biggest first. If equal, the Send Queue still goes first
1018          */
1019         qp->qp_sq_baseaddr = 0;
1020         qp->qp_rq_baseaddr = 0;
1021         if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1022                 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1023                 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1024                 qp->qp_rq_baseaddr = sq_size;
1025         } else {
1026                 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1027                 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1028                 qp->qp_sq_baseaddr = rq_size;
1029         }
1030 
1031         qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1032         if (qp->qp_sq_wqhdr == NULL) {
1033                 status = IBT_INSUFF_RESOURCE;
1034                 goto spec_qpalloc_fail6;
1035         }
1036         qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(1 << log_qp_rq_size);
1037         if (qp->qp_rq_wqhdr == NULL) {
1038                 status = IBT_INSUFF_RESOURCE;
1039                 goto spec_qpalloc_fail6;
1040         }
1041         qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1042         qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1043         qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1044         qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1045         qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1046         qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1047 
1048         /*
1049          * Register the memory for the special QP work queues.  The memory for
1050          * the special QP must be registered in the Hermon cMPT tables.  This
1051          * gives us the LKey to specify in the QP context later.  Note: The
1052          * memory for Hermon work queues (both Send and Recv) must be contiguous
1053          * and registered as a single memory region. Also, in order to meet the
1054          * alignment restriction, we pass the "mro_bind_override_addr" flag in
1055          * the call to hermon_mr_register(). This guarantees that the resulting
1056          * IB vaddr will be zero-based (modulo the offset into the first page).
1057          * If we fail here, we have a bunch of resource and reference count
1058          * cleanup to do.
1059          */
1060         flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1061             IBT_MR_NOSLEEP;
1062         mr_attr.mr_vaddr    = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1063         mr_attr.mr_len      = qp->qp_wqinfo.qa_size;
1064         mr_attr.mr_as       = NULL;
1065         mr_attr.mr_flags    = flag;
1066 
1067         mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
1068         mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1069         mr_op.mro_bind_override_addr = 1;
1070 
1071         status = hermon_mr_register(state, pd, &mr_attr, &mr, &mr_op,
1072             HERMON_QP_CMPT);
1073         if (status != DDI_SUCCESS) {
1074                 status = IBT_INSUFF_RESOURCE;
1075                 goto spec_qpalloc_fail6;
1076         }
1077 
1078         /*
1079          * Calculate the offset between the kernel virtual address space
1080          * and the IB virtual address space.  This will be used when
1081          * posting work requests to properly initialize each WQE.
1082          */
1083         qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1084             (uint64_t)mr->mr_bindinfo.bi_addr;
1085 
1086         /* set the prefetch - initially, not prefetching */
1087         qp->qp_no_prefetch = 1;
1088 
1089         if (qp->qp_no_prefetch)
1090                 qp->qp_sq_headroom = 2 * sq_wqe_size;
1091         else
1092                 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1093         /*
1094          * hdrm wqes must be integral since both sq_wqe_size &
1095          * HERMON_QP_OH_SIZE are power of 2
1096          */
1097         qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1098         /*
1099          * Fill in all the return arguments (if necessary).  This includes
1100          * real work queue sizes, real SGLs, and QP number (which will be
1101          * either zero or one, depending on the special QP type)
1102          */
1103         if (queuesz_p != NULL) {
1104                 queuesz_p->cs_sq     =
1105                     (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1106                 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1107                 queuesz_p->cs_rq     = (1 << log_qp_rq_size);
1108                 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1109         }
1110 
1111         /*
1112          * Fill in the rest of the Hermon Queue Pair handle.  We can update
1113          * the following fields for use in further operations on the QP.
1114          */
1115         qp->qp_qpcrsrcp              = qpc;
1116         qp->qp_rsrcp         = rsrc;
1117         qp->qp_state         = HERMON_QP_RESET;
1118         HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1119         qp->qp_pdhdl         = pd;
1120         qp->qp_mrhdl         = mr;
1121         qp->qp_sq_sigtype    = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1122             HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1123         qp->qp_is_special    = (type == IBT_SMI_SQP) ?
1124             HERMON_QP_SMI : HERMON_QP_GSI;
1125         qp->qp_uarpg         = uarpg;
1126         qp->qp_umap_dhp              = (devmap_cookie_t)NULL;
1127         qp->qp_sq_cqhdl              = sq_cq;
1128         qp->qp_sq_bufsz              = (1 << log_qp_sq_size);
1129         qp->qp_sq_buf                = sq_buf;
1130         qp->qp_sq_logqsz     = log_qp_sq_size;
1131         qp->qp_desc_off              = qp_desc_off;
1132         qp->qp_rq_cqhdl              = rq_cq;
1133         qp->qp_rq_bufsz              = (1 << log_qp_rq_size);
1134         qp->qp_rq_buf                = rq_buf;
1135         qp->qp_rq_logqsz     = log_qp_rq_size;
1136         qp->qp_portnum               = port;
1137         qp->qp_pkeyindx              = 0;
1138         qp->qp_forward_sqd_event  = 0;
1139         qp->qp_sqd_still_draining = 0;
1140         qp->qp_hdlrarg               = (void *)ibt_qphdl;
1141         qp->qp_mcg_refcnt    = 0;
1142         qp->qp_srqhdl                = NULL;
1143 
1144         /* All special QPs are UD QP service type */
1145         qp->qp_type = IBT_UD_RQP;
1146         qp->qp_serv_type = HERMON_QP_UD;
1147 
1148         /*
1149          * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1150          */
1151 
1152         /*
1153          * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1154          * set the quadword to all F's - high-order bit is owner (init to one)
1155          * and the rest for the headroom definition of prefetching
1156          *
1157          */
1158 
1159         wqesz_shift = qp->qp_sq_log_wqesz;
1160         thewqesz    = 1 << wqesz_shift;
1161         thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1162         for (i = 0; i < sq_depth; i++) {
1163                 /*
1164                  * for each stride, go through and every 64 bytes write the
1165                  * init value - having set the address once, just keep
1166                  * incrementing it
1167                  */
1168                 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1169                         *(uint32_t *)thewqe = 0xFFFFFFFF;
1170                 }
1171         }
1172 
1173 
1174         /* Zero out the QP context */
1175         bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1176 
1177         /*
1178          * Put QP handle in Hermon QPNum-to-QPHdl list.  Then fill in the
1179          * "qphdl" and return success
1180          */
1181         hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + port, qp);
1182 
1183         mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1184             DDI_INTR_PRI(state->hs_intrmsi_pri));
1185 
1186         *qphdl = qp;
1187 
1188         return (DDI_SUCCESS);
1189 
1190 /*
1191  * The following is cleanup for all possible failure cases in this routine
1192  */
1193 spec_qpalloc_fail6:
1194         hermon_queue_free(&qp->qp_wqinfo);
1195         if (qp->qp_sq_wqhdr)
1196                 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1197         if (qp->qp_rq_wqhdr)
1198                 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1199 spec_qpalloc_fail5a:
1200         hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1201 spec_qpalloc_fail5:
1202         hermon_rsrc_free(state, &rsrc);
1203 spec_qpalloc_fail4:
1204         if (hermon_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) {
1205                 HERMON_WARNING(state, "failed to free special QP rsrc");
1206         }
1207 spec_qpalloc_fail3:
1208         hermon_cq_refcnt_dec(rq_cq);
1209 spec_qpalloc_fail2:
1210         hermon_cq_refcnt_dec(sq_cq);
1211 spec_qpalloc_fail1:
1212         hermon_pd_refcnt_dec(pd);
1213 spec_qpalloc_fail:
1214         return (status);
1215 }
1216 
1217 
1218 /*
1219  * hermon_qp_alloc_range()
1220  *    Context: Can be called only from user or kernel context.
1221  */
1222 int
1223 hermon_qp_alloc_range(hermon_state_t *state, uint_t log2,
1224     hermon_qp_info_t *qpinfo, ibtl_qp_hdl_t *ibt_qphdl,
1225     ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
1226     hermon_qphdl_t *qphdl, uint_t sleepflag)
1227 {
1228         hermon_rsrc_t                   *qpc, *rsrc;
1229         hermon_rsrc_type_t              rsrc_type;
1230         hermon_qphdl_t                  qp;
1231         hermon_qp_range_t               *qp_range_p;
1232         ibt_qp_alloc_attr_t             *attr_p;
1233         ibt_qp_type_t                   type;
1234         hermon_qp_wq_type_t             swq_type;
1235         ibt_chan_sizes_t                *queuesz_p;
1236         ibt_mr_attr_t                   mr_attr;
1237         hermon_mr_options_t             mr_op;
1238         hermon_srqhdl_t                 srq;
1239         hermon_pdhdl_t                  pd;
1240         hermon_cqhdl_t                  sq_cq, rq_cq;
1241         hermon_mrhdl_t                  mr;
1242         uint64_t                        qp_desc_off;
1243         uint64_t                        *thewqe, thewqesz;
1244         uint32_t                        *sq_buf, *rq_buf;
1245         uint32_t                        log_qp_sq_size, log_qp_rq_size;
1246         uint32_t                        sq_size, rq_size;
1247         uint32_t                        sq_depth, rq_depth;
1248         uint32_t                        sq_wqe_size, rq_wqe_size, wqesz_shift;
1249         uint32_t                        max_sgl, max_recv_sgl, uarpg;
1250         uint_t                          qp_srq_en, i, j;
1251         int                             ii;     /* loop counter for range */
1252         int                             status, flag;
1253         uint_t                          serv_type;
1254 
1255         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
1256 
1257         /*
1258          * Extract the necessary info from the hermon_qp_info_t structure
1259          */
1260         attr_p    = qpinfo->qpi_attrp;
1261         type      = qpinfo->qpi_type;
1262         queuesz_p = qpinfo->qpi_queueszp;
1263 
1264         if (attr_p->qp_alloc_flags & IBT_QP_USES_RSS) {
1265                 if (log2 > state->hs_ibtfinfo.hca_attr->hca_rss_max_log2_table)
1266                         return (IBT_INSUFF_RESOURCE);
1267                 rsrc_type = HERMON_QPC;
1268                 serv_type = HERMON_QP_UD;
1269         } else if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1270                 if (log2 > state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp)
1271                         return (IBT_INSUFF_RESOURCE);
1272                 switch (attr_p->qp_fc.fc_hca_port) {
1273                 case 1:
1274                         rsrc_type = HERMON_QPC_FEXCH_PORT1;
1275                         break;
1276                 case 2:
1277                         rsrc_type = HERMON_QPC_FEXCH_PORT2;
1278                         break;
1279                 default:
1280                         return (IBT_INVALID_PARAM);
1281                 }
1282                 serv_type = HERMON_QP_FEXCH;
1283         } else
1284                 return (IBT_INVALID_PARAM);
1285 
1286         /*
1287          * Determine whether QP is being allocated for userland access or
1288          * whether it is being allocated for kernel access.  If the QP is
1289          * being allocated for userland access, fail (too complex for now).
1290          */
1291         if (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) {
1292                 return (IBT_NOT_SUPPORTED);
1293         } else {
1294                 uarpg = state->hs_kernel_uar_index;
1295         }
1296 
1297         /*
1298          * Determine whether QP is being associated with an SRQ
1299          */
1300         qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
1301         if (qp_srq_en) {
1302                 /*
1303                  * Check for valid SRQ handle pointers
1304                  */
1305                 if (attr_p->qp_ibc_srq_hdl == NULL) {
1306                         return (IBT_SRQ_HDL_INVALID);
1307                 }
1308                 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
1309         }
1310 
1311         /*
1312          * Check for valid QP service type (only UD supported)
1313          */
1314         if (type != IBT_UD_RQP) {
1315                 return (IBT_QP_SRV_TYPE_INVALID);
1316         }
1317 
1318         /*
1319          * Check for valid PD handle pointer
1320          */
1321         if (attr_p->qp_pd_hdl == NULL) {
1322                 return (IBT_PD_HDL_INVALID);
1323         }
1324         pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
1325 
1326         /*
1327          * If on an SRQ, check to make sure the PD is the same
1328          */
1329         if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
1330                 return (IBT_PD_HDL_INVALID);
1331         }
1332 
1333         /* set loop variable here, for freeing resources on error */
1334         ii = 0;
1335 
1336         /*
1337          * Allocate 2^log2 contiguous/aligned QP context entries.  This will
1338          * be filled in with all the necessary parameters to define the
1339          * Queue Pairs.  Unlike other Hermon hardware resources, ownership
1340          * is not immediately given to hardware in the final step here.
1341          * Instead, we must wait until the QP is later transitioned to the
1342          * "Init" state before passing the QP to hardware.  If we fail here,
1343          * we must undo all the reference count (CQ and PD).
1344          */
1345         status = hermon_rsrc_alloc(state, rsrc_type, 1 << log2, sleepflag,
1346             &qpc);
1347         if (status != DDI_SUCCESS) {
1348                 return (IBT_INSUFF_RESOURCE);
1349         }
1350 
1351         if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH)
1352                 /*
1353                  * Need to init the MKEYs for the FEXCH QPs.
1354                  *
1355                  * For FEXCH QP subranges, we return the QPN base as
1356                  * "relative" to the full FEXCH QP range for the port.
1357                  */
1358                 *(qpinfo->qpi_qpn) = hermon_fcoib_fexch_relative_qpn(state,
1359                     attr_p->qp_fc.fc_hca_port, qpc->hr_indx);
1360         else
1361                 *(qpinfo->qpi_qpn) = (ib_qpn_t)qpc->hr_indx;
1362 
1363         qp_range_p = kmem_alloc(sizeof (*qp_range_p),
1364             (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1365         if (qp_range_p == NULL) {
1366                 status = IBT_INSUFF_RESOURCE;
1367                 goto qpalloc_fail0;
1368         }
1369         mutex_init(&qp_range_p->hqpr_lock, NULL, MUTEX_DRIVER,
1370             DDI_INTR_PRI(state->hs_intrmsi_pri));
1371         mutex_enter(&qp_range_p->hqpr_lock);
1372         qp_range_p->hqpr_refcnt = 1 << log2;
1373         qp_range_p->hqpr_qpcrsrc = qpc;
1374         mutex_exit(&qp_range_p->hqpr_lock);
1375 
1376 for_each_qp:
1377 
1378         /* Increment the reference count on the protection domain (PD) */
1379         hermon_pd_refcnt_inc(pd);
1380 
1381         rq_cq = (hermon_cqhdl_t)recv_cq[ii];
1382         sq_cq = (hermon_cqhdl_t)send_cq[ii];
1383         if (sq_cq == NULL) {
1384                 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1385                         /* if no send completions, just use rq_cq */
1386                         sq_cq = rq_cq;
1387                 } else {
1388                         status = IBT_CQ_HDL_INVALID;
1389                         goto qpalloc_fail1;
1390                 }
1391         }
1392 
1393         /*
1394          * Increment the reference count on the CQs.  One or both of these
1395          * could return error if we determine that the given CQ is already
1396          * being used with a special (SMI/GSI) QP.
1397          */
1398         status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
1399         if (status != DDI_SUCCESS) {
1400                 status = IBT_CQ_HDL_INVALID;
1401                 goto qpalloc_fail1;
1402         }
1403         status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
1404         if (status != DDI_SUCCESS) {
1405                 status = IBT_CQ_HDL_INVALID;
1406                 goto qpalloc_fail2;
1407         }
1408 
1409         /*
1410          * Allocate the software structure for tracking the queue pair
1411          * (i.e. the Hermon Queue Pair handle).  If we fail here, we must
1412          * undo the reference counts and the previous resource allocation.
1413          */
1414         status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
1415         if (status != DDI_SUCCESS) {
1416                 status = IBT_INSUFF_RESOURCE;
1417                 goto qpalloc_fail4;
1418         }
1419         qp = (hermon_qphdl_t)rsrc->hr_addr;
1420         bzero(qp, sizeof (struct hermon_sw_qp_s));
1421         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1422         qp->qp_alloc_flags = attr_p->qp_alloc_flags;
1423 
1424         /*
1425          * Calculate the QP number from QPC index.  This routine handles
1426          * all of the operations necessary to keep track of used, unused,
1427          * and released QP numbers.
1428          */
1429         qp->qp_qpnum = qpc->hr_indx + ii;
1430         qp->qp_ring = qp->qp_qpnum << 8;
1431         qp->qp_qpn_hdl = NULL;
1432 
1433         /*
1434          * Allocate the doorbell record.  Hermon just needs one for the RQ,
1435          * if the QP is not associated with an SRQ, and use uarpg (above) as
1436          * the uar index
1437          */
1438 
1439         if (!qp_srq_en) {
1440                 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
1441                     &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
1442                 if (status != DDI_SUCCESS) {
1443                         status = IBT_INSUFF_RESOURCE;
1444                         goto qpalloc_fail6;
1445                 }
1446         }
1447 
1448         qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
1449 
1450         /*
1451          * We verify that the requested number of SGL is valid (i.e.
1452          * consistent with the device limits and/or software-configured
1453          * limits).  If not, then obviously the same cleanup needs to be done.
1454          */
1455         max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
1456         swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
1457         max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
1458         if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
1459             (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
1460                 status = IBT_HCA_SGL_EXCEEDED;
1461                 goto qpalloc_fail7;
1462         }
1463 
1464         /*
1465          * Determine this QP's WQE stride (for both the Send and Recv WQEs).
1466          * This will depend on the requested number of SGLs.  Note: this
1467          * has the side-effect of also calculating the real number of SGLs
1468          * (for the calculated WQE size).
1469          *
1470          * For QP's on an SRQ, we set these to 0.
1471          */
1472         if (qp_srq_en) {
1473                 qp->qp_rq_log_wqesz = 0;
1474                 qp->qp_rq_sgl = 0;
1475         } else {
1476                 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
1477                     max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
1478                     &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
1479         }
1480         hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
1481             max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
1482 
1483         sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1484 
1485         /* NOTE: currently policy in driver, later maybe IBTF interface */
1486         qp->qp_no_prefetch = 0;
1487 
1488         /*
1489          * for prefetching, we need to add the number of wqes in
1490          * the 2k area plus one to the number requested, but
1491          * ONLY for send queue.  If no_prefetch == 1 (prefetch off)
1492          * it's exactly TWO wqes for the headroom
1493          */
1494         if (qp->qp_no_prefetch)
1495                 qp->qp_sq_headroom = 2 * sq_wqe_size;
1496         else
1497                 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1498         /*
1499          * hdrm wqes must be integral since both sq_wqe_size &
1500          * HERMON_QP_OH_SIZE are power of 2
1501          */
1502         qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1503 
1504 
1505         /*
1506          * Calculate the appropriate size for the work queues.
1507          * For send queue, add in the headroom wqes to the calculation.
1508          * Note:  All Hermon QP work queues must be a power-of-2 in size.  Also
1509          * they may not be any smaller than HERMON_QP_MIN_SIZE.  This step is
1510          * to round the requested size up to the next highest power-of-2
1511          */
1512         /* first, adjust to a minimum and tell the caller the change */
1513         attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
1514             HERMON_QP_MIN_SIZE);
1515         attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
1516             HERMON_QP_MIN_SIZE);
1517         /*
1518          * now, calculate the alloc size, taking into account
1519          * the headroom for the sq
1520          */
1521         log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
1522         /* if the total is a power of two, reduce it */
1523         if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes))    {
1524                 log_qp_sq_size = log_qp_sq_size - 1;
1525         }
1526 
1527         log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
1528         if (ISP2(attr_p->qp_sizes.cs_rq)) {
1529                 log_qp_rq_size = log_qp_rq_size - 1;
1530         }
1531 
1532         /*
1533          * Next we verify that the rounded-up size is valid (i.e. consistent
1534          * with the device limits and/or software-configured limits).  If not,
1535          * then obviously we have a lot of cleanup to do before returning.
1536          *
1537          * NOTE: the first condition deals with the (test) case of cs_sq
1538          * being just less than 2^32.  In this case, the headroom addition
1539          * to the requested cs_sq will pass the test when it should not.
1540          * This test no longer lets that case slip through the check.
1541          */
1542         if ((attr_p->qp_sizes.cs_sq >
1543             (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
1544             (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
1545             (!qp_srq_en && (log_qp_rq_size >
1546             state->hs_cfg_profile->cp_log_max_qp_sz))) {
1547                 status = IBT_HCA_WR_EXCEEDED;
1548                 goto qpalloc_fail7;
1549         }
1550 
1551         /*
1552          * Allocate the memory for QP work queues. Since Hermon work queues
1553          * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
1554          * the work queue memory is very important.  We used to allocate
1555          * work queues (the combined receive and send queues) so that they
1556          * would be aligned on their combined size.  That alignment guaranteed
1557          * that they would never cross the 4GB boundary (Hermon work queues
1558          * are on the order of MBs at maximum).  Now we are able to relax
1559          * this alignment constraint by ensuring that the IB address assigned
1560          * to the queue memory (as a result of the hermon_mr_register() call)
1561          * is offset from zero.
1562          * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
1563          * guarantee the alignment, but when attempting to use IOMMU bypass
1564          * mode we found that we were not allowed to specify any alignment
1565          * that was more restrictive than the system page size.
1566          * So we avoided this constraint by passing two alignment values,
1567          * one for the memory allocation itself and the other for the DMA
1568          * handle (for later bind).  This used to cause more memory than
1569          * necessary to be allocated (in order to guarantee the more
1570          * restrictive alignment contraint).  But by guaranteeing the
1571          * zero-based IB virtual address for the queue, we are able to
1572          * conserve this memory.
1573          */
1574         sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1575         sq_depth    = 1 << log_qp_sq_size;
1576         sq_size     = sq_depth * sq_wqe_size;
1577 
1578         /* QP on SRQ sets these to 0 */
1579         if (qp_srq_en) {
1580                 rq_wqe_size = 0;
1581                 rq_size     = 0;
1582         } else {
1583                 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1584                 rq_depth    = 1 << log_qp_rq_size;
1585                 rq_size     = rq_depth * rq_wqe_size;
1586         }
1587 
1588         qp->qp_wqinfo.qa_size = sq_size + rq_size;
1589         qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1590         qp->qp_wqinfo.qa_bind_align  = PAGESIZE;
1591         qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1592         status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1593         if (status != DDI_SUCCESS) {
1594                 status = IBT_INSUFF_RESOURCE;
1595                 goto qpalloc_fail7;
1596         }
1597 
1598         /*
1599          * Sort WQs in memory according to stride (*q_wqe_size), largest first
1600          * If they are equal, still put the SQ first
1601          */
1602         qp->qp_sq_baseaddr = 0;
1603         qp->qp_rq_baseaddr = 0;
1604         if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1605                 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1606 
1607                 /* if this QP is on an SRQ, set the rq_buf to NULL */
1608                 if (qp_srq_en) {
1609                         rq_buf = NULL;
1610                 } else {
1611                         rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1612                         qp->qp_rq_baseaddr = sq_size;
1613                 }
1614         } else {
1615                 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1616                 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1617                 qp->qp_sq_baseaddr = rq_size;
1618         }
1619 
1620         qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1621         if (qp->qp_sq_wqhdr == NULL) {
1622                 status = IBT_INSUFF_RESOURCE;
1623                 goto qpalloc_fail8;
1624         }
1625         if (qp_srq_en) {
1626                 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
1627                 qp->qp_rq_wqavl.wqa_srq_en = 1;
1628                 qp->qp_rq_wqavl.wqa_srq = srq;
1629         } else {
1630                 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
1631                 if (qp->qp_rq_wqhdr == NULL) {
1632                         status = IBT_INSUFF_RESOURCE;
1633                         goto qpalloc_fail8;
1634                 }
1635                 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1636         }
1637         qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1638         qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1639         qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1640         qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1641         qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1642 
1643         /*
1644          * Register the memory for the QP work queues.  The memory for the
1645          * QP must be registered in the Hermon cMPT tables.  This gives us the
1646          * LKey to specify in the QP context later.  Note: The memory for
1647          * Hermon work queues (both Send and Recv) must be contiguous and
1648          * registered as a single memory region.  Note: If the QP memory is
1649          * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
1650          * meet the alignment restriction, we pass the "mro_bind_override_addr"
1651          * flag in the call to hermon_mr_register(). This guarantees that the
1652          * resulting IB vaddr will be zero-based (modulo the offset into the
1653          * first page). If we fail here, we still have the bunch of resource
1654          * and reference count cleanup to do.
1655          */
1656         flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1657             IBT_MR_NOSLEEP;
1658         mr_attr.mr_vaddr    = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1659         mr_attr.mr_len      = qp->qp_wqinfo.qa_size;
1660         mr_attr.mr_as       = NULL;
1661         mr_attr.mr_flags    = flag;
1662         /* HERMON_QUEUE_LOCATION_NORMAL */
1663         mr_op.mro_bind_type =
1664             state->hs_cfg_profile->cp_iommu_bypass;
1665         mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1666         mr_op.mro_bind_override_addr = 1;
1667         status = hermon_mr_register(state, pd, &mr_attr, &mr,
1668             &mr_op, HERMON_QP_CMPT);
1669         if (status != DDI_SUCCESS) {
1670                 status = IBT_INSUFF_RESOURCE;
1671                 goto qpalloc_fail9;
1672         }
1673 
1674         /*
1675          * Calculate the offset between the kernel virtual address space
1676          * and the IB virtual address space.  This will be used when
1677          * posting work requests to properly initialize each WQE.
1678          */
1679         qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1680             (uint64_t)mr->mr_bindinfo.bi_addr;
1681 
1682         /*
1683          * Fill in all the return arguments (if necessary).  This includes
1684          * real work queue sizes (in wqes), real SGLs, and QP number
1685          */
1686         if (queuesz_p != NULL) {
1687                 queuesz_p->cs_sq     =
1688                     (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1689                 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1690 
1691                 /* if this QP is on an SRQ, set these to 0 */
1692                 if (qp_srq_en) {
1693                         queuesz_p->cs_rq     = 0;
1694                         queuesz_p->cs_rq_sgl = 0;
1695                 } else {
1696                         queuesz_p->cs_rq     = (1 << log_qp_rq_size);
1697                         queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1698                 }
1699         }
1700 
1701         /*
1702          * Fill in the rest of the Hermon Queue Pair handle.
1703          */
1704         qp->qp_qpcrsrcp              = NULL;
1705         qp->qp_rsrcp         = rsrc;
1706         qp->qp_state         = HERMON_QP_RESET;
1707         HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1708         qp->qp_pdhdl         = pd;
1709         qp->qp_mrhdl         = mr;
1710         qp->qp_sq_sigtype    = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1711             HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1712         qp->qp_is_special    = 0;
1713         qp->qp_uarpg         = uarpg;
1714         qp->qp_umap_dhp              = (devmap_cookie_t)NULL;
1715         qp->qp_sq_cqhdl              = sq_cq;
1716         qp->qp_sq_bufsz              = (1 << log_qp_sq_size);
1717         qp->qp_sq_logqsz     = log_qp_sq_size;
1718         qp->qp_sq_buf                = sq_buf;
1719         qp->qp_desc_off              = qp_desc_off;
1720         qp->qp_rq_cqhdl              = rq_cq;
1721         qp->qp_rq_buf                = rq_buf;
1722         qp->qp_rlky          = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
1723             0;
1724 
1725         /* if this QP is on an SRQ, set rq_bufsz to 0 */
1726         if (qp_srq_en) {
1727                 qp->qp_rq_bufsz              = 0;
1728                 qp->qp_rq_logqsz     = 0;
1729         } else {
1730                 qp->qp_rq_bufsz              = (1 << log_qp_rq_size);
1731                 qp->qp_rq_logqsz     = log_qp_rq_size;
1732         }
1733 
1734         qp->qp_forward_sqd_event  = 0;
1735         qp->qp_sqd_still_draining = 0;
1736         qp->qp_hdlrarg               = (void *)ibt_qphdl[ii];
1737         qp->qp_mcg_refcnt    = 0;
1738 
1739         /*
1740          * If this QP is to be associated with an SRQ, set the SRQ handle
1741          */
1742         if (qp_srq_en) {
1743                 qp->qp_srqhdl = srq;
1744                 hermon_srq_refcnt_inc(qp->qp_srqhdl);
1745         } else {
1746                 qp->qp_srqhdl = NULL;
1747         }
1748 
1749         qp->qp_type = IBT_UD_RQP;
1750         qp->qp_serv_type = serv_type;
1751 
1752         /*
1753          * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1754          */
1755 
1756         /*
1757          * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1758          * set the quadword to all F's - high-order bit is owner (init to one)
1759          * and the rest for the headroom definition of prefetching.
1760          */
1761         if ((attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) == 0) {
1762                 wqesz_shift = qp->qp_sq_log_wqesz;
1763                 thewqesz    = 1 << wqesz_shift;
1764                 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1765                 for (i = 0; i < sq_depth; i++) {
1766                         /*
1767                          * for each stride, go through and every 64 bytes
1768                          * write the init value - having set the address
1769                          * once, just keep incrementing it
1770                          */
1771                         for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1772                                 *(uint32_t *)thewqe = 0xFFFFFFFF;
1773                         }
1774                 }
1775         }
1776 
1777         /* Zero out the QP context */
1778         bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1779 
1780         /*
1781          * Put QP handle in Hermon QPNum-to-QPHdl list.  Then fill in the
1782          * "qphdl" and return success
1783          */
1784         hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + ii, qp);
1785 
1786         mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1787             DDI_INTR_PRI(state->hs_intrmsi_pri));
1788 
1789         qp->qp_rangep = qp_range_p;
1790 
1791         qphdl[ii] = qp;
1792 
1793         if (++ii < (1 << log2))
1794                 goto for_each_qp;
1795 
1796         return (DDI_SUCCESS);
1797 
1798 /*
1799  * The following is cleanup for all possible failure cases in this routine
1800  */
1801 qpalloc_fail9:
1802         hermon_queue_free(&qp->qp_wqinfo);
1803 qpalloc_fail8:
1804         if (qp->qp_sq_wqhdr)
1805                 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1806         if (qp->qp_rq_wqhdr)
1807                 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1808 qpalloc_fail7:
1809         if (!qp_srq_en) {
1810                 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1811         }
1812 
1813 qpalloc_fail6:
1814         hermon_rsrc_free(state, &rsrc);
1815 qpalloc_fail4:
1816         hermon_cq_refcnt_dec(rq_cq);
1817 qpalloc_fail2:
1818         hermon_cq_refcnt_dec(sq_cq);
1819 qpalloc_fail1:
1820         hermon_pd_refcnt_dec(pd);
1821 qpalloc_fail0:
1822         if (ii == 0) {
1823                 if (qp_range_p)
1824                         kmem_free(qp_range_p, sizeof (*qp_range_p));
1825                 hermon_rsrc_free(state, &qpc);
1826         } else {
1827                 /* qp_range_p and qpc rsrc will be freed in hermon_qp_free */
1828 
1829                 mutex_enter(&qp->qp_rangep->hqpr_lock);
1830                 qp_range_p->hqpr_refcnt = ii;
1831                 mutex_exit(&qp->qp_rangep->hqpr_lock);
1832                 while (--ii >= 0) {
1833                         ibc_qpn_hdl_t qpn_hdl;
1834                         int free_status;
1835 
1836                         free_status = hermon_qp_free(state, &qphdl[ii],
1837                             IBC_FREE_QP_AND_QPN, &qpn_hdl, sleepflag);
1838                         if (free_status != DDI_SUCCESS)
1839                                 cmn_err(CE_CONT, "!qp_range: status 0x%x: "
1840                                     "error status %x during free",
1841                                     status, free_status);
1842                 }
1843         }
1844 
1845         return (status);
1846 }
1847 
1848 
1849 /*
1850  * hermon_qp_free()
1851  *    This function frees up the QP resources.  Depending on the value
1852  *    of the "free_qp_flags", the QP number may not be released until
1853  *    a subsequent call to hermon_qp_release_qpn().
1854  *
1855  *    Context: Can be called only from user or kernel context.
1856  */
1857 /* ARGSUSED */
1858 int
1859 hermon_qp_free(hermon_state_t *state, hermon_qphdl_t *qphdl,
1860     ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh,
1861     uint_t sleepflag)
1862 {
1863         hermon_rsrc_t           *qpc, *rsrc;
1864         hermon_umap_db_entry_t  *umapdb;
1865         hermon_qpn_entry_t      *entry;
1866         hermon_pdhdl_t          pd;
1867         hermon_mrhdl_t          mr;
1868         hermon_cqhdl_t          sq_cq, rq_cq;
1869         hermon_srqhdl_t         srq;
1870         hermon_qphdl_t          qp;
1871         uint64_t                value;
1872         uint_t                  type, port;
1873         uint_t                  maxprot;
1874         uint_t                  qp_srq_en;
1875         int                     status;
1876 
1877         /*
1878          * Pull all the necessary information from the Hermon Queue Pair
1879          * handle.  This is necessary here because the resource for the
1880          * QP handle is going to be freed up as part of this operation.
1881          */
1882         qp      = *qphdl;
1883         mutex_enter(&qp->qp_lock);
1884         qpc     = qp->qp_qpcrsrcp;   /* NULL if part of a "range" */
1885         rsrc    = qp->qp_rsrcp;
1886         pd      = qp->qp_pdhdl;
1887         srq     = qp->qp_srqhdl;
1888         mr      = qp->qp_mrhdl;
1889         rq_cq   = qp->qp_rq_cqhdl;
1890         sq_cq   = qp->qp_sq_cqhdl;
1891         port    = qp->qp_portnum;
1892         qp_srq_en = qp->qp_alloc_flags & IBT_QP_USES_SRQ;
1893 
1894         /*
1895          * If the QP is part of an MCG, then we fail the qp_free
1896          */
1897         if (qp->qp_mcg_refcnt != 0) {
1898                 mutex_exit(&qp->qp_lock);
1899                 status = ibc_get_ci_failure(0);
1900                 goto qpfree_fail;
1901         }
1902 
1903         /*
1904          * If the QP is not already in "Reset" state, then transition to
1905          * "Reset".  This is necessary because software does not reclaim
1906          * ownership of the QP context until the QP is in the "Reset" state.
1907          * If the ownership transfer fails for any reason, then it is an
1908          * indication that something (either in HW or SW) has gone seriously
1909          * wrong.  So we print a warning message and return.
1910          */
1911         if (qp->qp_state != HERMON_QP_RESET) {
1912                 if (hermon_qp_to_reset(state, qp) != DDI_SUCCESS) {
1913                         mutex_exit(&qp->qp_lock);
1914                         HERMON_WARNING(state, "failed to reset QP context");
1915                         status = ibc_get_ci_failure(0);
1916                         goto qpfree_fail;
1917                 }
1918                 qp->qp_state = HERMON_QP_RESET;
1919                 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1920 
1921                 /*
1922                  * Do any additional handling necessary for the transition
1923                  * to the "Reset" state (e.g. update the WRID lists)
1924                  */
1925                 if (hermon_wrid_to_reset_handling(state, qp) != DDI_SUCCESS) {
1926                         mutex_exit(&qp->qp_lock);
1927                         HERMON_WARNING(state, "failed to reset QP WRID list");
1928                         status = ibc_get_ci_failure(0);
1929                         goto qpfree_fail;
1930                 }
1931         }
1932 
1933         /*
1934          * If this was a user-mappable QP, then we need to remove its entry
1935          * from the "userland resources database".  If it is also currently
1936          * mmap()'d out to a user process, then we need to call
1937          * devmap_devmem_remap() to remap the QP memory to an invalid mapping.
1938          * We also need to invalidate the QP tracking information for the
1939          * user mapping.
1940          */
1941         if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
1942                 status = hermon_umap_db_find(state->hs_instance, qp->qp_qpnum,
1943                     MLNX_UMAP_QPMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
1944                     &umapdb);
1945                 if (status != DDI_SUCCESS) {
1946                         mutex_exit(&qp->qp_lock);
1947                         HERMON_WARNING(state, "failed to find in database");
1948                         return (ibc_get_ci_failure(0));
1949                 }
1950                 hermon_umap_db_free(umapdb);
1951                 if (qp->qp_umap_dhp != NULL) {
1952                         maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1953                         status = devmap_devmem_remap(qp->qp_umap_dhp,
1954                             state->hs_dip, 0, 0, qp->qp_wqinfo.qa_size,
1955                             maxprot, DEVMAP_MAPPING_INVALID, NULL);
1956                         if (status != DDI_SUCCESS) {
1957                                 mutex_exit(&qp->qp_lock);
1958                                 HERMON_WARNING(state, "failed in QP memory "
1959                                     "devmap_devmem_remap()");
1960                                 return (ibc_get_ci_failure(0));
1961                         }
1962                         qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1963                 }
1964         }
1965 
1966 
1967         /*
1968          * Put NULL into the Hermon QPNum-to-QPHdl list.  This will allow any
1969          * in-progress events to detect that the QP corresponding to this
1970          * number has been freed.  Note: it does depend in whether we are
1971          * freeing a special QP or not.
1972          */
1973         if (qpc == NULL) {
1974                 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1975                     qp->qp_qpnum, NULL);
1976         } else if (qp->qp_is_special) {
1977                 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1978                     qpc->hr_indx + port, NULL);
1979         } else {
1980                 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1981                     qpc->hr_indx, NULL);
1982         }
1983 
1984         /*
1985          * Drop the QP lock
1986          *    At this point the lock is no longer necessary.  We cannot
1987          *    protect from multiple simultaneous calls to free the same QP.
1988          *    In addition, since the QP lock is contained in the QP "software
1989          *    handle" resource, which we will free (see below), it is
1990          *    important that we have no further references to that memory.
1991          */
1992         mutex_exit(&qp->qp_lock);
1993         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1994 
1995         /*
1996          * Free the QP resources
1997          *    Start by deregistering and freeing the memory for work queues.
1998          *    Next free any previously allocated context information
1999          *    (depending on QP type)
2000          *    Finally, decrement the necessary reference counts.
2001          * If this fails for any reason, then it is an indication that
2002          * something (either in HW or SW) has gone seriously wrong.  So we
2003          * print a warning message and return.
2004          */
2005         status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2006             sleepflag);
2007         if (status != DDI_SUCCESS) {
2008                 HERMON_WARNING(state, "failed to deregister QP memory");
2009                 status = ibc_get_ci_failure(0);
2010                 goto qpfree_fail;
2011         }
2012 
2013         /* Free the memory for the QP */
2014         hermon_queue_free(&qp->qp_wqinfo);
2015 
2016         if (qp->qp_sq_wqhdr)
2017                 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
2018         if (qp->qp_rq_wqhdr)
2019                 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
2020 
2021         /* Free the dbr */
2022         if (!qp_srq_en) {
2023                 hermon_dbr_free(state, qp->qp_uarpg, qp->qp_rq_vdbr);
2024         }
2025 
2026         /*
2027          * Free up the remainder of the QP resources.  Note: we have a few
2028          * different resources to free up depending on whether the QP is a
2029          * special QP or not.  As described above, if any of these fail for
2030          * any reason it is an indication that something (either in HW or SW)
2031          * has gone seriously wrong.  So we print a warning message and
2032          * return.
2033          */
2034         if (qp->qp_is_special) {
2035                 type = (qp->qp_is_special == HERMON_QP_SMI) ?
2036                     IBT_SMI_SQP : IBT_GSI_SQP;
2037 
2038                 /* Free up resources for the special QP */
2039                 status = hermon_special_qp_rsrc_free(state, type, port);
2040                 if (status != DDI_SUCCESS) {
2041                         HERMON_WARNING(state, "failed to free special QP rsrc");
2042                         status = ibc_get_ci_failure(0);
2043                         goto qpfree_fail;
2044                 }
2045 
2046         } else if (qp->qp_rangep) {
2047                 int refcnt;
2048                 mutex_enter(&qp->qp_rangep->hqpr_lock);
2049                 refcnt = --qp->qp_rangep->hqpr_refcnt;
2050                 mutex_exit(&qp->qp_rangep->hqpr_lock);
2051                 if (refcnt == 0) {
2052                         mutex_destroy(&qp->qp_rangep->hqpr_lock);
2053                         hermon_rsrc_free(state, &qp->qp_rangep->hqpr_qpcrsrc);
2054                         kmem_free(qp->qp_rangep, sizeof (*qp->qp_rangep));
2055                 }
2056                 qp->qp_rangep = NULL;
2057         } else if (qp->qp_qpn_hdl == NULL) {
2058                 hermon_rsrc_free(state, &qpc);
2059         } else {
2060                 /*
2061                  * Check the flags and determine whether to release the
2062                  * QPN or not, based on their value.
2063                  */
2064                 if (free_qp_flags == IBC_FREE_QP_ONLY) {
2065                         entry = qp->qp_qpn_hdl;
2066                         hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2067                             HERMON_QPN_FREE_ONLY);
2068                         *qpnh = (ibc_qpn_hdl_t)entry;
2069                 } else {
2070                         hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2071                             HERMON_QPN_RELEASE);
2072                 }
2073         }
2074 
2075         mutex_destroy(&qp->qp_sq_lock);
2076 
2077         /* Free the Hermon Queue Pair handle */
2078         hermon_rsrc_free(state, &rsrc);
2079 
2080         /* Decrement the reference counts on CQs, PD and SRQ (if needed) */
2081         hermon_cq_refcnt_dec(rq_cq);
2082         hermon_cq_refcnt_dec(sq_cq);
2083         hermon_pd_refcnt_dec(pd);
2084         if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
2085                 hermon_srq_refcnt_dec(srq);
2086         }
2087 
2088         /* Set the qphdl pointer to NULL and return success */
2089         *qphdl = NULL;
2090 
2091         return (DDI_SUCCESS);
2092 
2093 qpfree_fail:
2094         return (status);
2095 }
2096 
2097 
2098 /*
2099  * hermon_qp_query()
2100  *    Context: Can be called from interrupt or base context.
2101  */
2102 int
2103 hermon_qp_query(hermon_state_t *state, hermon_qphdl_t qp,
2104     ibt_qp_query_attr_t *attr_p)
2105 {
2106         ibt_cep_state_t         qp_state;
2107         ibt_qp_ud_attr_t        *ud;
2108         ibt_qp_rc_attr_t        *rc;
2109         ibt_qp_uc_attr_t        *uc;
2110         ibt_cep_flags_t         enable_flags;
2111         hermon_hw_addr_path_t   *qpc_path, *qpc_alt_path;
2112         ibt_cep_path_t          *path_ptr, *alt_path_ptr;
2113         hermon_hw_qpc_t         *qpc;
2114         int                     status;
2115         uint_t                  tmp_sched_q, tmp_alt_sched_q;
2116 
2117         mutex_enter(&qp->qp_lock);
2118 
2119         /*
2120          * Grab the temporary QPC entry from QP software state
2121          */
2122         qpc = &qp->qpc;
2123 
2124         /* Convert the current Hermon QP state to IBTF QP state */
2125         switch (qp->qp_state) {
2126         case HERMON_QP_RESET:
2127                 qp_state = IBT_STATE_RESET;             /* "Reset" */
2128                 break;
2129         case HERMON_QP_INIT:
2130                 qp_state = IBT_STATE_INIT;              /* Initialized */
2131                 break;
2132         case HERMON_QP_RTR:
2133                 qp_state = IBT_STATE_RTR;               /* Ready to Receive */
2134                 break;
2135         case HERMON_QP_RTS:
2136                 qp_state = IBT_STATE_RTS;               /* Ready to Send */
2137                 break;
2138         case HERMON_QP_SQERR:
2139                 qp_state = IBT_STATE_SQE;               /* Send Queue Error */
2140                 break;
2141         case HERMON_QP_SQD:
2142                 if (qp->qp_sqd_still_draining) {
2143                         qp_state = IBT_STATE_SQDRAIN;   /* SQ Draining */
2144                 } else {
2145                         qp_state = IBT_STATE_SQD;       /* SQ Drained */
2146                 }
2147                 break;
2148         case HERMON_QP_ERR:
2149                 qp_state = IBT_STATE_ERROR;             /* Error */
2150                 break;
2151         default:
2152                 mutex_exit(&qp->qp_lock);
2153                 return (ibc_get_ci_failure(0));
2154         }
2155         attr_p->qp_info.qp_state = qp_state;
2156 
2157         /* SRQ Hook. */
2158         attr_p->qp_srq = NULL;
2159 
2160         /*
2161          * The following QP information is always returned, regardless of
2162          * the current QP state.  Note: Some special handling is necessary
2163          * for calculating the QP number on special QP (QP0 and QP1).
2164          */
2165         attr_p->qp_sq_cq    =
2166             (qp->qp_sq_cqhdl == NULL) ? NULL : qp->qp_sq_cqhdl->cq_hdlrarg;
2167         attr_p->qp_rq_cq    =
2168             (qp->qp_rq_cqhdl == NULL) ? NULL : qp->qp_rq_cqhdl->cq_hdlrarg;
2169         if (qp->qp_is_special) {
2170                 attr_p->qp_qpn = (qp->qp_is_special == HERMON_QP_SMI) ? 0 : 1;
2171         } else {
2172                 attr_p->qp_qpn = (ib_qpn_t)qp->qp_qpnum;
2173         }
2174         attr_p->qp_sq_sgl   = qp->qp_sq_sgl;
2175         attr_p->qp_rq_sgl   = qp->qp_rq_sgl;
2176         attr_p->qp_info.qp_sq_sz = qp->qp_sq_bufsz - qp->qp_sq_hdrmwqes;
2177         attr_p->qp_info.qp_rq_sz = qp->qp_rq_bufsz;
2178 
2179         /*
2180          * If QP is currently in the "Reset" state, then only the above are
2181          * returned
2182          */
2183         if (qp_state == IBT_STATE_RESET) {
2184                 mutex_exit(&qp->qp_lock);
2185                 return (DDI_SUCCESS);
2186         }
2187 
2188         /*
2189          * Post QUERY_QP command to firmware
2190          *
2191          * We do a HERMON_NOSLEEP here because we are holding the "qp_lock".
2192          * Since we may be in the interrupt context (or subsequently raised
2193          * to interrupt level by priority inversion), we do not want to block
2194          * in this routine waiting for success.
2195          */
2196         tmp_sched_q = qpc->pri_addr_path.sched_q;
2197         tmp_alt_sched_q = qpc->alt_addr_path.sched_q;
2198         status = hermon_cmn_query_cmd_post(state, QUERY_QP, 0, qp->qp_qpnum,
2199             qpc, sizeof (hermon_hw_qpc_t), HERMON_CMD_NOSLEEP_SPIN);
2200         if (status != HERMON_CMD_SUCCESS) {
2201                 mutex_exit(&qp->qp_lock);
2202                 cmn_err(CE_WARN, "hermon%d: hermon_qp_query: QUERY_QP "
2203                     "command failed: %08x\n", state->hs_instance, status);
2204                 if (status == HERMON_CMD_INVALID_STATUS) {
2205                         hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2206                 }
2207                 return (ibc_get_ci_failure(0));
2208         }
2209         qpc->pri_addr_path.sched_q = tmp_sched_q;
2210         qpc->alt_addr_path.sched_q = tmp_alt_sched_q;
2211 
2212         /*
2213          * Fill in the additional QP info based on the QP's transport type.
2214          */
2215         if (qp->qp_type == IBT_UD_RQP) {
2216 
2217                 /* Fill in the UD-specific info */
2218                 ud = &attr_p->qp_info.qp_transport.ud;
2219                 ud->ud_qkey  = (ib_qkey_t)qpc->qkey;
2220                 ud->ud_sq_psn        = qpc->next_snd_psn;
2221                 ud->ud_pkey_ix       = qpc->pri_addr_path.pkey_indx;
2222                 /* port+1 for port 1/2 */
2223                 ud->ud_port  =
2224                     (uint8_t)(((qpc->pri_addr_path.sched_q >> 6) & 0x01) + 1);
2225 
2226                 attr_p->qp_info.qp_trans = IBT_UD_SRV;
2227 
2228                 if (qp->qp_serv_type == HERMON_QP_FEXCH) {
2229                         ibt_pmr_desc_t *pmr;
2230                         uint64_t heart_beat;
2231 
2232                         _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
2233                         pmr = &attr_p->qp_query_fexch.fq_uni_mem_desc;
2234                         pmr->pmd_iova = 0;
2235                         pmr->pmd_lkey = pmr->pmd_rkey =
2236                             hermon_fcoib_qpn_to_mkey(state, qp->qp_qpnum);
2237                         pmr->pmd_phys_buf_list_sz =
2238                             state->hs_fcoib.hfc_mtts_per_mpt;
2239                         pmr->pmd_sync_required = 0;
2240 
2241                         pmr = &attr_p->qp_query_fexch.fq_bi_mem_desc;
2242                         pmr->pmd_iova = 0;
2243                         pmr->pmd_lkey = 0;
2244                         pmr->pmd_rkey = 0;
2245                         pmr->pmd_phys_buf_list_sz = 0;
2246                         pmr->pmd_sync_required = 0;
2247 
2248                         attr_p->qp_query_fexch.fq_flags =
2249                             ((hermon_get_heart_beat_rq_cmd_post(state,
2250                             qp->qp_qpnum, &heart_beat) == HERMON_CMD_SUCCESS) &&
2251                             (heart_beat == 0)) ? IBT_FEXCH_HEART_BEAT_OK :
2252                             IBT_FEXCH_NO_FLAGS;
2253 
2254                         ud->ud_fc = qp->qp_fc_attr;
2255                 } else if (qp->qp_serv_type == HERMON_QP_FCMND ||
2256                     qp->qp_serv_type == HERMON_QP_RFCI) {
2257                         ud->ud_fc = qp->qp_fc_attr;
2258                 }
2259 
2260         } else if (qp->qp_serv_type == HERMON_QP_RC) {
2261 
2262                 /* Fill in the RC-specific info */
2263                 rc = &attr_p->qp_info.qp_transport.rc;
2264                 rc->rc_sq_psn        = qpc->next_snd_psn;
2265                 rc->rc_rq_psn        = qpc->next_rcv_psn;
2266                 rc->rc_dst_qpn       = qpc->rem_qpn;
2267 
2268                 /* Grab the path migration state information */
2269                 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2270                         rc->rc_mig_state = IBT_STATE_MIGRATED;
2271                 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2272                         rc->rc_mig_state = IBT_STATE_REARMED;
2273                 } else {
2274                         rc->rc_mig_state = IBT_STATE_ARMED;
2275                 }
2276                 rc->rc_rdma_ra_out = (1 << qpc->sra_max);
2277                 rc->rc_rdma_ra_in  = (1 << qpc->rra_max);
2278                 rc->rc_min_rnr_nak = qpc->min_rnr_nak;
2279                 rc->rc_path_mtu         = qpc->mtu;
2280                 rc->rc_retry_cnt   = qpc->retry_cnt;
2281 
2282                 /* Get the common primary address path fields */
2283                 qpc_path = &qpc->pri_addr_path;
2284                 path_ptr = &rc->rc_path;
2285                 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2286                     HERMON_ADDRPATH_QP);
2287 
2288                 /* Fill in the additional primary address path fields */
2289                 path_ptr->cep_pkey_ix           = qpc_path->pkey_indx;
2290                 path_ptr->cep_hca_port_num =
2291                     path_ptr->cep_adds_vect.av_port_num =
2292                     (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2293                 path_ptr->cep_timeout           = qpc_path->ack_timeout;
2294 
2295                 /* Get the common alternate address path fields */
2296                 qpc_alt_path = &qpc->alt_addr_path;
2297                 alt_path_ptr = &rc->rc_alt_path;
2298                 hermon_get_addr_path(state, qpc_alt_path,
2299                     &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2300 
2301                 /* Fill in the additional alternate address path fields */
2302                 alt_path_ptr->cep_pkey_ix    = qpc_alt_path->pkey_indx;
2303                 alt_path_ptr->cep_hca_port_num       =
2304                     alt_path_ptr->cep_adds_vect.av_port_num =
2305                     (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2306                 alt_path_ptr->cep_timeout    = qpc_alt_path->ack_timeout;
2307 
2308                 /* Get the RNR retry time from primary path */
2309                 rc->rc_rnr_retry_cnt = qpc->rnr_retry;
2310 
2311                 /* Set the enable flags based on RDMA/Atomic enable bits */
2312                 enable_flags = IBT_CEP_NO_FLAGS;
2313                 enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD);
2314                 enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2315                 enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC);
2316                 attr_p->qp_info.qp_flags = enable_flags;
2317 
2318                 attr_p->qp_info.qp_trans = IBT_RC_SRV;
2319 
2320         } else if (qp->qp_serv_type == HERMON_QP_UC) {
2321 
2322                 /* Fill in the UC-specific info */
2323                 uc = &attr_p->qp_info.qp_transport.uc;
2324                 uc->uc_sq_psn        = qpc->next_snd_psn;
2325                 uc->uc_rq_psn        = qpc->next_rcv_psn;
2326                 uc->uc_dst_qpn       = qpc->rem_qpn;
2327 
2328                 /* Grab the path migration state information */
2329                 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2330                         uc->uc_mig_state = IBT_STATE_MIGRATED;
2331                 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2332                         uc->uc_mig_state = IBT_STATE_REARMED;
2333                 } else {
2334                         uc->uc_mig_state = IBT_STATE_ARMED;
2335                 }
2336                 uc->uc_path_mtu = qpc->mtu;
2337 
2338                 /* Get the common primary address path fields */
2339                 qpc_path = &qpc->pri_addr_path;
2340                 path_ptr = &uc->uc_path;
2341                 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2342                     HERMON_ADDRPATH_QP);
2343 
2344                 /* Fill in the additional primary address path fields */
2345                 path_ptr->cep_pkey_ix           = qpc_path->pkey_indx;
2346                 path_ptr->cep_hca_port_num =
2347                     path_ptr->cep_adds_vect.av_port_num =
2348                     (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2349 
2350                 /* Get the common alternate address path fields */
2351                 qpc_alt_path = &qpc->alt_addr_path;
2352                 alt_path_ptr = &uc->uc_alt_path;
2353                 hermon_get_addr_path(state, qpc_alt_path,
2354                     &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2355 
2356                 /* Fill in the additional alternate address path fields */
2357                 alt_path_ptr->cep_pkey_ix    = qpc_alt_path->pkey_indx;
2358                 alt_path_ptr->cep_hca_port_num       =
2359                     alt_path_ptr->cep_adds_vect.av_port_num =
2360                     (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2361 
2362                 /*
2363                  * Set the enable flags based on RDMA enable bits (by
2364                  * definition UC doesn't support Atomic or RDMA Read)
2365                  */
2366                 enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2367                 attr_p->qp_info.qp_flags = enable_flags;
2368 
2369                 attr_p->qp_info.qp_trans = IBT_UC_SRV;
2370 
2371         } else {
2372                 HERMON_WARNING(state, "unexpected QP transport type");
2373                 mutex_exit(&qp->qp_lock);
2374                 return (ibc_get_ci_failure(0));
2375         }
2376 
2377         /*
2378          * Under certain circumstances it is possible for the Hermon hardware
2379          * to transition to one of the error states without software directly
2380          * knowing about it.  The QueryQP() call is the one place where we
2381          * have an opportunity to sample and update our view of the QP state.
2382          */
2383         if (qpc->state == HERMON_QP_SQERR) {
2384                 attr_p->qp_info.qp_state = IBT_STATE_SQE;
2385                 qp->qp_state = HERMON_QP_SQERR;
2386                 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_SQERR);
2387         }
2388         if (qpc->state == HERMON_QP_ERR) {
2389                 attr_p->qp_info.qp_state = IBT_STATE_ERROR;
2390                 qp->qp_state = HERMON_QP_ERR;
2391                 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_ERR);
2392         }
2393         mutex_exit(&qp->qp_lock);
2394 
2395         return (DDI_SUCCESS);
2396 }
2397 
2398 
2399 /*
2400  * hermon_qp_create_qpn()
2401  *    Context: Can be called from interrupt or base context.
2402  */
2403 static int
2404 hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
2405     hermon_rsrc_t *qpc)
2406 {
2407         hermon_qpn_entry_t      query;
2408         hermon_qpn_entry_t      *entry;
2409         avl_index_t             where;
2410 
2411         /*
2412          * Build a query (for the AVL tree lookup) and attempt to find
2413          * a previously added entry that has a matching QPC index.  If
2414          * no matching entry is found, then allocate, initialize, and
2415          * add an entry to the AVL tree.
2416          * If a matching entry is found, then increment its QPN counter
2417          * and reference counter.
2418          */
2419         query.qpn_indx = qpc->hr_indx;
2420         mutex_enter(&state->hs_qpn_avl_lock);
2421         entry = (hermon_qpn_entry_t *)avl_find(&state->hs_qpn_avl,
2422             &query, &where);
2423         if (entry == NULL) {
2424                 /*
2425                  * Allocate and initialize a QPN entry, then insert
2426                  * it into the AVL tree.
2427                  */
2428                 entry = (hermon_qpn_entry_t *)kmem_zalloc(
2429                     sizeof (hermon_qpn_entry_t), KM_NOSLEEP);
2430                 if (entry == NULL) {
2431                         mutex_exit(&state->hs_qpn_avl_lock);
2432                         return (DDI_FAILURE);
2433                 }
2434                 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
2435 
2436                 entry->qpn_indx         = qpc->hr_indx;
2437                 entry->qpn_refcnt  = 0;
2438                 entry->qpn_counter = 0;
2439 
2440                 avl_insert(&state->hs_qpn_avl, entry, where);
2441         }
2442 
2443         /*
2444          * Make the AVL tree entry point to the QP context resource that
2445          * it will be responsible for tracking
2446          */
2447         entry->qpn_qpc = qpc;
2448 
2449         /*
2450          * Setup the QP handle to point to the AVL tree entry.  Then
2451          * generate the new QP number from the entry's QPN counter value
2452          * and the hardware's QP context table index.
2453          */
2454         qp->qp_qpn_hdl       = entry;
2455         qp->qp_qpnum = ((entry->qpn_counter <<
2456             state->hs_cfg_profile->cp_log_num_qp) | qpc->hr_indx) &
2457             HERMON_QP_MAXNUMBER_MSK;
2458         qp->qp_ring = qp->qp_qpnum << 8;
2459 
2460         /*
2461          * Increment the reference counter and QPN counter.  The QPN
2462          * counter always indicates the next available number for use.
2463          */
2464         entry->qpn_counter++;
2465         entry->qpn_refcnt++;
2466 
2467         mutex_exit(&state->hs_qpn_avl_lock);
2468 
2469         return (DDI_SUCCESS);
2470 }
2471 
2472 
2473 /*
2474  * hermon_qp_release_qpn()
2475  *    Context: Can be called only from user or kernel context.
2476  */
2477 void
2478 hermon_qp_release_qpn(hermon_state_t *state, hermon_qpn_entry_t *entry,
2479     int flags)
2480 {
2481         ASSERT(entry != NULL);
2482 
2483         mutex_enter(&state->hs_qpn_avl_lock);
2484 
2485         /*
2486          * If we are releasing the QP number here, then we decrement the
2487          * reference count and check for zero references.  If there are
2488          * zero references, then we free the QPC context (if it hadn't
2489          * already been freed during a HERMON_QPN_FREE_ONLY free, i.e. for
2490          * reuse with another similar QP number) and remove the tracking
2491          * structure from the QP number AVL tree and free the structure.
2492          * If we are not releasing the QP number here, then, as long as we
2493          * have not exhausted the usefulness of the QPC context (that is,
2494          * re-used it too many times without the reference count having
2495          * gone to zero), we free up the QPC context for use by another
2496          * thread (which will use it to construct a different QP number
2497          * from the same QPC table index).
2498          */
2499         if (flags == HERMON_QPN_RELEASE) {
2500                 entry->qpn_refcnt--;
2501 
2502                 /*
2503                  * If the reference count is zero, then we free the QPC
2504                  * context (if it hadn't already been freed in an early
2505                  * step, e.g. HERMON_QPN_FREE_ONLY) and remove/free the
2506                  * tracking structure from the QP number AVL tree.
2507                  */
2508                 if (entry->qpn_refcnt == 0) {
2509                         if (entry->qpn_qpc != NULL) {
2510                                 hermon_rsrc_free(state, &entry->qpn_qpc);
2511                         }
2512 
2513                         /*
2514                          * If the current entry has served it's useful
2515                          * purpose (i.e. been reused the maximum allowable
2516                          * number of times), then remove it from QP number
2517                          * AVL tree and free it up.
2518                          */
2519                         if (entry->qpn_counter >= (1 <<
2520                             (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2521                                 avl_remove(&state->hs_qpn_avl, entry);
2522                                 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2523                         }
2524                 }
2525 
2526         } else if (flags == HERMON_QPN_FREE_ONLY) {
2527                 /*
2528                  * Even if we are not freeing the QP number, that will not
2529                  * always prevent us from releasing the QPC context.  In fact,
2530                  * since the QPC context only forms part of the whole QPN,
2531                  * we want to free it up for use by other consumers.  But
2532                  * if the reference count is non-zero (which it will always
2533                  * be when we are doing HERMON_QPN_FREE_ONLY) and the counter
2534                  * has reached its maximum value, then we cannot reuse the
2535                  * QPC context until the reference count eventually reaches
2536                  * zero (in HERMON_QPN_RELEASE, above).
2537                  */
2538                 if (entry->qpn_counter < (1 <<
2539                     (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2540                         hermon_rsrc_free(state, &entry->qpn_qpc);
2541                 }
2542         }
2543         mutex_exit(&state->hs_qpn_avl_lock);
2544 }
2545 
2546 
2547 /*
2548  * hermon_qpn_avl_compare()
2549  *    Context: Can be called from user or kernel context.
2550  */
2551 static int
2552 hermon_qpn_avl_compare(const void *q, const void *e)
2553 {
2554         hermon_qpn_entry_t      *entry, *query;
2555 
2556         entry = (hermon_qpn_entry_t *)e;
2557         query = (hermon_qpn_entry_t *)q;
2558 
2559         if (query->qpn_indx < entry->qpn_indx) {
2560                 return (-1);
2561         } else if (query->qpn_indx > entry->qpn_indx) {
2562                 return (+1);
2563         } else {
2564                 return (0);
2565         }
2566 }
2567 
2568 
2569 /*
2570  * hermon_qpn_avl_init()
2571  *    Context: Only called from attach() path context
2572  */
2573 void
2574 hermon_qpn_avl_init(hermon_state_t *state)
2575 {
2576         /* Initialize the lock used for QP number (QPN) AVL tree access */
2577         mutex_init(&state->hs_qpn_avl_lock, NULL, MUTEX_DRIVER,
2578             DDI_INTR_PRI(state->hs_intrmsi_pri));
2579 
2580         /* Initialize the AVL tree for the QP number (QPN) storage */
2581         avl_create(&state->hs_qpn_avl, hermon_qpn_avl_compare,
2582             sizeof (hermon_qpn_entry_t),
2583             offsetof(hermon_qpn_entry_t, qpn_avlnode));
2584 }
2585 
2586 
2587 /*
2588  * hermon_qpn_avl_fini()
2589  *    Context: Only called from attach() and/or detach() path contexts
2590  */
2591 void
2592 hermon_qpn_avl_fini(hermon_state_t *state)
2593 {
2594         hermon_qpn_entry_t      *entry;
2595         void                    *cookie;
2596 
2597         /*
2598          * Empty all entries (if necessary) and destroy the AVL tree
2599          * that was used for QP number (QPN) tracking.
2600          */
2601         cookie = NULL;
2602         while ((entry = (hermon_qpn_entry_t *)avl_destroy_nodes(
2603             &state->hs_qpn_avl, &cookie)) != NULL) {
2604                 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2605         }
2606         avl_destroy(&state->hs_qpn_avl);
2607 
2608         /* Destroy the lock used for QP number (QPN) AVL tree access */
2609         mutex_destroy(&state->hs_qpn_avl_lock);
2610 }
2611 
2612 
2613 /*
2614  * hermon_qphdl_from_qpnum()
2615  *    Context: Can be called from interrupt or base context.
2616  *
2617  *    This routine is important because changing the unconstrained
2618  *    portion of the QP number is critical to the detection of a
2619  *    potential race condition in the QP event handler code (i.e. the case
2620  *    where a QP is freed and alloc'd again before an event for the
2621  *    "old" QP can be handled).
2622  *
2623  *    While this is not a perfect solution (not sure that one exists)
2624  *    it does help to mitigate the chance that this race condition will
2625  *    cause us to deliver a "stale" event to the new QP owner.  Note:
2626  *    this solution does not scale well because the number of constrained
2627  *    bits increases (and, hence, the number of unconstrained bits
2628  *    decreases) as the number of supported QPs grows.  For small and
2629  *    intermediate values, it should hopefully provide sufficient
2630  *    protection.
2631  */
2632 hermon_qphdl_t
2633 hermon_qphdl_from_qpnum(hermon_state_t *state, uint_t qpnum)
2634 {
2635         uint_t  qpindx, qpmask;
2636 
2637         /* Calculate the QP table index from the qpnum */
2638         qpmask = (1 << state->hs_cfg_profile->cp_log_num_qp) - 1;
2639         qpindx = qpnum & qpmask;
2640         return (hermon_icm_num_to_hdl(state, HERMON_QPC, qpindx));
2641 }
2642 
2643 
2644 /*
2645  * hermon_special_qp_rsrc_alloc
2646  *    Context: Can be called from interrupt or base context.
2647  */
2648 static int
2649 hermon_special_qp_rsrc_alloc(hermon_state_t *state, ibt_sqp_type_t type,
2650     uint_t port, hermon_rsrc_t **qp_rsrc)
2651 {
2652         uint_t          mask, flags;
2653         int             status;
2654 
2655         mutex_enter(&state->hs_spec_qplock);
2656         flags = state->hs_spec_qpflags;
2657         if (type == IBT_SMI_SQP) {
2658                 /*
2659                  * Check here to see if the driver has been configured
2660                  * to instruct the Hermon firmware to handle all incoming
2661                  * SMP messages (i.e. messages sent to SMA).  If so,
2662                  * then we will treat QP0 as if it has already been
2663                  * allocated (for internal use).  Otherwise, if we allow
2664                  * the allocation to happen, it will cause unexpected
2665                  * behaviors (e.g. Hermon SMA becomes unresponsive).
2666                  */
2667                 if (state->hs_cfg_profile->cp_qp0_agents_in_fw != 0) {
2668                         mutex_exit(&state->hs_spec_qplock);
2669                         return (IBT_QP_IN_USE);
2670                 }
2671 
2672                 /*
2673                  * If this is the first QP0 allocation, then post
2674                  * a CONF_SPECIAL_QP firmware command
2675                  */
2676                 if ((flags & HERMON_SPECIAL_QP0_RSRC_MASK) == 0) {
2677                         status = hermon_conf_special_qp_cmd_post(state,
2678                             state->hs_spec_qp0->hr_indx, HERMON_CMD_QP_SMI,
2679                             HERMON_CMD_NOSLEEP_SPIN,
2680                             HERMON_CMD_SPEC_QP_OPMOD(
2681                             state->hs_cfg_profile->cp_qp0_agents_in_fw,
2682                             state->hs_cfg_profile->cp_qp1_agents_in_fw));
2683                         if (status != HERMON_CMD_SUCCESS) {
2684                                 mutex_exit(&state->hs_spec_qplock);
2685                                 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2686                                     "command failed: %08x\n",
2687                                     state->hs_instance, status);
2688                                 return (IBT_INSUFF_RESOURCE);
2689                         }
2690                 }
2691 
2692                 /*
2693                  * Now check (and, if necessary, modify) the flags to indicate
2694                  * whether the allocation was successful
2695                  */
2696                 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2697                 if (flags & mask) {
2698                         mutex_exit(&state->hs_spec_qplock);
2699                         return (IBT_QP_IN_USE);
2700                 }
2701                 state->hs_spec_qpflags |= mask;
2702                 *qp_rsrc = state->hs_spec_qp0;
2703 
2704         } else {
2705                 /*
2706                  * If this is the first QP1 allocation, then post
2707                  * a CONF_SPECIAL_QP firmware command
2708                  */
2709                 if ((flags & HERMON_SPECIAL_QP1_RSRC_MASK) == 0) {
2710                         status = hermon_conf_special_qp_cmd_post(state,
2711                             state->hs_spec_qp1->hr_indx, HERMON_CMD_QP_GSI,
2712                             HERMON_CMD_NOSLEEP_SPIN,
2713                             HERMON_CMD_SPEC_QP_OPMOD(
2714                             state->hs_cfg_profile->cp_qp0_agents_in_fw,
2715                             state->hs_cfg_profile->cp_qp1_agents_in_fw));
2716                         if (status != HERMON_CMD_SUCCESS) {
2717                                 mutex_exit(&state->hs_spec_qplock);
2718                                 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2719                                     "command failed: %08x\n",
2720                                     state->hs_instance, status);
2721                                 return (IBT_INSUFF_RESOURCE);
2722                         }
2723                 }
2724 
2725                 /*
2726                  * Now check (and, if necessary, modify) the flags to indicate
2727                  * whether the allocation was successful
2728                  */
2729                 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2730                 if (flags & mask) {
2731                         mutex_exit(&state->hs_spec_qplock);
2732                         return (IBT_QP_IN_USE);
2733                 }
2734                 state->hs_spec_qpflags |= mask;
2735                 *qp_rsrc = state->hs_spec_qp1;
2736         }
2737 
2738         mutex_exit(&state->hs_spec_qplock);
2739         return (DDI_SUCCESS);
2740 }
2741 
2742 
2743 /*
2744  * hermon_special_qp_rsrc_free
2745  *    Context: Can be called from interrupt or base context.
2746  */
2747 static int
2748 hermon_special_qp_rsrc_free(hermon_state_t *state, ibt_sqp_type_t type,
2749     uint_t port)
2750 {
2751         uint_t          mask, flags;
2752         int             status;
2753 
2754         mutex_enter(&state->hs_spec_qplock);
2755         if (type == IBT_SMI_SQP) {
2756                 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2757                 state->hs_spec_qpflags &= ~mask;
2758                 flags = state->hs_spec_qpflags;
2759 
2760                 /*
2761                  * If this is the last QP0 free, then post a CONF_SPECIAL_QP
2762                  * NOW, If this is the last Special QP free, then post a
2763                  * CONF_SPECIAL_QP firmware command - it'll stop them all
2764                  */
2765                 if (flags) {
2766                         status = hermon_conf_special_qp_cmd_post(state, 0,
2767                             HERMON_CMD_QP_SMI, HERMON_CMD_NOSLEEP_SPIN, 0);
2768                         if (status != HERMON_CMD_SUCCESS) {
2769                                 mutex_exit(&state->hs_spec_qplock);
2770                                 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2771                                     "command failed: %08x\n",
2772                                     state->hs_instance, status);
2773                                 if (status == HERMON_CMD_INVALID_STATUS) {
2774                                         hermon_fm_ereport(state, HCA_SYS_ERR,
2775                                             HCA_ERR_SRV_LOST);
2776                                 }
2777                                 return (ibc_get_ci_failure(0));
2778                         }
2779                 }
2780         } else {
2781                 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2782                 state->hs_spec_qpflags &= ~mask;
2783                 flags = state->hs_spec_qpflags;
2784 
2785                 /*
2786                  * If this is the last QP1 free, then post a CONF_SPECIAL_QP
2787                  * NOW, if this is the last special QP free, then post a
2788                  * CONF_SPECIAL_QP firmware command - it'll stop them all
2789                  */
2790                 if (flags) {
2791                         status = hermon_conf_special_qp_cmd_post(state, 0,
2792                             HERMON_CMD_QP_GSI, HERMON_CMD_NOSLEEP_SPIN, 0);
2793                         if (status != HERMON_CMD_SUCCESS) {
2794                                 mutex_exit(&state->hs_spec_qplock);
2795                                 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2796                                     "command failed: %08x\n",
2797                                     state->hs_instance, status);
2798                                 if (status == HERMON_CMD_INVALID_STATUS) {
2799                                         hermon_fm_ereport(state, HCA_SYS_ERR,
2800                                             HCA_ERR_SRV_LOST);
2801                                 }
2802                                 return (ibc_get_ci_failure(0));
2803                         }
2804                 }
2805         }
2806 
2807         mutex_exit(&state->hs_spec_qplock);
2808         return (DDI_SUCCESS);
2809 }
2810 
2811 
2812 /*
2813  * hermon_qp_sgl_to_logwqesz()
2814  *    Context: Can be called from interrupt or base context.
2815  */
2816 static void
2817 hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
2818     uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
2819     uint_t *logwqesz, uint_t *max_sgl)
2820 {
2821         uint_t  max_size, log2, actual_sgl;
2822 
2823         switch (wq_type) {
2824         case HERMON_QP_WQ_TYPE_SENDQ_UD:
2825                 /*
2826                  * Use requested maximum SGL to calculate max descriptor size
2827                  * (while guaranteeing that the descriptor size is a
2828                  * power-of-2 cachelines).
2829                  */
2830                 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2831                 log2 = highbit(max_size);
2832                 if (ISP2(max_size)) {
2833                         log2 = log2 - 1;
2834                 }
2835 
2836                 /* Make sure descriptor is at least the minimum size */
2837                 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2838 
2839                 /* Calculate actual number of SGL (given WQE size) */
2840                 actual_sgl = ((1 << log2) -
2841                     sizeof (hermon_hw_snd_wqe_ctrl_t)) >> 4;
2842                 break;
2843 
2844         case HERMON_QP_WQ_TYPE_SENDQ_CONN:
2845                 /*
2846                  * Use requested maximum SGL to calculate max descriptor size
2847                  * (while guaranteeing that the descriptor size is a
2848                  * power-of-2 cachelines).
2849                  */
2850                 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2851                 log2 = highbit(max_size);
2852                 if (ISP2(max_size)) {
2853                         log2 = log2 - 1;
2854                 }
2855 
2856                 /* Make sure descriptor is at least the minimum size */
2857                 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2858 
2859                 /* Calculate actual number of SGL (given WQE size) */
2860                 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SND_HDRS) >> 4;
2861                 break;
2862 
2863         case HERMON_QP_WQ_TYPE_RECVQ:
2864                 /*
2865                  * Same as above (except for Recv WQEs)
2866                  */
2867                 max_size = (HERMON_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
2868                 log2 = highbit(max_size);
2869                 if (ISP2(max_size)) {
2870                         log2 = log2 - 1;
2871                 }
2872 
2873                 /* Make sure descriptor is at least the minimum size */
2874                 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2875 
2876                 /* Calculate actual number of SGL (given WQE size) */
2877                 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_RCV_HDRS) >> 4;
2878                 break;
2879 
2880         case HERMON_QP_WQ_TYPE_SENDMLX_QP0:
2881                 /*
2882                  * Same as above (except for MLX transport WQEs).  For these
2883                  * WQEs we have to account for the space consumed by the
2884                  * "inline" packet headers.  (This is smaller than for QP1
2885                  * below because QP0 is not allowed to send packets with a GRH.
2886                  */
2887                 max_size = (HERMON_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4));
2888                 log2 = highbit(max_size);
2889                 if (ISP2(max_size)) {
2890                         log2 = log2 - 1;
2891                 }
2892 
2893                 /* Make sure descriptor is at least the minimum size */
2894                 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2895 
2896                 /* Calculate actual number of SGL (given WQE size) */
2897                 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP0_HDRS) >> 4;
2898                 break;
2899 
2900         case HERMON_QP_WQ_TYPE_SENDMLX_QP1:
2901                 /*
2902                  * Same as above.  For these WQEs we again have to account for
2903                  * the space consumed by the "inline" packet headers.  (This
2904                  * is larger than for QP0 above because we have to account for
2905                  * the possibility of a GRH in each packet - and this
2906                  * introduces an alignment issue that causes us to consume
2907                  * an additional 8 bytes).
2908                  */
2909                 max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4));
2910                 log2 = highbit(max_size);
2911                 if (ISP2(max_size)) {
2912                         log2 = log2 - 1;
2913                 }
2914 
2915                 /* Make sure descriptor is at least the minimum size */
2916                 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2917 
2918                 /* Calculate actual number of SGL (given WQE size) */
2919                 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4;
2920                 break;
2921 
2922         default:
2923                 HERMON_WARNING(state, "unexpected work queue type");
2924                 break;
2925         }
2926 
2927         /* Fill in the return values */
2928         *logwqesz = log2;
2929         *max_sgl  = min(real_max_sgl, actual_sgl);
2930 }