73 tavor_qphdl_t *qphdl;
74 ibt_mr_attr_t mr_attr;
75 tavor_mr_options_t mr_op;
76 tavor_srqhdl_t srq;
77 tavor_pdhdl_t pd;
78 tavor_cqhdl_t sq_cq, rq_cq;
79 tavor_mrhdl_t mr;
80 uint64_t value, qp_desc_off;
81 uint32_t *sq_buf, *rq_buf;
82 uint32_t log_qp_sq_size, log_qp_rq_size;
83 uint32_t sq_size, rq_size;
84 uint32_t sq_wqe_size, rq_wqe_size;
85 uint32_t max_rdb, max_sgl, uarpg;
86 uint_t wq_location, dma_xfer_mode, qp_is_umap;
87 uint_t qp_srq_en;
88 int status, flag;
89 char *errormsg;
90
91 TAVOR_TNF_ENTER(tavor_qp_alloc);
92
93 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
94
95 /*
96 * Check the "options" flag. Currently this flag tells the driver
97 * whether or not the QP's work queues should be come from normal
98 * system memory or whether they should be allocated from DDR memory.
99 */
100 if (op == NULL) {
101 wq_location = TAVOR_QUEUE_LOCATION_NORMAL;
102 } else {
103 wq_location = op->qpo_wq_loc;
104 }
105
106 /*
107 * Extract the necessary info from the tavor_qp_info_t structure
108 */
109 attr_p = qpinfo->qpi_attrp;
110 type = qpinfo->qpi_type;
111 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
112 queuesz_p = qpinfo->qpi_queueszp;
113 qpn = qpinfo->qpi_qpn;
114 qphdl = &qpinfo->qpi_qphdl;
233 */
234 status = tavor_rsrc_alloc(state, TAVOR_QPC, 1, sleepflag, &qpc);
235 if (status != DDI_SUCCESS) {
236 /* Set "status" and "errormsg" and goto failure */
237 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP context");
238 goto qpalloc_fail3;
239 }
240
241 /*
242 * Allocate the software structure for tracking the queue pair
243 * (i.e. the Tavor Queue Pair handle). If we fail here, we must
244 * undo the reference counts and the previous resource allocation.
245 */
246 status = tavor_rsrc_alloc(state, TAVOR_QPHDL, 1, sleepflag, &rsrc);
247 if (status != DDI_SUCCESS) {
248 /* Set "status" and "errormsg" and goto failure */
249 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP handle");
250 goto qpalloc_fail4;
251 }
252 qp = (tavor_qphdl_t)rsrc->tr_addr;
253 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
254
255 /*
256 * Calculate the QP number from QPC index. This routine handles
257 * all of the operations necessary to keep track of used, unused,
258 * and released QP numbers.
259 */
260 status = tavor_qp_create_qpn(state, qp, qpc);
261 if (status != DDI_SUCCESS) {
262 /* Set "status" and "errormsg" and goto failure */
263 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QPN create");
264 goto qpalloc_fail5;
265 }
266
267 /*
268 * If this will be a user-mappable QP, then allocate an entry for
269 * the "userland resources database". This will later be added to
270 * the database (after all further QP operations are successful).
271 * If we fail here, we must undo the reference counts and the
272 * previous resource allocation.
273 */
768 */
769 status = tavor_special_qp_rsrc_alloc(state, type, port, &qpc);
770 if (status != DDI_SUCCESS) {
771 /* Set "status" and "errormsg" and goto failure */
772 TAVOR_TNF_FAIL(status, "failed special QP rsrc");
773 goto spec_qpalloc_fail3;
774 }
775
776 /*
777 * Allocate the software structure for tracking the special queue
778 * pair (i.e. the Tavor Queue Pair handle). If we fail here, we
779 * must undo the reference counts and the previous resource allocation.
780 */
781 status = tavor_rsrc_alloc(state, TAVOR_QPHDL, 1, sleepflag, &rsrc);
782 if (status != DDI_SUCCESS) {
783 /* Set "status" and "errormsg" and goto failure */
784 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP handle");
785 goto spec_qpalloc_fail4;
786 }
787 qp = (tavor_qphdl_t)rsrc->tr_addr;
788 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
789
790 /*
791 * Actual QP number is a combination of the index of the QPC and
792 * the port number. This is because the special QP contexts must
793 * be allocated two-at-a-time.
794 */
795 qp->qp_qpnum = qpc->tr_indx + port;
796
797 /*
798 * Calculate the appropriate size for the work queues.
799 * Note: All Tavor QP work queues must be a power-of-2 in size. Also
800 * they may not be any smaller than TAVOR_QP_MIN_SIZE. This step is
801 * to round the requested size up to the next highest power-of-2
802 */
803 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, TAVOR_QP_MIN_SIZE);
804 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, TAVOR_QP_MIN_SIZE);
805 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
806 if (ISP2(attr_p->qp_sizes.cs_sq)) {
807 log_qp_sq_size = log_qp_sq_size - 1;
808 }
1161 * Put NULL into the Tavor QPNum-to-QPHdl list. This will allow any
1162 * in-progress events to detect that the QP corresponding to this
1163 * number has been freed. Note: it does depend in whether we are
1164 * freeing a special QP or not.
1165 */
1166 if (qp->qp_is_special) {
1167 state->ts_qphdl[qpc->tr_indx + port] = NULL;
1168 } else {
1169 state->ts_qphdl[qpc->tr_indx] = NULL;
1170 }
1171
1172 /*
1173 * Drop the QP lock
1174 * At this point the lock is no longer necessary. We cannot
1175 * protect from multiple simultaneous calls to free the same QP.
1176 * In addition, since the QP lock is contained in the QP "software
1177 * handle" resource, which we will free (see below), it is
1178 * important that we have no further references to that memory.
1179 */
1180 mutex_exit(&qp->qp_lock);
1181 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1182
1183 /*
1184 * Free the QP resources
1185 * Start by deregistering and freeing the memory for work queues.
1186 * Next free any previously allocated context information
1187 * (depending on QP type)
1188 * Finally, decrement the necessary reference counts.
1189 * If this fails for any reason, then it is an indication that
1190 * something (either in HW or SW) has gone seriously wrong. So we
1191 * print a warning message and return.
1192 */
1193 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1194 sleepflag);
1195 if (status != DDI_SUCCESS) {
1196 TAVOR_WARNING(state, "failed to deregister QP memory");
1197 /* Set "status" and "errormsg" and goto failure */
1198 TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "failed deregister mr");
1199 goto qpfree_fail;
1200 }
1201
1549 * add an entry to the AVL tree.
1550 * If a matching entry is found, then increment its QPN counter
1551 * and reference counter.
1552 */
1553 query.qpn_indx = qpc->tr_indx;
1554 mutex_enter(&state->ts_qpn_avl_lock);
1555 entry = (tavor_qpn_entry_t *)avl_find(&state->ts_qpn_avl,
1556 &query, &where);
1557 if (entry == NULL) {
1558 /*
1559 * Allocate and initialize a QPN entry, then insert
1560 * it into the AVL tree.
1561 */
1562 entry = (tavor_qpn_entry_t *)kmem_zalloc(
1563 sizeof (tavor_qpn_entry_t), KM_NOSLEEP);
1564 if (entry == NULL) {
1565 mutex_exit(&state->ts_qpn_avl_lock);
1566 TAVOR_TNF_EXIT(tavor_qp_create_qpn);
1567 return (DDI_FAILURE);
1568 }
1569 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
1570
1571 entry->qpn_indx = qpc->tr_indx;
1572 entry->qpn_refcnt = 0;
1573 entry->qpn_counter = 0;
1574
1575 avl_insert(&state->ts_qpn_avl, entry, where);
1576 }
1577
1578 /*
1579 * Make the AVL tree entry point to the QP context resource that
1580 * it will be responsible for tracking
1581 */
1582 entry->qpn_qpc = qpc;
1583
1584 /*
1585 * Setup the QP handle to point to the AVL tree entry. Then
1586 * generate the new QP number from the entry's QPN counter value
1587 * and the hardware's QP context table index.
1588 */
1589 qp->qp_qpn_hdl = entry;
|
73 tavor_qphdl_t *qphdl;
74 ibt_mr_attr_t mr_attr;
75 tavor_mr_options_t mr_op;
76 tavor_srqhdl_t srq;
77 tavor_pdhdl_t pd;
78 tavor_cqhdl_t sq_cq, rq_cq;
79 tavor_mrhdl_t mr;
80 uint64_t value, qp_desc_off;
81 uint32_t *sq_buf, *rq_buf;
82 uint32_t log_qp_sq_size, log_qp_rq_size;
83 uint32_t sq_size, rq_size;
84 uint32_t sq_wqe_size, rq_wqe_size;
85 uint32_t max_rdb, max_sgl, uarpg;
86 uint_t wq_location, dma_xfer_mode, qp_is_umap;
87 uint_t qp_srq_en;
88 int status, flag;
89 char *errormsg;
90
91 TAVOR_TNF_ENTER(tavor_qp_alloc);
92
93 /*
94 * Check the "options" flag. Currently this flag tells the driver
95 * whether or not the QP's work queues should be come from normal
96 * system memory or whether they should be allocated from DDR memory.
97 */
98 if (op == NULL) {
99 wq_location = TAVOR_QUEUE_LOCATION_NORMAL;
100 } else {
101 wq_location = op->qpo_wq_loc;
102 }
103
104 /*
105 * Extract the necessary info from the tavor_qp_info_t structure
106 */
107 attr_p = qpinfo->qpi_attrp;
108 type = qpinfo->qpi_type;
109 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
110 queuesz_p = qpinfo->qpi_queueszp;
111 qpn = qpinfo->qpi_qpn;
112 qphdl = &qpinfo->qpi_qphdl;
231 */
232 status = tavor_rsrc_alloc(state, TAVOR_QPC, 1, sleepflag, &qpc);
233 if (status != DDI_SUCCESS) {
234 /* Set "status" and "errormsg" and goto failure */
235 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP context");
236 goto qpalloc_fail3;
237 }
238
239 /*
240 * Allocate the software structure for tracking the queue pair
241 * (i.e. the Tavor Queue Pair handle). If we fail here, we must
242 * undo the reference counts and the previous resource allocation.
243 */
244 status = tavor_rsrc_alloc(state, TAVOR_QPHDL, 1, sleepflag, &rsrc);
245 if (status != DDI_SUCCESS) {
246 /* Set "status" and "errormsg" and goto failure */
247 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP handle");
248 goto qpalloc_fail4;
249 }
250 qp = (tavor_qphdl_t)rsrc->tr_addr;
251
252 /*
253 * Calculate the QP number from QPC index. This routine handles
254 * all of the operations necessary to keep track of used, unused,
255 * and released QP numbers.
256 */
257 status = tavor_qp_create_qpn(state, qp, qpc);
258 if (status != DDI_SUCCESS) {
259 /* Set "status" and "errormsg" and goto failure */
260 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QPN create");
261 goto qpalloc_fail5;
262 }
263
264 /*
265 * If this will be a user-mappable QP, then allocate an entry for
266 * the "userland resources database". This will later be added to
267 * the database (after all further QP operations are successful).
268 * If we fail here, we must undo the reference counts and the
269 * previous resource allocation.
270 */
765 */
766 status = tavor_special_qp_rsrc_alloc(state, type, port, &qpc);
767 if (status != DDI_SUCCESS) {
768 /* Set "status" and "errormsg" and goto failure */
769 TAVOR_TNF_FAIL(status, "failed special QP rsrc");
770 goto spec_qpalloc_fail3;
771 }
772
773 /*
774 * Allocate the software structure for tracking the special queue
775 * pair (i.e. the Tavor Queue Pair handle). If we fail here, we
776 * must undo the reference counts and the previous resource allocation.
777 */
778 status = tavor_rsrc_alloc(state, TAVOR_QPHDL, 1, sleepflag, &rsrc);
779 if (status != DDI_SUCCESS) {
780 /* Set "status" and "errormsg" and goto failure */
781 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP handle");
782 goto spec_qpalloc_fail4;
783 }
784 qp = (tavor_qphdl_t)rsrc->tr_addr;
785
786 /*
787 * Actual QP number is a combination of the index of the QPC and
788 * the port number. This is because the special QP contexts must
789 * be allocated two-at-a-time.
790 */
791 qp->qp_qpnum = qpc->tr_indx + port;
792
793 /*
794 * Calculate the appropriate size for the work queues.
795 * Note: All Tavor QP work queues must be a power-of-2 in size. Also
796 * they may not be any smaller than TAVOR_QP_MIN_SIZE. This step is
797 * to round the requested size up to the next highest power-of-2
798 */
799 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, TAVOR_QP_MIN_SIZE);
800 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, TAVOR_QP_MIN_SIZE);
801 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
802 if (ISP2(attr_p->qp_sizes.cs_sq)) {
803 log_qp_sq_size = log_qp_sq_size - 1;
804 }
1157 * Put NULL into the Tavor QPNum-to-QPHdl list. This will allow any
1158 * in-progress events to detect that the QP corresponding to this
1159 * number has been freed. Note: it does depend in whether we are
1160 * freeing a special QP or not.
1161 */
1162 if (qp->qp_is_special) {
1163 state->ts_qphdl[qpc->tr_indx + port] = NULL;
1164 } else {
1165 state->ts_qphdl[qpc->tr_indx] = NULL;
1166 }
1167
1168 /*
1169 * Drop the QP lock
1170 * At this point the lock is no longer necessary. We cannot
1171 * protect from multiple simultaneous calls to free the same QP.
1172 * In addition, since the QP lock is contained in the QP "software
1173 * handle" resource, which we will free (see below), it is
1174 * important that we have no further references to that memory.
1175 */
1176 mutex_exit(&qp->qp_lock);
1177
1178 /*
1179 * Free the QP resources
1180 * Start by deregistering and freeing the memory for work queues.
1181 * Next free any previously allocated context information
1182 * (depending on QP type)
1183 * Finally, decrement the necessary reference counts.
1184 * If this fails for any reason, then it is an indication that
1185 * something (either in HW or SW) has gone seriously wrong. So we
1186 * print a warning message and return.
1187 */
1188 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1189 sleepflag);
1190 if (status != DDI_SUCCESS) {
1191 TAVOR_WARNING(state, "failed to deregister QP memory");
1192 /* Set "status" and "errormsg" and goto failure */
1193 TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "failed deregister mr");
1194 goto qpfree_fail;
1195 }
1196
1544 * add an entry to the AVL tree.
1545 * If a matching entry is found, then increment its QPN counter
1546 * and reference counter.
1547 */
1548 query.qpn_indx = qpc->tr_indx;
1549 mutex_enter(&state->ts_qpn_avl_lock);
1550 entry = (tavor_qpn_entry_t *)avl_find(&state->ts_qpn_avl,
1551 &query, &where);
1552 if (entry == NULL) {
1553 /*
1554 * Allocate and initialize a QPN entry, then insert
1555 * it into the AVL tree.
1556 */
1557 entry = (tavor_qpn_entry_t *)kmem_zalloc(
1558 sizeof (tavor_qpn_entry_t), KM_NOSLEEP);
1559 if (entry == NULL) {
1560 mutex_exit(&state->ts_qpn_avl_lock);
1561 TAVOR_TNF_EXIT(tavor_qp_create_qpn);
1562 return (DDI_FAILURE);
1563 }
1564
1565 entry->qpn_indx = qpc->tr_indx;
1566 entry->qpn_refcnt = 0;
1567 entry->qpn_counter = 0;
1568
1569 avl_insert(&state->ts_qpn_avl, entry, where);
1570 }
1571
1572 /*
1573 * Make the AVL tree entry point to the QP context resource that
1574 * it will be responsible for tracking
1575 */
1576 entry->qpn_qpc = qpc;
1577
1578 /*
1579 * Setup the QP handle to point to the AVL tree entry. Then
1580 * generate the new QP number from the entry's QPN counter value
1581 * and the hardware's QP context table index.
1582 */
1583 qp->qp_qpn_hdl = entry;
|