Print this page
8368 remove warlock leftovers from usr/src/uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/adapters/hermon/hermon_qp.c
+++ new/usr/src/uts/common/io/ib/adapters/hermon/hermon_qp.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25
26 26 /*
27 27 * hermon_qp.c
28 28 * Hermon Queue Pair Processing Routines
29 29 *
30 30 * Implements all the routines necessary for allocating, freeing, and
31 31 * querying the Hermon queue pairs.
32 32 */
33 33
34 34 #include <sys/types.h>
35 35 #include <sys/conf.h>
36 36 #include <sys/ddi.h>
37 37 #include <sys/sunddi.h>
38 38 #include <sys/modctl.h>
39 39 #include <sys/bitmap.h>
40 40 #include <sys/sysmacros.h>
41 41
42 42 #include <sys/ib/adapters/hermon/hermon.h>
43 43 #include <sys/ib/ib_pkt_hdrs.h>
44 44
45 45 static int hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
46 46 hermon_rsrc_t *qpc);
47 47 static int hermon_qpn_avl_compare(const void *q, const void *e);
48 48 static int hermon_special_qp_rsrc_alloc(hermon_state_t *state,
49 49 ibt_sqp_type_t type, uint_t port, hermon_rsrc_t **qp_rsrc);
50 50 static int hermon_special_qp_rsrc_free(hermon_state_t *state,
51 51 ibt_sqp_type_t type, uint_t port);
52 52 static void hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
53 53 uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
54 54 uint_t *logwqesz, uint_t *max_sgl);
55 55
56 56 /*
57 57 * hermon_qp_alloc()
58 58 * Context: Can be called only from user or kernel context.
59 59 */
60 60 int
61 61 hermon_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
62 62 uint_t sleepflag)
63 63 {
64 64 hermon_rsrc_t *qpc, *rsrc;
65 65 hermon_rsrc_type_t rsrc_type;
66 66 hermon_umap_db_entry_t *umapdb;
67 67 hermon_qphdl_t qp;
68 68 ibt_qp_alloc_attr_t *attr_p;
69 69 ibt_qp_alloc_flags_t alloc_flags;
70 70 ibt_qp_type_t type;
71 71 hermon_qp_wq_type_t swq_type;
72 72 ibtl_qp_hdl_t ibt_qphdl;
73 73 ibt_chan_sizes_t *queuesz_p;
74 74 ib_qpn_t *qpn;
75 75 hermon_qphdl_t *qphdl;
76 76 ibt_mr_attr_t mr_attr;
77 77 hermon_mr_options_t mr_op;
78 78 hermon_srqhdl_t srq;
79 79 hermon_pdhdl_t pd;
80 80 hermon_cqhdl_t sq_cq, rq_cq;
81 81 hermon_mrhdl_t mr;
82 82 uint64_t value, qp_desc_off;
83 83 uint64_t *thewqe, thewqesz;
|
↓ open down ↓ |
83 lines elided |
↑ open up ↑ |
84 84 uint32_t *sq_buf, *rq_buf;
85 85 uint32_t log_qp_sq_size, log_qp_rq_size;
86 86 uint32_t sq_size, rq_size;
87 87 uint32_t sq_depth, rq_depth;
88 88 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
89 89 uint32_t max_sgl, max_recv_sgl, uarpg;
90 90 uint_t qp_is_umap;
91 91 uint_t qp_srq_en, i, j;
92 92 int status, flag;
93 93
94 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
95 -
96 94 /*
97 95 * Extract the necessary info from the hermon_qp_info_t structure
98 96 */
99 97 attr_p = qpinfo->qpi_attrp;
100 98 type = qpinfo->qpi_type;
101 99 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
102 100 queuesz_p = qpinfo->qpi_queueszp;
103 101 qpn = qpinfo->qpi_qpn;
104 102 qphdl = &qpinfo->qpi_qphdl;
105 103 alloc_flags = attr_p->qp_alloc_flags;
106 104
107 105 /*
108 106 * Verify correctness of alloc_flags.
109 107 *
110 108 * 1. FEXCH and RSS are only allocated via qp_range.
111 109 */
112 110 if (alloc_flags & (IBT_QP_USES_FEXCH | IBT_QP_USES_RSS)) {
113 111 return (IBT_INVALID_PARAM);
114 112 }
115 113 rsrc_type = HERMON_QPC;
116 114 qp_is_umap = 0;
117 115
118 116 /* 2. Make sure only one of these flags is set. */
119 117 switch (alloc_flags &
120 118 (IBT_QP_USER_MAP | IBT_QP_USES_RFCI | IBT_QP_USES_FCMD)) {
121 119 case IBT_QP_USER_MAP:
122 120 qp_is_umap = 1;
123 121 break;
124 122 case IBT_QP_USES_RFCI:
125 123 if (type != IBT_UD_RQP)
126 124 return (IBT_INVALID_PARAM);
127 125
128 126 switch (attr_p->qp_fc.fc_hca_port) {
129 127 case 1:
130 128 rsrc_type = HERMON_QPC_RFCI_PORT1;
131 129 break;
132 130 case 2:
133 131 rsrc_type = HERMON_QPC_RFCI_PORT2;
134 132 break;
135 133 default:
136 134 return (IBT_INVALID_PARAM);
137 135 }
138 136 break;
139 137 case IBT_QP_USES_FCMD:
140 138 if (type != IBT_UD_RQP)
141 139 return (IBT_INVALID_PARAM);
142 140 break;
143 141 case 0:
144 142 break;
145 143 default:
146 144 return (IBT_INVALID_PARAM); /* conflicting flags set */
147 145 }
148 146
149 147 /*
150 148 * Determine whether QP is being allocated for userland access or
151 149 * whether it is being allocated for kernel access. If the QP is
152 150 * being allocated for userland access, then lookup the UAR
153 151 * page number for the current process. Note: If this is not found
154 152 * (e.g. if the process has not previously open()'d the Hermon driver),
155 153 * then an error is returned.
156 154 */
157 155 if (qp_is_umap) {
158 156 status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
159 157 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
160 158 if (status != DDI_SUCCESS) {
161 159 return (IBT_INVALID_PARAM);
162 160 }
163 161 uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
164 162 } else {
165 163 uarpg = state->hs_kernel_uar_index;
166 164 }
167 165
168 166 /*
169 167 * Determine whether QP is being associated with an SRQ
170 168 */
171 169 qp_srq_en = (alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
172 170 if (qp_srq_en) {
173 171 /*
174 172 * Check for valid SRQ handle pointers
175 173 */
176 174 if (attr_p->qp_ibc_srq_hdl == NULL) {
177 175 status = IBT_SRQ_HDL_INVALID;
178 176 goto qpalloc_fail;
179 177 }
180 178 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
181 179 }
182 180
183 181 /*
184 182 * Check for valid QP service type (only UD/RC/UC supported)
185 183 */
186 184 if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) &&
187 185 (type != IBT_UC_RQP))) {
188 186 status = IBT_QP_SRV_TYPE_INVALID;
189 187 goto qpalloc_fail;
190 188 }
191 189
192 190
193 191 /*
194 192 * Check for valid PD handle pointer
195 193 */
196 194 if (attr_p->qp_pd_hdl == NULL) {
197 195 status = IBT_PD_HDL_INVALID;
198 196 goto qpalloc_fail;
199 197 }
200 198 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
201 199
202 200 /*
203 201 * If on an SRQ, check to make sure the PD is the same
204 202 */
205 203 if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
206 204 status = IBT_PD_HDL_INVALID;
207 205 goto qpalloc_fail;
208 206 }
209 207
210 208 /* Increment the reference count on the protection domain (PD) */
211 209 hermon_pd_refcnt_inc(pd);
212 210
213 211 /*
214 212 * Check for valid CQ handle pointers
215 213 *
216 214 * FCMD QPs do not require a receive cq handle.
217 215 */
218 216 if (attr_p->qp_ibc_scq_hdl == NULL) {
219 217 status = IBT_CQ_HDL_INVALID;
220 218 goto qpalloc_fail1;
221 219 }
222 220 sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
223 221 if ((attr_p->qp_ibc_rcq_hdl == NULL)) {
224 222 if ((alloc_flags & IBT_QP_USES_FCMD) == 0) {
225 223 status = IBT_CQ_HDL_INVALID;
226 224 goto qpalloc_fail1;
227 225 }
228 226 rq_cq = sq_cq; /* just use the send cq */
229 227 } else
230 228 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
231 229
232 230 /*
233 231 * Increment the reference count on the CQs. One or both of these
234 232 * could return error if we determine that the given CQ is already
235 233 * being used with a special (SMI/GSI) QP.
236 234 */
237 235 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
238 236 if (status != DDI_SUCCESS) {
239 237 status = IBT_CQ_HDL_INVALID;
240 238 goto qpalloc_fail1;
241 239 }
242 240 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
243 241 if (status != DDI_SUCCESS) {
244 242 status = IBT_CQ_HDL_INVALID;
245 243 goto qpalloc_fail2;
246 244 }
247 245
248 246 /*
249 247 * Allocate an QP context entry. This will be filled in with all
250 248 * the necessary parameters to define the Queue Pair. Unlike
251 249 * other Hermon hardware resources, ownership is not immediately
252 250 * given to hardware in the final step here. Instead, we must
253 251 * wait until the QP is later transitioned to the "Init" state before
254 252 * passing the QP to hardware. If we fail here, we must undo all
255 253 * the reference count (CQ and PD).
256 254 */
257 255 status = hermon_rsrc_alloc(state, rsrc_type, 1, sleepflag, &qpc);
258 256 if (status != DDI_SUCCESS) {
259 257 status = IBT_INSUFF_RESOURCE;
260 258 goto qpalloc_fail3;
261 259 }
262 260
263 261 /*
264 262 * Allocate the software structure for tracking the queue pair
|
↓ open down ↓ |
159 lines elided |
↑ open up ↑ |
265 263 * (i.e. the Hermon Queue Pair handle). If we fail here, we must
266 264 * undo the reference counts and the previous resource allocation.
267 265 */
268 266 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
269 267 if (status != DDI_SUCCESS) {
270 268 status = IBT_INSUFF_RESOURCE;
271 269 goto qpalloc_fail4;
272 270 }
273 271 qp = (hermon_qphdl_t)rsrc->hr_addr;
274 272 bzero(qp, sizeof (struct hermon_sw_qp_s));
275 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
276 273
277 274 qp->qp_alloc_flags = alloc_flags;
278 275
279 276 /*
280 277 * Calculate the QP number from QPC index. This routine handles
281 278 * all of the operations necessary to keep track of used, unused,
282 279 * and released QP numbers.
283 280 */
284 281 if (type == IBT_UD_RQP) {
285 282 qp->qp_qpnum = qpc->hr_indx;
286 283 qp->qp_ring = qp->qp_qpnum << 8;
287 284 qp->qp_qpn_hdl = NULL;
288 285 } else {
289 286 status = hermon_qp_create_qpn(state, qp, qpc);
290 287 if (status != DDI_SUCCESS) {
291 288 status = IBT_INSUFF_RESOURCE;
292 289 goto qpalloc_fail5;
293 290 }
294 291 }
295 292
296 293 /*
297 294 * If this will be a user-mappable QP, then allocate an entry for
298 295 * the "userland resources database". This will later be added to
299 296 * the database (after all further QP operations are successful).
300 297 * If we fail here, we must undo the reference counts and the
301 298 * previous resource allocation.
302 299 */
303 300 if (qp_is_umap) {
304 301 umapdb = hermon_umap_db_alloc(state->hs_instance, qp->qp_qpnum,
305 302 MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
306 303 if (umapdb == NULL) {
307 304 status = IBT_INSUFF_RESOURCE;
308 305 goto qpalloc_fail6;
309 306 }
310 307 }
311 308
312 309 /*
313 310 * Allocate the doorbell record. Hermon just needs one for the RQ,
314 311 * if the QP is not associated with an SRQ, and use uarpg (above) as
315 312 * the uar index
316 313 */
317 314
318 315 if (!qp_srq_en) {
319 316 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
320 317 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
321 318 if (status != DDI_SUCCESS) {
322 319 status = IBT_INSUFF_RESOURCE;
323 320 goto qpalloc_fail6;
324 321 }
325 322 }
326 323
327 324 qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
328 325
329 326 /*
330 327 * We verify that the requested number of SGL is valid (i.e.
331 328 * consistent with the device limits and/or software-configured
332 329 * limits). If not, then obviously the same cleanup needs to be done.
333 330 */
334 331 if (type == IBT_UD_RQP) {
335 332 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
336 333 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
337 334 } else {
338 335 max_sgl = state->hs_ibtfinfo.hca_attr->hca_conn_send_sgl_sz;
339 336 swq_type = HERMON_QP_WQ_TYPE_SENDQ_CONN;
340 337 }
341 338 max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
342 339 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
343 340 (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
344 341 status = IBT_HCA_SGL_EXCEEDED;
345 342 goto qpalloc_fail7;
346 343 }
347 344
348 345 /*
349 346 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
350 347 * This will depend on the requested number of SGLs. Note: this
351 348 * has the side-effect of also calculating the real number of SGLs
352 349 * (for the calculated WQE size).
353 350 *
354 351 * For QP's on an SRQ, we set these to 0.
355 352 */
356 353 if (qp_srq_en) {
357 354 qp->qp_rq_log_wqesz = 0;
358 355 qp->qp_rq_sgl = 0;
359 356 } else {
360 357 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
361 358 max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
362 359 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
363 360 }
364 361 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
365 362 max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
366 363
367 364 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
368 365
369 366 /* NOTE: currently policy in driver, later maybe IBTF interface */
370 367 qp->qp_no_prefetch = 0;
371 368
372 369 /*
373 370 * for prefetching, we need to add the number of wqes in
374 371 * the 2k area plus one to the number requested, but
375 372 * ONLY for send queue. If no_prefetch == 1 (prefetch off)
376 373 * it's exactly TWO wqes for the headroom
377 374 */
378 375 if (qp->qp_no_prefetch)
379 376 qp->qp_sq_headroom = 2 * sq_wqe_size;
380 377 else
381 378 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
382 379 /*
383 380 * hdrm wqes must be integral since both sq_wqe_size &
384 381 * HERMON_QP_OH_SIZE are power of 2
385 382 */
386 383 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
387 384
388 385
389 386 /*
390 387 * Calculate the appropriate size for the work queues.
391 388 * For send queue, add in the headroom wqes to the calculation.
392 389 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
393 390 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
394 391 * to round the requested size up to the next highest power-of-2
395 392 */
396 393 /* first, adjust to a minimum and tell the caller the change */
397 394 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
398 395 HERMON_QP_MIN_SIZE);
399 396 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
400 397 HERMON_QP_MIN_SIZE);
401 398 /*
402 399 * now, calculate the alloc size, taking into account
403 400 * the headroom for the sq
404 401 */
405 402 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
406 403 /* if the total is a power of two, reduce it */
407 404 if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes)) {
408 405 log_qp_sq_size = log_qp_sq_size - 1;
409 406 }
410 407
411 408 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
412 409 if (ISP2(attr_p->qp_sizes.cs_rq)) {
413 410 log_qp_rq_size = log_qp_rq_size - 1;
414 411 }
415 412
416 413 /*
417 414 * Next we verify that the rounded-up size is valid (i.e. consistent
418 415 * with the device limits and/or software-configured limits). If not,
419 416 * then obviously we have a lot of cleanup to do before returning.
420 417 *
421 418 * NOTE: the first condition deals with the (test) case of cs_sq
422 419 * being just less than 2^32. In this case, the headroom addition
423 420 * to the requested cs_sq will pass the test when it should not.
424 421 * This test no longer lets that case slip through the check.
425 422 */
426 423 if ((attr_p->qp_sizes.cs_sq >
427 424 (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
428 425 (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
429 426 (!qp_srq_en && (log_qp_rq_size >
430 427 state->hs_cfg_profile->cp_log_max_qp_sz))) {
431 428 status = IBT_HCA_WR_EXCEEDED;
432 429 goto qpalloc_fail7;
433 430 }
434 431
435 432 /*
436 433 * Allocate the memory for QP work queues. Since Hermon work queues
437 434 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
438 435 * the work queue memory is very important. We used to allocate
439 436 * work queues (the combined receive and send queues) so that they
440 437 * would be aligned on their combined size. That alignment guaranteed
441 438 * that they would never cross the 4GB boundary (Hermon work queues
442 439 * are on the order of MBs at maximum). Now we are able to relax
443 440 * this alignment constraint by ensuring that the IB address assigned
444 441 * to the queue memory (as a result of the hermon_mr_register() call)
445 442 * is offset from zero.
446 443 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
447 444 * guarantee the alignment, but when attempting to use IOMMU bypass
448 445 * mode we found that we were not allowed to specify any alignment
449 446 * that was more restrictive than the system page size.
450 447 * So we avoided this constraint by passing two alignment values,
451 448 * one for the memory allocation itself and the other for the DMA
452 449 * handle (for later bind). This used to cause more memory than
453 450 * necessary to be allocated (in order to guarantee the more
454 451 * restrictive alignment contraint). But by guaranteeing the
455 452 * zero-based IB virtual address for the queue, we are able to
456 453 * conserve this memory.
457 454 */
458 455 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
459 456 sq_depth = 1 << log_qp_sq_size;
460 457 sq_size = sq_depth * sq_wqe_size;
461 458
462 459 /* QP on SRQ sets these to 0 */
463 460 if (qp_srq_en) {
464 461 rq_wqe_size = 0;
465 462 rq_size = 0;
466 463 } else {
467 464 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
468 465 rq_depth = 1 << log_qp_rq_size;
469 466 rq_size = rq_depth * rq_wqe_size;
470 467 }
471 468
472 469 qp->qp_wqinfo.qa_size = sq_size + rq_size;
473 470
474 471 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
475 472 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
476 473
477 474 if (qp_is_umap) {
478 475 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
479 476 } else {
480 477 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
481 478 }
482 479 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
483 480 if (status != DDI_SUCCESS) {
484 481 status = IBT_INSUFF_RESOURCE;
485 482 goto qpalloc_fail7;
486 483 }
487 484
488 485 /*
489 486 * Sort WQs in memory according to stride (*q_wqe_size), largest first
490 487 * If they are equal, still put the SQ first
491 488 */
492 489 qp->qp_sq_baseaddr = 0;
493 490 qp->qp_rq_baseaddr = 0;
494 491 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
495 492 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
496 493
497 494 /* if this QP is on an SRQ, set the rq_buf to NULL */
498 495 if (qp_srq_en) {
499 496 rq_buf = NULL;
500 497 } else {
501 498 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
502 499 qp->qp_rq_baseaddr = sq_size;
503 500 }
504 501 } else {
505 502 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
506 503 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
507 504 qp->qp_sq_baseaddr = rq_size;
508 505 }
509 506
510 507 if (qp_is_umap == 0) {
511 508 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
512 509 if (qp->qp_sq_wqhdr == NULL) {
513 510 status = IBT_INSUFF_RESOURCE;
514 511 goto qpalloc_fail8;
515 512 }
516 513 if (qp_srq_en) {
517 514 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
518 515 qp->qp_rq_wqavl.wqa_srq_en = 1;
519 516 qp->qp_rq_wqavl.wqa_srq = srq;
520 517 } else {
521 518 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
522 519 if (qp->qp_rq_wqhdr == NULL) {
523 520 status = IBT_INSUFF_RESOURCE;
524 521 goto qpalloc_fail8;
525 522 }
526 523 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
527 524 }
528 525 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
529 526 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
530 527 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
531 528 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
532 529 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
533 530 }
534 531
535 532 /*
536 533 * Register the memory for the QP work queues. The memory for the
537 534 * QP must be registered in the Hermon cMPT tables. This gives us the
538 535 * LKey to specify in the QP context later. Note: The memory for
539 536 * Hermon work queues (both Send and Recv) must be contiguous and
540 537 * registered as a single memory region. Note: If the QP memory is
541 538 * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
542 539 * meet the alignment restriction, we pass the "mro_bind_override_addr"
543 540 * flag in the call to hermon_mr_register(). This guarantees that the
544 541 * resulting IB vaddr will be zero-based (modulo the offset into the
545 542 * first page). If we fail here, we still have the bunch of resource
546 543 * and reference count cleanup to do.
547 544 */
548 545 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
549 546 IBT_MR_NOSLEEP;
550 547 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
551 548 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
552 549 mr_attr.mr_as = NULL;
553 550 mr_attr.mr_flags = flag;
554 551 if (qp_is_umap) {
555 552 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
556 553 } else {
557 554 /* HERMON_QUEUE_LOCATION_NORMAL */
558 555 mr_op.mro_bind_type =
559 556 state->hs_cfg_profile->cp_iommu_bypass;
560 557 }
561 558 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
562 559 mr_op.mro_bind_override_addr = 1;
563 560 status = hermon_mr_register(state, pd, &mr_attr, &mr,
564 561 &mr_op, HERMON_QP_CMPT);
565 562 if (status != DDI_SUCCESS) {
566 563 status = IBT_INSUFF_RESOURCE;
567 564 goto qpalloc_fail9;
568 565 }
569 566
570 567 /*
571 568 * Calculate the offset between the kernel virtual address space
572 569 * and the IB virtual address space. This will be used when
573 570 * posting work requests to properly initialize each WQE.
574 571 */
575 572 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
576 573 (uint64_t)mr->mr_bindinfo.bi_addr;
577 574
578 575 /*
579 576 * Fill in all the return arguments (if necessary). This includes
580 577 * real work queue sizes (in wqes), real SGLs, and QP number
581 578 */
582 579 if (queuesz_p != NULL) {
583 580 queuesz_p->cs_sq =
584 581 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
585 582 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
586 583
587 584 /* if this QP is on an SRQ, set these to 0 */
588 585 if (qp_srq_en) {
589 586 queuesz_p->cs_rq = 0;
590 587 queuesz_p->cs_rq_sgl = 0;
591 588 } else {
592 589 queuesz_p->cs_rq = (1 << log_qp_rq_size);
593 590 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
594 591 }
595 592 }
596 593 if (qpn != NULL) {
597 594 *qpn = (ib_qpn_t)qp->qp_qpnum;
598 595 }
599 596
600 597 /*
601 598 * Fill in the rest of the Hermon Queue Pair handle.
602 599 */
603 600 qp->qp_qpcrsrcp = qpc;
604 601 qp->qp_rsrcp = rsrc;
605 602 qp->qp_state = HERMON_QP_RESET;
606 603 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
607 604 qp->qp_pdhdl = pd;
608 605 qp->qp_mrhdl = mr;
609 606 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
610 607 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
611 608 qp->qp_is_special = 0;
612 609 qp->qp_uarpg = uarpg;
613 610 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
614 611 qp->qp_sq_cqhdl = sq_cq;
615 612 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
616 613 qp->qp_sq_logqsz = log_qp_sq_size;
617 614 qp->qp_sq_buf = sq_buf;
618 615 qp->qp_desc_off = qp_desc_off;
619 616 qp->qp_rq_cqhdl = rq_cq;
620 617 qp->qp_rq_buf = rq_buf;
621 618 qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
622 619 0;
623 620
624 621 /* if this QP is on an SRQ, set rq_bufsz to 0 */
625 622 if (qp_srq_en) {
626 623 qp->qp_rq_bufsz = 0;
627 624 qp->qp_rq_logqsz = 0;
628 625 } else {
629 626 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
630 627 qp->qp_rq_logqsz = log_qp_rq_size;
631 628 }
632 629
633 630 qp->qp_forward_sqd_event = 0;
634 631 qp->qp_sqd_still_draining = 0;
635 632 qp->qp_hdlrarg = (void *)ibt_qphdl;
636 633 qp->qp_mcg_refcnt = 0;
637 634
638 635 /*
639 636 * If this QP is to be associated with an SRQ, set the SRQ handle
640 637 */
641 638 if (qp_srq_en) {
642 639 qp->qp_srqhdl = srq;
643 640 hermon_srq_refcnt_inc(qp->qp_srqhdl);
644 641 } else {
645 642 qp->qp_srqhdl = NULL;
646 643 }
647 644
648 645 /* Determine the QP service type */
649 646 qp->qp_type = type;
650 647 if (type == IBT_RC_RQP) {
651 648 qp->qp_serv_type = HERMON_QP_RC;
652 649 } else if (type == IBT_UD_RQP) {
653 650 if (alloc_flags & IBT_QP_USES_RFCI)
654 651 qp->qp_serv_type = HERMON_QP_RFCI;
655 652 else if (alloc_flags & IBT_QP_USES_FCMD)
656 653 qp->qp_serv_type = HERMON_QP_FCMND;
657 654 else
658 655 qp->qp_serv_type = HERMON_QP_UD;
659 656 } else {
660 657 qp->qp_serv_type = HERMON_QP_UC;
661 658 }
662 659
663 660 /*
664 661 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
665 662 */
666 663
667 664 /*
668 665 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
669 666 * set the quadword to all F's - high-order bit is owner (init to one)
670 667 * and the rest for the headroom definition of prefetching
671 668 *
672 669 */
673 670 wqesz_shift = qp->qp_sq_log_wqesz;
674 671 thewqesz = 1 << wqesz_shift;
675 672 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
676 673 if (qp_is_umap == 0) {
677 674 for (i = 0; i < sq_depth; i++) {
678 675 /*
679 676 * for each stride, go through and every 64 bytes
680 677 * write the init value - having set the address
681 678 * once, just keep incrementing it
682 679 */
683 680 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
684 681 *(uint32_t *)thewqe = 0xFFFFFFFF;
685 682 }
686 683 }
687 684 }
688 685
689 686 /* Zero out the QP context */
690 687 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
691 688
692 689 /*
693 690 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
694 691 * "qphdl" and return success
695 692 */
696 693 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx, qp);
697 694
698 695 /*
699 696 * If this is a user-mappable QP, then we need to insert the previously
700 697 * allocated entry into the "userland resources database". This will
701 698 * allow for later lookup during devmap() (i.e. mmap()) calls.
702 699 */
703 700 if (qp_is_umap) {
704 701 hermon_umap_db_add(umapdb);
705 702 }
706 703 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
707 704 DDI_INTR_PRI(state->hs_intrmsi_pri));
708 705
709 706 *qphdl = qp;
710 707
711 708 return (DDI_SUCCESS);
712 709
713 710 /*
714 711 * The following is cleanup for all possible failure cases in this routine
715 712 */
716 713 qpalloc_fail9:
717 714 hermon_queue_free(&qp->qp_wqinfo);
718 715 qpalloc_fail8:
719 716 if (qp->qp_sq_wqhdr)
720 717 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
721 718 if (qp->qp_rq_wqhdr)
722 719 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
723 720 qpalloc_fail7:
724 721 if (qp_is_umap) {
725 722 hermon_umap_db_free(umapdb);
726 723 }
727 724 if (!qp_srq_en) {
728 725 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
729 726 }
730 727
731 728 qpalloc_fail6:
732 729 /*
733 730 * Releasing the QPN will also free up the QPC context. Update
734 731 * the QPC context pointer to indicate this.
735 732 */
736 733 if (qp->qp_qpn_hdl) {
737 734 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
738 735 HERMON_QPN_RELEASE);
739 736 } else {
740 737 hermon_rsrc_free(state, &qpc);
741 738 }
742 739 qpc = NULL;
743 740 qpalloc_fail5:
744 741 hermon_rsrc_free(state, &rsrc);
745 742 qpalloc_fail4:
746 743 if (qpc) {
747 744 hermon_rsrc_free(state, &qpc);
748 745 }
749 746 qpalloc_fail3:
750 747 hermon_cq_refcnt_dec(rq_cq);
751 748 qpalloc_fail2:
752 749 hermon_cq_refcnt_dec(sq_cq);
753 750 qpalloc_fail1:
754 751 hermon_pd_refcnt_dec(pd);
755 752 qpalloc_fail:
756 753 return (status);
757 754 }
758 755
759 756
760 757
761 758 /*
762 759 * hermon_special_qp_alloc()
763 760 * Context: Can be called only from user or kernel context.
764 761 */
765 762 int
766 763 hermon_special_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo,
767 764 uint_t sleepflag)
768 765 {
769 766 hermon_rsrc_t *qpc, *rsrc;
770 767 hermon_qphdl_t qp;
771 768 ibt_qp_alloc_attr_t *attr_p;
772 769 ibt_sqp_type_t type;
773 770 uint8_t port;
774 771 ibtl_qp_hdl_t ibt_qphdl;
775 772 ibt_chan_sizes_t *queuesz_p;
776 773 hermon_qphdl_t *qphdl;
777 774 ibt_mr_attr_t mr_attr;
778 775 hermon_mr_options_t mr_op;
779 776 hermon_pdhdl_t pd;
780 777 hermon_cqhdl_t sq_cq, rq_cq;
781 778 hermon_mrhdl_t mr;
782 779 uint64_t qp_desc_off;
783 780 uint64_t *thewqe, thewqesz;
784 781 uint32_t *sq_buf, *rq_buf;
785 782 uint32_t log_qp_sq_size, log_qp_rq_size;
786 783 uint32_t sq_size, rq_size, max_sgl;
787 784 uint32_t uarpg;
788 785 uint32_t sq_depth;
789 786 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
790 787 int status, flag, i, j;
791 788
792 789 /*
793 790 * Extract the necessary info from the hermon_qp_info_t structure
794 791 */
795 792 attr_p = qpinfo->qpi_attrp;
796 793 type = qpinfo->qpi_type;
797 794 port = qpinfo->qpi_port;
798 795 ibt_qphdl = qpinfo->qpi_ibt_qphdl;
799 796 queuesz_p = qpinfo->qpi_queueszp;
800 797 qphdl = &qpinfo->qpi_qphdl;
801 798
802 799 /*
803 800 * Check for valid special QP type (only SMI & GSI supported)
804 801 */
805 802 if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) {
806 803 status = IBT_QP_SPECIAL_TYPE_INVALID;
807 804 goto spec_qpalloc_fail;
808 805 }
809 806
810 807 /*
811 808 * Check for valid port number
812 809 */
813 810 if (!hermon_portnum_is_valid(state, port)) {
814 811 status = IBT_HCA_PORT_INVALID;
815 812 goto spec_qpalloc_fail;
816 813 }
817 814 port = port - 1;
818 815
819 816 /*
820 817 * Check for valid PD handle pointer
821 818 */
822 819 if (attr_p->qp_pd_hdl == NULL) {
823 820 status = IBT_PD_HDL_INVALID;
824 821 goto spec_qpalloc_fail;
825 822 }
826 823 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
827 824
828 825 /* Increment the reference count on the PD */
829 826 hermon_pd_refcnt_inc(pd);
830 827
831 828 /*
832 829 * Check for valid CQ handle pointers
833 830 */
834 831 if ((attr_p->qp_ibc_scq_hdl == NULL) ||
835 832 (attr_p->qp_ibc_rcq_hdl == NULL)) {
836 833 status = IBT_CQ_HDL_INVALID;
837 834 goto spec_qpalloc_fail1;
838 835 }
839 836 sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl;
840 837 rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl;
841 838
842 839 /*
843 840 * Increment the reference count on the CQs. One or both of these
844 841 * could return error if we determine that the given CQ is already
845 842 * being used with a non-special QP (i.e. a normal QP).
846 843 */
847 844 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_SPECIAL);
848 845 if (status != DDI_SUCCESS) {
849 846 status = IBT_CQ_HDL_INVALID;
850 847 goto spec_qpalloc_fail1;
851 848 }
852 849 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_SPECIAL);
853 850 if (status != DDI_SUCCESS) {
854 851 status = IBT_CQ_HDL_INVALID;
855 852 goto spec_qpalloc_fail2;
856 853 }
857 854
858 855 /*
859 856 * Allocate the special QP resources. Essentially, this allocation
860 857 * amounts to checking if the request special QP has already been
861 858 * allocated. If successful, the QP context return is an actual
862 859 * QP context that has been "aliased" to act as a special QP of the
863 860 * appropriate type (and for the appropriate port). Just as in
864 861 * hermon_qp_alloc() above, ownership for this QP context is not
865 862 * immediately given to hardware in the final step here. Instead, we
866 863 * wait until the QP is later transitioned to the "Init" state before
867 864 * passing the QP to hardware. If we fail here, we must undo all
868 865 * the reference count (CQ and PD).
869 866 */
870 867 status = hermon_special_qp_rsrc_alloc(state, type, port, &qpc);
871 868 if (status != DDI_SUCCESS) {
872 869 goto spec_qpalloc_fail3;
873 870 }
874 871
875 872 /*
876 873 * Allocate the software structure for tracking the special queue
877 874 * pair (i.e. the Hermon Queue Pair handle). If we fail here, we
878 875 * must undo the reference counts and the previous resource allocation.
|
↓ open down ↓ |
593 lines elided |
↑ open up ↑ |
879 876 */
880 877 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
881 878 if (status != DDI_SUCCESS) {
882 879 status = IBT_INSUFF_RESOURCE;
883 880 goto spec_qpalloc_fail4;
884 881 }
885 882 qp = (hermon_qphdl_t)rsrc->hr_addr;
886 883
887 884 bzero(qp, sizeof (struct hermon_sw_qp_s));
888 885
889 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
890 886 qp->qp_alloc_flags = attr_p->qp_alloc_flags;
891 887
892 888 /*
893 889 * Actual QP number is a combination of the index of the QPC and
894 890 * the port number. This is because the special QP contexts must
895 891 * be allocated two-at-a-time.
896 892 */
897 893 qp->qp_qpnum = qpc->hr_indx + port;
898 894 qp->qp_ring = qp->qp_qpnum << 8;
899 895
900 896 uarpg = state->hs_kernel_uar_index; /* must be for spec qp */
901 897 /*
902 898 * Allocate the doorbell record. Hermon uses only one for the RQ so
903 899 * alloc a qp doorbell, using uarpg (above) as the uar index
904 900 */
905 901
906 902 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
907 903 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
908 904 if (status != DDI_SUCCESS) {
909 905 status = IBT_INSUFF_RESOURCE;
910 906 goto spec_qpalloc_fail5;
911 907 }
912 908 /*
913 909 * Calculate the appropriate size for the work queues.
914 910 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
915 911 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
916 912 * to round the requested size up to the next highest power-of-2
917 913 */
918 914 attr_p->qp_sizes.cs_sq =
919 915 max(attr_p->qp_sizes.cs_sq, HERMON_QP_MIN_SIZE);
920 916 attr_p->qp_sizes.cs_rq =
921 917 max(attr_p->qp_sizes.cs_rq, HERMON_QP_MIN_SIZE);
922 918 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
923 919 if (ISP2(attr_p->qp_sizes.cs_sq)) {
924 920 log_qp_sq_size = log_qp_sq_size - 1;
925 921 }
926 922 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
927 923 if (ISP2(attr_p->qp_sizes.cs_rq)) {
928 924 log_qp_rq_size = log_qp_rq_size - 1;
929 925 }
930 926
931 927 /*
932 928 * Next we verify that the rounded-up size is valid (i.e. consistent
933 929 * with the device limits and/or software-configured limits). If not,
934 930 * then obviously we have a bit of cleanup to do before returning.
935 931 */
936 932 if ((log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
937 933 (log_qp_rq_size > state->hs_cfg_profile->cp_log_max_qp_sz)) {
938 934 status = IBT_HCA_WR_EXCEEDED;
939 935 goto spec_qpalloc_fail5a;
940 936 }
941 937
942 938 /*
943 939 * Next we verify that the requested number of SGL is valid (i.e.
944 940 * consistent with the device limits and/or software-configured
945 941 * limits). If not, then obviously the same cleanup needs to be done.
946 942 */
947 943 max_sgl = state->hs_cfg_profile->cp_wqe_real_max_sgl;
948 944 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
949 945 (attr_p->qp_sizes.cs_rq_sgl > max_sgl)) {
950 946 status = IBT_HCA_SGL_EXCEEDED;
951 947 goto spec_qpalloc_fail5a;
952 948 }
953 949
954 950 /*
955 951 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
956 952 * This will depend on the requested number of SGLs. Note: this
957 953 * has the side-effect of also calculating the real number of SGLs
958 954 * (for the calculated WQE size).
959 955 */
960 956 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
961 957 max_sgl, HERMON_QP_WQ_TYPE_RECVQ,
962 958 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
963 959 if (type == IBT_SMI_SQP) {
964 960 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
965 961 max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP0,
966 962 &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
967 963 } else {
968 964 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
969 965 max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP1,
970 966 &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
971 967 }
972 968
973 969 /*
974 970 * Allocate the memory for QP work queues. Since Hermon work queues
975 971 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
976 972 * the work queue memory is very important. We used to allocate
977 973 * work queues (the combined receive and send queues) so that they
978 974 * would be aligned on their combined size. That alignment guaranteed
979 975 * that they would never cross the 4GB boundary (Hermon work queues
980 976 * are on the order of MBs at maximum). Now we are able to relax
981 977 * this alignment constraint by ensuring that the IB address assigned
982 978 * to the queue memory (as a result of the hermon_mr_register() call)
983 979 * is offset from zero.
984 980 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
985 981 * guarantee the alignment, but when attempting to use IOMMU bypass
986 982 * mode we found that we were not allowed to specify any alignment
987 983 * that was more restrictive than the system page size.
988 984 * So we avoided this constraint by passing two alignment values,
989 985 * one for the memory allocation itself and the other for the DMA
990 986 * handle (for later bind). This used to cause more memory than
991 987 * necessary to be allocated (in order to guarantee the more
992 988 * restrictive alignment contraint). But by guaranteeing the
993 989 * zero-based IB virtual address for the queue, we are able to
994 990 * conserve this memory.
995 991 */
996 992 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
997 993 sq_depth = 1 << log_qp_sq_size;
998 994 sq_size = (1 << log_qp_sq_size) * sq_wqe_size;
999 995
1000 996 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1001 997 rq_size = (1 << log_qp_rq_size) * rq_wqe_size;
1002 998
1003 999 qp->qp_wqinfo.qa_size = sq_size + rq_size;
1004 1000
1005 1001 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1006 1002 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
1007 1003 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1008 1004
1009 1005 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1010 1006 if (status != NULL) {
1011 1007 status = IBT_INSUFF_RESOURCE;
1012 1008 goto spec_qpalloc_fail5a;
1013 1009 }
1014 1010
1015 1011 /*
1016 1012 * Sort WQs in memory according to depth, stride (*q_wqe_size),
1017 1013 * biggest first. If equal, the Send Queue still goes first
1018 1014 */
1019 1015 qp->qp_sq_baseaddr = 0;
1020 1016 qp->qp_rq_baseaddr = 0;
1021 1017 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1022 1018 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1023 1019 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1024 1020 qp->qp_rq_baseaddr = sq_size;
1025 1021 } else {
1026 1022 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1027 1023 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1028 1024 qp->qp_sq_baseaddr = rq_size;
1029 1025 }
1030 1026
1031 1027 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1032 1028 if (qp->qp_sq_wqhdr == NULL) {
1033 1029 status = IBT_INSUFF_RESOURCE;
1034 1030 goto spec_qpalloc_fail6;
1035 1031 }
1036 1032 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(1 << log_qp_rq_size);
1037 1033 if (qp->qp_rq_wqhdr == NULL) {
1038 1034 status = IBT_INSUFF_RESOURCE;
1039 1035 goto spec_qpalloc_fail6;
1040 1036 }
1041 1037 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1042 1038 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1043 1039 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1044 1040 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1045 1041 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1046 1042 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1047 1043
1048 1044 /*
1049 1045 * Register the memory for the special QP work queues. The memory for
1050 1046 * the special QP must be registered in the Hermon cMPT tables. This
1051 1047 * gives us the LKey to specify in the QP context later. Note: The
1052 1048 * memory for Hermon work queues (both Send and Recv) must be contiguous
1053 1049 * and registered as a single memory region. Also, in order to meet the
1054 1050 * alignment restriction, we pass the "mro_bind_override_addr" flag in
1055 1051 * the call to hermon_mr_register(). This guarantees that the resulting
1056 1052 * IB vaddr will be zero-based (modulo the offset into the first page).
1057 1053 * If we fail here, we have a bunch of resource and reference count
1058 1054 * cleanup to do.
1059 1055 */
1060 1056 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1061 1057 IBT_MR_NOSLEEP;
1062 1058 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1063 1059 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
1064 1060 mr_attr.mr_as = NULL;
1065 1061 mr_attr.mr_flags = flag;
1066 1062
1067 1063 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
1068 1064 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1069 1065 mr_op.mro_bind_override_addr = 1;
1070 1066
1071 1067 status = hermon_mr_register(state, pd, &mr_attr, &mr, &mr_op,
1072 1068 HERMON_QP_CMPT);
1073 1069 if (status != DDI_SUCCESS) {
1074 1070 status = IBT_INSUFF_RESOURCE;
1075 1071 goto spec_qpalloc_fail6;
1076 1072 }
1077 1073
1078 1074 /*
1079 1075 * Calculate the offset between the kernel virtual address space
1080 1076 * and the IB virtual address space. This will be used when
1081 1077 * posting work requests to properly initialize each WQE.
1082 1078 */
1083 1079 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1084 1080 (uint64_t)mr->mr_bindinfo.bi_addr;
1085 1081
1086 1082 /* set the prefetch - initially, not prefetching */
1087 1083 qp->qp_no_prefetch = 1;
1088 1084
1089 1085 if (qp->qp_no_prefetch)
1090 1086 qp->qp_sq_headroom = 2 * sq_wqe_size;
1091 1087 else
1092 1088 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1093 1089 /*
1094 1090 * hdrm wqes must be integral since both sq_wqe_size &
1095 1091 * HERMON_QP_OH_SIZE are power of 2
1096 1092 */
1097 1093 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1098 1094 /*
1099 1095 * Fill in all the return arguments (if necessary). This includes
1100 1096 * real work queue sizes, real SGLs, and QP number (which will be
1101 1097 * either zero or one, depending on the special QP type)
1102 1098 */
1103 1099 if (queuesz_p != NULL) {
1104 1100 queuesz_p->cs_sq =
1105 1101 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1106 1102 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1107 1103 queuesz_p->cs_rq = (1 << log_qp_rq_size);
1108 1104 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1109 1105 }
1110 1106
1111 1107 /*
1112 1108 * Fill in the rest of the Hermon Queue Pair handle. We can update
1113 1109 * the following fields for use in further operations on the QP.
1114 1110 */
1115 1111 qp->qp_qpcrsrcp = qpc;
1116 1112 qp->qp_rsrcp = rsrc;
1117 1113 qp->qp_state = HERMON_QP_RESET;
1118 1114 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1119 1115 qp->qp_pdhdl = pd;
1120 1116 qp->qp_mrhdl = mr;
1121 1117 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1122 1118 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1123 1119 qp->qp_is_special = (type == IBT_SMI_SQP) ?
1124 1120 HERMON_QP_SMI : HERMON_QP_GSI;
1125 1121 qp->qp_uarpg = uarpg;
1126 1122 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1127 1123 qp->qp_sq_cqhdl = sq_cq;
1128 1124 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
1129 1125 qp->qp_sq_buf = sq_buf;
1130 1126 qp->qp_sq_logqsz = log_qp_sq_size;
1131 1127 qp->qp_desc_off = qp_desc_off;
1132 1128 qp->qp_rq_cqhdl = rq_cq;
1133 1129 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
1134 1130 qp->qp_rq_buf = rq_buf;
1135 1131 qp->qp_rq_logqsz = log_qp_rq_size;
1136 1132 qp->qp_portnum = port;
1137 1133 qp->qp_pkeyindx = 0;
1138 1134 qp->qp_forward_sqd_event = 0;
1139 1135 qp->qp_sqd_still_draining = 0;
1140 1136 qp->qp_hdlrarg = (void *)ibt_qphdl;
1141 1137 qp->qp_mcg_refcnt = 0;
1142 1138 qp->qp_srqhdl = NULL;
1143 1139
1144 1140 /* All special QPs are UD QP service type */
1145 1141 qp->qp_type = IBT_UD_RQP;
1146 1142 qp->qp_serv_type = HERMON_QP_UD;
1147 1143
1148 1144 /*
1149 1145 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1150 1146 */
1151 1147
1152 1148 /*
1153 1149 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1154 1150 * set the quadword to all F's - high-order bit is owner (init to one)
1155 1151 * and the rest for the headroom definition of prefetching
1156 1152 *
1157 1153 */
1158 1154
1159 1155 wqesz_shift = qp->qp_sq_log_wqesz;
1160 1156 thewqesz = 1 << wqesz_shift;
1161 1157 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1162 1158 for (i = 0; i < sq_depth; i++) {
1163 1159 /*
1164 1160 * for each stride, go through and every 64 bytes write the
1165 1161 * init value - having set the address once, just keep
1166 1162 * incrementing it
1167 1163 */
1168 1164 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1169 1165 *(uint32_t *)thewqe = 0xFFFFFFFF;
1170 1166 }
1171 1167 }
1172 1168
1173 1169
1174 1170 /* Zero out the QP context */
1175 1171 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1176 1172
1177 1173 /*
1178 1174 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
1179 1175 * "qphdl" and return success
1180 1176 */
1181 1177 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + port, qp);
1182 1178
1183 1179 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1184 1180 DDI_INTR_PRI(state->hs_intrmsi_pri));
1185 1181
1186 1182 *qphdl = qp;
1187 1183
1188 1184 return (DDI_SUCCESS);
1189 1185
1190 1186 /*
1191 1187 * The following is cleanup for all possible failure cases in this routine
1192 1188 */
1193 1189 spec_qpalloc_fail6:
1194 1190 hermon_queue_free(&qp->qp_wqinfo);
1195 1191 if (qp->qp_sq_wqhdr)
1196 1192 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1197 1193 if (qp->qp_rq_wqhdr)
1198 1194 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1199 1195 spec_qpalloc_fail5a:
1200 1196 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1201 1197 spec_qpalloc_fail5:
1202 1198 hermon_rsrc_free(state, &rsrc);
1203 1199 spec_qpalloc_fail4:
1204 1200 if (hermon_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) {
1205 1201 HERMON_WARNING(state, "failed to free special QP rsrc");
1206 1202 }
1207 1203 spec_qpalloc_fail3:
1208 1204 hermon_cq_refcnt_dec(rq_cq);
1209 1205 spec_qpalloc_fail2:
1210 1206 hermon_cq_refcnt_dec(sq_cq);
1211 1207 spec_qpalloc_fail1:
1212 1208 hermon_pd_refcnt_dec(pd);
1213 1209 spec_qpalloc_fail:
1214 1210 return (status);
1215 1211 }
1216 1212
1217 1213
1218 1214 /*
1219 1215 * hermon_qp_alloc_range()
1220 1216 * Context: Can be called only from user or kernel context.
1221 1217 */
1222 1218 int
1223 1219 hermon_qp_alloc_range(hermon_state_t *state, uint_t log2,
1224 1220 hermon_qp_info_t *qpinfo, ibtl_qp_hdl_t *ibt_qphdl,
1225 1221 ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
1226 1222 hermon_qphdl_t *qphdl, uint_t sleepflag)
1227 1223 {
1228 1224 hermon_rsrc_t *qpc, *rsrc;
1229 1225 hermon_rsrc_type_t rsrc_type;
1230 1226 hermon_qphdl_t qp;
1231 1227 hermon_qp_range_t *qp_range_p;
1232 1228 ibt_qp_alloc_attr_t *attr_p;
1233 1229 ibt_qp_type_t type;
1234 1230 hermon_qp_wq_type_t swq_type;
1235 1231 ibt_chan_sizes_t *queuesz_p;
1236 1232 ibt_mr_attr_t mr_attr;
1237 1233 hermon_mr_options_t mr_op;
1238 1234 hermon_srqhdl_t srq;
1239 1235 hermon_pdhdl_t pd;
1240 1236 hermon_cqhdl_t sq_cq, rq_cq;
1241 1237 hermon_mrhdl_t mr;
1242 1238 uint64_t qp_desc_off;
1243 1239 uint64_t *thewqe, thewqesz;
1244 1240 uint32_t *sq_buf, *rq_buf;
|
↓ open down ↓ |
345 lines elided |
↑ open up ↑ |
1245 1241 uint32_t log_qp_sq_size, log_qp_rq_size;
1246 1242 uint32_t sq_size, rq_size;
1247 1243 uint32_t sq_depth, rq_depth;
1248 1244 uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift;
1249 1245 uint32_t max_sgl, max_recv_sgl, uarpg;
1250 1246 uint_t qp_srq_en, i, j;
1251 1247 int ii; /* loop counter for range */
1252 1248 int status, flag;
1253 1249 uint_t serv_type;
1254 1250
1255 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
1256 -
1257 1251 /*
1258 1252 * Extract the necessary info from the hermon_qp_info_t structure
1259 1253 */
1260 1254 attr_p = qpinfo->qpi_attrp;
1261 1255 type = qpinfo->qpi_type;
1262 1256 queuesz_p = qpinfo->qpi_queueszp;
1263 1257
1264 1258 if (attr_p->qp_alloc_flags & IBT_QP_USES_RSS) {
1265 1259 if (log2 > state->hs_ibtfinfo.hca_attr->hca_rss_max_log2_table)
1266 1260 return (IBT_INSUFF_RESOURCE);
1267 1261 rsrc_type = HERMON_QPC;
1268 1262 serv_type = HERMON_QP_UD;
1269 1263 } else if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1270 1264 if (log2 > state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp)
1271 1265 return (IBT_INSUFF_RESOURCE);
1272 1266 switch (attr_p->qp_fc.fc_hca_port) {
1273 1267 case 1:
1274 1268 rsrc_type = HERMON_QPC_FEXCH_PORT1;
1275 1269 break;
1276 1270 case 2:
1277 1271 rsrc_type = HERMON_QPC_FEXCH_PORT2;
1278 1272 break;
1279 1273 default:
1280 1274 return (IBT_INVALID_PARAM);
1281 1275 }
1282 1276 serv_type = HERMON_QP_FEXCH;
1283 1277 } else
1284 1278 return (IBT_INVALID_PARAM);
1285 1279
1286 1280 /*
1287 1281 * Determine whether QP is being allocated for userland access or
1288 1282 * whether it is being allocated for kernel access. If the QP is
1289 1283 * being allocated for userland access, fail (too complex for now).
1290 1284 */
1291 1285 if (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) {
1292 1286 return (IBT_NOT_SUPPORTED);
1293 1287 } else {
1294 1288 uarpg = state->hs_kernel_uar_index;
1295 1289 }
1296 1290
1297 1291 /*
1298 1292 * Determine whether QP is being associated with an SRQ
1299 1293 */
1300 1294 qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
1301 1295 if (qp_srq_en) {
1302 1296 /*
1303 1297 * Check for valid SRQ handle pointers
1304 1298 */
1305 1299 if (attr_p->qp_ibc_srq_hdl == NULL) {
1306 1300 return (IBT_SRQ_HDL_INVALID);
1307 1301 }
1308 1302 srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl;
1309 1303 }
1310 1304
1311 1305 /*
1312 1306 * Check for valid QP service type (only UD supported)
1313 1307 */
1314 1308 if (type != IBT_UD_RQP) {
1315 1309 return (IBT_QP_SRV_TYPE_INVALID);
1316 1310 }
1317 1311
1318 1312 /*
1319 1313 * Check for valid PD handle pointer
1320 1314 */
1321 1315 if (attr_p->qp_pd_hdl == NULL) {
1322 1316 return (IBT_PD_HDL_INVALID);
1323 1317 }
1324 1318 pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl;
1325 1319
1326 1320 /*
1327 1321 * If on an SRQ, check to make sure the PD is the same
1328 1322 */
1329 1323 if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
1330 1324 return (IBT_PD_HDL_INVALID);
1331 1325 }
1332 1326
1333 1327 /* set loop variable here, for freeing resources on error */
1334 1328 ii = 0;
1335 1329
1336 1330 /*
1337 1331 * Allocate 2^log2 contiguous/aligned QP context entries. This will
1338 1332 * be filled in with all the necessary parameters to define the
1339 1333 * Queue Pairs. Unlike other Hermon hardware resources, ownership
1340 1334 * is not immediately given to hardware in the final step here.
1341 1335 * Instead, we must wait until the QP is later transitioned to the
1342 1336 * "Init" state before passing the QP to hardware. If we fail here,
1343 1337 * we must undo all the reference count (CQ and PD).
1344 1338 */
1345 1339 status = hermon_rsrc_alloc(state, rsrc_type, 1 << log2, sleepflag,
1346 1340 &qpc);
1347 1341 if (status != DDI_SUCCESS) {
1348 1342 return (IBT_INSUFF_RESOURCE);
1349 1343 }
1350 1344
1351 1345 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH)
1352 1346 /*
1353 1347 * Need to init the MKEYs for the FEXCH QPs.
1354 1348 *
1355 1349 * For FEXCH QP subranges, we return the QPN base as
1356 1350 * "relative" to the full FEXCH QP range for the port.
1357 1351 */
1358 1352 *(qpinfo->qpi_qpn) = hermon_fcoib_fexch_relative_qpn(state,
1359 1353 attr_p->qp_fc.fc_hca_port, qpc->hr_indx);
1360 1354 else
1361 1355 *(qpinfo->qpi_qpn) = (ib_qpn_t)qpc->hr_indx;
1362 1356
1363 1357 qp_range_p = kmem_alloc(sizeof (*qp_range_p),
1364 1358 (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1365 1359 if (qp_range_p == NULL) {
1366 1360 status = IBT_INSUFF_RESOURCE;
1367 1361 goto qpalloc_fail0;
1368 1362 }
1369 1363 mutex_init(&qp_range_p->hqpr_lock, NULL, MUTEX_DRIVER,
1370 1364 DDI_INTR_PRI(state->hs_intrmsi_pri));
1371 1365 mutex_enter(&qp_range_p->hqpr_lock);
1372 1366 qp_range_p->hqpr_refcnt = 1 << log2;
1373 1367 qp_range_p->hqpr_qpcrsrc = qpc;
1374 1368 mutex_exit(&qp_range_p->hqpr_lock);
1375 1369
1376 1370 for_each_qp:
1377 1371
1378 1372 /* Increment the reference count on the protection domain (PD) */
1379 1373 hermon_pd_refcnt_inc(pd);
1380 1374
1381 1375 rq_cq = (hermon_cqhdl_t)recv_cq[ii];
1382 1376 sq_cq = (hermon_cqhdl_t)send_cq[ii];
1383 1377 if (sq_cq == NULL) {
1384 1378 if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) {
1385 1379 /* if no send completions, just use rq_cq */
1386 1380 sq_cq = rq_cq;
1387 1381 } else {
1388 1382 status = IBT_CQ_HDL_INVALID;
1389 1383 goto qpalloc_fail1;
1390 1384 }
1391 1385 }
1392 1386
1393 1387 /*
1394 1388 * Increment the reference count on the CQs. One or both of these
1395 1389 * could return error if we determine that the given CQ is already
1396 1390 * being used with a special (SMI/GSI) QP.
1397 1391 */
1398 1392 status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL);
1399 1393 if (status != DDI_SUCCESS) {
1400 1394 status = IBT_CQ_HDL_INVALID;
1401 1395 goto qpalloc_fail1;
1402 1396 }
1403 1397 status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL);
1404 1398 if (status != DDI_SUCCESS) {
1405 1399 status = IBT_CQ_HDL_INVALID;
1406 1400 goto qpalloc_fail2;
1407 1401 }
1408 1402
1409 1403 /*
1410 1404 * Allocate the software structure for tracking the queue pair
|
↓ open down ↓ |
144 lines elided |
↑ open up ↑ |
1411 1405 * (i.e. the Hermon Queue Pair handle). If we fail here, we must
1412 1406 * undo the reference counts and the previous resource allocation.
1413 1407 */
1414 1408 status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc);
1415 1409 if (status != DDI_SUCCESS) {
1416 1410 status = IBT_INSUFF_RESOURCE;
1417 1411 goto qpalloc_fail4;
1418 1412 }
1419 1413 qp = (hermon_qphdl_t)rsrc->hr_addr;
1420 1414 bzero(qp, sizeof (struct hermon_sw_qp_s));
1421 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1422 1415 qp->qp_alloc_flags = attr_p->qp_alloc_flags;
1423 1416
1424 1417 /*
1425 1418 * Calculate the QP number from QPC index. This routine handles
1426 1419 * all of the operations necessary to keep track of used, unused,
1427 1420 * and released QP numbers.
1428 1421 */
1429 1422 qp->qp_qpnum = qpc->hr_indx + ii;
1430 1423 qp->qp_ring = qp->qp_qpnum << 8;
1431 1424 qp->qp_qpn_hdl = NULL;
1432 1425
1433 1426 /*
1434 1427 * Allocate the doorbell record. Hermon just needs one for the RQ,
1435 1428 * if the QP is not associated with an SRQ, and use uarpg (above) as
1436 1429 * the uar index
1437 1430 */
1438 1431
1439 1432 if (!qp_srq_en) {
1440 1433 status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl,
1441 1434 &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset);
1442 1435 if (status != DDI_SUCCESS) {
1443 1436 status = IBT_INSUFF_RESOURCE;
1444 1437 goto qpalloc_fail6;
1445 1438 }
1446 1439 }
1447 1440
1448 1441 qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO);
1449 1442
1450 1443 /*
1451 1444 * We verify that the requested number of SGL is valid (i.e.
1452 1445 * consistent with the device limits and/or software-configured
1453 1446 * limits). If not, then obviously the same cleanup needs to be done.
1454 1447 */
1455 1448 max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz;
1456 1449 swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD;
1457 1450 max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz;
1458 1451 if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) ||
1459 1452 (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) {
1460 1453 status = IBT_HCA_SGL_EXCEEDED;
1461 1454 goto qpalloc_fail7;
1462 1455 }
1463 1456
1464 1457 /*
1465 1458 * Determine this QP's WQE stride (for both the Send and Recv WQEs).
1466 1459 * This will depend on the requested number of SGLs. Note: this
1467 1460 * has the side-effect of also calculating the real number of SGLs
1468 1461 * (for the calculated WQE size).
1469 1462 *
1470 1463 * For QP's on an SRQ, we set these to 0.
1471 1464 */
1472 1465 if (qp_srq_en) {
1473 1466 qp->qp_rq_log_wqesz = 0;
1474 1467 qp->qp_rq_sgl = 0;
1475 1468 } else {
1476 1469 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
1477 1470 max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ,
1478 1471 &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
1479 1472 }
1480 1473 hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
1481 1474 max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
1482 1475
1483 1476 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1484 1477
1485 1478 /* NOTE: currently policy in driver, later maybe IBTF interface */
1486 1479 qp->qp_no_prefetch = 0;
1487 1480
1488 1481 /*
1489 1482 * for prefetching, we need to add the number of wqes in
1490 1483 * the 2k area plus one to the number requested, but
1491 1484 * ONLY for send queue. If no_prefetch == 1 (prefetch off)
1492 1485 * it's exactly TWO wqes for the headroom
1493 1486 */
1494 1487 if (qp->qp_no_prefetch)
1495 1488 qp->qp_sq_headroom = 2 * sq_wqe_size;
1496 1489 else
1497 1490 qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE;
1498 1491 /*
1499 1492 * hdrm wqes must be integral since both sq_wqe_size &
1500 1493 * HERMON_QP_OH_SIZE are power of 2
1501 1494 */
1502 1495 qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size);
1503 1496
1504 1497
1505 1498 /*
1506 1499 * Calculate the appropriate size for the work queues.
1507 1500 * For send queue, add in the headroom wqes to the calculation.
1508 1501 * Note: All Hermon QP work queues must be a power-of-2 in size. Also
1509 1502 * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is
1510 1503 * to round the requested size up to the next highest power-of-2
1511 1504 */
1512 1505 /* first, adjust to a minimum and tell the caller the change */
1513 1506 attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq,
1514 1507 HERMON_QP_MIN_SIZE);
1515 1508 attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq,
1516 1509 HERMON_QP_MIN_SIZE);
1517 1510 /*
1518 1511 * now, calculate the alloc size, taking into account
1519 1512 * the headroom for the sq
1520 1513 */
1521 1514 log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes);
1522 1515 /* if the total is a power of two, reduce it */
1523 1516 if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes)) {
1524 1517 log_qp_sq_size = log_qp_sq_size - 1;
1525 1518 }
1526 1519
1527 1520 log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
1528 1521 if (ISP2(attr_p->qp_sizes.cs_rq)) {
1529 1522 log_qp_rq_size = log_qp_rq_size - 1;
1530 1523 }
1531 1524
1532 1525 /*
1533 1526 * Next we verify that the rounded-up size is valid (i.e. consistent
1534 1527 * with the device limits and/or software-configured limits). If not,
1535 1528 * then obviously we have a lot of cleanup to do before returning.
1536 1529 *
1537 1530 * NOTE: the first condition deals with the (test) case of cs_sq
1538 1531 * being just less than 2^32. In this case, the headroom addition
1539 1532 * to the requested cs_sq will pass the test when it should not.
1540 1533 * This test no longer lets that case slip through the check.
1541 1534 */
1542 1535 if ((attr_p->qp_sizes.cs_sq >
1543 1536 (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) ||
1544 1537 (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) ||
1545 1538 (!qp_srq_en && (log_qp_rq_size >
1546 1539 state->hs_cfg_profile->cp_log_max_qp_sz))) {
1547 1540 status = IBT_HCA_WR_EXCEEDED;
1548 1541 goto qpalloc_fail7;
1549 1542 }
1550 1543
1551 1544 /*
1552 1545 * Allocate the memory for QP work queues. Since Hermon work queues
1553 1546 * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
1554 1547 * the work queue memory is very important. We used to allocate
1555 1548 * work queues (the combined receive and send queues) so that they
1556 1549 * would be aligned on their combined size. That alignment guaranteed
1557 1550 * that they would never cross the 4GB boundary (Hermon work queues
1558 1551 * are on the order of MBs at maximum). Now we are able to relax
1559 1552 * this alignment constraint by ensuring that the IB address assigned
1560 1553 * to the queue memory (as a result of the hermon_mr_register() call)
1561 1554 * is offset from zero.
1562 1555 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
1563 1556 * guarantee the alignment, but when attempting to use IOMMU bypass
1564 1557 * mode we found that we were not allowed to specify any alignment
1565 1558 * that was more restrictive than the system page size.
1566 1559 * So we avoided this constraint by passing two alignment values,
1567 1560 * one for the memory allocation itself and the other for the DMA
1568 1561 * handle (for later bind). This used to cause more memory than
1569 1562 * necessary to be allocated (in order to guarantee the more
1570 1563 * restrictive alignment contraint). But by guaranteeing the
1571 1564 * zero-based IB virtual address for the queue, we are able to
1572 1565 * conserve this memory.
1573 1566 */
1574 1567 sq_wqe_size = 1 << qp->qp_sq_log_wqesz;
1575 1568 sq_depth = 1 << log_qp_sq_size;
1576 1569 sq_size = sq_depth * sq_wqe_size;
1577 1570
1578 1571 /* QP on SRQ sets these to 0 */
1579 1572 if (qp_srq_en) {
1580 1573 rq_wqe_size = 0;
1581 1574 rq_size = 0;
1582 1575 } else {
1583 1576 rq_wqe_size = 1 << qp->qp_rq_log_wqesz;
1584 1577 rq_depth = 1 << log_qp_rq_size;
1585 1578 rq_size = rq_depth * rq_wqe_size;
1586 1579 }
1587 1580
1588 1581 qp->qp_wqinfo.qa_size = sq_size + rq_size;
1589 1582 qp->qp_wqinfo.qa_alloc_align = PAGESIZE;
1590 1583 qp->qp_wqinfo.qa_bind_align = PAGESIZE;
1591 1584 qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
1592 1585 status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
1593 1586 if (status != DDI_SUCCESS) {
1594 1587 status = IBT_INSUFF_RESOURCE;
1595 1588 goto qpalloc_fail7;
1596 1589 }
1597 1590
1598 1591 /*
1599 1592 * Sort WQs in memory according to stride (*q_wqe_size), largest first
1600 1593 * If they are equal, still put the SQ first
1601 1594 */
1602 1595 qp->qp_sq_baseaddr = 0;
1603 1596 qp->qp_rq_baseaddr = 0;
1604 1597 if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) {
1605 1598 sq_buf = qp->qp_wqinfo.qa_buf_aligned;
1606 1599
1607 1600 /* if this QP is on an SRQ, set the rq_buf to NULL */
1608 1601 if (qp_srq_en) {
1609 1602 rq_buf = NULL;
1610 1603 } else {
1611 1604 rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
1612 1605 qp->qp_rq_baseaddr = sq_size;
1613 1606 }
1614 1607 } else {
1615 1608 rq_buf = qp->qp_wqinfo.qa_buf_aligned;
1616 1609 sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
1617 1610 qp->qp_sq_baseaddr = rq_size;
1618 1611 }
1619 1612
1620 1613 qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth);
1621 1614 if (qp->qp_sq_wqhdr == NULL) {
1622 1615 status = IBT_INSUFF_RESOURCE;
1623 1616 goto qpalloc_fail8;
1624 1617 }
1625 1618 if (qp_srq_en) {
1626 1619 qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr;
1627 1620 qp->qp_rq_wqavl.wqa_srq_en = 1;
1628 1621 qp->qp_rq_wqavl.wqa_srq = srq;
1629 1622 } else {
1630 1623 qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth);
1631 1624 if (qp->qp_rq_wqhdr == NULL) {
1632 1625 status = IBT_INSUFF_RESOURCE;
1633 1626 goto qpalloc_fail8;
1634 1627 }
1635 1628 qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr;
1636 1629 }
1637 1630 qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum;
1638 1631 qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND;
1639 1632 qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr;
1640 1633 qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum;
1641 1634 qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV;
1642 1635
1643 1636 /*
1644 1637 * Register the memory for the QP work queues. The memory for the
1645 1638 * QP must be registered in the Hermon cMPT tables. This gives us the
1646 1639 * LKey to specify in the QP context later. Note: The memory for
1647 1640 * Hermon work queues (both Send and Recv) must be contiguous and
1648 1641 * registered as a single memory region. Note: If the QP memory is
1649 1642 * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to
1650 1643 * meet the alignment restriction, we pass the "mro_bind_override_addr"
1651 1644 * flag in the call to hermon_mr_register(). This guarantees that the
1652 1645 * resulting IB vaddr will be zero-based (modulo the offset into the
1653 1646 * first page). If we fail here, we still have the bunch of resource
1654 1647 * and reference count cleanup to do.
1655 1648 */
1656 1649 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
1657 1650 IBT_MR_NOSLEEP;
1658 1651 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1659 1652 mr_attr.mr_len = qp->qp_wqinfo.qa_size;
1660 1653 mr_attr.mr_as = NULL;
1661 1654 mr_attr.mr_flags = flag;
1662 1655 /* HERMON_QUEUE_LOCATION_NORMAL */
1663 1656 mr_op.mro_bind_type =
1664 1657 state->hs_cfg_profile->cp_iommu_bypass;
1665 1658 mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
1666 1659 mr_op.mro_bind_override_addr = 1;
1667 1660 status = hermon_mr_register(state, pd, &mr_attr, &mr,
1668 1661 &mr_op, HERMON_QP_CMPT);
1669 1662 if (status != DDI_SUCCESS) {
1670 1663 status = IBT_INSUFF_RESOURCE;
1671 1664 goto qpalloc_fail9;
1672 1665 }
1673 1666
1674 1667 /*
1675 1668 * Calculate the offset between the kernel virtual address space
1676 1669 * and the IB virtual address space. This will be used when
1677 1670 * posting work requests to properly initialize each WQE.
1678 1671 */
1679 1672 qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
1680 1673 (uint64_t)mr->mr_bindinfo.bi_addr;
1681 1674
1682 1675 /*
1683 1676 * Fill in all the return arguments (if necessary). This includes
1684 1677 * real work queue sizes (in wqes), real SGLs, and QP number
1685 1678 */
1686 1679 if (queuesz_p != NULL) {
1687 1680 queuesz_p->cs_sq =
1688 1681 (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes;
1689 1682 queuesz_p->cs_sq_sgl = qp->qp_sq_sgl;
1690 1683
1691 1684 /* if this QP is on an SRQ, set these to 0 */
1692 1685 if (qp_srq_en) {
1693 1686 queuesz_p->cs_rq = 0;
1694 1687 queuesz_p->cs_rq_sgl = 0;
1695 1688 } else {
1696 1689 queuesz_p->cs_rq = (1 << log_qp_rq_size);
1697 1690 queuesz_p->cs_rq_sgl = qp->qp_rq_sgl;
1698 1691 }
1699 1692 }
1700 1693
1701 1694 /*
1702 1695 * Fill in the rest of the Hermon Queue Pair handle.
1703 1696 */
1704 1697 qp->qp_qpcrsrcp = NULL;
1705 1698 qp->qp_rsrcp = rsrc;
1706 1699 qp->qp_state = HERMON_QP_RESET;
1707 1700 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1708 1701 qp->qp_pdhdl = pd;
1709 1702 qp->qp_mrhdl = mr;
1710 1703 qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
1711 1704 HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED;
1712 1705 qp->qp_is_special = 0;
1713 1706 qp->qp_uarpg = uarpg;
1714 1707 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1715 1708 qp->qp_sq_cqhdl = sq_cq;
1716 1709 qp->qp_sq_bufsz = (1 << log_qp_sq_size);
1717 1710 qp->qp_sq_logqsz = log_qp_sq_size;
1718 1711 qp->qp_sq_buf = sq_buf;
1719 1712 qp->qp_desc_off = qp_desc_off;
1720 1713 qp->qp_rq_cqhdl = rq_cq;
1721 1714 qp->qp_rq_buf = rq_buf;
1722 1715 qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) !=
1723 1716 0;
1724 1717
1725 1718 /* if this QP is on an SRQ, set rq_bufsz to 0 */
1726 1719 if (qp_srq_en) {
1727 1720 qp->qp_rq_bufsz = 0;
1728 1721 qp->qp_rq_logqsz = 0;
1729 1722 } else {
1730 1723 qp->qp_rq_bufsz = (1 << log_qp_rq_size);
1731 1724 qp->qp_rq_logqsz = log_qp_rq_size;
1732 1725 }
1733 1726
1734 1727 qp->qp_forward_sqd_event = 0;
1735 1728 qp->qp_sqd_still_draining = 0;
1736 1729 qp->qp_hdlrarg = (void *)ibt_qphdl[ii];
1737 1730 qp->qp_mcg_refcnt = 0;
1738 1731
1739 1732 /*
1740 1733 * If this QP is to be associated with an SRQ, set the SRQ handle
1741 1734 */
1742 1735 if (qp_srq_en) {
1743 1736 qp->qp_srqhdl = srq;
1744 1737 hermon_srq_refcnt_inc(qp->qp_srqhdl);
1745 1738 } else {
1746 1739 qp->qp_srqhdl = NULL;
1747 1740 }
1748 1741
1749 1742 qp->qp_type = IBT_UD_RQP;
1750 1743 qp->qp_serv_type = serv_type;
1751 1744
1752 1745 /*
1753 1746 * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed
1754 1747 */
1755 1748
1756 1749 /*
1757 1750 * Initialize the SQ WQEs - all that needs to be done is every 64 bytes
1758 1751 * set the quadword to all F's - high-order bit is owner (init to one)
1759 1752 * and the rest for the headroom definition of prefetching.
1760 1753 */
1761 1754 if ((attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) == 0) {
1762 1755 wqesz_shift = qp->qp_sq_log_wqesz;
1763 1756 thewqesz = 1 << wqesz_shift;
1764 1757 thewqe = (uint64_t *)(void *)(qp->qp_sq_buf);
1765 1758 for (i = 0; i < sq_depth; i++) {
1766 1759 /*
1767 1760 * for each stride, go through and every 64 bytes
1768 1761 * write the init value - having set the address
1769 1762 * once, just keep incrementing it
1770 1763 */
1771 1764 for (j = 0; j < thewqesz; j += 64, thewqe += 8) {
1772 1765 *(uint32_t *)thewqe = 0xFFFFFFFF;
1773 1766 }
1774 1767 }
1775 1768 }
1776 1769
1777 1770 /* Zero out the QP context */
1778 1771 bzero(&qp->qpc, sizeof (hermon_hw_qpc_t));
1779 1772
1780 1773 /*
1781 1774 * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the
1782 1775 * "qphdl" and return success
1783 1776 */
1784 1777 hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + ii, qp);
1785 1778
1786 1779 mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER,
1787 1780 DDI_INTR_PRI(state->hs_intrmsi_pri));
1788 1781
1789 1782 qp->qp_rangep = qp_range_p;
1790 1783
1791 1784 qphdl[ii] = qp;
1792 1785
1793 1786 if (++ii < (1 << log2))
1794 1787 goto for_each_qp;
1795 1788
1796 1789 return (DDI_SUCCESS);
1797 1790
1798 1791 /*
1799 1792 * The following is cleanup for all possible failure cases in this routine
1800 1793 */
1801 1794 qpalloc_fail9:
1802 1795 hermon_queue_free(&qp->qp_wqinfo);
1803 1796 qpalloc_fail8:
1804 1797 if (qp->qp_sq_wqhdr)
1805 1798 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
1806 1799 if (qp->qp_rq_wqhdr)
1807 1800 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
1808 1801 qpalloc_fail7:
1809 1802 if (!qp_srq_en) {
1810 1803 hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr);
1811 1804 }
1812 1805
1813 1806 qpalloc_fail6:
1814 1807 hermon_rsrc_free(state, &rsrc);
1815 1808 qpalloc_fail4:
1816 1809 hermon_cq_refcnt_dec(rq_cq);
1817 1810 qpalloc_fail2:
1818 1811 hermon_cq_refcnt_dec(sq_cq);
1819 1812 qpalloc_fail1:
1820 1813 hermon_pd_refcnt_dec(pd);
1821 1814 qpalloc_fail0:
1822 1815 if (ii == 0) {
1823 1816 if (qp_range_p)
1824 1817 kmem_free(qp_range_p, sizeof (*qp_range_p));
1825 1818 hermon_rsrc_free(state, &qpc);
1826 1819 } else {
1827 1820 /* qp_range_p and qpc rsrc will be freed in hermon_qp_free */
1828 1821
1829 1822 mutex_enter(&qp->qp_rangep->hqpr_lock);
1830 1823 qp_range_p->hqpr_refcnt = ii;
1831 1824 mutex_exit(&qp->qp_rangep->hqpr_lock);
1832 1825 while (--ii >= 0) {
1833 1826 ibc_qpn_hdl_t qpn_hdl;
1834 1827 int free_status;
1835 1828
1836 1829 free_status = hermon_qp_free(state, &qphdl[ii],
1837 1830 IBC_FREE_QP_AND_QPN, &qpn_hdl, sleepflag);
1838 1831 if (free_status != DDI_SUCCESS)
1839 1832 cmn_err(CE_CONT, "!qp_range: status 0x%x: "
1840 1833 "error status %x during free",
1841 1834 status, free_status);
1842 1835 }
1843 1836 }
1844 1837
1845 1838 return (status);
1846 1839 }
1847 1840
1848 1841
1849 1842 /*
1850 1843 * hermon_qp_free()
1851 1844 * This function frees up the QP resources. Depending on the value
1852 1845 * of the "free_qp_flags", the QP number may not be released until
1853 1846 * a subsequent call to hermon_qp_release_qpn().
1854 1847 *
1855 1848 * Context: Can be called only from user or kernel context.
1856 1849 */
1857 1850 /* ARGSUSED */
1858 1851 int
1859 1852 hermon_qp_free(hermon_state_t *state, hermon_qphdl_t *qphdl,
1860 1853 ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh,
1861 1854 uint_t sleepflag)
1862 1855 {
1863 1856 hermon_rsrc_t *qpc, *rsrc;
1864 1857 hermon_umap_db_entry_t *umapdb;
1865 1858 hermon_qpn_entry_t *entry;
1866 1859 hermon_pdhdl_t pd;
1867 1860 hermon_mrhdl_t mr;
1868 1861 hermon_cqhdl_t sq_cq, rq_cq;
1869 1862 hermon_srqhdl_t srq;
1870 1863 hermon_qphdl_t qp;
1871 1864 uint64_t value;
1872 1865 uint_t type, port;
1873 1866 uint_t maxprot;
1874 1867 uint_t qp_srq_en;
1875 1868 int status;
1876 1869
1877 1870 /*
1878 1871 * Pull all the necessary information from the Hermon Queue Pair
1879 1872 * handle. This is necessary here because the resource for the
1880 1873 * QP handle is going to be freed up as part of this operation.
1881 1874 */
1882 1875 qp = *qphdl;
1883 1876 mutex_enter(&qp->qp_lock);
1884 1877 qpc = qp->qp_qpcrsrcp; /* NULL if part of a "range" */
1885 1878 rsrc = qp->qp_rsrcp;
1886 1879 pd = qp->qp_pdhdl;
1887 1880 srq = qp->qp_srqhdl;
1888 1881 mr = qp->qp_mrhdl;
1889 1882 rq_cq = qp->qp_rq_cqhdl;
1890 1883 sq_cq = qp->qp_sq_cqhdl;
1891 1884 port = qp->qp_portnum;
1892 1885 qp_srq_en = qp->qp_alloc_flags & IBT_QP_USES_SRQ;
1893 1886
1894 1887 /*
1895 1888 * If the QP is part of an MCG, then we fail the qp_free
1896 1889 */
1897 1890 if (qp->qp_mcg_refcnt != 0) {
1898 1891 mutex_exit(&qp->qp_lock);
1899 1892 status = ibc_get_ci_failure(0);
1900 1893 goto qpfree_fail;
1901 1894 }
1902 1895
1903 1896 /*
1904 1897 * If the QP is not already in "Reset" state, then transition to
1905 1898 * "Reset". This is necessary because software does not reclaim
1906 1899 * ownership of the QP context until the QP is in the "Reset" state.
1907 1900 * If the ownership transfer fails for any reason, then it is an
1908 1901 * indication that something (either in HW or SW) has gone seriously
1909 1902 * wrong. So we print a warning message and return.
1910 1903 */
1911 1904 if (qp->qp_state != HERMON_QP_RESET) {
1912 1905 if (hermon_qp_to_reset(state, qp) != DDI_SUCCESS) {
1913 1906 mutex_exit(&qp->qp_lock);
1914 1907 HERMON_WARNING(state, "failed to reset QP context");
1915 1908 status = ibc_get_ci_failure(0);
1916 1909 goto qpfree_fail;
1917 1910 }
1918 1911 qp->qp_state = HERMON_QP_RESET;
1919 1912 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET);
1920 1913
1921 1914 /*
1922 1915 * Do any additional handling necessary for the transition
1923 1916 * to the "Reset" state (e.g. update the WRID lists)
1924 1917 */
1925 1918 if (hermon_wrid_to_reset_handling(state, qp) != DDI_SUCCESS) {
1926 1919 mutex_exit(&qp->qp_lock);
1927 1920 HERMON_WARNING(state, "failed to reset QP WRID list");
1928 1921 status = ibc_get_ci_failure(0);
1929 1922 goto qpfree_fail;
1930 1923 }
1931 1924 }
1932 1925
1933 1926 /*
1934 1927 * If this was a user-mappable QP, then we need to remove its entry
1935 1928 * from the "userland resources database". If it is also currently
1936 1929 * mmap()'d out to a user process, then we need to call
1937 1930 * devmap_devmem_remap() to remap the QP memory to an invalid mapping.
1938 1931 * We also need to invalidate the QP tracking information for the
1939 1932 * user mapping.
1940 1933 */
1941 1934 if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
1942 1935 status = hermon_umap_db_find(state->hs_instance, qp->qp_qpnum,
1943 1936 MLNX_UMAP_QPMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
1944 1937 &umapdb);
1945 1938 if (status != DDI_SUCCESS) {
1946 1939 mutex_exit(&qp->qp_lock);
1947 1940 HERMON_WARNING(state, "failed to find in database");
1948 1941 return (ibc_get_ci_failure(0));
1949 1942 }
1950 1943 hermon_umap_db_free(umapdb);
1951 1944 if (qp->qp_umap_dhp != NULL) {
1952 1945 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1953 1946 status = devmap_devmem_remap(qp->qp_umap_dhp,
1954 1947 state->hs_dip, 0, 0, qp->qp_wqinfo.qa_size,
1955 1948 maxprot, DEVMAP_MAPPING_INVALID, NULL);
1956 1949 if (status != DDI_SUCCESS) {
1957 1950 mutex_exit(&qp->qp_lock);
1958 1951 HERMON_WARNING(state, "failed in QP memory "
1959 1952 "devmap_devmem_remap()");
1960 1953 return (ibc_get_ci_failure(0));
1961 1954 }
1962 1955 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
1963 1956 }
1964 1957 }
1965 1958
1966 1959
1967 1960 /*
1968 1961 * Put NULL into the Hermon QPNum-to-QPHdl list. This will allow any
1969 1962 * in-progress events to detect that the QP corresponding to this
1970 1963 * number has been freed. Note: it does depend in whether we are
1971 1964 * freeing a special QP or not.
1972 1965 */
1973 1966 if (qpc == NULL) {
1974 1967 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1975 1968 qp->qp_qpnum, NULL);
1976 1969 } else if (qp->qp_is_special) {
1977 1970 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1978 1971 qpc->hr_indx + port, NULL);
1979 1972 } else {
1980 1973 hermon_icm_set_num_to_hdl(state, HERMON_QPC,
1981 1974 qpc->hr_indx, NULL);
1982 1975 }
|
↓ open down ↓ |
551 lines elided |
↑ open up ↑ |
1983 1976
1984 1977 /*
1985 1978 * Drop the QP lock
1986 1979 * At this point the lock is no longer necessary. We cannot
1987 1980 * protect from multiple simultaneous calls to free the same QP.
1988 1981 * In addition, since the QP lock is contained in the QP "software
1989 1982 * handle" resource, which we will free (see below), it is
1990 1983 * important that we have no further references to that memory.
1991 1984 */
1992 1985 mutex_exit(&qp->qp_lock);
1993 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp))
1994 1986
1995 1987 /*
1996 1988 * Free the QP resources
1997 1989 * Start by deregistering and freeing the memory for work queues.
1998 1990 * Next free any previously allocated context information
1999 1991 * (depending on QP type)
2000 1992 * Finally, decrement the necessary reference counts.
2001 1993 * If this fails for any reason, then it is an indication that
2002 1994 * something (either in HW or SW) has gone seriously wrong. So we
2003 1995 * print a warning message and return.
2004 1996 */
2005 1997 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2006 1998 sleepflag);
2007 1999 if (status != DDI_SUCCESS) {
2008 2000 HERMON_WARNING(state, "failed to deregister QP memory");
2009 2001 status = ibc_get_ci_failure(0);
2010 2002 goto qpfree_fail;
2011 2003 }
2012 2004
2013 2005 /* Free the memory for the QP */
2014 2006 hermon_queue_free(&qp->qp_wqinfo);
2015 2007
2016 2008 if (qp->qp_sq_wqhdr)
2017 2009 hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr);
2018 2010 if (qp->qp_rq_wqhdr)
2019 2011 hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr);
2020 2012
2021 2013 /* Free the dbr */
2022 2014 if (!qp_srq_en) {
2023 2015 hermon_dbr_free(state, qp->qp_uarpg, qp->qp_rq_vdbr);
2024 2016 }
2025 2017
2026 2018 /*
2027 2019 * Free up the remainder of the QP resources. Note: we have a few
2028 2020 * different resources to free up depending on whether the QP is a
2029 2021 * special QP or not. As described above, if any of these fail for
2030 2022 * any reason it is an indication that something (either in HW or SW)
2031 2023 * has gone seriously wrong. So we print a warning message and
2032 2024 * return.
2033 2025 */
2034 2026 if (qp->qp_is_special) {
2035 2027 type = (qp->qp_is_special == HERMON_QP_SMI) ?
2036 2028 IBT_SMI_SQP : IBT_GSI_SQP;
2037 2029
2038 2030 /* Free up resources for the special QP */
2039 2031 status = hermon_special_qp_rsrc_free(state, type, port);
2040 2032 if (status != DDI_SUCCESS) {
2041 2033 HERMON_WARNING(state, "failed to free special QP rsrc");
2042 2034 status = ibc_get_ci_failure(0);
2043 2035 goto qpfree_fail;
2044 2036 }
2045 2037
2046 2038 } else if (qp->qp_rangep) {
2047 2039 int refcnt;
2048 2040 mutex_enter(&qp->qp_rangep->hqpr_lock);
2049 2041 refcnt = --qp->qp_rangep->hqpr_refcnt;
2050 2042 mutex_exit(&qp->qp_rangep->hqpr_lock);
2051 2043 if (refcnt == 0) {
2052 2044 mutex_destroy(&qp->qp_rangep->hqpr_lock);
2053 2045 hermon_rsrc_free(state, &qp->qp_rangep->hqpr_qpcrsrc);
2054 2046 kmem_free(qp->qp_rangep, sizeof (*qp->qp_rangep));
2055 2047 }
2056 2048 qp->qp_rangep = NULL;
2057 2049 } else if (qp->qp_qpn_hdl == NULL) {
2058 2050 hermon_rsrc_free(state, &qpc);
2059 2051 } else {
2060 2052 /*
2061 2053 * Check the flags and determine whether to release the
2062 2054 * QPN or not, based on their value.
2063 2055 */
2064 2056 if (free_qp_flags == IBC_FREE_QP_ONLY) {
2065 2057 entry = qp->qp_qpn_hdl;
2066 2058 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2067 2059 HERMON_QPN_FREE_ONLY);
2068 2060 *qpnh = (ibc_qpn_hdl_t)entry;
2069 2061 } else {
2070 2062 hermon_qp_release_qpn(state, qp->qp_qpn_hdl,
2071 2063 HERMON_QPN_RELEASE);
2072 2064 }
2073 2065 }
2074 2066
2075 2067 mutex_destroy(&qp->qp_sq_lock);
2076 2068
2077 2069 /* Free the Hermon Queue Pair handle */
2078 2070 hermon_rsrc_free(state, &rsrc);
2079 2071
2080 2072 /* Decrement the reference counts on CQs, PD and SRQ (if needed) */
2081 2073 hermon_cq_refcnt_dec(rq_cq);
2082 2074 hermon_cq_refcnt_dec(sq_cq);
2083 2075 hermon_pd_refcnt_dec(pd);
2084 2076 if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
2085 2077 hermon_srq_refcnt_dec(srq);
2086 2078 }
2087 2079
2088 2080 /* Set the qphdl pointer to NULL and return success */
2089 2081 *qphdl = NULL;
2090 2082
2091 2083 return (DDI_SUCCESS);
2092 2084
2093 2085 qpfree_fail:
2094 2086 return (status);
2095 2087 }
2096 2088
2097 2089
2098 2090 /*
2099 2091 * hermon_qp_query()
2100 2092 * Context: Can be called from interrupt or base context.
2101 2093 */
2102 2094 int
2103 2095 hermon_qp_query(hermon_state_t *state, hermon_qphdl_t qp,
2104 2096 ibt_qp_query_attr_t *attr_p)
2105 2097 {
2106 2098 ibt_cep_state_t qp_state;
2107 2099 ibt_qp_ud_attr_t *ud;
2108 2100 ibt_qp_rc_attr_t *rc;
2109 2101 ibt_qp_uc_attr_t *uc;
2110 2102 ibt_cep_flags_t enable_flags;
2111 2103 hermon_hw_addr_path_t *qpc_path, *qpc_alt_path;
2112 2104 ibt_cep_path_t *path_ptr, *alt_path_ptr;
2113 2105 hermon_hw_qpc_t *qpc;
2114 2106 int status;
2115 2107 uint_t tmp_sched_q, tmp_alt_sched_q;
2116 2108
2117 2109 mutex_enter(&qp->qp_lock);
2118 2110
2119 2111 /*
2120 2112 * Grab the temporary QPC entry from QP software state
2121 2113 */
2122 2114 qpc = &qp->qpc;
2123 2115
2124 2116 /* Convert the current Hermon QP state to IBTF QP state */
2125 2117 switch (qp->qp_state) {
2126 2118 case HERMON_QP_RESET:
2127 2119 qp_state = IBT_STATE_RESET; /* "Reset" */
2128 2120 break;
2129 2121 case HERMON_QP_INIT:
2130 2122 qp_state = IBT_STATE_INIT; /* Initialized */
2131 2123 break;
2132 2124 case HERMON_QP_RTR:
2133 2125 qp_state = IBT_STATE_RTR; /* Ready to Receive */
2134 2126 break;
2135 2127 case HERMON_QP_RTS:
2136 2128 qp_state = IBT_STATE_RTS; /* Ready to Send */
2137 2129 break;
2138 2130 case HERMON_QP_SQERR:
2139 2131 qp_state = IBT_STATE_SQE; /* Send Queue Error */
2140 2132 break;
2141 2133 case HERMON_QP_SQD:
2142 2134 if (qp->qp_sqd_still_draining) {
2143 2135 qp_state = IBT_STATE_SQDRAIN; /* SQ Draining */
2144 2136 } else {
2145 2137 qp_state = IBT_STATE_SQD; /* SQ Drained */
2146 2138 }
2147 2139 break;
2148 2140 case HERMON_QP_ERR:
2149 2141 qp_state = IBT_STATE_ERROR; /* Error */
2150 2142 break;
2151 2143 default:
2152 2144 mutex_exit(&qp->qp_lock);
2153 2145 return (ibc_get_ci_failure(0));
2154 2146 }
2155 2147 attr_p->qp_info.qp_state = qp_state;
2156 2148
2157 2149 /* SRQ Hook. */
2158 2150 attr_p->qp_srq = NULL;
2159 2151
2160 2152 /*
2161 2153 * The following QP information is always returned, regardless of
2162 2154 * the current QP state. Note: Some special handling is necessary
2163 2155 * for calculating the QP number on special QP (QP0 and QP1).
2164 2156 */
2165 2157 attr_p->qp_sq_cq =
2166 2158 (qp->qp_sq_cqhdl == NULL) ? NULL : qp->qp_sq_cqhdl->cq_hdlrarg;
2167 2159 attr_p->qp_rq_cq =
2168 2160 (qp->qp_rq_cqhdl == NULL) ? NULL : qp->qp_rq_cqhdl->cq_hdlrarg;
2169 2161 if (qp->qp_is_special) {
2170 2162 attr_p->qp_qpn = (qp->qp_is_special == HERMON_QP_SMI) ? 0 : 1;
2171 2163 } else {
2172 2164 attr_p->qp_qpn = (ib_qpn_t)qp->qp_qpnum;
2173 2165 }
2174 2166 attr_p->qp_sq_sgl = qp->qp_sq_sgl;
2175 2167 attr_p->qp_rq_sgl = qp->qp_rq_sgl;
2176 2168 attr_p->qp_info.qp_sq_sz = qp->qp_sq_bufsz - qp->qp_sq_hdrmwqes;
2177 2169 attr_p->qp_info.qp_rq_sz = qp->qp_rq_bufsz;
2178 2170
2179 2171 /*
2180 2172 * If QP is currently in the "Reset" state, then only the above are
2181 2173 * returned
2182 2174 */
2183 2175 if (qp_state == IBT_STATE_RESET) {
2184 2176 mutex_exit(&qp->qp_lock);
2185 2177 return (DDI_SUCCESS);
2186 2178 }
2187 2179
2188 2180 /*
2189 2181 * Post QUERY_QP command to firmware
2190 2182 *
2191 2183 * We do a HERMON_NOSLEEP here because we are holding the "qp_lock".
2192 2184 * Since we may be in the interrupt context (or subsequently raised
2193 2185 * to interrupt level by priority inversion), we do not want to block
2194 2186 * in this routine waiting for success.
2195 2187 */
2196 2188 tmp_sched_q = qpc->pri_addr_path.sched_q;
2197 2189 tmp_alt_sched_q = qpc->alt_addr_path.sched_q;
2198 2190 status = hermon_cmn_query_cmd_post(state, QUERY_QP, 0, qp->qp_qpnum,
2199 2191 qpc, sizeof (hermon_hw_qpc_t), HERMON_CMD_NOSLEEP_SPIN);
2200 2192 if (status != HERMON_CMD_SUCCESS) {
2201 2193 mutex_exit(&qp->qp_lock);
2202 2194 cmn_err(CE_WARN, "hermon%d: hermon_qp_query: QUERY_QP "
2203 2195 "command failed: %08x\n", state->hs_instance, status);
2204 2196 if (status == HERMON_CMD_INVALID_STATUS) {
2205 2197 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2206 2198 }
2207 2199 return (ibc_get_ci_failure(0));
2208 2200 }
2209 2201 qpc->pri_addr_path.sched_q = tmp_sched_q;
2210 2202 qpc->alt_addr_path.sched_q = tmp_alt_sched_q;
2211 2203
2212 2204 /*
2213 2205 * Fill in the additional QP info based on the QP's transport type.
2214 2206 */
2215 2207 if (qp->qp_type == IBT_UD_RQP) {
2216 2208
2217 2209 /* Fill in the UD-specific info */
2218 2210 ud = &attr_p->qp_info.qp_transport.ud;
2219 2211 ud->ud_qkey = (ib_qkey_t)qpc->qkey;
2220 2212 ud->ud_sq_psn = qpc->next_snd_psn;
2221 2213 ud->ud_pkey_ix = qpc->pri_addr_path.pkey_indx;
|
↓ open down ↓ |
218 lines elided |
↑ open up ↑ |
2222 2214 /* port+1 for port 1/2 */
2223 2215 ud->ud_port =
2224 2216 (uint8_t)(((qpc->pri_addr_path.sched_q >> 6) & 0x01) + 1);
2225 2217
2226 2218 attr_p->qp_info.qp_trans = IBT_UD_SRV;
2227 2219
2228 2220 if (qp->qp_serv_type == HERMON_QP_FEXCH) {
2229 2221 ibt_pmr_desc_t *pmr;
2230 2222 uint64_t heart_beat;
2231 2223
2232 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
2233 2224 pmr = &attr_p->qp_query_fexch.fq_uni_mem_desc;
2234 2225 pmr->pmd_iova = 0;
2235 2226 pmr->pmd_lkey = pmr->pmd_rkey =
2236 2227 hermon_fcoib_qpn_to_mkey(state, qp->qp_qpnum);
2237 2228 pmr->pmd_phys_buf_list_sz =
2238 2229 state->hs_fcoib.hfc_mtts_per_mpt;
2239 2230 pmr->pmd_sync_required = 0;
2240 2231
2241 2232 pmr = &attr_p->qp_query_fexch.fq_bi_mem_desc;
2242 2233 pmr->pmd_iova = 0;
2243 2234 pmr->pmd_lkey = 0;
2244 2235 pmr->pmd_rkey = 0;
2245 2236 pmr->pmd_phys_buf_list_sz = 0;
2246 2237 pmr->pmd_sync_required = 0;
2247 2238
2248 2239 attr_p->qp_query_fexch.fq_flags =
2249 2240 ((hermon_get_heart_beat_rq_cmd_post(state,
2250 2241 qp->qp_qpnum, &heart_beat) == HERMON_CMD_SUCCESS) &&
2251 2242 (heart_beat == 0)) ? IBT_FEXCH_HEART_BEAT_OK :
2252 2243 IBT_FEXCH_NO_FLAGS;
2253 2244
2254 2245 ud->ud_fc = qp->qp_fc_attr;
2255 2246 } else if (qp->qp_serv_type == HERMON_QP_FCMND ||
2256 2247 qp->qp_serv_type == HERMON_QP_RFCI) {
2257 2248 ud->ud_fc = qp->qp_fc_attr;
2258 2249 }
2259 2250
2260 2251 } else if (qp->qp_serv_type == HERMON_QP_RC) {
2261 2252
2262 2253 /* Fill in the RC-specific info */
2263 2254 rc = &attr_p->qp_info.qp_transport.rc;
2264 2255 rc->rc_sq_psn = qpc->next_snd_psn;
2265 2256 rc->rc_rq_psn = qpc->next_rcv_psn;
2266 2257 rc->rc_dst_qpn = qpc->rem_qpn;
2267 2258
2268 2259 /* Grab the path migration state information */
2269 2260 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2270 2261 rc->rc_mig_state = IBT_STATE_MIGRATED;
2271 2262 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2272 2263 rc->rc_mig_state = IBT_STATE_REARMED;
2273 2264 } else {
2274 2265 rc->rc_mig_state = IBT_STATE_ARMED;
2275 2266 }
2276 2267 rc->rc_rdma_ra_out = (1 << qpc->sra_max);
2277 2268 rc->rc_rdma_ra_in = (1 << qpc->rra_max);
2278 2269 rc->rc_min_rnr_nak = qpc->min_rnr_nak;
2279 2270 rc->rc_path_mtu = qpc->mtu;
2280 2271 rc->rc_retry_cnt = qpc->retry_cnt;
2281 2272
2282 2273 /* Get the common primary address path fields */
2283 2274 qpc_path = &qpc->pri_addr_path;
2284 2275 path_ptr = &rc->rc_path;
2285 2276 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2286 2277 HERMON_ADDRPATH_QP);
2287 2278
2288 2279 /* Fill in the additional primary address path fields */
2289 2280 path_ptr->cep_pkey_ix = qpc_path->pkey_indx;
2290 2281 path_ptr->cep_hca_port_num =
2291 2282 path_ptr->cep_adds_vect.av_port_num =
2292 2283 (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2293 2284 path_ptr->cep_timeout = qpc_path->ack_timeout;
2294 2285
2295 2286 /* Get the common alternate address path fields */
2296 2287 qpc_alt_path = &qpc->alt_addr_path;
2297 2288 alt_path_ptr = &rc->rc_alt_path;
2298 2289 hermon_get_addr_path(state, qpc_alt_path,
2299 2290 &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2300 2291
2301 2292 /* Fill in the additional alternate address path fields */
2302 2293 alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
2303 2294 alt_path_ptr->cep_hca_port_num =
2304 2295 alt_path_ptr->cep_adds_vect.av_port_num =
2305 2296 (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2306 2297 alt_path_ptr->cep_timeout = qpc_alt_path->ack_timeout;
2307 2298
2308 2299 /* Get the RNR retry time from primary path */
2309 2300 rc->rc_rnr_retry_cnt = qpc->rnr_retry;
2310 2301
2311 2302 /* Set the enable flags based on RDMA/Atomic enable bits */
2312 2303 enable_flags = IBT_CEP_NO_FLAGS;
2313 2304 enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD);
2314 2305 enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2315 2306 enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC);
2316 2307 attr_p->qp_info.qp_flags = enable_flags;
2317 2308
2318 2309 attr_p->qp_info.qp_trans = IBT_RC_SRV;
2319 2310
2320 2311 } else if (qp->qp_serv_type == HERMON_QP_UC) {
2321 2312
2322 2313 /* Fill in the UC-specific info */
2323 2314 uc = &attr_p->qp_info.qp_transport.uc;
2324 2315 uc->uc_sq_psn = qpc->next_snd_psn;
2325 2316 uc->uc_rq_psn = qpc->next_rcv_psn;
2326 2317 uc->uc_dst_qpn = qpc->rem_qpn;
2327 2318
2328 2319 /* Grab the path migration state information */
2329 2320 if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) {
2330 2321 uc->uc_mig_state = IBT_STATE_MIGRATED;
2331 2322 } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) {
2332 2323 uc->uc_mig_state = IBT_STATE_REARMED;
2333 2324 } else {
2334 2325 uc->uc_mig_state = IBT_STATE_ARMED;
2335 2326 }
2336 2327 uc->uc_path_mtu = qpc->mtu;
2337 2328
2338 2329 /* Get the common primary address path fields */
2339 2330 qpc_path = &qpc->pri_addr_path;
2340 2331 path_ptr = &uc->uc_path;
2341 2332 hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
2342 2333 HERMON_ADDRPATH_QP);
2343 2334
2344 2335 /* Fill in the additional primary address path fields */
2345 2336 path_ptr->cep_pkey_ix = qpc_path->pkey_indx;
2346 2337 path_ptr->cep_hca_port_num =
2347 2338 path_ptr->cep_adds_vect.av_port_num =
2348 2339 (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1);
2349 2340
2350 2341 /* Get the common alternate address path fields */
2351 2342 qpc_alt_path = &qpc->alt_addr_path;
2352 2343 alt_path_ptr = &uc->uc_alt_path;
2353 2344 hermon_get_addr_path(state, qpc_alt_path,
2354 2345 &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP);
2355 2346
2356 2347 /* Fill in the additional alternate address path fields */
2357 2348 alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
2358 2349 alt_path_ptr->cep_hca_port_num =
2359 2350 alt_path_ptr->cep_adds_vect.av_port_num =
2360 2351 (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1);
2361 2352
2362 2353 /*
2363 2354 * Set the enable flags based on RDMA enable bits (by
2364 2355 * definition UC doesn't support Atomic or RDMA Read)
2365 2356 */
2366 2357 enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
2367 2358 attr_p->qp_info.qp_flags = enable_flags;
2368 2359
2369 2360 attr_p->qp_info.qp_trans = IBT_UC_SRV;
2370 2361
2371 2362 } else {
2372 2363 HERMON_WARNING(state, "unexpected QP transport type");
2373 2364 mutex_exit(&qp->qp_lock);
2374 2365 return (ibc_get_ci_failure(0));
2375 2366 }
2376 2367
2377 2368 /*
2378 2369 * Under certain circumstances it is possible for the Hermon hardware
2379 2370 * to transition to one of the error states without software directly
2380 2371 * knowing about it. The QueryQP() call is the one place where we
2381 2372 * have an opportunity to sample and update our view of the QP state.
2382 2373 */
2383 2374 if (qpc->state == HERMON_QP_SQERR) {
2384 2375 attr_p->qp_info.qp_state = IBT_STATE_SQE;
2385 2376 qp->qp_state = HERMON_QP_SQERR;
2386 2377 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_SQERR);
2387 2378 }
2388 2379 if (qpc->state == HERMON_QP_ERR) {
2389 2380 attr_p->qp_info.qp_state = IBT_STATE_ERROR;
2390 2381 qp->qp_state = HERMON_QP_ERR;
2391 2382 HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_ERR);
2392 2383 }
2393 2384 mutex_exit(&qp->qp_lock);
2394 2385
2395 2386 return (DDI_SUCCESS);
2396 2387 }
2397 2388
2398 2389
2399 2390 /*
2400 2391 * hermon_qp_create_qpn()
2401 2392 * Context: Can be called from interrupt or base context.
2402 2393 */
2403 2394 static int
2404 2395 hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp,
2405 2396 hermon_rsrc_t *qpc)
2406 2397 {
2407 2398 hermon_qpn_entry_t query;
2408 2399 hermon_qpn_entry_t *entry;
2409 2400 avl_index_t where;
2410 2401
2411 2402 /*
2412 2403 * Build a query (for the AVL tree lookup) and attempt to find
2413 2404 * a previously added entry that has a matching QPC index. If
2414 2405 * no matching entry is found, then allocate, initialize, and
2415 2406 * add an entry to the AVL tree.
2416 2407 * If a matching entry is found, then increment its QPN counter
2417 2408 * and reference counter.
2418 2409 */
2419 2410 query.qpn_indx = qpc->hr_indx;
2420 2411 mutex_enter(&state->hs_qpn_avl_lock);
2421 2412 entry = (hermon_qpn_entry_t *)avl_find(&state->hs_qpn_avl,
2422 2413 &query, &where);
2423 2414 if (entry == NULL) {
|
↓ open down ↓ |
181 lines elided |
↑ open up ↑ |
2424 2415 /*
2425 2416 * Allocate and initialize a QPN entry, then insert
2426 2417 * it into the AVL tree.
2427 2418 */
2428 2419 entry = (hermon_qpn_entry_t *)kmem_zalloc(
2429 2420 sizeof (hermon_qpn_entry_t), KM_NOSLEEP);
2430 2421 if (entry == NULL) {
2431 2422 mutex_exit(&state->hs_qpn_avl_lock);
2432 2423 return (DDI_FAILURE);
2433 2424 }
2434 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
2435 2425
2436 2426 entry->qpn_indx = qpc->hr_indx;
2437 2427 entry->qpn_refcnt = 0;
2438 2428 entry->qpn_counter = 0;
2439 2429
2440 2430 avl_insert(&state->hs_qpn_avl, entry, where);
2441 2431 }
2442 2432
2443 2433 /*
2444 2434 * Make the AVL tree entry point to the QP context resource that
2445 2435 * it will be responsible for tracking
2446 2436 */
2447 2437 entry->qpn_qpc = qpc;
2448 2438
2449 2439 /*
2450 2440 * Setup the QP handle to point to the AVL tree entry. Then
2451 2441 * generate the new QP number from the entry's QPN counter value
2452 2442 * and the hardware's QP context table index.
2453 2443 */
2454 2444 qp->qp_qpn_hdl = entry;
2455 2445 qp->qp_qpnum = ((entry->qpn_counter <<
2456 2446 state->hs_cfg_profile->cp_log_num_qp) | qpc->hr_indx) &
2457 2447 HERMON_QP_MAXNUMBER_MSK;
2458 2448 qp->qp_ring = qp->qp_qpnum << 8;
2459 2449
2460 2450 /*
2461 2451 * Increment the reference counter and QPN counter. The QPN
2462 2452 * counter always indicates the next available number for use.
2463 2453 */
2464 2454 entry->qpn_counter++;
2465 2455 entry->qpn_refcnt++;
2466 2456
2467 2457 mutex_exit(&state->hs_qpn_avl_lock);
2468 2458
2469 2459 return (DDI_SUCCESS);
2470 2460 }
2471 2461
2472 2462
2473 2463 /*
2474 2464 * hermon_qp_release_qpn()
2475 2465 * Context: Can be called only from user or kernel context.
2476 2466 */
2477 2467 void
2478 2468 hermon_qp_release_qpn(hermon_state_t *state, hermon_qpn_entry_t *entry,
2479 2469 int flags)
2480 2470 {
2481 2471 ASSERT(entry != NULL);
2482 2472
2483 2473 mutex_enter(&state->hs_qpn_avl_lock);
2484 2474
2485 2475 /*
2486 2476 * If we are releasing the QP number here, then we decrement the
2487 2477 * reference count and check for zero references. If there are
2488 2478 * zero references, then we free the QPC context (if it hadn't
2489 2479 * already been freed during a HERMON_QPN_FREE_ONLY free, i.e. for
2490 2480 * reuse with another similar QP number) and remove the tracking
2491 2481 * structure from the QP number AVL tree and free the structure.
2492 2482 * If we are not releasing the QP number here, then, as long as we
2493 2483 * have not exhausted the usefulness of the QPC context (that is,
2494 2484 * re-used it too many times without the reference count having
2495 2485 * gone to zero), we free up the QPC context for use by another
2496 2486 * thread (which will use it to construct a different QP number
2497 2487 * from the same QPC table index).
2498 2488 */
2499 2489 if (flags == HERMON_QPN_RELEASE) {
2500 2490 entry->qpn_refcnt--;
2501 2491
2502 2492 /*
2503 2493 * If the reference count is zero, then we free the QPC
2504 2494 * context (if it hadn't already been freed in an early
2505 2495 * step, e.g. HERMON_QPN_FREE_ONLY) and remove/free the
2506 2496 * tracking structure from the QP number AVL tree.
2507 2497 */
2508 2498 if (entry->qpn_refcnt == 0) {
2509 2499 if (entry->qpn_qpc != NULL) {
2510 2500 hermon_rsrc_free(state, &entry->qpn_qpc);
2511 2501 }
2512 2502
2513 2503 /*
2514 2504 * If the current entry has served it's useful
2515 2505 * purpose (i.e. been reused the maximum allowable
2516 2506 * number of times), then remove it from QP number
2517 2507 * AVL tree and free it up.
2518 2508 */
2519 2509 if (entry->qpn_counter >= (1 <<
2520 2510 (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2521 2511 avl_remove(&state->hs_qpn_avl, entry);
2522 2512 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2523 2513 }
2524 2514 }
2525 2515
2526 2516 } else if (flags == HERMON_QPN_FREE_ONLY) {
2527 2517 /*
2528 2518 * Even if we are not freeing the QP number, that will not
2529 2519 * always prevent us from releasing the QPC context. In fact,
2530 2520 * since the QPC context only forms part of the whole QPN,
2531 2521 * we want to free it up for use by other consumers. But
2532 2522 * if the reference count is non-zero (which it will always
2533 2523 * be when we are doing HERMON_QPN_FREE_ONLY) and the counter
2534 2524 * has reached its maximum value, then we cannot reuse the
2535 2525 * QPC context until the reference count eventually reaches
2536 2526 * zero (in HERMON_QPN_RELEASE, above).
2537 2527 */
2538 2528 if (entry->qpn_counter < (1 <<
2539 2529 (24 - state->hs_cfg_profile->cp_log_num_qp))) {
2540 2530 hermon_rsrc_free(state, &entry->qpn_qpc);
2541 2531 }
2542 2532 }
2543 2533 mutex_exit(&state->hs_qpn_avl_lock);
2544 2534 }
2545 2535
2546 2536
2547 2537 /*
2548 2538 * hermon_qpn_avl_compare()
2549 2539 * Context: Can be called from user or kernel context.
2550 2540 */
2551 2541 static int
2552 2542 hermon_qpn_avl_compare(const void *q, const void *e)
2553 2543 {
2554 2544 hermon_qpn_entry_t *entry, *query;
2555 2545
2556 2546 entry = (hermon_qpn_entry_t *)e;
2557 2547 query = (hermon_qpn_entry_t *)q;
2558 2548
2559 2549 if (query->qpn_indx < entry->qpn_indx) {
2560 2550 return (-1);
2561 2551 } else if (query->qpn_indx > entry->qpn_indx) {
2562 2552 return (+1);
2563 2553 } else {
2564 2554 return (0);
2565 2555 }
2566 2556 }
2567 2557
2568 2558
2569 2559 /*
2570 2560 * hermon_qpn_avl_init()
2571 2561 * Context: Only called from attach() path context
2572 2562 */
2573 2563 void
2574 2564 hermon_qpn_avl_init(hermon_state_t *state)
2575 2565 {
2576 2566 /* Initialize the lock used for QP number (QPN) AVL tree access */
2577 2567 mutex_init(&state->hs_qpn_avl_lock, NULL, MUTEX_DRIVER,
2578 2568 DDI_INTR_PRI(state->hs_intrmsi_pri));
2579 2569
2580 2570 /* Initialize the AVL tree for the QP number (QPN) storage */
2581 2571 avl_create(&state->hs_qpn_avl, hermon_qpn_avl_compare,
2582 2572 sizeof (hermon_qpn_entry_t),
2583 2573 offsetof(hermon_qpn_entry_t, qpn_avlnode));
2584 2574 }
2585 2575
2586 2576
2587 2577 /*
2588 2578 * hermon_qpn_avl_fini()
2589 2579 * Context: Only called from attach() and/or detach() path contexts
2590 2580 */
2591 2581 void
2592 2582 hermon_qpn_avl_fini(hermon_state_t *state)
2593 2583 {
2594 2584 hermon_qpn_entry_t *entry;
2595 2585 void *cookie;
2596 2586
2597 2587 /*
2598 2588 * Empty all entries (if necessary) and destroy the AVL tree
2599 2589 * that was used for QP number (QPN) tracking.
2600 2590 */
2601 2591 cookie = NULL;
2602 2592 while ((entry = (hermon_qpn_entry_t *)avl_destroy_nodes(
2603 2593 &state->hs_qpn_avl, &cookie)) != NULL) {
2604 2594 kmem_free(entry, sizeof (hermon_qpn_entry_t));
2605 2595 }
2606 2596 avl_destroy(&state->hs_qpn_avl);
2607 2597
2608 2598 /* Destroy the lock used for QP number (QPN) AVL tree access */
2609 2599 mutex_destroy(&state->hs_qpn_avl_lock);
2610 2600 }
2611 2601
2612 2602
2613 2603 /*
2614 2604 * hermon_qphdl_from_qpnum()
2615 2605 * Context: Can be called from interrupt or base context.
2616 2606 *
2617 2607 * This routine is important because changing the unconstrained
2618 2608 * portion of the QP number is critical to the detection of a
2619 2609 * potential race condition in the QP event handler code (i.e. the case
2620 2610 * where a QP is freed and alloc'd again before an event for the
2621 2611 * "old" QP can be handled).
2622 2612 *
2623 2613 * While this is not a perfect solution (not sure that one exists)
2624 2614 * it does help to mitigate the chance that this race condition will
2625 2615 * cause us to deliver a "stale" event to the new QP owner. Note:
2626 2616 * this solution does not scale well because the number of constrained
2627 2617 * bits increases (and, hence, the number of unconstrained bits
2628 2618 * decreases) as the number of supported QPs grows. For small and
2629 2619 * intermediate values, it should hopefully provide sufficient
2630 2620 * protection.
2631 2621 */
2632 2622 hermon_qphdl_t
2633 2623 hermon_qphdl_from_qpnum(hermon_state_t *state, uint_t qpnum)
2634 2624 {
2635 2625 uint_t qpindx, qpmask;
2636 2626
2637 2627 /* Calculate the QP table index from the qpnum */
2638 2628 qpmask = (1 << state->hs_cfg_profile->cp_log_num_qp) - 1;
2639 2629 qpindx = qpnum & qpmask;
2640 2630 return (hermon_icm_num_to_hdl(state, HERMON_QPC, qpindx));
2641 2631 }
2642 2632
2643 2633
2644 2634 /*
2645 2635 * hermon_special_qp_rsrc_alloc
2646 2636 * Context: Can be called from interrupt or base context.
2647 2637 */
2648 2638 static int
2649 2639 hermon_special_qp_rsrc_alloc(hermon_state_t *state, ibt_sqp_type_t type,
2650 2640 uint_t port, hermon_rsrc_t **qp_rsrc)
2651 2641 {
2652 2642 uint_t mask, flags;
2653 2643 int status;
2654 2644
2655 2645 mutex_enter(&state->hs_spec_qplock);
2656 2646 flags = state->hs_spec_qpflags;
2657 2647 if (type == IBT_SMI_SQP) {
2658 2648 /*
2659 2649 * Check here to see if the driver has been configured
2660 2650 * to instruct the Hermon firmware to handle all incoming
2661 2651 * SMP messages (i.e. messages sent to SMA). If so,
2662 2652 * then we will treat QP0 as if it has already been
2663 2653 * allocated (for internal use). Otherwise, if we allow
2664 2654 * the allocation to happen, it will cause unexpected
2665 2655 * behaviors (e.g. Hermon SMA becomes unresponsive).
2666 2656 */
2667 2657 if (state->hs_cfg_profile->cp_qp0_agents_in_fw != 0) {
2668 2658 mutex_exit(&state->hs_spec_qplock);
2669 2659 return (IBT_QP_IN_USE);
2670 2660 }
2671 2661
2672 2662 /*
2673 2663 * If this is the first QP0 allocation, then post
2674 2664 * a CONF_SPECIAL_QP firmware command
2675 2665 */
2676 2666 if ((flags & HERMON_SPECIAL_QP0_RSRC_MASK) == 0) {
2677 2667 status = hermon_conf_special_qp_cmd_post(state,
2678 2668 state->hs_spec_qp0->hr_indx, HERMON_CMD_QP_SMI,
2679 2669 HERMON_CMD_NOSLEEP_SPIN,
2680 2670 HERMON_CMD_SPEC_QP_OPMOD(
2681 2671 state->hs_cfg_profile->cp_qp0_agents_in_fw,
2682 2672 state->hs_cfg_profile->cp_qp1_agents_in_fw));
2683 2673 if (status != HERMON_CMD_SUCCESS) {
2684 2674 mutex_exit(&state->hs_spec_qplock);
2685 2675 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2686 2676 "command failed: %08x\n",
2687 2677 state->hs_instance, status);
2688 2678 return (IBT_INSUFF_RESOURCE);
2689 2679 }
2690 2680 }
2691 2681
2692 2682 /*
2693 2683 * Now check (and, if necessary, modify) the flags to indicate
2694 2684 * whether the allocation was successful
2695 2685 */
2696 2686 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2697 2687 if (flags & mask) {
2698 2688 mutex_exit(&state->hs_spec_qplock);
2699 2689 return (IBT_QP_IN_USE);
2700 2690 }
2701 2691 state->hs_spec_qpflags |= mask;
2702 2692 *qp_rsrc = state->hs_spec_qp0;
2703 2693
2704 2694 } else {
2705 2695 /*
2706 2696 * If this is the first QP1 allocation, then post
2707 2697 * a CONF_SPECIAL_QP firmware command
2708 2698 */
2709 2699 if ((flags & HERMON_SPECIAL_QP1_RSRC_MASK) == 0) {
2710 2700 status = hermon_conf_special_qp_cmd_post(state,
2711 2701 state->hs_spec_qp1->hr_indx, HERMON_CMD_QP_GSI,
2712 2702 HERMON_CMD_NOSLEEP_SPIN,
2713 2703 HERMON_CMD_SPEC_QP_OPMOD(
2714 2704 state->hs_cfg_profile->cp_qp0_agents_in_fw,
2715 2705 state->hs_cfg_profile->cp_qp1_agents_in_fw));
2716 2706 if (status != HERMON_CMD_SUCCESS) {
2717 2707 mutex_exit(&state->hs_spec_qplock);
2718 2708 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2719 2709 "command failed: %08x\n",
2720 2710 state->hs_instance, status);
2721 2711 return (IBT_INSUFF_RESOURCE);
2722 2712 }
2723 2713 }
2724 2714
2725 2715 /*
2726 2716 * Now check (and, if necessary, modify) the flags to indicate
2727 2717 * whether the allocation was successful
2728 2718 */
2729 2719 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2730 2720 if (flags & mask) {
2731 2721 mutex_exit(&state->hs_spec_qplock);
2732 2722 return (IBT_QP_IN_USE);
2733 2723 }
2734 2724 state->hs_spec_qpflags |= mask;
2735 2725 *qp_rsrc = state->hs_spec_qp1;
2736 2726 }
2737 2727
2738 2728 mutex_exit(&state->hs_spec_qplock);
2739 2729 return (DDI_SUCCESS);
2740 2730 }
2741 2731
2742 2732
2743 2733 /*
2744 2734 * hermon_special_qp_rsrc_free
2745 2735 * Context: Can be called from interrupt or base context.
2746 2736 */
2747 2737 static int
2748 2738 hermon_special_qp_rsrc_free(hermon_state_t *state, ibt_sqp_type_t type,
2749 2739 uint_t port)
2750 2740 {
2751 2741 uint_t mask, flags;
2752 2742 int status;
2753 2743
2754 2744 mutex_enter(&state->hs_spec_qplock);
2755 2745 if (type == IBT_SMI_SQP) {
2756 2746 mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port));
2757 2747 state->hs_spec_qpflags &= ~mask;
2758 2748 flags = state->hs_spec_qpflags;
2759 2749
2760 2750 /*
2761 2751 * If this is the last QP0 free, then post a CONF_SPECIAL_QP
2762 2752 * NOW, If this is the last Special QP free, then post a
2763 2753 * CONF_SPECIAL_QP firmware command - it'll stop them all
2764 2754 */
2765 2755 if (flags) {
2766 2756 status = hermon_conf_special_qp_cmd_post(state, 0,
2767 2757 HERMON_CMD_QP_SMI, HERMON_CMD_NOSLEEP_SPIN, 0);
2768 2758 if (status != HERMON_CMD_SUCCESS) {
2769 2759 mutex_exit(&state->hs_spec_qplock);
2770 2760 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2771 2761 "command failed: %08x\n",
2772 2762 state->hs_instance, status);
2773 2763 if (status == HERMON_CMD_INVALID_STATUS) {
2774 2764 hermon_fm_ereport(state, HCA_SYS_ERR,
2775 2765 HCA_ERR_SRV_LOST);
2776 2766 }
2777 2767 return (ibc_get_ci_failure(0));
2778 2768 }
2779 2769 }
2780 2770 } else {
2781 2771 mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port));
2782 2772 state->hs_spec_qpflags &= ~mask;
2783 2773 flags = state->hs_spec_qpflags;
2784 2774
2785 2775 /*
2786 2776 * If this is the last QP1 free, then post a CONF_SPECIAL_QP
2787 2777 * NOW, if this is the last special QP free, then post a
2788 2778 * CONF_SPECIAL_QP firmware command - it'll stop them all
2789 2779 */
2790 2780 if (flags) {
2791 2781 status = hermon_conf_special_qp_cmd_post(state, 0,
2792 2782 HERMON_CMD_QP_GSI, HERMON_CMD_NOSLEEP_SPIN, 0);
2793 2783 if (status != HERMON_CMD_SUCCESS) {
2794 2784 mutex_exit(&state->hs_spec_qplock);
2795 2785 cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP "
2796 2786 "command failed: %08x\n",
2797 2787 state->hs_instance, status);
2798 2788 if (status == HERMON_CMD_INVALID_STATUS) {
2799 2789 hermon_fm_ereport(state, HCA_SYS_ERR,
2800 2790 HCA_ERR_SRV_LOST);
2801 2791 }
2802 2792 return (ibc_get_ci_failure(0));
2803 2793 }
2804 2794 }
2805 2795 }
2806 2796
2807 2797 mutex_exit(&state->hs_spec_qplock);
2808 2798 return (DDI_SUCCESS);
2809 2799 }
2810 2800
2811 2801
2812 2802 /*
2813 2803 * hermon_qp_sgl_to_logwqesz()
2814 2804 * Context: Can be called from interrupt or base context.
2815 2805 */
2816 2806 static void
2817 2807 hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
2818 2808 uint_t real_max_sgl, hermon_qp_wq_type_t wq_type,
2819 2809 uint_t *logwqesz, uint_t *max_sgl)
2820 2810 {
2821 2811 uint_t max_size, log2, actual_sgl;
2822 2812
2823 2813 switch (wq_type) {
2824 2814 case HERMON_QP_WQ_TYPE_SENDQ_UD:
2825 2815 /*
2826 2816 * Use requested maximum SGL to calculate max descriptor size
2827 2817 * (while guaranteeing that the descriptor size is a
2828 2818 * power-of-2 cachelines).
2829 2819 */
2830 2820 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2831 2821 log2 = highbit(max_size);
2832 2822 if (ISP2(max_size)) {
2833 2823 log2 = log2 - 1;
2834 2824 }
2835 2825
2836 2826 /* Make sure descriptor is at least the minimum size */
2837 2827 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2838 2828
2839 2829 /* Calculate actual number of SGL (given WQE size) */
2840 2830 actual_sgl = ((1 << log2) -
2841 2831 sizeof (hermon_hw_snd_wqe_ctrl_t)) >> 4;
2842 2832 break;
2843 2833
2844 2834 case HERMON_QP_WQ_TYPE_SENDQ_CONN:
2845 2835 /*
2846 2836 * Use requested maximum SGL to calculate max descriptor size
2847 2837 * (while guaranteeing that the descriptor size is a
2848 2838 * power-of-2 cachelines).
2849 2839 */
2850 2840 max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
2851 2841 log2 = highbit(max_size);
2852 2842 if (ISP2(max_size)) {
2853 2843 log2 = log2 - 1;
2854 2844 }
2855 2845
2856 2846 /* Make sure descriptor is at least the minimum size */
2857 2847 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2858 2848
2859 2849 /* Calculate actual number of SGL (given WQE size) */
2860 2850 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SND_HDRS) >> 4;
2861 2851 break;
2862 2852
2863 2853 case HERMON_QP_WQ_TYPE_RECVQ:
2864 2854 /*
2865 2855 * Same as above (except for Recv WQEs)
2866 2856 */
2867 2857 max_size = (HERMON_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
2868 2858 log2 = highbit(max_size);
2869 2859 if (ISP2(max_size)) {
2870 2860 log2 = log2 - 1;
2871 2861 }
2872 2862
2873 2863 /* Make sure descriptor is at least the minimum size */
2874 2864 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2875 2865
2876 2866 /* Calculate actual number of SGL (given WQE size) */
2877 2867 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_RCV_HDRS) >> 4;
2878 2868 break;
2879 2869
2880 2870 case HERMON_QP_WQ_TYPE_SENDMLX_QP0:
2881 2871 /*
2882 2872 * Same as above (except for MLX transport WQEs). For these
2883 2873 * WQEs we have to account for the space consumed by the
2884 2874 * "inline" packet headers. (This is smaller than for QP1
2885 2875 * below because QP0 is not allowed to send packets with a GRH.
2886 2876 */
2887 2877 max_size = (HERMON_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4));
2888 2878 log2 = highbit(max_size);
2889 2879 if (ISP2(max_size)) {
2890 2880 log2 = log2 - 1;
2891 2881 }
2892 2882
2893 2883 /* Make sure descriptor is at least the minimum size */
2894 2884 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2895 2885
2896 2886 /* Calculate actual number of SGL (given WQE size) */
2897 2887 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP0_HDRS) >> 4;
2898 2888 break;
2899 2889
2900 2890 case HERMON_QP_WQ_TYPE_SENDMLX_QP1:
2901 2891 /*
2902 2892 * Same as above. For these WQEs we again have to account for
2903 2893 * the space consumed by the "inline" packet headers. (This
2904 2894 * is larger than for QP0 above because we have to account for
2905 2895 * the possibility of a GRH in each packet - and this
2906 2896 * introduces an alignment issue that causes us to consume
2907 2897 * an additional 8 bytes).
2908 2898 */
2909 2899 max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4));
2910 2900 log2 = highbit(max_size);
2911 2901 if (ISP2(max_size)) {
2912 2902 log2 = log2 - 1;
2913 2903 }
2914 2904
2915 2905 /* Make sure descriptor is at least the minimum size */
2916 2906 log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
2917 2907
2918 2908 /* Calculate actual number of SGL (given WQE size) */
2919 2909 actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4;
2920 2910 break;
2921 2911
2922 2912 default:
2923 2913 HERMON_WARNING(state, "unexpected work queue type");
2924 2914 break;
2925 2915 }
2926 2916
2927 2917 /* Fill in the return values */
2928 2918 *logwqesz = log2;
2929 2919 *max_sgl = min(real_max_sgl, actual_sgl);
2930 2920 }
|
↓ open down ↓ |
486 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX