64 ibt_cq_attr_t *cq_attr, uint_t *actual_size, tavor_cqhdl_t *cqhdl,
65 uint_t sleepflag)
66 {
67 tavor_rsrc_t *cqc, *rsrc;
68 tavor_umap_db_entry_t *umapdb;
69 tavor_hw_cqc_t cqc_entry;
70 tavor_cqhdl_t cq;
71 ibt_mr_attr_t mr_attr;
72 tavor_mr_options_t op;
73 tavor_pdhdl_t pd;
74 tavor_mrhdl_t mr;
75 tavor_hw_cqe_t *buf;
76 uint64_t addr, value;
77 uint32_t log_cq_size, lkey, uarpg;
78 uint_t dma_xfer_mode, cq_sync, cq_is_umap;
79 int status, i, flag;
80 char *errormsg;
81
82 TAVOR_TNF_ENTER(tavor_cq_alloc);
83
84 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq_attr))
85
86 /*
87 * Determine whether CQ is being allocated for userland access or
88 * whether it is being allocated for kernel access. If the CQ is
89 * being allocated for userland access, then lookup the UAR doorbell
90 * page number for the current process. Note: If this is not found
91 * (e.g. if the process has not previously open()'d the Tavor driver),
92 * then an error is returned.
93 */
94 cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
95 if (cq_is_umap) {
96 status = tavor_umap_db_find(state->ts_instance, ddi_get_pid(),
97 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
98 if (status != DDI_SUCCESS) {
99 /* Set "status" and "errormsg" and goto failure */
100 TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "failed UAR page");
101 goto cqalloc_fail;
102 }
103 uarpg = ((tavor_rsrc_t *)(uintptr_t)value)->tr_indx;
104 }
105
119 status = tavor_rsrc_alloc(state, TAVOR_CQC, 1, sleepflag, &cqc);
120 if (status != DDI_SUCCESS) {
121 /* Set "status" and "errormsg" and goto failure */
122 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed CQ context");
123 goto cqalloc_fail1;
124 }
125
126 /*
127 * Allocate the software structure for tracking the completion queue
128 * (i.e. the Tavor Completion Queue handle). If we fail here, we must
129 * undo the protection domain reference count and the previous
130 * resource allocation.
131 */
132 status = tavor_rsrc_alloc(state, TAVOR_CQHDL, 1, sleepflag, &rsrc);
133 if (status != DDI_SUCCESS) {
134 /* Set "status" and "errormsg" and goto failure */
135 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed CQ handle");
136 goto cqalloc_fail2;
137 }
138 cq = (tavor_cqhdl_t)rsrc->tr_addr;
139 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
140 cq->cq_is_umap = cq_is_umap;
141
142 /* Use the index as CQ number */
143 cq->cq_cqnum = cqc->tr_indx;
144
145 /*
146 * If this will be a user-mappable CQ, then allocate an entry for
147 * the "userland resources database". This will later be added to
148 * the database (after all further CQ operations are successful).
149 * If we fail here, we must undo the reference counts and the
150 * previous resource allocation.
151 */
152 if (cq->cq_is_umap) {
153 umapdb = tavor_umap_db_alloc(state->ts_instance, cq->cq_cqnum,
154 MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
155 if (umapdb == NULL) {
156 /* Set "status" and "errormsg" and goto failure */
157 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
158 goto cqalloc_fail3;
159 }
191 * It is also worth noting that, unlike Tavor QP work queues,
192 * completion queues do not have the same strict alignment
193 * requirements. It is sufficient for the CQ memory to be both
194 * aligned to and bound to addresses which are a multiple of CQE size.
195 */
196 cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (tavor_hw_cqe_t);
197 cq->cq_cqinfo.qa_alloc_align = sizeof (tavor_hw_cqe_t);
198 cq->cq_cqinfo.qa_bind_align = sizeof (tavor_hw_cqe_t);
199 if (cq->cq_is_umap) {
200 cq->cq_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
201 } else {
202 cq->cq_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
203 }
204 status = tavor_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
205 if (status != DDI_SUCCESS) {
206 /* Set "status" and "errormsg" and goto failure */
207 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed completion queue");
208 goto cqalloc_fail4;
209 }
210 buf = (tavor_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
211 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
212
213 /*
214 * Initialize each of the Completion Queue Entries (CQE) by setting
215 * their ownership to hardware ("owner" bit set to HW). This is in
216 * preparation for the final transfer of ownership (below) of the
217 * CQ context itself.
218 */
219 for (i = 0; i < (1 << log_cq_size); i++) {
220 TAVOR_CQE_OWNER_SET_HW(cq, &buf[i]);
221 }
222
223 /*
224 * Register the memory for the CQ. The memory for the CQ must
225 * be registered in the Tavor TPT tables. This gives us the LKey
226 * to specify in the CQ context below. Note: If this is a user-
227 * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
228 */
229 flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
230 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
231 mr_attr.mr_len = cq->cq_cqinfo.qa_size;
232 mr_attr.mr_as = NULL;
233 mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
234 if (cq->cq_is_umap) {
235 dma_xfer_mode = DDI_DMA_CONSISTENT;
236 } else {
237 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
238 }
239 if (dma_xfer_mode == DDI_DMA_STREAMING) {
240 mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
241 }
242 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
243 op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl;
244 op.mro_bind_override_addr = 0;
245 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
246 if (status != DDI_SUCCESS) {
247 /* Set "status" and "errormsg" and goto failure */
248 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
249 goto cqalloc_fail5;
250 }
251 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
252 addr = mr->mr_bindinfo.bi_addr;
253 lkey = mr->mr_lkey;
254
255 /* Determine if later ddi_dma_sync will be necessary */
256 cq_sync = TAVOR_CQ_IS_SYNC_REQ(state, cq->cq_cqinfo);
257
258 /* Sync entire CQ for use by the hardware (if necessary). */
259 if (cq_sync) {
260 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
261 cq->cq_cqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
262 }
263
264 /*
265 * Fill in the CQC entry. This is the final step before passing
266 * ownership of the CQC entry to the Tavor hardware. We use all of
267 * the information collected/calculated above to fill in the
268 * requisite portions of the CQC. Note: If this CQ is going to be
269 * used for userland access, then we need to set the UAR page number
270 * appropriately (otherwise it's a "don't care")
271 */
467 cq->cq_umap_dhp = (devmap_cookie_t)NULL;
468 }
469 }
470
471 /*
472 * Put NULL into the Tavor CQNum-to-CQHdl list. This will allow any
473 * in-progress events to detect that the CQ corresponding to this
474 * number has been freed.
475 */
476 state->ts_cqhdl[cqc->tr_indx] = NULL;
477
478 /*
479 * While we hold the CQ lock, do a "forced reap" of the workQ WRID
480 * list. This cleans up all the structures associated with the WRID
481 * processing for this CQ. Once we complete, drop the lock and finish
482 * the deallocation of the CQ.
483 */
484 tavor_wrid_cq_force_reap(cq);
485
486 mutex_exit(&cq->cq_lock);
487 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
488
489 /*
490 * Reclaim CQC entry from hardware (using the Tavor HW2SW_CQ
491 * firmware command). If the ownership transfer fails for any reason,
492 * then it is an indication that something (either in HW or SW) has
493 * gone seriously wrong.
494 */
495 status = tavor_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
496 sizeof (tavor_hw_cqc_t), cqnum, sleepflag);
497 if (status != TAVOR_CMD_SUCCESS) {
498 TAVOR_WARNING(state, "failed to reclaim CQC ownership");
499 cmn_err(CE_CONT, "Tavor: HW2SW_CQ command failed: %08x\n",
500 status);
501 TNF_PROBE_1(tavor_cq_free_hw2sw_cq_cmd_fail,
502 TAVOR_TNF_ERROR, "", tnf_uint, status, status);
503 TAVOR_TNF_EXIT(tavor_cq_free);
504 return (ibc_get_ci_failure(0));
505 }
506
507 /*
597 * It is also worth noting that, unlike Tavor QP work queues,
598 * completion queues do not have the same strict alignment
599 * requirements. It is sufficient for the CQ memory to be both
600 * aligned to and bound to addresses which are a multiple of CQE size.
601 */
602 new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (tavor_hw_cqe_t);
603 new_cqinfo.qa_alloc_align = sizeof (tavor_hw_cqe_t);
604 new_cqinfo.qa_bind_align = sizeof (tavor_hw_cqe_t);
605 if (cq->cq_is_umap) {
606 new_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
607 } else {
608 new_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
609 }
610 status = tavor_queue_alloc(state, &new_cqinfo, sleepflag);
611 if (status != DDI_SUCCESS) {
612 /* Set "status" and "errormsg" and goto failure */
613 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed completion queue");
614 goto cqresize_fail;
615 }
616 buf = (tavor_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
617 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
618
619 /*
620 * Initialize each of the Completion Queue Entries (CQE) by setting
621 * their ownership to hardware ("owner" bit set to HW). This is in
622 * preparation for the final resize operation (below).
623 */
624 for (i = 0; i < (1 << log_cq_size); i++) {
625 TAVOR_CQE_OWNER_SET_HW(cq, &buf[i]);
626 }
627
628 /*
629 * Register the memory for the CQ. The memory for the CQ must
630 * be registered in the Tavor TPT tables. This gives us the LKey
631 * to specify in the CQ context below.
632 */
633 flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
634 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
635 mr_attr.mr_len = new_cqinfo.qa_size;
636 mr_attr.mr_as = NULL;
637 mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
638 if (cq->cq_is_umap) {
639 dma_xfer_mode = DDI_DMA_CONSISTENT;
640 } else {
641 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
642 }
643 if (dma_xfer_mode == DDI_DMA_STREAMING) {
644 mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
645 }
646 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
647 op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl;
648 op.mro_bind_override_addr = 0;
649 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
650 if (status != DDI_SUCCESS) {
651 tavor_queue_free(state, &new_cqinfo);
652 /* Set "status" and "errormsg" and goto failure */
653 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
654 goto cqresize_fail;
655 }
656 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
657
658 /* Determine if later ddi_dma_sync will be necessary */
659 cq_sync = TAVOR_CQ_IS_SYNC_REQ(state, new_cqinfo);
660
661 /* Sync entire "new" CQ for use by hardware (if necessary) */
662 if (cq_sync) {
663 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
664 new_cqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
665 }
666
667 /*
668 * Now we grab the CQ lock. Since we will be updating the actual
669 * CQ location and the producer/consumer indexes, we should hold
670 * the lock.
671 *
672 * We do a TAVOR_NOSLEEP here (and below), though, because we are
673 * holding the "cq_lock" and if we got raised to interrupt level
674 * by priority inversion, we would not want to block in this routine
675 * waiting for success.
676 */
|
64 ibt_cq_attr_t *cq_attr, uint_t *actual_size, tavor_cqhdl_t *cqhdl,
65 uint_t sleepflag)
66 {
67 tavor_rsrc_t *cqc, *rsrc;
68 tavor_umap_db_entry_t *umapdb;
69 tavor_hw_cqc_t cqc_entry;
70 tavor_cqhdl_t cq;
71 ibt_mr_attr_t mr_attr;
72 tavor_mr_options_t op;
73 tavor_pdhdl_t pd;
74 tavor_mrhdl_t mr;
75 tavor_hw_cqe_t *buf;
76 uint64_t addr, value;
77 uint32_t log_cq_size, lkey, uarpg;
78 uint_t dma_xfer_mode, cq_sync, cq_is_umap;
79 int status, i, flag;
80 char *errormsg;
81
82 TAVOR_TNF_ENTER(tavor_cq_alloc);
83
84 /*
85 * Determine whether CQ is being allocated for userland access or
86 * whether it is being allocated for kernel access. If the CQ is
87 * being allocated for userland access, then lookup the UAR doorbell
88 * page number for the current process. Note: If this is not found
89 * (e.g. if the process has not previously open()'d the Tavor driver),
90 * then an error is returned.
91 */
92 cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
93 if (cq_is_umap) {
94 status = tavor_umap_db_find(state->ts_instance, ddi_get_pid(),
95 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
96 if (status != DDI_SUCCESS) {
97 /* Set "status" and "errormsg" and goto failure */
98 TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "failed UAR page");
99 goto cqalloc_fail;
100 }
101 uarpg = ((tavor_rsrc_t *)(uintptr_t)value)->tr_indx;
102 }
103
117 status = tavor_rsrc_alloc(state, TAVOR_CQC, 1, sleepflag, &cqc);
118 if (status != DDI_SUCCESS) {
119 /* Set "status" and "errormsg" and goto failure */
120 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed CQ context");
121 goto cqalloc_fail1;
122 }
123
124 /*
125 * Allocate the software structure for tracking the completion queue
126 * (i.e. the Tavor Completion Queue handle). If we fail here, we must
127 * undo the protection domain reference count and the previous
128 * resource allocation.
129 */
130 status = tavor_rsrc_alloc(state, TAVOR_CQHDL, 1, sleepflag, &rsrc);
131 if (status != DDI_SUCCESS) {
132 /* Set "status" and "errormsg" and goto failure */
133 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed CQ handle");
134 goto cqalloc_fail2;
135 }
136 cq = (tavor_cqhdl_t)rsrc->tr_addr;
137 cq->cq_is_umap = cq_is_umap;
138
139 /* Use the index as CQ number */
140 cq->cq_cqnum = cqc->tr_indx;
141
142 /*
143 * If this will be a user-mappable CQ, then allocate an entry for
144 * the "userland resources database". This will later be added to
145 * the database (after all further CQ operations are successful).
146 * If we fail here, we must undo the reference counts and the
147 * previous resource allocation.
148 */
149 if (cq->cq_is_umap) {
150 umapdb = tavor_umap_db_alloc(state->ts_instance, cq->cq_cqnum,
151 MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
152 if (umapdb == NULL) {
153 /* Set "status" and "errormsg" and goto failure */
154 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
155 goto cqalloc_fail3;
156 }
188 * It is also worth noting that, unlike Tavor QP work queues,
189 * completion queues do not have the same strict alignment
190 * requirements. It is sufficient for the CQ memory to be both
191 * aligned to and bound to addresses which are a multiple of CQE size.
192 */
193 cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (tavor_hw_cqe_t);
194 cq->cq_cqinfo.qa_alloc_align = sizeof (tavor_hw_cqe_t);
195 cq->cq_cqinfo.qa_bind_align = sizeof (tavor_hw_cqe_t);
196 if (cq->cq_is_umap) {
197 cq->cq_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
198 } else {
199 cq->cq_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
200 }
201 status = tavor_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
202 if (status != DDI_SUCCESS) {
203 /* Set "status" and "errormsg" and goto failure */
204 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed completion queue");
205 goto cqalloc_fail4;
206 }
207 buf = (tavor_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
208
209 /*
210 * Initialize each of the Completion Queue Entries (CQE) by setting
211 * their ownership to hardware ("owner" bit set to HW). This is in
212 * preparation for the final transfer of ownership (below) of the
213 * CQ context itself.
214 */
215 for (i = 0; i < (1 << log_cq_size); i++) {
216 TAVOR_CQE_OWNER_SET_HW(cq, &buf[i]);
217 }
218
219 /*
220 * Register the memory for the CQ. The memory for the CQ must
221 * be registered in the Tavor TPT tables. This gives us the LKey
222 * to specify in the CQ context below. Note: If this is a user-
223 * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
224 */
225 flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
226 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
227 mr_attr.mr_len = cq->cq_cqinfo.qa_size;
228 mr_attr.mr_as = NULL;
229 mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
230 if (cq->cq_is_umap) {
231 dma_xfer_mode = DDI_DMA_CONSISTENT;
232 } else {
233 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
234 }
235 if (dma_xfer_mode == DDI_DMA_STREAMING) {
236 mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
237 }
238 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
239 op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl;
240 op.mro_bind_override_addr = 0;
241 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
242 if (status != DDI_SUCCESS) {
243 /* Set "status" and "errormsg" and goto failure */
244 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
245 goto cqalloc_fail5;
246 }
247 addr = mr->mr_bindinfo.bi_addr;
248 lkey = mr->mr_lkey;
249
250 /* Determine if later ddi_dma_sync will be necessary */
251 cq_sync = TAVOR_CQ_IS_SYNC_REQ(state, cq->cq_cqinfo);
252
253 /* Sync entire CQ for use by the hardware (if necessary). */
254 if (cq_sync) {
255 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
256 cq->cq_cqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
257 }
258
259 /*
260 * Fill in the CQC entry. This is the final step before passing
261 * ownership of the CQC entry to the Tavor hardware. We use all of
262 * the information collected/calculated above to fill in the
263 * requisite portions of the CQC. Note: If this CQ is going to be
264 * used for userland access, then we need to set the UAR page number
265 * appropriately (otherwise it's a "don't care")
266 */
462 cq->cq_umap_dhp = (devmap_cookie_t)NULL;
463 }
464 }
465
466 /*
467 * Put NULL into the Tavor CQNum-to-CQHdl list. This will allow any
468 * in-progress events to detect that the CQ corresponding to this
469 * number has been freed.
470 */
471 state->ts_cqhdl[cqc->tr_indx] = NULL;
472
473 /*
474 * While we hold the CQ lock, do a "forced reap" of the workQ WRID
475 * list. This cleans up all the structures associated with the WRID
476 * processing for this CQ. Once we complete, drop the lock and finish
477 * the deallocation of the CQ.
478 */
479 tavor_wrid_cq_force_reap(cq);
480
481 mutex_exit(&cq->cq_lock);
482
483 /*
484 * Reclaim CQC entry from hardware (using the Tavor HW2SW_CQ
485 * firmware command). If the ownership transfer fails for any reason,
486 * then it is an indication that something (either in HW or SW) has
487 * gone seriously wrong.
488 */
489 status = tavor_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
490 sizeof (tavor_hw_cqc_t), cqnum, sleepflag);
491 if (status != TAVOR_CMD_SUCCESS) {
492 TAVOR_WARNING(state, "failed to reclaim CQC ownership");
493 cmn_err(CE_CONT, "Tavor: HW2SW_CQ command failed: %08x\n",
494 status);
495 TNF_PROBE_1(tavor_cq_free_hw2sw_cq_cmd_fail,
496 TAVOR_TNF_ERROR, "", tnf_uint, status, status);
497 TAVOR_TNF_EXIT(tavor_cq_free);
498 return (ibc_get_ci_failure(0));
499 }
500
501 /*
591 * It is also worth noting that, unlike Tavor QP work queues,
592 * completion queues do not have the same strict alignment
593 * requirements. It is sufficient for the CQ memory to be both
594 * aligned to and bound to addresses which are a multiple of CQE size.
595 */
596 new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (tavor_hw_cqe_t);
597 new_cqinfo.qa_alloc_align = sizeof (tavor_hw_cqe_t);
598 new_cqinfo.qa_bind_align = sizeof (tavor_hw_cqe_t);
599 if (cq->cq_is_umap) {
600 new_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
601 } else {
602 new_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
603 }
604 status = tavor_queue_alloc(state, &new_cqinfo, sleepflag);
605 if (status != DDI_SUCCESS) {
606 /* Set "status" and "errormsg" and goto failure */
607 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed completion queue");
608 goto cqresize_fail;
609 }
610 buf = (tavor_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
611
612 /*
613 * Initialize each of the Completion Queue Entries (CQE) by setting
614 * their ownership to hardware ("owner" bit set to HW). This is in
615 * preparation for the final resize operation (below).
616 */
617 for (i = 0; i < (1 << log_cq_size); i++) {
618 TAVOR_CQE_OWNER_SET_HW(cq, &buf[i]);
619 }
620
621 /*
622 * Register the memory for the CQ. The memory for the CQ must
623 * be registered in the Tavor TPT tables. This gives us the LKey
624 * to specify in the CQ context below.
625 */
626 flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
627 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
628 mr_attr.mr_len = new_cqinfo.qa_size;
629 mr_attr.mr_as = NULL;
630 mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
631 if (cq->cq_is_umap) {
632 dma_xfer_mode = DDI_DMA_CONSISTENT;
633 } else {
634 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
635 }
636 if (dma_xfer_mode == DDI_DMA_STREAMING) {
637 mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
638 }
639 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
640 op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl;
641 op.mro_bind_override_addr = 0;
642 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
643 if (status != DDI_SUCCESS) {
644 tavor_queue_free(state, &new_cqinfo);
645 /* Set "status" and "errormsg" and goto failure */
646 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
647 goto cqresize_fail;
648 }
649
650 /* Determine if later ddi_dma_sync will be necessary */
651 cq_sync = TAVOR_CQ_IS_SYNC_REQ(state, new_cqinfo);
652
653 /* Sync entire "new" CQ for use by hardware (if necessary) */
654 if (cq_sync) {
655 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
656 new_cqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
657 }
658
659 /*
660 * Now we grab the CQ lock. Since we will be updating the actual
661 * CQ location and the producer/consumer indexes, we should hold
662 * the lock.
663 *
664 * We do a TAVOR_NOSLEEP here (and below), though, because we are
665 * holding the "cq_lock" and if we got raised to interrupt level
666 * by priority inversion, we would not want to block in this routine
667 * waiting for success.
668 */
|