57 ibt_srq_sizes_t *sizes;
58 ibt_srq_sizes_t *real_sizes;
59 hermon_srqhdl_t *srqhdl;
60 ibt_srq_flags_t flags;
61 hermon_rsrc_t *srqc, *rsrc;
62 hermon_hw_srqc_t srqc_entry;
63 uint32_t *buf;
64 hermon_srqhdl_t srq;
65 hermon_umap_db_entry_t *umapdb;
66 ibt_mr_attr_t mr_attr;
67 hermon_mr_options_t mr_op;
68 hermon_mrhdl_t mr;
69 uint64_t value, srq_desc_off;
70 uint32_t log_srq_size;
71 uint32_t uarpg;
72 uint_t srq_is_umap;
73 int flag, status;
74 uint_t max_sgl;
75 uint_t wqesz;
76 uint_t srq_wr_sz;
77 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sizes))
78
79 /*
80 * options-->wq_location used to be for location, now explicitly
81 * LOCATION_NORMAL
82 */
83
84 /*
85 * Extract the necessary info from the hermon_srq_info_t structure
86 */
87 real_sizes = srqinfo->srqi_real_sizes;
88 sizes = srqinfo->srqi_sizes;
89 pd = srqinfo->srqi_pd;
90 ibt_srqhdl = srqinfo->srqi_ibt_srqhdl;
91 flags = srqinfo->srqi_flags;
92 srqhdl = srqinfo->srqi_srqhdl;
93
94 /*
95 * Determine whether SRQ is being allocated for userland access or
96 * whether it is being allocated for kernel access. If the SRQ is
97 * being allocated for userland access, then lookup the UAR doorbell
113 }
114
115 /* Increase PD refcnt */
116 hermon_pd_refcnt_inc(pd);
117
118 /* Allocate an SRQ context entry */
119 status = hermon_rsrc_alloc(state, HERMON_SRQC, 1, sleepflag, &srqc);
120 if (status != DDI_SUCCESS) {
121 status = IBT_INSUFF_RESOURCE;
122 goto srqalloc_fail1;
123 }
124
125 /* Allocate the SRQ Handle entry */
126 status = hermon_rsrc_alloc(state, HERMON_SRQHDL, 1, sleepflag, &rsrc);
127 if (status != DDI_SUCCESS) {
128 status = IBT_INSUFF_RESOURCE;
129 goto srqalloc_fail2;
130 }
131
132 srq = (hermon_srqhdl_t)rsrc->hr_addr;
133 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq))
134
135 bzero(srq, sizeof (struct hermon_sw_srq_s));
136 /* Calculate the SRQ number */
137
138 /* just use the index, implicit in Hermon */
139 srq->srq_srqnum = srqc->hr_indx;
140
141 /*
142 * If this will be a user-mappable SRQ, then allocate an entry for
143 * the "userland resources database". This will later be added to
144 * the database (after all further SRQ operations are successful).
145 * If we fail here, we must undo the reference counts and the
146 * previous resource allocation.
147 */
148 if (srq_is_umap) {
149 umapdb = hermon_umap_db_alloc(state->hs_instance,
150 srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC,
151 (uint64_t)(uintptr_t)rsrc);
152 if (umapdb == NULL) {
153 status = IBT_INSUFF_RESOURCE;
239 * to make sure that all the resulting IB addresses will start at 0, for
240 * a zero-based queue. By making sure we are aligned on at least a
241 * page, any offset we use into our queue will be the same as when we
242 * perform hermon_srq_modify() operations later.
243 */
244 wqesz = (1 << srq->srq_wq_log_wqesz);
245 srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
246 srq->srq_wqinfo.qa_alloc_align = PAGESIZE;
247 srq->srq_wqinfo.qa_bind_align = PAGESIZE;
248 if (srq_is_umap) {
249 srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
250 } else {
251 srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
252 }
253 status = hermon_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
254 if (status != DDI_SUCCESS) {
255 status = IBT_INSUFF_RESOURCE;
256 goto srqalloc_fail4a;
257 }
258 buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
259 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
260
261 /*
262 * Register the memory for the SRQ work queues. The memory for the SRQ
263 * must be registered in the Hermon cMPT tables. This gives us the LKey
264 * to specify in the SRQ context later. Note: If the work queue is to
265 * be allocated from DDR memory, then only a "bypass" mapping is
266 * appropriate. And if the SRQ memory is user-mappable, then we force
267 * DDI_DMA_CONSISTENT mapping. Also, in order to meet the alignment
268 * restriction, we pass the "mro_bind_override_addr" flag in the call
269 * to hermon_mr_register(). This guarantees that the resulting IB vaddr
270 * will be zero-based (modulo the offset into the first page). If we
271 * fail here, we still have the bunch of resource and reference count
272 * cleanup to do.
273 */
274 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
275 IBT_MR_NOSLEEP;
276 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
277 mr_attr.mr_len = srq->srq_wqinfo.qa_size;
278 mr_attr.mr_as = NULL;
279 mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
280 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
281 mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
282 mr_op.mro_bind_override_addr = 1;
283 status = hermon_mr_register(state, pd, &mr_attr, &mr,
284 &mr_op, HERMON_SRQ_CMPT);
285 if (status != DDI_SUCCESS) {
286 status = IBT_INSUFF_RESOURCE;
287 goto srqalloc_fail5;
288 }
289 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
290
291 /*
292 * Calculate the offset between the kernel virtual address space
293 * and the IB virtual address space. This will be used when
294 * posting work requests to properly initialize each WQE.
295 */
296 srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
297 (uint64_t)mr->mr_bindinfo.bi_addr;
298
299 srq->srq_wq_wqhdr = hermon_wrid_wqhdr_create(1 << log_srq_size);
300
301 /*
302 * Fill in all the return arguments (if necessary). This includes
303 * real queue size and real SGLs.
304 */
305 if (real_sizes != NULL) {
306 real_sizes->srq_wr_sz = (1 << log_srq_size) - 1;
307 real_sizes->srq_sgl_sz = srq->srq_wq_sgl;
308 }
309
503 state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size,
504 maxprot, DEVMAP_MAPPING_INVALID, NULL);
505 if (status != DDI_SUCCESS) {
506 mutex_exit(&srq->srq_lock);
507 HERMON_WARNING(state, "failed in SRQ memory "
508 "devmap_devmem_remap()");
509 return (ibc_get_ci_failure(0));
510 }
511 srq->srq_umap_dhp = (devmap_cookie_t)NULL;
512 }
513 }
514
515 /*
516 * Put NULL into the Hermon SRQNum-to-SRQHdl list. This will allow any
517 * in-progress events to detect that the SRQ corresponding to this
518 * number has been freed.
519 */
520 hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, NULL);
521
522 mutex_exit(&srq->srq_lock);
523 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq));
524
525 /*
526 * Reclaim SRQC entry from hardware (using the Hermon HW2SW_SRQ
527 * firmware command). If the ownership transfer fails for any reason,
528 * then it is an indication that something (either in HW or SW) has
529 * gone seriously wrong.
530 */
531 status = hermon_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
532 sizeof (hermon_hw_srqc_t), srqnum, sleepflag);
533 if (status != HERMON_CMD_SUCCESS) {
534 HERMON_WARNING(state, "failed to reclaim SRQC ownership");
535 cmn_err(CE_CONT, "Hermon: HW2SW_SRQ command failed: %08x\n",
536 status);
537 if (status == HERMON_CMD_INVALID_STATUS) {
538 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
539 }
540 return (ibc_get_ci_failure(0));
541 }
542
543 /*
647 * to make sure that all the resulting IB addresses will start at 0,
648 * for a zero-based queue. By making sure we are aligned on at least a
649 * page, any offset we use into our queue will be the same as it was
650 * when we allocated it at hermon_srq_alloc() time.
651 */
652 wqesz = (1 << srq->srq_wq_log_wqesz);
653 new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
654 new_srqinfo.qa_alloc_align = PAGESIZE;
655 new_srqinfo.qa_bind_align = PAGESIZE;
656 if (srq->srq_is_umap) {
657 new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
658 } else {
659 new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
660 }
661 status = hermon_queue_alloc(state, &new_srqinfo, sleepflag);
662 if (status != DDI_SUCCESS) {
663 status = IBT_INSUFF_RESOURCE;
664 goto srqmodify_fail;
665 }
666 buf = (uint32_t *)new_srqinfo.qa_buf_aligned;
667 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
668
669 /*
670 * Allocate the memory for the new WRE list. This will be used later
671 * when we resize the wridlist based on the new SRQ size.
672 */
673 wre_new = kmem_zalloc((1 << log_srq_size) * sizeof (uint64_t),
674 sleepflag);
675 if (wre_new == NULL) {
676 status = IBT_INSUFF_RESOURCE;
677 goto srqmodify_fail;
678 }
679
680 /*
681 * Fill in the "bind" struct. This struct provides the majority
682 * of the information that will be used to distinguish between an
683 * "addr" binding (as is the case here) and a "buf" binding (see
684 * below). The "bind" struct is later passed to hermon_mr_mem_bind()
685 * which does most of the "heavy lifting" for the Hermon memory
686 * registration routines.
687 */
688 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(bind))
689 bzero(&bind, sizeof (hermon_bind_info_t));
690 bind.bi_type = HERMON_BINDHDL_VADDR;
691 bind.bi_addr = (uint64_t)(uintptr_t)buf;
692 bind.bi_len = new_srqinfo.qa_size;
693 bind.bi_as = NULL;
694 bind.bi_flags = sleepflag == HERMON_SLEEP ? IBT_MR_SLEEP :
695 IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
696 bind.bi_bypass = state->hs_cfg_profile->cp_iommu_bypass;
697
698 status = hermon_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
699 &mtt_pgsize_bits, 0); /* no relaxed ordering */
700 if (status != DDI_SUCCESS) {
701 status = status;
702 kmem_free(wre_new, (1 << log_srq_size) *
703 sizeof (uint64_t));
704 hermon_queue_free(&new_srqinfo);
705 goto srqmodify_fail;
706 }
707
708 /*
|
57 ibt_srq_sizes_t *sizes;
58 ibt_srq_sizes_t *real_sizes;
59 hermon_srqhdl_t *srqhdl;
60 ibt_srq_flags_t flags;
61 hermon_rsrc_t *srqc, *rsrc;
62 hermon_hw_srqc_t srqc_entry;
63 uint32_t *buf;
64 hermon_srqhdl_t srq;
65 hermon_umap_db_entry_t *umapdb;
66 ibt_mr_attr_t mr_attr;
67 hermon_mr_options_t mr_op;
68 hermon_mrhdl_t mr;
69 uint64_t value, srq_desc_off;
70 uint32_t log_srq_size;
71 uint32_t uarpg;
72 uint_t srq_is_umap;
73 int flag, status;
74 uint_t max_sgl;
75 uint_t wqesz;
76 uint_t srq_wr_sz;
77
78 /*
79 * options-->wq_location used to be for location, now explicitly
80 * LOCATION_NORMAL
81 */
82
83 /*
84 * Extract the necessary info from the hermon_srq_info_t structure
85 */
86 real_sizes = srqinfo->srqi_real_sizes;
87 sizes = srqinfo->srqi_sizes;
88 pd = srqinfo->srqi_pd;
89 ibt_srqhdl = srqinfo->srqi_ibt_srqhdl;
90 flags = srqinfo->srqi_flags;
91 srqhdl = srqinfo->srqi_srqhdl;
92
93 /*
94 * Determine whether SRQ is being allocated for userland access or
95 * whether it is being allocated for kernel access. If the SRQ is
96 * being allocated for userland access, then lookup the UAR doorbell
112 }
113
114 /* Increase PD refcnt */
115 hermon_pd_refcnt_inc(pd);
116
117 /* Allocate an SRQ context entry */
118 status = hermon_rsrc_alloc(state, HERMON_SRQC, 1, sleepflag, &srqc);
119 if (status != DDI_SUCCESS) {
120 status = IBT_INSUFF_RESOURCE;
121 goto srqalloc_fail1;
122 }
123
124 /* Allocate the SRQ Handle entry */
125 status = hermon_rsrc_alloc(state, HERMON_SRQHDL, 1, sleepflag, &rsrc);
126 if (status != DDI_SUCCESS) {
127 status = IBT_INSUFF_RESOURCE;
128 goto srqalloc_fail2;
129 }
130
131 srq = (hermon_srqhdl_t)rsrc->hr_addr;
132
133 bzero(srq, sizeof (struct hermon_sw_srq_s));
134 /* Calculate the SRQ number */
135
136 /* just use the index, implicit in Hermon */
137 srq->srq_srqnum = srqc->hr_indx;
138
139 /*
140 * If this will be a user-mappable SRQ, then allocate an entry for
141 * the "userland resources database". This will later be added to
142 * the database (after all further SRQ operations are successful).
143 * If we fail here, we must undo the reference counts and the
144 * previous resource allocation.
145 */
146 if (srq_is_umap) {
147 umapdb = hermon_umap_db_alloc(state->hs_instance,
148 srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC,
149 (uint64_t)(uintptr_t)rsrc);
150 if (umapdb == NULL) {
151 status = IBT_INSUFF_RESOURCE;
237 * to make sure that all the resulting IB addresses will start at 0, for
238 * a zero-based queue. By making sure we are aligned on at least a
239 * page, any offset we use into our queue will be the same as when we
240 * perform hermon_srq_modify() operations later.
241 */
242 wqesz = (1 << srq->srq_wq_log_wqesz);
243 srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
244 srq->srq_wqinfo.qa_alloc_align = PAGESIZE;
245 srq->srq_wqinfo.qa_bind_align = PAGESIZE;
246 if (srq_is_umap) {
247 srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
248 } else {
249 srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
250 }
251 status = hermon_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
252 if (status != DDI_SUCCESS) {
253 status = IBT_INSUFF_RESOURCE;
254 goto srqalloc_fail4a;
255 }
256 buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
257
258 /*
259 * Register the memory for the SRQ work queues. The memory for the SRQ
260 * must be registered in the Hermon cMPT tables. This gives us the LKey
261 * to specify in the SRQ context later. Note: If the work queue is to
262 * be allocated from DDR memory, then only a "bypass" mapping is
263 * appropriate. And if the SRQ memory is user-mappable, then we force
264 * DDI_DMA_CONSISTENT mapping. Also, in order to meet the alignment
265 * restriction, we pass the "mro_bind_override_addr" flag in the call
266 * to hermon_mr_register(). This guarantees that the resulting IB vaddr
267 * will be zero-based (modulo the offset into the first page). If we
268 * fail here, we still have the bunch of resource and reference count
269 * cleanup to do.
270 */
271 flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
272 IBT_MR_NOSLEEP;
273 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
274 mr_attr.mr_len = srq->srq_wqinfo.qa_size;
275 mr_attr.mr_as = NULL;
276 mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
277 mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
278 mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
279 mr_op.mro_bind_override_addr = 1;
280 status = hermon_mr_register(state, pd, &mr_attr, &mr,
281 &mr_op, HERMON_SRQ_CMPT);
282 if (status != DDI_SUCCESS) {
283 status = IBT_INSUFF_RESOURCE;
284 goto srqalloc_fail5;
285 }
286
287 /*
288 * Calculate the offset between the kernel virtual address space
289 * and the IB virtual address space. This will be used when
290 * posting work requests to properly initialize each WQE.
291 */
292 srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
293 (uint64_t)mr->mr_bindinfo.bi_addr;
294
295 srq->srq_wq_wqhdr = hermon_wrid_wqhdr_create(1 << log_srq_size);
296
297 /*
298 * Fill in all the return arguments (if necessary). This includes
299 * real queue size and real SGLs.
300 */
301 if (real_sizes != NULL) {
302 real_sizes->srq_wr_sz = (1 << log_srq_size) - 1;
303 real_sizes->srq_sgl_sz = srq->srq_wq_sgl;
304 }
305
499 state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size,
500 maxprot, DEVMAP_MAPPING_INVALID, NULL);
501 if (status != DDI_SUCCESS) {
502 mutex_exit(&srq->srq_lock);
503 HERMON_WARNING(state, "failed in SRQ memory "
504 "devmap_devmem_remap()");
505 return (ibc_get_ci_failure(0));
506 }
507 srq->srq_umap_dhp = (devmap_cookie_t)NULL;
508 }
509 }
510
511 /*
512 * Put NULL into the Hermon SRQNum-to-SRQHdl list. This will allow any
513 * in-progress events to detect that the SRQ corresponding to this
514 * number has been freed.
515 */
516 hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, NULL);
517
518 mutex_exit(&srq->srq_lock);
519
520 /*
521 * Reclaim SRQC entry from hardware (using the Hermon HW2SW_SRQ
522 * firmware command). If the ownership transfer fails for any reason,
523 * then it is an indication that something (either in HW or SW) has
524 * gone seriously wrong.
525 */
526 status = hermon_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
527 sizeof (hermon_hw_srqc_t), srqnum, sleepflag);
528 if (status != HERMON_CMD_SUCCESS) {
529 HERMON_WARNING(state, "failed to reclaim SRQC ownership");
530 cmn_err(CE_CONT, "Hermon: HW2SW_SRQ command failed: %08x\n",
531 status);
532 if (status == HERMON_CMD_INVALID_STATUS) {
533 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
534 }
535 return (ibc_get_ci_failure(0));
536 }
537
538 /*
642 * to make sure that all the resulting IB addresses will start at 0,
643 * for a zero-based queue. By making sure we are aligned on at least a
644 * page, any offset we use into our queue will be the same as it was
645 * when we allocated it at hermon_srq_alloc() time.
646 */
647 wqesz = (1 << srq->srq_wq_log_wqesz);
648 new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
649 new_srqinfo.qa_alloc_align = PAGESIZE;
650 new_srqinfo.qa_bind_align = PAGESIZE;
651 if (srq->srq_is_umap) {
652 new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
653 } else {
654 new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
655 }
656 status = hermon_queue_alloc(state, &new_srqinfo, sleepflag);
657 if (status != DDI_SUCCESS) {
658 status = IBT_INSUFF_RESOURCE;
659 goto srqmodify_fail;
660 }
661 buf = (uint32_t *)new_srqinfo.qa_buf_aligned;
662
663 /*
664 * Allocate the memory for the new WRE list. This will be used later
665 * when we resize the wridlist based on the new SRQ size.
666 */
667 wre_new = kmem_zalloc((1 << log_srq_size) * sizeof (uint64_t),
668 sleepflag);
669 if (wre_new == NULL) {
670 status = IBT_INSUFF_RESOURCE;
671 goto srqmodify_fail;
672 }
673
674 /*
675 * Fill in the "bind" struct. This struct provides the majority
676 * of the information that will be used to distinguish between an
677 * "addr" binding (as is the case here) and a "buf" binding (see
678 * below). The "bind" struct is later passed to hermon_mr_mem_bind()
679 * which does most of the "heavy lifting" for the Hermon memory
680 * registration routines.
681 */
682 bzero(&bind, sizeof (hermon_bind_info_t));
683 bind.bi_type = HERMON_BINDHDL_VADDR;
684 bind.bi_addr = (uint64_t)(uintptr_t)buf;
685 bind.bi_len = new_srqinfo.qa_size;
686 bind.bi_as = NULL;
687 bind.bi_flags = sleepflag == HERMON_SLEEP ? IBT_MR_SLEEP :
688 IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
689 bind.bi_bypass = state->hs_cfg_profile->cp_iommu_bypass;
690
691 status = hermon_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
692 &mtt_pgsize_bits, 0); /* no relaxed ordering */
693 if (status != DDI_SUCCESS) {
694 status = status;
695 kmem_free(wre_new, (1 << log_srq_size) *
696 sizeof (uint64_t));
697 hermon_queue_free(&new_srqinfo);
698 goto srqmodify_fail;
699 }
700
701 /*
|