1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * hermon_mr.c
28 * Hermon Memory Region/Window Routines
29 *
30 * Implements all the routines necessary to provide the requisite memory
31 * registration verbs. These include operations like RegisterMemRegion(),
32 * DeregisterMemRegion(), ReregisterMemRegion, RegisterSharedMemRegion,
33 * etc., that affect Memory Regions. It also includes the verbs that
34 * affect Memory Windows, including AllocMemWindow(), FreeMemWindow(),
35 * and QueryMemWindow().
36 */
37
38 #include <sys/types.h>
39 #include <sys/conf.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 #include <sys/modctl.h>
43 #include <sys/esunddi.h>
44
45 #include <sys/ib/adapters/hermon/hermon.h>
46
47 extern uint32_t hermon_kernel_data_ro;
48 extern uint32_t hermon_user_data_ro;
49 extern int hermon_rdma_debug;
50
51 /*
52 * Used by hermon_mr_keycalc() below to fill in the "unconstrained" portion
53 * of Hermon memory keys (LKeys and RKeys)
54 */
55 static uint_t hermon_memkey_cnt = 0x00;
56 #define HERMON_MEMKEY_SHIFT 24
57
58 /* initial state of an MPT */
59 #define HERMON_MPT_SW_OWNERSHIP 0xF /* memory regions */
60 #define HERMON_MPT_FREE 0x3 /* allocate lkey */
61
62 static int hermon_mr_common_reg(hermon_state_t *state, hermon_pdhdl_t pd,
63 hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op,
64 hermon_mpt_rsrc_type_t mpt_type);
65 static int hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr,
66 hermon_pdhdl_t pd, hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl_new,
67 hermon_mr_options_t *op);
68 static int hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr,
69 hermon_bind_info_t *bind, hermon_mr_options_t *op, uint64_t *mtt_addr,
70 uint_t sleep, uint_t *dereg_level);
71 static uint64_t hermon_mr_nummtt_needed(hermon_state_t *state,
72 hermon_bind_info_t *bind, uint_t *mtt_pgsize);
73 static int hermon_mr_mem_bind(hermon_state_t *state, hermon_bind_info_t *bind,
74 ddi_dma_handle_t dmahdl, uint_t sleep, uint_t is_buffer);
75 static void hermon_mr_mem_unbind(hermon_state_t *state,
76 hermon_bind_info_t *bind);
77 static int hermon_mr_fast_mtt_write(hermon_state_t *state, hermon_rsrc_t *mtt,
78 hermon_bind_info_t *bind, uint32_t mtt_pgsize_bits);
79 static int hermon_mr_fast_mtt_write_fmr(hermon_state_t *state,
80 hermon_rsrc_t *mtt, ibt_pmr_attr_t *mem_pattr, uint32_t mtt_pgsize_bits);
81 static uint_t hermon_mtt_refcnt_inc(hermon_rsrc_t *rsrc);
82 static uint_t hermon_mtt_refcnt_dec(hermon_rsrc_t *rsrc);
83
84
85 /*
86 * The Hermon umem_lockmemory() callback ops. When userland memory is
87 * registered, these callback ops are specified. The hermon_umap_umemlock_cb()
88 * callback will be called whenever the memory for the corresponding
89 * ddi_umem_cookie_t is being freed.
90 */
91 static struct umem_callback_ops hermon_umem_cbops = {
92 UMEM_CALLBACK_VERSION,
93 hermon_umap_umemlock_cb,
94 };
95
96
97
98 /*
99 * hermon_mr_register()
100 * Context: Can be called from interrupt or base context.
101 */
102 int
103 hermon_mr_register(hermon_state_t *state, hermon_pdhdl_t pd,
104 ibt_mr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op,
105 hermon_mpt_rsrc_type_t mpt_type)
106 {
107 hermon_bind_info_t bind;
108 int status;
109
110 /*
111 * Fill in the "bind" struct. This struct provides the majority
112 * of the information that will be used to distinguish between an
113 * "addr" binding (as is the case here) and a "buf" binding (see
114 * below). The "bind" struct is later passed to hermon_mr_mem_bind()
115 * which does most of the "heavy lifting" for the Hermon memory
116 * registration routines.
117 */
118 bind.bi_type = HERMON_BINDHDL_VADDR;
119 bind.bi_addr = mr_attr->mr_vaddr;
120 bind.bi_len = mr_attr->mr_len;
121 bind.bi_as = mr_attr->mr_as;
122 bind.bi_flags = mr_attr->mr_flags;
123 status = hermon_mr_common_reg(state, pd, &bind, mrhdl, op,
124 mpt_type);
125 return (status);
126 }
127
128
129 /*
130 * hermon_mr_register_buf()
131 * Context: Can be called from interrupt or base context.
132 */
133 int
134 hermon_mr_register_buf(hermon_state_t *state, hermon_pdhdl_t pd,
135 ibt_smr_attr_t *mr_attr, struct buf *buf, hermon_mrhdl_t *mrhdl,
136 hermon_mr_options_t *op, hermon_mpt_rsrc_type_t mpt_type)
137 {
138 hermon_bind_info_t bind;
139 int status;
140
141 /*
142 * Fill in the "bind" struct. This struct provides the majority
143 * of the information that will be used to distinguish between an
144 * "addr" binding (see above) and a "buf" binding (as is the case
145 * here). The "bind" struct is later passed to hermon_mr_mem_bind()
146 * which does most of the "heavy lifting" for the Hermon memory
147 * registration routines. Note: We have chosen to provide
148 * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
149 * not set). It is not critical what value we choose here as it need
150 * only be unique for the given RKey (which will happen by default),
151 * so the choice here is somewhat arbitrary.
152 */
153 bind.bi_type = HERMON_BINDHDL_BUF;
154 bind.bi_buf = buf;
155 if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) {
156 bind.bi_addr = mr_attr->mr_vaddr;
157 } else {
158 bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr;
159 }
160 bind.bi_as = NULL;
161 bind.bi_len = (uint64_t)buf->b_bcount;
162 bind.bi_flags = mr_attr->mr_flags;
163 status = hermon_mr_common_reg(state, pd, &bind, mrhdl, op, mpt_type);
164 return (status);
165 }
166
167
168 /*
169 * hermon_mr_register_shared()
170 * Context: Can be called from interrupt or base context.
171 */
172 int
173 hermon_mr_register_shared(hermon_state_t *state, hermon_mrhdl_t mrhdl,
174 hermon_pdhdl_t pd, ibt_smr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl_new)
175 {
176 hermon_rsrc_t *mpt, *mtt, *rsrc;
177 hermon_umap_db_entry_t *umapdb;
178 hermon_hw_dmpt_t mpt_entry;
179 hermon_mrhdl_t mr;
180 hermon_bind_info_t *bind;
181 ddi_umem_cookie_t umem_cookie;
182 size_t umem_len;
183 caddr_t umem_addr;
184 uint64_t mtt_addr, pgsize_msk;
185 uint_t sleep, mr_is_umem;
186 int status, umem_flags;
187
188 /*
189 * Check the sleep flag. Ensure that it is consistent with the
190 * current thread context (i.e. if we are currently in the interrupt
191 * context, then we shouldn't be attempting to sleep).
192 */
193 sleep = (mr_attr->mr_flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP :
194 HERMON_SLEEP;
195 if ((sleep == HERMON_SLEEP) &&
196 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
197 status = IBT_INVALID_PARAM;
198 goto mrshared_fail;
199 }
200
201 /* Increment the reference count on the protection domain (PD) */
202 hermon_pd_refcnt_inc(pd);
203
204 /*
205 * Allocate an MPT entry. This will be filled in with all the
206 * necessary parameters to define the shared memory region.
207 * Specifically, it will be made to reference the currently existing
208 * MTT entries and ownership of the MPT will be passed to the hardware
209 * in the last step below. If we fail here, we must undo the
210 * protection domain reference count.
211 */
212 status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
213 if (status != DDI_SUCCESS) {
214 status = IBT_INSUFF_RESOURCE;
215 goto mrshared_fail1;
216 }
217
218 /*
219 * Allocate the software structure for tracking the shared memory
220 * region (i.e. the Hermon Memory Region handle). If we fail here, we
221 * must undo the protection domain reference count and the previous
222 * resource allocation.
223 */
224 status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
225 if (status != DDI_SUCCESS) {
226 status = IBT_INSUFF_RESOURCE;
227 goto mrshared_fail2;
228 }
229 mr = (hermon_mrhdl_t)rsrc->hr_addr;
230 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
231
232 /*
233 * Setup and validate the memory region access flags. This means
234 * translating the IBTF's enable flags into the access flags that
235 * will be used in later operations.
236 */
237 mr->mr_accflag = 0;
238 if (mr_attr->mr_flags & IBT_MR_ENABLE_WINDOW_BIND)
239 mr->mr_accflag |= IBT_MR_WINDOW_BIND;
240 if (mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)
241 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
242 if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)
243 mr->mr_accflag |= IBT_MR_REMOTE_READ;
244 if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE)
245 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
246 if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
247 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
248
249 /*
250 * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
251 * from a certain number of "constrained" bits (the least significant
252 * bits) and some number of "unconstrained" bits. The constrained
253 * bits must be set to the index of the entry in the MPT table, but
254 * the unconstrained bits can be set to any value we wish. Note:
255 * if no remote access is required, then the RKey value is not filled
256 * in. Otherwise both Rkey and LKey are given the same value.
257 */
258 mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
259
260 /* Grab the MR lock for the current memory region */
261 mutex_enter(&mrhdl->mr_lock);
262
263 /*
264 * Check here to see if the memory region has already been partially
265 * deregistered as a result of a hermon_umap_umemlock_cb() callback.
266 * If so, this is an error, return failure.
267 */
268 if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
269 mutex_exit(&mrhdl->mr_lock);
270 status = IBT_MR_HDL_INVALID;
271 goto mrshared_fail3;
272 }
273
274 /*
275 * Determine if the original memory was from userland and, if so, pin
276 * the pages (again) with umem_lockmemory(). This will guarantee a
277 * separate callback for each of this shared region's MR handles.
278 * If this is userland memory, then allocate an entry in the
279 * "userland resources database". This will later be added to
280 * the database (after all further memory registration operations are
281 * successful). If we fail here, we must undo all the above setup.
282 */
283 mr_is_umem = mrhdl->mr_is_umem;
284 if (mr_is_umem) {
285 umem_len = ptob(btopr(mrhdl->mr_bindinfo.bi_len));
286 umem_addr = (caddr_t)((uintptr_t)mrhdl->mr_bindinfo.bi_addr &
287 ~PAGEOFFSET);
288 umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
289 DDI_UMEMLOCK_LONGTERM);
290 status = umem_lockmemory(umem_addr, umem_len, umem_flags,
291 &umem_cookie, &hermon_umem_cbops, NULL);
292 if (status != 0) {
293 mutex_exit(&mrhdl->mr_lock);
294 status = IBT_INSUFF_RESOURCE;
295 goto mrshared_fail3;
296 }
297
298 umapdb = hermon_umap_db_alloc(state->hs_instance,
299 (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
300 (uint64_t)(uintptr_t)rsrc);
301 if (umapdb == NULL) {
302 mutex_exit(&mrhdl->mr_lock);
303 status = IBT_INSUFF_RESOURCE;
304 goto mrshared_fail4;
305 }
306 }
307
308 /*
309 * Copy the MTT resource pointer (and additional parameters) from
310 * the original Hermon Memory Region handle. Note: this is normally
311 * where the hermon_mr_mem_bind() routine would be called, but because
312 * we already have bound and filled-in MTT entries it is simply a
313 * matter here of managing the MTT reference count and grabbing the
314 * address of the MTT table entries (for filling in the shared region's
315 * MPT entry).
316 */
317 mr->mr_mttrsrcp = mrhdl->mr_mttrsrcp;
318 mr->mr_logmttpgsz = mrhdl->mr_logmttpgsz;
319 mr->mr_bindinfo = mrhdl->mr_bindinfo;
320 mr->mr_mttrefcntp = mrhdl->mr_mttrefcntp;
321 mutex_exit(&mrhdl->mr_lock);
322 bind = &mr->mr_bindinfo;
323 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
324 mtt = mr->mr_mttrsrcp;
325
326 /*
327 * Increment the MTT reference count (to reflect the fact that
328 * the MTT is now shared)
329 */
330 (void) hermon_mtt_refcnt_inc(mr->mr_mttrefcntp);
331
332 /*
333 * Update the new "bind" virtual address. Do some extra work here
334 * to ensure proper alignment. That is, make sure that the page
335 * offset for the beginning of the old range is the same as the
336 * offset for this new mapping
337 */
338 pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1);
339 bind->bi_addr = ((mr_attr->mr_vaddr & ~pgsize_msk) |
340 (mr->mr_bindinfo.bi_addr & pgsize_msk));
341
342 /*
343 * Fill in the MPT entry. This is the final step before passing
344 * ownership of the MPT entry to the Hermon hardware. We use all of
345 * the information collected/calculated above to fill in the
346 * requisite portions of the MPT.
347 */
348 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
349 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
350 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
351 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
352 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
353 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
354 mpt_entry.lr = 1;
355 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
356 mpt_entry.entity_sz = mr->mr_logmttpgsz;
357 mpt_entry.mem_key = mr->mr_lkey;
358 mpt_entry.pd = pd->pd_pdnum;
359 mpt_entry.start_addr = bind->bi_addr;
360 mpt_entry.reg_win_len = bind->bi_len;
361 mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
362 mpt_entry.mtt_addr_h = mtt_addr >> 32;
363 mpt_entry.mtt_addr_l = mtt_addr >> 3;
364
365 /*
366 * Write the MPT entry to hardware. Lastly, we pass ownership of
367 * the entry to the hardware. Note: in general, this operation
368 * shouldn't fail. But if it does, we have to undo everything we've
369 * done above before returning error.
370 */
371 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
372 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
373 if (status != HERMON_CMD_SUCCESS) {
374 cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
375 status);
376 if (status == HERMON_CMD_INVALID_STATUS) {
377 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
378 }
379 status = ibc_get_ci_failure(0);
380 goto mrshared_fail5;
381 }
382
383 /*
384 * Fill in the rest of the Hermon Memory Region handle. Having
385 * successfully transferred ownership of the MPT, we can update the
386 * following fields for use in further operations on the MR.
387 */
388 mr->mr_mptrsrcp = mpt;
389 mr->mr_mttrsrcp = mtt;
390 mr->mr_mpt_type = HERMON_MPT_DMPT;
391 mr->mr_pdhdl = pd;
392 mr->mr_rsrcp = rsrc;
393 mr->mr_is_umem = mr_is_umem;
394 mr->mr_is_fmr = 0;
395 mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
396 mr->mr_umem_cbfunc = NULL;
397 mr->mr_umem_cbarg1 = NULL;
398 mr->mr_umem_cbarg2 = NULL;
399 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
400 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
401
402 /*
403 * If this is userland memory, then we need to insert the previously
404 * allocated entry into the "userland resources database". This will
405 * allow for later coordination between the hermon_umap_umemlock_cb()
406 * callback and hermon_mr_deregister().
407 */
408 if (mr_is_umem) {
409 hermon_umap_db_add(umapdb);
410 }
411
412 *mrhdl_new = mr;
413
414 return (DDI_SUCCESS);
415
416 /*
417 * The following is cleanup for all possible failure cases in this routine
418 */
419 mrshared_fail5:
420 (void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp);
421 if (mr_is_umem) {
422 hermon_umap_db_free(umapdb);
423 }
424 mrshared_fail4:
425 if (mr_is_umem) {
426 ddi_umem_unlock(umem_cookie);
427 }
428 mrshared_fail3:
429 hermon_rsrc_free(state, &rsrc);
430 mrshared_fail2:
431 hermon_rsrc_free(state, &mpt);
432 mrshared_fail1:
433 hermon_pd_refcnt_dec(pd);
434 mrshared_fail:
435 return (status);
436 }
437
438 /*
439 * hermon_mr_alloc_fmr()
440 * Context: Can be called from interrupt or base context.
441 */
442 int
443 hermon_mr_alloc_fmr(hermon_state_t *state, hermon_pdhdl_t pd,
444 hermon_fmrhdl_t fmr_pool, hermon_mrhdl_t *mrhdl)
445 {
446 hermon_rsrc_t *mpt, *mtt, *rsrc;
447 hermon_hw_dmpt_t mpt_entry;
448 hermon_mrhdl_t mr;
449 hermon_bind_info_t bind;
450 uint64_t mtt_addr;
451 uint64_t nummtt;
452 uint_t sleep, mtt_pgsize_bits;
453 int status;
454 offset_t i;
455 hermon_icm_table_t *icm_table;
456 hermon_dma_info_t *dma_info;
457 uint32_t index1, index2, rindx;
458
459 /*
460 * Check the sleep flag. Ensure that it is consistent with the
461 * current thread context (i.e. if we are currently in the interrupt
462 * context, then we shouldn't be attempting to sleep).
463 */
464 sleep = (fmr_pool->fmr_flags & IBT_MR_SLEEP) ? HERMON_SLEEP :
465 HERMON_NOSLEEP;
466 if ((sleep == HERMON_SLEEP) &&
467 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
468 return (IBT_INVALID_PARAM);
469 }
470
471 /* Increment the reference count on the protection domain (PD) */
472 hermon_pd_refcnt_inc(pd);
473
474 /*
475 * Allocate an MPT entry. This will be filled in with all the
476 * necessary parameters to define the FMR. Specifically, it will be
477 * made to reference the currently existing MTT entries and ownership
478 * of the MPT will be passed to the hardware in the last step below.
479 * If we fail here, we must undo the protection domain reference count.
480 */
481
482 status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
483 if (status != DDI_SUCCESS) {
484 status = IBT_INSUFF_RESOURCE;
485 goto fmralloc_fail1;
486 }
487
488 /*
489 * Allocate the software structure for tracking the fmr memory
490 * region (i.e. the Hermon Memory Region handle). If we fail here, we
491 * must undo the protection domain reference count and the previous
492 * resource allocation.
493 */
494 status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
495 if (status != DDI_SUCCESS) {
496 status = IBT_INSUFF_RESOURCE;
497 goto fmralloc_fail2;
498 }
499 mr = (hermon_mrhdl_t)rsrc->hr_addr;
500 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
501
502 /*
503 * Setup and validate the memory region access flags. This means
504 * translating the IBTF's enable flags into the access flags that
505 * will be used in later operations.
506 */
507 mr->mr_accflag = 0;
508 if (fmr_pool->fmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)
509 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
510 if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_READ)
511 mr->mr_accflag |= IBT_MR_REMOTE_READ;
512 if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_WRITE)
513 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
514 if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
515 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
516
517 /*
518 * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
519 * from a certain number of "constrained" bits (the least significant
520 * bits) and some number of "unconstrained" bits. The constrained
521 * bits must be set to the index of the entry in the MPT table, but
522 * the unconstrained bits can be set to any value we wish. Note:
523 * if no remote access is required, then the RKey value is not filled
524 * in. Otherwise both Rkey and LKey are given the same value.
525 */
526 mr->mr_fmr_key = 1; /* ready for the next reload */
527 mr->mr_rkey = mr->mr_lkey = mpt->hr_indx;
528
529 /*
530 * Determine number of pages spanned. This routine uses the
531 * information in the "bind" struct to determine the required
532 * number of MTT entries needed (and returns the suggested page size -
533 * as a "power-of-2" - for each MTT entry).
534 */
535 /* Assume address will be page aligned later */
536 bind.bi_addr = 0;
537 /* Calculate size based on given max pages */
538 bind.bi_len = fmr_pool->fmr_max_pages << PAGESHIFT;
539 nummtt = hermon_mr_nummtt_needed(state, &bind, &mtt_pgsize_bits);
540
541 /*
542 * Allocate the MTT entries. Use the calculations performed above to
543 * allocate the required number of MTT entries. If we fail here, we
544 * must not only undo all the previous resource allocation (and PD
545 * reference count), but we must also unbind the memory.
546 */
547 status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt, sleep, &mtt);
548 if (status != DDI_SUCCESS) {
549 IBTF_DPRINTF_L2("FMR", "FATAL: too few MTTs");
550 status = IBT_INSUFF_RESOURCE;
551 goto fmralloc_fail3;
552 }
553 mr->mr_logmttpgsz = mtt_pgsize_bits;
554
555 /*
556 * Fill in the MPT entry. This is the final step before passing
557 * ownership of the MPT entry to the Hermon hardware. We use all of
558 * the information collected/calculated above to fill in the
559 * requisite portions of the MPT.
560 */
561 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
562 mpt_entry.en_bind = 0;
563 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
564 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
565 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
566 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
567 mpt_entry.lr = 1;
568 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
569 mpt_entry.pd = pd->pd_pdnum;
570
571 mpt_entry.entity_sz = mr->mr_logmttpgsz;
572 mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
573 mpt_entry.fast_reg_en = 1;
574 mpt_entry.mtt_size = (uint_t)nummtt;
575 mpt_entry.mtt_addr_h = mtt_addr >> 32;
576 mpt_entry.mtt_addr_l = mtt_addr >> 3;
577 mpt_entry.mem_key = mr->mr_lkey;
578
579 /*
580 * FMR sets these to 0 for now. Later during actual fmr registration
581 * these values are filled in.
582 */
583 mpt_entry.start_addr = 0;
584 mpt_entry.reg_win_len = 0;
585
586 /*
587 * Write the MPT entry to hardware. Lastly, we pass ownership of
588 * the entry to the hardware. Note: in general, this operation
589 * shouldn't fail. But if it does, we have to undo everything we've
590 * done above before returning error.
591 */
592 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
593 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
594 if (status != HERMON_CMD_SUCCESS) {
595 cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
596 status);
597 if (status == HERMON_CMD_INVALID_STATUS) {
598 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
599 }
600 status = ibc_get_ci_failure(0);
601 goto fmralloc_fail4;
602 }
603
604 /*
605 * Fill in the rest of the Hermon Memory Region handle. Having
606 * successfully transferred ownership of the MPT, we can update the
607 * following fields for use in further operations on the MR. Also, set
608 * that this is an FMR region.
609 */
610 mr->mr_mptrsrcp = mpt;
611 mr->mr_mttrsrcp = mtt;
612
613 mr->mr_mpt_type = HERMON_MPT_DMPT;
614 mr->mr_pdhdl = pd;
615 mr->mr_rsrcp = rsrc;
616 mr->mr_is_fmr = 1;
617 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
618 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
619 mr->mr_mttaddr = mtt_addr;
620 (void) memcpy(&mr->mr_bindinfo, &bind, sizeof (hermon_bind_info_t));
621
622 /* initialize hr_addr for use during register/deregister/invalidate */
623 icm_table = &state->hs_icm[HERMON_DMPT];
624 rindx = mpt->hr_indx;
625 hermon_index(index1, index2, rindx, icm_table, i);
626 dma_info = icm_table->icm_dma[index1] + index2;
627 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mpt))
628 mpt->hr_addr = (void *)((uintptr_t)(dma_info->vaddr + i * mpt->hr_len));
629
630 *mrhdl = mr;
631
632 return (DDI_SUCCESS);
633
634 /*
635 * The following is cleanup for all possible failure cases in this routine
636 */
637 fmralloc_fail4:
638 kmem_free(mtt, sizeof (hermon_rsrc_t) * nummtt);
639 fmralloc_fail3:
640 hermon_rsrc_free(state, &rsrc);
641 fmralloc_fail2:
642 hermon_rsrc_free(state, &mpt);
643 fmralloc_fail1:
644 hermon_pd_refcnt_dec(pd);
645 fmralloc_fail:
646 return (status);
647 }
648
649
650 /*
651 * hermon_mr_register_physical_fmr()
652 * Context: Can be called from interrupt or base context.
653 */
654 /*ARGSUSED*/
655 int
656 hermon_mr_register_physical_fmr(hermon_state_t *state,
657 ibt_pmr_attr_t *mem_pattr_p, hermon_mrhdl_t mr, ibt_pmr_desc_t *mem_desc_p)
658 {
659 hermon_rsrc_t *mpt;
660 uint64_t *mpt_table;
661 int status;
662 uint32_t key;
663
664 mutex_enter(&mr->mr_lock);
665 mpt = mr->mr_mptrsrcp;
666 mpt_table = (uint64_t *)mpt->hr_addr;
667
668 /* Write MPT status to SW bit */
669 *(uint8_t *)mpt_table = 0xF0;
670
671 membar_producer();
672
673 /*
674 * Write the mapped addresses into the MTT entries. FMR needs to do
675 * this a little differently, so we call the fmr specific fast mtt
676 * write here.
677 */
678 status = hermon_mr_fast_mtt_write_fmr(state, mr->mr_mttrsrcp,
679 mem_pattr_p, mr->mr_logmttpgsz);
680 if (status != DDI_SUCCESS) {
681 mutex_exit(&mr->mr_lock);
682 status = ibc_get_ci_failure(0);
683 goto fmr_reg_fail1;
684 }
685
686 /*
687 * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
688 * from a certain number of "constrained" bits (the least significant
689 * bits) and some number of "unconstrained" bits. The constrained
690 * bits must be set to the index of the entry in the MPT table, but
691 * the unconstrained bits can be set to any value we wish. Note:
692 * if no remote access is required, then the RKey value is not filled
693 * in. Otherwise both Rkey and LKey are given the same value.
694 */
695 key = mpt->hr_indx | (mr->mr_fmr_key++ << HERMON_MEMKEY_SHIFT);
696 mr->mr_lkey = mr->mr_rkey = hermon_mr_key_swap(key);
697
698 /* write mem key value */
699 *(uint32_t *)&mpt_table[1] = htonl(key);
700
701 /* write length value */
702 mpt_table[3] = htonll(mem_pattr_p->pmr_len);
703
704 /* write start addr value */
705 mpt_table[2] = htonll(mem_pattr_p->pmr_iova);
706
707 /* write lkey value */
708 *(uint32_t *)&mpt_table[4] = htonl(key);
709
710 membar_producer();
711
712 /* Write MPT status to HW bit */
713 *(uint8_t *)mpt_table = 0x00;
714
715 /* Fill in return parameters */
716 mem_desc_p->pmd_lkey = mr->mr_lkey;
717 mem_desc_p->pmd_rkey = mr->mr_rkey;
718 mem_desc_p->pmd_iova = mem_pattr_p->pmr_iova;
719 mem_desc_p->pmd_phys_buf_list_sz = mem_pattr_p->pmr_len;
720
721 /* Fill in MR bindinfo struct for later sync or query operations */
722 mr->mr_bindinfo.bi_addr = mem_pattr_p->pmr_iova;
723 mr->mr_bindinfo.bi_flags = mem_pattr_p->pmr_flags & IBT_MR_NONCOHERENT;
724
725 mutex_exit(&mr->mr_lock);
726
727 return (DDI_SUCCESS);
728
729 fmr_reg_fail1:
730 /*
731 * Note, we fail here, and purposely leave the memory ownership in
732 * software. The memory tables may be corrupt, so we leave the region
733 * unregistered.
734 */
735 return (status);
736 }
737
738
739 /*
740 * hermon_mr_deregister()
741 * Context: Can be called from interrupt or base context.
742 */
743 /* ARGSUSED */
744 int
745 hermon_mr_deregister(hermon_state_t *state, hermon_mrhdl_t *mrhdl, uint_t level,
746 uint_t sleep)
747 {
748 hermon_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt;
749 hermon_umap_db_entry_t *umapdb;
750 hermon_pdhdl_t pd;
751 hermon_mrhdl_t mr;
752 hermon_bind_info_t *bind;
753 uint64_t value;
754 int status;
755 uint_t shared_mtt;
756
757 /*
758 * Check the sleep flag. Ensure that it is consistent with the
759 * current thread context (i.e. if we are currently in the interrupt
760 * context, then we shouldn't be attempting to sleep).
761 */
762 if ((sleep == HERMON_SLEEP) &&
763 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
764 status = IBT_INVALID_PARAM;
765 return (status);
766 }
767
768 /*
769 * Pull all the necessary information from the Hermon Memory Region
770 * handle. This is necessary here because the resource for the
771 * MR handle is going to be freed up as part of the this
772 * deregistration
773 */
774 mr = *mrhdl;
775 mutex_enter(&mr->mr_lock);
776 mpt = mr->mr_mptrsrcp;
777 mtt = mr->mr_mttrsrcp;
778 mtt_refcnt = mr->mr_mttrefcntp;
779 rsrc = mr->mr_rsrcp;
780 pd = mr->mr_pdhdl;
781 bind = &mr->mr_bindinfo;
782
783 /*
784 * Check here if the memory region is really an FMR. If so, this is a
785 * bad thing and we shouldn't be here. Return failure.
786 */
787 if (mr->mr_is_fmr) {
788 mutex_exit(&mr->mr_lock);
789 return (IBT_INVALID_PARAM);
790 }
791
792 /*
793 * Check here to see if the memory region has already been partially
794 * deregistered as a result of the hermon_umap_umemlock_cb() callback.
795 * If so, then jump to the end and free the remaining resources.
796 */
797 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
798 goto mrdereg_finish_cleanup;
799 }
800 if (hermon_rdma_debug & 0x4)
801 IBTF_DPRINTF_L2("mr", "dereg: mr %p key %x",
802 mr, mr->mr_rkey);
803
804 /*
805 * We must drop the "mr_lock" here to ensure that both SLEEP and
806 * NOSLEEP calls into the firmware work as expected. Also, if two
807 * threads are attemping to access this MR (via de-register,
808 * re-register, or otherwise), then we allow the firmware to enforce
809 * the checking, that only one deregister is valid.
810 */
811 mutex_exit(&mr->mr_lock);
812
813 /*
814 * Reclaim MPT entry from hardware (if necessary). Since the
815 * hermon_mr_deregister() routine is used in the memory region
816 * reregistration process as well, it is possible that we will
817 * not always wish to reclaim ownership of the MPT. Check the
818 * "level" arg and, if necessary, attempt to reclaim it. If
819 * the ownership transfer fails for any reason, we check to see
820 * what command status was returned from the hardware. The only
821 * "expected" error status is the one that indicates an attempt to
822 * deregister a memory region that has memory windows bound to it
823 */
824 if (level >= HERMON_MR_DEREG_ALL) {
825 if (mr->mr_mpt_type >= HERMON_MPT_DMPT) {
826 status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT,
827 NULL, 0, mpt->hr_indx, sleep);
828 if (status != HERMON_CMD_SUCCESS) {
829 if (status == HERMON_CMD_REG_BOUND) {
830 return (IBT_MR_IN_USE);
831 } else {
832 cmn_err(CE_CONT, "Hermon: HW2SW_MPT "
833 "command failed: %08x\n", status);
834 if (status ==
835 HERMON_CMD_INVALID_STATUS) {
836 hermon_fm_ereport(state,
837 HCA_SYS_ERR,
838 DDI_SERVICE_LOST);
839 }
840 return (IBT_INVALID_PARAM);
841 }
842 }
843 }
844 }
845
846 /*
847 * Re-grab the mr_lock here. Since further access to the protected
848 * 'mr' structure is needed, and we would have returned previously for
849 * the multiple deregistration case, we can safely grab the lock here.
850 */
851 mutex_enter(&mr->mr_lock);
852
853 /*
854 * If the memory had come from userland, then we do a lookup in the
855 * "userland resources database". On success, we free the entry, call
856 * ddi_umem_unlock(), and continue the cleanup. On failure (which is
857 * an indication that the umem_lockmemory() callback has called
858 * hermon_mr_deregister()), we call ddi_umem_unlock() and invalidate
859 * the "mr_umemcookie" field in the MR handle (this will be used
860 * later to detect that only partial cleaup still remains to be done
861 * on the MR handle).
862 */
863 if (mr->mr_is_umem) {
864 status = hermon_umap_db_find(state->hs_instance,
865 (uint64_t)(uintptr_t)mr->mr_umemcookie,
866 MLNX_UMAP_MRMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
867 &umapdb);
868 if (status == DDI_SUCCESS) {
869 hermon_umap_db_free(umapdb);
870 ddi_umem_unlock(mr->mr_umemcookie);
871 } else {
872 ddi_umem_unlock(mr->mr_umemcookie);
873 mr->mr_umemcookie = NULL;
874 }
875 }
876
877 /* mtt_refcnt is NULL in the case of hermon_dma_mr_register() */
878 if (mtt_refcnt != NULL) {
879 /*
880 * Decrement the MTT reference count. Since the MTT resource
881 * may be shared between multiple memory regions (as a result
882 * of a "RegisterSharedMR" verb) it is important that we not
883 * free up or unbind resources prematurely. If it's not shared
884 * (as indicated by the return status), then free the resource.
885 */
886 shared_mtt = hermon_mtt_refcnt_dec(mtt_refcnt);
887 if (!shared_mtt) {
888 hermon_rsrc_free(state, &mtt_refcnt);
889 }
890
891 /*
892 * Free up the MTT entries and unbind the memory. Here,
893 * as above, we attempt to free these resources only if
894 * it is appropriate to do so.
895 * Note, 'bind' is NULL in the alloc_lkey case.
896 */
897 if (!shared_mtt) {
898 if (level >= HERMON_MR_DEREG_NO_HW2SW_MPT) {
899 hermon_mr_mem_unbind(state, bind);
900 }
901 hermon_rsrc_free(state, &mtt);
902 }
903 }
904
905 /*
906 * If the MR handle has been invalidated, then drop the
907 * lock and return success. Note: This only happens because
908 * the umem_lockmemory() callback has been triggered. The
909 * cleanup here is partial, and further cleanup (in a
910 * subsequent hermon_mr_deregister() call) will be necessary.
911 */
912 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
913 mutex_exit(&mr->mr_lock);
914 return (DDI_SUCCESS);
915 }
916
917 mrdereg_finish_cleanup:
918 mutex_exit(&mr->mr_lock);
919
920 /* Free the Hermon Memory Region handle */
921 hermon_rsrc_free(state, &rsrc);
922
923 /* Free up the MPT entry resource */
924 if (mpt != NULL)
925 hermon_rsrc_free(state, &mpt);
926
927 /* Decrement the reference count on the protection domain (PD) */
928 hermon_pd_refcnt_dec(pd);
929
930 /* Set the mrhdl pointer to NULL and return success */
931 *mrhdl = NULL;
932
933 return (DDI_SUCCESS);
934 }
935
936 /*
937 * hermon_mr_dealloc_fmr()
938 * Context: Can be called from interrupt or base context.
939 */
940 /* ARGSUSED */
941 int
942 hermon_mr_dealloc_fmr(hermon_state_t *state, hermon_mrhdl_t *mrhdl)
943 {
944 hermon_rsrc_t *mpt, *mtt, *rsrc;
945 hermon_pdhdl_t pd;
946 hermon_mrhdl_t mr;
947
948 /*
949 * Pull all the necessary information from the Hermon Memory Region
950 * handle. This is necessary here because the resource for the
951 * MR handle is going to be freed up as part of the this
952 * deregistration
953 */
954 mr = *mrhdl;
955 mutex_enter(&mr->mr_lock);
956 mpt = mr->mr_mptrsrcp;
957 mtt = mr->mr_mttrsrcp;
958 rsrc = mr->mr_rsrcp;
959 pd = mr->mr_pdhdl;
960 mutex_exit(&mr->mr_lock);
961
962 /* Free the MTT entries */
963 hermon_rsrc_free(state, &mtt);
964
965 /* Free the Hermon Memory Region handle */
966 hermon_rsrc_free(state, &rsrc);
967
968 /* Free up the MPT entry resource */
969 hermon_rsrc_free(state, &mpt);
970
971 /* Decrement the reference count on the protection domain (PD) */
972 hermon_pd_refcnt_dec(pd);
973
974 /* Set the mrhdl pointer to NULL and return success */
975 *mrhdl = NULL;
976
977 return (DDI_SUCCESS);
978 }
979
980
981 /*
982 * hermon_mr_query()
983 * Context: Can be called from interrupt or base context.
984 */
985 /* ARGSUSED */
986 int
987 hermon_mr_query(hermon_state_t *state, hermon_mrhdl_t mr,
988 ibt_mr_query_attr_t *attr)
989 {
990 int status;
991 hermon_hw_dmpt_t mpt_entry;
992 uint32_t lkey;
993
994 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr))
995
996 mutex_enter(&mr->mr_lock);
997
998 /*
999 * Check here to see if the memory region has already been partially
1000 * deregistered as a result of a hermon_umap_umemlock_cb() callback.
1001 * If so, this is an error, return failure.
1002 */
1003 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
1004 mutex_exit(&mr->mr_lock);
1005 return (IBT_MR_HDL_INVALID);
1006 }
1007
1008 status = hermon_cmn_query_cmd_post(state, QUERY_MPT, 0,
1009 mr->mr_lkey >> 8, &mpt_entry, sizeof (hermon_hw_dmpt_t),
1010 HERMON_NOSLEEP);
1011 if (status != HERMON_CMD_SUCCESS) {
1012 cmn_err(CE_CONT, "Hermon: QUERY_MPT failed: status %x", status);
1013 mutex_exit(&mr->mr_lock);
1014 return (ibc_get_ci_failure(0));
1015 }
1016
1017 /* Update the mr sw struct from the hw struct. */
1018 lkey = mpt_entry.mem_key;
1019 mr->mr_lkey = mr->mr_rkey = (lkey >> 8) | (lkey << 24);
1020 mr->mr_bindinfo.bi_addr = mpt_entry.start_addr;
1021 mr->mr_bindinfo.bi_len = mpt_entry.reg_win_len;
1022 mr->mr_accflag = (mr->mr_accflag & IBT_MR_RO_DISABLED) |
1023 (mpt_entry.lw ? IBT_MR_LOCAL_WRITE : 0) |
1024 (mpt_entry.rr ? IBT_MR_REMOTE_READ : 0) |
1025 (mpt_entry.rw ? IBT_MR_REMOTE_WRITE : 0) |
1026 (mpt_entry.atomic ? IBT_MR_REMOTE_ATOMIC : 0) |
1027 (mpt_entry.en_bind ? IBT_MR_WINDOW_BIND : 0);
1028 mr->mr_mttaddr = ((uint64_t)mpt_entry.mtt_addr_h << 32) |
1029 (mpt_entry.mtt_addr_l << 3);
1030 mr->mr_logmttpgsz = mpt_entry.entity_sz;
1031
1032 /* Fill in the queried attributes */
1033 attr->mr_lkey_state =
1034 (mpt_entry.status == HERMON_MPT_FREE) ? IBT_KEY_FREE :
1035 (mpt_entry.status == HERMON_MPT_SW_OWNERSHIP) ? IBT_KEY_INVALID :
1036 IBT_KEY_VALID;
1037 attr->mr_phys_buf_list_sz = mpt_entry.mtt_size;
1038 attr->mr_attr_flags = mr->mr_accflag;
1039 attr->mr_pd = (ibt_pd_hdl_t)mr->mr_pdhdl;
1040
1041 /* Fill in the "local" attributes */
1042 attr->mr_lkey = (ibt_lkey_t)mr->mr_lkey;
1043 attr->mr_lbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
1044 attr->mr_lbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
1045
1046 /*
1047 * Fill in the "remote" attributes (if necessary). Note: the
1048 * remote attributes are only valid if the memory region has one
1049 * or more of the remote access flags set.
1050 */
1051 if ((mr->mr_accflag & IBT_MR_REMOTE_READ) ||
1052 (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ||
1053 (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) {
1054 attr->mr_rkey = (ibt_rkey_t)mr->mr_rkey;
1055 attr->mr_rbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
1056 attr->mr_rbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
1057 }
1058
1059 /*
1060 * If region is mapped for streaming (i.e. noncoherent), then set sync
1061 * is required
1062 */
1063 attr->mr_sync_required = (mr->mr_bindinfo.bi_flags &
1064 IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
1065
1066 mutex_exit(&mr->mr_lock);
1067 return (DDI_SUCCESS);
1068 }
1069
1070
1071 /*
1072 * hermon_mr_reregister()
1073 * Context: Can be called from interrupt or base context.
1074 */
1075 int
1076 hermon_mr_reregister(hermon_state_t *state, hermon_mrhdl_t mr,
1077 hermon_pdhdl_t pd, ibt_mr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl_new,
1078 hermon_mr_options_t *op)
1079 {
1080 hermon_bind_info_t bind;
1081 int status;
1082
1083 /*
1084 * Fill in the "bind" struct. This struct provides the majority
1085 * of the information that will be used to distinguish between an
1086 * "addr" binding (as is the case here) and a "buf" binding (see
1087 * below). The "bind" struct is later passed to hermon_mr_mem_bind()
1088 * which does most of the "heavy lifting" for the Hermon memory
1089 * registration (and reregistration) routines.
1090 */
1091 bind.bi_type = HERMON_BINDHDL_VADDR;
1092 bind.bi_addr = mr_attr->mr_vaddr;
1093 bind.bi_len = mr_attr->mr_len;
1094 bind.bi_as = mr_attr->mr_as;
1095 bind.bi_flags = mr_attr->mr_flags;
1096 status = hermon_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
1097 return (status);
1098 }
1099
1100
1101 /*
1102 * hermon_mr_reregister_buf()
1103 * Context: Can be called from interrupt or base context.
1104 */
1105 int
1106 hermon_mr_reregister_buf(hermon_state_t *state, hermon_mrhdl_t mr,
1107 hermon_pdhdl_t pd, ibt_smr_attr_t *mr_attr, struct buf *buf,
1108 hermon_mrhdl_t *mrhdl_new, hermon_mr_options_t *op)
1109 {
1110 hermon_bind_info_t bind;
1111 int status;
1112
1113 /*
1114 * Fill in the "bind" struct. This struct provides the majority
1115 * of the information that will be used to distinguish between an
1116 * "addr" binding (see above) and a "buf" binding (as is the case
1117 * here). The "bind" struct is later passed to hermon_mr_mem_bind()
1118 * which does most of the "heavy lifting" for the Hermon memory
1119 * registration routines. Note: We have chosen to provide
1120 * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
1121 * not set). It is not critical what value we choose here as it need
1122 * only be unique for the given RKey (which will happen by default),
1123 * so the choice here is somewhat arbitrary.
1124 */
1125 bind.bi_type = HERMON_BINDHDL_BUF;
1126 bind.bi_buf = buf;
1127 if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) {
1128 bind.bi_addr = mr_attr->mr_vaddr;
1129 } else {
1130 bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr;
1131 }
1132 bind.bi_len = (uint64_t)buf->b_bcount;
1133 bind.bi_flags = mr_attr->mr_flags;
1134 bind.bi_as = NULL;
1135 status = hermon_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
1136 return (status);
1137 }
1138
1139
1140 /*
1141 * hermon_mr_sync()
1142 * Context: Can be called from interrupt or base context.
1143 */
1144 /* ARGSUSED */
1145 int
1146 hermon_mr_sync(hermon_state_t *state, ibt_mr_sync_t *mr_segs, size_t num_segs)
1147 {
1148 hermon_mrhdl_t mrhdl;
1149 uint64_t seg_vaddr, seg_len, seg_end;
1150 uint64_t mr_start, mr_end;
1151 uint_t type;
1152 int status, i;
1153
1154 /* Process each of the ibt_mr_sync_t's */
1155 for (i = 0; i < num_segs; i++) {
1156 mrhdl = (hermon_mrhdl_t)mr_segs[i].ms_handle;
1157
1158 /* Check for valid memory region handle */
1159 if (mrhdl == NULL) {
1160 status = IBT_MR_HDL_INVALID;
1161 goto mrsync_fail;
1162 }
1163
1164 mutex_enter(&mrhdl->mr_lock);
1165
1166 /*
1167 * Check here to see if the memory region has already been
1168 * partially deregistered as a result of a
1169 * hermon_umap_umemlock_cb() callback. If so, this is an
1170 * error, return failure.
1171 */
1172 if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
1173 mutex_exit(&mrhdl->mr_lock);
1174 status = IBT_MR_HDL_INVALID;
1175 goto mrsync_fail;
1176 }
1177
1178 /* Check for valid bounds on sync request */
1179 seg_vaddr = mr_segs[i].ms_vaddr;
1180 seg_len = mr_segs[i].ms_len;
1181 seg_end = seg_vaddr + seg_len - 1;
1182 mr_start = mrhdl->mr_bindinfo.bi_addr;
1183 mr_end = mr_start + mrhdl->mr_bindinfo.bi_len - 1;
1184 if ((seg_vaddr < mr_start) || (seg_vaddr > mr_end)) {
1185 mutex_exit(&mrhdl->mr_lock);
1186 status = IBT_MR_VA_INVALID;
1187 goto mrsync_fail;
1188 }
1189 if ((seg_end < mr_start) || (seg_end > mr_end)) {
1190 mutex_exit(&mrhdl->mr_lock);
1191 status = IBT_MR_LEN_INVALID;
1192 goto mrsync_fail;
1193 }
1194
1195 /* Determine what type (i.e. direction) for sync */
1196 if (mr_segs[i].ms_flags & IBT_SYNC_READ) {
1197 type = DDI_DMA_SYNC_FORDEV;
1198 } else if (mr_segs[i].ms_flags & IBT_SYNC_WRITE) {
1199 type = DDI_DMA_SYNC_FORCPU;
1200 } else {
1201 mutex_exit(&mrhdl->mr_lock);
1202 status = IBT_INVALID_PARAM;
1203 goto mrsync_fail;
1204 }
1205
1206 (void) ddi_dma_sync(mrhdl->mr_bindinfo.bi_dmahdl,
1207 (off_t)(seg_vaddr - mr_start), (size_t)seg_len, type);
1208
1209 mutex_exit(&mrhdl->mr_lock);
1210 }
1211
1212 return (DDI_SUCCESS);
1213
1214 mrsync_fail:
1215 return (status);
1216 }
1217
1218
1219 /*
1220 * hermon_mw_alloc()
1221 * Context: Can be called from interrupt or base context.
1222 */
1223 int
1224 hermon_mw_alloc(hermon_state_t *state, hermon_pdhdl_t pd, ibt_mw_flags_t flags,
1225 hermon_mwhdl_t *mwhdl)
1226 {
1227 hermon_rsrc_t *mpt, *rsrc;
1228 hermon_hw_dmpt_t mpt_entry;
1229 hermon_mwhdl_t mw;
1230 uint_t sleep;
1231 int status;
1232
1233 if (state != NULL) /* XXX - bogus test that is always TRUE */
1234 return (IBT_INSUFF_RESOURCE);
1235
1236 /*
1237 * Check the sleep flag. Ensure that it is consistent with the
1238 * current thread context (i.e. if we are currently in the interrupt
1239 * context, then we shouldn't be attempting to sleep).
1240 */
1241 sleep = (flags & IBT_MW_NOSLEEP) ? HERMON_NOSLEEP : HERMON_SLEEP;
1242 if ((sleep == HERMON_SLEEP) &&
1243 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
1244 status = IBT_INVALID_PARAM;
1245 goto mwalloc_fail;
1246 }
1247
1248 /* Increment the reference count on the protection domain (PD) */
1249 hermon_pd_refcnt_inc(pd);
1250
1251 /*
1252 * Allocate an MPT entry (for use as a memory window). Since the
1253 * Hermon hardware uses the MPT entry for memory regions and for
1254 * memory windows, we will fill in this MPT with all the necessary
1255 * parameters for the memory window. And then (just as we do for
1256 * memory regions) ownership will be passed to the hardware in the
1257 * final step below. If we fail here, we must undo the protection
1258 * domain reference count.
1259 */
1260 status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
1261 if (status != DDI_SUCCESS) {
1262 status = IBT_INSUFF_RESOURCE;
1263 goto mwalloc_fail1;
1264 }
1265
1266 /*
1267 * Allocate the software structure for tracking the memory window (i.e.
1268 * the Hermon Memory Window handle). Note: This is actually the same
1269 * software structure used for tracking memory regions, but since many
1270 * of the same properties are needed, only a single structure is
1271 * necessary. If we fail here, we must undo the protection domain
1272 * reference count and the previous resource allocation.
1273 */
1274 status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
1275 if (status != DDI_SUCCESS) {
1276 status = IBT_INSUFF_RESOURCE;
1277 goto mwalloc_fail2;
1278 }
1279 mw = (hermon_mwhdl_t)rsrc->hr_addr;
1280 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw))
1281
1282 /*
1283 * Calculate an "unbound" RKey from MPT index. In much the same way
1284 * as we do for memory regions (above), this key is constructed from
1285 * a "constrained" (which depends on the MPT index) and an
1286 * "unconstrained" portion (which may be arbitrarily chosen).
1287 */
1288 mw->mr_rkey = hermon_mr_keycalc(mpt->hr_indx);
1289
1290 /*
1291 * Fill in the MPT entry. This is the final step before passing
1292 * ownership of the MPT entry to the Hermon hardware. We use all of
1293 * the information collected/calculated above to fill in the
1294 * requisite portions of the MPT. Note: fewer entries in the MPT
1295 * entry are necessary to allocate a memory window.
1296 */
1297 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
1298 mpt_entry.reg_win = HERMON_MPT_IS_WINDOW;
1299 mpt_entry.mem_key = mw->mr_rkey;
1300 mpt_entry.pd = pd->pd_pdnum;
1301 mpt_entry.lr = 1;
1302
1303 /*
1304 * Write the MPT entry to hardware. Lastly, we pass ownership of
1305 * the entry to the hardware. Note: in general, this operation
1306 * shouldn't fail. But if it does, we have to undo everything we've
1307 * done above before returning error.
1308 */
1309 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
1310 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
1311 if (status != HERMON_CMD_SUCCESS) {
1312 cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
1313 status);
1314 if (status == HERMON_CMD_INVALID_STATUS) {
1315 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
1316 }
1317 status = ibc_get_ci_failure(0);
1318 goto mwalloc_fail3;
1319 }
1320
1321 /*
1322 * Fill in the rest of the Hermon Memory Window handle. Having
1323 * successfully transferred ownership of the MPT, we can update the
1324 * following fields for use in further operations on the MW.
1325 */
1326 mw->mr_mptrsrcp = mpt;
1327 mw->mr_pdhdl = pd;
1328 mw->mr_rsrcp = rsrc;
1329 mw->mr_rkey = hermon_mr_key_swap(mw->mr_rkey);
1330 *mwhdl = mw;
1331
1332 return (DDI_SUCCESS);
1333
1334 mwalloc_fail3:
1335 hermon_rsrc_free(state, &rsrc);
1336 mwalloc_fail2:
1337 hermon_rsrc_free(state, &mpt);
1338 mwalloc_fail1:
1339 hermon_pd_refcnt_dec(pd);
1340 mwalloc_fail:
1341 return (status);
1342 }
1343
1344
1345 /*
1346 * hermon_mw_free()
1347 * Context: Can be called from interrupt or base context.
1348 */
1349 int
1350 hermon_mw_free(hermon_state_t *state, hermon_mwhdl_t *mwhdl, uint_t sleep)
1351 {
1352 hermon_rsrc_t *mpt, *rsrc;
1353 hermon_mwhdl_t mw;
1354 int status;
1355 hermon_pdhdl_t pd;
1356
1357 /*
1358 * Check the sleep flag. Ensure that it is consistent with the
1359 * current thread context (i.e. if we are currently in the interrupt
1360 * context, then we shouldn't be attempting to sleep).
1361 */
1362 if ((sleep == HERMON_SLEEP) &&
1363 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
1364 status = IBT_INVALID_PARAM;
1365 return (status);
1366 }
1367
1368 /*
1369 * Pull all the necessary information from the Hermon Memory Window
1370 * handle. This is necessary here because the resource for the
1371 * MW handle is going to be freed up as part of the this operation.
1372 */
1373 mw = *mwhdl;
1374 mutex_enter(&mw->mr_lock);
1375 mpt = mw->mr_mptrsrcp;
1376 rsrc = mw->mr_rsrcp;
1377 pd = mw->mr_pdhdl;
1378 mutex_exit(&mw->mr_lock);
1379 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw))
1380
1381 /*
1382 * Reclaim the MPT entry from hardware. Note: in general, it is
1383 * unexpected for this operation to return an error.
1384 */
1385 status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT, NULL,
1386 0, mpt->hr_indx, sleep);
1387 if (status != HERMON_CMD_SUCCESS) {
1388 cmn_err(CE_CONT, "Hermon: HW2SW_MPT command failed: %08x\n",
1389 status);
1390 if (status == HERMON_CMD_INVALID_STATUS) {
1391 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
1392 }
1393 return (ibc_get_ci_failure(0));
1394 }
1395
1396 /* Free the Hermon Memory Window handle */
1397 hermon_rsrc_free(state, &rsrc);
1398
1399 /* Free up the MPT entry resource */
1400 hermon_rsrc_free(state, &mpt);
1401
1402 /* Decrement the reference count on the protection domain (PD) */
1403 hermon_pd_refcnt_dec(pd);
1404
1405 /* Set the mwhdl pointer to NULL and return success */
1406 *mwhdl = NULL;
1407
1408 return (DDI_SUCCESS);
1409 }
1410
1411
1412 /*
1413 * hermon_mr_keycalc()
1414 * Context: Can be called from interrupt or base context.
1415 * NOTE: Produces a key in the form of
1416 * KKKKKKKK IIIIIIII IIIIIIII IIIIIIIII
1417 * where K == the arbitrary bits and I == the index
1418 */
1419 uint32_t
1420 hermon_mr_keycalc(uint32_t indx)
1421 {
1422 uint32_t tmp_key, tmp_indx;
1423
1424 /*
1425 * Generate a simple key from counter. Note: We increment this
1426 * static variable _intentionally_ without any kind of mutex around
1427 * it. First, single-threading all operations through a single lock
1428 * would be a bad idea (from a performance point-of-view). Second,
1429 * the upper "unconstrained" bits don't really have to be unique
1430 * because the lower bits are guaranteed to be (although we do make a
1431 * best effort to ensure that they are). Third, the window for the
1432 * race (where both threads read and update the counter at the same
1433 * time) is incredibly small.
1434 * And, lastly, we'd like to make this into a "random" key
1435 */
1436 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(hermon_memkey_cnt))
1437 tmp_key = (hermon_memkey_cnt++) << HERMON_MEMKEY_SHIFT;
1438 tmp_indx = indx & 0xffffff;
1439 return (tmp_key | tmp_indx);
1440 }
1441
1442
1443 /*
1444 * hermon_mr_key_swap()
1445 * Context: Can be called from interrupt or base context.
1446 * NOTE: Produces a key in the form of
1447 * IIIIIIII IIIIIIII IIIIIIIII KKKKKKKK
1448 * where K == the arbitrary bits and I == the index
1449 */
1450 uint32_t
1451 hermon_mr_key_swap(uint32_t indx)
1452 {
1453 /*
1454 * The memory key format to pass down to the hardware is
1455 * (key[7:0],index[23:0]), which defines the index to the
1456 * hardware resource. When the driver passes this as a memory
1457 * key, (i.e. to retrieve a resource) the format is
1458 * (index[23:0],key[7:0]).
1459 */
1460 return (((indx >> 24) & 0x000000ff) | ((indx << 8) & 0xffffff00));
1461 }
1462
1463 /*
1464 * hermon_mr_common_reg()
1465 * Context: Can be called from interrupt or base context.
1466 */
1467 static int
1468 hermon_mr_common_reg(hermon_state_t *state, hermon_pdhdl_t pd,
1469 hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op,
1470 hermon_mpt_rsrc_type_t mpt_type)
1471 {
1472 hermon_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt;
1473 hermon_umap_db_entry_t *umapdb;
1474 hermon_sw_refcnt_t *swrc_tmp;
1475 hermon_hw_dmpt_t mpt_entry;
1476 hermon_mrhdl_t mr;
1477 ibt_mr_flags_t flags;
1478 hermon_bind_info_t *bh;
1479 ddi_dma_handle_t bind_dmahdl;
1480 ddi_umem_cookie_t umem_cookie;
1481 size_t umem_len;
1482 caddr_t umem_addr;
1483 uint64_t mtt_addr, max_sz;
1484 uint_t sleep, mtt_pgsize_bits, bind_type, mr_is_umem;
1485 int status, umem_flags, bind_override_addr;
1486
1487 /*
1488 * Check the "options" flag. Currently this flag tells the driver
1489 * whether or not the region should be bound normally (i.e. with
1490 * entries written into the PCI IOMMU), whether it should be
1491 * registered to bypass the IOMMU, and whether or not the resulting
1492 * address should be "zero-based" (to aid the alignment restrictions
1493 * for QPs).
1494 */
1495 if (op == NULL) {
1496 bind_type = HERMON_BINDMEM_NORMAL;
1497 bind_dmahdl = NULL;
1498 bind_override_addr = 0;
1499 } else {
1500 bind_type = op->mro_bind_type;
1501 bind_dmahdl = op->mro_bind_dmahdl;
1502 bind_override_addr = op->mro_bind_override_addr;
1503 }
1504
1505 /* check what kind of mpt to use */
1506
1507 /* Extract the flags field from the hermon_bind_info_t */
1508 flags = bind->bi_flags;
1509
1510 /*
1511 * Check for invalid length. Check is the length is zero or if the
1512 * length is larger than the maximum configured value. Return error
1513 * if it is.
1514 */
1515 max_sz = ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_mrw_sz);
1516 if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) {
1517 status = IBT_MR_LEN_INVALID;
1518 goto mrcommon_fail;
1519 }
1520
1521 /*
1522 * Check the sleep flag. Ensure that it is consistent with the
1523 * current thread context (i.e. if we are currently in the interrupt
1524 * context, then we shouldn't be attempting to sleep).
1525 */
1526 sleep = (flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP: HERMON_SLEEP;
1527 if ((sleep == HERMON_SLEEP) &&
1528 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
1529 status = IBT_INVALID_PARAM;
1530 goto mrcommon_fail;
1531 }
1532
1533 /* Increment the reference count on the protection domain (PD) */
1534 hermon_pd_refcnt_inc(pd);
1535
1536 /*
1537 * Allocate an MPT entry. This will be filled in with all the
1538 * necessary parameters to define the memory region. And then
1539 * ownership will be passed to the hardware in the final step
1540 * below. If we fail here, we must undo the protection domain
1541 * reference count.
1542 */
1543 if (mpt_type == HERMON_MPT_DMPT) {
1544 status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
1545 if (status != DDI_SUCCESS) {
1546 status = IBT_INSUFF_RESOURCE;
1547 goto mrcommon_fail1;
1548 }
1549 } else {
1550 mpt = NULL;
1551 }
1552
1553 /*
1554 * Allocate the software structure for tracking the memory region (i.e.
1555 * the Hermon Memory Region handle). If we fail here, we must undo
1556 * the protection domain reference count and the previous resource
1557 * allocation.
1558 */
1559 status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
1560 if (status != DDI_SUCCESS) {
1561 status = IBT_INSUFF_RESOURCE;
1562 goto mrcommon_fail2;
1563 }
1564 mr = (hermon_mrhdl_t)rsrc->hr_addr;
1565 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
1566
1567 /*
1568 * Setup and validate the memory region access flags. This means
1569 * translating the IBTF's enable flags into the access flags that
1570 * will be used in later operations.
1571 */
1572 mr->mr_accflag = 0;
1573 if (flags & IBT_MR_ENABLE_WINDOW_BIND)
1574 mr->mr_accflag |= IBT_MR_WINDOW_BIND;
1575 if (flags & IBT_MR_ENABLE_LOCAL_WRITE)
1576 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
1577 if (flags & IBT_MR_ENABLE_REMOTE_READ)
1578 mr->mr_accflag |= IBT_MR_REMOTE_READ;
1579 if (flags & IBT_MR_ENABLE_REMOTE_WRITE)
1580 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
1581 if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
1582 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
1583
1584 /*
1585 * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
1586 * from a certain number of "constrained" bits (the least significant
1587 * bits) and some number of "unconstrained" bits. The constrained
1588 * bits must be set to the index of the entry in the MPT table, but
1589 * the unconstrained bits can be set to any value we wish. Note:
1590 * if no remote access is required, then the RKey value is not filled
1591 * in. Otherwise both Rkey and LKey are given the same value.
1592 */
1593 if (mpt)
1594 mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
1595
1596 /*
1597 * Determine if the memory is from userland and pin the pages
1598 * with umem_lockmemory() if necessary.
1599 * Then, if this is userland memory, allocate an entry in the
1600 * "userland resources database". This will later be added to
1601 * the database (after all further memory registration operations are
1602 * successful). If we fail here, we must undo the reference counts
1603 * and the previous resource allocations.
1604 */
1605 mr_is_umem = (((bind->bi_as != NULL) && (bind->bi_as != &kas)) ? 1 : 0);
1606 if (mr_is_umem) {
1607 umem_len = ptob(btopr(bind->bi_len +
1608 ((uintptr_t)bind->bi_addr & PAGEOFFSET)));
1609 umem_addr = (caddr_t)((uintptr_t)bind->bi_addr & ~PAGEOFFSET);
1610 umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
1611 DDI_UMEMLOCK_LONGTERM);
1612 status = umem_lockmemory(umem_addr, umem_len, umem_flags,
1613 &umem_cookie, &hermon_umem_cbops, NULL);
1614 if (status != 0) {
1615 status = IBT_INSUFF_RESOURCE;
1616 goto mrcommon_fail3;
1617 }
1618
1619 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
1620 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind->bi_buf))
1621
1622 bind->bi_buf = ddi_umem_iosetup(umem_cookie, 0, umem_len,
1623 B_WRITE, 0, 0, NULL, DDI_UMEM_SLEEP);
1624 if (bind->bi_buf == NULL) {
1625 status = IBT_INSUFF_RESOURCE;
1626 goto mrcommon_fail3;
1627 }
1628 bind->bi_type = HERMON_BINDHDL_UBUF;
1629 bind->bi_buf->b_flags |= B_READ;
1630
1631 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind->bi_buf))
1632 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind))
1633
1634 umapdb = hermon_umap_db_alloc(state->hs_instance,
1635 (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
1636 (uint64_t)(uintptr_t)rsrc);
1637 if (umapdb == NULL) {
1638 status = IBT_INSUFF_RESOURCE;
1639 goto mrcommon_fail4;
1640 }
1641 }
1642
1643 /*
1644 * Setup the bindinfo for the mtt bind call
1645 */
1646 bh = &mr->mr_bindinfo;
1647 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bh))
1648 bcopy(bind, bh, sizeof (hermon_bind_info_t));
1649 bh->bi_bypass = bind_type;
1650 status = hermon_mr_mtt_bind(state, bh, bind_dmahdl, &mtt,
1651 &mtt_pgsize_bits, mpt != NULL);
1652 if (status != DDI_SUCCESS) {
1653 /*
1654 * When mtt_bind fails, freerbuf has already been done,
1655 * so make sure not to call it again.
1656 */
1657 bind->bi_type = bh->bi_type;
1658 goto mrcommon_fail5;
1659 }
1660 mr->mr_logmttpgsz = mtt_pgsize_bits;
1661
1662 /*
1663 * Allocate MTT reference count (to track shared memory regions).
1664 * This reference count resource may never be used on the given
1665 * memory region, but if it is ever later registered as "shared"
1666 * memory region then this resource will be necessary. If we fail
1667 * here, we do pretty much the same as above to clean up.
1668 */
1669 status = hermon_rsrc_alloc(state, HERMON_REFCNT, 1, sleep,
1670 &mtt_refcnt);
1671 if (status != DDI_SUCCESS) {
1672 status = IBT_INSUFF_RESOURCE;
1673 goto mrcommon_fail6;
1674 }
1675 mr->mr_mttrefcntp = mtt_refcnt;
1676 swrc_tmp = (hermon_sw_refcnt_t *)mtt_refcnt->hr_addr;
1677 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_tmp))
1678 HERMON_MTT_REFCNT_INIT(swrc_tmp);
1679
1680 mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
1681
1682 /*
1683 * Fill in the MPT entry. This is the final step before passing
1684 * ownership of the MPT entry to the Hermon hardware. We use all of
1685 * the information collected/calculated above to fill in the
1686 * requisite portions of the MPT. Do this ONLY for DMPTs.
1687 */
1688 if (mpt == NULL)
1689 goto no_passown;
1690
1691 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
1692
1693 mpt_entry.status = HERMON_MPT_SW_OWNERSHIP;
1694 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
1695 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
1696 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
1697 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
1698 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
1699 mpt_entry.lr = 1;
1700 mpt_entry.phys_addr = 0;
1701 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
1702
1703 mpt_entry.entity_sz = mr->mr_logmttpgsz;
1704 mpt_entry.mem_key = mr->mr_lkey;
1705 mpt_entry.pd = pd->pd_pdnum;
1706 mpt_entry.rem_acc_en = 0;
1707 mpt_entry.fast_reg_en = 0;
1708 mpt_entry.en_inval = 0;
1709 mpt_entry.lkey = 0;
1710 mpt_entry.win_cnt = 0;
1711
1712 if (bind_override_addr == 0) {
1713 mpt_entry.start_addr = bh->bi_addr;
1714 } else {
1715 bh->bi_addr = bh->bi_addr & ((1 << mr->mr_logmttpgsz) - 1);
1716 mpt_entry.start_addr = bh->bi_addr;
1717 }
1718 mpt_entry.reg_win_len = bh->bi_len;
1719
1720 mpt_entry.mtt_addr_h = mtt_addr >> 32; /* only 8 more bits */
1721 mpt_entry.mtt_addr_l = mtt_addr >> 3; /* only 29 bits */
1722
1723 /*
1724 * Write the MPT entry to hardware. Lastly, we pass ownership of
1725 * the entry to the hardware if needed. Note: in general, this
1726 * operation shouldn't fail. But if it does, we have to undo
1727 * everything we've done above before returning error.
1728 *
1729 * For Hermon, this routine (which is common to the contexts) will only
1730 * set the ownership if needed - the process of passing the context
1731 * itself to HW will take care of setting up the MPT (based on type
1732 * and index).
1733 */
1734
1735 mpt_entry.bnd_qp = 0; /* dMPT for a qp, check for window */
1736 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
1737 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
1738 if (status != HERMON_CMD_SUCCESS) {
1739 cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
1740 status);
1741 if (status == HERMON_CMD_INVALID_STATUS) {
1742 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
1743 }
1744 status = ibc_get_ci_failure(0);
1745 goto mrcommon_fail7;
1746 }
1747 if (hermon_rdma_debug & 0x4)
1748 IBTF_DPRINTF_L2("mr", " reg: mr %p key %x",
1749 mr, hermon_mr_key_swap(mr->mr_rkey));
1750 no_passown:
1751
1752 /*
1753 * Fill in the rest of the Hermon Memory Region handle. Having
1754 * successfully transferred ownership of the MPT, we can update the
1755 * following fields for use in further operations on the MR.
1756 */
1757 mr->mr_mttaddr = mtt_addr;
1758
1759 mr->mr_log2_pgsz = (mr->mr_logmttpgsz - HERMON_PAGESHIFT);
1760 mr->mr_mptrsrcp = mpt;
1761 mr->mr_mttrsrcp = mtt;
1762 mr->mr_pdhdl = pd;
1763 mr->mr_rsrcp = rsrc;
1764 mr->mr_is_umem = mr_is_umem;
1765 mr->mr_is_fmr = 0;
1766 mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
1767 mr->mr_umem_cbfunc = NULL;
1768 mr->mr_umem_cbarg1 = NULL;
1769 mr->mr_umem_cbarg2 = NULL;
1770 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
1771 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
1772 mr->mr_mpt_type = mpt_type;
1773
1774 /*
1775 * If this is userland memory, then we need to insert the previously
1776 * allocated entry into the "userland resources database". This will
1777 * allow for later coordination between the hermon_umap_umemlock_cb()
1778 * callback and hermon_mr_deregister().
1779 */
1780 if (mr_is_umem) {
1781 hermon_umap_db_add(umapdb);
1782 }
1783
1784 *mrhdl = mr;
1785
1786 return (DDI_SUCCESS);
1787
1788 /*
1789 * The following is cleanup for all possible failure cases in this routine
1790 */
1791 mrcommon_fail7:
1792 hermon_rsrc_free(state, &mtt_refcnt);
1793 mrcommon_fail6:
1794 hermon_mr_mem_unbind(state, bh);
1795 bind->bi_type = bh->bi_type;
1796 mrcommon_fail5:
1797 if (mr_is_umem) {
1798 hermon_umap_db_free(umapdb);
1799 }
1800 mrcommon_fail4:
1801 if (mr_is_umem) {
1802 /*
1803 * Free up the memory ddi_umem_iosetup() allocates
1804 * internally.
1805 */
1806 if (bind->bi_type == HERMON_BINDHDL_UBUF) {
1807 freerbuf(bind->bi_buf);
1808 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
1809 bind->bi_type = HERMON_BINDHDL_NONE;
1810 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind))
1811 }
1812 ddi_umem_unlock(umem_cookie);
1813 }
1814 mrcommon_fail3:
1815 hermon_rsrc_free(state, &rsrc);
1816 mrcommon_fail2:
1817 if (mpt != NULL)
1818 hermon_rsrc_free(state, &mpt);
1819 mrcommon_fail1:
1820 hermon_pd_refcnt_dec(pd);
1821 mrcommon_fail:
1822 return (status);
1823 }
1824
1825 /*
1826 * hermon_dma_mr_register()
1827 * Context: Can be called from base context.
1828 */
1829 int
1830 hermon_dma_mr_register(hermon_state_t *state, hermon_pdhdl_t pd,
1831 ibt_dmr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl)
1832 {
1833 hermon_rsrc_t *mpt, *rsrc;
1834 hermon_hw_dmpt_t mpt_entry;
1835 hermon_mrhdl_t mr;
1836 ibt_mr_flags_t flags;
1837 uint_t sleep;
1838 int status;
1839
1840 /* Extract the flags field */
1841 flags = mr_attr->dmr_flags;
1842
1843 /*
1844 * Check the sleep flag. Ensure that it is consistent with the
1845 * current thread context (i.e. if we are currently in the interrupt
1846 * context, then we shouldn't be attempting to sleep).
1847 */
1848 sleep = (flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP: HERMON_SLEEP;
1849 if ((sleep == HERMON_SLEEP) &&
1850 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
1851 status = IBT_INVALID_PARAM;
1852 goto mrcommon_fail;
1853 }
1854
1855 /* Increment the reference count on the protection domain (PD) */
1856 hermon_pd_refcnt_inc(pd);
1857
1858 /*
1859 * Allocate an MPT entry. This will be filled in with all the
1860 * necessary parameters to define the memory region. And then
1861 * ownership will be passed to the hardware in the final step
1862 * below. If we fail here, we must undo the protection domain
1863 * reference count.
1864 */
1865 status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
1866 if (status != DDI_SUCCESS) {
1867 status = IBT_INSUFF_RESOURCE;
1868 goto mrcommon_fail1;
1869 }
1870
1871 /*
1872 * Allocate the software structure for tracking the memory region (i.e.
1873 * the Hermon Memory Region handle). If we fail here, we must undo
1874 * the protection domain reference count and the previous resource
1875 * allocation.
1876 */
1877 status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
1878 if (status != DDI_SUCCESS) {
1879 status = IBT_INSUFF_RESOURCE;
1880 goto mrcommon_fail2;
1881 }
1882 mr = (hermon_mrhdl_t)rsrc->hr_addr;
1883 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
1884 bzero(mr, sizeof (*mr));
1885
1886 /*
1887 * Setup and validate the memory region access flags. This means
1888 * translating the IBTF's enable flags into the access flags that
1889 * will be used in later operations.
1890 */
1891 mr->mr_accflag = 0;
1892 if (flags & IBT_MR_ENABLE_WINDOW_BIND)
1893 mr->mr_accflag |= IBT_MR_WINDOW_BIND;
1894 if (flags & IBT_MR_ENABLE_LOCAL_WRITE)
1895 mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
1896 if (flags & IBT_MR_ENABLE_REMOTE_READ)
1897 mr->mr_accflag |= IBT_MR_REMOTE_READ;
1898 if (flags & IBT_MR_ENABLE_REMOTE_WRITE)
1899 mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
1900 if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
1901 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
1902
1903 /*
1904 * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
1905 * from a certain number of "constrained" bits (the least significant
1906 * bits) and some number of "unconstrained" bits. The constrained
1907 * bits must be set to the index of the entry in the MPT table, but
1908 * the unconstrained bits can be set to any value we wish. Note:
1909 * if no remote access is required, then the RKey value is not filled
1910 * in. Otherwise both Rkey and LKey are given the same value.
1911 */
1912 if (mpt)
1913 mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
1914
1915 /*
1916 * Fill in the MPT entry. This is the final step before passing
1917 * ownership of the MPT entry to the Hermon hardware. We use all of
1918 * the information collected/calculated above to fill in the
1919 * requisite portions of the MPT. Do this ONLY for DMPTs.
1920 */
1921 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
1922
1923 mpt_entry.status = HERMON_MPT_SW_OWNERSHIP;
1924 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
1925 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
1926 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
1927 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
1928 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
1929 mpt_entry.lr = 1;
1930 mpt_entry.phys_addr = 1; /* critical bit for this */
1931 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
1932
1933 mpt_entry.entity_sz = mr->mr_logmttpgsz;
1934 mpt_entry.mem_key = mr->mr_lkey;
1935 mpt_entry.pd = pd->pd_pdnum;
1936 mpt_entry.rem_acc_en = 0;
1937 mpt_entry.fast_reg_en = 0;
1938 mpt_entry.en_inval = 0;
1939 mpt_entry.lkey = 0;
1940 mpt_entry.win_cnt = 0;
1941
1942 mpt_entry.start_addr = mr_attr->dmr_paddr;
1943 mpt_entry.reg_win_len = mr_attr->dmr_len;
1944 if (mr_attr->dmr_len == 0)
1945 mpt_entry.len_b64 = 1; /* needed for 2^^64 length */
1946
1947 mpt_entry.mtt_addr_h = 0;
1948 mpt_entry.mtt_addr_l = 0;
1949
1950 /*
1951 * Write the MPT entry to hardware. Lastly, we pass ownership of
1952 * the entry to the hardware if needed. Note: in general, this
1953 * operation shouldn't fail. But if it does, we have to undo
1954 * everything we've done above before returning error.
1955 *
1956 * For Hermon, this routine (which is common to the contexts) will only
1957 * set the ownership if needed - the process of passing the context
1958 * itself to HW will take care of setting up the MPT (based on type
1959 * and index).
1960 */
1961
1962 mpt_entry.bnd_qp = 0; /* dMPT for a qp, check for window */
1963 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
1964 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
1965 if (status != HERMON_CMD_SUCCESS) {
1966 cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
1967 status);
1968 if (status == HERMON_CMD_INVALID_STATUS) {
1969 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
1970 }
1971 status = ibc_get_ci_failure(0);
1972 goto mrcommon_fail7;
1973 }
1974
1975 /*
1976 * Fill in the rest of the Hermon Memory Region handle. Having
1977 * successfully transferred ownership of the MPT, we can update the
1978 * following fields for use in further operations on the MR.
1979 */
1980 mr->mr_mttaddr = 0;
1981
1982 mr->mr_log2_pgsz = 0;
1983 mr->mr_mptrsrcp = mpt;
1984 mr->mr_mttrsrcp = NULL;
1985 mr->mr_pdhdl = pd;
1986 mr->mr_rsrcp = rsrc;
1987 mr->mr_is_umem = 0;
1988 mr->mr_is_fmr = 0;
1989 mr->mr_umemcookie = NULL;
1990 mr->mr_umem_cbfunc = NULL;
1991 mr->mr_umem_cbarg1 = NULL;
1992 mr->mr_umem_cbarg2 = NULL;
1993 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
1994 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
1995 mr->mr_mpt_type = HERMON_MPT_DMPT;
1996
1997 *mrhdl = mr;
1998
1999 return (DDI_SUCCESS);
2000
2001 /*
2002 * The following is cleanup for all possible failure cases in this routine
2003 */
2004 mrcommon_fail7:
2005 hermon_rsrc_free(state, &rsrc);
2006 mrcommon_fail2:
2007 hermon_rsrc_free(state, &mpt);
2008 mrcommon_fail1:
2009 hermon_pd_refcnt_dec(pd);
2010 mrcommon_fail:
2011 return (status);
2012 }
2013
2014 /*
2015 * hermon_mr_alloc_lkey()
2016 * Context: Can be called from base context.
2017 */
2018 int
2019 hermon_mr_alloc_lkey(hermon_state_t *state, hermon_pdhdl_t pd,
2020 ibt_lkey_flags_t flags, uint_t nummtt, hermon_mrhdl_t *mrhdl)
2021 {
2022 hermon_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt;
2023 hermon_sw_refcnt_t *swrc_tmp;
2024 hermon_hw_dmpt_t mpt_entry;
2025 hermon_mrhdl_t mr;
2026 uint64_t mtt_addr;
2027 uint_t sleep;
2028 int status;
2029
2030 /* Increment the reference count on the protection domain (PD) */
2031 hermon_pd_refcnt_inc(pd);
2032
2033 sleep = (flags & IBT_KEY_NOSLEEP) ? HERMON_NOSLEEP: HERMON_SLEEP;
2034
2035 /*
2036 * Allocate an MPT entry. This will be filled in with "some" of the
2037 * necessary parameters to define the memory region. And then
2038 * ownership will be passed to the hardware in the final step
2039 * below. If we fail here, we must undo the protection domain
2040 * reference count.
2041 *
2042 * The MTTs will get filled in when the FRWR is processed.
2043 */
2044 status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
2045 if (status != DDI_SUCCESS) {
2046 status = IBT_INSUFF_RESOURCE;
2047 goto alloclkey_fail1;
2048 }
2049
2050 /*
2051 * Allocate the software structure for tracking the memory region (i.e.
2052 * the Hermon Memory Region handle). If we fail here, we must undo
2053 * the protection domain reference count and the previous resource
2054 * allocation.
2055 */
2056 status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
2057 if (status != DDI_SUCCESS) {
2058 status = IBT_INSUFF_RESOURCE;
2059 goto alloclkey_fail2;
2060 }
2061 mr = (hermon_mrhdl_t)rsrc->hr_addr;
2062 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2063 bzero(mr, sizeof (*mr));
2064 mr->mr_bindinfo.bi_type = HERMON_BINDHDL_LKEY;
2065
2066 mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
2067
2068 status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt, sleep, &mtt);
2069 if (status != DDI_SUCCESS) {
2070 status = IBT_INSUFF_RESOURCE;
2071 goto alloclkey_fail3;
2072 }
2073 mr->mr_logmttpgsz = PAGESHIFT;
2074
2075 /*
2076 * Allocate MTT reference count (to track shared memory regions).
2077 * This reference count resource may never be used on the given
2078 * memory region, but if it is ever later registered as "shared"
2079 * memory region then this resource will be necessary. If we fail
2080 * here, we do pretty much the same as above to clean up.
2081 */
2082 status = hermon_rsrc_alloc(state, HERMON_REFCNT, 1, sleep,
2083 &mtt_refcnt);
2084 if (status != DDI_SUCCESS) {
2085 status = IBT_INSUFF_RESOURCE;
2086 goto alloclkey_fail4;
2087 }
2088 mr->mr_mttrefcntp = mtt_refcnt;
2089 swrc_tmp = (hermon_sw_refcnt_t *)mtt_refcnt->hr_addr;
2090 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_tmp))
2091 HERMON_MTT_REFCNT_INIT(swrc_tmp);
2092
2093 mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
2094
2095 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
2096 mpt_entry.status = HERMON_MPT_FREE;
2097 mpt_entry.lw = 1;
2098 mpt_entry.lr = 1;
2099 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
2100 mpt_entry.entity_sz = mr->mr_logmttpgsz;
2101 mpt_entry.mem_key = mr->mr_lkey;
2102 mpt_entry.pd = pd->pd_pdnum;
2103 mpt_entry.fast_reg_en = 1;
2104 mpt_entry.rem_acc_en = 1;
2105 mpt_entry.en_inval = 1;
2106 if (flags & IBT_KEY_REMOTE) {
2107 mpt_entry.ren_inval = 1;
2108 }
2109 mpt_entry.mtt_size = nummtt;
2110 mpt_entry.mtt_addr_h = mtt_addr >> 32; /* only 8 more bits */
2111 mpt_entry.mtt_addr_l = mtt_addr >> 3; /* only 29 bits */
2112
2113 /*
2114 * Write the MPT entry to hardware. Lastly, we pass ownership of
2115 * the entry to the hardware if needed. Note: in general, this
2116 * operation shouldn't fail. But if it does, we have to undo
2117 * everything we've done above before returning error.
2118 *
2119 * For Hermon, this routine (which is common to the contexts) will only
2120 * set the ownership if needed - the process of passing the context
2121 * itself to HW will take care of setting up the MPT (based on type
2122 * and index).
2123 */
2124 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
2125 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
2126 if (status != HERMON_CMD_SUCCESS) {
2127 cmn_err(CE_CONT, "Hermon: alloc_lkey: SW2HW_MPT command "
2128 "failed: %08x\n", status);
2129 if (status == HERMON_CMD_INVALID_STATUS) {
2130 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2131 }
2132 status = ibc_get_ci_failure(0);
2133 goto alloclkey_fail5;
2134 }
2135
2136 /*
2137 * Fill in the rest of the Hermon Memory Region handle. Having
2138 * successfully transferred ownership of the MPT, we can update the
2139 * following fields for use in further operations on the MR.
2140 */
2141 mr->mr_accflag = IBT_MR_LOCAL_WRITE;
2142 mr->mr_mttaddr = mtt_addr;
2143 mr->mr_log2_pgsz = (mr->mr_logmttpgsz - HERMON_PAGESHIFT);
2144 mr->mr_mptrsrcp = mpt;
2145 mr->mr_mttrsrcp = mtt;
2146 mr->mr_pdhdl = pd;
2147 mr->mr_rsrcp = rsrc;
2148 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
2149 mr->mr_rkey = mr->mr_lkey;
2150 mr->mr_mpt_type = HERMON_MPT_DMPT;
2151
2152 *mrhdl = mr;
2153 return (DDI_SUCCESS);
2154
2155 alloclkey_fail5:
2156 hermon_rsrc_free(state, &mtt_refcnt);
2157 alloclkey_fail4:
2158 hermon_rsrc_free(state, &mtt);
2159 alloclkey_fail3:
2160 hermon_rsrc_free(state, &rsrc);
2161 alloclkey_fail2:
2162 hermon_rsrc_free(state, &mpt);
2163 alloclkey_fail1:
2164 hermon_pd_refcnt_dec(pd);
2165 return (status);
2166 }
2167
2168 /*
2169 * hermon_mr_fexch_mpt_init()
2170 * Context: Can be called from base context.
2171 *
2172 * This is the same as alloc_lkey, but not returning an mrhdl.
2173 */
2174 int
2175 hermon_mr_fexch_mpt_init(hermon_state_t *state, hermon_pdhdl_t pd,
2176 uint32_t mpt_indx, uint_t nummtt, uint64_t mtt_addr, uint_t sleep)
2177 {
2178 hermon_hw_dmpt_t mpt_entry;
2179 int status;
2180
2181 /*
2182 * The MTTs will get filled in when the FRWR is processed.
2183 */
2184
2185 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
2186 mpt_entry.status = HERMON_MPT_FREE;
2187 mpt_entry.lw = 1;
2188 mpt_entry.lr = 1;
2189 mpt_entry.rw = 1;
2190 mpt_entry.rr = 1;
2191 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
2192 mpt_entry.entity_sz = PAGESHIFT;
2193 mpt_entry.mem_key = mpt_indx;
2194 mpt_entry.pd = pd->pd_pdnum;
2195 mpt_entry.fast_reg_en = 1;
2196 mpt_entry.rem_acc_en = 1;
2197 mpt_entry.en_inval = 1;
2198 mpt_entry.ren_inval = 1;
2199 mpt_entry.mtt_size = nummtt;
2200 mpt_entry.mtt_addr_h = mtt_addr >> 32; /* only 8 more bits */
2201 mpt_entry.mtt_addr_l = mtt_addr >> 3; /* only 29 bits */
2202
2203 /*
2204 * Write the MPT entry to hardware. Lastly, we pass ownership of
2205 * the entry to the hardware if needed. Note: in general, this
2206 * operation shouldn't fail. But if it does, we have to undo
2207 * everything we've done above before returning error.
2208 *
2209 * For Hermon, this routine (which is common to the contexts) will only
2210 * set the ownership if needed - the process of passing the context
2211 * itself to HW will take care of setting up the MPT (based on type
2212 * and index).
2213 */
2214 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
2215 sizeof (hermon_hw_dmpt_t), mpt_indx, sleep);
2216 if (status != HERMON_CMD_SUCCESS) {
2217 cmn_err(CE_CONT, "Hermon: fexch_mpt_init: SW2HW_MPT command "
2218 "failed: %08x\n", status);
2219 if (status == HERMON_CMD_INVALID_STATUS) {
2220 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2221 }
2222 status = ibc_get_ci_failure(0);
2223 return (status);
2224 }
2225 /* Increment the reference count on the protection domain (PD) */
2226 hermon_pd_refcnt_inc(pd);
2227
2228 return (DDI_SUCCESS);
2229 }
2230
2231 /*
2232 * hermon_mr_fexch_mpt_fini()
2233 * Context: Can be called from base context.
2234 *
2235 * This is the same as deregister_mr, without an mrhdl.
2236 */
2237 int
2238 hermon_mr_fexch_mpt_fini(hermon_state_t *state, hermon_pdhdl_t pd,
2239 uint32_t mpt_indx, uint_t sleep)
2240 {
2241 int status;
2242
2243 status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT,
2244 NULL, 0, mpt_indx, sleep);
2245 if (status != DDI_SUCCESS) {
2246 cmn_err(CE_CONT, "Hermon: fexch_mpt_fini: HW2SW_MPT command "
2247 "failed: %08x\n", status);
2248 if (status == HERMON_CMD_INVALID_STATUS) {
2249 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2250 }
2251 status = ibc_get_ci_failure(0);
2252 return (status);
2253 }
2254
2255 /* Decrement the reference count on the protection domain (PD) */
2256 hermon_pd_refcnt_dec(pd);
2257
2258 return (DDI_SUCCESS);
2259 }
2260
2261 /*
2262 * hermon_mr_mtt_bind()
2263 * Context: Can be called from interrupt or base context.
2264 */
2265 int
2266 hermon_mr_mtt_bind(hermon_state_t *state, hermon_bind_info_t *bind,
2267 ddi_dma_handle_t bind_dmahdl, hermon_rsrc_t **mtt, uint_t *mtt_pgsize_bits,
2268 uint_t is_buffer)
2269 {
2270 uint64_t nummtt;
2271 uint_t sleep;
2272 int status;
2273
2274 /*
2275 * Check the sleep flag. Ensure that it is consistent with the
2276 * current thread context (i.e. if we are currently in the interrupt
2277 * context, then we shouldn't be attempting to sleep).
2278 */
2279 sleep = (bind->bi_flags & IBT_MR_NOSLEEP) ?
2280 HERMON_NOSLEEP : HERMON_SLEEP;
2281 if ((sleep == HERMON_SLEEP) &&
2282 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
2283 status = IBT_INVALID_PARAM;
2284 goto mrmttbind_fail;
2285 }
2286
2287 /*
2288 * Bind the memory and determine the mapped addresses. This is
2289 * the first of two routines that do all the "heavy lifting" for
2290 * the Hermon memory registration routines. The hermon_mr_mem_bind()
2291 * routine takes the "bind" struct with all its fields filled
2292 * in and returns a list of DMA cookies (for the PCI mapped addresses
2293 * corresponding to the specified address region) which are used by
2294 * the hermon_mr_fast_mtt_write() routine below. If we fail here, we
2295 * must undo all the previous resource allocation (and PD reference
2296 * count).
2297 */
2298 status = hermon_mr_mem_bind(state, bind, bind_dmahdl, sleep, is_buffer);
2299 if (status != DDI_SUCCESS) {
2300 status = IBT_INSUFF_RESOURCE;
2301 goto mrmttbind_fail;
2302 }
2303
2304 /*
2305 * Determine number of pages spanned. This routine uses the
2306 * information in the "bind" struct to determine the required
2307 * number of MTT entries needed (and returns the suggested page size -
2308 * as a "power-of-2" - for each MTT entry).
2309 */
2310 nummtt = hermon_mr_nummtt_needed(state, bind, mtt_pgsize_bits);
2311
2312 /*
2313 * Allocate the MTT entries. Use the calculations performed above to
2314 * allocate the required number of MTT entries. If we fail here, we
2315 * must not only undo all the previous resource allocation (and PD
2316 * reference count), but we must also unbind the memory.
2317 */
2318 status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt, sleep, mtt);
2319 if (status != DDI_SUCCESS) {
2320 status = IBT_INSUFF_RESOURCE;
2321 goto mrmttbind_fail2;
2322 }
2323
2324 /*
2325 * Write the mapped addresses into the MTT entries. This is part two
2326 * of the "heavy lifting" routines that we talked about above. Note:
2327 * we pass the suggested page size from the earlier operation here.
2328 * And if we fail here, we again do pretty much the same huge clean up.
2329 */
2330 status = hermon_mr_fast_mtt_write(state, *mtt, bind, *mtt_pgsize_bits);
2331 if (status != DDI_SUCCESS) {
2332 /*
2333 * hermon_mr_fast_mtt_write() returns DDI_FAILURE
2334 * only if it detects a HW error during DMA.
2335 */
2336 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2337 status = ibc_get_ci_failure(0);
2338 goto mrmttbind_fail3;
2339 }
2340 return (DDI_SUCCESS);
2341
2342 /*
2343 * The following is cleanup for all possible failure cases in this routine
2344 */
2345 mrmttbind_fail3:
2346 hermon_rsrc_free(state, mtt);
2347 mrmttbind_fail2:
2348 hermon_mr_mem_unbind(state, bind);
2349 mrmttbind_fail:
2350 return (status);
2351 }
2352
2353
2354 /*
2355 * hermon_mr_mtt_unbind()
2356 * Context: Can be called from interrupt or base context.
2357 */
2358 int
2359 hermon_mr_mtt_unbind(hermon_state_t *state, hermon_bind_info_t *bind,
2360 hermon_rsrc_t *mtt)
2361 {
2362 /*
2363 * Free up the MTT entries and unbind the memory. Here, as above, we
2364 * attempt to free these resources only if it is appropriate to do so.
2365 */
2366 hermon_mr_mem_unbind(state, bind);
2367 hermon_rsrc_free(state, &mtt);
2368
2369 return (DDI_SUCCESS);
2370 }
2371
2372
2373 /*
2374 * hermon_mr_common_rereg()
2375 * Context: Can be called from interrupt or base context.
2376 */
2377 static int
2378 hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr,
2379 hermon_pdhdl_t pd, hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl_new,
2380 hermon_mr_options_t *op)
2381 {
2382 hermon_rsrc_t *mpt;
2383 ibt_mr_attr_flags_t acc_flags_to_use;
2384 ibt_mr_flags_t flags;
2385 hermon_pdhdl_t pd_to_use;
2386 hermon_hw_dmpt_t mpt_entry;
2387 uint64_t mtt_addr_to_use, vaddr_to_use, len_to_use;
2388 uint_t sleep, dereg_level;
2389 int status;
2390
2391 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
2392
2393 /*
2394 * Check here to see if the memory region corresponds to a userland
2395 * mapping. Reregistration of userland memory regions is not
2396 * currently supported. Return failure.
2397 */
2398 if (mr->mr_is_umem) {
2399 status = IBT_MR_HDL_INVALID;
2400 goto mrrereg_fail;
2401 }
2402
2403 mutex_enter(&mr->mr_lock);
2404
2405 /* Pull MPT resource pointer from the Hermon Memory Region handle */
2406 mpt = mr->mr_mptrsrcp;
2407
2408 /* Extract the flags field from the hermon_bind_info_t */
2409 flags = bind->bi_flags;
2410
2411 /*
2412 * Check the sleep flag. Ensure that it is consistent with the
2413 * current thread context (i.e. if we are currently in the interrupt
2414 * context, then we shouldn't be attempting to sleep).
2415 */
2416 sleep = (flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP: HERMON_SLEEP;
2417 if ((sleep == HERMON_SLEEP) &&
2418 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
2419 mutex_exit(&mr->mr_lock);
2420 status = IBT_INVALID_PARAM;
2421 goto mrrereg_fail;
2422 }
2423
2424 /*
2425 * First step is to temporarily invalidate the MPT entry. This
2426 * regains ownership from the hardware, and gives us the opportunity
2427 * to modify the entry. Note: The HW2SW_MPT command returns the
2428 * current MPT entry contents. These are saved away here because
2429 * they will be reused in a later step below. If the region has
2430 * bound memory windows that we fail returning an "in use" error code.
2431 * Otherwise, this is an unexpected error and we deregister the
2432 * memory region and return error.
2433 *
2434 * We use HERMON_CMD_NOSLEEP_SPIN here always because we must protect
2435 * against holding the lock around this rereg call in all contexts.
2436 */
2437 status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT, &mpt_entry,
2438 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, HERMON_CMD_NOSLEEP_SPIN);
2439 if (status != HERMON_CMD_SUCCESS) {
2440 mutex_exit(&mr->mr_lock);
2441 if (status == HERMON_CMD_REG_BOUND) {
2442 return (IBT_MR_IN_USE);
2443 } else {
2444 cmn_err(CE_CONT, "Hermon: HW2SW_MPT command failed: "
2445 "%08x\n", status);
2446 if (status == HERMON_CMD_INVALID_STATUS) {
2447 hermon_fm_ereport(state, HCA_SYS_ERR,
2448 HCA_ERR_SRV_LOST);
2449 }
2450 /*
2451 * Call deregister and ensure that all current
2452 * resources get freed up
2453 */
2454 if (hermon_mr_deregister(state, &mr,
2455 HERMON_MR_DEREG_ALL, sleep) != DDI_SUCCESS) {
2456 HERMON_WARNING(state, "failed to deregister "
2457 "memory region");
2458 }
2459 return (ibc_get_ci_failure(0));
2460 }
2461 }
2462
2463 /*
2464 * If we're changing the protection domain, then validate the new one
2465 */
2466 if (flags & IBT_MR_CHANGE_PD) {
2467
2468 /* Check for valid PD handle pointer */
2469 if (pd == NULL) {
2470 mutex_exit(&mr->mr_lock);
2471 /*
2472 * Call deregister and ensure that all current
2473 * resources get properly freed up. Unnecessary
2474 * here to attempt to regain software ownership
2475 * of the MPT entry as that has already been
2476 * done above.
2477 */
2478 if (hermon_mr_deregister(state, &mr,
2479 HERMON_MR_DEREG_NO_HW2SW_MPT, sleep) !=
2480 DDI_SUCCESS) {
2481 HERMON_WARNING(state, "failed to deregister "
2482 "memory region");
2483 }
2484 status = IBT_PD_HDL_INVALID;
2485 goto mrrereg_fail;
2486 }
2487
2488 /* Use the new PD handle in all operations below */
2489 pd_to_use = pd;
2490
2491 } else {
2492 /* Use the current PD handle in all operations below */
2493 pd_to_use = mr->mr_pdhdl;
2494 }
2495
2496 /*
2497 * If we're changing access permissions, then validate the new ones
2498 */
2499 if (flags & IBT_MR_CHANGE_ACCESS) {
2500 /*
2501 * Validate the access flags. Both remote write and remote
2502 * atomic require the local write flag to be set
2503 */
2504 if (((flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
2505 (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
2506 !(flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
2507 mutex_exit(&mr->mr_lock);
2508 /*
2509 * Call deregister and ensure that all current
2510 * resources get properly freed up. Unnecessary
2511 * here to attempt to regain software ownership
2512 * of the MPT entry as that has already been
2513 * done above.
2514 */
2515 if (hermon_mr_deregister(state, &mr,
2516 HERMON_MR_DEREG_NO_HW2SW_MPT, sleep) !=
2517 DDI_SUCCESS) {
2518 HERMON_WARNING(state, "failed to deregister "
2519 "memory region");
2520 }
2521 status = IBT_MR_ACCESS_REQ_INVALID;
2522 goto mrrereg_fail;
2523 }
2524
2525 /*
2526 * Setup and validate the memory region access flags. This
2527 * means translating the IBTF's enable flags into the access
2528 * flags that will be used in later operations.
2529 */
2530 acc_flags_to_use = 0;
2531 if (flags & IBT_MR_ENABLE_WINDOW_BIND)
2532 acc_flags_to_use |= IBT_MR_WINDOW_BIND;
2533 if (flags & IBT_MR_ENABLE_LOCAL_WRITE)
2534 acc_flags_to_use |= IBT_MR_LOCAL_WRITE;
2535 if (flags & IBT_MR_ENABLE_REMOTE_READ)
2536 acc_flags_to_use |= IBT_MR_REMOTE_READ;
2537 if (flags & IBT_MR_ENABLE_REMOTE_WRITE)
2538 acc_flags_to_use |= IBT_MR_REMOTE_WRITE;
2539 if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
2540 acc_flags_to_use |= IBT_MR_REMOTE_ATOMIC;
2541
2542 } else {
2543 acc_flags_to_use = mr->mr_accflag;
2544 }
2545
2546 /*
2547 * If we're modifying the translation, then figure out whether
2548 * we can reuse the current MTT resources. This means calling
2549 * hermon_mr_rereg_xlat_helper() which does most of the heavy lifting
2550 * for the reregistration. If the current memory region contains
2551 * sufficient MTT entries for the new regions, then it will be
2552 * reused and filled in. Otherwise, new entries will be allocated,
2553 * the old ones will be freed, and the new entries will be filled
2554 * in. Note: If we're not modifying the translation, then we
2555 * should already have all the information we need to update the MPT.
2556 * Also note: If hermon_mr_rereg_xlat_helper() fails, it will return
2557 * a "dereg_level" which is the level of cleanup that needs to be
2558 * passed to hermon_mr_deregister() to finish the cleanup.
2559 */
2560 if (flags & IBT_MR_CHANGE_TRANSLATION) {
2561 status = hermon_mr_rereg_xlat_helper(state, mr, bind, op,
2562 &mtt_addr_to_use, sleep, &dereg_level);
2563 if (status != DDI_SUCCESS) {
2564 mutex_exit(&mr->mr_lock);
2565 /*
2566 * Call deregister and ensure that all resources get
2567 * properly freed up.
2568 */
2569 if (hermon_mr_deregister(state, &mr, dereg_level,
2570 sleep) != DDI_SUCCESS) {
2571 HERMON_WARNING(state, "failed to deregister "
2572 "memory region");
2573 }
2574 goto mrrereg_fail;
2575 }
2576 vaddr_to_use = mr->mr_bindinfo.bi_addr;
2577 len_to_use = mr->mr_bindinfo.bi_len;
2578 } else {
2579 mtt_addr_to_use = mr->mr_mttaddr;
2580 vaddr_to_use = mr->mr_bindinfo.bi_addr;
2581 len_to_use = mr->mr_bindinfo.bi_len;
2582 }
2583
2584 /*
2585 * Calculate new keys (Lkey, Rkey) from MPT index. Just like they were
2586 * when the region was first registered, each key is formed from
2587 * "constrained" bits and "unconstrained" bits. Note: If no remote
2588 * access is required, then the RKey value is not filled in. Otherwise
2589 * both Rkey and LKey are given the same value.
2590 */
2591 mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
2592 if ((acc_flags_to_use & IBT_MR_REMOTE_READ) ||
2593 (acc_flags_to_use & IBT_MR_REMOTE_WRITE) ||
2594 (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC)) {
2595 mr->mr_rkey = mr->mr_lkey;
2596 } else
2597 mr->mr_rkey = 0;
2598
2599 /*
2600 * Fill in the MPT entry. This is the final step before passing
2601 * ownership of the MPT entry to the Hermon hardware. We use all of
2602 * the information collected/calculated above to fill in the
2603 * requisite portions of the MPT.
2604 */
2605 bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
2606
2607 mpt_entry.status = HERMON_MPT_SW_OWNERSHIP;
2608 mpt_entry.en_bind = (acc_flags_to_use & IBT_MR_WINDOW_BIND) ? 1 : 0;
2609 mpt_entry.atomic = (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
2610 mpt_entry.rw = (acc_flags_to_use & IBT_MR_REMOTE_WRITE) ? 1 : 0;
2611 mpt_entry.rr = (acc_flags_to_use & IBT_MR_REMOTE_READ) ? 1 : 0;
2612 mpt_entry.lw = (acc_flags_to_use & IBT_MR_LOCAL_WRITE) ? 1 : 0;
2613 mpt_entry.lr = 1;
2614 mpt_entry.phys_addr = 0;
2615 mpt_entry.reg_win = HERMON_MPT_IS_REGION;
2616
2617 mpt_entry.entity_sz = mr->mr_logmttpgsz;
2618 mpt_entry.mem_key = mr->mr_lkey;
2619 mpt_entry.pd = pd_to_use->pd_pdnum;
2620
2621 mpt_entry.start_addr = vaddr_to_use;
2622 mpt_entry.reg_win_len = len_to_use;
2623 mpt_entry.mtt_addr_h = mtt_addr_to_use >> 32;
2624 mpt_entry.mtt_addr_l = mtt_addr_to_use >> 3;
2625
2626 /*
2627 * Write the updated MPT entry to hardware
2628 *
2629 * We use HERMON_CMD_NOSLEEP_SPIN here always because we must protect
2630 * against holding the lock around this rereg call in all contexts.
2631 */
2632 status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
2633 sizeof (hermon_hw_dmpt_t), mpt->hr_indx, HERMON_CMD_NOSLEEP_SPIN);
2634 if (status != HERMON_CMD_SUCCESS) {
2635 mutex_exit(&mr->mr_lock);
2636 cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
2637 status);
2638 if (status == HERMON_CMD_INVALID_STATUS) {
2639 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2640 }
2641 /*
2642 * Call deregister and ensure that all current resources get
2643 * properly freed up. Unnecessary here to attempt to regain
2644 * software ownership of the MPT entry as that has already
2645 * been done above.
2646 */
2647 if (hermon_mr_deregister(state, &mr,
2648 HERMON_MR_DEREG_NO_HW2SW_MPT, sleep) != DDI_SUCCESS) {
2649 HERMON_WARNING(state, "failed to deregister memory "
2650 "region");
2651 }
2652 return (ibc_get_ci_failure(0));
2653 }
2654
2655 /*
2656 * If we're changing PD, then update their reference counts now.
2657 * This means decrementing the reference count on the old PD and
2658 * incrementing the reference count on the new PD.
2659 */
2660 if (flags & IBT_MR_CHANGE_PD) {
2661 hermon_pd_refcnt_dec(mr->mr_pdhdl);
2662 hermon_pd_refcnt_inc(pd);
2663 }
2664
2665 /*
2666 * Update the contents of the Hermon Memory Region handle to reflect
2667 * what has been changed.
2668 */
2669 mr->mr_pdhdl = pd_to_use;
2670 mr->mr_accflag = acc_flags_to_use;
2671 mr->mr_is_umem = 0;
2672 mr->mr_is_fmr = 0;
2673 mr->mr_umemcookie = NULL;
2674 mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey);
2675 mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey);
2676
2677 /* New MR handle is same as the old */
2678 *mrhdl_new = mr;
2679 mutex_exit(&mr->mr_lock);
2680
2681 return (DDI_SUCCESS);
2682
2683 mrrereg_fail:
2684 return (status);
2685 }
2686
2687
2688 /*
2689 * hermon_mr_rereg_xlat_helper
2690 * Context: Can be called from interrupt or base context.
2691 * Note: This routine expects the "mr_lock" to be held when it
2692 * is called. Upon returning failure, this routine passes information
2693 * about what "dereg_level" should be passed to hermon_mr_deregister().
2694 */
2695 static int
2696 hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr,
2697 hermon_bind_info_t *bind, hermon_mr_options_t *op, uint64_t *mtt_addr,
2698 uint_t sleep, uint_t *dereg_level)
2699 {
2700 hermon_rsrc_t *mtt, *mtt_refcnt;
2701 hermon_sw_refcnt_t *swrc_old, *swrc_new;
2702 ddi_dma_handle_t dmahdl;
2703 uint64_t nummtt_needed, nummtt_in_currrsrc, max_sz;
2704 uint_t mtt_pgsize_bits, bind_type, reuse_dmahdl;
2705 int status;
2706
2707 ASSERT(MUTEX_HELD(&mr->mr_lock));
2708
2709 /*
2710 * Check the "options" flag. Currently this flag tells the driver
2711 * whether or not the region should be bound normally (i.e. with
2712 * entries written into the PCI IOMMU) or whether it should be
2713 * registered to bypass the IOMMU.
2714 */
2715 if (op == NULL) {
2716 bind_type = HERMON_BINDMEM_NORMAL;
2717 } else {
2718 bind_type = op->mro_bind_type;
2719 }
2720
2721 /*
2722 * Check for invalid length. Check is the length is zero or if the
2723 * length is larger than the maximum configured value. Return error
2724 * if it is.
2725 */
2726 max_sz = ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_mrw_sz);
2727 if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) {
2728 /*
2729 * Deregister will be called upon returning failure from this
2730 * routine. This will ensure that all current resources get
2731 * properly freed up. Unnecessary to attempt to regain
2732 * software ownership of the MPT entry as that has already
2733 * been done above (in hermon_mr_reregister())
2734 */
2735 *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT;
2736
2737 status = IBT_MR_LEN_INVALID;
2738 goto mrrereghelp_fail;
2739 }
2740
2741 /*
2742 * Determine the number of pages necessary for new region and the
2743 * number of pages supported by the current MTT resources
2744 */
2745 nummtt_needed = hermon_mr_nummtt_needed(state, bind, &mtt_pgsize_bits);
2746 nummtt_in_currrsrc = mr->mr_mttrsrcp->hr_len >> HERMON_MTT_SIZE_SHIFT;
2747
2748 /*
2749 * Depending on whether we have enough pages or not, the next step is
2750 * to fill in a set of MTT entries that reflect the new mapping. In
2751 * the first case below, we already have enough entries. This means
2752 * we need to unbind the memory from the previous mapping, bind the
2753 * memory for the new mapping, write the new MTT entries, and update
2754 * the mr to reflect the changes.
2755 * In the second case below, we do not have enough entries in the
2756 * current mapping. So, in this case, we need not only to unbind the
2757 * current mapping, but we need to free up the MTT resources associated
2758 * with that mapping. After we've successfully done that, we continue
2759 * by binding the new memory, allocating new MTT entries, writing the
2760 * new MTT entries, and updating the mr to reflect the changes.
2761 */
2762
2763 /*
2764 * If this region is being shared (i.e. MTT refcount != 1), then we
2765 * can't reuse the current MTT resources regardless of their size.
2766 * Instead we'll need to alloc new ones (below) just as if there
2767 * hadn't been enough room in the current entries.
2768 */
2769 swrc_old = (hermon_sw_refcnt_t *)mr->mr_mttrefcntp->hr_addr;
2770 if (HERMON_MTT_IS_NOT_SHARED(swrc_old) &&
2771 (nummtt_needed <= nummtt_in_currrsrc)) {
2772
2773 /*
2774 * Unbind the old mapping for this memory region, but retain
2775 * the ddi_dma_handle_t (if possible) for reuse in the bind
2776 * operation below. Note: If original memory region was
2777 * bound for IOMMU bypass and the new region can not use
2778 * bypass, then a new DMA handle will be necessary.
2779 */
2780 if (HERMON_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
2781 mr->mr_bindinfo.bi_free_dmahdl = 0;
2782 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2783 dmahdl = mr->mr_bindinfo.bi_dmahdl;
2784 reuse_dmahdl = 1;
2785 } else {
2786 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2787 dmahdl = NULL;
2788 reuse_dmahdl = 0;
2789 }
2790
2791 /*
2792 * Bind the new memory and determine the mapped addresses.
2793 * As described, this routine and hermon_mr_fast_mtt_write()
2794 * do the majority of the work for the memory registration
2795 * operations. Note: When we successfully finish the binding,
2796 * we will set the "bi_free_dmahdl" flag to indicate that
2797 * even though we may have reused the ddi_dma_handle_t we do
2798 * wish it to be freed up at some later time. Note also that
2799 * if we fail, we may need to cleanup the ddi_dma_handle_t.
2800 */
2801 bind->bi_bypass = bind_type;
2802 status = hermon_mr_mem_bind(state, bind, dmahdl, sleep, 1);
2803 if (status != DDI_SUCCESS) {
2804 if (reuse_dmahdl) {
2805 ddi_dma_free_handle(&dmahdl);
2806 }
2807
2808 /*
2809 * Deregister will be called upon returning failure
2810 * from this routine. This will ensure that all
2811 * current resources get properly freed up.
2812 * Unnecessary to attempt to regain software ownership
2813 * of the MPT entry as that has already been done
2814 * above (in hermon_mr_reregister()). Also unnecessary
2815 * to attempt to unbind the memory.
2816 */
2817 *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
2818
2819 status = IBT_INSUFF_RESOURCE;
2820 goto mrrereghelp_fail;
2821 }
2822 if (reuse_dmahdl) {
2823 bind->bi_free_dmahdl = 1;
2824 }
2825
2826 /*
2827 * Using the new mapping, but reusing the current MTT
2828 * resources, write the updated entries to MTT
2829 */
2830 mtt = mr->mr_mttrsrcp;
2831 status = hermon_mr_fast_mtt_write(state, mtt, bind,
2832 mtt_pgsize_bits);
2833 if (status != DDI_SUCCESS) {
2834 /*
2835 * Deregister will be called upon returning failure
2836 * from this routine. This will ensure that all
2837 * current resources get properly freed up.
2838 * Unnecessary to attempt to regain software ownership
2839 * of the MPT entry as that has already been done
2840 * above (in hermon_mr_reregister()). Also unnecessary
2841 * to attempt to unbind the memory.
2842 *
2843 * But we do need to unbind the newly bound memory
2844 * before returning.
2845 */
2846 hermon_mr_mem_unbind(state, bind);
2847 *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
2848
2849 /*
2850 * hermon_mr_fast_mtt_write() returns DDI_FAILURE
2851 * only if it detects a HW error during DMA.
2852 */
2853 hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
2854 status = ibc_get_ci_failure(0);
2855 goto mrrereghelp_fail;
2856 }
2857
2858 /* Put the updated information into the Mem Region handle */
2859 mr->mr_bindinfo = *bind;
2860 mr->mr_logmttpgsz = mtt_pgsize_bits;
2861
2862 } else {
2863 /*
2864 * Check if the memory region MTT is shared by any other MRs.
2865 * Since the resource may be shared between multiple memory
2866 * regions (as a result of a "RegisterSharedMR()" verb) it is
2867 * important that we not unbind any resources prematurely.
2868 */
2869 if (!HERMON_MTT_IS_SHARED(swrc_old)) {
2870 /*
2871 * Unbind the old mapping for this memory region, but
2872 * retain the ddi_dma_handle_t for reuse in the bind
2873 * operation below. Note: This can only be done here
2874 * because the region being reregistered is not
2875 * currently shared. Also if original memory region
2876 * was bound for IOMMU bypass and the new region can
2877 * not use bypass, then a new DMA handle will be
2878 * necessary.
2879 */
2880 if (HERMON_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
2881 mr->mr_bindinfo.bi_free_dmahdl = 0;
2882 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2883 dmahdl = mr->mr_bindinfo.bi_dmahdl;
2884 reuse_dmahdl = 1;
2885 } else {
2886 hermon_mr_mem_unbind(state, &mr->mr_bindinfo);
2887 dmahdl = NULL;
2888 reuse_dmahdl = 0;
2889 }
2890 } else {
2891 dmahdl = NULL;
2892 reuse_dmahdl = 0;
2893 }
2894
2895 /*
2896 * Bind the new memory and determine the mapped addresses.
2897 * As described, this routine and hermon_mr_fast_mtt_write()
2898 * do the majority of the work for the memory registration
2899 * operations. Note: When we successfully finish the binding,
2900 * we will set the "bi_free_dmahdl" flag to indicate that
2901 * even though we may have reused the ddi_dma_handle_t we do
2902 * wish it to be freed up at some later time. Note also that
2903 * if we fail, we may need to cleanup the ddi_dma_handle_t.
2904 */
2905 bind->bi_bypass = bind_type;
2906 status = hermon_mr_mem_bind(state, bind, dmahdl, sleep, 1);
2907 if (status != DDI_SUCCESS) {
2908 if (reuse_dmahdl) {
2909 ddi_dma_free_handle(&dmahdl);
2910 }
2911
2912 /*
2913 * Deregister will be called upon returning failure
2914 * from this routine. This will ensure that all
2915 * current resources get properly freed up.
2916 * Unnecessary to attempt to regain software ownership
2917 * of the MPT entry as that has already been done
2918 * above (in hermon_mr_reregister()). Also unnecessary
2919 * to attempt to unbind the memory.
2920 */
2921 *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
2922
2923 status = IBT_INSUFF_RESOURCE;
2924 goto mrrereghelp_fail;
2925 }
2926 if (reuse_dmahdl) {
2927 bind->bi_free_dmahdl = 1;
2928 }
2929
2930 /*
2931 * Allocate the new MTT entries resource
2932 */
2933 status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt_needed,
2934 sleep, &mtt);
2935 if (status != DDI_SUCCESS) {
2936 /*
2937 * Deregister will be called upon returning failure
2938 * from this routine. This will ensure that all
2939 * current resources get properly freed up.
2940 * Unnecessary to attempt to regain software ownership
2941 * of the MPT entry as that has already been done
2942 * above (in hermon_mr_reregister()). Also unnecessary
2943 * to attempt to unbind the memory.
2944 *
2945 * But we do need to unbind the newly bound memory
2946 * before returning.
2947 */
2948 hermon_mr_mem_unbind(state, bind);
2949 *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
2950
2951 status = IBT_INSUFF_RESOURCE;
2952 goto mrrereghelp_fail;
2953 }
2954
2955 /*
2956 * Allocate MTT reference count (to track shared memory
2957 * regions). As mentioned elsewhere above, this reference
2958 * count resource may never be used on the given memory region,
2959 * but if it is ever later registered as a "shared" memory
2960 * region then this resource will be necessary. Note: This
2961 * is only necessary here if the existing memory region is
2962 * already being shared (because otherwise we already have
2963 * a useable reference count resource).
2964 */
2965 if (HERMON_MTT_IS_SHARED(swrc_old)) {
2966 status = hermon_rsrc_alloc(state, HERMON_REFCNT, 1,
2967 sleep, &mtt_refcnt);
2968 if (status != DDI_SUCCESS) {
2969 /*
2970 * Deregister will be called upon returning
2971 * failure from this routine. This will ensure
2972 * that all current resources get properly
2973 * freed up. Unnecessary to attempt to regain
2974 * software ownership of the MPT entry as that
2975 * has already been done above (in
2976 * hermon_mr_reregister()). Also unnecessary
2977 * to attempt to unbind the memory.
2978 *
2979 * But we need to unbind the newly bound
2980 * memory and free up the newly allocated MTT
2981 * entries before returning.
2982 */
2983 hermon_mr_mem_unbind(state, bind);
2984 hermon_rsrc_free(state, &mtt);
2985 *dereg_level =
2986 HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
2987
2988 status = IBT_INSUFF_RESOURCE;
2989 goto mrrereghelp_fail;
2990 }
2991 swrc_new = (hermon_sw_refcnt_t *)mtt_refcnt->hr_addr;
2992 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_new))
2993 HERMON_MTT_REFCNT_INIT(swrc_new);
2994 } else {
2995 mtt_refcnt = mr->mr_mttrefcntp;
2996 }
2997
2998 /*
2999 * Using the new mapping and the new MTT resources, write the
3000 * updated entries to MTT
3001 */
3002 status = hermon_mr_fast_mtt_write(state, mtt, bind,
3003 mtt_pgsize_bits);
3004 if (status != DDI_SUCCESS) {
3005 /*
3006 * Deregister will be called upon returning failure
3007 * from this routine. This will ensure that all
3008 * current resources get properly freed up.
3009 * Unnecessary to attempt to regain software ownership
3010 * of the MPT entry as that has already been done
3011 * above (in hermon_mr_reregister()). Also unnecessary
3012 * to attempt to unbind the memory.
3013 *
3014 * But we need to unbind the newly bound memory,
3015 * free up the newly allocated MTT entries, and
3016 * (possibly) free the new MTT reference count
3017 * resource before returning.
3018 */
3019 if (HERMON_MTT_IS_SHARED(swrc_old)) {
3020 hermon_rsrc_free(state, &mtt_refcnt);
3021 }
3022 hermon_mr_mem_unbind(state, bind);
3023 hermon_rsrc_free(state, &mtt);
3024 *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
3025
3026 status = IBT_INSUFF_RESOURCE;
3027 goto mrrereghelp_fail;
3028 }
3029
3030 /*
3031 * Check if the memory region MTT is shared by any other MRs.
3032 * Since the resource may be shared between multiple memory
3033 * regions (as a result of a "RegisterSharedMR()" verb) it is
3034 * important that we not free up any resources prematurely.
3035 */
3036 if (HERMON_MTT_IS_SHARED(swrc_old)) {
3037 /* Decrement MTT reference count for "old" region */
3038 (void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp);
3039 } else {
3040 /* Free up the old MTT entries resource */
3041 hermon_rsrc_free(state, &mr->mr_mttrsrcp);
3042 }
3043
3044 /* Put the updated information into the mrhdl */
3045 mr->mr_bindinfo = *bind;
3046 mr->mr_logmttpgsz = mtt_pgsize_bits;
3047 mr->mr_mttrsrcp = mtt;
3048 mr->mr_mttrefcntp = mtt_refcnt;
3049 }
3050
3051 /*
3052 * Calculate and return the updated MTT address (in the DDR address
3053 * space). This will be used by the caller (hermon_mr_reregister) in
3054 * the updated MPT entry
3055 */
3056 *mtt_addr = mtt->hr_indx << HERMON_MTT_SIZE_SHIFT;
3057
3058 return (DDI_SUCCESS);
3059
3060 mrrereghelp_fail:
3061 return (status);
3062 }
3063
3064
3065 /*
3066 * hermon_mr_nummtt_needed()
3067 * Context: Can be called from interrupt or base context.
3068 */
3069 /* ARGSUSED */
3070 static uint64_t
3071 hermon_mr_nummtt_needed(hermon_state_t *state, hermon_bind_info_t *bind,
3072 uint_t *mtt_pgsize_bits)
3073 {
3074 uint64_t pg_offset_mask;
3075 uint64_t pg_offset, tmp_length;
3076
3077 /*
3078 * For now we specify the page size as 8Kb (the default page size for
3079 * the sun4u architecture), or 4Kb for x86. Figure out optimal page
3080 * size by examining the dmacookies
3081 */
3082 *mtt_pgsize_bits = PAGESHIFT;
3083
3084 pg_offset_mask = ((uint64_t)1 << *mtt_pgsize_bits) - 1;
3085 pg_offset = bind->bi_addr & pg_offset_mask;
3086 tmp_length = pg_offset + (bind->bi_len - 1);
3087 return ((tmp_length >> *mtt_pgsize_bits) + 1);
3088 }
3089
3090
3091 /*
3092 * hermon_mr_mem_bind()
3093 * Context: Can be called from interrupt or base context.
3094 */
3095 static int
3096 hermon_mr_mem_bind(hermon_state_t *state, hermon_bind_info_t *bind,
3097 ddi_dma_handle_t dmahdl, uint_t sleep, uint_t is_buffer)
3098 {
3099 ddi_dma_attr_t dma_attr;
3100 int (*callback)(caddr_t);
3101 int status;
3102
3103 /* bi_type must be set to a meaningful value to get a bind handle */
3104 ASSERT(bind->bi_type == HERMON_BINDHDL_VADDR ||
3105 bind->bi_type == HERMON_BINDHDL_BUF ||
3106 bind->bi_type == HERMON_BINDHDL_UBUF);
3107
3108 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
3109
3110 /* Set the callback flag appropriately */
3111 callback = (sleep == HERMON_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
3112
3113 /*
3114 * Initialize many of the default DMA attributes. Then, if we're
3115 * bypassing the IOMMU, set the DDI_DMA_FORCE_PHYSICAL flag.
3116 */
3117 if (dmahdl == NULL) {
3118 hermon_dma_attr_init(state, &dma_attr);
3119 #ifdef __sparc
3120 if (bind->bi_bypass == HERMON_BINDMEM_BYPASS) {
3121 dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
3122 }
3123 #endif
3124
3125 /* set RO if needed - tunable set and 'is_buffer' is non-0 */
3126 if (is_buffer) {
3127 if (! (bind->bi_flags & IBT_MR_DISABLE_RO)) {
3128 if ((bind->bi_type != HERMON_BINDHDL_UBUF) &&
3129 (hermon_kernel_data_ro ==
3130 HERMON_RO_ENABLED)) {
3131 dma_attr.dma_attr_flags |=
3132 DDI_DMA_RELAXED_ORDERING;
3133 }
3134 if (((bind->bi_type == HERMON_BINDHDL_UBUF) &&
3135 (hermon_user_data_ro ==
3136 HERMON_RO_ENABLED))) {
3137 dma_attr.dma_attr_flags |=
3138 DDI_DMA_RELAXED_ORDERING;
3139 }
3140 }
3141 }
3142
3143 /* Allocate a DMA handle for the binding */
3144 status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
3145 callback, NULL, &bind->bi_dmahdl);
3146 if (status != DDI_SUCCESS) {
3147 return (status);
3148 }
3149 bind->bi_free_dmahdl = 1;
3150
3151 } else {
3152 bind->bi_dmahdl = dmahdl;
3153 bind->bi_free_dmahdl = 0;
3154 }
3155
3156
3157 /*
3158 * Bind the memory to get the PCI mapped addresses. The decision
3159 * to call ddi_dma_addr_bind_handle() or ddi_dma_buf_bind_handle()
3160 * is determined by the "bi_type" flag. Note: if the bind operation
3161 * fails then we have to free up the DMA handle and return error.
3162 */
3163 if (bind->bi_type == HERMON_BINDHDL_VADDR) {
3164 status = ddi_dma_addr_bind_handle(bind->bi_dmahdl, NULL,
3165 (caddr_t)(uintptr_t)bind->bi_addr, bind->bi_len,
3166 (DDI_DMA_RDWR | DDI_DMA_CONSISTENT), callback, NULL,
3167 &bind->bi_dmacookie, &bind->bi_cookiecnt);
3168
3169 } else { /* HERMON_BINDHDL_BUF or HERMON_BINDHDL_UBUF */
3170
3171 status = ddi_dma_buf_bind_handle(bind->bi_dmahdl,
3172 bind->bi_buf, (DDI_DMA_RDWR | DDI_DMA_CONSISTENT), callback,
3173 NULL, &bind->bi_dmacookie, &bind->bi_cookiecnt);
3174 }
3175 if (status != DDI_DMA_MAPPED) {
3176 if (bind->bi_free_dmahdl != 0) {
3177 ddi_dma_free_handle(&bind->bi_dmahdl);
3178 }
3179 return (status);
3180 }
3181
3182 return (DDI_SUCCESS);
3183 }
3184
3185
3186 /*
3187 * hermon_mr_mem_unbind()
3188 * Context: Can be called from interrupt or base context.
3189 */
3190 static void
3191 hermon_mr_mem_unbind(hermon_state_t *state, hermon_bind_info_t *bind)
3192 {
3193 int status;
3194
3195 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
3196 /* there is nothing to unbind for alloc_lkey */
3197 if (bind->bi_type == HERMON_BINDHDL_LKEY)
3198 return;
3199
3200 /*
3201 * In case of HERMON_BINDHDL_UBUF, the memory bi_buf points to
3202 * is actually allocated by ddi_umem_iosetup() internally, then
3203 * it's required to free it here. Reset bi_type to HERMON_BINDHDL_NONE
3204 * not to free it again later.
3205 */
3206 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
3207 if (bind->bi_type == HERMON_BINDHDL_UBUF) {
3208 freerbuf(bind->bi_buf);
3209 bind->bi_type = HERMON_BINDHDL_NONE;
3210 }
3211 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind))
3212
3213 /*
3214 * Unbind the DMA memory for the region
3215 *
3216 * Note: The only way ddi_dma_unbind_handle() currently
3217 * can return an error is if the handle passed in is invalid.
3218 * Since this should never happen, we choose to return void
3219 * from this function! If this does return an error, however,
3220 * then we print a warning message to the console.
3221 */
3222 status = ddi_dma_unbind_handle(bind->bi_dmahdl);
3223 if (status != DDI_SUCCESS) {
3224 HERMON_WARNING(state, "failed to unbind DMA mapping");
3225 return;
3226 }
3227
3228 /* Free up the DMA handle */
3229 if (bind->bi_free_dmahdl != 0) {
3230 ddi_dma_free_handle(&bind->bi_dmahdl);
3231 }
3232 }
3233
3234
3235 /*
3236 * hermon_mr_fast_mtt_write()
3237 * Context: Can be called from interrupt or base context.
3238 */
3239 static int
3240 hermon_mr_fast_mtt_write(hermon_state_t *state, hermon_rsrc_t *mtt,
3241 hermon_bind_info_t *bind, uint32_t mtt_pgsize_bits)
3242 {
3243 hermon_icm_table_t *icm_table;
3244 hermon_dma_info_t *dma_info;
3245 uint32_t index1, index2, rindx;
3246 ddi_dma_cookie_t dmacookie;
3247 uint_t cookie_cnt;
3248 uint64_t *mtt_table;
3249 uint64_t mtt_entry;
3250 uint64_t addr, endaddr;
3251 uint64_t pagesize;
3252 offset_t i, start;
3253 uint_t per_span;
3254 int sync_needed;
3255
3256 /*
3257 * XXX According to the PRM, we are to use the WRITE_MTT
3258 * command to write out MTTs. Tavor does not do this,
3259 * instead taking advantage of direct access to the MTTs,
3260 * and knowledge that Mellanox FMR relies on our ability
3261 * to write directly to the MTTs without any further
3262 * notification to the firmware. Likewise, we will choose
3263 * to not use the WRITE_MTT command, but to simply write
3264 * out the MTTs.
3265 */
3266
3267 /* Calculate page size from the suggested value passed in */
3268 pagesize = ((uint64_t)1 << mtt_pgsize_bits);
3269
3270 /* Walk the "cookie list" and fill in the MTT table entries */
3271 dmacookie = bind->bi_dmacookie;
3272 cookie_cnt = bind->bi_cookiecnt;
3273
3274 icm_table = &state->hs_icm[HERMON_MTT];
3275 rindx = mtt->hr_indx;
3276 hermon_index(index1, index2, rindx, icm_table, i);
3277 start = i;
3278
3279 per_span = icm_table->span;
3280 dma_info = icm_table->icm_dma[index1] + index2;
3281 mtt_table = (uint64_t *)(uintptr_t)dma_info->vaddr;
3282
3283 sync_needed = 0;
3284 while (cookie_cnt-- > 0) {
3285 addr = dmacookie.dmac_laddress;
3286 endaddr = addr + (dmacookie.dmac_size - 1);
3287 addr = addr & ~((uint64_t)pagesize - 1);
3288
3289 while (addr <= endaddr) {
3290
3291 /*
3292 * Fill in the mapped addresses (calculated above) and
3293 * set HERMON_MTT_ENTRY_PRESENT flag for each MTT entry.
3294 */
3295 mtt_entry = addr | HERMON_MTT_ENTRY_PRESENT;
3296 mtt_table[i] = htonll(mtt_entry);
3297 i++;
3298 rindx++;
3299
3300 if (i == per_span) {
3301
3302 (void) ddi_dma_sync(dma_info->dma_hdl,
3303 start * sizeof (hermon_hw_mtt_t),
3304 (i - start) * sizeof (hermon_hw_mtt_t),
3305 DDI_DMA_SYNC_FORDEV);
3306
3307 if ((addr + pagesize > endaddr) &&
3308 (cookie_cnt == 0))
3309 return (DDI_SUCCESS);
3310
3311 hermon_index(index1, index2, rindx, icm_table,
3312 i);
3313 start = i * sizeof (hermon_hw_mtt_t);
3314 dma_info = icm_table->icm_dma[index1] + index2;
3315 mtt_table =
3316 (uint64_t *)(uintptr_t)dma_info->vaddr;
3317
3318 sync_needed = 0;
3319 } else {
3320 sync_needed = 1;
3321 }
3322
3323 addr += pagesize;
3324 if (addr == 0) {
3325 static int do_once = 1;
3326 _NOTE(SCHEME_PROTECTS_DATA("safe sharing",
3327 do_once))
3328 if (do_once) {
3329 do_once = 0;
3330 cmn_err(CE_NOTE, "probable error in "
3331 "dma_cookie address from caller\n");
3332 }
3333 break;
3334 }
3335 }
3336
3337 /*
3338 * When we've reached the end of the current DMA cookie,
3339 * jump to the next cookie (if there are more)
3340 */
3341 if (cookie_cnt != 0) {
3342 ddi_dma_nextcookie(bind->bi_dmahdl, &dmacookie);
3343 }
3344 }
3345
3346 /* done all the cookies, now sync the memory for the device */
3347 if (sync_needed)
3348 (void) ddi_dma_sync(dma_info->dma_hdl,
3349 start * sizeof (hermon_hw_mtt_t),
3350 (i - start) * sizeof (hermon_hw_mtt_t),
3351 DDI_DMA_SYNC_FORDEV);
3352
3353 return (DDI_SUCCESS);
3354 }
3355
3356 /*
3357 * hermon_mr_fast_mtt_write_fmr()
3358 * Context: Can be called from interrupt or base context.
3359 */
3360 /* ARGSUSED */
3361 static int
3362 hermon_mr_fast_mtt_write_fmr(hermon_state_t *state, hermon_rsrc_t *mtt,
3363 ibt_pmr_attr_t *mem_pattr, uint32_t mtt_pgsize_bits)
3364 {
3365 hermon_icm_table_t *icm_table;
3366 hermon_dma_info_t *dma_info;
3367 uint32_t index1, index2, rindx;
3368 uint64_t *mtt_table;
3369 offset_t i, j;
3370 uint_t per_span;
3371
3372 icm_table = &state->hs_icm[HERMON_MTT];
3373 rindx = mtt->hr_indx;
3374 hermon_index(index1, index2, rindx, icm_table, i);
3375 per_span = icm_table->span;
3376 dma_info = icm_table->icm_dma[index1] + index2;
3377 mtt_table = (uint64_t *)(uintptr_t)dma_info->vaddr;
3378
3379 /*
3380 * Fill in the MTT table entries
3381 */
3382 for (j = 0; j < mem_pattr->pmr_num_buf; j++) {
3383 mtt_table[i] = mem_pattr->pmr_addr_list[j].p_laddr;
3384 i++;
3385 rindx++;
3386 if (i == per_span) {
3387 hermon_index(index1, index2, rindx, icm_table, i);
3388 dma_info = icm_table->icm_dma[index1] + index2;
3389 mtt_table = (uint64_t *)(uintptr_t)dma_info->vaddr;
3390 }
3391 }
3392
3393 return (DDI_SUCCESS);
3394 }
3395
3396
3397 /*
3398 * hermon_mtt_refcnt_inc()
3399 * Context: Can be called from interrupt or base context.
3400 */
3401 static uint_t
3402 hermon_mtt_refcnt_inc(hermon_rsrc_t *rsrc)
3403 {
3404 hermon_sw_refcnt_t *rc;
3405
3406 rc = (hermon_sw_refcnt_t *)rsrc->hr_addr;
3407 return (atomic_inc_uint_nv(&rc->swrc_refcnt));
3408 }
3409
3410
3411 /*
3412 * hermon_mtt_refcnt_dec()
3413 * Context: Can be called from interrupt or base context.
3414 */
3415 static uint_t
3416 hermon_mtt_refcnt_dec(hermon_rsrc_t *rsrc)
3417 {
3418 hermon_sw_refcnt_t *rc;
3419
3420 rc = (hermon_sw_refcnt_t *)rsrc->hr_addr;
3421 return (atomic_dec_uint_nv(&rc->swrc_refcnt));
3422 }