Print this page
8368 remove warlock leftovers from usr/src/uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/adapters/tavor/tavor_umap.c
+++ new/usr/src/uts/common/io/ib/adapters/tavor/tavor_umap.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /*
28 28 * tavor_umap.c
29 29 * Tavor Userland Mapping Routines
30 30 *
31 31 * Implements all the routines necessary for enabling direct userland
32 32 * access to the Tavor hardware. This includes all routines necessary for
33 33 * maintaining the "userland resources database" and all the support routines
34 34 * for the devmap calls.
35 35 */
36 36
37 37 #include <sys/types.h>
38 38 #include <sys/conf.h>
39 39 #include <sys/ddi.h>
40 40 #include <sys/sunddi.h>
41 41 #include <sys/modctl.h>
42 42 #include <sys/file.h>
43 43 #include <sys/avl.h>
44 44 #include <sys/sysmacros.h>
45 45
46 46 #include <sys/ib/adapters/tavor/tavor.h>
47 47
48 48 /* Tavor HCA state pointer (extern) */
49 49 extern void *tavor_statep;
50 50
51 51 /* Tavor HCA Userland Resource Database (extern) */
52 52 extern tavor_umap_db_t tavor_userland_rsrc_db;
53 53
54 54 static int tavor_umap_uarpg(tavor_state_t *state, devmap_cookie_t dhp,
55 55 tavor_rsrc_t *rsrcp, size_t *maplen, int *err);
56 56 static int tavor_umap_cqmem(tavor_state_t *state, devmap_cookie_t dhp,
57 57 tavor_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
58 58 static int tavor_umap_qpmem(tavor_state_t *state, devmap_cookie_t dhp,
59 59 tavor_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
60 60 static int tavor_umap_srqmem(tavor_state_t *state, devmap_cookie_t dhp,
61 61 tavor_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
62 62 static int tavor_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
63 63 offset_t off, size_t len, void **pvtp);
64 64 static int tavor_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp,
65 65 devmap_cookie_t new_dhp, void **new_pvtp);
66 66 static void tavor_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp,
67 67 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
68 68 devmap_cookie_t new_dhp2, void **pvtp2);
69 69 static int tavor_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
70 70 offset_t off, size_t len, void **pvtp);
71 71 static int tavor_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
72 72 devmap_cookie_t new_dhp, void **new_pvtp);
73 73 static void tavor_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp,
74 74 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
75 75 devmap_cookie_t new_dhp2, void **pvtp2);
76 76 static ibt_status_t tavor_umap_mr_data_in(tavor_mrhdl_t mr,
77 77 ibt_mr_data_in_t *data, size_t data_sz);
78 78 static ibt_status_t tavor_umap_cq_data_out(tavor_cqhdl_t cq,
79 79 mlnx_umap_cq_data_out_t *data, size_t data_sz);
80 80 static ibt_status_t tavor_umap_qp_data_out(tavor_qphdl_t qp,
81 81 mlnx_umap_qp_data_out_t *data, size_t data_sz);
82 82 static ibt_status_t tavor_umap_srq_data_out(tavor_srqhdl_t srq,
83 83 mlnx_umap_srq_data_out_t *data, size_t data_sz);
84 84 static int tavor_umap_db_compare(const void *query, const void *entry);
85 85 static ibt_status_t tavor_umap_pd_data_out(tavor_pdhdl_t pd,
86 86 mlnx_umap_pd_data_out_t *data, size_t data_sz);
87 87
88 88
89 89 /*
90 90 * These callbacks are passed to devmap_umem_setup() and devmap_devmem_setup(),
91 91 * respectively. They are used to handle (among other things) partial
92 92 * unmappings and to provide a method for invalidating mappings inherited
93 93 * as a result of a fork(2) system call.
94 94 */
95 95 static struct devmap_callback_ctl tavor_devmap_umem_cbops = {
96 96 DEVMAP_OPS_REV,
97 97 tavor_devmap_umem_map,
98 98 NULL,
99 99 tavor_devmap_umem_dup,
100 100 tavor_devmap_umem_unmap
101 101 };
102 102 static struct devmap_callback_ctl tavor_devmap_devmem_cbops = {
103 103 DEVMAP_OPS_REV,
104 104 tavor_devmap_devmem_map,
105 105 NULL,
106 106 tavor_devmap_devmem_dup,
107 107 tavor_devmap_devmem_unmap
108 108 };
109 109
110 110 /*
111 111 * tavor_devmap()
112 112 * Context: Can be called from user context.
113 113 */
114 114 /* ARGSUSED */
115 115 int
116 116 tavor_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
117 117 size_t *maplen, uint_t model)
118 118 {
119 119 tavor_state_t *state;
120 120 tavor_rsrc_t *rsrcp;
121 121 minor_t instance;
122 122 uint64_t key, value;
123 123 uint_t type;
124 124 int err, status;
125 125
126 126 TAVOR_TNF_ENTER(tavor_devmap);
127 127
128 128 /* Get Tavor softstate structure from instance */
129 129 instance = TAVOR_DEV_INSTANCE(dev);
130 130 state = ddi_get_soft_state(tavor_statep, instance);
131 131 if (state == NULL) {
132 132 TNF_PROBE_0(tavor_devmap_gss_fail, TAVOR_TNF_ERROR, "");
133 133 TAVOR_TNF_EXIT(tavor_devmap);
134 134 return (ENXIO);
135 135 }
136 136
137 137 /*
138 138 * Access to Tavor devmap interface is not allowed in
139 139 * "maintenance mode".
140 140 */
141 141 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
142 142 TNF_PROBE_0(tavor_devmap_maintenance_mode_fail,
143 143 TAVOR_TNF_ERROR, "");
144 144 TAVOR_TNF_EXIT(tavor_devmap);
145 145 return (EFAULT);
146 146 }
147 147
148 148 /*
149 149 * The bottom bits of "offset" are undefined (number depends on
150 150 * system PAGESIZE). Shifting these off leaves us with a "key".
151 151 * The "key" is actually a combination of both a real key value
152 152 * (for the purpose of database lookup) and a "type" value. We
153 153 * extract this information before doing the database lookup.
154 154 */
155 155 key = off >> PAGESHIFT;
156 156 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
157 157 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
158 158 status = tavor_umap_db_find(instance, key, type, &value, 0, NULL);
159 159 if (status == DDI_SUCCESS) {
160 160 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
161 161
162 162 switch (type) {
163 163 case MLNX_UMAP_UARPG_RSRC:
164 164 /*
165 165 * Double check that process who open()'d Tavor is
166 166 * same process attempting to mmap() UAR page.
167 167 */
168 168 if (key != ddi_get_pid()) {
169 169 TNF_PROBE_0(tavor_devmap_uarpg_invpid_fail,
170 170 TAVOR_TNF_ERROR, "");
171 171 TAVOR_TNF_EXIT(tavor_devmap);
172 172 return (EINVAL);
173 173 }
174 174
175 175 /* Map the UAR page out for userland access */
176 176 status = tavor_umap_uarpg(state, dhp, rsrcp, maplen,
177 177 &err);
178 178 if (status != DDI_SUCCESS) {
179 179 TNF_PROBE_0(tavor_devmap_uarpg_map_fail,
180 180 TAVOR_TNF_ERROR, "");
181 181 TAVOR_TNF_EXIT(tavor_devmap);
182 182 return (err);
183 183 }
184 184 break;
185 185
186 186 case MLNX_UMAP_CQMEM_RSRC:
187 187 /* Map the CQ memory out for userland access */
188 188 status = tavor_umap_cqmem(state, dhp, rsrcp, off,
189 189 maplen, &err);
190 190 if (status != DDI_SUCCESS) {
191 191 TNF_PROBE_0(tavor_devmap_cqmem_map_fail,
192 192 TAVOR_TNF_ERROR, "");
193 193 TAVOR_TNF_EXIT(tavor_devmap);
194 194 return (err);
195 195 }
196 196 break;
197 197
198 198 case MLNX_UMAP_QPMEM_RSRC:
199 199 /* Map the QP memory out for userland access */
200 200 status = tavor_umap_qpmem(state, dhp, rsrcp, off,
201 201 maplen, &err);
202 202 if (status != DDI_SUCCESS) {
203 203 TNF_PROBE_0(tavor_devmap_qpmem_map_fail,
204 204 TAVOR_TNF_ERROR, "");
205 205 TAVOR_TNF_EXIT(tavor_devmap);
206 206 return (err);
207 207 }
208 208 break;
209 209
210 210 case MLNX_UMAP_SRQMEM_RSRC:
211 211 /* Map the SRQ memory out for userland access */
212 212 status = tavor_umap_srqmem(state, dhp, rsrcp, off,
213 213 maplen, &err);
214 214 if (status != DDI_SUCCESS) {
215 215 TNF_PROBE_0(tavor_devmap_srqmem_map_fail,
216 216 TAVOR_TNF_ERROR, "");
217 217 TAVOR_TNF_EXIT(tavor_devmap);
218 218 return (err);
219 219 }
220 220 break;
221 221
222 222 default:
223 223 TAVOR_WARNING(state, "unexpected rsrc type in devmap");
224 224 TNF_PROBE_0(tavor_devmap_invrsrc_fail,
225 225 TAVOR_TNF_ERROR, "");
226 226 TAVOR_TNF_EXIT(tavor_devmap);
227 227 return (EINVAL);
228 228 }
229 229 } else {
230 230 TNF_PROBE_0(tavor_devmap_umap_lookup_fail, TAVOR_TNF_ERROR, "");
231 231 TAVOR_TNF_EXIT(tavor_devmap);
232 232 return (EINVAL);
233 233 }
234 234
235 235 TAVOR_TNF_EXIT(tavor_devmap);
236 236 return (0);
237 237 }
238 238
239 239
240 240 /*
241 241 * tavor_umap_uarpg()
242 242 * Context: Can be called from user context.
243 243 */
244 244 static int
245 245 tavor_umap_uarpg(tavor_state_t *state, devmap_cookie_t dhp,
246 246 tavor_rsrc_t *rsrcp, size_t *maplen, int *err)
247 247 {
248 248 int status;
249 249 uint_t maxprot;
250 250
251 251 TAVOR_TNF_ENTER(tavor_umap_uarpg);
252 252
253 253 /* Map out the UAR page (doorbell page) */
254 254 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
255 255 status = devmap_devmem_setup(dhp, state->ts_dip,
256 256 &tavor_devmap_devmem_cbops, TAVOR_UAR_BAR, (rsrcp->tr_indx <<
257 257 PAGESHIFT), PAGESIZE, maxprot, DEVMAP_ALLOW_REMAP,
258 258 &state->ts_reg_accattr);
259 259 if (status < 0) {
260 260 *err = status;
261 261 TNF_PROBE_0(tavor_umap_uarpg_devmap_fail, TAVOR_TNF_ERROR, "");
262 262 TAVOR_TNF_EXIT(tavor_umap_uarpg);
263 263 return (DDI_FAILURE);
264 264 }
265 265
266 266 *maplen = PAGESIZE;
267 267 TAVOR_TNF_EXIT(tavor_umap_uarpg);
268 268 return (DDI_SUCCESS);
269 269 }
270 270
271 271
272 272 /*
273 273 * tavor_umap_cqmem()
274 274 * Context: Can be called from user context.
275 275 */
276 276 /* ARGSUSED */
277 277 static int
278 278 tavor_umap_cqmem(tavor_state_t *state, devmap_cookie_t dhp,
279 279 tavor_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
280 280 {
281 281 tavor_cqhdl_t cq;
282 282 size_t size;
283 283 uint_t maxprot;
284 284 int status;
285 285
286 286 TAVOR_TNF_ENTER(tavor_umap_cqmem);
287 287
288 288 /* Extract the Tavor CQ handle pointer from the tavor_rsrc_t */
289 289 cq = (tavor_cqhdl_t)rsrcp->tr_addr;
290 290
291 291 /* Round-up the CQ size to system page size */
292 292 size = ptob(btopr(cq->cq_cqinfo.qa_size));
293 293
294 294 /* Map out the CQ memory */
295 295 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
296 296 status = devmap_umem_setup(dhp, state->ts_dip,
297 297 &tavor_devmap_umem_cbops, cq->cq_cqinfo.qa_umemcookie, 0, size,
298 298 maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
299 299 if (status < 0) {
300 300 *err = status;
301 301 TNF_PROBE_0(tavor_umap_cqmem_devmap_fail, TAVOR_TNF_ERROR, "");
302 302 TAVOR_TNF_EXIT(tavor_umap_cqmem);
303 303 return (DDI_FAILURE);
304 304 }
305 305 *maplen = size;
306 306
307 307 TAVOR_TNF_EXIT(tavor_umap_cqmem);
308 308 return (DDI_SUCCESS);
309 309 }
310 310
311 311
312 312 /*
313 313 * tavor_umap_qpmem()
314 314 * Context: Can be called from user context.
315 315 */
316 316 /* ARGSUSED */
317 317 static int
318 318 tavor_umap_qpmem(tavor_state_t *state, devmap_cookie_t dhp,
319 319 tavor_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
320 320 {
321 321 tavor_qphdl_t qp;
322 322 offset_t offset;
323 323 size_t size;
324 324 uint_t maxprot;
325 325 int status;
326 326
327 327 TAVOR_TNF_ENTER(tavor_umap_qpmem);
328 328
329 329 /* Extract the Tavor QP handle pointer from the tavor_rsrc_t */
330 330 qp = (tavor_qphdl_t)rsrcp->tr_addr;
331 331
332 332 /*
333 333 * Calculate the offset of the first work queue (send or recv) into
334 334 * the memory (ddi_umem_alloc()) allocated previously for the QP.
335 335 */
336 336 offset = (offset_t)((uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
337 337 (uintptr_t)qp->qp_wqinfo.qa_buf_real);
338 338
339 339 /* Round-up the QP work queue sizes to system page size */
340 340 size = ptob(btopr(qp->qp_wqinfo.qa_size));
341 341
342 342 /* Map out the QP memory */
343 343 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
344 344 status = devmap_umem_setup(dhp, state->ts_dip,
345 345 &tavor_devmap_umem_cbops, qp->qp_wqinfo.qa_umemcookie, offset,
346 346 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
347 347 if (status < 0) {
348 348 *err = status;
349 349 TNF_PROBE_0(tavor_umap_qpmem_devmap_fail, TAVOR_TNF_ERROR, "");
350 350 TAVOR_TNF_EXIT(tavor_umap_qpmem);
351 351 return (DDI_FAILURE);
352 352 }
353 353 *maplen = size;
354 354
355 355 TAVOR_TNF_EXIT(tavor_umap_qpmem);
356 356 return (DDI_SUCCESS);
357 357 }
358 358
359 359
360 360 /*
361 361 * tavor_umap_srqmem()
362 362 * Context: Can be called from user context.
363 363 */
364 364 /* ARGSUSED */
365 365 static int
366 366 tavor_umap_srqmem(tavor_state_t *state, devmap_cookie_t dhp,
367 367 tavor_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
368 368 {
369 369 tavor_srqhdl_t srq;
370 370 offset_t offset;
371 371 size_t size;
372 372 uint_t maxprot;
373 373 int status;
374 374
375 375 TAVOR_TNF_ENTER(tavor_umap_srqmem);
376 376
377 377 /* Extract the Tavor SRQ handle pointer from the tavor_rsrc_t */
378 378 srq = (tavor_srqhdl_t)rsrcp->tr_addr;
379 379
380 380 /*
381 381 * Calculate the offset of the first shared recv queue into the memory
382 382 * (ddi_umem_alloc()) allocated previously for the SRQ.
383 383 */
384 384 offset = (offset_t)((uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
385 385 (uintptr_t)srq->srq_wqinfo.qa_buf_real);
386 386
387 387 /* Round-up the SRQ work queue sizes to system page size */
388 388 size = ptob(btopr(srq->srq_wqinfo.qa_size));
389 389
390 390 /* Map out the QP memory */
391 391 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
392 392 status = devmap_umem_setup(dhp, state->ts_dip,
393 393 &tavor_devmap_umem_cbops, srq->srq_wqinfo.qa_umemcookie, offset,
394 394 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
395 395 if (status < 0) {
396 396 *err = status;
397 397 TNF_PROBE_0(tavor_umap_srqmem_devmap_fail, TAVOR_TNF_ERROR, "");
398 398 TAVOR_TNF_EXIT(tavor_umap_srqmem);
399 399 return (DDI_FAILURE);
400 400 }
401 401 *maplen = size;
402 402
403 403 TAVOR_TNF_EXIT(tavor_umap_srqmem);
404 404 return (DDI_SUCCESS);
405 405 }
406 406
407 407
408 408 /*
409 409 * tavor_devmap_umem_map()
410 410 * Context: Can be called from kernel context.
411 411 */
412 412 /* ARGSUSED */
413 413 static int
414 414 tavor_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
415 415 offset_t off, size_t len, void **pvtp)
416 416 {
417 417 tavor_state_t *state;
418 418 tavor_devmap_track_t *dvm_track;
419 419 tavor_cqhdl_t cq;
420 420 tavor_qphdl_t qp;
421 421 tavor_srqhdl_t srq;
422 422 minor_t instance;
423 423 uint64_t key;
424 424 uint_t type;
425 425
426 426 TAVOR_TNF_ENTER(tavor_devmap_umem_map);
427 427
428 428 /* Get Tavor softstate structure from instance */
429 429 instance = TAVOR_DEV_INSTANCE(dev);
430 430 state = ddi_get_soft_state(tavor_statep, instance);
431 431 if (state == NULL) {
432 432 TNF_PROBE_0(tavor_devmap_umem_map_gss_fail, TAVOR_TNF_ERROR,
433 433 "");
434 434 TAVOR_TNF_EXIT(tavor_devmap_umem_map);
435 435 return (ENXIO);
436 436 }
437 437
438 438 /*
439 439 * The bottom bits of "offset" are undefined (number depends on
440 440 * system PAGESIZE). Shifting these off leaves us with a "key".
441 441 * The "key" is actually a combination of both a real key value
442 442 * (for the purpose of database lookup) and a "type" value. Although
443 443 * we are not going to do any database lookup per se, we do want
444 444 * to extract the "key" and the "type" (to enable faster lookup of
445 445 * the appropriate CQ or QP handle).
446 446 */
|
↓ open down ↓ |
446 lines elided |
↑ open up ↑ |
447 447 key = off >> PAGESHIFT;
448 448 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
449 449 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
450 450
451 451 /*
452 452 * Allocate an entry to track the mapping and unmapping (specifically,
453 453 * partial unmapping) of this resource.
454 454 */
455 455 dvm_track = (tavor_devmap_track_t *)kmem_zalloc(
456 456 sizeof (tavor_devmap_track_t), KM_SLEEP);
457 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
458 457 dvm_track->tdt_offset = off;
459 458 dvm_track->tdt_state = state;
460 459 dvm_track->tdt_refcnt = 1;
461 460 mutex_init(&dvm_track->tdt_lock, NULL, MUTEX_DRIVER,
462 461 DDI_INTR_PRI(state->ts_intrmsi_pri));
463 462
464 463 /*
465 464 * Depending of the type of resource that has been mapped out, we
466 465 * need to update the QP or CQ handle to reflect that it has, in
467 466 * fact, been mapped. This allows the driver code which frees a QP
468 467 * or a CQ to know whether it is appropriate to do a
469 468 * devmap_devmem_remap() to invalidate the userland mapping for the
470 469 * corresponding queue's memory.
471 470 */
472 471 if (type == MLNX_UMAP_CQMEM_RSRC) {
473 472
474 473 /* Use "key" (CQ number) to do fast lookup of CQ handle */
475 474 cq = tavor_cqhdl_from_cqnum(state, key);
476 475
477 476 /*
478 477 * Update the handle to the userland mapping. Note: If
479 478 * the CQ already has a valid userland mapping, then stop
480 479 * and return failure.
481 480 */
482 481 mutex_enter(&cq->cq_lock);
483 482 if (cq->cq_umap_dhp == NULL) {
484 483 cq->cq_umap_dhp = dhp;
485 484 dvm_track->tdt_size = cq->cq_cqinfo.qa_size;
486 485 mutex_exit(&cq->cq_lock);
487 486 } else {
488 487 mutex_exit(&cq->cq_lock);
489 488 goto umem_map_fail;
490 489 }
491 490
492 491 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
493 492
494 493 /* Use "key" (QP number) to do fast lookup of QP handle */
495 494 qp = tavor_qphdl_from_qpnum(state, key);
496 495
497 496 /*
498 497 * Update the handle to the userland mapping. Note: If
499 498 * the CQ already has a valid userland mapping, then stop
500 499 * and return failure.
501 500 */
502 501 mutex_enter(&qp->qp_lock);
503 502 if (qp->qp_umap_dhp == NULL) {
504 503 qp->qp_umap_dhp = dhp;
505 504 dvm_track->tdt_size = qp->qp_wqinfo.qa_size;
506 505 mutex_exit(&qp->qp_lock);
507 506 } else {
508 507 mutex_exit(&qp->qp_lock);
509 508 goto umem_map_fail;
510 509 }
511 510
512 511 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
513 512
514 513 /* Use "key" (SRQ number) to do fast lookup on SRQ handle */
515 514 srq = tavor_srqhdl_from_srqnum(state, key);
516 515
517 516 /*
518 517 * Update the handle to the userland mapping. Note: If the
519 518 * SRQ already has a valid userland mapping, then stop and
520 519 * return failure.
521 520 */
522 521 mutex_enter(&srq->srq_lock);
523 522 if (srq->srq_umap_dhp == NULL) {
524 523 srq->srq_umap_dhp = dhp;
525 524 dvm_track->tdt_size = srq->srq_wqinfo.qa_size;
526 525 mutex_exit(&srq->srq_lock);
527 526 } else {
528 527 mutex_exit(&srq->srq_lock);
529 528 goto umem_map_fail;
530 529 }
531 530 }
532 531
533 532 /*
534 533 * Pass the private "Tavor devmap tracking structure" back. This
535 534 * pointer will be returned in subsequent "unmap" callbacks.
536 535 */
537 536 *pvtp = dvm_track;
538 537
539 538 TAVOR_TNF_EXIT(tavor_devmap_umem_map);
540 539 return (DDI_SUCCESS);
541 540
542 541 umem_map_fail:
543 542 mutex_destroy(&dvm_track->tdt_lock);
544 543 kmem_free(dvm_track, sizeof (tavor_devmap_track_t));
545 544 TAVOR_TNF_EXIT(tavor_devmap_umem_map);
546 545 return (DDI_FAILURE);
547 546 }
548 547
549 548
550 549 /*
551 550 * tavor_devmap_umem_dup()
552 551 * Context: Can be called from kernel context.
553 552 */
554 553 /* ARGSUSED */
555 554 static int
556 555 tavor_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp,
557 556 void **new_pvtp)
558 557 {
559 558 tavor_state_t *state;
560 559 tavor_devmap_track_t *dvm_track, *new_dvm_track;
|
↓ open down ↓ |
93 lines elided |
↑ open up ↑ |
561 560 uint_t maxprot;
562 561 int status;
563 562
564 563 TAVOR_TNF_ENTER(tavor_devmap_umem_dup);
565 564
566 565 /*
567 566 * Extract the Tavor softstate pointer from "Tavor devmap tracking
568 567 * structure" (in "pvtp").
569 568 */
570 569 dvm_track = (tavor_devmap_track_t *)pvtp;
571 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
572 570 state = dvm_track->tdt_state;
573 571
574 572 /*
575 573 * Since this devmap_dup() entry point is generally called
576 574 * when a process does fork(2), it is incumbent upon the driver
577 575 * to insure that the child does not inherit a valid copy of
578 576 * the parent's QP or CQ resource. This is accomplished by using
579 577 * devmap_devmem_remap() to invalidate the child's mapping to the
580 578 * kernel memory.
581 579 */
582 580 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
583 581 status = devmap_devmem_remap(new_dhp, state->ts_dip, 0, 0,
584 582 dvm_track->tdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
585 583 if (status != DDI_SUCCESS) {
586 584 TAVOR_WARNING(state, "failed in tavor_devmap_umem_dup()");
587 585 TAVOR_TNF_EXIT(tavor_devmap_umem_dup);
588 586 return (status);
589 587 }
590 588
|
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
591 589 /*
592 590 * Allocate a new entry to track the subsequent unmapping
593 591 * (specifically, all partial unmappings) of the child's newly
594 592 * invalidated resource. Note: Setting the "tdt_size" field to
595 593 * zero here is an indication to the devmap_unmap() entry point
596 594 * that this mapping is invalid, and that its subsequent unmapping
597 595 * should not affect any of the parent's CQ or QP resources.
598 596 */
599 597 new_dvm_track = (tavor_devmap_track_t *)kmem_zalloc(
600 598 sizeof (tavor_devmap_track_t), KM_SLEEP);
601 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
602 599 new_dvm_track->tdt_offset = 0;
603 600 new_dvm_track->tdt_state = state;
604 601 new_dvm_track->tdt_refcnt = 1;
605 602 new_dvm_track->tdt_size = 0;
606 603 mutex_init(&new_dvm_track->tdt_lock, NULL, MUTEX_DRIVER,
607 604 DDI_INTR_PRI(state->ts_intrmsi_pri));
608 605 *new_pvtp = new_dvm_track;
609 606
610 607 TAVOR_TNF_EXIT(tavor_devmap_umem_dup);
611 608 return (DDI_SUCCESS);
612 609 }
613 610
614 611
615 612 /*
616 613 * tavor_devmap_umem_unmap()
617 614 * Context: Can be called from kernel context.
618 615 */
619 616 /* ARGSUSED */
620 617 static void
621 618 tavor_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
622 619 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
623 620 devmap_cookie_t new_dhp2, void **pvtp2)
624 621 {
625 622 tavor_state_t *state;
626 623 tavor_rsrc_t *rsrcp;
627 624 tavor_devmap_track_t *dvm_track;
628 625 tavor_cqhdl_t cq;
629 626 tavor_qphdl_t qp;
630 627 tavor_srqhdl_t srq;
631 628 uint64_t key, value;
632 629 uint_t type;
|
↓ open down ↓ |
21 lines elided |
↑ open up ↑ |
633 630 uint_t size;
634 631 int status;
635 632
636 633 TAVOR_TNF_ENTER(tavor_devmap_umem_unmap);
637 634
638 635 /*
639 636 * Extract the Tavor softstate pointer from "Tavor devmap tracking
640 637 * structure" (in "pvtp").
641 638 */
642 639 dvm_track = (tavor_devmap_track_t *)pvtp;
643 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
644 640 state = dvm_track->tdt_state;
645 641
646 642 /*
647 643 * Extract the "offset" from the "Tavor devmap tracking structure".
648 644 * Note: The input argument "off" is ignored here because the
649 645 * Tavor mapping interfaces define a very specific meaning to
650 646 * each "logical offset". Also extract the "key" and "type" encoded
651 647 * in the logical offset.
652 648 */
653 649 key = dvm_track->tdt_offset >> PAGESHIFT;
654 650 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
655 651 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
656 652
657 653 /*
658 654 * Extract the "size" of the mapping. If this size is determined
659 655 * to be zero, then it is an indication of a previously invalidated
660 656 * mapping, and no CQ or QP resources should be affected.
661 657 */
662 658 size = dvm_track->tdt_size;
663 659
664 660 /*
665 661 * If only the "middle portion of a given mapping is being unmapped,
666 662 * then we are effectively creating one new piece of mapped memory.
667 663 * (Original region is divided into three pieces of which the middle
668 664 * piece is being removed. This leaves two pieces. Since we started
669 665 * with one piece and now have two pieces, we need to increment the
670 666 * counter in the "Tavor devmap tracking structure".
671 667 *
672 668 * If, however, the whole mapped region is being unmapped, then we
673 669 * have started with one region which we are completely removing.
674 670 * In this case, we need to decrement the counter in the "Tavor
675 671 * devmap tracking structure".
676 672 *
677 673 * In each of the remaining cases, we will have started with one
678 674 * mapped region and ended with one (different) region. So no counter
679 675 * modification is necessary.
680 676 */
681 677 mutex_enter(&dvm_track->tdt_lock);
682 678 if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
683 679 dvm_track->tdt_refcnt--;
684 680 } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
685 681 dvm_track->tdt_refcnt++;
686 682 }
687 683 mutex_exit(&dvm_track->tdt_lock);
688 684
689 685 /*
690 686 * For each of the cases where the region is being divided, then we
691 687 * need to pass back the "Tavor devmap tracking structure". This way
692 688 * we get it back when each of the remaining pieces is subsequently
693 689 * unmapped.
694 690 */
695 691 if (new_dhp1 != NULL) {
696 692 *pvtp1 = pvtp;
697 693 }
698 694 if (new_dhp2 != NULL) {
699 695 *pvtp2 = pvtp;
700 696 }
701 697
702 698 /*
703 699 * If the "Tavor devmap tracking structure" is no longer being
704 700 * referenced, then free it up. Otherwise, return.
705 701 */
706 702 if (dvm_track->tdt_refcnt == 0) {
707 703 mutex_destroy(&dvm_track->tdt_lock);
708 704 kmem_free(dvm_track, sizeof (tavor_devmap_track_t));
709 705
710 706 /*
711 707 * If the mapping was invalid (see explanation above), then
712 708 * no further processing is necessary.
713 709 */
714 710 if (size == 0) {
715 711 TAVOR_TNF_EXIT(tavor_devmap_umem_unmap);
716 712 return;
717 713 }
718 714 } else {
719 715 TAVOR_TNF_EXIT(tavor_devmap_umem_unmap);
720 716 return;
721 717 }
722 718
723 719 /*
724 720 * Now that we can guarantee that the user memory is fully unmapped,
725 721 * we can use the "key" and "type" values to try to find the entry
726 722 * in the "userland resources database". If it's found, then it
727 723 * indicates that the queue memory (CQ or QP) has not yet been freed.
728 724 * In this case, we update the corresponding CQ or QP handle to
729 725 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
730 726 * If it's _not_ found, then it indicates that the CQ or QP memory
731 727 * was, in fact, freed before it was unmapped (thus requiring a
732 728 * previous invalidation by remapping - which will already have
733 729 * been done in the free routine).
734 730 */
735 731 status = tavor_umap_db_find(state->ts_instance, key, type, &value,
736 732 0, NULL);
737 733 if (status == DDI_SUCCESS) {
738 734 /*
739 735 * Depending on the type of the mapped resource (CQ or QP),
740 736 * update handle to indicate that no invalidation remapping
741 737 * will be necessary.
742 738 */
743 739 if (type == MLNX_UMAP_CQMEM_RSRC) {
744 740
745 741 /* Use "value" to convert to CQ handle */
746 742 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
747 743 cq = (tavor_cqhdl_t)rsrcp->tr_addr;
748 744
749 745 /*
750 746 * Invalidate the handle to the userland mapping.
751 747 * Note: We must ensure that the mapping being
752 748 * unmapped here is the current one for the CQ. It
753 749 * is possible that it might not be if this CQ has
754 750 * been resized and the previous CQ memory has not
755 751 * yet been unmapped. But in that case, because of
756 752 * the devmap_devmem_remap(), there is no longer any
757 753 * association between the mapping and the real CQ
758 754 * kernel memory.
759 755 */
760 756 mutex_enter(&cq->cq_lock);
761 757 if (cq->cq_umap_dhp == dhp) {
762 758 cq->cq_umap_dhp = (devmap_cookie_t)NULL;
763 759 }
764 760 mutex_exit(&cq->cq_lock);
765 761
766 762 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
767 763
768 764 /* Use "value" to convert to QP handle */
769 765 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
770 766 qp = (tavor_qphdl_t)rsrcp->tr_addr;
771 767
772 768 /*
773 769 * Invalidate the handle to the userland mapping.
774 770 * Note: we ensure that the mapping being unmapped
775 771 * here is the current one for the QP. This is
776 772 * more of a sanity check here since, unlike CQs
777 773 * (above) we do not support resize of QPs.
778 774 */
779 775 mutex_enter(&qp->qp_lock);
780 776 if (qp->qp_umap_dhp == dhp) {
781 777 qp->qp_umap_dhp = (devmap_cookie_t)NULL;
782 778 }
783 779 mutex_exit(&qp->qp_lock);
784 780
785 781 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
786 782
787 783 /* Use "value" to convert to SRQ handle */
788 784 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
789 785 srq = (tavor_srqhdl_t)rsrcp->tr_addr;
790 786
791 787 /*
792 788 * Invalidate the handle to the userland mapping.
793 789 * Note: we ensure that the mapping being unmapped
794 790 * here is the current one for the QP. This is
795 791 * more of a sanity check here since, unlike CQs
796 792 * (above) we do not support resize of QPs.
797 793 */
798 794 mutex_enter(&srq->srq_lock);
799 795 if (srq->srq_umap_dhp == dhp) {
800 796 srq->srq_umap_dhp = (devmap_cookie_t)NULL;
801 797 }
802 798 mutex_exit(&srq->srq_lock);
803 799 }
804 800 }
805 801
806 802 TAVOR_TNF_EXIT(tavor_devmap_umem_unmap);
807 803 }
808 804
809 805
810 806 /*
811 807 * tavor_devmap_devmem_map()
812 808 * Context: Can be called from kernel context.
813 809 */
814 810 /* ARGSUSED */
815 811 static int
816 812 tavor_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
817 813 offset_t off, size_t len, void **pvtp)
818 814 {
819 815 tavor_state_t *state;
820 816 tavor_devmap_track_t *dvm_track;
821 817 minor_t instance;
822 818
823 819 TAVOR_TNF_ENTER(tavor_devmap_devmem_map);
824 820
825 821 /* Get Tavor softstate structure from instance */
826 822 instance = TAVOR_DEV_INSTANCE(dev);
827 823 state = ddi_get_soft_state(tavor_statep, instance);
828 824 if (state == NULL) {
829 825 TNF_PROBE_0(tavor_devmap_devmem_map_gss_fail, TAVOR_TNF_ERROR,
830 826 "");
831 827 TAVOR_TNF_EXIT(tavor_devmap_devmem_map);
832 828 return (ENXIO);
833 829 }
834 830
|
↓ open down ↓ |
181 lines elided |
↑ open up ↑ |
835 831 /*
836 832 * Allocate an entry to track the mapping and unmapping of this
837 833 * resource. Note: We don't need to initialize the "refcnt" or
838 834 * "offset" fields here, nor do we need to initialize the mutex
839 835 * used with the "refcnt". Since UAR pages are single pages, they
840 836 * are not subject to "partial" unmappings. This makes these other
841 837 * fields unnecessary.
842 838 */
843 839 dvm_track = (tavor_devmap_track_t *)kmem_zalloc(
844 840 sizeof (tavor_devmap_track_t), KM_SLEEP);
845 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
846 841 dvm_track->tdt_state = state;
847 842 dvm_track->tdt_size = PAGESIZE;
848 843
849 844 /*
850 845 * Pass the private "Tavor devmap tracking structure" back. This
851 846 * pointer will be returned in a subsequent "unmap" callback.
852 847 */
853 848 *pvtp = dvm_track;
854 849
855 850 TAVOR_TNF_EXIT(tavor_devmap_devmem_map);
856 851 return (DDI_SUCCESS);
857 852 }
858 853
859 854
860 855 /*
861 856 * tavor_devmap_devmem_dup()
862 857 * Context: Can be called from kernel context.
863 858 */
864 859 /* ARGSUSED */
865 860 static int
866 861 tavor_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
867 862 devmap_cookie_t new_dhp, void **new_pvtp)
868 863 {
869 864 tavor_state_t *state;
870 865 tavor_devmap_track_t *dvm_track;
871 866 uint_t maxprot;
872 867 int status;
873 868
874 869 TAVOR_TNF_ENTER(tavor_devmap_devmem_dup);
875 870
876 871 /*
877 872 * Extract the Tavor softstate pointer from "Tavor devmap tracking
878 873 * structure" (in "pvtp"). Note: If the tracking structure is NULL
|
↓ open down ↓ |
23 lines elided |
↑ open up ↑ |
879 874 * here, it means that the mapping corresponds to an invalid mapping.
880 875 * In this case, it can be safely ignored ("new_pvtp" set to NULL).
881 876 */
882 877 dvm_track = (tavor_devmap_track_t *)pvtp;
883 878 if (dvm_track == NULL) {
884 879 *new_pvtp = NULL;
885 880 TAVOR_TNF_EXIT(tavor_devmap_devmem_dup);
886 881 return (DDI_SUCCESS);
887 882 }
888 883
889 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
890 884 state = dvm_track->tdt_state;
891 885
892 886 /*
893 887 * Since this devmap_dup() entry point is generally called
894 888 * when a process does fork(2), it is incumbent upon the driver
895 889 * to insure that the child does not inherit a valid copy of
896 890 * the parent's resource. This is accomplished by using
897 891 * devmap_devmem_remap() to invalidate the child's mapping to the
898 892 * kernel memory.
899 893 */
900 894 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
901 895 status = devmap_devmem_remap(new_dhp, state->ts_dip, 0, 0,
902 896 dvm_track->tdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
903 897 if (status != DDI_SUCCESS) {
904 898 TAVOR_WARNING(state, "failed in tavor_devmap_devmem_dup()");
905 899 TAVOR_TNF_EXIT(tavor_devmap_devmem_dup);
906 900 return (status);
907 901 }
908 902
909 903 /*
910 904 * Since the region is invalid, there is no need for us to
911 905 * allocate and continue to track an additional "Tavor devmap
912 906 * tracking structure". Instead we return NULL here, which is an
913 907 * indication to the devmap_unmap() entry point that this entry
914 908 * can be safely ignored.
915 909 */
916 910 *new_pvtp = NULL;
917 911
918 912 TAVOR_TNF_EXIT(tavor_devmap_devmem_dup);
919 913 return (DDI_SUCCESS);
920 914 }
921 915
922 916
923 917 /*
924 918 * tavor_devmap_devmem_unmap()
925 919 * Context: Can be called from kernel context.
926 920 */
927 921 /* ARGSUSED */
928 922 static void
929 923 tavor_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
930 924 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
931 925 devmap_cookie_t new_dhp2, void **pvtp2)
932 926 {
933 927 tavor_devmap_track_t *dvm_track;
934 928
|
↓ open down ↓ |
35 lines elided |
↑ open up ↑ |
935 929 TAVOR_TNF_ENTER(tavor_devmap_devmem_unmap);
936 930
937 931 /*
938 932 * Free up the "Tavor devmap tracking structure" (in "pvtp").
939 933 * There cannot be "partial" unmappings here because all UAR pages
940 934 * are single pages. Note: If the tracking structure is NULL here,
941 935 * it means that the mapping corresponds to an invalid mapping. In
942 936 * this case, it can be safely ignored.
943 937 */
944 938 dvm_track = (tavor_devmap_track_t *)pvtp;
945 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
946 939 if (dvm_track == NULL) {
947 940 TAVOR_TNF_EXIT(tavor_devmap_devmem_unmap);
948 941 return;
949 942 }
950 943
951 944 kmem_free(dvm_track, sizeof (tavor_devmap_track_t));
952 945 TAVOR_TNF_EXIT(tavor_devmap_devmem_unmap);
953 946 }
954 947
955 948
956 949 /*
957 950 * tavor_umap_ci_data_in()
958 951 * Context: Can be called from user or kernel context.
959 952 */
960 953 /* ARGSUSED */
961 954 ibt_status_t
962 955 tavor_umap_ci_data_in(tavor_state_t *state, ibt_ci_data_flags_t flags,
963 956 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
964 957 {
965 958 int status;
966 959
967 960 TAVOR_TNF_ENTER(tavor_umap_ci_data_in);
968 961
969 962 /*
970 963 * Depending on the type of object about which additional information
971 964 * is being provided (currently only MR is supported), we call the
972 965 * appropriate resource-specific function.
973 966 */
974 967 switch (object) {
975 968 case IBT_HDL_MR:
976 969 status = tavor_umap_mr_data_in((tavor_mrhdl_t)hdl,
977 970 (ibt_mr_data_in_t *)data_p, data_sz);
978 971 if (status != DDI_SUCCESS) {
979 972 TNF_PROBE_0(tavor_umap_mr_data_in_fail,
980 973 TAVOR_TNF_ERROR, "");
981 974 TAVOR_TNF_EXIT(tavor_umap_ci_data_in);
982 975 return (status);
983 976 }
984 977 break;
985 978
986 979 /*
987 980 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
988 981 * since the Tavor driver does not support these.
989 982 */
990 983 case IBT_HDL_HCA:
991 984 case IBT_HDL_QP:
992 985 case IBT_HDL_CQ:
993 986 case IBT_HDL_PD:
994 987 case IBT_HDL_MW:
995 988 case IBT_HDL_AH:
996 989 case IBT_HDL_SCHED:
997 990 case IBT_HDL_EEC:
998 991 case IBT_HDL_RDD:
999 992 case IBT_HDL_SRQ:
1000 993 TNF_PROBE_0(tavor_umap_ci_data_in_unsupp_type,
1001 994 TAVOR_TNF_ERROR, "");
1002 995 TAVOR_TNF_EXIT(tavor_umap_ci_data_in);
1003 996 return (IBT_NOT_SUPPORTED);
1004 997
1005 998 /*
1006 999 * Any other types are invalid.
1007 1000 */
1008 1001 default:
1009 1002 TNF_PROBE_0(tavor_umap_ci_data_in_invtype_fail,
1010 1003 TAVOR_TNF_ERROR, "");
1011 1004 TAVOR_TNF_EXIT(tavor_umap_ci_data_in);
1012 1005 return (IBT_INVALID_PARAM);
1013 1006 }
1014 1007
1015 1008 TAVOR_TNF_EXIT(tavor_umap_ci_data_in);
1016 1009 return (DDI_SUCCESS);
1017 1010 }
1018 1011
1019 1012
1020 1013 /*
1021 1014 * tavor_umap_mr_data_in()
1022 1015 * Context: Can be called from user or kernel context.
1023 1016 */
1024 1017 static ibt_status_t
1025 1018 tavor_umap_mr_data_in(tavor_mrhdl_t mr, ibt_mr_data_in_t *data,
1026 1019 size_t data_sz)
1027 1020 {
1028 1021 TAVOR_TNF_ENTER(tavor_umap_mr_data_in);
1029 1022
1030 1023 if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) {
1031 1024 TNF_PROBE_0(tavor_umap_mr_data_in_ver_fail,
1032 1025 TAVOR_TNF_ERROR, "");
1033 1026 TAVOR_TNF_EXIT(tavor_umap_mr_data_in);
1034 1027 return (IBT_NOT_SUPPORTED);
1035 1028 }
1036 1029
1037 1030 /* Check for valid MR handle pointer */
1038 1031 if (mr == NULL) {
1039 1032 TNF_PROBE_0(tavor_umap_mr_data_in_invmrhdl_fail,
1040 1033 TAVOR_TNF_ERROR, "");
1041 1034 TAVOR_TNF_EXIT(tavor_umap_mr_data_in);
|
↓ open down ↓ |
86 lines elided |
↑ open up ↑ |
1042 1035 return (IBT_MR_HDL_INVALID);
1043 1036 }
1044 1037
1045 1038 /* Check for valid MR input structure size */
1046 1039 if (data_sz < sizeof (ibt_mr_data_in_t)) {
1047 1040 TNF_PROBE_0(tavor_umap_mr_data_in_invdatasz_fail,
1048 1041 TAVOR_TNF_ERROR, "");
1049 1042 TAVOR_TNF_EXIT(tavor_umap_mr_data_in);
1050 1043 return (IBT_INSUFF_RESOURCE);
1051 1044 }
1052 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1053 1045
1054 1046 /*
1055 1047 * Ensure that the MR corresponds to userland memory and that it is
1056 1048 * a currently valid memory region as well.
1057 1049 */
1058 1050 mutex_enter(&mr->mr_lock);
1059 1051 if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) {
1060 1052 mutex_exit(&mr->mr_lock);
1061 1053 TNF_PROBE_0(tavor_umap_mr_data_in_invumem_fail,
1062 1054 TAVOR_TNF_ERROR, "");
1063 1055 TAVOR_TNF_EXIT(tavor_umap_mr_data_in);
1064 1056 return (IBT_MR_HDL_INVALID);
1065 1057 }
1066 1058
1067 1059 /*
1068 1060 * If it has passed all the above checks, then extract the callback
1069 1061 * function and argument from the input structure. Copy them into
1070 1062 * the MR handle. This function will be called only if the memory
1071 1063 * corresponding to the MR handle gets a umem_lockmemory() callback.
1072 1064 */
1073 1065 mr->mr_umem_cbfunc = data->mr_func;
1074 1066 mr->mr_umem_cbarg1 = data->mr_arg1;
1075 1067 mr->mr_umem_cbarg2 = data->mr_arg2;
1076 1068 mutex_exit(&mr->mr_lock);
1077 1069
1078 1070 TAVOR_TNF_EXIT(tavor_umap_cq_data_out);
1079 1071 return (DDI_SUCCESS);
1080 1072 }
1081 1073
1082 1074
1083 1075 /*
1084 1076 * tavor_umap_ci_data_out()
1085 1077 * Context: Can be called from user or kernel context.
1086 1078 */
1087 1079 /* ARGSUSED */
1088 1080 ibt_status_t
1089 1081 tavor_umap_ci_data_out(tavor_state_t *state, ibt_ci_data_flags_t flags,
1090 1082 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1091 1083 {
1092 1084 int status;
1093 1085
1094 1086 TAVOR_TNF_ENTER(tavor_umap_ci_data_out);
1095 1087
1096 1088 /*
1097 1089 * Depending on the type of object about which additional information
1098 1090 * is being requested (CQ or QP), we call the appropriate resource-
1099 1091 * specific mapping function.
1100 1092 */
1101 1093 switch (object) {
1102 1094 case IBT_HDL_CQ:
1103 1095 status = tavor_umap_cq_data_out((tavor_cqhdl_t)hdl,
1104 1096 (mlnx_umap_cq_data_out_t *)data_p, data_sz);
1105 1097 if (status != DDI_SUCCESS) {
1106 1098 TNF_PROBE_0(tavor_umap_cq_data_out_fail,
1107 1099 TAVOR_TNF_ERROR, "");
1108 1100 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1109 1101 return (status);
1110 1102 }
1111 1103 break;
1112 1104
1113 1105 case IBT_HDL_QP:
1114 1106 status = tavor_umap_qp_data_out((tavor_qphdl_t)hdl,
1115 1107 (mlnx_umap_qp_data_out_t *)data_p, data_sz);
1116 1108 if (status != DDI_SUCCESS) {
1117 1109 TNF_PROBE_0(tavor_umap_qp_data_out_fail,
1118 1110 TAVOR_TNF_ERROR, "");
1119 1111 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1120 1112 return (status);
1121 1113 }
1122 1114 break;
1123 1115
1124 1116 case IBT_HDL_SRQ:
1125 1117 status = tavor_umap_srq_data_out((tavor_srqhdl_t)hdl,
1126 1118 (mlnx_umap_srq_data_out_t *)data_p, data_sz);
1127 1119 if (status != DDI_SUCCESS) {
1128 1120 TNF_PROBE_0(tavor_umap_srq_data_out_fail,
1129 1121 TAVOR_TNF_ERROR, "");
1130 1122 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1131 1123 return (status);
1132 1124 }
1133 1125 break;
1134 1126
1135 1127 /*
1136 1128 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
1137 1129 * since the Tavor driver does not support these.
1138 1130 */
1139 1131 case IBT_HDL_PD:
1140 1132 status = tavor_umap_pd_data_out((tavor_pdhdl_t)hdl,
1141 1133 (mlnx_umap_pd_data_out_t *)data_p, data_sz);
1142 1134 if (status != DDI_SUCCESS) {
1143 1135 TNF_PROBE_0(tavor_umap_pd_data_out_fail,
1144 1136 TAVOR_TNF_ERROR, "");
1145 1137 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1146 1138 return (status);
1147 1139 }
1148 1140 break;
1149 1141
1150 1142 case IBT_HDL_HCA:
1151 1143 case IBT_HDL_MR:
1152 1144 case IBT_HDL_MW:
1153 1145 case IBT_HDL_AH:
1154 1146 case IBT_HDL_SCHED:
1155 1147 case IBT_HDL_EEC:
1156 1148 case IBT_HDL_RDD:
1157 1149 TNF_PROBE_0(tavor_umap_ci_data_out_unsupp_type,
1158 1150 TAVOR_TNF_ERROR, "");
1159 1151 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1160 1152 return (IBT_NOT_SUPPORTED);
1161 1153
1162 1154 /*
1163 1155 * Any other types are invalid.
1164 1156 */
1165 1157 default:
1166 1158 TNF_PROBE_0(tavor_umap_ci_data_out_invtype_fail,
1167 1159 TAVOR_TNF_ERROR, "");
1168 1160 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1169 1161 return (IBT_INVALID_PARAM);
1170 1162 }
1171 1163
1172 1164 TAVOR_TNF_EXIT(tavor_umap_ci_data_out);
1173 1165 return (DDI_SUCCESS);
1174 1166 }
1175 1167
1176 1168
1177 1169 /*
1178 1170 * tavor_umap_cq_data_out()
1179 1171 * Context: Can be called from user or kernel context.
1180 1172 */
1181 1173 static ibt_status_t
1182 1174 tavor_umap_cq_data_out(tavor_cqhdl_t cq, mlnx_umap_cq_data_out_t *data,
1183 1175 size_t data_sz)
1184 1176 {
1185 1177 TAVOR_TNF_ENTER(tavor_umap_cq_data_out);
1186 1178
1187 1179 /* Check for valid CQ handle pointer */
1188 1180 if (cq == NULL) {
1189 1181 TNF_PROBE_0(tavor_umap_cq_data_out_invcqhdl_fail,
1190 1182 TAVOR_TNF_ERROR, "");
1191 1183 TAVOR_TNF_EXIT(tavor_umap_cq_data_out);
|
↓ open down ↓ |
129 lines elided |
↑ open up ↑ |
1192 1184 return (IBT_CQ_HDL_INVALID);
1193 1185 }
1194 1186
1195 1187 /* Check for valid CQ mapping structure size */
1196 1188 if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) {
1197 1189 TNF_PROBE_0(tavor_umap_cq_data_out_invdatasz_fail,
1198 1190 TAVOR_TNF_ERROR, "");
1199 1191 TAVOR_TNF_EXIT(tavor_umap_cq_data_out);
1200 1192 return (IBT_INSUFF_RESOURCE);
1201 1193 }
1202 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1203 1194
1204 1195 /*
1205 1196 * If it has passed all the above checks, then fill in all the useful
1206 1197 * mapping information (including the mapping offset that will be
1207 1198 * passed back to the devmap() interface during a subsequent mmap()
1208 1199 * call.
1209 1200 *
1210 1201 * The "offset" for CQ mmap()'s looks like this:
1211 1202 * +----------------------------------------+--------+--------------+
1212 1203 * | CQ Number | 0x33 | Reserved (0) |
1213 1204 * +----------------------------------------+--------+--------------+
1214 1205 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1215 1206 *
1216 1207 * This returns information about the mapping offset, the length of
1217 1208 * the CQ memory, the CQ number (for use in later CQ doorbells), the
1218 1209 * number of CQEs the CQ memory can hold, and the size of each CQE.
1219 1210 */
1220 1211 data->mcq_rev = MLNX_UMAP_IF_VERSION;
1221 1212 data->mcq_mapoffset = ((((uint64_t)cq->cq_cqnum <<
1222 1213 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_CQMEM_RSRC) << PAGESHIFT);
1223 1214 data->mcq_maplen = cq->cq_cqinfo.qa_size;
1224 1215 data->mcq_cqnum = cq->cq_cqnum;
1225 1216 data->mcq_numcqe = cq->cq_bufsz;
1226 1217 data->mcq_cqesz = sizeof (tavor_hw_cqe_t);
1227 1218
1228 1219 TAVOR_TNF_EXIT(tavor_umap_cq_data_out);
1229 1220 return (DDI_SUCCESS);
1230 1221 }
1231 1222
1232 1223
1233 1224 /*
1234 1225 * tavor_umap_qp_data_out()
1235 1226 * Context: Can be called from user or kernel context.
1236 1227 */
1237 1228 static ibt_status_t
1238 1229 tavor_umap_qp_data_out(tavor_qphdl_t qp, mlnx_umap_qp_data_out_t *data,
1239 1230 size_t data_sz)
1240 1231 {
1241 1232 TAVOR_TNF_ENTER(tavor_umap_qp_data_out);
1242 1233
1243 1234 /* Check for valid QP handle pointer */
1244 1235 if (qp == NULL) {
1245 1236 TNF_PROBE_0(tavor_umap_qp_data_out_invqphdl_fail,
1246 1237 TAVOR_TNF_ERROR, "");
1247 1238 TAVOR_TNF_EXIT(tavor_umap_qp_data_out);
|
↓ open down ↓ |
35 lines elided |
↑ open up ↑ |
1248 1239 return (IBT_QP_HDL_INVALID);
1249 1240 }
1250 1241
1251 1242 /* Check for valid QP mapping structure size */
1252 1243 if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) {
1253 1244 TNF_PROBE_0(tavor_umap_qp_data_out_invdatasz_fail,
1254 1245 TAVOR_TNF_ERROR, "");
1255 1246 TAVOR_TNF_EXIT(tavor_umap_qp_data_out);
1256 1247 return (IBT_INSUFF_RESOURCE);
1257 1248 }
1258 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1259 1249
1260 1250 /*
1261 1251 * If it has passed all the checks, then fill in all the useful
1262 1252 * mapping information (including the mapping offset that will be
1263 1253 * passed back to the devmap() interface during a subsequent mmap()
1264 1254 * call.
1265 1255 *
1266 1256 * The "offset" for QP mmap()'s looks like this:
1267 1257 * +----------------------------------------+--------+--------------+
1268 1258 * | QP Number | 0x44 | Reserved (0) |
1269 1259 * +----------------------------------------+--------+--------------+
1270 1260 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1271 1261 *
1272 1262 * This returns information about the mapping offset, the length of
1273 1263 * the QP memory, and the QP number (for use in later send and recv
1274 1264 * doorbells). It also returns the following information for both
1275 1265 * the receive work queue and the send work queue, respectively: the
1276 1266 * offset (from the base mapped address) of the start of the given
1277 1267 * work queue, the 64-bit IB virtual address that corresponds to
1278 1268 * the base mapped address (needed for posting WQEs though the
1279 1269 * QP doorbells), the number of WQEs the given work queue can hold,
1280 1270 * and the size of each WQE for the given work queue.
1281 1271 */
1282 1272 data->mqp_rev = MLNX_UMAP_IF_VERSION;
1283 1273 data->mqp_mapoffset = ((((uint64_t)qp->qp_qpnum <<
1284 1274 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_QPMEM_RSRC) << PAGESHIFT);
1285 1275 data->mqp_maplen = qp->qp_wqinfo.qa_size;
1286 1276 data->mqp_qpnum = qp->qp_qpnum;
1287 1277
1288 1278 /*
1289 1279 * If this QP is associated with a shared receive queue (SRQ),
1290 1280 * then return invalid RecvQ parameters. Otherwise, return
1291 1281 * the proper parameter values.
1292 1282 */
1293 1283 if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
1294 1284 data->mqp_rq_off = (uint32_t)qp->qp_wqinfo.qa_size;
1295 1285 data->mqp_rq_desc_addr = (uint32_t)qp->qp_wqinfo.qa_size;
1296 1286 data->mqp_rq_numwqe = 0;
1297 1287 data->mqp_rq_wqesz = 0;
1298 1288 } else {
1299 1289 data->mqp_rq_off = (uintptr_t)qp->qp_rq_buf -
1300 1290 (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1301 1291 data->mqp_rq_desc_addr = (uint32_t)((uintptr_t)qp->qp_rq_buf -
1302 1292 qp->qp_desc_off);
1303 1293 data->mqp_rq_numwqe = qp->qp_rq_bufsz;
1304 1294 data->mqp_rq_wqesz = (1 << qp->qp_rq_log_wqesz);
1305 1295 }
1306 1296 data->mqp_sq_off = (uintptr_t)qp->qp_sq_buf -
1307 1297 (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1308 1298 data->mqp_sq_desc_addr = (uint32_t)((uintptr_t)qp->qp_sq_buf -
1309 1299 qp->qp_desc_off);
1310 1300 data->mqp_sq_numwqe = qp->qp_sq_bufsz;
1311 1301 data->mqp_sq_wqesz = (1 << qp->qp_sq_log_wqesz);
1312 1302
1313 1303 TAVOR_TNF_EXIT(tavor_umap_qp_data_out);
1314 1304 return (DDI_SUCCESS);
1315 1305 }
1316 1306
1317 1307
1318 1308 /*
1319 1309 * tavor_umap_srq_data_out()
1320 1310 * Context: Can be called from user or kernel context.
1321 1311 */
1322 1312 static ibt_status_t
1323 1313 tavor_umap_srq_data_out(tavor_srqhdl_t srq, mlnx_umap_srq_data_out_t *data,
1324 1314 size_t data_sz)
1325 1315 {
1326 1316 TAVOR_TNF_ENTER(tavor_umap_srq_data_out);
1327 1317
1328 1318 /* Check for valid SRQ handle pointer */
1329 1319 if (srq == NULL) {
1330 1320 TNF_PROBE_0(tavor_umap_srq_data_out_invsrqhdl_fail,
1331 1321 TAVOR_TNF_ERROR, "");
1332 1322 TAVOR_TNF_EXIT(tavor_umap_srq_data_out);
|
↓ open down ↓ |
64 lines elided |
↑ open up ↑ |
1333 1323 return (IBT_SRQ_HDL_INVALID);
1334 1324 }
1335 1325
1336 1326 /* Check for valid SRQ mapping structure size */
1337 1327 if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) {
1338 1328 TNF_PROBE_0(tavor_umap_srq_data_out_invdatasz_fail,
1339 1329 TAVOR_TNF_ERROR, "");
1340 1330 TAVOR_TNF_EXIT(tavor_umap_srq_data_out);
1341 1331 return (IBT_INSUFF_RESOURCE);
1342 1332 }
1343 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1344 1333
1345 1334 /*
1346 1335 * If it has passed all the checks, then fill in all the useful
1347 1336 * mapping information (including the mapping offset that will be
1348 1337 * passed back to the devmap() interface during a subsequent mmap()
1349 1338 * call.
1350 1339 *
1351 1340 * The "offset" for SRQ mmap()'s looks like this:
1352 1341 * +----------------------------------------+--------+--------------+
1353 1342 * | SRQ Number | 0x66 | Reserved (0) |
1354 1343 * +----------------------------------------+--------+--------------+
1355 1344 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1356 1345 *
1357 1346 * This returns information about the mapping offset, the length of the
1358 1347 * SRQ memory, and the SRQ number (for use in later send and recv
1359 1348 * doorbells). It also returns the following information for the
1360 1349 * shared receive queue: the offset (from the base mapped address) of
1361 1350 * the start of the given work queue, the 64-bit IB virtual address
1362 1351 * that corresponds to the base mapped address (needed for posting WQEs
1363 1352 * though the QP doorbells), the number of WQEs the given work queue
1364 1353 * can hold, and the size of each WQE for the given work queue.
1365 1354 */
1366 1355 data->msrq_rev = MLNX_UMAP_IF_VERSION;
1367 1356 data->msrq_mapoffset = ((((uint64_t)srq->srq_srqnum <<
1368 1357 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_SRQMEM_RSRC) << PAGESHIFT);
1369 1358 data->msrq_maplen = srq->srq_wqinfo.qa_size;
1370 1359 data->msrq_srqnum = srq->srq_srqnum;
1371 1360
1372 1361 data->msrq_desc_addr = (uint32_t)((uintptr_t)srq->srq_wq_buf -
1373 1362 srq->srq_desc_off);
1374 1363 data->msrq_numwqe = srq->srq_wq_bufsz;
1375 1364 data->msrq_wqesz = (1 << srq->srq_wq_log_wqesz);
1376 1365
1377 1366 TAVOR_TNF_EXIT(tavor_umap_srq_data_out);
1378 1367 return (DDI_SUCCESS);
1379 1368 }
1380 1369
1381 1370 /*
1382 1371 * tavor_umap_pd_data_out()
1383 1372 * Context: Can be called from user or kernel context.
1384 1373 */
1385 1374 static ibt_status_t
1386 1375 tavor_umap_pd_data_out(tavor_pdhdl_t pd, mlnx_umap_pd_data_out_t *data,
1387 1376 size_t data_sz)
1388 1377 {
1389 1378 TAVOR_TNF_ENTER(tavor_umap_pd_data_out);
1390 1379
1391 1380 /* Check for valid PD handle pointer */
1392 1381 if (pd == NULL) {
1393 1382 TNF_PROBE_0(tavor_umap_pd_data_out_invpdhdl_fail,
1394 1383 TAVOR_TNF_ERROR, "");
1395 1384 TAVOR_TNF_EXIT(tavor_umap_pd_data_out);
|
↓ open down ↓ |
42 lines elided |
↑ open up ↑ |
1396 1385 return (IBT_PD_HDL_INVALID);
1397 1386 }
1398 1387
1399 1388 /* Check for valid PD mapping structure size */
1400 1389 if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) {
1401 1390 TNF_PROBE_0(tavor_umap_pd_data_out_invdatasz_fail,
1402 1391 TAVOR_TNF_ERROR, "");
1403 1392 TAVOR_TNF_EXIT(tavor_umap_pd_data_out);
1404 1393 return (IBT_INSUFF_RESOURCE);
1405 1394 }
1406 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1407 1395
1408 1396 /*
1409 1397 * If it has passed all the checks, then fill the PD table index
1410 1398 * (the PD table allocated index for the PD pd_pdnum)
1411 1399 */
1412 1400 data->mpd_rev = MLNX_UMAP_IF_VERSION;
1413 1401 data->mpd_pdnum = pd->pd_pdnum;
1414 1402
1415 1403 TAVOR_TNF_EXIT(tavor_umap_pd_data_out);
1416 1404 return (DDI_SUCCESS);
1417 1405 }
1418 1406
1419 1407 /*
1420 1408 * tavor_umap_db_init()
1421 1409 * Context: Only called from attach() path context
1422 1410 */
1423 1411 void
1424 1412 tavor_umap_db_init(void)
1425 1413 {
1426 1414 TAVOR_TNF_ENTER(tavor_umap_db_init);
1427 1415
1428 1416 /*
1429 1417 * Initialize the lock used by the Tavor "userland resources database"
1430 1418 * This is used to ensure atomic access to add, remove, and find
1431 1419 * entries in the database.
1432 1420 */
1433 1421 mutex_init(&tavor_userland_rsrc_db.tdl_umapdb_lock, NULL,
1434 1422 MUTEX_DRIVER, NULL);
1435 1423
1436 1424 /*
1437 1425 * Initialize the AVL tree used for the "userland resources
1438 1426 * database". Using an AVL tree here provides the ability to
1439 1427 * scale the database size to large numbers of resources. The
1440 1428 * entries in the tree are "tavor_umap_db_entry_t".
1441 1429 * The tree is searched with the help of the
1442 1430 * tavor_umap_db_compare() routine.
1443 1431 */
1444 1432 avl_create(&tavor_userland_rsrc_db.tdl_umapdb_avl,
1445 1433 tavor_umap_db_compare, sizeof (tavor_umap_db_entry_t),
1446 1434 offsetof(tavor_umap_db_entry_t, tdbe_avlnode));
1447 1435
1448 1436 TAVOR_TNF_EXIT(tavor_umap_db_init);
1449 1437 }
1450 1438
1451 1439
1452 1440 /*
1453 1441 * tavor_umap_db_fini()
1454 1442 * Context: Only called from attach() and/or detach() path contexts
1455 1443 */
1456 1444 void
1457 1445 tavor_umap_db_fini(void)
1458 1446 {
1459 1447 TAVOR_TNF_ENTER(tavor_umap_db_fini);
1460 1448
1461 1449 /* Destroy the AVL tree for the "userland resources database" */
1462 1450 avl_destroy(&tavor_userland_rsrc_db.tdl_umapdb_avl);
1463 1451
1464 1452 /* Destroy the lock for the "userland resources database" */
1465 1453 mutex_destroy(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1466 1454
1467 1455 TAVOR_TNF_EXIT(tavor_umap_db_fini);
1468 1456 }
1469 1457
1470 1458
1471 1459 /*
1472 1460 * tavor_umap_db_alloc()
1473 1461 * Context: Can be called from user or kernel context.
1474 1462 */
1475 1463 tavor_umap_db_entry_t *
1476 1464 tavor_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value)
1477 1465 {
1478 1466 tavor_umap_db_entry_t *umapdb;
|
↓ open down ↓ |
62 lines elided |
↑ open up ↑ |
1479 1467
1480 1468 TAVOR_TNF_ENTER(tavor_umap_db_alloc);
1481 1469
1482 1470 /* Allocate an entry to add to the "userland resources database" */
1483 1471 umapdb = kmem_zalloc(sizeof (tavor_umap_db_entry_t), KM_NOSLEEP);
1484 1472 if (umapdb == NULL) {
1485 1473 TNF_PROBE_0(tavor_umap_db_alloc_kmz_fail, TAVOR_TNF_ERROR, "");
1486 1474 TAVOR_TNF_EXIT(tavor_umap_db_alloc);
1487 1475 return (NULL);
1488 1476 }
1489 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1490 1477
1491 1478 /* Fill in the fields in the database entry */
1492 1479 umapdb->tdbe_common.tdb_instance = instance;
1493 1480 umapdb->tdbe_common.tdb_type = type;
1494 1481 umapdb->tdbe_common.tdb_key = key;
1495 1482 umapdb->tdbe_common.tdb_value = value;
1496 1483
1497 1484 TAVOR_TNF_EXIT(tavor_umap_db_alloc);
1498 1485 return (umapdb);
1499 1486 }
1500 1487
1501 1488
1502 1489 /*
1503 1490 * tavor_umap_db_free()
1504 1491 * Context: Can be called from user or kernel context.
1505 1492 */
1506 1493 void
1507 1494 tavor_umap_db_free(tavor_umap_db_entry_t *umapdb)
1508 1495 {
1509 1496 TAVOR_TNF_ENTER(tavor_umap_db_free);
1510 1497
1511 1498 /* Free the database entry */
1512 1499 kmem_free(umapdb, sizeof (tavor_umap_db_entry_t));
1513 1500
1514 1501 TAVOR_TNF_EXIT(tavor_umap_db_free);
1515 1502 }
1516 1503
1517 1504
1518 1505 /*
1519 1506 * tavor_umap_db_add()
1520 1507 * Context: Can be called from user or kernel context.
1521 1508 */
1522 1509 void
1523 1510 tavor_umap_db_add(tavor_umap_db_entry_t *umapdb)
1524 1511 {
1525 1512 TAVOR_TNF_ENTER(tavor_umap_db_add);
1526 1513
1527 1514 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1528 1515 tavor_umap_db_add_nolock(umapdb);
1529 1516 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1530 1517
1531 1518 TAVOR_TNF_EXIT(tavor_umap_db_add);
1532 1519 }
1533 1520
1534 1521
1535 1522 /*
1536 1523 * tavor_umap_db_add_nolock()
1537 1524 * Context: Can be called from user or kernel context.
1538 1525 */
|
↓ open down ↓ |
39 lines elided |
↑ open up ↑ |
1539 1526 void
1540 1527 tavor_umap_db_add_nolock(tavor_umap_db_entry_t *umapdb)
1541 1528 {
1542 1529 tavor_umap_db_query_t query;
1543 1530 avl_index_t where;
1544 1531
1545 1532 TAVOR_TNF_ENTER(tavor_umap_db_add_nolock);
1546 1533
1547 1534 ASSERT(MUTEX_HELD(&tavor_userland_rsrc_db.tdl_umapdb_lock));
1548 1535
1549 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1550 -
1551 1536 /*
1552 1537 * Copy the common portion of the "to-be-added" database entry
1553 1538 * into the "tavor_umap_db_query_t" structure. We use this structure
1554 1539 * (with no flags set) to find the appropriate location in the
1555 1540 * "userland resources database" for the new entry to be added.
1556 1541 *
1557 1542 * Note: we expect that this entry should not be found in the
1558 1543 * database (unless something bad has happened).
1559 1544 */
1560 1545 query.tqdb_common = umapdb->tdbe_common;
1561 1546 query.tqdb_flags = 0;
1562 1547 (void) avl_find(&tavor_userland_rsrc_db.tdl_umapdb_avl, &query,
1563 1548 &where);
1564 1549
1565 1550 /*
1566 1551 * Now, using the "where" field from the avl_find() operation
1567 1552 * above, we will insert the new database entry ("umapdb").
1568 1553 */
1569 1554 avl_insert(&tavor_userland_rsrc_db.tdl_umapdb_avl, umapdb,
1570 1555 where);
1571 1556
1572 1557 TAVOR_TNF_EXIT(tavor_umap_db_add_nolock);
1573 1558 }
1574 1559
1575 1560
1576 1561 /*
1577 1562 * tavor_umap_db_find()
1578 1563 * Context: Can be called from user or kernel context.
1579 1564 */
1580 1565 int
1581 1566 tavor_umap_db_find(uint_t instance, uint64_t key, uint_t type,
1582 1567 uint64_t *value, uint_t flag, tavor_umap_db_entry_t **umapdb)
1583 1568 {
1584 1569 int status;
1585 1570
1586 1571 TAVOR_TNF_ENTER(tavor_umap_db_find);
1587 1572
1588 1573 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1589 1574 status = tavor_umap_db_find_nolock(instance, key, type, value, flag,
1590 1575 umapdb);
1591 1576 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1592 1577
1593 1578 TAVOR_TNF_EXIT(tavor_umap_db_find);
1594 1579 return (status);
1595 1580 }
1596 1581
1597 1582
1598 1583 /*
1599 1584 * tavor_umap_db_find_nolock()
1600 1585 * Context: Can be called from user or kernel context.
1601 1586 */
1602 1587 int
1603 1588 tavor_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type,
1604 1589 uint64_t *value, uint_t flags, tavor_umap_db_entry_t **umapdb)
1605 1590 {
1606 1591 tavor_umap_db_query_t query;
1607 1592 tavor_umap_db_entry_t *entry;
1608 1593 avl_index_t where;
1609 1594
1610 1595 TAVOR_TNF_ENTER(tavor_umap_db_find_nolock);
1611 1596
1612 1597 ASSERT(MUTEX_HELD(&tavor_userland_rsrc_db.tdl_umapdb_lock));
1613 1598
1614 1599 /*
1615 1600 * Fill in key, type, instance, and flags values of the
1616 1601 * tavor_umap_db_query_t in preparation for the database
1617 1602 * lookup.
1618 1603 */
1619 1604 query.tqdb_flags = flags;
1620 1605 query.tqdb_common.tdb_key = key;
1621 1606 query.tqdb_common.tdb_type = type;
1622 1607 query.tqdb_common.tdb_instance = instance;
1623 1608
|
↓ open down ↓ |
63 lines elided |
↑ open up ↑ |
1624 1609 /*
1625 1610 * Perform the database query. If no entry is found, then
1626 1611 * return failure, else continue.
1627 1612 */
1628 1613 entry = (tavor_umap_db_entry_t *)avl_find(
1629 1614 &tavor_userland_rsrc_db.tdl_umapdb_avl, &query, &where);
1630 1615 if (entry == NULL) {
1631 1616 TAVOR_TNF_EXIT(tavor_umap_db_find_nolock);
1632 1617 return (DDI_FAILURE);
1633 1618 }
1634 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
1635 1619
1636 1620 /*
1637 1621 * If the flags argument specifies that the entry should
1638 1622 * be removed if found, then call avl_remove() to remove
1639 1623 * the entry from the database.
1640 1624 */
1641 1625 if (flags & TAVOR_UMAP_DB_REMOVE) {
1642 1626
1643 1627 avl_remove(&tavor_userland_rsrc_db.tdl_umapdb_avl, entry);
1644 1628
1645 1629 /*
1646 1630 * The database entry is returned with the expectation
1647 1631 * that the caller will use tavor_umap_db_free() to
1648 1632 * free the entry's memory. ASSERT that this is non-NULL.
1649 1633 * NULL pointer should never be passed for the
1650 1634 * TAVOR_UMAP_DB_REMOVE case.
1651 1635 */
1652 1636 ASSERT(umapdb != NULL);
1653 1637 }
1654 1638
1655 1639 /*
1656 1640 * If the caller would like visibility to the database entry
1657 1641 * (indicated through the use of a non-NULL "umapdb" argument),
1658 1642 * then fill it in.
1659 1643 */
1660 1644 if (umapdb != NULL) {
1661 1645 *umapdb = entry;
1662 1646 }
1663 1647
1664 1648 /* Extract value field from database entry and return success */
1665 1649 *value = entry->tdbe_common.tdb_value;
1666 1650
1667 1651 TAVOR_TNF_EXIT(tavor_umap_db_find_nolock);
1668 1652 return (DDI_SUCCESS);
1669 1653 }
1670 1654
1671 1655
1672 1656 /*
1673 1657 * tavor_umap_umemlock_cb()
1674 1658 * Context: Can be called from callback context.
1675 1659 */
1676 1660 void
1677 1661 tavor_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie)
1678 1662 {
1679 1663 tavor_umap_db_entry_t *umapdb;
1680 1664 tavor_state_t *state;
1681 1665 tavor_rsrc_t *rsrcp;
1682 1666 tavor_mrhdl_t mr;
1683 1667 uint64_t value;
1684 1668 uint_t instance;
1685 1669 int status;
1686 1670 void (*mr_callback)(void *, void *);
1687 1671 void *mr_cbarg1, *mr_cbarg2;
1688 1672
1689 1673 TAVOR_TNF_ENTER(tavor_umap_umemlock_cb);
1690 1674
|
↓ open down ↓ |
46 lines elided |
↑ open up ↑ |
1691 1675 /*
1692 1676 * If this was userland memory, then we need to remove its entry
1693 1677 * from the "userland resources database". Note: We use the
1694 1678 * TAVOR_UMAP_DB_IGNORE_INSTANCE flag here because we don't know
1695 1679 * which instance was used when the entry was added (but we want
1696 1680 * to know after the entry is found using the other search criteria).
1697 1681 */
1698 1682 status = tavor_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie,
1699 1683 MLNX_UMAP_MRMEM_RSRC, &value, (TAVOR_UMAP_DB_REMOVE |
1700 1684 TAVOR_UMAP_DB_IGNORE_INSTANCE), &umapdb);
1701 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1702 1685 if (status == DDI_SUCCESS) {
1703 1686 instance = umapdb->tdbe_common.tdb_instance;
1704 1687 state = ddi_get_soft_state(tavor_statep, instance);
1705 1688 if (state == NULL) {
1706 1689 cmn_err(CE_WARN, "Unable to match Tavor instance\n");
1707 1690 TNF_PROBE_0(tavor_umap_umemlock_cb_gss_fail,
1708 1691 TAVOR_TNF_ERROR, "");
1709 1692 TAVOR_TNF_EXIT(tavor_umap_umemlock_cb);
1710 1693 return;
1711 1694 }
1712 1695
1713 1696 /* Free the database entry */
1714 1697 tavor_umap_db_free(umapdb);
1715 1698
1716 1699 /* Use "value" to convert to an MR handle */
1717 1700 rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
1718 1701 mr = (tavor_mrhdl_t)rsrcp->tr_addr;
1719 1702
1720 1703 /*
1721 1704 * If a callback has been provided, call it first. This
1722 1705 * callback is expected to do any cleanup necessary to
1723 1706 * guarantee that the subsequent MR deregister (below)
1724 1707 * will succeed. Specifically, this means freeing up memory
1725 1708 * windows which might have been associated with the MR.
1726 1709 */
1727 1710 mutex_enter(&mr->mr_lock);
1728 1711 mr_callback = mr->mr_umem_cbfunc;
1729 1712 mr_cbarg1 = mr->mr_umem_cbarg1;
1730 1713 mr_cbarg2 = mr->mr_umem_cbarg2;
1731 1714 mutex_exit(&mr->mr_lock);
1732 1715 if (mr_callback != NULL) {
1733 1716 mr_callback(mr_cbarg1, mr_cbarg2);
1734 1717 }
1735 1718
1736 1719 /*
1737 1720 * Then call tavor_mr_deregister() to release the resources
1738 1721 * associated with the MR handle. Note: Because this routine
1739 1722 * will also check for whether the ddi_umem_cookie_t is in the
1740 1723 * database, it will take responsibility for disabling the
1741 1724 * memory region and calling ddi_umem_unlock().
1742 1725 */
1743 1726 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1744 1727 TAVOR_SLEEP);
1745 1728 if (status != DDI_SUCCESS) {
1746 1729 TAVOR_WARNING(state, "Unexpected failure in "
1747 1730 "deregister from callback\n");
1748 1731 TNF_PROBE_0(tavor_umap_umemlock_cb_dereg_fail,
1749 1732 TAVOR_TNF_ERROR, "");
1750 1733 TAVOR_TNF_EXIT(tavor_umap_umemlock_cb);
1751 1734 }
1752 1735 }
1753 1736
1754 1737 TAVOR_TNF_EXIT(tavor_umap_umemlock_cb);
1755 1738 }
1756 1739
1757 1740
1758 1741 /*
1759 1742 * tavor_umap_db_compare()
|
↓ open down ↓ |
48 lines elided |
↑ open up ↑ |
1760 1743 * Context: Can be called from user or kernel context.
1761 1744 */
1762 1745 static int
1763 1746 tavor_umap_db_compare(const void *q, const void *e)
1764 1747 {
1765 1748 tavor_umap_db_common_t *entry_common, *query_common;
1766 1749 uint_t query_flags;
1767 1750
1768 1751 TAVOR_TNF_ENTER(tavor_umap_db_compare);
1769 1752
1770 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((tavor_umap_db_query_t *)q)))
1771 -
1772 1753 entry_common = &((tavor_umap_db_entry_t *)e)->tdbe_common;
1773 1754 query_common = &((tavor_umap_db_query_t *)q)->tqdb_common;
1774 1755 query_flags = ((tavor_umap_db_query_t *)q)->tqdb_flags;
1775 1756
1776 1757 /*
1777 1758 * The first comparison is done on the "key" value in "query"
1778 1759 * and "entry". If they are not equal, then the appropriate
1779 1760 * search direction is returned. Else, we continue by
1780 1761 * comparing "type".
1781 1762 */
1782 1763 if (query_common->tdb_key < entry_common->tdb_key) {
1783 1764 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1784 1765 return (-1);
1785 1766 } else if (query_common->tdb_key > entry_common->tdb_key) {
1786 1767 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1787 1768 return (+1);
1788 1769 }
1789 1770
1790 1771 /*
1791 1772 * If the search reaches this point, then "query" and "entry"
1792 1773 * have equal key values. So we continue be comparing their
1793 1774 * "type" values. Again, if they are not equal, then the
1794 1775 * appropriate search direction is returned. Else, we continue
1795 1776 * by comparing "instance".
1796 1777 */
1797 1778 if (query_common->tdb_type < entry_common->tdb_type) {
1798 1779 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1799 1780 return (-1);
1800 1781 } else if (query_common->tdb_type > entry_common->tdb_type) {
1801 1782 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1802 1783 return (+1);
1803 1784 }
1804 1785
1805 1786 /*
1806 1787 * If the search reaches this point, then "query" and "entry"
1807 1788 * have exactly the same key and type values. Now we consult
1808 1789 * the "flags" field in the query to determine whether the
1809 1790 * "instance" is relevant to the search. If the
1810 1791 * TAVOR_UMAP_DB_IGNORE_INSTANCE flags is set, then return
1811 1792 * success (0) here. Otherwise, continue the search by comparing
1812 1793 * instance values and returning the appropriate search direction.
1813 1794 */
1814 1795 if (query_flags & TAVOR_UMAP_DB_IGNORE_INSTANCE) {
1815 1796 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1816 1797 return (0);
1817 1798 }
1818 1799
1819 1800 /*
1820 1801 * If the search has reached this point, then "query" and "entry"
1821 1802 * can only be differentiated by their instance values. If these
1822 1803 * are not equal, then return the appropriate search direction.
1823 1804 * Else, we return success (0).
1824 1805 */
1825 1806 if (query_common->tdb_instance < entry_common->tdb_instance) {
1826 1807 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1827 1808 return (-1);
1828 1809 } else if (query_common->tdb_instance > entry_common->tdb_instance) {
1829 1810 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1830 1811 return (+1);
1831 1812 }
1832 1813
1833 1814 /* Everything matches... so return success */
1834 1815 TAVOR_TNF_EXIT(tavor_umap_db_compare);
1835 1816 return (0);
1836 1817 }
1837 1818
1838 1819
1839 1820 /*
1840 1821 * tavor_umap_db_set_onclose_cb()
1841 1822 * Context: Can be called from user or kernel context.
1842 1823 */
1843 1824 int
1844 1825 tavor_umap_db_set_onclose_cb(dev_t dev, uint64_t flag,
1845 1826 void (*callback)(void *), void *arg)
1846 1827 {
1847 1828 tavor_umap_db_priv_t *priv;
1848 1829 tavor_umap_db_entry_t *umapdb;
1849 1830 minor_t instance;
1850 1831 uint64_t value;
1851 1832 int status;
1852 1833
1853 1834 TAVOR_TNF_ENTER(tavor_umap_db_set_onclose_cb);
1854 1835
1855 1836 instance = TAVOR_DEV_INSTANCE(dev);
1856 1837 if (instance == -1) {
1857 1838 TNF_PROBE_0(tavor_umap_db_set_onclose_cb_inst_fail,
1858 1839 TAVOR_TNF_ERROR, "");
1859 1840 TAVOR_TNF_EXIT(tavor_umap_db_set_onclose_cb);
1860 1841 return (DDI_FAILURE);
1861 1842 }
1862 1843
1863 1844 if (flag != TAVOR_ONCLOSE_FLASH_INPROGRESS) {
1864 1845 TNF_PROBE_0(tavor_umap_db_set_onclose_cb_invflag_fail,
1865 1846 TAVOR_TNF_ERROR, "");
1866 1847 TAVOR_TNF_EXIT(tavor_umap_db_set_onclose_cb);
1867 1848 return (DDI_FAILURE);
1868 1849 }
1869 1850
1870 1851 /*
1871 1852 * Grab the lock for the "userland resources database" and find
1872 1853 * the entry corresponding to this minor number. Once it's found,
1873 1854 * allocate (if necessary) and add an entry (in the "tdb_priv"
1874 1855 * field) to indicate that further processing may be needed during
1875 1856 * Tavor's close() handling.
1876 1857 */
1877 1858 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1878 1859 status = tavor_umap_db_find_nolock(instance, dev,
1879 1860 MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
1880 1861 if (status != DDI_SUCCESS) {
1881 1862 TNF_PROBE_0(tavor_umap_db_set_onclose_cb_find_fail,
1882 1863 TAVOR_TNF_ERROR, "");
1883 1864 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1884 1865 TAVOR_TNF_EXIT(tavor_umap_db_set_onclose_cb);
1885 1866 return (DDI_FAILURE);
1886 1867 }
1887 1868
1888 1869 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv;
1889 1870 if (priv == NULL) {
1890 1871 priv = (tavor_umap_db_priv_t *)kmem_zalloc(
1891 1872 sizeof (tavor_umap_db_priv_t), KM_NOSLEEP);
1892 1873 if (priv == NULL) {
1893 1874 TNF_PROBE_0(tavor_umap_db_set_onclose_cb_kmz_fail,
1894 1875 TAVOR_TNF_ERROR, "");
1895 1876 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1896 1877 TAVOR_TNF_EXIT(tavor_umap_db_set_onclose_cb);
1897 1878 return (DDI_FAILURE);
1898 1879 }
1899 1880 }
1900 1881
1901 1882 /*
1902 1883 * Save away the callback and argument to be used during Tavor's
1903 1884 * close() processing.
1904 1885 */
1905 1886 priv->tdp_cb = callback;
1906 1887 priv->tdp_arg = arg;
1907 1888
1908 1889 umapdb->tdbe_common.tdb_priv = (void *)priv;
1909 1890 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1910 1891
1911 1892 TAVOR_TNF_EXIT(tavor_umap_db_set_onclose_cb);
1912 1893 return (DDI_SUCCESS);
1913 1894 }
1914 1895
1915 1896
1916 1897 /*
1917 1898 * tavor_umap_db_clear_onclose_cb()
1918 1899 * Context: Can be called from user or kernel context.
1919 1900 */
1920 1901 int
1921 1902 tavor_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag)
1922 1903 {
1923 1904 tavor_umap_db_priv_t *priv;
1924 1905 tavor_umap_db_entry_t *umapdb;
1925 1906 minor_t instance;
1926 1907 uint64_t value;
1927 1908 int status;
1928 1909
1929 1910 TAVOR_TNF_ENTER(tavor_umap_db_set_onclose_cb);
1930 1911
1931 1912 instance = TAVOR_DEV_INSTANCE(dev);
1932 1913 if (instance == -1) {
1933 1914 TNF_PROBE_0(tavor_umap_db_clear_onclose_cb_inst_fail,
1934 1915 TAVOR_TNF_ERROR, "");
1935 1916 TAVOR_TNF_EXIT(tavor_umap_db_clear_onclose_cb);
1936 1917 return (DDI_FAILURE);
1937 1918 }
1938 1919
1939 1920 if (flag != TAVOR_ONCLOSE_FLASH_INPROGRESS) {
1940 1921 TNF_PROBE_0(tavor_umap_db_clear_onclose_cb_invflag_fail,
1941 1922 TAVOR_TNF_ERROR, "");
1942 1923 TAVOR_TNF_EXIT(tavor_umap_db_clear_onclose_cb);
1943 1924 return (DDI_FAILURE);
1944 1925 }
1945 1926
1946 1927 /*
1947 1928 * Grab the lock for the "userland resources database" and find
1948 1929 * the entry corresponding to this minor number. Once it's found,
1949 1930 * remove the entry (in the "tdb_priv" field) that indicated the
1950 1931 * need for further processing during Tavor's close(). Free the
1951 1932 * entry, if appropriate.
1952 1933 */
1953 1934 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1954 1935 status = tavor_umap_db_find_nolock(instance, dev,
1955 1936 MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
1956 1937 if (status != DDI_SUCCESS) {
1957 1938 TNF_PROBE_0(tavor_umap_db_clear_onclose_cb_find_fail,
1958 1939 TAVOR_TNF_ERROR, "");
1959 1940 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1960 1941 TAVOR_TNF_EXIT(tavor_umap_db_clear_onclose_cb);
1961 1942 return (DDI_FAILURE);
1962 1943 }
1963 1944
1964 1945 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv;
1965 1946 if (priv != NULL) {
1966 1947 kmem_free(priv, sizeof (tavor_umap_db_priv_t));
1967 1948 priv = NULL;
1968 1949 }
1969 1950
1970 1951 umapdb->tdbe_common.tdb_priv = (void *)priv;
1971 1952 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
1972 1953 return (DDI_SUCCESS);
1973 1954 }
1974 1955
1975 1956
1976 1957 /*
1977 1958 * tavor_umap_db_clear_onclose_cb()
1978 1959 * Context: Can be called from user or kernel context.
1979 1960 */
1980 1961 void
1981 1962 tavor_umap_db_handle_onclose_cb(tavor_umap_db_priv_t *priv)
1982 1963 {
1983 1964 void (*callback)(void *);
1984 1965
1985 1966 ASSERT(MUTEX_HELD(&tavor_userland_rsrc_db.tdl_umapdb_lock));
1986 1967
1987 1968 /*
1988 1969 * Call the callback.
1989 1970 * Note: Currently there is only one callback (in "tdp_cb"), but
1990 1971 * in the future there may be more, depending on what other types
1991 1972 * of interaction there are between userland processes and the
1992 1973 * driver.
1993 1974 */
1994 1975 callback = priv->tdp_cb;
1995 1976 callback(priv->tdp_arg);
1996 1977 }
|
↓ open down ↓ |
215 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX