Print this page
8368 remove warlock leftovers from usr/src/uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/adapters/tavor/tavor_rsrc.c
+++ new/usr/src/uts/common/io/ib/adapters/tavor/tavor_rsrc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /*
28 28 * tavor_rsrc.c
29 29 * Tavor Resource Management Routines
30 30 *
31 31 * Implements all the routines necessary for setup, teardown, and
32 32 * alloc/free of all Tavor resources, including those that are managed
33 33 * by Tavor hardware or which live in Tavor's direct attached DDR memory.
34 34 */
35 35
36 36 #include <sys/sysmacros.h>
37 37 #include <sys/types.h>
38 38 #include <sys/conf.h>
39 39 #include <sys/ddi.h>
40 40 #include <sys/sunddi.h>
41 41 #include <sys/modctl.h>
42 42 #include <sys/vmem.h>
43 43 #include <sys/bitmap.h>
44 44
45 45 #include <sys/ib/adapters/tavor/tavor.h>
46 46
47 47 /*
48 48 * The following routines are used for initializing and destroying
49 49 * the resource pools used by the Tavor resource allocation routines.
50 50 * They consist of four classes of object:
51 51 *
52 52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor
53 53 * command interface routines. Mailboxes are used to pass information
54 54 * back and forth to the Tavor firmware. Either type of mailbox may
55 55 * be allocated from Tavor's direct attached DDR memory or from system
56 56 * memory (although currently all "In" mailboxes are in DDR and all "out"
57 57 * mailboxes come from system memory.
58 58 *
59 59 * HW entry objects: These objects represent resources required by the Tavor
60 60 * hardware. These objects include things like Queue Pair contexts (QPC),
61 61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
62 62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
63 63 * Protection Table entries (MPT), Memory Translation Table entries (MTT).
64 64 *
65 65 * What these objects all have in common is that they are each required
66 66 * to come from DDR memory, they are always allocated from tables, and
67 67 * they are not to be directly accessed (read or written) by driver
68 68 * software.
69 69 * One notable exceptions to this rule are the Extended QP contexts (EQPC),
70 70 * and the UAR scratch area (UAR_SCR), both of which are not directly
71 71 * accessible through the Tavor resource allocation routines, but both
72 72 * of which are also required to reside in DDR memory and are not to be
73 73 * manipulated by driver software (they are privately managed by Tavor
74 74 * hardware).
75 75 * The other notable exceptions are the UAR pages (UAR_PG) which are
76 76 * allocated from the UAR address space rather than DDR, and the UD
77 77 * address vectors (UDAV) which are similar to the common object types
78 78 * with the major difference being that UDAVs _are_ directly read and
79 79 * written by driver software.
80 80 *
81 81 * SW handle objects: These objects represent resources required by Tavor
82 82 * driver software. They are primarily software tracking structures,
83 83 * which are allocated from system memory (using kmem_cache). Several of
84 84 * the objects have both a "constructor" and "destructor" method
85 85 * associated with them (see below).
86 86 *
87 87 * Protection Domain (PD) handle objects: These objects are very much like
88 88 * a SW handle object with the notable difference that all PD handle
89 89 * objects have an actual Protection Domain number (PD) associated with
90 90 * them (and the PD number is allocated/managed through a separate
91 91 * vmem_arena specifically set aside for this purpose.
92 92 */
93 93
94 94 static int tavor_rsrc_mbox_init(tavor_state_t *state,
95 95 tavor_rsrc_mbox_info_t *info);
96 96 static void tavor_rsrc_mbox_fini(tavor_state_t *state,
97 97 tavor_rsrc_mbox_info_t *info);
98 98
99 99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
100 100 tavor_rsrc_hw_entry_info_t *info);
101 101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
102 102 tavor_rsrc_hw_entry_info_t *info);
103 103
104 104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
105 105 tavor_rsrc_sw_hdl_info_t *info);
106 106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
107 107 tavor_rsrc_sw_hdl_info_t *info);
108 108
109 109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
110 110 tavor_rsrc_sw_hdl_info_t *info);
111 111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
112 112 tavor_rsrc_sw_hdl_info_t *info);
113 113
114 114 /*
115 115 * The following routines are used for allocating and freeing the specific
116 116 * types of objects described above from their associated resource pools.
117 117 */
118 118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
119 119 uint_t num, tavor_rsrc_t *hdl);
120 120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
121 121 tavor_rsrc_t *hdl);
122 122
123 123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
124 124 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
125 125 uint_t sleepflag, tavor_rsrc_t *hdl);
126 126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
127 127 tavor_rsrc_t *hdl);
128 128
129 129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
130 130 uint_t sleepflag, tavor_rsrc_t *hdl);
131 131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
132 132 tavor_rsrc_t *hdl);
133 133
134 134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
135 135 uint_t sleepflag, tavor_rsrc_t *hdl);
136 136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
137 137 tavor_rsrc_t *hdl);
138 138
139 139 /*
140 140 * The following routines are the constructors and destructors for several
141 141 * of the SW handle type objects. For certain types of SW handles objects
142 142 * (all of which are implemented using kmem_cache), we need to do some
143 143 * special field initialization (specifically, mutex_init/destroy). These
144 144 * routines enable that init and teardown.
145 145 */
146 146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
147 147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
148 148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
149 149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
150 150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
151 151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
152 152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
153 153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
154 154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
155 155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
156 156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
157 157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
158 158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
159 159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);
160 160
161 161 /*
162 162 * Special routine to calculate and return the size of a MCG object based
163 163 * on current driver configuration (specifically, the number of QP per MCG
164 164 * that has been configured.
165 165 */
166 166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
167 167 uint_t *mcg_size_shift);
168 168
169 169
170 170 /*
171 171 * tavor_rsrc_alloc()
172 172 *
173 173 * Context: Can be called from interrupt or base context.
174 174 * The "sleepflag" parameter is used by all object allocators to
175 175 * determine whether to SLEEP for resources or not.
176 176 */
177 177 int
178 178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
179 179 uint_t sleepflag, tavor_rsrc_t **hdl)
180 180 {
181 181 tavor_rsrc_pool_info_t *rsrc_pool;
182 182 tavor_rsrc_t *tmp_rsrc_hdl;
183 183 int flag, status = DDI_FAILURE;
184 184
185 185 TAVOR_TNF_ENTER(tavor_rsrc_alloc);
186 186
187 187 ASSERT(state != NULL);
188 188 ASSERT(hdl != NULL);
189 189
190 190 rsrc_pool = &state->ts_rsrc_hdl[rsrc];
191 191 ASSERT(rsrc_pool != NULL);
192 192
193 193 /*
|
↓ open down ↓ |
193 lines elided |
↑ open up ↑ |
194 194 * Allocate space for the object used to track the resource handle
195 195 */
196 196 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
197 197 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
198 198 flag);
199 199 if (tmp_rsrc_hdl == NULL) {
200 200 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, "");
201 201 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
202 202 return (DDI_FAILURE);
203 203 }
204 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
205 204
206 205 /*
207 206 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call
208 207 * to know what type of resource is being freed.
209 208 */
210 209 tmp_rsrc_hdl->rsrc_type = rsrc;
211 210
212 211 /*
213 212 * Depending on resource type, call the appropriate alloc routine
214 213 */
215 214 switch (rsrc_pool->rsrc_type) {
216 215 case TAVOR_IN_MBOX:
217 216 case TAVOR_OUT_MBOX:
218 217 case TAVOR_INTR_IN_MBOX:
219 218 case TAVOR_INTR_OUT_MBOX:
220 219 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
221 220 break;
222 221
223 222 case TAVOR_QPC:
224 223 case TAVOR_CQC:
225 224 case TAVOR_SRQC:
226 225 case TAVOR_EQC:
227 226 case TAVOR_RDB:
228 227 /*
229 228 * Because these objects are NOT accessed by Tavor driver
230 229 * software, we set the acc_handle parameter to zero. But
231 230 * if they are allocated in multiples, we specify here that
232 231 * they must be aligned on a more restrictive boundary.
233 232 */
234 233 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
235 234 sleepflag, tmp_rsrc_hdl);
236 235 break;
237 236
238 237 case TAVOR_MPT:
239 238 /*
240 239 * Because these MPT objects are sometimes accessed by Tavor
241 240 * driver software (FMR), we set the acc_handle parameter. But
242 241 * if they are allocated in multiples, we specify here that
243 242 * they must be aligned on a more restrictive boundary.
244 243 */
245 244 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
246 245 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
247 246 break;
248 247
249 248 case TAVOR_MCG:
250 249 /*
251 250 * Tavor MCG entries are also NOT accessed by Tavor driver
252 251 * software, but because MCG entries do not have the same
253 252 * alignnment restrictions we loosen the constraint here.
254 253 */
255 254 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
256 255 sleepflag, tmp_rsrc_hdl);
257 256 break;
258 257
259 258 case TAVOR_MTT:
260 259 case TAVOR_UDAV:
261 260 /*
262 261 * Because MTT segments are among the few HW resources that
263 262 * may be allocated in odd numbers, we specify a less
264 263 * restrictive alignment than for the above resources.
265 264 *
266 265 * Also because both UDAV and MTT segment objects are read
267 266 * and/or written by Tavor driver software, we set the
268 267 * acc_handle parameter to point to the ddi_acc_handle_t for
269 268 * the Tavor DDR memory.
270 269 */
271 270 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
272 271 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
273 272 break;
274 273
275 274 case TAVOR_UARPG:
276 275 /*
277 276 * Because UAR pages are written by Tavor driver software (for
278 277 * doorbells), we set the acc_handle parameter to point to
279 278 * the ddi_acc_handle_t for the Tavor UAR memory.
280 279 */
281 280 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
282 281 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
283 282 break;
284 283
285 284 case TAVOR_MRHDL:
286 285 case TAVOR_EQHDL:
287 286 case TAVOR_CQHDL:
288 287 case TAVOR_SRQHDL:
289 288 case TAVOR_AHHDL:
290 289 case TAVOR_QPHDL:
291 290 case TAVOR_REFCNT:
292 291 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
293 292 tmp_rsrc_hdl);
294 293 break;
295 294
296 295 case TAVOR_PDHDL:
297 296 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
298 297 tmp_rsrc_hdl);
299 298 break;
300 299
301 300 default:
302 301 TAVOR_WARNING(state, "unexpected resource type in alloc");
303 302 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail,
304 303 TAVOR_TNF_ERROR, "");
305 304 break;
306 305 }
307 306
308 307 /*
309 308 * If the resource allocation failed, then free the special resource
310 309 * tracking structure and return failure. Otherwise return the
311 310 * handle for the resource tracking structure.
312 311 */
313 312 if (status != DDI_SUCCESS) {
314 313 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
315 314 tmp_rsrc_hdl = NULL;
316 315 TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "",
317 316 tnf_uint, rsrc_type, rsrc_pool->rsrc_type);
318 317 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
319 318 return (DDI_FAILURE);
320 319 } else {
321 320 *hdl = tmp_rsrc_hdl;
322 321 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
323 322 return (DDI_SUCCESS);
324 323 }
325 324 }
326 325
327 326
328 327 /*
329 328 * tavor_rsrc_free()
330 329 * Context: Can be called from interrupt or base context.
331 330 */
332 331 void
333 332 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
334 333 {
335 334 tavor_rsrc_pool_info_t *rsrc_pool;
336 335
337 336 TAVOR_TNF_ENTER(tavor_rsrc_free);
338 337
339 338 ASSERT(state != NULL);
340 339 ASSERT(hdl != NULL);
341 340
342 341 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
343 342 ASSERT(rsrc_pool != NULL);
344 343
345 344 /*
346 345 * Depending on resource type, call the appropriate free routine
347 346 */
348 347 switch (rsrc_pool->rsrc_type) {
349 348 case TAVOR_IN_MBOX:
350 349 case TAVOR_OUT_MBOX:
351 350 case TAVOR_INTR_IN_MBOX:
352 351 case TAVOR_INTR_OUT_MBOX:
353 352 tavor_rsrc_mbox_free(rsrc_pool, *hdl);
354 353 break;
355 354
356 355 case TAVOR_QPC:
357 356 case TAVOR_CQC:
358 357 case TAVOR_SRQC:
359 358 case TAVOR_EQC:
360 359 case TAVOR_RDB:
361 360 case TAVOR_MCG:
362 361 case TAVOR_MPT:
363 362 case TAVOR_MTT:
364 363 case TAVOR_UDAV:
365 364 case TAVOR_UARPG:
366 365 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
367 366 break;
368 367
369 368 case TAVOR_MRHDL:
370 369 case TAVOR_EQHDL:
371 370 case TAVOR_CQHDL:
372 371 case TAVOR_SRQHDL:
373 372 case TAVOR_AHHDL:
374 373 case TAVOR_QPHDL:
375 374 case TAVOR_REFCNT:
376 375 tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
377 376 break;
378 377
379 378 case TAVOR_PDHDL:
380 379 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
381 380 break;
382 381
383 382 default:
384 383 TAVOR_WARNING(state, "unexpected resource type in free");
385 384 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail,
386 385 TAVOR_TNF_ERROR, "");
387 386 break;
388 387 }
389 388
390 389 /*
391 390 * Free the special resource tracking structure, set the handle to
392 391 * NULL, and return.
393 392 */
394 393 kmem_cache_free(state->ts_rsrc_cache, *hdl);
395 394 *hdl = NULL;
396 395
397 396 TAVOR_TNF_EXIT(tavor_rsrc_free);
398 397 }
399 398
400 399
401 400 /*
402 401 * tavor_rsrc_init_phase1()
403 402 *
404 403 * Completes the first phase of Tavor resource/configuration init.
405 404 * This involves creating the kmem_cache for the "tavor_rsrc_t"
406 405 * structs, allocating the space for the resource pool handles,
407 406 * and setting up the "Out" mailboxes.
408 407 *
409 408 * When this function completes, the Tavor driver is ready to
410 409 * post the following commands which return information only in the
411 410 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
412 411 * If any of these commands are to be posted at this time, they must be
413 412 * done so only when "spinning" (as the outstanding command list and
414 413 * EQ setup code has not yet run)
415 414 *
416 415 * Context: Only called from attach() path context
417 416 */
418 417 int
419 418 tavor_rsrc_init_phase1(tavor_state_t *state)
420 419 {
421 420 tavor_rsrc_pool_info_t *rsrc_pool;
422 421 tavor_rsrc_mbox_info_t mbox_info;
423 422 tavor_rsrc_cleanup_level_t cleanup;
424 423 tavor_cfg_profile_t *cfgprof;
425 424 uint64_t num, size;
426 425 int status;
427 426 char *errormsg, *rsrc_name;
428 427
429 428 TAVOR_TNF_ENTER(tavor_rsrc_init_phase1);
430 429
431 430 ASSERT(state != NULL);
432 431
433 432 /* This is where Phase 1 of resource initialization begins */
434 433 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;
435 434
436 435 /* Build kmem cache name from Tavor instance */
437 436 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
438 437 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);
439 438
440 439 /*
441 440 * Create the kmem_cache for "tavor_rsrc_t" structures
442 441 * (kmem_cache_create will SLEEP until successful)
443 442 */
444 443 state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
445 444 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
446 445
447 446 /*
448 447 * Allocate an array of tavor_rsrc_pool_info_t's (used in all
449 448 * subsequent resource allocations)
450 449 */
451 450 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
452 451 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);
453 452
454 453 cfgprof = state->ts_cfg_profile;
455 454
456 455 /*
457 456 * Initialize the resource pool for "Out" mailboxes. Notice that
458 457 * the number of "Out" mailboxes, their size, and their location
459 458 * (DDR or system memory) is configurable. By default, however,
460 459 * all "Out" mailboxes are located in system memory only (because
461 460 * they are primarily read from and never written to)
462 461 */
463 462 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
464 463 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
465 464 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
466 465 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX;
467 466 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
468 467 rsrc_pool->rsrc_pool_size = (size * num);
469 468 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
470 469 rsrc_pool->rsrc_quantum = size;
471 470 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
472 471 rsrc_pool->rsrc_state = state;
473 472 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
474 473 mbox_info.mbi_num = num;
475 474 mbox_info.mbi_size = size;
476 475 mbox_info.mbi_rsrcpool = rsrc_pool;
477 476 mbox_info.mbi_rsrcname = rsrc_name;
478 477 status = tavor_rsrc_mbox_init(state, &mbox_info);
479 478 if (status != DDI_SUCCESS) {
480 479 tavor_rsrc_fini(state, cleanup);
481 480 /* Set "status" and "errormsg" and goto failure */
482 481 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes");
483 482 goto rsrcinitp1_fail;
484 483 }
485 484 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;
486 485
487 486 /*
488 487 * Initialize the Tavor "Out" mailbox list. This step actually uses
489 488 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
490 489 * "Out" mailboxes, bind them for DMA access, and arrange them into
491 490 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
492 491 * for more details)
493 492 */
494 493 status = tavor_outmbox_list_init(state);
495 494 if (status != DDI_SUCCESS) {
496 495 tavor_rsrc_fini(state, cleanup);
497 496 /* Set "status" and "errormsg" and goto failure */
498 497 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list");
499 498 goto rsrcinitp1_fail;
500 499 }
501 500 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;
502 501
503 502 /*
504 503 * Initialize the resource pool for interrupt "Out" mailboxes. Notice
505 504 * that the number of interrupt "Out" mailboxes, their size, and their
506 505 * location (DDR or system memory) is configurable. By default,
507 506 * however, all interrupt "Out" mailboxes are located in system memory
508 507 * only (because they are primarily read from and never written to)
509 508 */
510 509 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
511 510 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
512 511 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
513 512 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX;
514 513 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
515 514 rsrc_pool->rsrc_pool_size = (size * num);
516 515 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
517 516 rsrc_pool->rsrc_quantum = size;
518 517 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
519 518 rsrc_pool->rsrc_state = state;
520 519 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
521 520 mbox_info.mbi_num = num;
522 521 mbox_info.mbi_size = size;
523 522 mbox_info.mbi_rsrcpool = rsrc_pool;
524 523 mbox_info.mbi_rsrcname = rsrc_name;
525 524 status = tavor_rsrc_mbox_init(state, &mbox_info);
526 525 if (status != DDI_SUCCESS) {
527 526 tavor_rsrc_fini(state, cleanup);
528 527 /* Set "status" and "errormsg" and goto failure */
529 528 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes");
530 529 goto rsrcinitp1_fail;
531 530 }
532 531 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;
533 532
534 533 /*
535 534 * Initialize the Tavor "Out" mailbox list. This step actually uses
536 535 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
537 536 * "Out" mailboxes, bind them for DMA access, and arrange them into
538 537 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
539 538 * for more details)
540 539 */
541 540 status = tavor_intr_outmbox_list_init(state);
542 541 if (status != DDI_SUCCESS) {
543 542 tavor_rsrc_fini(state, cleanup);
544 543 /* Set "status" and "errormsg" and goto failure */
545 544 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list");
546 545 goto rsrcinitp1_fail;
547 546 }
548 547 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
549 548
550 549 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
551 550 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
552 551 return (DDI_SUCCESS);
553 552
554 553 rsrcinitp1_fail:
555 554 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
556 555 TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "",
557 556 tnf_string, msg, errormsg);
558 557 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
559 558 return (status);
560 559 }
561 560
562 561
563 562 /*
564 563 * tavor_rsrc_init_phase2()
565 564 * Context: Only called from attach() path context
566 565 */
567 566 int
568 567 tavor_rsrc_init_phase2(tavor_state_t *state)
569 568 {
570 569 tavor_rsrc_sw_hdl_info_t hdl_info;
571 570 tavor_rsrc_hw_entry_info_t entry_info;
572 571 tavor_rsrc_mbox_info_t mbox_info;
573 572 tavor_rsrc_pool_info_t *rsrc_pool;
574 573 tavor_rsrc_cleanup_level_t cleanup;
575 574 tavor_cfg_profile_t *cfgprof;
576 575 uint64_t num, max, size, num_prealloc;
577 576 uint64_t ddr_size, fw_size;
578 577 uint_t mcg_size, mcg_size_shift;
579 578 uint_t uarscr_size, mttsegment_sz;
580 579 int status;
581 580 char *errormsg, *rsrc_name;
582 581
583 582 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2);
584 583
585 584 ASSERT(state != NULL);
586 585
587 586 /* Phase 2 initialization begins where Phase 1 left off */
588 587 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
589 588
590 589 /*
591 590 * Calculate the extent of the DDR size and portion of which that
592 591 * is already reserved for Tavor firmware. (Note: this information
593 592 * is available because the QUERY_DDR and QUERY_FW commands have
594 593 * been posted to Tavor firmware prior to calling this routine)
595 594 */
596 595 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
597 596 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;
598 597
599 598 /* Build the DDR vmem arena name from Tavor instance */
600 599 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
601 600 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);
602 601
603 602 /*
604 603 * Do a vmem_create for the entire DDR range (not including the
605 604 * portion consumed by Tavor firmware). This creates the vmem arena
606 605 * from which all other DDR objects (specifically, tables of HW
607 606 * entries) will be allocated.
608 607 */
609 608 state->ts_ddrvmem = vmem_create(rsrc_name,
610 609 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
611 610 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
612 611 if (state->ts_ddrvmem == NULL) {
613 612 tavor_rsrc_fini(state, cleanup);
614 613 /* Set "status" and "errormsg" and goto failure */
615 614 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem");
616 615 goto rsrcinitp2_fail;
617 616 }
618 617 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;
619 618
620 619 /*
621 620 * Initialize the resource pools for all objects that exist in
622 621 * Tavor DDR memory. This includes ("In") mailboxes, context tables
623 622 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
624 623 */
625 624 cfgprof = state->ts_cfg_profile;
626 625
627 626 /*
628 627 * Initialize the resource pool for the MPT table entries. Notice
629 628 * that the number of MPTs is configurable. The configured value must
630 629 * be less that the maximum value (obtained from the QUERY_DEV_LIM
631 630 * command) or the initialization will fail. Note also that a certain
632 631 * number of MPTs must be set aside for Tavor firmware use.
633 632 */
634 633 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
635 634 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
636 635 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
637 636 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
638 637 rsrc_pool->rsrc_type = TAVOR_MPT;
639 638 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
640 639 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
641 640 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT;
642 641 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE;
643 642 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num);
644 643 rsrc_pool->rsrc_state = state;
645 644 rsrc_pool->rsrc_start = NULL;
646 645 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
647 646 entry_info.hwi_num = num;
648 647 entry_info.hwi_max = max;
649 648 entry_info.hwi_prealloc = num_prealloc;
650 649 entry_info.hwi_rsrcpool = rsrc_pool;
651 650 entry_info.hwi_rsrcname = rsrc_name;
652 651 status = tavor_rsrc_hw_entries_init(state, &entry_info);
653 652 if (status != DDI_SUCCESS) {
654 653 tavor_rsrc_fini(state, cleanup);
655 654 /* Set "status" and "errormsg" and goto failure */
656 655 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table");
657 656 goto rsrcinitp2_fail;
658 657 }
659 658 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;
660 659
661 660 /*
662 661 * Initialize the resource pool for the MTT table entries. Notice
663 662 * that the number of MTTs is configurable. The configured value must
664 663 * be less that the maximum value (obtained from the QUERY_DEV_LIM
665 664 * command) or the initialization will fail. Note also that a certain
666 665 * number of MTT segments must be set aside for Tavor firmware use.
667 666 */
668 667 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
669 668 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
670 669 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
671 670 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
672 671 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
673 672 rsrc_pool->rsrc_type = TAVOR_MTT;
674 673 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
675 674 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
676 675 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT;
677 676 rsrc_pool->rsrc_quantum = mttsegment_sz;
678 677 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num);
679 678 rsrc_pool->rsrc_state = state;
680 679 rsrc_pool->rsrc_start = NULL;
681 680 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
682 681 entry_info.hwi_num = num;
683 682 entry_info.hwi_max = max;
684 683 entry_info.hwi_prealloc = num_prealloc;
685 684 entry_info.hwi_rsrcpool = rsrc_pool;
686 685 entry_info.hwi_rsrcname = rsrc_name;
687 686 status = tavor_rsrc_hw_entries_init(state, &entry_info);
688 687 if (status != DDI_SUCCESS) {
689 688 tavor_rsrc_fini(state, cleanup);
690 689 /* Set "status" and "errormsg" and goto failure */
691 690 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table");
692 691 goto rsrcinitp2_fail;
693 692 }
694 693 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;
695 694
696 695 /*
697 696 * Initialize the resource pool for the QPC table entries. Notice
698 697 * that the number of QPs is configurable. The configured value must
699 698 * be less that the maximum value (obtained from the QUERY_DEV_LIM
700 699 * command) or the initialization will fail. Note also that a certain
701 700 * number of QP contexts must be set aside for Tavor firmware use.
702 701 */
703 702 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
704 703 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
705 704 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
706 705 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
707 706 rsrc_pool->rsrc_type = TAVOR_QPC;
708 707 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
709 708 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
710 709 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT;
711 710 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE;
712 711 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num);
713 712 rsrc_pool->rsrc_state = state;
714 713 rsrc_pool->rsrc_start = NULL;
715 714 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
716 715 entry_info.hwi_num = num;
717 716 entry_info.hwi_max = max;
718 717 entry_info.hwi_prealloc = num_prealloc;
719 718 entry_info.hwi_rsrcpool = rsrc_pool;
720 719 entry_info.hwi_rsrcname = rsrc_name;
721 720 status = tavor_rsrc_hw_entries_init(state, &entry_info);
722 721 if (status != DDI_SUCCESS) {
723 722 tavor_rsrc_fini(state, cleanup);
724 723 /* Set "status" and "errormsg" and goto failure */
725 724 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table");
726 725 goto rsrcinitp2_fail;
727 726 }
728 727 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;
729 728
730 729 /*
731 730 * Initialize the resource pool for the RDB table entries. Notice
732 731 * that the number of RDBs is configurable. The configured value must
733 732 * be less that the maximum value (obtained from the QUERY_DEV_LIM
734 733 * command) or the initialization will fail.
735 734 */
736 735 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
737 736 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
738 737 num_prealloc = 0;
739 738 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
740 739 rsrc_pool->rsrc_type = TAVOR_RDB;
741 740 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
742 741 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
743 742 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT;
744 743 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE;
745 744 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num);
746 745 rsrc_pool->rsrc_state = state;
747 746 rsrc_pool->rsrc_start = NULL;
748 747 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
749 748 entry_info.hwi_num = num;
750 749 entry_info.hwi_max = max;
751 750 entry_info.hwi_prealloc = num_prealloc;
752 751 entry_info.hwi_rsrcpool = rsrc_pool;
753 752 entry_info.hwi_rsrcname = rsrc_name;
754 753 status = tavor_rsrc_hw_entries_init(state, &entry_info);
755 754 if (status != DDI_SUCCESS) {
756 755 tavor_rsrc_fini(state, cleanup);
757 756 /* Set "status" and "errormsg" and goto failure */
758 757 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table");
759 758 goto rsrcinitp2_fail;
760 759 }
761 760 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;
762 761
763 762 /*
764 763 * Initialize the resource pool for the CQC table entries. Notice
765 764 * that the number of CQs is configurable. The configured value must
766 765 * be less that the maximum value (obtained from the QUERY_DEV_LIM
767 766 * command) or the initialization will fail. Note also that a certain
768 767 * number of CQ contexts must be set aside for Tavor firmware use.
769 768 */
770 769 num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
771 770 max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
772 771 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
773 772 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
774 773 rsrc_pool->rsrc_type = TAVOR_CQC;
775 774 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
776 775 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
777 776 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT;
778 777 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE;
779 778 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num);
780 779 rsrc_pool->rsrc_state = state;
781 780 rsrc_pool->rsrc_start = NULL;
782 781 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
783 782 entry_info.hwi_num = num;
784 783 entry_info.hwi_max = max;
785 784 entry_info.hwi_prealloc = num_prealloc;
786 785 entry_info.hwi_rsrcpool = rsrc_pool;
787 786 entry_info.hwi_rsrcname = rsrc_name;
788 787 status = tavor_rsrc_hw_entries_init(state, &entry_info);
789 788 if (status != DDI_SUCCESS) {
790 789 tavor_rsrc_fini(state, cleanup);
791 790 /* Set "status" and "errormsg" and goto failure */
792 791 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table");
793 792 goto rsrcinitp2_fail;
794 793 }
795 794 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;
796 795
797 796 /*
798 797 * Initialize the resource pool for the Extended QPC table entries.
799 798 * Notice that the number of EQPCs must be the same as the number
800 799 * of QP contexts. So the initialization is constructed in a
801 800 * similar way as above (for TAVOR_QPC). One notable difference
802 801 * here, however, is that by setting the rsrc_quantum field to
803 802 * zero (indicating a zero-sized object) we indicate that the
804 803 * object is not allocatable. The EQPC table is, in fact, managed
805 804 * internally by the hardware and it is, therefore, unnecessary to
806 805 * initialize an additional vmem_arena for this type of object.
807 806 */
808 807 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
809 808 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
810 809 num_prealloc = 0;
811 810 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
812 811 rsrc_pool->rsrc_type = TAVOR_EQPC;
813 812 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
814 813 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
815 814 rsrc_pool->rsrc_shift = 0;
816 815 rsrc_pool->rsrc_quantum = 0;
817 816 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE;
818 817 rsrc_pool->rsrc_state = state;
819 818 rsrc_pool->rsrc_start = NULL;
820 819 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
821 820 entry_info.hwi_num = num;
822 821 entry_info.hwi_max = max;
823 822 entry_info.hwi_prealloc = num_prealloc;
824 823 entry_info.hwi_rsrcpool = rsrc_pool;
825 824 entry_info.hwi_rsrcname = rsrc_name;
826 825 status = tavor_rsrc_hw_entries_init(state, &entry_info);
827 826 if (status != DDI_SUCCESS) {
828 827 tavor_rsrc_fini(state, cleanup);
829 828 /* Set "status" and "errormsg" and goto failure */
830 829 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table");
831 830 goto rsrcinitp2_fail;
832 831 }
833 832 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;
834 833
835 834 /*
836 835 * Initialize the resource pool for the UD address vector table
837 836 * entries. Notice that the number of UDAVs is configurable. The
838 837 * configured value must be less that the maximum value (obtained
839 838 * from the QUERY_DEV_LIM command) or the initialization will fail.
840 839 */
841 840 num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
842 841 max = ((uint64_t)1 << state->ts_devlim.log_max_av);
843 842 num_prealloc = 0;
844 843 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
845 844 rsrc_pool->rsrc_type = TAVOR_UDAV;
846 845 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
847 846 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
848 847 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT;
849 848 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE;
850 849 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE;
851 850 rsrc_pool->rsrc_state = state;
852 851 rsrc_pool->rsrc_start = NULL;
853 852 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
854 853 entry_info.hwi_num = num;
855 854 entry_info.hwi_max = max;
856 855 entry_info.hwi_prealloc = num_prealloc;
857 856 entry_info.hwi_rsrcpool = rsrc_pool;
858 857 entry_info.hwi_rsrcname = rsrc_name;
859 858 status = tavor_rsrc_hw_entries_init(state, &entry_info);
860 859 if (status != DDI_SUCCESS) {
861 860 tavor_rsrc_fini(state, cleanup);
862 861 /* Set "status" and "errormsg" and goto failure */
863 862 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table");
864 863 goto rsrcinitp2_fail;
865 864 }
866 865 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;
867 866
868 867 /*
869 868 * Initialize the resource pool for the UAR scratch table entries.
870 869 * Notice that the number of UARSCRs is configurable. The configured
871 870 * value must be less that the maximum value (obtained from the
872 871 * QUERY_DEV_LIM command) or the initialization will fail.
873 872 * Like the EQPCs above, UARSCR objects are not allocatable. The
874 873 * UARSCR table is also managed internally by the hardware and it
875 874 * is, therefore, unnecessary to initialize an additional vmem_arena
876 875 * for this type of object. We indicate this by setting the
877 876 * rsrc_quantum field to zero (indicating a zero-sized object).
878 877 */
879 878 uarscr_size = state->ts_devlim.uarscr_entry_sz;
880 879 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
881 880 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
882 881 PAGESHIFT));
883 882 num_prealloc = 0;
884 883 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
885 884 rsrc_pool->rsrc_type = TAVOR_UAR_SCR;
886 885 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
887 886 rsrc_pool->rsrc_pool_size = (uarscr_size * num);
888 887 rsrc_pool->rsrc_shift = 0;
889 888 rsrc_pool->rsrc_quantum = 0;
890 889 rsrc_pool->rsrc_align = uarscr_size;
891 890 rsrc_pool->rsrc_state = state;
892 891 rsrc_pool->rsrc_start = NULL;
893 892 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
894 893 entry_info.hwi_num = num;
895 894 entry_info.hwi_max = max;
896 895 entry_info.hwi_prealloc = num_prealloc;
897 896 entry_info.hwi_rsrcpool = rsrc_pool;
898 897 entry_info.hwi_rsrcname = rsrc_name;
899 898 status = tavor_rsrc_hw_entries_init(state, &entry_info);
900 899 if (status != DDI_SUCCESS) {
901 900 tavor_rsrc_fini(state, cleanup);
902 901 /* Set "status" and "errormsg" and goto failure */
903 902 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table");
904 903 goto rsrcinitp2_fail;
905 904 }
906 905 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;
907 906
908 907 /*
909 908 * Initialize the resource pool for the SRQC table entries. Notice
910 909 * that the number of SRQs is configurable. The configured value must
911 910 * be less that the maximum value (obtained from the QUERY_DEV_LIM
912 911 * command) or the initialization will fail. Note also that a certain
913 912 * number of SRQ contexts must be set aside for Tavor firmware use.
914 913 *
915 914 * Note: We only allocate these resources if SRQ is enabled in the
916 915 * config profile; see below.
917 916 */
918 917 num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
919 918 max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
920 919 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);
921 920
922 921 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
923 922 rsrc_pool->rsrc_type = TAVOR_SRQC;
924 923 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
925 924 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
926 925 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT;
927 926 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE;
928 927 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num);
929 928 rsrc_pool->rsrc_state = state;
930 929 rsrc_pool->rsrc_start = NULL;
931 930 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
932 931 entry_info.hwi_num = num;
933 932 entry_info.hwi_max = max;
934 933 entry_info.hwi_prealloc = num_prealloc;
935 934 entry_info.hwi_rsrcpool = rsrc_pool;
936 935 entry_info.hwi_rsrcname = rsrc_name;
937 936
938 937 /*
939 938 * SRQ support is configurable. Only if SRQ is enabled (the default)
940 939 * do we actually try to configure these resources. Otherwise, we
941 940 * simply set the cleanup level and continue on to the next resource
942 941 */
943 942 if (state->ts_cfg_profile->cp_srq_enable != 0) {
944 943 status = tavor_rsrc_hw_entries_init(state, &entry_info);
945 944 if (status != DDI_SUCCESS) {
946 945 tavor_rsrc_fini(state, cleanup);
947 946 /* Set "status" and "errormsg" and goto failure */
948 947 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table");
949 948 goto rsrcinitp2_fail;
950 949 }
951 950 }
952 951 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;
953 952
954 953 /*
955 954 * Initialize the resource pool for "In" mailboxes. Notice that
956 955 * the number of "In" mailboxes, their size, and their location
957 956 * (DDR or system memory) is configurable. By default, however,
958 957 * all "In" mailboxes are located in system memory only (because
959 958 * they are primarily written to and rarely read from)
960 959 */
961 960 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
962 961 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
963 962 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
964 963 rsrc_pool->rsrc_type = TAVOR_IN_MBOX;
965 964 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
966 965 rsrc_pool->rsrc_pool_size = (size * num);
967 966 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
968 967 rsrc_pool->rsrc_quantum = size;
969 968 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
970 969 rsrc_pool->rsrc_state = state;
971 970 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
972 971 mbox_info.mbi_num = num;
973 972 mbox_info.mbi_size = size;
974 973 mbox_info.mbi_rsrcpool = rsrc_pool;
975 974 mbox_info.mbi_rsrcname = rsrc_name;
976 975 status = tavor_rsrc_mbox_init(state, &mbox_info);
977 976 if (status != DDI_SUCCESS) {
978 977 tavor_rsrc_fini(state, cleanup);
979 978 /* Set "status" and "errormsg" and goto failure */
980 979 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes");
981 980 goto rsrcinitp2_fail;
982 981 }
983 982 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;
984 983
985 984 /*
986 985 * Initialize the Tavor "In" mailbox list. This step actually uses
987 986 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
988 987 * "In" mailboxes, bind them for DMA access, and arrange them into
989 988 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
990 989 * for more details)
991 990 */
992 991 status = tavor_inmbox_list_init(state);
993 992 if (status != DDI_SUCCESS) {
994 993 tavor_rsrc_fini(state, cleanup);
995 994 /* Set "status" and "errormsg" and goto failure */
996 995 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list");
997 996 goto rsrcinitp2_fail;
998 997 }
999 998 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;
1000 999
1001 1000 /*
1002 1001 * Initialize the resource pool for interrupt "In" mailboxes. Notice
1003 1002 * that the number of interrupt "In" mailboxes, their size, and their
1004 1003 * location (DDR or system memory) is configurable. By default,
1005 1004 * however, all interrupt "In" mailboxes are located in system memory
1006 1005 * only (because they are primarily written to and rarely read from)
1007 1006 */
1008 1007 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
1009 1008 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
1010 1009 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
1011 1010 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX;
1012 1011 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1013 1012 rsrc_pool->rsrc_pool_size = (size * num);
1014 1013 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
1015 1014 rsrc_pool->rsrc_quantum = size;
1016 1015 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
1017 1016 rsrc_pool->rsrc_state = state;
1018 1017 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
1019 1018 mbox_info.mbi_num = num;
1020 1019 mbox_info.mbi_size = size;
1021 1020 mbox_info.mbi_rsrcpool = rsrc_pool;
1022 1021 mbox_info.mbi_rsrcname = rsrc_name;
1023 1022 status = tavor_rsrc_mbox_init(state, &mbox_info);
1024 1023 if (status != DDI_SUCCESS) {
1025 1024 tavor_rsrc_fini(state, cleanup);
1026 1025 /* Set "status" and "errormsg" and goto failure */
1027 1026 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes");
1028 1027 goto rsrcinitp2_fail;
1029 1028 }
1030 1029 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;
1031 1030
1032 1031 /*
1033 1032 * Initialize the Tavor interrupt "In" mailbox list. This step
1034 1033 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
1035 1034 * preallocate the interrupt "In" mailboxes, bind them for DMA access,
1036 1035 * and arrange them into an easily accessed fast-allocation mechanism
1037 1036 * (see tavor_cmd.c for more details)
1038 1037 */
1039 1038 status = tavor_intr_inmbox_list_init(state);
1040 1039 if (status != DDI_SUCCESS) {
1041 1040 tavor_rsrc_fini(state, cleanup);
1042 1041 /* Set "status" and "errormsg" and goto failure */
1043 1042 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list");
1044 1043 goto rsrcinitp2_fail;
1045 1044 }
1046 1045 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;
1047 1046
1048 1047 /*
1049 1048 * Initialize the Tavor command handling interfaces. This step
1050 1049 * sets up the outstanding command tracking mechanism for easy access
1051 1050 * and fast allocation (see tavor_cmd.c for more details).
1052 1051 */
1053 1052 status = tavor_outstanding_cmdlist_init(state);
1054 1053 if (status != DDI_SUCCESS) {
1055 1054 tavor_rsrc_fini(state, cleanup);
1056 1055 /* Set "status" and "errormsg" and goto failure */
1057 1056 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list");
1058 1057 goto rsrcinitp2_fail;
1059 1058 }
1060 1059 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;
1061 1060
1062 1061 /*
1063 1062 * Calculate (and validate) the size of Multicast Group (MCG) entries
1064 1063 */
1065 1064 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
1066 1065 if (status != DDI_SUCCESS) {
1067 1066 tavor_rsrc_fini(state, cleanup);
1068 1067 /* Set "status" and "errormsg" and goto failure */
1069 1068 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size");
1070 1069 goto rsrcinitp2_fail;
1071 1070 }
1072 1071 mcg_size = TAVOR_MCGMEM_SZ(state);
1073 1072
1074 1073 /*
1075 1074 * Initialize the resource pool for the MCG table entries. Notice
1076 1075 * that the number of MCGs is configurable. The configured value must
1077 1076 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1078 1077 * command) or the initialization will fail. Note also that a certain
1079 1078 * number of MCGs must be set aside for Tavor firmware use (they
1080 1079 * correspond to the number of MCGs used by the internal hash
1081 1080 * function.
1082 1081 */
1083 1082 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1084 1083 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1085 1084 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
1086 1085 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
1087 1086 rsrc_pool->rsrc_type = TAVOR_MCG;
1088 1087 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1089 1088 rsrc_pool->rsrc_pool_size = (mcg_size * num);
1090 1089 rsrc_pool->rsrc_shift = mcg_size_shift;
1091 1090 rsrc_pool->rsrc_quantum = mcg_size;
1092 1091 rsrc_pool->rsrc_align = mcg_size;
1093 1092 rsrc_pool->rsrc_state = state;
1094 1093 rsrc_pool->rsrc_start = NULL;
1095 1094 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
1096 1095 entry_info.hwi_num = num;
1097 1096 entry_info.hwi_max = max;
1098 1097 entry_info.hwi_prealloc = num_prealloc;
1099 1098 entry_info.hwi_rsrcpool = rsrc_pool;
1100 1099 entry_info.hwi_rsrcname = rsrc_name;
1101 1100 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1102 1101 if (status != DDI_SUCCESS) {
1103 1102 tavor_rsrc_fini(state, cleanup);
1104 1103 /* Set "status" and "errormsg" and goto failure */
1105 1104 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table");
1106 1105 goto rsrcinitp2_fail;
1107 1106 }
1108 1107 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;
1109 1108
1110 1109 /*
1111 1110 * Initialize the resource pool for the EQC table entries. Notice
1112 1111 * that the number of EQs is hardcoded. The hardcoded value should
1113 1112 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1114 1113 * command) or the initialization will fail.
1115 1114 */
1116 1115 num = TAVOR_NUM_EQ;
1117 1116 max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1118 1117 num_prealloc = 0;
1119 1118 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
1120 1119 rsrc_pool->rsrc_type = TAVOR_EQC;
1121 1120 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1122 1121 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
1123 1122 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT;
1124 1123 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE;
1125 1124 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num);
1126 1125 rsrc_pool->rsrc_state = state;
1127 1126 rsrc_pool->rsrc_start = NULL;
1128 1127 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
1129 1128 entry_info.hwi_num = num;
1130 1129 entry_info.hwi_max = max;
1131 1130 entry_info.hwi_prealloc = num_prealloc;
1132 1131 entry_info.hwi_rsrcpool = rsrc_pool;
1133 1132 entry_info.hwi_rsrcname = rsrc_name;
1134 1133 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1135 1134 if (status != DDI_SUCCESS) {
1136 1135 tavor_rsrc_fini(state, cleanup);
1137 1136 /* Set "status" and "errormsg" and goto failure */
1138 1137 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table");
1139 1138 goto rsrcinitp2_fail;
1140 1139 }
1141 1140 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;
1142 1141
1143 1142 /*
1144 1143 * Initialize the resource pools for all objects that exist in
1145 1144 * system memory. This includes PD handles, MR handle, EQ handles,
1146 1145 * QP handles, etc. These objects are almost entirely managed using
1147 1146 * kmem_cache routines. (See comment above for more detail)
1148 1147 */
1149 1148
1150 1149 /*
1151 1150 * Initialize the resource pool for the PD handles. Notice
1152 1151 * that the number of PDHDLs is configurable. The configured value
1153 1152 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1154 1153 * command) or the initialization will fail. Note also that the PD
1155 1154 * handle has constructor and destructor methods associated with it.
1156 1155 */
1157 1156 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1158 1157 rsrc_pool->rsrc_type = TAVOR_PDHDL;
1159 1158 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1160 1159 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s);
1161 1160 rsrc_pool->rsrc_state = state;
1162 1161 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
1163 1162 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1164 1163 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
1165 1164 hdl_info.swi_rsrcpool = rsrc_pool;
1166 1165 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
1167 1166 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor;
1168 1167 hdl_info.swi_rsrcname = rsrc_name;
1169 1168 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1170 1169 status = tavor_rsrc_pd_handles_init(state, &hdl_info);
1171 1170 if (status != DDI_SUCCESS) {
1172 1171 tavor_rsrc_fini(state, cleanup);
1173 1172 /* Set "status" and "errormsg" and goto failure */
1174 1173 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle");
1175 1174 goto rsrcinitp2_fail;
1176 1175 }
1177 1176 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;
1178 1177
1179 1178 /*
1180 1179 * Initialize the resource pool for the MR handles. Notice
1181 1180 * that the number of MRHDLs is configurable. The configured value
1182 1181 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1183 1182 * command) or the initialization will fail.
1184 1183 */
1185 1184 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1186 1185 rsrc_pool->rsrc_type = TAVOR_MRHDL;
1187 1186 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1188 1187 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s);
1189 1188 rsrc_pool->rsrc_state = state;
1190 1189 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
1191 1190 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1192 1191 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1193 1192 hdl_info.swi_rsrcpool = rsrc_pool;
1194 1193 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
1195 1194 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor;
1196 1195 hdl_info.swi_rsrcname = rsrc_name;
1197 1196 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1198 1197 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1199 1198 if (status != DDI_SUCCESS) {
1200 1199 tavor_rsrc_fini(state, cleanup);
1201 1200 /* Set "status" and "errormsg" and goto failure */
1202 1201 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle");
1203 1202 goto rsrcinitp2_fail;
1204 1203 }
1205 1204 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;
1206 1205
1207 1206 /*
1208 1207 * Initialize the resource pool for the EQ handles. Notice
1209 1208 * that the number of EQHDLs is hardcoded. The hardcoded value
1210 1209 * should be less that the maximum value (obtained from the
1211 1210 * QUERY_DEV_LIM command) or the initialization will fail.
1212 1211 */
1213 1212 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1214 1213 rsrc_pool->rsrc_type = TAVOR_EQHDL;
1215 1214 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1216 1215 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s);
1217 1216 rsrc_pool->rsrc_state = state;
1218 1217 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
1219 1218 hdl_info.swi_num = TAVOR_NUM_EQ;
1220 1219 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1221 1220 hdl_info.swi_rsrcpool = rsrc_pool;
1222 1221 hdl_info.swi_constructor = NULL;
1223 1222 hdl_info.swi_destructor = NULL;
1224 1223 hdl_info.swi_rsrcname = rsrc_name;
1225 1224 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1226 1225 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1227 1226 if (status != DDI_SUCCESS) {
1228 1227 tavor_rsrc_fini(state, cleanup);
1229 1228 /* Set "status" and "errormsg" and goto failure */
1230 1229 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle");
1231 1230 goto rsrcinitp2_fail;
1232 1231 }
1233 1232 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;
1234 1233
1235 1234 /*
1236 1235 * Initialize the resource pool for the CQ handles. Notice
1237 1236 * that the number of CQHDLs is configurable. The configured value
1238 1237 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1239 1238 * command) or the initialization will fail. Note also that the CQ
1240 1239 * handle has constructor and destructor methods associated with it.
1241 1240 */
1242 1241 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1243 1242 rsrc_pool->rsrc_type = TAVOR_CQHDL;
1244 1243 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1245 1244 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s);
1246 1245 rsrc_pool->rsrc_state = state;
1247 1246 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
1248 1247 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1249 1248 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
1250 1249 hdl_info.swi_rsrcpool = rsrc_pool;
1251 1250 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
1252 1251 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor;
1253 1252 hdl_info.swi_rsrcname = rsrc_name;
1254 1253 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1255 1254 TAVOR_SWHDL_TABLE_INIT);
1256 1255 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1257 1256 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1258 1257 if (status != DDI_SUCCESS) {
1259 1258 tavor_rsrc_fini(state, cleanup);
1260 1259 /* Set "status" and "errormsg" and goto failure */
1261 1260 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle");
1262 1261 goto rsrcinitp2_fail;
1263 1262 }
1264 1263
1265 1264 /*
1266 1265 * Save away the pointer to the central list of CQ handle pointers
1267 1266 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1268 1267 * lookup during EQ event processing. The table is a list of
1269 1268 * tavor_cqhdl_t allocated by the above routine because of the
1270 1269 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t
1271 1270 * as the number of CQs.
1272 1271 */
1273 1272 state->ts_cqhdl = hdl_info.swi_table_ptr;
1274 1273 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;
1275 1274
1276 1275 /*
1277 1276 * Initialize the resource pool for the SRQ handles. Notice
1278 1277 * that the number of SRQHDLs is configurable. The configured value
1279 1278 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1280 1279 * command) or the initialization will fail. Note also that the SRQ
1281 1280 * handle has constructor and destructor methods associated with it.
1282 1281 *
1283 1282 * Note: We only allocate these resources if SRQ is enabled in the
1284 1283 * config profile; see below.
1285 1284 */
1286 1285 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1287 1286 rsrc_pool->rsrc_type = TAVOR_SRQHDL;
1288 1287 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1289 1288 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s);
1290 1289 rsrc_pool->rsrc_state = state;
1291 1290 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
1292 1291 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1293 1292 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
1294 1293 hdl_info.swi_rsrcpool = rsrc_pool;
1295 1294 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
1296 1295 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor;
1297 1296 hdl_info.swi_rsrcname = rsrc_name;
1298 1297 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1299 1298 TAVOR_SWHDL_TABLE_INIT);
1300 1299 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1301 1300
1302 1301 /*
1303 1302 * SRQ support is configurable. Only if SRQ is enabled (the default)
1304 1303 * do we actually try to configure these resources. Otherwise, we
1305 1304 * simply set the cleanup level and continue on to the next resource
1306 1305 */
1307 1306 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1308 1307 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1309 1308 if (status != DDI_SUCCESS) {
1310 1309 tavor_rsrc_fini(state, cleanup);
1311 1310 /* Set "status" and "errormsg" and goto failure */
1312 1311 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle");
1313 1312 goto rsrcinitp2_fail;
1314 1313 }
1315 1314
1316 1315 /*
1317 1316 * Save away the pointer to the central list of SRQ handle
1318 1317 * pointers This this is used as a mechanism to enable fast
1319 1318 * SRQnumber-to-SRQhandle lookup. The table is a list of
1320 1319 * tavor_srqhdl_t allocated by the above routine because of the
1321 1320 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many
1322 1321 * tavor_srqhdl_t as the number of SRQs.
1323 1322 */
1324 1323 state->ts_srqhdl = hdl_info.swi_table_ptr;
1325 1324 }
1326 1325 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;
1327 1326
1328 1327 /*
1329 1328 * Initialize the resource pool for the address handles. Notice
1330 1329 * that the number of AHHDLs is configurable. The configured value
1331 1330 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1332 1331 * command) or the initialization will fail.
1333 1332 */
1334 1333 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1335 1334 rsrc_pool->rsrc_type = TAVOR_AHHDL;
1336 1335 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1337 1336 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s);
1338 1337 rsrc_pool->rsrc_state = state;
1339 1338 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
1340 1339 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
1341 1340 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
1342 1341 hdl_info.swi_rsrcpool = rsrc_pool;
1343 1342 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
1344 1343 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor;
1345 1344 hdl_info.swi_rsrcname = rsrc_name;
1346 1345 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1347 1346 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1348 1347 if (status != DDI_SUCCESS) {
1349 1348 tavor_rsrc_fini(state, cleanup);
1350 1349 /* Set "status" and "errormsg" and goto failure */
1351 1350 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle");
1352 1351 goto rsrcinitp2_fail;
1353 1352 }
1354 1353 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;
1355 1354
1356 1355 /*
1357 1356 * Initialize the resource pool for the QP handles. Notice
1358 1357 * that the number of QPHDLs is configurable. The configured value
1359 1358 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1360 1359 * command) or the initialization will fail. Note also that the QP
1361 1360 * handle has constructor and destructor methods associated with it.
1362 1361 */
1363 1362 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1364 1363 rsrc_pool->rsrc_type = TAVOR_QPHDL;
1365 1364 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1366 1365 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s);
1367 1366 rsrc_pool->rsrc_state = state;
1368 1367 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
1369 1368 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1370 1369 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
1371 1370 hdl_info.swi_rsrcpool = rsrc_pool;
1372 1371 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
1373 1372 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor;
1374 1373 hdl_info.swi_rsrcname = rsrc_name;
1375 1374 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1376 1375 TAVOR_SWHDL_TABLE_INIT);
1377 1376 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1378 1377 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1379 1378 if (status != DDI_SUCCESS) {
1380 1379 tavor_rsrc_fini(state, cleanup);
1381 1380 /* Set "status" and "errormsg" and goto failure */
1382 1381 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle");
1383 1382 goto rsrcinitp2_fail;
1384 1383 }
1385 1384
1386 1385 /*
1387 1386 * Save away the pointer to the central list of QP handle pointers
1388 1387 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1389 1388 * lookup during CQ event processing. The table is a list of
1390 1389 * tavor_qphdl_t allocated by the above routine because of the
1391 1390 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t
1392 1391 * as the number of QPs.
1393 1392 */
1394 1393 state->ts_qphdl = hdl_info.swi_table_ptr;
1395 1394 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;
1396 1395
1397 1396 /*
1398 1397 * Initialize the resource pool for the reference count handles.
1399 1398 * Notice that the number of REFCNTs is configurable, but it's value
1400 1399 * is set to the number of MPTs. Since REFCNTs are used to support
1401 1400 * shared memory regions, it is possible that we might require as
1402 1401 * one REFCNT for every MPT.
1403 1402 */
1404 1403 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1405 1404 rsrc_pool->rsrc_type = TAVOR_REFCNT;
1406 1405 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1407 1406 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t);
1408 1407 rsrc_pool->rsrc_state = state;
1409 1408 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
1410 1409 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1411 1410 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1412 1411 hdl_info.swi_rsrcpool = rsrc_pool;
1413 1412 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
1414 1413 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor;
1415 1414 hdl_info.swi_rsrcname = rsrc_name;
1416 1415 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1417 1416 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1418 1417 if (status != DDI_SUCCESS) {
1419 1418 tavor_rsrc_fini(state, cleanup);
1420 1419 /* Set "status" and "errormsg" and goto failure */
1421 1420 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle");
1422 1421 goto rsrcinitp2_fail;
1423 1422 }
1424 1423 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;
1425 1424
1426 1425 /*
1427 1426 * Initialize the resource pool for the MCG handles. Notice that for
1428 1427 * these MCG handles, we are allocating a table of structures (used to
1429 1428 * keep track of the MCG entries that are being written to hardware
1430 1429 * and to speed up multicast attach/detach operations).
1431 1430 */
1432 1431 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1433 1432 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1434 1433 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
1435 1434 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1436 1435 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1437 1436 if (status != DDI_SUCCESS) {
1438 1437 tavor_rsrc_fini(state, cleanup);
1439 1438 /* Set "status" and "errormsg" and goto failure */
1440 1439 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle");
1441 1440 goto rsrcinitp2_fail;
1442 1441 }
1443 1442 state->ts_mcghdl = hdl_info.swi_table_ptr;
1444 1443 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;
1445 1444
1446 1445 /*
1447 1446 * Initialize the resource pools for all objects that exist in
1448 1447 * UAR memory. The only objects that are allocated from UAR memory
1449 1448 * are the UAR pages which are used for holding Tavor hardware's
1450 1449 * doorbell registers.
1451 1450 */
1452 1451
1453 1452 /*
1454 1453 * Initialize the resource pool for the UAR pages. Notice
1455 1454 * that the number of UARPGs is configurable. The configured value
1456 1455 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1457 1456 * command) or the initialization will fail. Note also that by
1458 1457 * specifying the rsrc_start parameter in advance, we direct the
1459 1458 * initialization routine not to attempt to allocated space from the
1460 1459 * Tavor DDR vmem_arena.
1461 1460 */
1462 1461 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1463 1462 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
1464 1463 PAGESHIFT));
1465 1464 num_prealloc = 0;
1466 1465 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1467 1466 rsrc_pool->rsrc_type = TAVOR_UARPG;
1468 1467 rsrc_pool->rsrc_loc = TAVOR_IN_UAR;
1469 1468 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1470 1469 rsrc_pool->rsrc_shift = PAGESHIFT;
1471 1470 rsrc_pool->rsrc_quantum = PAGESIZE;
1472 1471 rsrc_pool->rsrc_align = PAGESIZE;
1473 1472 rsrc_pool->rsrc_state = state;
1474 1473 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr;
1475 1474 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
1476 1475 entry_info.hwi_num = num;
1477 1476 entry_info.hwi_max = max;
1478 1477 entry_info.hwi_prealloc = num_prealloc;
1479 1478 entry_info.hwi_rsrcpool = rsrc_pool;
1480 1479 entry_info.hwi_rsrcname = rsrc_name;
1481 1480 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1482 1481 if (status != DDI_SUCCESS) {
1483 1482 tavor_rsrc_fini(state, cleanup);
1484 1483 /* Set "status" and "errormsg" and goto failure */
1485 1484 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table");
1486 1485 goto rsrcinitp2_fail;
1487 1486 }
1488 1487 cleanup = TAVOR_RSRC_CLEANUP_ALL;
1489 1488
1490 1489 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1491 1490 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1492 1491 return (DDI_SUCCESS);
1493 1492
1494 1493 rsrcinitp2_fail:
1495 1494 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1496 1495 TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "",
1497 1496 tnf_string, msg, errormsg);
1498 1497 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1499 1498 return (status);
1500 1499 }
1501 1500
1502 1501
1503 1502 /*
1504 1503 * tavor_rsrc_fini()
1505 1504 * Context: Only called from attach() and/or detach() path contexts
1506 1505 */
1507 1506 void
1508 1507 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
1509 1508 {
1510 1509 tavor_rsrc_sw_hdl_info_t hdl_info;
1511 1510 tavor_rsrc_hw_entry_info_t entry_info;
1512 1511 tavor_rsrc_mbox_info_t mbox_info;
1513 1512 tavor_cfg_profile_t *cfgprof;
1514 1513
1515 1514 TAVOR_TNF_ENTER(tavor_rsrc_fini);
1516 1515
1517 1516 ASSERT(state != NULL);
1518 1517
1519 1518 cfgprof = state->ts_cfg_profile;
1520 1519
1521 1520 switch (clean) {
1522 1521 /*
1523 1522 * If we add more resources that need to be cleaned up here, we should
1524 1523 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1525 1524 * corresponds to the last resource allocated).
1526 1525 */
1527 1526 case TAVOR_RSRC_CLEANUP_ALL:
1528 1527 /* Cleanup the UAR page resource pool */
1529 1528 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1530 1529 tavor_rsrc_hw_entries_fini(state, &entry_info);
1531 1530 /* FALLTHROUGH */
1532 1531
1533 1532 case TAVOR_RSRC_CLEANUP_LEVEL30:
1534 1533 /* Cleanup the central MCG handle pointers list */
1535 1534 hdl_info.swi_rsrcpool = NULL;
1536 1535 hdl_info.swi_table_ptr = state->ts_mcghdl;
1537 1536 hdl_info.swi_num =
1538 1537 ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1539 1538 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1540 1539 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1541 1540 /* FALLTHROUGH */
1542 1541
1543 1542 case TAVOR_RSRC_CLEANUP_LEVEL29:
1544 1543 /* Cleanup the reference count resource pool */
1545 1544 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1546 1545 hdl_info.swi_table_ptr = NULL;
1547 1546 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1548 1547 /* FALLTHROUGH */
1549 1548
1550 1549 case TAVOR_RSRC_CLEANUP_LEVEL28:
1551 1550 /* Cleanup the QP handle resource pool */
1552 1551 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1553 1552 hdl_info.swi_table_ptr = state->ts_qphdl;
1554 1553 hdl_info.swi_num =
1555 1554 ((uint64_t)1 << cfgprof->cp_log_num_qp);
1556 1555 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1557 1556 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1558 1557 /* FALLTHROUGH */
1559 1558
1560 1559 case TAVOR_RSRC_CLEANUP_LEVEL27:
1561 1560 /* Cleanup the address handle resource pool */
1562 1561 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1563 1562 hdl_info.swi_table_ptr = NULL;
1564 1563 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1565 1564 /* FALLTHROUGH */
1566 1565
1567 1566 case TAVOR_RSRC_CLEANUP_LEVEL26:
1568 1567 /*
1569 1568 * Cleanup the SRQ handle resource pool.
1570 1569 *
1571 1570 * Note: We only clean up if SRQ is enabled. Otherwise we
1572 1571 * simply fallthrough to the next resource cleanup.
1573 1572 */
1574 1573 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1575 1574 hdl_info.swi_rsrcpool =
1576 1575 &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1577 1576 hdl_info.swi_table_ptr = state->ts_srqhdl;
1578 1577 hdl_info.swi_num =
1579 1578 ((uint64_t)1 << cfgprof->cp_log_num_srq);
1580 1579 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1581 1580 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1582 1581 }
1583 1582 /* FALLTHROUGH */
1584 1583
1585 1584 case TAVOR_RSRC_CLEANUP_LEVEL25:
1586 1585 /* Cleanup the CQ handle resource pool */
1587 1586 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1588 1587 hdl_info.swi_table_ptr = state->ts_cqhdl;
1589 1588 hdl_info.swi_num =
1590 1589 ((uint64_t)1 << cfgprof->cp_log_num_cq);
1591 1590 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1592 1591 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1593 1592 /* FALLTHROUGH */
1594 1593
1595 1594 case TAVOR_RSRC_CLEANUP_LEVEL24:
1596 1595 /* Cleanup the EQ handle resource pool */
1597 1596 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1598 1597 hdl_info.swi_table_ptr = NULL;
1599 1598 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1600 1599 /* FALLTHROUGH */
1601 1600
1602 1601 case TAVOR_RSRC_CLEANUP_LEVEL23:
1603 1602 /* Cleanup the MR handle resource pool */
1604 1603 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1605 1604 hdl_info.swi_table_ptr = NULL;
1606 1605 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1607 1606 /* FALLTHROUGH */
1608 1607
1609 1608 case TAVOR_RSRC_CLEANUP_LEVEL22:
1610 1609 /* Cleanup the PD handle resource pool */
1611 1610 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1612 1611 hdl_info.swi_table_ptr = NULL;
1613 1612 tavor_rsrc_pd_handles_fini(state, &hdl_info);
1614 1613 /* FALLTHROUGH */
1615 1614
1616 1615 case TAVOR_RSRC_CLEANUP_LEVEL21:
1617 1616 /* Cleanup the EQC table resource pool */
1618 1617 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
1619 1618 tavor_rsrc_hw_entries_fini(state, &entry_info);
1620 1619 /* FALLTHROUGH */
1621 1620
1622 1621 case TAVOR_RSRC_CLEANUP_LEVEL20:
1623 1622 /* Cleanup the MCG table resource pool */
1624 1623 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
1625 1624 tavor_rsrc_hw_entries_fini(state, &entry_info);
1626 1625 /* FALLTHROUGH */
1627 1626
1628 1627 case TAVOR_RSRC_CLEANUP_LEVEL19:
1629 1628 /* Cleanup the outstanding command list */
1630 1629 tavor_outstanding_cmdlist_fini(state);
1631 1630 /* FALLTHROUGH */
1632 1631
1633 1632 case TAVOR_RSRC_CLEANUP_LEVEL18:
1634 1633 /* Cleanup the "In" mailbox list */
1635 1634 tavor_intr_inmbox_list_fini(state);
1636 1635 /* FALLTHROUGH */
1637 1636
1638 1637 case TAVOR_RSRC_CLEANUP_LEVEL17:
1639 1638 /* Cleanup the interrupt "In" mailbox resource pool */
1640 1639 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1641 1640 TAVOR_INTR_IN_MBOX];
1642 1641 tavor_rsrc_mbox_fini(state, &mbox_info);
1643 1642 /* FALLTHROUGH */
1644 1643
1645 1644 case TAVOR_RSRC_CLEANUP_LEVEL16:
1646 1645 /* Cleanup the "In" mailbox list */
1647 1646 tavor_inmbox_list_fini(state);
1648 1647 /* FALLTHROUGH */
1649 1648
1650 1649 case TAVOR_RSRC_CLEANUP_LEVEL15:
1651 1650 /* Cleanup the "In" mailbox resource pool */
1652 1651 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
1653 1652 tavor_rsrc_mbox_fini(state, &mbox_info);
1654 1653 /* FALLTHROUGH */
1655 1654
1656 1655 case TAVOR_RSRC_CLEANUP_LEVEL14:
1657 1656 /*
1658 1657 * Cleanup the SRQC table resource pool.
1659 1658 *
1660 1659 * Note: We only clean up if SRQ is enabled. Otherwise we
1661 1660 * simply fallthrough to the next resource cleanup.
1662 1661 */
1663 1662 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1664 1663 entry_info.hwi_rsrcpool =
1665 1664 &state->ts_rsrc_hdl[TAVOR_SRQC];
1666 1665 tavor_rsrc_hw_entries_fini(state, &entry_info);
1667 1666 }
1668 1667 /* FALLTHROUGH */
1669 1668
1670 1669 case TAVOR_RSRC_CLEANUP_LEVEL13:
1671 1670 /* Cleanup the UAR scratch table resource pool */
1672 1671 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
1673 1672 tavor_rsrc_hw_entries_fini(state, &entry_info);
1674 1673 /* FALLTHROUGH */
1675 1674
1676 1675 case TAVOR_RSRC_CLEANUP_LEVEL12:
1677 1676 /* Cleanup the UDAV table resource pool */
1678 1677 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
1679 1678 tavor_rsrc_hw_entries_fini(state, &entry_info);
1680 1679 /* FALLTHROUGH */
1681 1680
1682 1681 case TAVOR_RSRC_CLEANUP_LEVEL11:
1683 1682 /* Cleanup the EQPC table resource pool */
1684 1683 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
1685 1684 tavor_rsrc_hw_entries_fini(state, &entry_info);
1686 1685 /* FALLTHROUGH */
1687 1686
1688 1687 case TAVOR_RSRC_CLEANUP_LEVEL10:
1689 1688 /* Cleanup the CQC table resource pool */
1690 1689 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
1691 1690 tavor_rsrc_hw_entries_fini(state, &entry_info);
1692 1691 /* FALLTHROUGH */
1693 1692
1694 1693 case TAVOR_RSRC_CLEANUP_LEVEL9:
1695 1694 /* Cleanup the RDB table resource pool */
1696 1695 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
1697 1696 tavor_rsrc_hw_entries_fini(state, &entry_info);
1698 1697 /* FALLTHROUGH */
1699 1698
1700 1699 case TAVOR_RSRC_CLEANUP_LEVEL8:
1701 1700 /* Cleanup the QPC table resource pool */
1702 1701 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
1703 1702 tavor_rsrc_hw_entries_fini(state, &entry_info);
1704 1703 /* FALLTHROUGH */
1705 1704
1706 1705 case TAVOR_RSRC_CLEANUP_LEVEL7:
1707 1706 /* Cleanup the MTT table resource pool */
1708 1707 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
1709 1708 tavor_rsrc_hw_entries_fini(state, &entry_info);
1710 1709 /* FALLTHROUGH */
1711 1710
1712 1711 case TAVOR_RSRC_CLEANUP_LEVEL6:
1713 1712 /* Cleanup the MPT table resource pool */
1714 1713 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
1715 1714 tavor_rsrc_hw_entries_fini(state, &entry_info);
1716 1715 /* FALLTHROUGH */
1717 1716
1718 1717 case TAVOR_RSRC_CLEANUP_LEVEL5:
1719 1718 /* Destroy the vmem arena for DDR memory */
1720 1719 vmem_destroy(state->ts_ddrvmem);
1721 1720 break;
1722 1721
1723 1722 /*
1724 1723 * The cleanup below comes from the "Phase 1" initialization step.
1725 1724 * (see tavor_rsrc_init_phase1() above)
1726 1725 */
1727 1726 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
1728 1727 /* Cleanup the interrupt "Out" mailbox list */
1729 1728 tavor_intr_outmbox_list_fini(state);
1730 1729 /* FALLTHROUGH */
1731 1730
1732 1731 case TAVOR_RSRC_CLEANUP_LEVEL3:
1733 1732 /* Cleanup the "Out" mailbox resource pool */
1734 1733 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1735 1734 TAVOR_INTR_OUT_MBOX];
1736 1735 tavor_rsrc_mbox_fini(state, &mbox_info);
1737 1736 /* FALLTHROUGH */
1738 1737
1739 1738 case TAVOR_RSRC_CLEANUP_LEVEL2:
1740 1739 /* Cleanup the "Out" mailbox list */
1741 1740 tavor_outmbox_list_fini(state);
1742 1741 /* FALLTHROUGH */
1743 1742
1744 1743 case TAVOR_RSRC_CLEANUP_LEVEL1:
1745 1744 /* Cleanup the "Out" mailbox resource pool */
1746 1745 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
1747 1746 tavor_rsrc_mbox_fini(state, &mbox_info);
1748 1747 /* FALLTHROUGH */
1749 1748
1750 1749 case TAVOR_RSRC_CLEANUP_LEVEL0:
1751 1750 /* Free the array of tavor_rsrc_pool_info_t's */
1752 1751 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
1753 1752 sizeof (tavor_rsrc_pool_info_t));
1754 1753 kmem_cache_destroy(state->ts_rsrc_cache);
1755 1754 break;
1756 1755
1757 1756 default:
1758 1757 TAVOR_WARNING(state, "unexpected resource cleanup level");
1759 1758 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, "");
1760 1759 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1761 1760 return;
1762 1761 }
1763 1762
1764 1763 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1765 1764 }
1766 1765
1767 1766
1768 1767 /*
1769 1768 * tavor_rsrc_mbox_init()
1770 1769 * Context: Only called from attach() path context
1771 1770 */
1772 1771 static int
1773 1772 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1774 1773 {
1775 1774 tavor_rsrc_pool_info_t *rsrc_pool;
1776 1775 tavor_rsrc_priv_mbox_t *priv;
1777 1776 vmem_t *vmp;
1778 1777 uint64_t offset;
1779 1778 uint_t dma_xfer_mode;
1780 1779
1781 1780 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init);
1782 1781
1783 1782 ASSERT(state != NULL);
1784 1783 ASSERT(info != NULL);
1785 1784
1786 1785 rsrc_pool = info->mbi_rsrcpool;
1787 1786 ASSERT(rsrc_pool != NULL);
1788 1787
1789 1788 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1790 1789
1791 1790 /* Allocate and initialize mailbox private structure */
1792 1791 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
1793 1792 priv->pmb_dip = state->ts_dip;
1794 1793 priv->pmb_acchdl = state->ts_reg_ddrhdl;
1795 1794 priv->pmb_devaccattr = state->ts_reg_accattr;
1796 1795 priv->pmb_xfer_mode = dma_xfer_mode;
1797 1796
1798 1797 /*
1799 1798 * Initialize many of the default DMA attributes. Then set alignment
1800 1799 * and scatter-gather restrictions specific for mailbox memory.
1801 1800 */
1802 1801 tavor_dma_attr_init(&priv->pmb_dmaattr);
1803 1802 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN;
1804 1803 priv->pmb_dmaattr.dma_attr_sgllen = 1;
1805 1804
1806 1805 rsrc_pool->rsrc_private = priv;
1807 1806
1808 1807 /* Is object in DDR memory or system memory? */
1809 1808 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1810 1809 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1811 1810 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1812 1811 0, 0, NULL, NULL, VM_SLEEP);
1813 1812 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1814 1813 /* Unable to alloc space for mailboxes */
1815 1814 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1816 1815 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail,
1817 1816 TAVOR_TNF_ERROR, "");
1818 1817 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1819 1818 return (DDI_FAILURE);
1820 1819 }
1821 1820
1822 1821 /* Calculate offset and starting point (in DDR) */
1823 1822 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1824 1823 state->ts_ddr.ddr_baseaddr);
1825 1824 rsrc_pool->rsrc_start =
1826 1825 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1827 1826 offset);
1828 1827
1829 1828 /* Create new vmem arena for the mailboxes */
1830 1829 vmp = vmem_create(info->mbi_rsrcname,
1831 1830 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1832 1831 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1833 1832 if (vmp == NULL) {
1834 1833 /* Unable to create vmem arena */
1835 1834 vmem_xfree(state->ts_ddrvmem,
1836 1835 rsrc_pool->rsrc_ddr_offset,
1837 1836 rsrc_pool->rsrc_pool_size);
1838 1837 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1839 1838 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail,
1840 1839 TAVOR_TNF_ERROR, "");
1841 1840 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1842 1841 return (DDI_FAILURE);
1843 1842 }
1844 1843 rsrc_pool->rsrc_vmp = vmp;
1845 1844 } else {
1846 1845 rsrc_pool->rsrc_ddr_offset = NULL;
1847 1846 rsrc_pool->rsrc_start = NULL;
1848 1847 rsrc_pool->rsrc_vmp = NULL;
1849 1848 }
1850 1849
1851 1850 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1852 1851 return (DDI_SUCCESS);
1853 1852 }
1854 1853
1855 1854
1856 1855 /*
1857 1856 * tavor_rsrc_mbox_fini()
1858 1857 * Context: Only called from attach() and/or detach() path contexts
1859 1858 */
1860 1859 static void
1861 1860 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1862 1861 {
1863 1862 tavor_rsrc_pool_info_t *rsrc_pool;
1864 1863
1865 1864 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini);
1866 1865
1867 1866 ASSERT(state != NULL);
1868 1867 ASSERT(info != NULL);
1869 1868
1870 1869 rsrc_pool = info->mbi_rsrcpool;
1871 1870 ASSERT(rsrc_pool != NULL);
1872 1871
1873 1872 /* If mailboxes are DDR memory, then destroy and free up vmem */
1874 1873 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1875 1874
1876 1875 /* Destroy the specially created mbox vmem arena */
1877 1876 vmem_destroy(rsrc_pool->rsrc_vmp);
1878 1877
1879 1878 /* Free up the region from the ddr_vmem arena */
1880 1879 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1881 1880 rsrc_pool->rsrc_pool_size);
1882 1881 }
1883 1882
1884 1883 /* Free up the private struct */
1885 1884 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
1886 1885
1887 1886 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini);
1888 1887 }
1889 1888
1890 1889
1891 1890 /*
1892 1891 * tavor_rsrc_hw_entries_init()
1893 1892 * Context: Only called from attach() path context
1894 1893 */
1895 1894 static int
1896 1895 tavor_rsrc_hw_entries_init(tavor_state_t *state,
1897 1896 tavor_rsrc_hw_entry_info_t *info)
1898 1897 {
1899 1898 tavor_rsrc_pool_info_t *rsrc_pool;
1900 1899 tavor_rsrc_t *rsvd_rsrc = NULL;
1901 1900 vmem_t *vmp;
1902 1901 uint64_t num_hwentry, max_hwentry, num_prealloc;
1903 1902 uint64_t offset;
1904 1903 int status;
1905 1904
1906 1905 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init);
1907 1906
1908 1907 ASSERT(state != NULL);
1909 1908 ASSERT(info != NULL);
1910 1909
1911 1910 rsrc_pool = info->hwi_rsrcpool;
1912 1911 ASSERT(rsrc_pool != NULL);
1913 1912 num_hwentry = info->hwi_num;
1914 1913 max_hwentry = info->hwi_max;
1915 1914 num_prealloc = info->hwi_prealloc;
1916 1915
1917 1916 /* Make sure number of HW entries makes sense */
1918 1917 if (num_hwentry > max_hwentry) {
1919 1918 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail,
1920 1919 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW "
1921 1920 "entries exceeds device maximum", tnf_uint, maxhw,
1922 1921 max_hwentry);
1923 1922 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1924 1923 return (DDI_FAILURE);
1925 1924 }
1926 1925
1927 1926 /*
1928 1927 * Determine if we need to allocate DDR space to set up the
1929 1928 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already
1930 1929 * been initialized (as is the case for the UAR page init).
1931 1930 */
1932 1931 if (rsrc_pool->rsrc_start == NULL) {
1933 1932 /* Make sure HW entries table is aligned as specified */
1934 1933 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1935 1934 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1936 1935 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
1937 1936 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1938 1937 /* Unable to alloc space for aligned HW table */
1939 1938 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail,
1940 1939 TAVOR_TNF_ERROR, "");
1941 1940 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1942 1941 return (DDI_FAILURE);
1943 1942 }
1944 1943
1945 1944 /* Calculate offset and starting point (in DDR) */
1946 1945 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1947 1946 state->ts_ddr.ddr_baseaddr);
1948 1947 rsrc_pool->rsrc_start =
1949 1948 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1950 1949 offset);
1951 1950 } else {
1952 1951 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
1953 1952 }
1954 1953
1955 1954 /*
1956 1955 * Create new vmem arena for the HW entries table (if rsrc_quantum
1957 1956 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW
1958 1957 * entries are not going to be dynamically allocatable (i.e. they
1959 1958 * won't be allocated/freed through tavor_rsrc_alloc/free). This
1960 1959 * latter option is used for EQPC and UARSCR resource which are, in
1961 1960 * fact, managed by the Tavor hardware.
1962 1961 */
1963 1962 if (rsrc_pool->rsrc_quantum != 0) {
1964 1963 vmp = vmem_create(info->hwi_rsrcname,
1965 1964 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1966 1965 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1967 1966 if (vmp == NULL) {
1968 1967 /* Unable to create vmem arena */
1969 1968 if (rsrc_pool->rsrc_ddr_offset !=
1970 1969 rsrc_pool->rsrc_start) {
1971 1970 vmem_xfree(state->ts_ddrvmem,
1972 1971 rsrc_pool->rsrc_ddr_offset,
1973 1972 rsrc_pool->rsrc_pool_size);
1974 1973 }
1975 1974 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail,
1976 1975 TAVOR_TNF_ERROR, "");
1977 1976 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1978 1977 return (DDI_FAILURE);
1979 1978 }
1980 1979 rsrc_pool->rsrc_vmp = vmp;
1981 1980 } else {
1982 1981 rsrc_pool->rsrc_vmp = NULL;
1983 1982 }
1984 1983
1985 1984 /* The first HW entries may be reserved by Tavor firmware */
1986 1985 if (num_prealloc != 0) {
1987 1986 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
1988 1987 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
1989 1988 if (status != DDI_SUCCESS) {
1990 1989 /* Unable to preallocate the reserved HW entries */
1991 1990 if (rsrc_pool->rsrc_vmp != NULL) {
1992 1991 vmem_destroy(rsrc_pool->rsrc_vmp);
1993 1992 }
1994 1993 if (rsrc_pool->rsrc_ddr_offset !=
1995 1994 rsrc_pool->rsrc_start) {
1996 1995 vmem_xfree(state->ts_ddrvmem,
1997 1996 rsrc_pool->rsrc_ddr_offset,
1998 1997 rsrc_pool->rsrc_pool_size);
1999 1998 }
2000 1999 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail,
2001 2000 TAVOR_TNF_ERROR, "");
2002 2001 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2003 2002 return (DDI_FAILURE);
2004 2003 }
2005 2004 }
2006 2005 rsrc_pool->rsrc_private = rsvd_rsrc;
2007 2006
2008 2007 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2009 2008 return (DDI_SUCCESS);
2010 2009 }
2011 2010
2012 2011
2013 2012 /*
2014 2013 * tavor_rsrc_hw_entries_fini()
2015 2014 * Context: Only called from attach() and/or detach() path contexts
2016 2015 */
2017 2016 static void
2018 2017 tavor_rsrc_hw_entries_fini(tavor_state_t *state,
2019 2018 tavor_rsrc_hw_entry_info_t *info)
2020 2019 {
2021 2020 tavor_rsrc_pool_info_t *rsrc_pool;
2022 2021 tavor_rsrc_t *rsvd_rsrc;
2023 2022
2024 2023 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini);
2025 2024
2026 2025 ASSERT(state != NULL);
2027 2026 ASSERT(info != NULL);
2028 2027
2029 2028 rsrc_pool = info->hwi_rsrcpool;
2030 2029 ASSERT(rsrc_pool != NULL);
2031 2030
2032 2031 /* Free up any "reserved" (i.e. preallocated) HW entries */
2033 2032 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
2034 2033 if (rsvd_rsrc != NULL) {
2035 2034 tavor_rsrc_free(state, &rsvd_rsrc);
2036 2035 }
2037 2036
2038 2037 /*
2039 2038 * If we've actually setup a vmem arena for the HW entries, then
2040 2039 * destroy it now
2041 2040 */
2042 2041 if (rsrc_pool->rsrc_vmp != NULL) {
2043 2042 vmem_destroy(rsrc_pool->rsrc_vmp);
2044 2043 }
2045 2044
2046 2045 /*
2047 2046 * Determine if a region was allocated from the tavor_ddr_vmem
2048 2047 * arena (and free it up if necessary)
2049 2048 */
2050 2049 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
2051 2050 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
2052 2051 rsrc_pool->rsrc_pool_size);
2053 2052 }
2054 2053
2055 2054 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini);
2056 2055 }
2057 2056
2058 2057
2059 2058 /*
2060 2059 * tavor_rsrc_sw_handles_init()
2061 2060 * Context: Only called from attach() path context
2062 2061 */
2063 2062 /* ARGSUSED */
2064 2063 static int
2065 2064 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2066 2065 {
2067 2066 tavor_rsrc_pool_info_t *rsrc_pool;
2068 2067 uint64_t num_swhdl, max_swhdl, prealloc_sz;
2069 2068
2070 2069 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init);
2071 2070
2072 2071 ASSERT(state != NULL);
2073 2072 ASSERT(info != NULL);
2074 2073
2075 2074 rsrc_pool = info->swi_rsrcpool;
2076 2075 ASSERT(rsrc_pool != NULL);
2077 2076 num_swhdl = info->swi_num;
2078 2077 max_swhdl = info->swi_max;
2079 2078 prealloc_sz = info->swi_prealloc_sz;
2080 2079
2081 2080 /* Make sure number of SW handles makes sense */
2082 2081 if (num_swhdl > max_swhdl) {
2083 2082 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail,
2084 2083 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW "
2085 2084 "handles exceeds maximum", tnf_uint, maxsw, max_swhdl);
2086 2085 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2087 2086 return (DDI_FAILURE);
2088 2087 }
2089 2088
2090 2089 /*
2091 2090 * Depending on the flags parameter, create a kmem_cache for some
2092 2091 * number of software handle structures. Note: kmem_cache_create()
2093 2092 * will SLEEP until successful.
2094 2093 */
2095 2094 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
2096 2095 rsrc_pool->rsrc_private = kmem_cache_create(
2097 2096 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
2098 2097 info->swi_constructor, info->swi_destructor, NULL,
2099 2098 rsrc_pool->rsrc_state, NULL, 0);
2100 2099 }
2101 2100
2102 2101 /* Allocate the central list of SW handle pointers */
2103 2102 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
2104 2103 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
2105 2104 KM_SLEEP);
2106 2105 }
2107 2106
2108 2107 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2109 2108 return (DDI_SUCCESS);
2110 2109 }
2111 2110
2112 2111
2113 2112 /*
2114 2113 * tavor_rsrc_sw_handles_fini()
2115 2114 * Context: Only called from attach() and/or detach() path contexts
2116 2115 */
2117 2116 /* ARGSUSED */
2118 2117 static void
2119 2118 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2120 2119 {
2121 2120 tavor_rsrc_pool_info_t *rsrc_pool;
2122 2121 uint64_t num_swhdl, prealloc_sz;
2123 2122
2124 2123 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini);
2125 2124
2126 2125 ASSERT(state != NULL);
2127 2126 ASSERT(info != NULL);
2128 2127
2129 2128 rsrc_pool = info->swi_rsrcpool;
2130 2129 num_swhdl = info->swi_num;
2131 2130 prealloc_sz = info->swi_prealloc_sz;
2132 2131
2133 2132 /*
2134 2133 * If a "software handle" kmem_cache exists for this resource, then
2135 2134 * destroy it now
2136 2135 */
2137 2136 if (rsrc_pool != NULL) {
2138 2137 kmem_cache_destroy(rsrc_pool->rsrc_private);
2139 2138 }
2140 2139
2141 2140 /* Free up this central list of SW handle pointers */
2142 2141 if (info->swi_table_ptr != NULL) {
2143 2142 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
2144 2143 }
2145 2144
2146 2145 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini);
2147 2146 }
2148 2147
2149 2148
2150 2149 /*
2151 2150 * tavor_rsrc_pd_handles_init()
2152 2151 * Context: Only called from attach() path context
2153 2152 */
2154 2153 static int
2155 2154 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2156 2155 {
2157 2156 tavor_rsrc_pool_info_t *rsrc_pool;
2158 2157 vmem_t *vmp;
2159 2158 char vmem_name[TAVOR_RSRC_NAME_MAXLEN];
2160 2159 int status;
2161 2160
2162 2161 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init);
2163 2162
2164 2163 ASSERT(state != NULL);
2165 2164 ASSERT(info != NULL);
2166 2165
2167 2166 rsrc_pool = info->swi_rsrcpool;
2168 2167 ASSERT(rsrc_pool != NULL);
2169 2168
2170 2169 /* Initialize the resource pool for software handle table */
2171 2170 status = tavor_rsrc_sw_handles_init(state, info);
2172 2171 if (status != DDI_SUCCESS) {
2173 2172 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2174 2173 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2175 2174 return (DDI_FAILURE);
2176 2175 }
2177 2176
2178 2177 /* Build vmem arena name from Tavor instance */
2179 2178 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);
2180 2179
2181 2180 /* Create new vmem arena for PD numbers */
2182 2181 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
2183 2182 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
2184 2183 if (vmp == NULL) {
2185 2184 /* Unable to create vmem arena */
2186 2185 info->swi_table_ptr = NULL;
2187 2186 tavor_rsrc_sw_handles_fini(state, info);
2188 2187 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail,
2189 2188 TAVOR_TNF_ERROR, "");
2190 2189 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2191 2190 return (DDI_FAILURE);
2192 2191 }
2193 2192 rsrc_pool->rsrc_vmp = vmp;
2194 2193
2195 2194 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2196 2195 return (DDI_SUCCESS);
2197 2196 }
2198 2197
2199 2198
2200 2199 /*
2201 2200 * tavor_rsrc_pd_handles_fini()
2202 2201 * Context: Only called from attach() and/or detach() path contexts
2203 2202 */
2204 2203 static void
2205 2204 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2206 2205 {
2207 2206 tavor_rsrc_pool_info_t *rsrc_pool;
2208 2207
2209 2208 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini);
2210 2209
2211 2210 ASSERT(state != NULL);
2212 2211 ASSERT(info != NULL);
2213 2212
2214 2213 rsrc_pool = info->swi_rsrcpool;
2215 2214
2216 2215 /* Destroy the specially created UAR scratch table vmem arena */
2217 2216 vmem_destroy(rsrc_pool->rsrc_vmp);
2218 2217
2219 2218 /* Destroy the "tavor_sw_pd_t" kmem_cache */
2220 2219 tavor_rsrc_sw_handles_fini(state, info);
2221 2220
2222 2221 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini);
2223 2222 }
2224 2223
2225 2224
2226 2225 /*
2227 2226 * tavor_rsrc_mbox_alloc()
2228 2227 * Context: Only called from attach() path context
2229 2228 */
2230 2229 static int
2231 2230 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2232 2231 tavor_rsrc_t *hdl)
2233 2232 {
2234 2233 tavor_rsrc_priv_mbox_t *priv;
2235 2234 void *addr;
2236 2235 caddr_t kaddr;
2237 2236 uint64_t offset;
2238 2237 size_t real_len, temp_len;
2239 2238 int status;
2240 2239
2241 2240 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc);
2242 2241
2243 2242 ASSERT(pool_info != NULL);
2244 2243 ASSERT(hdl != NULL);
2245 2244
2246 2245 /* Get the private pointer for the mailboxes */
2247 2246 priv = pool_info->rsrc_private;
2248 2247 ASSERT(priv != NULL);
2249 2248
2250 2249 /*
2251 2250 * Allocate a DMA handle for the mailbox. This will be used for
2252 2251 * two purposes (potentially). First, it could be used below in
2253 2252 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2254 2253 * system memory. Second, it is definitely used later to bind
2255 2254 * the mailbox for DMA access from/by the hardware.
2256 2255 */
2257 2256 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
2258 2257 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
2259 2258 if (status != DDI_SUCCESS) {
2260 2259 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR,
2261 2260 "", tnf_uint, status, status);
2262 2261 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2263 2262 return (DDI_FAILURE);
2264 2263 }
2265 2264
2266 2265 /* Is mailbox in DDR memory or system memory? */
2267 2266 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2268 2267 /* Use vmem_alloc() to get DDR address of mbox */
2269 2268 hdl->tr_len = (num * pool_info->rsrc_quantum);
2270 2269 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
2271 2270 VM_SLEEP);
2272 2271 if (addr == NULL) {
2273 2272 /* No more DDR available for mailbox entries */
2274 2273 ddi_dma_free_handle(&hdl->tr_dmahdl);
2275 2274 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail,
2276 2275 TAVOR_TNF_ERROR, "");
2277 2276 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2278 2277 return (DDI_FAILURE);
2279 2278 }
2280 2279 hdl->tr_acchdl = priv->pmb_acchdl;
2281 2280
2282 2281 /* Calculate kernel virtual address (from the DDR offset) */
2283 2282 offset = ((uintptr_t)addr -
2284 2283 (uintptr_t)pool_info->rsrc_ddr_offset);
2285 2284 hdl->tr_addr = (void *)(uintptr_t)(offset +
2286 2285 (uintptr_t)pool_info->rsrc_start);
2287 2286
2288 2287 } else { /* TAVOR_IN_SYSMEM */
2289 2288
2290 2289 /* Use ddi_dma_mem_alloc() to get memory for mailbox */
2291 2290 temp_len = (num * pool_info->rsrc_quantum);
2292 2291 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
2293 2292 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
2294 2293 NULL, &kaddr, &real_len, &hdl->tr_acchdl);
2295 2294 if (status != DDI_SUCCESS) {
2296 2295 /* No more sys memory available for mailbox entries */
2297 2296 ddi_dma_free_handle(&hdl->tr_dmahdl);
2298 2297 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail,
2299 2298 TAVOR_TNF_ERROR, "");
2300 2299 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2301 2300 return (DDI_FAILURE);
2302 2301 }
2303 2302 hdl->tr_addr = (void *)kaddr;
2304 2303 hdl->tr_len = real_len;
2305 2304 }
2306 2305
2307 2306 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2308 2307 return (DDI_SUCCESS);
2309 2308 }
2310 2309
2311 2310
2312 2311 /*
2313 2312 * tavor_rsrc_mbox_free()
2314 2313 * Context: Can be called from interrupt or base context.
2315 2314 */
2316 2315 static void
2317 2316 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2318 2317 {
2319 2318 void *addr;
2320 2319 uint64_t offset;
2321 2320
2322 2321 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free);
2323 2322
2324 2323 ASSERT(pool_info != NULL);
2325 2324 ASSERT(hdl != NULL);
2326 2325
2327 2326 /* Is mailbox in DDR memory or system memory? */
2328 2327 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2329 2328
2330 2329 /* Calculate the allocated address (the mbox's DDR offset) */
2331 2330 offset = ((uintptr_t)hdl->tr_addr -
2332 2331 (uintptr_t)pool_info->rsrc_start);
2333 2332 addr = (void *)(uintptr_t)(offset +
2334 2333 (uintptr_t)pool_info->rsrc_ddr_offset);
2335 2334
2336 2335 /* Use vmem_free() to free up DDR memory for mailbox */
2337 2336 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);
2338 2337
2339 2338 } else { /* TAVOR_IN_SYSMEM */
2340 2339
2341 2340 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2342 2341 ddi_dma_mem_free(&hdl->tr_acchdl);
2343 2342 }
2344 2343
2345 2344 /* Free the DMA handle for the mailbox */
2346 2345 ddi_dma_free_handle(&hdl->tr_dmahdl);
2347 2346
2348 2347 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free);
2349 2348 }
2350 2349
2351 2350
2352 2351 /*
2353 2352 * tavor_rsrc_hw_entry_alloc()
2354 2353 * Context: Can be called from interrupt or base context.
2355 2354 */
2356 2355 static int
2357 2356 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2358 2357 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
2359 2358 tavor_rsrc_t *hdl)
2360 2359 {
2361 2360 void *addr;
2362 2361 uint64_t offset;
2363 2362 uint32_t align;
2364 2363 int flag;
2365 2364
2366 2365 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc);
2367 2366
2368 2367 ASSERT(pool_info != NULL);
2369 2368 ASSERT(hdl != NULL);
2370 2369
2371 2370 /*
2372 2371 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2373 2372 * use dma_handle (because they are in Tavor locally attached DDR
2374 2373 * memory) and, generally, don't use the acc_handle (because the
2375 2374 * entries are not directly accessed by software). The exceptions
2376 2375 * to this rule are the UARPG and UDAV entries.
2377 2376 */
2378 2377
2379 2378 /*
2380 2379 * Use vmem_xalloc() to get a properly aligned pointer (based on
2381 2380 * the number requested) to the HW entry(ies). This handles the
2382 2381 * cases (for special QPCs and for RDB entries) where we need more
2383 2382 * than one and need to ensure that they are properly aligned.
2384 2383 */
2385 2384 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2386 2385 hdl->tr_len = (num * pool_info->rsrc_quantum);
2387 2386 align = (num_align * pool_info->rsrc_quantum);
2388 2387 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
2389 2388 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
2390 2389 if (addr == NULL) {
2391 2390 /* No more HW entries available */
2392 2391 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail,
2393 2392 TAVOR_TNF_ERROR, "");
2394 2393 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2395 2394 return (DDI_FAILURE);
2396 2395 }
2397 2396
2398 2397 /* If an access handle was provided, fill it in */
2399 2398 if (acc_handle != 0) {
2400 2399 hdl->tr_acchdl = acc_handle;
2401 2400 }
2402 2401
2403 2402 /* Calculate vaddr and HW table index (from the DDR offset) */
2404 2403 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
2405 2404 hdl->tr_addr = (void *)(uintptr_t)(offset +
2406 2405 (uintptr_t)pool_info->rsrc_start);
2407 2406 hdl->tr_indx = (offset >> pool_info->rsrc_shift);
2408 2407
2409 2408 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2410 2409 return (DDI_SUCCESS);
2411 2410 }
2412 2411
2413 2412
2414 2413 /*
2415 2414 * tavor_rsrc_hw_entry_free()
2416 2415 * Context: Can be called from interrupt or base context.
2417 2416 */
2418 2417 static void
2419 2418 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2420 2419 {
2421 2420 void *addr;
2422 2421 uint64_t offset;
2423 2422
2424 2423 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free);
2425 2424
2426 2425 ASSERT(pool_info != NULL);
2427 2426 ASSERT(hdl != NULL);
2428 2427
2429 2428 /* Calculate the allocated address (the entry's DDR offset) */
2430 2429 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
2431 2430 addr = (void *)(uintptr_t)(offset +
2432 2431 (uintptr_t)pool_info->rsrc_ddr_offset);
2433 2432
2434 2433 /* Use vmem_xfree() to free up the HW table entry */
2435 2434 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
2436 2435
2437 2436 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free);
2438 2437 }
2439 2438
2440 2439
2441 2440 /*
2442 2441 * tavor_rsrc_swhdl_alloc()
2443 2442 * Context: Can be called from interrupt or base context.
2444 2443 */
2445 2444 static int
2446 2445 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2447 2446 tavor_rsrc_t *hdl)
2448 2447 {
2449 2448 void *addr;
2450 2449 int flag;
2451 2450
2452 2451 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc);
2453 2452
2454 2453 ASSERT(pool_info != NULL);
2455 2454 ASSERT(hdl != NULL);
2456 2455
2457 2456 /* Allocate the software handle structure */
2458 2457 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2459 2458 addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2460 2459 if (addr == NULL) {
2461 2460 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR,
2462 2461 "");
2463 2462 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2464 2463 return (DDI_FAILURE);
2465 2464 }
2466 2465 hdl->tr_len = pool_info->rsrc_quantum;
2467 2466 hdl->tr_addr = addr;
2468 2467
2469 2468 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2470 2469 return (DDI_SUCCESS);
2471 2470 }
2472 2471
2473 2472
2474 2473 /*
2475 2474 * tavor_rsrc_swhdl_free()
2476 2475 * Context: Can be called from interrupt or base context.
2477 2476 */
2478 2477 static void
2479 2478 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2480 2479 {
2481 2480 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free);
2482 2481
2483 2482 ASSERT(pool_info != NULL);
2484 2483 ASSERT(hdl != NULL);
2485 2484
2486 2485 /* Free the software handle structure */
2487 2486 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
2488 2487
2489 2488 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free);
2490 2489 }
2491 2490
2492 2491
2493 2492 /*
2494 2493 * tavor_rsrc_pdhdl_alloc()
2495 2494 * Context: Can be called from interrupt or base context.
2496 2495 */
2497 2496 static int
2498 2497 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2499 2498 tavor_rsrc_t *hdl)
2500 2499 {
2501 2500 tavor_pdhdl_t addr;
2502 2501 void *tmpaddr;
2503 2502 int flag, status;
2504 2503
2505 2504 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc);
2506 2505
2507 2506 ASSERT(pool_info != NULL);
|
↓ open down ↓ |
2293 lines elided |
↑ open up ↑ |
2508 2507 ASSERT(hdl != NULL);
2509 2508
2510 2509 /* Allocate the software handle */
2511 2510 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2512 2511 if (status != DDI_SUCCESS) {
2513 2512 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2514 2513 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2515 2514 return (DDI_FAILURE);
2516 2515 }
2517 2516 addr = (tavor_pdhdl_t)hdl->tr_addr;
2518 - _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2519 2517
2520 2518 /* Allocate a PD number for the handle */
2521 2519 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2522 2520 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2523 2521 if (tmpaddr == NULL) {
2524 2522 /* No more PD number entries available */
2525 2523 tavor_rsrc_swhdl_free(pool_info, hdl);
2526 2524 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail,
2527 2525 TAVOR_TNF_ERROR, "");
2528 2526 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2529 2527 return (DDI_FAILURE);
2530 2528 }
2531 2529 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2532 2530 addr->pd_rsrcp = hdl;
2533 2531 hdl->tr_indx = addr->pd_pdnum;
2534 2532
2535 2533 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2536 2534 return (DDI_SUCCESS);
2537 2535 }
2538 2536
2539 2537
2540 2538 /*
2541 2539 * tavor_rsrc_pdhdl_free()
2542 2540 * Context: Can be called from interrupt or base context.
2543 2541 */
2544 2542 static void
2545 2543 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2546 2544 {
2547 2545 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free);
2548 2546
2549 2547 ASSERT(pool_info != NULL);
2550 2548 ASSERT(hdl != NULL);
2551 2549
2552 2550 /* Use vmem_free() to free up the PD number */
2553 2551 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);
2554 2552
2555 2553 /* Free the software handle structure */
2556 2554 tavor_rsrc_swhdl_free(pool_info, hdl);
2557 2555
2558 2556 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free);
2559 2557 }
2560 2558
2561 2559
2562 2560 /*
2563 2561 * tavor_rsrc_pdhdl_constructor()
2564 2562 * Context: Can be called from interrupt or base context.
2565 2563 */
2566 2564 /* ARGSUSED */
2567 2565 static int
2568 2566 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2569 2567 {
2570 2568 tavor_pdhdl_t pdhdl;
2571 2569 tavor_state_t *state;
2572 2570
2573 2571 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor);
2574 2572
2575 2573 pdhdl = (tavor_pdhdl_t)pd;
2576 2574 state = (tavor_state_t *)priv;
2577 2575
2578 2576 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2579 2577 DDI_INTR_PRI(state->ts_intrmsi_pri));
2580 2578
2581 2579 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor);
2582 2580 return (DDI_SUCCESS);
2583 2581 }
2584 2582
2585 2583
2586 2584 /*
2587 2585 * tavor_rsrc_pdhdl_destructor()
2588 2586 * Context: Can be called from interrupt or base context.
2589 2587 */
2590 2588 /* ARGSUSED */
2591 2589 static void
2592 2590 tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
2593 2591 {
2594 2592 tavor_pdhdl_t pdhdl;
2595 2593
2596 2594 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor);
2597 2595
2598 2596 pdhdl = (tavor_pdhdl_t)pd;
2599 2597
2600 2598 mutex_destroy(&pdhdl->pd_lock);
2601 2599
2602 2600 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor);
2603 2601 }
2604 2602
2605 2603
2606 2604 /*
2607 2605 * tavor_rsrc_cqhdl_constructor()
2608 2606 * Context: Can be called from interrupt or base context.
2609 2607 */
2610 2608 /* ARGSUSED */
2611 2609 static int
2612 2610 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2613 2611 {
2614 2612 tavor_cqhdl_t cqhdl;
2615 2613 tavor_state_t *state;
2616 2614
2617 2615 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor);
2618 2616
2619 2617 cqhdl = (tavor_cqhdl_t)cq;
2620 2618 state = (tavor_state_t *)priv;
2621 2619
2622 2620 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2623 2621 DDI_INTR_PRI(state->ts_intrmsi_pri));
2624 2622 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
2625 2623 DDI_INTR_PRI(state->ts_intrmsi_pri));
2626 2624
2627 2625 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor);
2628 2626 return (DDI_SUCCESS);
2629 2627 }
2630 2628
2631 2629
2632 2630 /*
2633 2631 * tavor_rsrc_cqhdl_destructor()
2634 2632 * Context: Can be called from interrupt or base context.
2635 2633 */
2636 2634 /* ARGSUSED */
2637 2635 static void
2638 2636 tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
2639 2637 {
2640 2638 tavor_cqhdl_t cqhdl;
2641 2639
2642 2640 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor);
2643 2641
2644 2642 cqhdl = (tavor_cqhdl_t)cq;
2645 2643
2646 2644 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
2647 2645 mutex_destroy(&cqhdl->cq_lock);
2648 2646
2649 2647 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor);
2650 2648 }
2651 2649
2652 2650
2653 2651 /*
2654 2652 * tavor_rsrc_qphdl_constructor()
2655 2653 * Context: Can be called from interrupt or base context.
2656 2654 */
2657 2655 /* ARGSUSED */
2658 2656 static int
2659 2657 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2660 2658 {
2661 2659 tavor_qphdl_t qphdl;
2662 2660 tavor_state_t *state;
2663 2661
2664 2662 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor);
2665 2663
2666 2664 qphdl = (tavor_qphdl_t)qp;
2667 2665 state = (tavor_state_t *)priv;
2668 2666
2669 2667 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2670 2668 DDI_INTR_PRI(state->ts_intrmsi_pri));
2671 2669
2672 2670 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor);
2673 2671 return (DDI_SUCCESS);
2674 2672 }
2675 2673
2676 2674
2677 2675 /*
2678 2676 * tavor_rsrc_qphdl_destructor()
2679 2677 * Context: Can be called from interrupt or base context.
2680 2678 */
2681 2679 /* ARGSUSED */
2682 2680 static void
2683 2681 tavor_rsrc_qphdl_destructor(void *qp, void *priv)
2684 2682 {
2685 2683 tavor_qphdl_t qphdl;
2686 2684
2687 2685 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2688 2686
2689 2687 qphdl = (tavor_qphdl_t)qp;
2690 2688
2691 2689 mutex_destroy(&qphdl->qp_lock);
2692 2690
2693 2691 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2694 2692 }
2695 2693
2696 2694
2697 2695 /*
2698 2696 * tavor_rsrc_srqhdl_constructor()
2699 2697 * Context: Can be called from interrupt or base context.
2700 2698 */
2701 2699 /* ARGSUSED */
2702 2700 static int
2703 2701 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2704 2702 {
2705 2703 tavor_srqhdl_t srqhdl;
2706 2704 tavor_state_t *state;
2707 2705
2708 2706 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor);
2709 2707
2710 2708 srqhdl = (tavor_srqhdl_t)srq;
2711 2709 state = (tavor_state_t *)priv;
2712 2710
2713 2711 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2714 2712 DDI_INTR_PRI(state->ts_intrmsi_pri));
2715 2713
2716 2714 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor);
2717 2715 return (DDI_SUCCESS);
2718 2716 }
2719 2717
2720 2718
2721 2719 /*
2722 2720 * tavor_rsrc_srqhdl_destructor()
2723 2721 * Context: Can be called from interrupt or base context.
2724 2722 */
2725 2723 /* ARGSUSED */
2726 2724 static void
2727 2725 tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
2728 2726 {
2729 2727 tavor_srqhdl_t srqhdl;
2730 2728
2731 2729 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor);
2732 2730
2733 2731 srqhdl = (tavor_srqhdl_t)srq;
2734 2732
2735 2733 mutex_destroy(&srqhdl->srq_lock);
2736 2734
2737 2735 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor);
2738 2736 }
2739 2737
2740 2738
2741 2739 /*
2742 2740 * tavor_rsrc_refcnt_constructor()
2743 2741 * Context: Can be called from interrupt or base context.
2744 2742 */
2745 2743 /* ARGSUSED */
2746 2744 static int
2747 2745 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2748 2746 {
2749 2747 tavor_sw_refcnt_t *refcnt;
2750 2748 tavor_state_t *state;
2751 2749
2752 2750 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor);
2753 2751
2754 2752 refcnt = (tavor_sw_refcnt_t *)rc;
2755 2753 state = (tavor_state_t *)priv;
2756 2754
2757 2755 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2758 2756 DDI_INTR_PRI(state->ts_intrmsi_pri));
2759 2757
2760 2758 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor);
2761 2759 return (DDI_SUCCESS);
2762 2760 }
2763 2761
2764 2762
2765 2763 /*
2766 2764 * tavor_rsrc_refcnt_destructor()
2767 2765 * Context: Can be called from interrupt or base context.
2768 2766 */
2769 2767 /* ARGSUSED */
2770 2768 static void
2771 2769 tavor_rsrc_refcnt_destructor(void *rc, void *priv)
2772 2770 {
2773 2771 tavor_sw_refcnt_t *refcnt;
2774 2772
2775 2773 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2776 2774
2777 2775 refcnt = (tavor_sw_refcnt_t *)rc;
2778 2776
2779 2777 mutex_destroy(&refcnt->swrc_lock);
2780 2778
2781 2779 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2782 2780 }
2783 2781
2784 2782
2785 2783 /*
2786 2784 * tavor_rsrc_ahhdl_constructor()
2787 2785 * Context: Can be called from interrupt or base context.
2788 2786 */
2789 2787 /* ARGSUSED */
2790 2788 static int
2791 2789 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2792 2790 {
2793 2791 tavor_ahhdl_t ahhdl;
2794 2792 tavor_state_t *state;
2795 2793
2796 2794 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor);
2797 2795
2798 2796 ahhdl = (tavor_ahhdl_t)ah;
2799 2797 state = (tavor_state_t *)priv;
2800 2798
2801 2799 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2802 2800 DDI_INTR_PRI(state->ts_intrmsi_pri));
2803 2801
2804 2802 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor);
2805 2803 return (DDI_SUCCESS);
2806 2804 }
2807 2805
2808 2806
2809 2807 /*
2810 2808 * tavor_rsrc_ahhdl_destructor()
2811 2809 * Context: Can be called from interrupt or base context.
2812 2810 */
2813 2811 /* ARGSUSED */
2814 2812 static void
2815 2813 tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
2816 2814 {
2817 2815 tavor_ahhdl_t ahhdl;
2818 2816
2819 2817 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2820 2818
2821 2819 ahhdl = (tavor_ahhdl_t)ah;
2822 2820
2823 2821 mutex_destroy(&ahhdl->ah_lock);
2824 2822
2825 2823 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2826 2824 }
2827 2825
2828 2826
2829 2827 /*
2830 2828 * tavor_rsrc_mrhdl_constructor()
2831 2829 * Context: Can be called from interrupt or base context.
2832 2830 */
2833 2831 /* ARGSUSED */
2834 2832 static int
2835 2833 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2836 2834 {
2837 2835 tavor_mrhdl_t mrhdl;
2838 2836 tavor_state_t *state;
2839 2837
2840 2838 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor);
2841 2839
2842 2840 mrhdl = (tavor_mrhdl_t)mr;
2843 2841 state = (tavor_state_t *)priv;
2844 2842
2845 2843 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2846 2844 DDI_INTR_PRI(state->ts_intrmsi_pri));
2847 2845
2848 2846 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor);
2849 2847 return (DDI_SUCCESS);
2850 2848 }
2851 2849
2852 2850
2853 2851 /*
2854 2852 * tavor_rsrc_mrhdl_destructor()
2855 2853 * Context: Can be called from interrupt or base context.
2856 2854 */
2857 2855 /* ARGSUSED */
2858 2856 static void
2859 2857 tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
2860 2858 {
2861 2859 tavor_mrhdl_t mrhdl;
2862 2860
2863 2861 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2864 2862
2865 2863 mrhdl = (tavor_mrhdl_t)mr;
2866 2864
2867 2865 mutex_destroy(&mrhdl->mr_lock);
2868 2866
2869 2867 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2870 2868 }
2871 2869
2872 2870
2873 2871 /*
2874 2872 * tavor_rsrc_mcg_entry_get_size()
2875 2873 */
2876 2874 static int
2877 2875 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
2878 2876 {
2879 2877 uint_t num_qp_per_mcg, max_qp_per_mcg, log2;
2880 2878
2881 2879 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size);
2882 2880
2883 2881 /*
2884 2882 * Round the configured number of QP per MCG to next larger
2885 2883 * power-of-2 size and update.
2886 2884 */
2887 2885 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
2888 2886 log2 = highbit(num_qp_per_mcg);
2889 2887 if (ISP2(num_qp_per_mcg)) {
2890 2888 log2 = log2 - 1;
2891 2889 }
2892 2890 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2893 2891
2894 2892 /* Now make sure number of QP per MCG makes sense */
2895 2893 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
2896 2894 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
2897 2895 if (num_qp_per_mcg > max_qp_per_mcg) {
2898 2896 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail,
2899 2897 TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg);
2900 2898 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2901 2899 return (DDI_FAILURE);
2902 2900 }
2903 2901
2904 2902 /* Return the (shift) size of an individual MCG HW entry */
2905 2903 *mcg_size_shift = log2 + 2;
2906 2904
2907 2905 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2908 2906 return (DDI_SUCCESS);
2909 2907 }
|
↓ open down ↓ |
381 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX