Print this page
10686 Debug macros causes smatch issues
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/comstar/port/srpt/srpt_impl.h
+++ new/usr/src/uts/common/io/comstar/port/srpt/srpt_impl.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 */
25 25
26 +/*
27 + * Copyright 2019, Joyent, Inc.
28 + */
29 +
26 30 #ifndef _SRPT_IMPL_H_
27 31 #define _SRPT_IMPL_H_
28 32
29 33 /*
30 34 * Prototypes and data structures for the SRP Target Port Provider.
31 35 */
32 36
33 37 #include <sys/types.h>
34 38 #include <sys/ddi.h>
35 39 #include <sys/ib/ibtl/ibti.h>
36 40 #include <sys/modctl.h>
37 41
38 42 #include <sys/stmf.h>
39 43 #include <sys/stmf_ioctl.h>
40 44 #include <sys/portif.h>
41 45
42 46 #include <sys/ib/mgt/ibdma/ibdma.h>
43 47
44 48 #ifdef __cplusplus
45 49 extern "C" {
46 50 #endif
47 51
48 52 /* Format the session identifier */
49 53 #define ALIAS_STR(s, a, b) \
50 54 ((void) snprintf((s), sizeof ((s)), "%016llx:%016llx", \
51 55 (u_longlong_t)(a), (u_longlong_t)(b)))
52 56
53 57 /* Format the EUI name */
54 58 #define EUI_STR(s, a) \
55 59 ((void) snprintf((s), sizeof ((s)), "eui.%016llX", (u_longlong_t)(a)))
56 60
57 61 /*
58 62 * We should/could consider making some of these values tunables.
59 63 * Specifically, SEND_MSG_SIZE and SEND_MSG_DEPTH.
60 64 */
61 65 enum {
62 66 SRPT_DEFAULT_IOC_SRQ_SIZE = 4096,
63 67 SRPT_DEFAULT_SEND_MSG_DEPTH = 128,
64 68 /*
65 69 * SEND_MSG_SIZE must be a multiple of 64 as it is registered
66 70 * as memory regions with IB. To support a scatter/gather table
67 71 * size of 32, the size must be at not less than 960. To support
68 72 * the maximum scatter/gather table size of 255, the IU must
69 73 * be at least 4160 bytes.
70 74 */
71 75 SRPT_DEFAULT_SEND_MSG_SIZE = 4160,
72 76 SRPT_DEFAULT_MAX_RDMA_SIZE = 65536,
73 77 SRPT_MIN_T_I_IU_LEN = 52,
74 78 SRPT_EUI_ID_LEN = 20,
75 79 SRPT_RECV_WC_POLL_SIZE = 16,
76 80 SRPT_SEND_WC_POLL_SIZE = 16,
77 81 SRPT_MAX_OUT_IO_PER_CMD = 16,
78 82 SRPT_FENCE_SEND = 1,
79 83 SRPT_NO_FENCE_SEND = 0
80 84 };
81 85
82 86 struct srpt_target_port_s;
83 87
84 88 #define SRPT_ALIAS_LEN (SRP_PORT_ID_LEN * 2 + 2)
|
↓ open down ↓ |
49 lines elided |
↑ open up ↑ |
85 89
86 90 /*
87 91 * SRP Session - represents a SCSI I_T_Nexus.
88 92 *
89 93 * Sessions map 1 or more initiator logins to a specific I/O
90 94 * Controller SCSI Target Port. Targets create sessions
91 95 * at initiator login and release when no longer referenced
92 96 * by a login.
93 97 */
94 98 typedef struct srpt_session_s {
95 - krwlock_t ss_rwlock;
99 + krwlock_t ss_rwlock;
96 100 list_node_t ss_node;
97 101
98 102 /*
99 103 * ADVANCED FEATURE, NOT YET SUPPORTED.
100 104 * In multi-channel mode, multiple RDMA communication
101 105 * channels may reference the same SCSI session. When
102 106 * a channel releases its reference to the SCSI session,
103 107 * it should have no tasks associated with the session.
104 108 *
105 109 * If multi-channel is implemented, add a channel list
106 110 * to this object instead of tracking it on the target.
107 111 *
108 112 * Will also need a session state & mode. Mode is to
109 113 * track if the session is MULTI or SINGLE channel.
110 114 */
111 115
112 116 stmf_scsi_session_t *ss_ss;
113 117 struct srpt_target_port_s *ss_tgt;
114 118 list_t ss_task_list;
115 119
116 120 /*
117 121 * SRP Initiator and target identifiers are 128-bit.
118 122 *
119 123 * The specification defines the initiator to be 64-bits of
120 124 * ID extension and 64 bits of GUID, but these are really
121 125 * just a recommendation. Generally the extension is used
122 126 * to create unique I_T_Nexus from the same initiator and
123 127 * target. Initiators are inconsistent on the GUID they
124 128 * use, some use the HCA Node, some the HCA port.
125 129 *
126 130 * The specification defines the target to be 64-bits of
127 131 * service ID followed by 64-bits of I/O Controller GUID.
128 132 * In the case where there is a single default target
129 133 * service, they will be the same (our default).
130 134 */
131 135 uint8_t ss_i_id[SRP_PORT_ID_LEN];
132 136 uint8_t ss_t_id[SRP_PORT_ID_LEN];
133 137
134 138 /* So we can see the full 128-bit initiator login from stmfadm */
135 139 char ss_i_alias[SRPT_ALIAS_LEN];
136 140 uint8_t ss_hw_port;
137 141
138 142 char ss_t_alias[SRPT_ALIAS_LEN];
139 143 char ss_i_name[SRPT_EUI_ID_LEN + 1];
140 144 char ss_t_name[SRPT_EUI_ID_LEN + 1];
141 145 char ss_i_gid[SRPT_ALIAS_LEN];
142 146 char ss_t_gid[SRPT_ALIAS_LEN];
143 147 } srpt_session_t;
144 148
145 149 /*
146 150 * Send work request types.
147 151 */
148 152 typedef enum srpt_swqe_type_e {
149 153 SRPT_SWQE_TYPE_DATA = 1,
150 154 SRPT_SWQE_TYPE_RESP
151 155 } srpt_swqe_type_t;
152 156
153 157 typedef struct srpt_swqe_s {
154 158 srpt_swqe_type_t sw_type;
155 159 void *sw_addr;
156 160 ibt_wrid_t sw_next;
157 161 } srpt_swqe_t;
158 162
159 163 /*
|
↓ open down ↓ |
54 lines elided |
↑ open up ↑ |
160 164 * SRP Channel - the RDMA communications channel associated with
161 165 * a specific SRP login.
162 166 */
163 167 typedef enum srpt_channel_state_e {
164 168 SRPT_CHANNEL_CONNECTING = 0,
165 169 SRPT_CHANNEL_CONNECTED,
166 170 SRPT_CHANNEL_DISCONNECTING
167 171 } srpt_channel_state_t;
168 172
169 173 typedef struct srpt_channel_s {
170 - krwlock_t ch_rwlock;
174 + krwlock_t ch_rwlock;
171 175
172 176 kmutex_t ch_reflock;
173 177 uint_t ch_refcnt;
174 178 kcondvar_t ch_cv_complete;
175 179 uint_t ch_cv_waiters;
176 180
177 181 list_node_t ch_stp_node;
178 182 srpt_channel_state_t ch_state;
179 183 ibt_cq_hdl_t ch_scq_hdl;
180 184 ibt_cq_hdl_t ch_rcq_hdl;
181 185 ibt_channel_hdl_t ch_chan_hdl;
182 186 ibt_chan_sizes_t ch_sizes;
183 187
184 188 uint32_t ch_req_lim_delta;
185 189 uint32_t ch_ti_iu_len;
186 190 struct srpt_target_port_s *ch_tgt;
187 191 srpt_session_t *ch_session;
188 192
189 193 /*
190 194 * Map IB send WQE request IDs to the
191 195 * apporpriate operation type (for errors).
192 196 */
193 197 kmutex_t ch_swqe_lock;
194 198 srpt_swqe_t *ch_swqe;
195 199 uint32_t ch_num_swqe;
196 200 uint32_t ch_head;
197 201 uint32_t ch_tail;
198 202 uint32_t ch_swqe_posted;
199 203 } srpt_channel_t;
200 204
201 205 /*
202 206 * SRP Information Unit (IU). Each IU structure contains
203 207 * the buffer for the IU itself (received over the RC
204 208 * channel), and all of the context required by the target
205 209 * to process this request represented by the IU.
206 210 * Available IU structures are managed on the I/O Controller
207 211 * shared receive queue.
208 212 */
209 213 enum {
210 214 SRPT_IU_STMF_ABORTING = 1 << 0, /* STMF called abort */
211 215 SRPT_IU_SRP_ABORTING = 1 << 1, /* SRP initiator aborting */
212 216 SRPT_IU_ABORTED = 1 << 2, /* Task has been aborted */
213 217 SRPT_IU_RESP_SENT = 1 << 3 /* Response queued */
214 218 };
215 219
216 220 typedef struct srpt_iu_s {
217 221 /*
218 222 * The buffer for the IU itself. When unused (a
219 223 * reference count of zero), this buffer is posted
220 224 * on the I/O Controllers SRPT SRQ.
221 225 */
222 226 void *iu_buf;
223 227 ibt_wr_ds_t iu_sge;
224 228 struct srpt_ioc_s *iu_ioc;
225 229 uint_t iu_pool_ndx;
226 230 kmutex_t iu_lock;
227 231
228 232 /*
229 233 * The following are reset for each IU request
230 234 * processed by this buffer.
231 235 */
232 236 list_node_t iu_ss_task_node;
233 237 srpt_channel_t *iu_ch;
234 238
235 239 uint_t iu_num_rdescs;
236 240 srp_direct_desc_t *iu_rdescs;
237 241 uint_t iu_tot_xfer_len;
238 242
239 243 uint64_t iu_tag;
240 244 uint_t iu_flags;
241 245 uint32_t iu_sq_posted_cnt;
242 246 scsi_task_t *iu_stmf_task;
243 247 } srpt_iu_t;
244 248
245 249 /*
246 250 * SRP SCSI Target Port. By default each HCA creates a single
247 251 * SCSI Target Port based on the associated I/O Controller
248 252 * (HCA) node GUID and made available through each physical
249 253 * hardware port of the I/O Controller.
250 254 */
251 255 typedef enum srpt_target_state_e {
252 256 SRPT_TGT_STATE_OFFLINE = 0,
253 257 SRPT_TGT_STATE_ONLINING,
254 258 SRPT_TGT_STATE_ONLINE,
255 259 SRPT_TGT_STATE_OFFLINING
256 260 } srpt_target_state_t;
257 261
258 262 typedef struct srpt_hw_port_s {
259 263 ibt_sbind_hdl_t hwp_bind_hdl;
260 264 ib_gid_t hwp_gid;
261 265 } srpt_hw_port_t;
262 266
263 267 typedef struct srpt_target_port_s {
264 268 stmf_local_port_t *tp_lport;
265 269 struct srpt_ioc_s *tp_ioc;
266 270
267 271 kmutex_t tp_lock;
268 272 srpt_target_state_t tp_state;
269 273 kcondvar_t tp_offline_complete;
270 274 uint_t tp_drv_disabled;
271 275
272 276 /*
273 277 * We are using a simple list for channels right now, we
274 278 * probably should switch this over to the AVL
275 279 * implementation eventually (but lookups are not done
276 280 * in the data path so this is not urgent).
277 281 */
278 282 kmutex_t tp_ch_list_lock;
279 283 list_t tp_ch_list;
280 284
281 285 /*
282 286 * A list of active sessions. Session lifetime is
283 287 * determined by having active channels, but track
284 288 * them here for easier determination to when a
285 289 * target can truly be offlined, and as a step toward
286 290 * being session-focused rather than channel-focused.
287 291 * If we ever truly support multi-channel, move the
288 292 * channels to be part of the session object.
289 293 *
290 294 * Sessions should remain on this list until they
291 295 * are deregistered from STMF. This allows the target
292 296 * to properly track when it can consider itself 'offline'.
293 297 */
294 298 kmutex_t tp_sess_list_lock;
295 299 kcondvar_t tp_sess_complete;
296 300 list_t tp_sess_list;
297 301
298 302 uint_t tp_srp_enabled;
299 303 ibt_srv_hdl_t tp_ibt_svc_hdl;
300 304 ibt_srv_desc_t tp_ibt_svc_desc;
301 305 ib_svc_id_t tp_ibt_svc_id;
302 306 scsi_devid_desc_t *tp_scsi_devid;
303 307 uint8_t tp_srp_port_id[SRP_PORT_ID_LEN];
304 308
305 309 uint_t tp_nports;
306 310 srpt_hw_port_t *tp_hw_port;
307 311 /*
308 312 * track the number of active ports so we can offline the target if
309 313 * none
310 314 */
311 315 uint32_t tp_num_active_ports;
312 316 /* state STMF wants the target in. We may be offline due to no ports */
313 317 srpt_target_state_t tp_requested_state;
314 318 } srpt_target_port_t;
|
↓ open down ↓ |
134 lines elided |
↑ open up ↑ |
315 319
316 320 /*
317 321 * SRP Target hardware device. A SRP Target hardware device
318 322 * is an IB HCA. All ports of the HCA comprise a single
319 323 * I/O Controller that is registered with the IB Device
320 324 * Managment Agent.
321 325 */
322 326 typedef struct srpt_ioc_s {
323 327 list_node_t ioc_node;
324 328
325 - krwlock_t ioc_rwlock;
329 + krwlock_t ioc_rwlock;
326 330 ibt_hca_hdl_t ioc_ibt_hdl;
327 331 ibt_hca_attr_t ioc_attr;
328 332 ib_guid_t ioc_guid;
329 333
330 334 /*
331 335 * By default each HCA is a single SRP.T10 service based on
332 336 * the HCA GUID. We have implemented the target here as a
333 337 * pointer to facilitate moving to a list of targets if
334 338 * appropriate down the road.
335 339 */
336 340 srpt_target_port_t *ioc_tgt_port;
337 341
338 342
339 343 /*
340 344 * Each HCA registers a single I/O Controller with the
341 345 * IB Device Management Agent.
342 346 */
343 347 ibdma_hdl_t ioc_ibdma_hdl;
344 348 ib_dm_ioc_ctrl_profile_t ioc_profile;
345 349 ib_dm_srv_t ioc_svc;
346 350
347 351 ibt_pd_hdl_t ioc_pd_hdl;
348 352 ibt_srq_sizes_t ioc_srq_attr;
349 353 ibt_srq_hdl_t ioc_srq_hdl;
350 354
351 355 /*
352 356 * The I/O Controller pool of IU resources allocated
353 357 * at controller creation.
354 358 */
355 359 uint32_t ioc_num_iu_entries;
356 360 srpt_iu_t *ioc_iu_pool;
357 361 ibt_mr_hdl_t ioc_iu_mr_hdl;
358 362 void *ioc_iu_bufs; /* iu buffer space */
359 363
360 364 /*
361 365 * Each I/O Controller has it's own data buffer
362 366 * vmem arena. Pool is created at controller creation,
363 367 * and expanded as required. This keeps IB memory
364 368 * registrations to a minimum in the data path.
365 369 */
366 370 struct srpt_vmem_pool_s *ioc_dbuf_pool;
367 371 stmf_dbuf_store_t *ioc_stmf_ds;
368 372 } srpt_ioc_t;
369 373
370 374 /*
371 375 * Memory regions
372 376 */
373 377 typedef struct srpt_mr_s {
374 378 ibt_mr_hdl_t mr_hdl;
375 379 ib_vaddr_t mr_va;
376 380 ib_memlen_t mr_len;
377 381 ibt_lkey_t mr_lkey;
378 382 ibt_rkey_t mr_rkey;
379 383 avl_node_t mr_avl;
380 384 } srpt_mr_t;
381 385
382 386 /*
383 387 * SRP Target vmem arena definition
384 388 */
385 389 typedef struct srpt_vmem_pool_s {
386 390 srpt_ioc_t *svp_ioc;
387 391 ib_memlen_t svp_chunksize;
388 392 vmem_t *svp_vmem;
389 393 uint64_t svp_total_size;
390 394 uint64_t svp_max_size;
391 395 avl_tree_t svp_mr_list;
392 396 krwlock_t svp_lock;
393 397 ibt_mr_flags_t svp_flags;
394 398 } srpt_vmem_pool_t;
395 399
396 400 /*
397 401 * SRP port provider data buffer, allocated and freed
398 402 * via calls to the IOC datastore.
399 403 */
400 404 typedef struct srpt_ds_dbuf_s {
401 405 stmf_data_buf_t *db_stmf_buf;
402 406 srpt_ioc_t *db_ioc;
403 407 ibt_mr_hdl_t db_mr_hdl;
404 408 ibt_wr_ds_t db_sge;
405 409 srpt_iu_t *db_iu;
406 410 } srpt_ds_dbuf_t;
407 411
408 412 /*
409 413 * SRP Target service state
410 414 */
411 415 typedef enum {
412 416 SRPT_SVC_DISABLED,
413 417 SRPT_SVC_ENABLED
414 418 } srpt_svc_state_t;
415 419
416 420 typedef struct {
417 421 ddi_modhandle_t ibdmah;
418 422 ibdma_hdl_t (*ibdma_register)(ib_guid_t,
419 423 ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *);
420 424 ibdma_status_t (*ibdma_unregister)(ibdma_hdl_t);
421 425 ibdma_status_t (*ibdma_update)(ibdma_hdl_t,
422 426 ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *);
423 427 } srpt_ibdma_ops_t;
424 428
425 429 /*
426 430 * SRP Target protocol driver context data structure, maintaining
427 431 * the global state of the protocol.
428 432 */
429 433 typedef struct srpt_ctxt_s {
430 434 dev_info_t *sc_dip;
431 435 krwlock_t sc_rwlock;
432 436 srpt_svc_state_t sc_svc_state;
433 437
434 438 ibt_clnt_hdl_t sc_ibt_hdl;
435 439
436 440 /*
437 441 * SRP Target I/O Controllers. Each IBT HCA represents an
438 442 * I/O Controller. Must hold rwlock as a writer to update.
439 443 */
440 444 list_t sc_ioc_list;
441 445 uint_t sc_num_iocs;
442 446
443 447 /* Back-end COMSTAR port provider interface. */
444 448 stmf_port_provider_t *sc_pp;
445 449
446 450 /* IBDMA entry points */
447 451 srpt_ibdma_ops_t sc_ibdma_ops;
448 452
449 453 /*
450 454 * List of explicitly configured HCAs and their configurable
451 455 * attributes.
452 456 */
453 457 nvlist_t *sc_cfg_hca_nv;
454 458 } srpt_ctxt_t;
455 459
456 460 typedef struct srpt_iu_data_s {
457 461 union {
458 462 uint8_t srp_op;
459 463 srp_cmd_req_t srp_cmd;
460 464 srp_tsk_mgmt_t srp_tsk_mgmt;
461 465 srp_i_logout_t srp_i_logout;
462 466 srp_rsp_t srp_rsp;
463 467 } rx_iu;
464 468 } srpt_iu_data_t;
465 469
466 470 extern srpt_ctxt_t *srpt_ctxt;
467 471
468 472 /*
469 473 * For Non recoverable or Major Errors
470 474 */
471 475 #define SRPT_LOG_L0 0
472 476
473 477 /*
474 478 * For additional information on Non recoverable errors and
475 479 * warnings/informational message for sys-admin types.
476 480 */
477 481 #define SRPT_LOG_L1 1
478 482
479 483 /*
480 484 * debug only
481 485 * for more verbose trace than L1, for e.g. recoverable errors,
482 486 * or intersting trace
483 487 */
484 488 #define SRPT_LOG_L2 2
485 489
486 490 /*
487 491 * debug only
488 492 * for more verbose trace than L2, for e.g. printing function entries....
489 493 */
490 494 #define SRPT_LOG_L3 3
491 495
492 496 /*
493 497 * debug only
494 498 * for more verbose trace than L3, for e.g. printing minor function entries...
495 499 */
496 500 #define SRPT_LOG_L4 4
497 501
498 502 /*
499 503 * srpt_errlevel can be set in the debugger to enable additional logging.
500 504 * You can also add set srpt:srpt_errlevel={0,1,2,3,4} in /etc/system.
501 505 * The default log level is L1.
502 506 */
503 507 #define SRPT_LOG_DEFAULT_LEVEL SRPT_LOG_L1
504 508
505 509 extern uint_t srpt_errlevel;
506 510
507 511
508 512 #define SRPT_DPRINTF_L0(...) cmn_err(CE_WARN, __VA_ARGS__)
509 513 #define SRPT_DPRINTF_L1(...) cmn_err(CE_NOTE, __VA_ARGS__)
510 514 #define SRPT_DPRINTF_L2(...) if (srpt_errlevel >= SRPT_LOG_L2) { \
|
↓ open down ↓ |
175 lines elided |
↑ open up ↑ |
511 515 cmn_err(CE_NOTE, __VA_ARGS__);\
512 516 }
513 517 #ifdef DEBUG
514 518 #define SRPT_DPRINTF_L3(...) if (srpt_errlevel >= SRPT_LOG_L3) { \
515 519 cmn_err(CE_NOTE, __VA_ARGS__);\
516 520 }
517 521 #define SRPT_DPRINTF_L4(...) if (srpt_errlevel >= SRPT_LOG_L4) { \
518 522 cmn_err(CE_NOTE, __VA_ARGS__);\
519 523 }
520 524 #else
521 -#define SRPT_DPRINTF_L3 0 &&
522 -#define SRPT_DPRINTF_L4 0 &&
525 +#define SRPT_DPRINTF_L3(...)
526 +#define SRPT_DPRINTF_L4(...)
523 527 #endif
524 528
525 529 #ifdef __cplusplus
526 530 }
527 531 #endif
528 532
529 533 #endif /* _SRPT_IMPL_H_ */
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX