1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008-2013 Solarflare Communications Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Solaris 10 9/10 (U9) is required to build this driver, as earlier Solaris
29 * releases do not ship with the required header files for GLDv3. The driver
30 * will run on Solaris 10 10/09 (U8) and later versions.
31 */
32
33
34 #ifndef _SYS_SFXGE_H
35 #define _SYS_SFXGE_H
36
37 #ifdef __cplusplus
38 extern "C" {
39 #endif
40
41 #include <sys/types.h>
42 #include <sys/ddi.h>
43 #include <sys/sunddi.h>
44 #include <sys/stream.h>
45 #include <sys/ethernet.h>
46 #include <sys/cpuvar.h>
47
48 #ifdef _USE_GLD_V3
49 #include <sys/mac.h>
50 #include <sys/mac_ether.h>
51 #include <sys/mac_provider.h>
52 #endif
53
54 #ifdef _USE_GLD_V2
55 #include <sys/gld.h>
56 #endif
57
58 #include "sfxge_ioc.h"
59 #include "sfxge_debug.h"
60
61 #include "efx.h"
62 #include "efx_regs.h"
63
64 #ifdef _USE_GLD_V3_SOL10
65 #include "compat.h"
66 #endif
67
68 #if defined(_USE_MAC_PRIV_PROP) && !defined(_USE_GLD_V3_PROPS)
69 #error "The _USE_MAC_PRIV_PROP build option is dependent on _USE_GLD_V3_PROPS"
70 #endif
71
72 #ifdef _KERNEL
73
74 #define SFXGE_DRIVER_NAME "sfxge"
75
76 #define SFXGE_CPU_CACHE_SIZE 64
77
78 #define IS_POW2(x) ((x) && ! ((x) & ((x) - 1)))
79
80 typedef struct sfxge_s sfxge_t;
81
82 typedef enum sfxge_intr_state_e {
83 SFXGE_INTR_UNINITIALIZED = 0,
84 SFXGE_INTR_INITIALIZED,
85 SFXGE_INTR_TESTING,
86 SFXGE_INTR_STARTED
87 } sfxge_intr_state_t;
88
89 typedef struct sfxge_intr_s {
90 ddi_intr_handle_t *si_table;
91 int si_table_size;
92 int si_nalloc;
93 int si_type;
94 int si_cap;
95 efsys_mem_t si_mem;
96 uint64_t si_mask;
97 sfxge_intr_state_t si_state;
98 uint32_t si_zero_count;
99 int si_intr_pri;
100 } sfxge_intr_t;
101
102 typedef enum sfxge_promisc_type_e {
103 SFXGE_PROMISC_OFF = 0,
104 SFXGE_PROMISC_ALL_MULTI,
105 SFXGE_PROMISC_ALL_PHYS
106 } sfxge_promisc_type_t;
107
108 typedef enum sfxge_link_duplex_e {
109 SFXGE_LINK_DUPLEX_UNKNOWN = 0,
110 SFXGE_LINK_DUPLEX_HALF,
111 SFXGE_LINK_DUPLEX_FULL
112 } sfxge_link_duplex_t;
113
114 typedef enum sfxge_unicst_type_e {
115 SFXGE_UNICST_BIA = 0,
116 SFXGE_UNICST_LAA,
117 SFXGE_UNICST_NTYPES
118 } sfxge_unicst_type_t;
119
120 typedef struct sfxge_phy_s {
121 kstat_t *sp_ksp;
122 kstat_named_t *sp_stat;
123 uint32_t *sp_statbuf;
124 efsys_mem_t sp_mem;
125 } sfxge_phy_t;
126
127 typedef enum sfxge_mac_state_e {
128 SFXGE_MAC_UNINITIALIZED = 0,
129 SFXGE_MAC_INITIALIZED,
130 SFXGE_MAC_STARTED
131 } sfxge_mac_state_t;
132
133 typedef struct sfxge_mac_s {
134 sfxge_t *sm_sp;
135 efsys_mem_t sm_mem;
136 kstat_t *sm_ksp;
137 kstat_named_t *sm_stat;
138 uint8_t sm_bia[ETHERADDRL];
139 uint8_t sm_laa[ETHERADDRL];
140 boolean_t sm_laa_valid;
141 unsigned int sm_fcntl;
142 sfxge_promisc_type_t sm_promisc;
143 unsigned int sm_bucket[EFX_MAC_HASH_BITS];
144 clock_t sm_lbolt;
145 kmutex_t sm_lock;
146 efx_link_mode_t sm_link_mode;
147 unsigned int sm_link_speed;
148 sfxge_link_duplex_t sm_link_duplex;
149 boolean_t sm_link_up;
150 boolean_t sm_link_poll_reqd;
151 kcondvar_t sm_link_poll_kv;
152 boolean_t sm_mac_stats_timer_reqd;
153 boolean_t sm_mac_stats_pend;
154 ddi_taskq_t *sm_tqp;
155 sfxge_mac_state_t sm_state;
156 sfxge_phy_t sm_phy;
157 uint32_t sm_phy_cap_to_set;
158 uint32_t sm_phy_cap_to_unset;
159 } sfxge_mac_t;
160
161 typedef enum sfxge_mon_state_e {
162 SFXGE_MON_UNINITIALIZED = 0,
163 SFXGE_MON_INITIALIZED,
164 SFXGE_MON_STARTED
165 } sfxge_mon_state_t;
166
167 typedef struct sfxge_mon_s {
168 sfxge_t *sm_sp;
169 efx_mon_type_t sm_type;
170 unsigned int sm_devid;
171 kstat_t *sm_ksp;
172 kstat_named_t *sm_stat;
173 efx_mon_stat_value_t *sm_statbuf;
174 kmutex_t sm_lock;
175 sfxge_mon_state_t sm_state;
176 efsys_mem_t sm_mem;
177 } sfxge_mon_t;
178
179 typedef enum sfxge_sram_state_e {
180 SFXGE_SRAM_UNINITIALIZED = 0,
181 SFXGE_SRAM_INITIALIZED,
182 SFXGE_SRAM_STARTED
183 } sfxge_sram_state_t;
184
185 typedef struct sfxge_sram_s {
186 sfxge_t *ss_sp;
187 kmutex_t ss_lock;
188 vmem_t *ss_buf_tbl;
189 unsigned int ss_count;
190 sfxge_sram_state_t ss_state;
191 } sfxge_sram_t;
192
193 typedef enum sfxge_mcdi_state_e {
194 SFXGE_MCDI_UNINITIALIZED = 0,
195 SFXGE_MCDI_INITIALIZED,
196 SFXGE_MCDI_BUSY,
197 SFXGE_MCDI_COMPLETED
198 } sfxge_mcdi_state_t;
199
200 typedef struct sfxge_mcdi_s {
201 sfxge_t *sm_sp;
202 kmutex_t sm_lock;
203 sfxge_mcdi_state_t sm_state;
204 efx_mcdi_transport_t sm_emt;
205 kcondvar_t sm_kv; /* MCDI poll complete */
206 } sfxge_mcdi_t;
207
208 #define SFXGE_NEVS 4096
209 #define SFXGE_NDESCS 1024
210 #define SFXGE_TX_NDESCS SFXGE_NDESCS
211 #define SFXGE_DEFAULT_RXQ_SIZE 1024
212
213 #define SFXGE_DEFAULT_MODERATION 30
214
215 typedef enum sfxge_evq_state_e {
216 SFXGE_EVQ_UNINITIALIZED = 0,
217 SFXGE_EVQ_INITIALIZED,
218 SFXGE_EVQ_STARTING,
219 SFXGE_EVQ_STARTED
220 } sfxge_evq_state_t;
221
222 #define SFXGE_EV_BATCH (SFXGE_NEVS / 4)
223
224 typedef struct sfxge_txq_s sfxge_txq_t;
225
226 typedef struct sfxge_evq_s {
227 union {
228 struct {
229 sfxge_t *__se_sp;
230 unsigned int __se_index;
231 efsys_mem_t __se_mem;
232 unsigned int __se_id;
233 kstat_t *__se_ksp;
234 kstat_named_t *__se_stat;
235 efx_ev_callbacks_t __se_eec;
236 sfxge_evq_state_t __se_state;
237 boolean_t __se_exception;
238 } __se_s1;
239 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
240 } __se_u1;
241 union {
242 struct {
243 kmutex_t __se_lock;
244 kcondvar_t __se_init_kv;
245 efx_evq_t *__se_eep;
246 unsigned int __se_count;
247 unsigned int __se_rx;
248 unsigned int __se_tx;
249 sfxge_txq_t *__se_stp;
250 sfxge_txq_t **__se_stpp;
251 processorid_t __se_cpu_id;
252 #ifdef _USE_CPU_PHYSID
253 id_t __se_chip_id;
254 id_t __se_core_id;
255 id_t __se_cache_id;
256 #endif
257 uint16_t __se_ev_batch;
258 } __se_s2;
259 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE];
260 } __se_u2;
261 } sfxge_evq_t;
262
263 #define se_sp __se_u1.__se_s1.__se_sp
264 #define se_index __se_u1.__se_s1.__se_index
265 #define se_mem __se_u1.__se_s1.__se_mem
266 #define se_id __se_u1.__se_s1.__se_id
267 #define se_ksp __se_u1.__se_s1.__se_ksp
268 #define se_stat __se_u1.__se_s1.__se_stat
269 #define se_eec __se_u1.__se_s1.__se_eec
270 #define se_state __se_u1.__se_s1.__se_state
271 #define se_exception __se_u1.__se_s1.__se_exception
272
273 #define se_lock __se_u2.__se_s2.__se_lock
274 #define se_init_kv __se_u2.__se_s2.__se_init_kv
275 #define se_eep __se_u2.__se_s2.__se_eep
276 #define se_count __se_u2.__se_s2.__se_count
277 #define se_rx __se_u2.__se_s2.__se_rx
278 #define se_tx __se_u2.__se_s2.__se_tx
279 #define se_stp __se_u2.__se_s2.__se_stp
280 #define se_stpp __se_u2.__se_s2.__se_stpp
281 #define se_cpu_id __se_u2.__se_s2.__se_cpu_id
282 #ifdef _USE_CPU_PHYSID
283 #define se_chip_id __se_u2.__se_s2.__se_chip_id
284 #define se_core_id __se_u2.__se_s2.__se_core_id
285 #define se_cache_id __se_u2.__se_s2.__se_cache_id
286 #endif
287 #define se_ev_batch __se_u2.__se_s2.__se_ev_batch
288
289 #define SFXGE_MAGIC_RESERVED 0x8000
290
291 #define SFXGE_MAGIC_DMAQ_LABEL_WIDTH 5
292 #define SFXGE_MAGIC_DMAQ_LABEL_MASK ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)
293
294 #define SFXGE_MAGIC_RX_QFLUSH_DONE \
295 (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
296
297 #define SFXGE_MAGIC_RX_QFLUSH_FAILED \
298 (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
299
300 #define SFXGE_MAGIC_RX_QFPP_TRIM \
301 (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
302
303 #define SFXGE_MAGIC_TX_QFLUSH_DONE \
304 (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
305
306 typedef struct sfxge_rxq_s sfxge_rxq_t;
307
308 #define SFXGE_IP_ALIGN 2
309
310 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */
311
312 typedef struct sfxge_rx_packet_s sfxge_rx_packet_t;
313
314 struct sfxge_rx_packet_s {
315 union {
316 struct {
317 frtn_t __srp_free;
318 uint16_t __srp_flags;
319 uint16_t __srp_size;
320 mblk_t *__srp_mp;
321 struct ether_header *__srp_etherhp;
322 struct ip *__srp_iphp;
323 struct tcphdr *__srp_thp;
324 size_t __srp_off;
325 } __srp_s1;
326 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE];
327 } __srp_u1;
328 union {
329 struct {
330 sfxge_rxq_t *__srp_srp;
331 ddi_dma_handle_t __srp_dma_handle;
332 ddi_acc_handle_t __srp_acc_handle;
333 unsigned char *__srp_base;
334 size_t __srp_mblksize;
335 uint64_t __srp_addr;
336 boolean_t __srp_recycle;
337 caddr_t __srp_putp;
338 } __srp_s2;
339 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
340 } __srp_u2;
341 };
342
343 #define srp_free __srp_u1.__srp_s1.__srp_free
344 #define srp_flags __srp_u1.__srp_s1.__srp_flags
345 #define srp_size __srp_u1.__srp_s1.__srp_size
346 #define srp_mp __srp_u1.__srp_s1.__srp_mp
347 #define srp_etherhp __srp_u1.__srp_s1.__srp_etherhp
348 #define srp_iphp __srp_u1.__srp_s1.__srp_iphp
349 #define srp_thp __srp_u1.__srp_s1.__srp_thp
350 #define srp_off __srp_u1.__srp_s1.__srp_off
351
352 #define srp_srp __srp_u2.__srp_s2.__srp_srp
353 #define srp_dma_handle __srp_u2.__srp_s2.__srp_dma_handle
354 #define srp_acc_handle __srp_u2.__srp_s2.__srp_acc_handle
355 #define srp_base __srp_u2.__srp_s2.__srp_base
356 #define srp_mblksize __srp_u2.__srp_s2.__srp_mblksize
357 #define srp_addr __srp_u2.__srp_s2.__srp_addr
358 #define srp_recycle __srp_u2.__srp_s2.__srp_recycle
359 #define srp_putp __srp_u2.__srp_s2.__srp_putp
360
361 #define SFXGE_RX_FPP_NSLOTS 8
362 #define SFXGE_RX_FPP_MASK (SFXGE_RX_FPP_NSLOTS - 1)
363
364 /* Free packet pool putlist (dynamically allocated) */
365 typedef struct sfxge_rx_fpp_putlist_s {
366 kmutex_t srfpl_lock;
367 unsigned int srfpl_count;
368 mblk_t *srfpl_putp;
369 mblk_t **srfpl_putpp;
370 } sfxge_rx_fpp_putlist_t;
371
372 /* Free packet pool */
373 typedef struct sfxge_rx_fpp_s {
374 caddr_t srfpp_putp;
375 unsigned int srfpp_loaned;
376 mblk_t *srfpp_get;
377 unsigned int srfpp_count;
378 unsigned int srfpp_min;
379 /* Low water mark: Don't trim to below this */
380 unsigned int srfpp_lowat;
381 } sfxge_rx_fpp_t;
382
383 typedef struct sfxge_rx_flow_s sfxge_rx_flow_t;
384
385 struct sfxge_rx_flow_s {
386 uint32_t srf_tag;
387 /* in-order segment count */
388 unsigned int srf_count;
389 uint16_t srf_tci;
390 uint32_t srf_saddr;
391 uint32_t srf_daddr;
392 uint16_t srf_sport;
393 uint16_t srf_dport;
394 /* sequence number */
395 uint32_t srf_seq;
396 clock_t srf_lbolt;
397 mblk_t *srf_mp;
398 mblk_t **srf_mpp;
399 struct ether_header *srf_etherhp;
400 struct ip *srf_iphp;
401 struct tcphdr *srf_first_thp;
402 struct tcphdr *srf_last_thp;
403 size_t srf_len;
404 sfxge_rx_flow_t *srf_next;
405 };
406
407 #define SFXGE_MAX_FLOW 1024
408 #define SFXGE_SLOW_START 20
409
410 typedef enum sfxge_flush_state_e {
411 SFXGE_FLUSH_INACTIVE = 0,
412 SFXGE_FLUSH_DONE,
413 SFXGE_FLUSH_PENDING,
414 SFXGE_FLUSH_FAILED
415 } sfxge_flush_state_t;
416
417 typedef enum sfxge_rxq_state_e {
418 SFXGE_RXQ_UNINITIALIZED = 0,
419 SFXGE_RXQ_INITIALIZED,
420 SFXGE_RXQ_STARTED
421 } sfxge_rxq_state_t;
422
423
424 #define SFXGE_RX_BATCH 128
425 #define SFXGE_RX_NSTATS 8 /* note that *esballoc share one kstat */
426
427 struct sfxge_rxq_s {
428 union {
429 struct {
430 sfxge_t *__sr_sp;
431 unsigned int __sr_index;
432 efsys_mem_t __sr_mem;
433 unsigned int __sr_id;
434 unsigned int __sr_lowat;
435 unsigned int __sr_hiwat;
436 volatile timeout_id_t __sr_tid;
437 sfxge_rxq_state_t __sr_state;
438 } __sr_s1;
439 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
440 } __sr_u1;
441 union {
442 struct {
443 sfxge_rx_packet_t **__sr_srpp;
444 unsigned int __sr_added;
445 unsigned int __sr_pending;
446 unsigned int __sr_completed;
447 unsigned int __sr_loopback;
448 mblk_t *__sr_mp;
449 mblk_t **__sr_mpp;
450 sfxge_rx_flow_t *__sr_flow;
451 sfxge_rx_flow_t *__sr_srfp;
452 sfxge_rx_flow_t **__sr_srfpp;
453 clock_t __sr_rto;
454 } __sr_s2;
455 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
456 } __sr_u2;
457 union {
458 struct {
459 sfxge_rx_fpp_t __sr_fpp;
460 efx_rxq_t *__sr_erp;
461 volatile sfxge_flush_state_t __sr_flush;
462 kcondvar_t __sr_flush_kv;
463 kstat_t *__sr_ksp;
464 } __sr_s3;
465 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE];
466 } __sr_u3;
467 struct {
468 /* NB must match SFXGE_RX_NSTATS */
469 uint32_t srk_rx_pkt_mem_limit;
470 uint32_t srk_kcache_alloc_nomem;
471 uint32_t srk_dma_alloc_nomem;
472 uint32_t srk_dma_alloc_fail;
473 uint32_t srk_dma_bind_nomem;
474 uint32_t srk_dma_bind_fail;
475 /* Following two are mutually exclusive */
476 #ifdef _USE_XESBALLOC
477 uint32_t srk_xesballoc_fail;
478 #endif
479 #ifdef _USE_DESBALLOC
480 uint32_t srk_desballoc_fail;
481 #endif
482 uint32_t srk_rxq_empty_discard;
483 } sr_kstat;
484 };
485
486 #define sr_sp __sr_u1.__sr_s1.__sr_sp
487 #define sr_index __sr_u1.__sr_s1.__sr_index
488 #define sr_mem __sr_u1.__sr_s1.__sr_mem
489 #define sr_id __sr_u1.__sr_s1.__sr_id
490 #define sr_mrh __sr_u1.__sr_s1.__sr_mrh
491 #define sr_lowat __sr_u1.__sr_s1.__sr_lowat
492 #define sr_hiwat __sr_u1.__sr_s1.__sr_hiwat
493 #define sr_tid __sr_u1.__sr_s1.__sr_tid
494 #define sr_state __sr_u1.__sr_s1.__sr_state
495
496 #define sr_srpp __sr_u2.__sr_s2.__sr_srpp
497 #define sr_added __sr_u2.__sr_s2.__sr_added
498 #define sr_pending __sr_u2.__sr_s2.__sr_pending
499 #define sr_completed __sr_u2.__sr_s2.__sr_completed
500 #define sr_loopback __sr_u2.__sr_s2.__sr_loopback
501 #define sr_mp __sr_u2.__sr_s2.__sr_mp
502 #define sr_mpp __sr_u2.__sr_s2.__sr_mpp
503 #define sr_flow __sr_u2.__sr_s2.__sr_flow
504 #define sr_srfp __sr_u2.__sr_s2.__sr_srfp
505 #define sr_srfpp __sr_u2.__sr_s2.__sr_srfpp
506 #define sr_rto __sr_u2.__sr_s2.__sr_rto
507
508 #define sr_fpp __sr_u3.__sr_s3.__sr_fpp
509 #define sr_erp __sr_u3.__sr_s3.__sr_erp
510 #define sr_flush __sr_u3.__sr_s3.__sr_flush
511 #define sr_flush_kv __sr_u3.__sr_s3.__sr_flush_kv
512 #define sr_ksp __sr_u3.__sr_s3.__sr_ksp
513
514 typedef struct sfxge_tx_packet_s sfxge_tx_packet_t;
515
516 struct sfxge_tx_packet_s {
517 sfxge_tx_packet_t *stp_next;
518 mblk_t *stp_mp;
519 struct ether_header *stp_etherhp;
520 struct ip *stp_iphp;
521 struct tcphdr *stp_thp;
522 size_t stp_off;
523 size_t stp_size;
524 size_t stp_mss;
525 uint32_t stp_dpl_put_len;
526 };
527
528 #define SFXGE_TX_FPP_MAX 64
529
530 typedef struct sfxge_tx_fpp_s {
531 sfxge_tx_packet_t *stf_stpp;
532 unsigned int stf_count;
533 } sfxge_tx_fpp_t;
534
535 typedef struct sfxge_tx_mapping_s sfxge_tx_mapping_t;
536
537 #define SFXGE_TX_MAPPING_NADDR (((1 << 16) >> 12) + 2)
538
539 struct sfxge_tx_mapping_s {
540 sfxge_tx_mapping_t *stm_next;
541 sfxge_t *stm_sp;
542 mblk_t *stm_mp;
543 ddi_dma_handle_t stm_dma_handle;
544 caddr_t stm_base;
545 size_t stm_size;
546 size_t stm_off;
547 uint64_t stm_addr[SFXGE_TX_MAPPING_NADDR];
548 };
549
550 typedef struct sfxge_tx_fmp_s {
551 sfxge_tx_mapping_t *stf_stmp;
552 unsigned int stf_count;
553 } sfxge_tx_fmp_t;
554
555 typedef struct sfxge_tx_buffer_s sfxge_tx_buffer_t;
556
557 struct sfxge_tx_buffer_s {
558 sfxge_tx_buffer_t *stb_next;
559 size_t stb_off;
560 efsys_mem_t stb_esm;
561 };
562
563 #define SFXGE_TX_BUFFER_SIZE 0x400
564 #define SFXGE_TX_HEADER_SIZE 0x100
565 #define SFXGE_TX_COPY_THRESHOLD 0x200
566
567 typedef struct sfxge_tx_fbp_s {
568 sfxge_tx_buffer_t *stf_stbp;
569 unsigned int stf_count;
570 } sfxge_tx_fbp_t;
571
572 typedef struct sfxge_tx_dpl_s {
573 uintptr_t std_put;
574 sfxge_tx_packet_t *std_get;
575 sfxge_tx_packet_t **std_getp;
576 unsigned int std_count; /* only get list count */
577 unsigned int get_pkt_limit;
578 unsigned int put_pkt_limit;
579 unsigned int get_full_count;
580 unsigned int put_full_count;
581 } sfxge_tx_dpl_t;
582
583 typedef enum sfxge_txq_state_e {
584 SFXGE_TXQ_UNINITIALIZED = 0,
585 SFXGE_TXQ_INITIALIZED,
586 SFXGE_TXQ_STARTED
587 } sfxge_txq_state_t;
588
589 typedef enum sfxge_txq_type_e {
590 SFXGE_TXQ_NON_CKSUM = 0,
591 SFXGE_TXQ_IP_CKSUM,
592 SFXGE_TXQ_IP_TCP_UDP_CKSUM,
593 SFXGE_TXQ_NTYPES
594 } sfxge_txq_type_t;
595
596 #define SFXGE_TXQ_UNBLOCK_LEVEL1 (EFX_TXQ_LIMIT(SFXGE_NDESCS) / 4)
597 #define SFXGE_TXQ_UNBLOCK_LEVEL2 0
598 #define SFXGE_TXQ_NOT_BLOCKED -1
599
600 #define SFXGE_TX_BATCH 64
601
602 struct sfxge_txq_s {
603 union {
604 struct {
605 sfxge_t *__st_sp;
606 unsigned int __st_index;
607 sfxge_txq_type_t __st_type;
608 unsigned int __st_evq;
609 efsys_mem_t __st_mem;
610 unsigned int __st_id;
611 kstat_t *__st_ksp;
612 kstat_named_t *__st_stat;
613 sfxge_txq_state_t __st_state;
614 } __st_s1;
615 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 2];
616 } __st_u1;
617 union {
618 struct {
619 sfxge_tx_dpl_t __st_dpl;
620 } __st_s2;
621 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
622 } __st_u2;
623 union {
624 struct {
625 kmutex_t __st_lock;
626 /* mapping pool - sfxge_tx_mapping_t */
627 sfxge_tx_fmp_t __st_fmp;
628 /* buffer pool - sfxge_tx_buffer_t */
629 sfxge_tx_fbp_t __st_fbp;
630 /* packet pool - sfxge_tx_packet_t */
631 sfxge_tx_fpp_t __st_fpp;
632 efx_buffer_t *__st_eb;
633 unsigned int __st_n;
634 efx_txq_t *__st_etp;
635 sfxge_tx_mapping_t **__st_stmp;
636 sfxge_tx_buffer_t **__st_stbp;
637 mblk_t **__st_mp;
638 unsigned int __st_added;
639 unsigned int __st_reaped;
640 int __st_unblock;
641 } __st_s3;
642 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 3];
643 } __st_u3;
644 union {
645 struct {
646 sfxge_txq_t *__st_next;
647 unsigned int __st_pending;
648 unsigned int __st_completed;
649 volatile sfxge_flush_state_t __st_flush;
650
651 } __st_s4;
652 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
653 } __st_u4;
654 };
655
656 #define st_sp __st_u1.__st_s1.__st_sp
657 #define st_index __st_u1.__st_s1.__st_index
658 #define st_type __st_u1.__st_s1.__st_type
659 #define st_evq __st_u1.__st_s1.__st_evq
660 #define st_mem __st_u1.__st_s1.__st_mem
661 #define st_id __st_u1.__st_s1.__st_id
662 #define st_ksp __st_u1.__st_s1.__st_ksp
663 #define st_stat __st_u1.__st_s1.__st_stat
664 #define st_state __st_u1.__st_s1.__st_state
665
666 #define st_dpl __st_u2.__st_s2.__st_dpl
667
668 #define st_lock __st_u3.__st_s3.__st_lock
669 #define st_fmp __st_u3.__st_s3.__st_fmp
670 #define st_fbp __st_u3.__st_s3.__st_fbp
671 #define st_fpp __st_u3.__st_s3.__st_fpp
672 #define st_eb __st_u3.__st_s3.__st_eb
673 #define st_n __st_u3.__st_s3.__st_n
674 #define st_etp __st_u3.__st_s3.__st_etp
675 #define st_stmp __st_u3.__st_s3.__st_stmp
676 #define st_stbp __st_u3.__st_s3.__st_stbp
677 #define st_mp __st_u3.__st_s3.__st_mp
678 #define st_added __st_u3.__st_s3.__st_added
679 #define st_reaped __st_u3.__st_s3.__st_reaped
680 #define st_unblock __st_u3.__st_s3.__st_unblock
681
682 #define st_next __st_u4.__st_s4.__st_next
683 #define st_pending __st_u4.__st_s4.__st_pending
684 #define st_completed __st_u4.__st_s4.__st_completed
685 #define st_flush __st_u4.__st_s4.__st_flush
686
687 typedef enum sfxge_rx_scale_state_e {
688 SFXGE_RX_SCALE_UNINITIALIZED = 0,
689 SFXGE_RX_SCALE_INITIALIZED,
690 SFXGE_RX_SCALE_STARTED
691 } sfxge_rx_scale_state_t;
692
693 #define SFXGE_RX_SCALE_MAX EFX_RSS_TBL_SIZE
694
695 typedef struct sfxge_rx_scale_s {
696 kmutex_t srs_lock;
697 unsigned int *srs_cpu;
698 #ifdef _USE_CPU_PHYSID
699 unsigned int *srs_core;
700 unsigned int *srs_cache;
701 unsigned int *srs_chip;
702 #endif
703 unsigned int srs_tbl[SFXGE_RX_SCALE_MAX];
704 unsigned int srs_count;
705 kstat_t *srs_ksp;
706 sfxge_rx_scale_state_t srs_state;
707 } sfxge_rx_scale_t;
708
709
710 #if defined(_USE_GLD_V2) || defined(_USE_GLD_V3_SOL10)
711 typedef struct sfxge_ndd_param_s {
712 sfxge_t *snp_sp;
713 unsigned int snp_id;
714 const char *snp_name;
715 int (*snp_get)(queue_t *, mblk_t *, caddr_t, cred_t *);
716 int (*snp_set)(queue_t *, mblk_t *, char *, caddr_t,
717 cred_t *);
718 } sfxge_ndd_param_t;
719
720 #endif
721 typedef enum sfxge_rx_coalesce_mode_e {
722 SFXGE_RX_COALESCE_OFF = 0,
723 SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
724 SFXGE_RX_COALESCE_ALLOW_PUSH = 2
725 } sfxge_rx_coalesce_mode_t;
726
727 typedef enum sfxge_vpd_type_e {
728 SFXGE_VPD_ID = 0,
729 SFXGE_VPD_PN = 1,
730 SFXGE_VPD_SN = 2,
731 SFXGE_VPD_EC = 3,
732 SFXGE_VPD_MN = 4,
733 SFXGE_VPD_VD = 5,
734 SFXGE_VPD_MAX = 6,
735 } sfxge_vpd_type_t;
736
737 typedef struct sfxge_vpd_kstat_s {
738 kstat_t *svk_ksp;
739 kstat_named_t svk_stat[SFXGE_VPD_MAX];
740 efx_vpd_value_t *svk_vv;
741 } sfxge_vpd_kstat_t;
742
743 typedef struct sfxge_cfg_kstat_s {
744 struct {
745 kstat_named_t sck_mac;
746 kstat_named_t sck_version;
747 } kstat;
748 struct {
749 char sck_mac[64 + 1];
750 } buf;
751 } sfxge_cfg_kstat_t;
752
753 typedef enum sfxge_state_e {
754 SFXGE_UNINITIALIZED = 0,
755 SFXGE_INITIALIZED,
756 SFXGE_REGISTERED,
757 SFXGE_STARTING,
758 SFXGE_STARTED,
759 SFXGE_STOPPING
760 } sfxge_state_t;
761
762 typedef enum sfxge_hw_err_e {
763 SFXGE_HW_OK = 0,
764 SFXGE_HW_ERR,
765 } sfxge_hw_err_t;
766
767 typedef enum sfxge_action_on_hw_err_e {
768 SFXGE_RECOVER = 0,
769 SFXGE_INVISIBLE = 1,
770 SFXGE_LEAVE_DEAD = 2,
771 } sfxge_action_on_hw_err_t;
772
773 typedef char *sfxge_mac_priv_prop_t;
774
775 struct sfxge_s {
776 kmutex_t s_state_lock;
777 sfxge_state_t s_state;
778 dev_info_t *s_dip;
779 ddi_taskq_t *s_tqp;
780 ddi_acc_handle_t s_pci_handle;
781 uint16_t s_pci_venid;
782 uint16_t s_pci_devid;
783 efx_family_t s_family;
784 unsigned int s_pcie_nlanes;
785 unsigned int s_pcie_linkspeed;
786 kmutex_t s_nic_lock;
787 efsys_bar_t s_bar;
788 sfxge_intr_t s_intr;
789 sfxge_mac_t s_mac;
790 sfxge_mon_t s_mon;
791 sfxge_sram_t s_sram;
792 sfxge_mcdi_t s_mcdi;
793 kmem_cache_t *s_eq0c; /* eventQ 0 */
794 kmem_cache_t *s_eqXc; /* all other eventQs */
795 sfxge_evq_t *s_sep[SFXGE_RX_SCALE_MAX];
796 unsigned int s_ev_moderation;
797 kmem_cache_t *s_rqc;
798 sfxge_rxq_t *s_srp[SFXGE_RX_SCALE_MAX];
799 sfxge_rx_scale_t s_rx_scale;
800 size_t s_rx_prefix_size;
801 size_t s_rx_buffer_size;
802 size_t s_rx_buffer_align;
803 sfxge_rx_coalesce_mode_t s_rx_coalesce_mode;
804 int64_t s_rx_pkt_mem_max;
805 volatile uint64_t s_rx_pkt_mem_alloc;
806 kmem_cache_t *s_rpc;
807 kmem_cache_t *s_tqc;
808 int s_tx_qcount;
809 sfxge_txq_t *s_stp[SFXGE_TXQ_NTYPES +
810 SFXGE_RX_SCALE_MAX];
811 kmem_cache_t *s_tpc;
812 int s_tx_flush_pending;
813 kmutex_t s_tx_flush_lock;
814 kcondvar_t s_tx_flush_kv;
815 kmem_cache_t *s_tbc;
816 kmem_cache_t *s_tmc;
817 efx_nic_t *s_enp;
818 sfxge_vpd_kstat_t s_vpd_kstat;
819 sfxge_cfg_kstat_t s_cfg_kstat;
820 kstat_t *s_cfg_ksp;
821 size_t s_mtu;
822 int s_rxq_poll_usec;
823 #ifdef _USE_GLD_V3
824 mac_callbacks_t s_mc;
825 mac_handle_t s_mh;
826 #ifdef _USE_MAC_PRIV_PROP
827 sfxge_mac_priv_prop_t *s_mac_priv_props;
828 int s_mac_priv_props_alloc;
829 #endif
830 #endif
831 #if defined(_USE_GLD_V2) || defined(_USE_GLD_V3)
832 sfxge_ndd_param_t *s_ndp;
833 kstat_named_t *s_nd_stat;
834 caddr_t s_ndh;
835 kstat_t *s_nd_ksp;
836 #endif
837 #ifdef _USE_GLD_V2
838 gld_mac_info_t *s_gmip;
839 kmutex_t s_rx_lock;
840 mblk_t *s_mp;
841 mblk_t **s_mpp;
842 #endif
843 uint32_t s_num_restarts;
844 uint32_t s_num_restarts_hw_err;
845 sfxge_hw_err_t s_hw_err;
846 sfxge_action_on_hw_err_t s_action_on_hw_err;
847 uint16_t s_rxq_size;
848 uint16_t s_evq0_size;
849 uint16_t s_evqX_size;
850 };
851
852 typedef struct sfxge_dma_buffer_attr_s {
853 dev_info_t *sdba_dip;
854 ddi_dma_attr_t *sdba_dattrp;
855 int (*sdba_callback) (caddr_t);
856 size_t sdba_length;
857 uint_t sdba_memflags;
858 ddi_device_acc_attr_t *sdba_devaccp;
859 uint_t sdba_bindflags;
860 int sdba_maxcookies;
861 boolean_t sdba_zeroinit;
862 } sfxge_dma_buffer_attr_t;
863
864 extern const char sfxge_ident[];
865 extern uint8_t sfxge_brdcst[];
866
867 extern kmutex_t sfxge_global_lock;
868
869 extern unsigned int *sfxge_cpu;
870 #ifdef _USE_CPU_PHYSID
871 extern unsigned int *sfxge_core;
872 extern unsigned int *sfxge_cache;
873 extern unsigned int *sfxge_chip;
874 #endif
875
876 extern int sfxge_start(sfxge_t *, boolean_t);
877 extern void sfxge_stop(sfxge_t *);
878 extern void sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
879 extern int sfxge_restart_dispatch(sfxge_t *, uint_t,
880 sfxge_hw_err_t, const char *, uint32_t);
881
882 extern void sfxge_gld_link_update(sfxge_t *);
883 extern void sfxge_gld_mtu_update(sfxge_t *);
884 extern void sfxge_gld_rx_post(sfxge_t *, unsigned int,
885 mblk_t *);
886 extern void sfxge_gld_rx_push(sfxge_t *);
887 extern int sfxge_gld_register(sfxge_t *);
888 extern int sfxge_gld_unregister(sfxge_t *);
889
890 extern int sfxge_gld_nd_register(sfxge_t *);
891 extern void sfxge_gld_nd_unregister(sfxge_t *);
892 #ifdef _USE_MAC_PRIV_PROP
893 extern void sfxge_gld_priv_prop_rename(sfxge_t *);
894 #endif
895
896 extern int sfxge_dma_buffer_create(efsys_mem_t *,
897 const sfxge_dma_buffer_attr_t *);
898 extern void sfxge_dma_buffer_destroy(efsys_mem_t *);
899
900 extern int sfxge_intr_init(sfxge_t *);
901 extern int sfxge_intr_start(sfxge_t *);
902 extern void sfxge_intr_stop(sfxge_t *);
903 extern void sfxge_intr_fini(sfxge_t *);
904 extern void sfxge_intr_fatal(sfxge_t *);
905
906 extern int sfxge_ev_init(sfxge_t *);
907 extern int sfxge_ev_start(sfxge_t *);
908 extern void sfxge_ev_moderation_get(sfxge_t *,
909 unsigned int *);
910 extern int sfxge_ev_moderation_set(sfxge_t *,
911 unsigned int);
912 extern int sfxge_ev_qmoderate(sfxge_t *, unsigned int,
913 unsigned int);
914 extern int sfxge_ev_qpoll(sfxge_t *, unsigned int);
915 extern int sfxge_ev_qprime(sfxge_t *, unsigned int);
916 extern void sfxge_ev_stop(sfxge_t *);
917 extern void sfxge_ev_fini(sfxge_t *);
918
919 extern int sfxge_mon_init(sfxge_t *);
920 extern int sfxge_mon_start(sfxge_t *);
921 extern void sfxge_mon_stop(sfxge_t *);
922 extern void sfxge_mon_fini(sfxge_t *);
923
924 extern int sfxge_mac_init(sfxge_t *);
925 extern int sfxge_mac_start(sfxge_t *, boolean_t);
926 extern void sfxge_mac_stat_get(sfxge_t *, unsigned int,
927 uint64_t *);
928 extern void sfxge_mac_link_check(sfxge_t *, boolean_t *);
929 extern void sfxge_mac_link_speed_get(sfxge_t *,
930 unsigned int *);
931 extern void sfxge_mac_link_duplex_get(sfxge_t *,
932 sfxge_link_duplex_t *);
933 extern void sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
934 extern int sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
935 extern int sfxge_mac_unicst_get(sfxge_t *,
936 sfxge_unicst_type_t, uint8_t *);
937 extern int sfxge_mac_unicst_set(sfxge_t *,
938 uint8_t *);
939 extern int sfxge_mac_promisc_set(sfxge_t *,
940 sfxge_promisc_type_t);
941 extern int sfxge_mac_multicst_add(sfxge_t *,
942 uint8_t *);
943 extern int sfxge_mac_multicst_remove(sfxge_t *,
944 uint8_t *);
945 extern int sfxge_mac_ioctl(sfxge_t *, sfxge_mac_ioc_t *);
946 extern void sfxge_mac_stop(sfxge_t *);
947 extern void sfxge_mac_fini(sfxge_t *);
948 extern void sfxge_mac_link_update(sfxge_t *sp,
949 efx_link_mode_t mode);
950
951 extern int sfxge_mcdi_init(sfxge_t *sp);
952 extern void sfxge_mcdi_fini(sfxge_t *sp);
953 extern int sfxge_mcdi_ioctl(sfxge_t *sp,
954 sfxge_mcdi_ioc_t *smip);
955
956 extern int sfxge_phy_init(sfxge_t *);
957 extern void sfxge_phy_link_mode_get(sfxge_t *,
958 efx_link_mode_t *);
959 extern int sfxge_phy_ioctl(sfxge_t *, sfxge_phy_ioc_t *);
960 extern int sfxge_phy_bist_ioctl(sfxge_t *,
961 sfxge_phy_bist_ioc_t *);
962 extern void sfxge_phy_fini(sfxge_t *);
963 extern int sfxge_phy_kstat_init(sfxge_t *sp);
964 extern void sfxge_phy_kstat_fini(sfxge_t *sp);
965 extern uint8_t sfxge_phy_lp_cap_test(sfxge_t *sp,
966 uint32_t field);
967 extern int sfxge_phy_cap_apply(sfxge_t *sp,
968 boolean_t use_default);
969 extern uint8_t sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
970 uint32_t field, boolean_t *mutablep);
971 extern int sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
972 int set);
973 #ifdef _USE_MAC_PRIV_PROP
974 extern int sfxge_phy_prop_get(sfxge_t *sp, unsigned int id,
975 uint32_t flags, uint32_t *valp);
976 extern int sfxge_phy_prop_set(sfxge_t *sp, unsigned int id,
977 uint32_t val);
978 #endif
979
980 extern int sfxge_rx_init(sfxge_t *);
981 extern int sfxge_rx_start(sfxge_t *);
982 extern void sfxge_rx_coalesce_mode_get(sfxge_t *,
983 sfxge_rx_coalesce_mode_t *);
984 extern int sfxge_rx_coalesce_mode_set(sfxge_t *,
985 sfxge_rx_coalesce_mode_t);
986 extern unsigned int sfxge_rx_scale_prop_get(sfxge_t *);
987 extern void sfxge_rx_scale_update(void *);
988 extern int sfxge_rx_scale_count_get(sfxge_t *,
989 unsigned int *);
990 extern int sfxge_rx_scale_count_set(sfxge_t *,
991 unsigned int);
992 extern void sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
993 extern void sfxge_rx_qflush_done(sfxge_rxq_t *);
994 extern void sfxge_rx_qflush_failed(sfxge_rxq_t *);
995 extern void sfxge_rx_qfpp_trim(sfxge_rxq_t *);
996 extern int sfxge_rx_ioctl(sfxge_t *, sfxge_rx_ioc_t *);
997 extern void sfxge_rx_stop(sfxge_t *);
998 extern unsigned int sfxge_rx_loaned(sfxge_t *);
999 extern void sfxge_rx_fini(sfxge_t *);
1000
1001 extern int sfxge_tx_init(sfxge_t *);
1002 extern int sfxge_tx_start(sfxge_t *);
1003 extern int sfxge_tx_packet_add(sfxge_t *, mblk_t *);
1004 extern void sfxge_tx_qcomplete(sfxge_txq_t *);
1005 extern void sfxge_tx_qflush_done(sfxge_txq_t *);
1006 extern int sfxge_tx_ioctl(sfxge_t *, sfxge_tx_ioc_t *);
1007 extern void sfxge_tx_stop(sfxge_t *);
1008 extern void sfxge_tx_fini(sfxge_t *);
1009 extern void sfxge_tx_qdpl_flush(sfxge_txq_t *stp);
1010
1011 extern void sfxge_sram_init(sfxge_t *);
1012 extern int sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
1013 uint32_t *);
1014 extern int sfxge_sram_start(sfxge_t *);
1015 extern int sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
1016 efsys_mem_t *, size_t);
1017 extern void sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
1018 size_t);
1019 extern void sfxge_sram_stop(sfxge_t *);
1020 extern void sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
1021 size_t);
1022 extern int sfxge_sram_ioctl(sfxge_t *, sfxge_sram_ioc_t *);
1023 extern void sfxge_sram_fini(sfxge_t *);
1024
1025 extern void sfxge_tcp_parse(mblk_t *,
1026 struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *);
1027
1028 #define SFXGE_TCP_HASH(_raddr, _rport, _laddr, _lport, _hash) \
1029 do { \
1030 uint32_t raddr = (uint32_t)(_raddr); \
1031 uint32_t rport = (uint32_t)(_rport); \
1032 uint32_t laddr = (uint32_t)(_laddr); \
1033 uint32_t lport = (uint32_t)(_lport); \
1034 uint32_t key; \
1035 uint32_t lfsr; \
1036 unsigned int index; \
1037 \
1038 key = laddr ^ \
1039 ((lport << 16) | (raddr >> 16)) ^ \
1040 ((raddr << 16) | rport); \
1041 \
1042 lfsr = 0xffffffff; \
1043 for (index = 0; index < 32; index++) { \
1044 uint32_t input; \
1045 uint32_t key_bit32; \
1046 uint32_t lfsr_bit16; \
1047 uint32_t lfsr_bit3; \
1048 \
1049 key_bit32 = key >> 31; \
1050 key <<= 1; \
1051 \
1052 lfsr_bit16 = (lfsr >> 15) & 1; \
1053 lfsr_bit3 = (lfsr >> 2) & 1; \
1054 \
1055 input = (lfsr_bit16 ^ lfsr_bit3) ^ key_bit32; \
1056 ASSERT((input & ~1) == 0); \
1057 \
1058 lfsr = (lfsr << 1) | input; \
1059 } \
1060 \
1061 (_hash) = (uint16_t)lfsr; \
1062 _NOTE(CONSTANTCONDITION) \
1063 } while (B_FALSE)
1064
1065 extern int sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);
1066
1067 extern int sfxge_pci_init(sfxge_t *);
1068 extern void sfxge_pcie_check_link(sfxge_t *, unsigned int,
1069 unsigned int);
1070 extern int sfxge_pci_ioctl(sfxge_t *, sfxge_pci_ioc_t *);
1071 extern void sfxge_pci_fini(sfxge_t *);
1072
1073 extern int sfxge_bar_init(sfxge_t *);
1074 extern int sfxge_bar_ioctl(sfxge_t *, sfxge_bar_ioc_t *);
1075 extern void sfxge_bar_fini(sfxge_t *);
1076
1077 extern int sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);
1078
1079 #endif /* _KERNEL */
1080
1081 #ifdef __cplusplus
1082 }
1083 #endif
1084
1085 #endif /* _SYS_SFXGE_H */