1 /*
2 * Copyright (c) 2008-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #ifndef _SYS_SFXGE_H
32 #define _SYS_SFXGE_H
33
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37
38 #include <sys/types.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/stream.h>
42 #include <sys/ethernet.h>
43 #include <sys/cpuvar.h>
44
45 #include <sys/mac.h>
46 #include <sys/mac_ether.h>
47 #include <sys/mac_provider.h>
48
49 #include "sfxge_ioc.h"
50 #include "sfxge_debug.h"
51
52 #include "efx.h"
53 #include "efx_regs.h"
54
55 #ifdef _KERNEL
56
57 #define SFXGE_DRIVER_NAME "sfxge"
58
59 #define SFXGE_CPU_CACHE_SIZE 64
60
61 typedef struct sfxge_s sfxge_t;
62
63 typedef enum sfxge_intr_state_e {
64 SFXGE_INTR_UNINITIALIZED = 0,
65 SFXGE_INTR_INITIALIZED,
66 SFXGE_INTR_TESTING,
67 SFXGE_INTR_STARTED
68 } sfxge_intr_state_t;
69
70 typedef struct sfxge_intr_s {
71 ddi_intr_handle_t *si_table;
72 int si_table_size;
73 int si_nalloc;
74 int si_type;
75 int si_cap;
76 efsys_mem_t si_mem;
77 uint64_t si_mask;
78 sfxge_intr_state_t si_state;
79 uint32_t si_zero_count;
80 int si_intr_pri;
81 } sfxge_intr_t;
82
83 typedef enum sfxge_promisc_type_e {
84 SFXGE_PROMISC_OFF = 0,
85 SFXGE_PROMISC_ALL_MULTI,
86 SFXGE_PROMISC_ALL_PHYS
87 } sfxge_promisc_type_t;
88
89 typedef enum sfxge_link_duplex_e {
90 SFXGE_LINK_DUPLEX_UNKNOWN = 0,
91 SFXGE_LINK_DUPLEX_HALF,
92 SFXGE_LINK_DUPLEX_FULL
93 } sfxge_link_duplex_t;
94
95 typedef enum sfxge_unicst_type_e {
96 SFXGE_UNICST_BIA = 0,
97 SFXGE_UNICST_LAA,
98 SFXGE_UNICST_NTYPES
99 } sfxge_unicst_type_t;
100
101 typedef struct sfxge_phy_s {
102 kstat_t *sp_ksp;
103 kstat_named_t *sp_stat;
104 uint32_t *sp_statbuf;
105 efsys_mem_t sp_mem;
106 } sfxge_phy_t;
107
108 typedef enum sfxge_mac_state_e {
109 SFXGE_MAC_UNINITIALIZED = 0,
110 SFXGE_MAC_INITIALIZED,
111 SFXGE_MAC_STARTED
112 } sfxge_mac_state_t;
113
114 typedef struct sfxge_mac_s {
115 sfxge_t *sm_sp;
116 efsys_mem_t sm_mem;
117 kstat_t *sm_ksp;
118 kstat_named_t *sm_stat;
119 uint8_t sm_bia[ETHERADDRL];
120 uint8_t sm_laa[ETHERADDRL];
121 boolean_t sm_laa_valid;
122 unsigned int sm_fcntl;
123 sfxge_promisc_type_t sm_promisc;
124 uint8_t sm_mcast_addr[EFX_MAC_MULTICAST_LIST_MAX *
125 ETHERADDRL]; /* List of multicast addresses to filter on */
126 int sm_mcast_count;
127 clock_t sm_lbolt;
128 kmutex_t sm_lock;
129 efx_link_mode_t sm_link_mode;
130 unsigned int sm_link_speed;
131 sfxge_link_duplex_t sm_link_duplex;
132 boolean_t sm_link_up;
133 boolean_t sm_link_poll_reqd;
134 kcondvar_t sm_link_poll_kv;
135 boolean_t sm_mac_stats_timer_reqd;
136 boolean_t sm_mac_stats_pend;
137 ddi_taskq_t *sm_tqp;
138 sfxge_mac_state_t sm_state;
139 sfxge_phy_t sm_phy;
140 uint32_t sm_phy_cap_to_set;
141 uint32_t sm_phy_cap_to_unset;
142 } sfxge_mac_t;
143
144 typedef enum sfxge_mon_state_e {
145 SFXGE_MON_UNINITIALIZED = 0,
146 SFXGE_MON_INITIALIZED,
147 SFXGE_MON_STARTED
148 } sfxge_mon_state_t;
149
150 typedef struct sfxge_mon_s {
151 sfxge_t *sm_sp;
152 efx_mon_type_t sm_type;
153 unsigned int sm_devid;
154 kstat_t *sm_ksp;
155 kstat_named_t *sm_stat;
156 efx_mon_stat_value_t *sm_statbuf;
157 kmutex_t sm_lock;
158 sfxge_mon_state_t sm_state;
159 efsys_mem_t sm_mem;
160 int sm_polling;
161 } sfxge_mon_t;
162
163 typedef enum sfxge_sram_state_e {
164 SFXGE_SRAM_UNINITIALIZED = 0,
165 SFXGE_SRAM_INITIALIZED,
166 SFXGE_SRAM_STARTED
167 } sfxge_sram_state_t;
168
169 typedef struct sfxge_sram_s {
170 sfxge_t *ss_sp;
171 kmutex_t ss_lock;
172 struct map *ss_buf_tbl_map;
173 unsigned int ss_count;
174 sfxge_sram_state_t ss_state;
175 } sfxge_sram_t;
176
177 typedef enum sfxge_mcdi_state_e {
178 SFXGE_MCDI_UNINITIALIZED = 0,
179 SFXGE_MCDI_INITIALIZED,
180 SFXGE_MCDI_BUSY,
181 SFXGE_MCDI_COMPLETED
182 } sfxge_mcdi_state_t;
183
184 typedef struct sfxge_mcdi_s {
185 sfxge_t *sm_sp;
186 kmutex_t sm_lock;
187 sfxge_mcdi_state_t sm_state;
188 efx_mcdi_transport_t sm_emt;
189 efsys_mem_t sm_mem;
190 kcondvar_t sm_kv; /* MCDI poll complete */
191 } sfxge_mcdi_t;
192
193 #define SFXGE_NEVS 4096
194 #define SFXGE_RX_NDESCS 1024
195 #define SFXGE_TX_NDESCS 1024
196 #define SFXGE_TX_NLABELS EFX_EV_TX_NLABELS
197
198 #define SFXGE_DEFAULT_RXQ_SIZE 1024
199 #define SFXGE_DEFAULT_MODERATION 30
200
201 typedef enum sfxge_evq_state_e {
202 SFXGE_EVQ_UNINITIALIZED = 0,
203 SFXGE_EVQ_INITIALIZED,
204 SFXGE_EVQ_STARTING,
205 SFXGE_EVQ_STARTED
206 } sfxge_evq_state_t;
207
208 #define SFXGE_EV_BATCH (SFXGE_NEVS / 4)
209
210 typedef struct sfxge_txq_s sfxge_txq_t;
211
212 typedef struct sfxge_evq_s {
213 union {
214 struct {
215 sfxge_t *__se_sp;
216 unsigned int __se_index;
217 efsys_mem_t __se_mem;
218 unsigned int __se_id;
219 kstat_t *__se_ksp;
220 kstat_named_t *__se_stat;
221 efx_ev_callbacks_t __se_eec;
222 sfxge_evq_state_t __se_state;
223 boolean_t __se_exception;
224 } __se_s1;
225 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
226 } __se_u1;
227 union {
228 struct {
229 kmutex_t __se_lock;
230 kcondvar_t __se_init_kv;
231 efx_evq_t *__se_eep;
232 unsigned int __se_count;
233 unsigned int __se_rx;
234 unsigned int __se_tx;
235 sfxge_txq_t *__se_stp;
236 sfxge_txq_t **__se_stpp;
237 processorid_t __se_cpu_id;
238 uint16_t __se_ev_batch;
239 } __se_s2;
240 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE];
241 } __se_u2;
242 union {
243 struct {
244 sfxge_txq_t *__se_label_stp[SFXGE_TX_NLABELS];
245 } __se_s3;
246 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
247 } __se_u3;
248 } sfxge_evq_t;
249
250 #define se_sp __se_u1.__se_s1.__se_sp
251 #define se_index __se_u1.__se_s1.__se_index
252 #define se_mem __se_u1.__se_s1.__se_mem
253 #define se_id __se_u1.__se_s1.__se_id
254 #define se_ksp __se_u1.__se_s1.__se_ksp
255 #define se_stat __se_u1.__se_s1.__se_stat
256 #define se_eec __se_u1.__se_s1.__se_eec
257 #define se_state __se_u1.__se_s1.__se_state
258 #define se_exception __se_u1.__se_s1.__se_exception
259
260 #define se_lock __se_u2.__se_s2.__se_lock
261 #define se_init_kv __se_u2.__se_s2.__se_init_kv
262 #define se_eep __se_u2.__se_s2.__se_eep
263 #define se_count __se_u2.__se_s2.__se_count
264 #define se_rx __se_u2.__se_s2.__se_rx
265 #define se_tx __se_u2.__se_s2.__se_tx
266 #define se_stp __se_u2.__se_s2.__se_stp
267 #define se_stpp __se_u2.__se_s2.__se_stpp
268 #define se_cpu_id __se_u2.__se_s2.__se_cpu_id
269 #define se_ev_batch __se_u2.__se_s2.__se_ev_batch
270
271 #define se_label_stp __se_u3.__se_s3.__se_label_stp
272
273
274 #define SFXGE_MAGIC_RESERVED 0x8000
275
276 #define SFXGE_MAGIC_DMAQ_LABEL_WIDTH 5
277 #define SFXGE_MAGIC_DMAQ_LABEL_MASK ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)
278
279 #define SFXGE_MAGIC_RX_QFLUSH_DONE \
280 (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
281
282 #define SFXGE_MAGIC_RX_QFLUSH_FAILED \
283 (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
284
285 #define SFXGE_MAGIC_RX_QFPP_TRIM \
286 (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
287
288 #define SFXGE_MAGIC_TX_QFLUSH_DONE \
289 (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
290
291 typedef struct sfxge_rxq_s sfxge_rxq_t;
292
293 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */
294
295 typedef struct sfxge_rx_packet_s sfxge_rx_packet_t;
296
297 struct sfxge_rx_packet_s {
298 union {
299 struct {
300 frtn_t __srp_free;
301 uint16_t __srp_flags;
302 uint16_t __srp_size;
303 mblk_t *__srp_mp;
304 struct ether_header *__srp_etherhp;
305 struct ip *__srp_iphp;
306 struct tcphdr *__srp_thp;
307 size_t __srp_off;
308 } __srp_s1;
309 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE];
310 } __srp_u1;
311 union {
312 struct {
313 sfxge_rxq_t *__srp_srp;
314 ddi_dma_handle_t __srp_dma_handle;
315 ddi_acc_handle_t __srp_acc_handle;
316 unsigned char *__srp_base;
317 size_t __srp_mblksize;
318 uint64_t __srp_addr;
319 boolean_t __srp_recycle;
320 caddr_t __srp_putp;
321 } __srp_s2;
322 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
323 } __srp_u2;
324 };
325
326 #define srp_free __srp_u1.__srp_s1.__srp_free
327 #define srp_flags __srp_u1.__srp_s1.__srp_flags
328 #define srp_size __srp_u1.__srp_s1.__srp_size
329 #define srp_mp __srp_u1.__srp_s1.__srp_mp
330 #define srp_etherhp __srp_u1.__srp_s1.__srp_etherhp
331 #define srp_iphp __srp_u1.__srp_s1.__srp_iphp
332 #define srp_thp __srp_u1.__srp_s1.__srp_thp
333 #define srp_off __srp_u1.__srp_s1.__srp_off
334
335 #define srp_srp __srp_u2.__srp_s2.__srp_srp
336 #define srp_dma_handle __srp_u2.__srp_s2.__srp_dma_handle
337 #define srp_acc_handle __srp_u2.__srp_s2.__srp_acc_handle
338 #define srp_base __srp_u2.__srp_s2.__srp_base
339 #define srp_mblksize __srp_u2.__srp_s2.__srp_mblksize
340 #define srp_addr __srp_u2.__srp_s2.__srp_addr
341 #define srp_recycle __srp_u2.__srp_s2.__srp_recycle
342 #define srp_putp __srp_u2.__srp_s2.__srp_putp
343
344 #define SFXGE_RX_FPP_NSLOTS 8
345 #define SFXGE_RX_FPP_MASK (SFXGE_RX_FPP_NSLOTS - 1)
346
347 /* Free packet pool putlist (dynamically allocated) */
348 typedef struct sfxge_rx_fpp_putlist_s {
349 kmutex_t srfpl_lock;
350 unsigned int srfpl_count;
351 mblk_t *srfpl_putp;
352 mblk_t **srfpl_putpp;
353 } sfxge_rx_fpp_putlist_t;
354
355 /* Free packet pool */
356 typedef struct sfxge_rx_fpp_s {
357 caddr_t srfpp_putp;
358 unsigned int srfpp_loaned;
359 mblk_t *srfpp_get;
360 unsigned int srfpp_count;
361 unsigned int srfpp_min;
362 /* Low water mark: Don't trim to below this */
363 unsigned int srfpp_lowat;
364 } sfxge_rx_fpp_t;
365
366 typedef struct sfxge_rx_flow_s sfxge_rx_flow_t;
367
368 struct sfxge_rx_flow_s {
369 uint32_t srf_tag;
370 /* in-order segment count */
371 unsigned int srf_count;
372 uint16_t srf_tci;
373 uint32_t srf_saddr;
374 uint32_t srf_daddr;
375 uint16_t srf_sport;
376 uint16_t srf_dport;
377 /* sequence number */
378 uint32_t srf_seq;
379 clock_t srf_lbolt;
380 mblk_t *srf_mp;
381 mblk_t **srf_mpp;
382 struct ether_header *srf_etherhp;
383 struct ip *srf_iphp;
384 struct tcphdr *srf_first_thp;
385 struct tcphdr *srf_last_thp;
386 size_t srf_len;
387 sfxge_rx_flow_t *srf_next;
388 };
389
390 #define SFXGE_MAX_FLOW 1024
391 #define SFXGE_SLOW_START 20
392
393 typedef enum sfxge_flush_state_e {
394 SFXGE_FLUSH_INACTIVE = 0,
395 SFXGE_FLUSH_DONE,
396 SFXGE_FLUSH_PENDING,
397 SFXGE_FLUSH_FAILED
398 } sfxge_flush_state_t;
399
400 typedef enum sfxge_rxq_state_e {
401 SFXGE_RXQ_UNINITIALIZED = 0,
402 SFXGE_RXQ_INITIALIZED,
403 SFXGE_RXQ_STARTED
404 } sfxge_rxq_state_t;
405
406
407 #define SFXGE_RX_BATCH 128
408 #define SFXGE_RX_NSTATS 8 /* note that *esballoc share one kstat */
409
410 struct sfxge_rxq_s {
411 union {
412 struct {
413 sfxge_t *__sr_sp;
414 unsigned int __sr_index;
415 efsys_mem_t __sr_mem;
416 unsigned int __sr_id;
417 unsigned int __sr_lowat;
418 unsigned int __sr_hiwat;
419 volatile timeout_id_t __sr_tid;
420 sfxge_rxq_state_t __sr_state;
421 } __sr_s1;
422 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
423 } __sr_u1;
424 union {
425 struct {
426 sfxge_rx_packet_t **__sr_srpp;
427 unsigned int __sr_added;
428 unsigned int __sr_pushed;
429 unsigned int __sr_pending;
430 unsigned int __sr_completed;
431 unsigned int __sr_loopback;
432 mblk_t *__sr_mp;
433 mblk_t **__sr_mpp;
434 sfxge_rx_flow_t *__sr_flow;
435 sfxge_rx_flow_t *__sr_srfp;
436 sfxge_rx_flow_t **__sr_srfpp;
437 clock_t __sr_rto;
438 } __sr_s2;
439 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
440 } __sr_u2;
441 union {
442 struct {
443 sfxge_rx_fpp_t __sr_fpp;
444 efx_rxq_t *__sr_erp;
445 volatile sfxge_flush_state_t __sr_flush;
446 kcondvar_t __sr_flush_kv;
447 kstat_t *__sr_ksp;
448 } __sr_s3;
449 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE];
450 } __sr_u3;
451 struct {
452 /* NB must match SFXGE_RX_NSTATS */
453 uint32_t srk_rx_pkt_mem_limit;
454 uint32_t srk_kcache_alloc_nomem;
455 uint32_t srk_dma_alloc_nomem;
456 uint32_t srk_dma_alloc_fail;
457 uint32_t srk_dma_bind_nomem;
458 uint32_t srk_dma_bind_fail;
459 uint32_t srk_desballoc_fail;
460 uint32_t srk_rxq_empty_discard;
461 } sr_kstat;
462 };
463
464 #define sr_sp __sr_u1.__sr_s1.__sr_sp
465 #define sr_index __sr_u1.__sr_s1.__sr_index
466 #define sr_mem __sr_u1.__sr_s1.__sr_mem
467 #define sr_id __sr_u1.__sr_s1.__sr_id
468 #define sr_mrh __sr_u1.__sr_s1.__sr_mrh
469 #define sr_lowat __sr_u1.__sr_s1.__sr_lowat
470 #define sr_hiwat __sr_u1.__sr_s1.__sr_hiwat
471 #define sr_tid __sr_u1.__sr_s1.__sr_tid
472 #define sr_state __sr_u1.__sr_s1.__sr_state
473
474 #define sr_srpp __sr_u2.__sr_s2.__sr_srpp
475 #define sr_added __sr_u2.__sr_s2.__sr_added
476 #define sr_pushed __sr_u2.__sr_s2.__sr_pushed
477 #define sr_pending __sr_u2.__sr_s2.__sr_pending
478 #define sr_completed __sr_u2.__sr_s2.__sr_completed
479 #define sr_loopback __sr_u2.__sr_s2.__sr_loopback
480 #define sr_mp __sr_u2.__sr_s2.__sr_mp
481 #define sr_mpp __sr_u2.__sr_s2.__sr_mpp
482 #define sr_flow __sr_u2.__sr_s2.__sr_flow
483 #define sr_srfp __sr_u2.__sr_s2.__sr_srfp
484 #define sr_srfpp __sr_u2.__sr_s2.__sr_srfpp
485 #define sr_rto __sr_u2.__sr_s2.__sr_rto
486
487 #define sr_fpp __sr_u3.__sr_s3.__sr_fpp
488 #define sr_erp __sr_u3.__sr_s3.__sr_erp
489 #define sr_flush __sr_u3.__sr_s3.__sr_flush
490 #define sr_flush_kv __sr_u3.__sr_s3.__sr_flush_kv
491 #define sr_ksp __sr_u3.__sr_s3.__sr_ksp
492
493 typedef struct sfxge_tx_packet_s sfxge_tx_packet_t;
494
495 /* Packet type from parsing transmit packet */
496 typedef enum sfxge_packet_type_e {
497 SFXGE_PACKET_TYPE_UNKNOWN = 0,
498 SFXGE_PACKET_TYPE_IPV4_TCP,
499 SFXGE_PACKET_TYPE_IPV4_UDP,
500 SFXGE_PACKET_TYPE_IPV4_SCTP,
501 SFXGE_PACKET_TYPE_IPV4_OTHER,
502 SFXGE_PACKET_NTYPES
503 } sfxge_packet_type_t;
504
505 struct sfxge_tx_packet_s {
506 sfxge_tx_packet_t *stp_next;
507 mblk_t *stp_mp;
508 struct ether_header *stp_etherhp;
509 struct ip *stp_iphp;
510 struct tcphdr *stp_thp;
511 size_t stp_off;
512 size_t stp_size;
513 size_t stp_mss;
514 uint32_t stp_dpl_put_len;
515 };
516
517 #define SFXGE_TX_FPP_MAX 64
518
519 typedef struct sfxge_tx_fpp_s {
520 sfxge_tx_packet_t *stf_stpp;
521 unsigned int stf_count;
522 } sfxge_tx_fpp_t;
523
524 typedef struct sfxge_tx_mapping_s sfxge_tx_mapping_t;
525
526 #define SFXGE_TX_MAPPING_NADDR (((1 << 16) >> 12) + 2)
527
528 struct sfxge_tx_mapping_s {
529 sfxge_tx_mapping_t *stm_next;
530 sfxge_t *stm_sp;
531 mblk_t *stm_mp;
532 ddi_dma_handle_t stm_dma_handle;
533 caddr_t stm_base;
534 size_t stm_size;
535 size_t stm_off;
536 uint64_t stm_addr[SFXGE_TX_MAPPING_NADDR];
537 };
538
539 typedef struct sfxge_tx_fmp_s {
540 sfxge_tx_mapping_t *stf_stmp;
541 unsigned int stf_count;
542 } sfxge_tx_fmp_t;
543
544 typedef struct sfxge_tx_buffer_s sfxge_tx_buffer_t;
545
546 struct sfxge_tx_buffer_s {
547 sfxge_tx_buffer_t *stb_next;
548 size_t stb_off;
549 efsys_mem_t stb_esm;
550 };
551
552 #define SFXGE_TX_BUFFER_SIZE 0x400
553 #define SFXGE_TX_HEADER_SIZE 0x100
554 #define SFXGE_TX_COPY_THRESHOLD 0x200
555
556 typedef struct sfxge_tx_fbp_s {
557 sfxge_tx_buffer_t *stf_stbp;
558 unsigned int stf_count;
559 } sfxge_tx_fbp_t;
560
561 typedef struct sfxge_tx_dpl_s {
562 uintptr_t std_put;
563 sfxge_tx_packet_t *std_get;
564 sfxge_tx_packet_t **std_getp;
565 unsigned int std_count; /* only get list count */
566 unsigned int get_pkt_limit;
567 unsigned int put_pkt_limit;
568 unsigned int get_full_count;
569 unsigned int put_full_count;
570 } sfxge_tx_dpl_t;
571
572 typedef enum sfxge_txq_state_e {
573 SFXGE_TXQ_UNINITIALIZED = 0,
574 SFXGE_TXQ_INITIALIZED,
575 SFXGE_TXQ_STARTED,
576 SFXGE_TXQ_FLUSH_PENDING,
577 SFXGE_TXQ_FLUSH_DONE,
578 SFXGE_TXQ_FLUSH_FAILED
579 } sfxge_txq_state_t;
580
581 typedef enum sfxge_txq_type_e {
582 SFXGE_TXQ_NON_CKSUM = 0,
583 SFXGE_TXQ_IP_CKSUM,
584 SFXGE_TXQ_IP_TCP_UDP_CKSUM,
585 SFXGE_TXQ_NTYPES
586 } sfxge_txq_type_t;
587
588 #define SFXGE_TXQ_UNBLOCK_LEVEL1 (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4)
589 #define SFXGE_TXQ_UNBLOCK_LEVEL2 0
590 #define SFXGE_TXQ_NOT_BLOCKED -1
591
592 #define SFXGE_TX_BATCH 64
593
594 struct sfxge_txq_s {
595 union {
596 struct {
597 sfxge_t *__st_sp;
598 unsigned int __st_index;
599 unsigned int __st_label;
600 sfxge_txq_type_t __st_type;
601 unsigned int __st_evq;
602 efsys_mem_t __st_mem;
603 unsigned int __st_id;
604 kstat_t *__st_ksp;
605 kstat_named_t *__st_stat;
606 sfxge_txq_state_t __st_state;
607 } __st_s1;
608 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 2];
609 } __st_u1;
610 union {
611 struct {
612 sfxge_tx_dpl_t __st_dpl;
613 } __st_s2;
614 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
615 } __st_u2;
616 union {
617 struct {
618 kmutex_t __st_lock;
619 /* mapping pool - sfxge_tx_mapping_t */
620 sfxge_tx_fmp_t __st_fmp;
621 /* buffer pool - sfxge_tx_buffer_t */
622 sfxge_tx_fbp_t __st_fbp;
623 /* packet pool - sfxge_tx_packet_t */
624 sfxge_tx_fpp_t __st_fpp;
625 efx_buffer_t *__st_eb;
626 unsigned int __st_n;
627 efx_txq_t *__st_etp;
628 sfxge_tx_mapping_t **__st_stmp;
629 sfxge_tx_buffer_t **__st_stbp;
630 mblk_t **__st_mp;
631 unsigned int __st_added;
632 unsigned int __st_reaped;
633 int __st_unblock;
634 } __st_s3;
635 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 3];
636 } __st_u3;
637 union {
638 struct {
639 sfxge_txq_t *__st_next;
640 unsigned int __st_pending;
641 unsigned int __st_completed;
642
643 } __st_s4;
644 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
645 } __st_u4;
646 };
647
648 #define st_sp __st_u1.__st_s1.__st_sp
649 #define st_index __st_u1.__st_s1.__st_index
650 #define st_label __st_u1.__st_s1.__st_label
651 #define st_type __st_u1.__st_s1.__st_type
652 #define st_evq __st_u1.__st_s1.__st_evq
653 #define st_mem __st_u1.__st_s1.__st_mem
654 #define st_id __st_u1.__st_s1.__st_id
655 #define st_ksp __st_u1.__st_s1.__st_ksp
656 #define st_stat __st_u1.__st_s1.__st_stat
657 #define st_state __st_u1.__st_s1.__st_state
658
659 #define st_dpl __st_u2.__st_s2.__st_dpl
660
661 #define st_lock __st_u3.__st_s3.__st_lock
662 #define st_fmp __st_u3.__st_s3.__st_fmp
663 #define st_fbp __st_u3.__st_s3.__st_fbp
664 #define st_fpp __st_u3.__st_s3.__st_fpp
665 #define st_eb __st_u3.__st_s3.__st_eb
666 #define st_n __st_u3.__st_s3.__st_n
667 #define st_etp __st_u3.__st_s3.__st_etp
668 #define st_stmp __st_u3.__st_s3.__st_stmp
669 #define st_stbp __st_u3.__st_s3.__st_stbp
670 #define st_mp __st_u3.__st_s3.__st_mp
671 #define st_added __st_u3.__st_s3.__st_added
672 #define st_reaped __st_u3.__st_s3.__st_reaped
673 #define st_unblock __st_u3.__st_s3.__st_unblock
674
675 #define st_next __st_u4.__st_s4.__st_next
676 #define st_pending __st_u4.__st_s4.__st_pending
677 #define st_completed __st_u4.__st_s4.__st_completed
678
679 typedef enum sfxge_rx_scale_state_e {
680 SFXGE_RX_SCALE_UNINITIALIZED = 0,
681 SFXGE_RX_SCALE_INITIALIZED,
682 SFXGE_RX_SCALE_STARTED
683 } sfxge_rx_scale_state_t;
684
685 #define SFXGE_RX_SCALE_MAX EFX_RSS_TBL_SIZE
686
687 typedef struct sfxge_rx_scale_s {
688 kmutex_t srs_lock;
689 unsigned int *srs_cpu;
690 unsigned int srs_tbl[SFXGE_RX_SCALE_MAX];
691 unsigned int srs_count;
692 kstat_t *srs_ksp;
693 sfxge_rx_scale_state_t srs_state;
694 } sfxge_rx_scale_t;
695
696
697 typedef enum sfxge_rx_coalesce_mode_e {
698 SFXGE_RX_COALESCE_OFF = 0,
699 SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
700 SFXGE_RX_COALESCE_ALLOW_PUSH = 2
701 } sfxge_rx_coalesce_mode_t;
702
703 typedef enum sfxge_vpd_type_e {
704 SFXGE_VPD_ID = 0,
705 SFXGE_VPD_PN = 1,
706 SFXGE_VPD_SN = 2,
707 SFXGE_VPD_EC = 3,
708 SFXGE_VPD_MN = 4,
709 SFXGE_VPD_VD = 5,
710 SFXGE_VPD_VE = 6,
711 SFXGE_VPD_MAX = 7,
712 } sfxge_vpd_type_t;
713
714 typedef struct sfxge_vpd_kstat_s {
715 kstat_t *svk_ksp;
716 kstat_named_t svk_stat[SFXGE_VPD_MAX];
717 efx_vpd_value_t *svk_vv;
718 } sfxge_vpd_kstat_t;
719
720 typedef struct sfxge_cfg_kstat_s {
721 struct {
722 kstat_named_t sck_mac;
723 kstat_named_t sck_version;
724 } kstat;
725 struct {
726 char sck_mac[64 + 1];
727 } buf;
728 } sfxge_cfg_kstat_t;
729
730 typedef enum sfxge_state_e {
731 SFXGE_UNINITIALIZED = 0,
732 SFXGE_INITIALIZED,
733 SFXGE_REGISTERED,
734 SFXGE_STARTING,
735 SFXGE_STARTED,
736 SFXGE_STOPPING
737 } sfxge_state_t;
738
739 typedef enum sfxge_hw_err_e {
740 SFXGE_HW_OK = 0,
741 SFXGE_HW_ERR,
742 } sfxge_hw_err_t;
743
744 typedef enum sfxge_action_on_hw_err_e {
745 SFXGE_RECOVER = 0,
746 SFXGE_INVISIBLE = 1,
747 SFXGE_LEAVE_DEAD = 2,
748 } sfxge_action_on_hw_err_t;
749
750 typedef char *sfxge_mac_priv_prop_t;
751
752 #define SFXGE_TOEPLITZ_KEY_LEN 40
753
754 struct sfxge_s {
755 kmutex_t s_state_lock;
756 sfxge_state_t s_state;
757 dev_info_t *s_dip;
758 ddi_taskq_t *s_tqp;
759 ddi_acc_handle_t s_pci_handle;
760 uint16_t s_pci_venid;
761 uint16_t s_pci_devid;
762 #if EFSYS_OPT_MCDI_LOGGING
763 unsigned int s_bus_addr;
764 #endif
765 efx_family_t s_family;
766 unsigned int s_pcie_nlanes;
767 unsigned int s_pcie_linkspeed;
768 kmutex_t s_nic_lock;
769 efsys_bar_t s_bar;
770 sfxge_intr_t s_intr;
771 sfxge_mac_t s_mac;
772 sfxge_mon_t s_mon;
773 sfxge_sram_t s_sram;
774 sfxge_mcdi_t s_mcdi;
775 kmem_cache_t *s_eq0c; /* eventQ 0 */
776 kmem_cache_t *s_eqXc; /* all other eventQs */
777 sfxge_evq_t *s_sep[SFXGE_RX_SCALE_MAX];
778 unsigned int s_ev_moderation;
779 kmem_cache_t *s_rqc;
780 sfxge_rxq_t *s_srp[SFXGE_RX_SCALE_MAX];
781 sfxge_rx_scale_t s_rx_scale;
782 size_t s_rx_prefix_size;
783 size_t s_rx_buffer_size;
784 size_t s_rx_buffer_align;
785 sfxge_rx_coalesce_mode_t s_rx_coalesce_mode;
786 int64_t s_rx_pkt_mem_max;
787 volatile uint64_t s_rx_pkt_mem_alloc;
788 kmem_cache_t *s_rpc;
789 kmem_cache_t *s_tqc;
790 unsigned int s_tx_scale_base[SFXGE_TXQ_NTYPES];
791 unsigned int s_tx_scale_max[SFXGE_TXQ_NTYPES];
792 int s_tx_qcount;
793 sfxge_txq_t *s_stp[SFXGE_RX_SCALE_MAX *
794 SFXGE_TXQ_NTYPES]; /* Sparse array */
795 kmem_cache_t *s_tpc;
796 int s_tx_flush_pending;
797 kmutex_t s_tx_flush_lock;
798 kcondvar_t s_tx_flush_kv;
799 kmem_cache_t *s_tbc;
800 kmem_cache_t *s_tmc;
801 efx_nic_t *s_enp;
802 sfxge_vpd_kstat_t s_vpd_kstat;
803 sfxge_cfg_kstat_t s_cfg_kstat;
804 kstat_t *s_cfg_ksp;
805 size_t s_mtu;
806 int s_rxq_poll_usec;
807 mac_callbacks_t s_mc;
808 mac_handle_t s_mh;
809 sfxge_mac_priv_prop_t *s_mac_priv_props;
810 int s_mac_priv_props_alloc;
811 volatile uint32_t s_nested_restarts;
812 uint32_t s_num_restarts;
813 uint32_t s_num_restarts_hw_err;
814 sfxge_hw_err_t s_hw_err;
815 sfxge_action_on_hw_err_t s_action_on_hw_err;
816 uint16_t s_rxq_size;
817 uint16_t s_evq0_size;
818 uint16_t s_evqX_size;
819 #if EFSYS_OPT_MCDI_LOGGING
820 int s_mcdi_logging;
821 #endif
822 const uint32_t *s_toeplitz_cache;
823 };
824
825 typedef struct sfxge_dma_buffer_attr_s {
826 dev_info_t *sdba_dip;
827 ddi_dma_attr_t *sdba_dattrp;
828 int (*sdba_callback) (caddr_t);
829 size_t sdba_length;
830 uint_t sdba_memflags;
831 ddi_device_acc_attr_t *sdba_devaccp;
832 uint_t sdba_bindflags;
833 int sdba_maxcookies;
834 boolean_t sdba_zeroinit;
835 } sfxge_dma_buffer_attr_t;
836
837 extern const char sfxge_ident[];
838 extern uint8_t sfxge_brdcst[];
839
840 extern kmutex_t sfxge_global_lock;
841
842 extern unsigned int *sfxge_cpu;
843
844 extern int sfxge_start(sfxge_t *, boolean_t);
845 extern void sfxge_stop(sfxge_t *);
846 extern void sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
847 extern int sfxge_restart_dispatch(sfxge_t *, uint_t,
848 sfxge_hw_err_t, const char *, uint32_t);
849
850 extern void sfxge_gld_link_update(sfxge_t *);
851 extern void sfxge_gld_mtu_update(sfxge_t *);
852 extern void sfxge_gld_rx_post(sfxge_t *, unsigned int,
853 mblk_t *);
854 extern void sfxge_gld_rx_push(sfxge_t *);
855 extern int sfxge_gld_register(sfxge_t *);
856 extern int sfxge_gld_unregister(sfxge_t *);
857
858 extern int sfxge_dma_buffer_create(efsys_mem_t *,
859 const sfxge_dma_buffer_attr_t *);
860 extern void sfxge_dma_buffer_destroy(efsys_mem_t *);
861
862 extern int sfxge_intr_init(sfxge_t *);
863 extern int sfxge_intr_start(sfxge_t *);
864 extern void sfxge_intr_stop(sfxge_t *);
865 extern void sfxge_intr_fini(sfxge_t *);
866 extern void sfxge_intr_fatal(sfxge_t *);
867
868 extern int sfxge_ev_init(sfxge_t *);
869 extern int sfxge_ev_start(sfxge_t *);
870 extern void sfxge_ev_moderation_get(sfxge_t *,
871 unsigned int *);
872 extern int sfxge_ev_moderation_set(sfxge_t *,
873 unsigned int);
874 extern int sfxge_ev_qmoderate(sfxge_t *, unsigned int,
875 unsigned int);
876 extern int sfxge_ev_qpoll(sfxge_t *, unsigned int);
877 extern int sfxge_ev_qprime(sfxge_t *, unsigned int);
878 extern void sfxge_ev_stop(sfxge_t *);
879 extern void sfxge_ev_fini(sfxge_t *);
880 extern int sfxge_ev_txlabel_alloc(sfxge_t *sp,
881 unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp);
882 extern int sfxge_ev_txlabel_free(sfxge_t *sp,
883 unsigned int evq, sfxge_txq_t *stp, unsigned int label);
884
885 extern int sfxge_mon_init(sfxge_t *);
886 extern int sfxge_mon_start(sfxge_t *);
887 extern void sfxge_mon_stop(sfxge_t *);
888 extern void sfxge_mon_fini(sfxge_t *);
889
890 extern int sfxge_mac_init(sfxge_t *);
891 extern int sfxge_mac_start(sfxge_t *, boolean_t);
892 extern void sfxge_mac_stat_get(sfxge_t *, unsigned int,
893 uint64_t *);
894 extern void sfxge_mac_link_check(sfxge_t *, boolean_t *);
895 extern void sfxge_mac_link_speed_get(sfxge_t *,
896 unsigned int *);
897 extern void sfxge_mac_link_duplex_get(sfxge_t *,
898 sfxge_link_duplex_t *);
899 extern void sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
900 extern int sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
901 extern int sfxge_mac_unicst_get(sfxge_t *,
902 sfxge_unicst_type_t, uint8_t *);
903 extern int sfxge_mac_unicst_set(sfxge_t *,
904 uint8_t *);
905 extern int sfxge_mac_promisc_set(sfxge_t *,
906 sfxge_promisc_type_t);
907 extern int sfxge_mac_multicst_add(sfxge_t *,
908 uint8_t const *addr);
909 extern int sfxge_mac_multicst_remove(sfxge_t *,
910 uint8_t const *addr);
911 extern void sfxge_mac_stop(sfxge_t *);
912 extern void sfxge_mac_fini(sfxge_t *);
913 extern void sfxge_mac_link_update(sfxge_t *sp,
914 efx_link_mode_t mode);
915
916 extern int sfxge_mcdi_init(sfxge_t *sp);
917 extern void sfxge_mcdi_fini(sfxge_t *sp);
918 extern int sfxge_mcdi_ioctl(sfxge_t *sp,
919 sfxge_mcdi_ioc_t *smip);
920 extern int sfxge_mcdi2_ioctl(sfxge_t *sp,
921 sfxge_mcdi2_ioc_t *smip);
922
923 extern int sfxge_phy_init(sfxge_t *);
924 extern void sfxge_phy_link_mode_get(sfxge_t *,
925 efx_link_mode_t *);
926 extern void sfxge_phy_fini(sfxge_t *);
927 extern int sfxge_phy_kstat_init(sfxge_t *sp);
928 extern void sfxge_phy_kstat_fini(sfxge_t *sp);
929 extern uint8_t sfxge_phy_lp_cap_test(sfxge_t *sp,
930 uint32_t field);
931 extern int sfxge_phy_cap_apply(sfxge_t *sp,
932 boolean_t use_default);
933 extern uint8_t sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
934 uint32_t field, boolean_t *mutablep);
935 extern int sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
936 int set);
937
938 extern int sfxge_rx_init(sfxge_t *);
939 extern int sfxge_rx_start(sfxge_t *);
940 extern void sfxge_rx_coalesce_mode_get(sfxge_t *,
941 sfxge_rx_coalesce_mode_t *);
942 extern int sfxge_rx_coalesce_mode_set(sfxge_t *,
943 sfxge_rx_coalesce_mode_t);
944 extern unsigned int sfxge_rx_scale_prop_get(sfxge_t *);
945 extern void sfxge_rx_scale_update(void *);
946 extern int sfxge_rx_scale_count_get(sfxge_t *,
947 unsigned int *);
948 extern int sfxge_rx_scale_count_set(sfxge_t *,
949 unsigned int);
950 extern void sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
951 extern void sfxge_rx_qflush_done(sfxge_rxq_t *);
952 extern void sfxge_rx_qflush_failed(sfxge_rxq_t *);
953 extern void sfxge_rx_qfpp_trim(sfxge_rxq_t *);
954 extern void sfxge_rx_stop(sfxge_t *);
955 extern unsigned int sfxge_rx_loaned(sfxge_t *);
956 extern void sfxge_rx_fini(sfxge_t *);
957
958 extern int sfxge_tx_init(sfxge_t *);
959 extern int sfxge_tx_start(sfxge_t *);
960 extern int sfxge_tx_packet_add(sfxge_t *, mblk_t *);
961 extern void sfxge_tx_qcomplete(sfxge_txq_t *);
962 extern void sfxge_tx_qflush_done(sfxge_txq_t *);
963 extern void sfxge_tx_stop(sfxge_t *);
964 extern void sfxge_tx_fini(sfxge_t *);
965 extern void sfxge_tx_qdpl_flush(sfxge_txq_t *stp);
966
967 extern void sfxge_sram_init(sfxge_t *);
968 extern int sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
969 uint32_t *);
970 extern int sfxge_sram_start(sfxge_t *);
971 extern int sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
972 efsys_mem_t *, size_t);
973 extern void sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
974 size_t);
975 extern void sfxge_sram_stop(sfxge_t *);
976 extern void sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
977 size_t);
978 extern void sfxge_sram_fini(sfxge_t *);
979
980 extern sfxge_packet_type_t sfxge_pkthdr_parse(mblk_t *,
981 struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *,
982 uint16_t *, uint16_t *);
983
984 extern int sfxge_toeplitz_hash_init(sfxge_t *);
985 extern void sfxge_toeplitz_hash_fini(sfxge_t *);
986 extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int,
987 uint8_t *, uint16_t, uint8_t *, uint16_t);
988
989 /*
990 * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection.
991 * To compute the same hash value as Siena/Huntington hardware, the inputs
992 * must be in big endian (network) byte order.
993 */
994 #define SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash) \
995 do { \
996 (_hash) = sfxge_toeplitz_hash(_sp, \
997 sizeof (struct in_addr), \
998 (uint8_t *)(_raddr), \
999 (_rport), \
1000 (uint8_t *)(_laddr), \
1001 (_lport)); \
1002 _NOTE(CONSTANTCONDITION) \
1003 } while (B_FALSE)
1004
1005 /*
1006 * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection.
1007 * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers.
1008 * For other IPv4 non-TCP packets, use zero for the port numbers.
1009 */
1010 #define SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash) \
1011 SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash))
1012
1013
1014 extern int sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);
1015
1016 extern int sfxge_pci_init(sfxge_t *);
1017 extern void sfxge_pcie_check_link(sfxge_t *, unsigned int,
1018 unsigned int);
1019 extern void sfxge_pci_fini(sfxge_t *);
1020
1021 extern int sfxge_bar_init(sfxge_t *);
1022 extern void sfxge_bar_fini(sfxge_t *);
1023
1024 extern int sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);
1025
1026
1027 #endif /* _KERNEL */
1028
1029 #ifdef __cplusplus
1030 }
1031 #endif
1032
1033 #endif /* _SYS_SFXGE_H */