1 /*
2 * Copyright (c) 2008-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #ifndef _SYS_SFXGE_H
32 #define _SYS_SFXGE_H
33
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37
38 #include <sys/types.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/stream.h>
42 #include <sys/ethernet.h>
43 #include <sys/cpuvar.h>
44
45 #include <sys/mac.h>
46 #include <sys/mac_ether.h>
47 #include <sys/mac_provider.h>
48
49 #include "sfxge_ioc.h"
50 #include "sfxge_debug.h"
51
52 #include "efx.h"
53 #include "efx_regs.h"
54
55 #ifdef _KERNEL
56
57 #define SFXGE_DRIVER_NAME "sfxge"
58
59 #define SFXGE_CPU_CACHE_SIZE 64
60
61 typedef struct sfxge_s sfxge_t;
62
63 typedef enum sfxge_intr_state_e {
64 SFXGE_INTR_UNINITIALIZED = 0,
65 SFXGE_INTR_INITIALIZED,
66 SFXGE_INTR_TESTING,
67 SFXGE_INTR_STARTED
68 } sfxge_intr_state_t;
69
70 typedef struct sfxge_intr_s {
71 ddi_intr_handle_t *si_table;
72 int si_table_size;
73 int si_nalloc;
74 int si_type;
75 int si_cap;
76 efsys_mem_t si_mem;
77 uint64_t si_mask;
78 sfxge_intr_state_t si_state;
79 uint32_t si_zero_count;
80 int si_intr_pri;
81 } sfxge_intr_t;
82
83 typedef enum sfxge_promisc_type_e {
84 SFXGE_PROMISC_OFF = 0,
85 SFXGE_PROMISC_ALL_MULTI,
86 SFXGE_PROMISC_ALL_PHYS
87 } sfxge_promisc_type_t;
88
89 #define SFXGE_MCAST_LIST_MAX 256
90
91 typedef enum sfxge_link_duplex_e {
92 SFXGE_LINK_DUPLEX_UNKNOWN = 0,
93 SFXGE_LINK_DUPLEX_HALF,
94 SFXGE_LINK_DUPLEX_FULL
95 } sfxge_link_duplex_t;
96
97 typedef enum sfxge_unicst_type_e {
98 SFXGE_UNICST_BIA = 0,
99 SFXGE_UNICST_LAA,
100 SFXGE_UNICST_NTYPES
101 } sfxge_unicst_type_t;
102
103 typedef struct sfxge_phy_s {
104 kstat_t *sp_ksp;
105 kstat_named_t *sp_stat;
106 uint32_t *sp_statbuf;
107 efsys_mem_t sp_mem;
108 } sfxge_phy_t;
109
110 typedef enum sfxge_mac_state_e {
111 SFXGE_MAC_UNINITIALIZED = 0,
112 SFXGE_MAC_INITIALIZED,
113 SFXGE_MAC_STARTED
114 } sfxge_mac_state_t;
115
116 typedef struct sfxge_mac_s {
117 sfxge_t *sm_sp;
118 efsys_mem_t sm_mem;
119 kstat_t *sm_ksp;
120 kstat_named_t *sm_stat;
121 uint8_t sm_bia[ETHERADDRL];
122 uint8_t sm_laa[ETHERADDRL];
123 boolean_t sm_laa_valid;
124 unsigned int sm_fcntl;
125 sfxge_promisc_type_t sm_promisc;
126 uint8_t sm_mcast_addr[SFXGE_MCAST_LIST_MAX *
127 ETHERADDRL]; /* List of multicast addresses to filter on */
128 int sm_mcast_count;
129 clock_t sm_lbolt;
130 kmutex_t sm_lock;
131 efx_link_mode_t sm_link_mode;
132 unsigned int sm_link_speed;
133 sfxge_link_duplex_t sm_link_duplex;
134 boolean_t sm_link_up;
135 boolean_t sm_link_poll_reqd;
136 kcondvar_t sm_link_poll_kv;
137 boolean_t sm_mac_stats_timer_reqd;
138 boolean_t sm_mac_stats_pend;
139 ddi_taskq_t *sm_tqp;
140 sfxge_mac_state_t sm_state;
141 sfxge_phy_t sm_phy;
142 uint32_t sm_phy_cap_to_set;
143 uint32_t sm_phy_cap_to_unset;
144 } sfxge_mac_t;
145
146 typedef enum sfxge_mon_state_e {
147 SFXGE_MON_UNINITIALIZED = 0,
148 SFXGE_MON_INITIALIZED,
149 SFXGE_MON_STARTED
150 } sfxge_mon_state_t;
151
152 typedef struct sfxge_mon_s {
153 sfxge_t *sm_sp;
154 efx_mon_type_t sm_type;
155 unsigned int sm_devid;
156 kstat_t *sm_ksp;
157 kstat_named_t *sm_stat;
158 efx_mon_stat_value_t *sm_statbuf;
159 kmutex_t sm_lock;
160 sfxge_mon_state_t sm_state;
161 efsys_mem_t sm_mem;
162 } sfxge_mon_t;
163
164 typedef enum sfxge_sram_state_e {
165 SFXGE_SRAM_UNINITIALIZED = 0,
166 SFXGE_SRAM_INITIALIZED,
167 SFXGE_SRAM_STARTED
168 } sfxge_sram_state_t;
169
170 typedef struct sfxge_sram_s {
171 sfxge_t *ss_sp;
172 kmutex_t ss_lock;
173 vmem_t *ss_buf_tbl;
174 unsigned int ss_count;
175 sfxge_sram_state_t ss_state;
176 } sfxge_sram_t;
177
178 typedef enum sfxge_mcdi_state_e {
179 SFXGE_MCDI_UNINITIALIZED = 0,
180 SFXGE_MCDI_INITIALIZED,
181 SFXGE_MCDI_BUSY,
182 SFXGE_MCDI_COMPLETED
183 } sfxge_mcdi_state_t;
184
185 typedef struct sfxge_mcdi_s {
186 sfxge_t *sm_sp;
187 kmutex_t sm_lock;
188 sfxge_mcdi_state_t sm_state;
189 efx_mcdi_transport_t sm_emt;
190 efsys_mem_t sm_mem;
191 kcondvar_t sm_kv; /* MCDI poll complete */
192 } sfxge_mcdi_t;
193
194 #define SFXGE_NEVS 4096
195 #define SFXGE_RX_NDESCS 1024
196 #define SFXGE_TX_NDESCS 1024
197 #define SFXGE_TX_NLABELS EFX_EV_TX_NLABELS
198
199 #define SFXGE_DEFAULT_RXQ_SIZE 1024
200 #define SFXGE_DEFAULT_MODERATION 30
201
202 typedef enum sfxge_evq_state_e {
203 SFXGE_EVQ_UNINITIALIZED = 0,
204 SFXGE_EVQ_INITIALIZED,
205 SFXGE_EVQ_STARTING,
206 SFXGE_EVQ_STARTED
207 } sfxge_evq_state_t;
208
209 #define SFXGE_EV_BATCH (SFXGE_NEVS / 4)
210
211 typedef struct sfxge_txq_s sfxge_txq_t;
212
213 typedef struct sfxge_evq_s {
214 union {
215 struct {
216 sfxge_t *__se_sp;
217 unsigned int __se_index;
218 efsys_mem_t __se_mem;
219 unsigned int __se_id;
220 kstat_t *__se_ksp;
221 kstat_named_t *__se_stat;
222 efx_ev_callbacks_t __se_eec;
223 sfxge_evq_state_t __se_state;
224 boolean_t __se_exception;
225 } __se_s1;
226 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
227 } __se_u1;
228 union {
229 struct {
230 kmutex_t __se_lock;
231 kcondvar_t __se_init_kv;
232 efx_evq_t *__se_eep;
233 unsigned int __se_count;
234 unsigned int __se_rx;
235 unsigned int __se_tx;
236 sfxge_txq_t *__se_stp;
237 sfxge_txq_t **__se_stpp;
238 processorid_t __se_cpu_id;
239 uint16_t __se_ev_batch;
240 } __se_s2;
241 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE];
242 } __se_u2;
243 union {
244 struct {
245 sfxge_txq_t *__se_label_stp[SFXGE_TX_NLABELS];
246 } __se_s3;
247 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
248 } __se_u3;
249 } sfxge_evq_t;
250
251 #define se_sp __se_u1.__se_s1.__se_sp
252 #define se_index __se_u1.__se_s1.__se_index
253 #define se_mem __se_u1.__se_s1.__se_mem
254 #define se_id __se_u1.__se_s1.__se_id
255 #define se_ksp __se_u1.__se_s1.__se_ksp
256 #define se_stat __se_u1.__se_s1.__se_stat
257 #define se_eec __se_u1.__se_s1.__se_eec
258 #define se_state __se_u1.__se_s1.__se_state
259 #define se_exception __se_u1.__se_s1.__se_exception
260
261 #define se_lock __se_u2.__se_s2.__se_lock
262 #define se_init_kv __se_u2.__se_s2.__se_init_kv
263 #define se_eep __se_u2.__se_s2.__se_eep
264 #define se_count __se_u2.__se_s2.__se_count
265 #define se_rx __se_u2.__se_s2.__se_rx
266 #define se_tx __se_u2.__se_s2.__se_tx
267 #define se_stp __se_u2.__se_s2.__se_stp
268 #define se_stpp __se_u2.__se_s2.__se_stpp
269 #define se_cpu_id __se_u2.__se_s2.__se_cpu_id
270 #define se_ev_batch __se_u2.__se_s2.__se_ev_batch
271
272 #define se_label_stp __se_u3.__se_s3.__se_label_stp
273
274
275 #define SFXGE_MAGIC_RESERVED 0x8000
276
277 #define SFXGE_MAGIC_DMAQ_LABEL_WIDTH 5
278 #define SFXGE_MAGIC_DMAQ_LABEL_MASK ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)
279
280 #define SFXGE_MAGIC_RX_QFLUSH_DONE \
281 (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
282
283 #define SFXGE_MAGIC_RX_QFLUSH_FAILED \
284 (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
285
286 #define SFXGE_MAGIC_RX_QFPP_TRIM \
287 (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
288
289 #define SFXGE_MAGIC_TX_QFLUSH_DONE \
290 (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
291
292 typedef struct sfxge_rxq_s sfxge_rxq_t;
293
294 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */
295
296 typedef struct sfxge_rx_packet_s sfxge_rx_packet_t;
297
298 struct sfxge_rx_packet_s {
299 union {
300 struct {
301 frtn_t __srp_free;
302 uint16_t __srp_flags;
303 uint16_t __srp_size;
304 mblk_t *__srp_mp;
305 struct ether_header *__srp_etherhp;
306 struct ip *__srp_iphp;
307 struct tcphdr *__srp_thp;
308 size_t __srp_off;
309 } __srp_s1;
310 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE];
311 } __srp_u1;
312 union {
313 struct {
314 sfxge_rxq_t *__srp_srp;
315 ddi_dma_handle_t __srp_dma_handle;
316 ddi_acc_handle_t __srp_acc_handle;
317 unsigned char *__srp_base;
318 size_t __srp_mblksize;
319 uint64_t __srp_addr;
320 boolean_t __srp_recycle;
321 caddr_t __srp_putp;
322 } __srp_s2;
323 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
324 } __srp_u2;
325 };
326
327 #define srp_free __srp_u1.__srp_s1.__srp_free
328 #define srp_flags __srp_u1.__srp_s1.__srp_flags
329 #define srp_size __srp_u1.__srp_s1.__srp_size
330 #define srp_mp __srp_u1.__srp_s1.__srp_mp
331 #define srp_etherhp __srp_u1.__srp_s1.__srp_etherhp
332 #define srp_iphp __srp_u1.__srp_s1.__srp_iphp
333 #define srp_thp __srp_u1.__srp_s1.__srp_thp
334 #define srp_off __srp_u1.__srp_s1.__srp_off
335
336 #define srp_srp __srp_u2.__srp_s2.__srp_srp
337 #define srp_dma_handle __srp_u2.__srp_s2.__srp_dma_handle
338 #define srp_acc_handle __srp_u2.__srp_s2.__srp_acc_handle
339 #define srp_base __srp_u2.__srp_s2.__srp_base
340 #define srp_mblksize __srp_u2.__srp_s2.__srp_mblksize
341 #define srp_addr __srp_u2.__srp_s2.__srp_addr
342 #define srp_recycle __srp_u2.__srp_s2.__srp_recycle
343 #define srp_putp __srp_u2.__srp_s2.__srp_putp
344
345 #define SFXGE_RX_FPP_NSLOTS 8
346 #define SFXGE_RX_FPP_MASK (SFXGE_RX_FPP_NSLOTS - 1)
347
348 /* Free packet pool putlist (dynamically allocated) */
349 typedef struct sfxge_rx_fpp_putlist_s {
350 kmutex_t srfpl_lock;
351 unsigned int srfpl_count;
352 mblk_t *srfpl_putp;
353 mblk_t **srfpl_putpp;
354 } sfxge_rx_fpp_putlist_t;
355
356 /* Free packet pool */
357 typedef struct sfxge_rx_fpp_s {
358 caddr_t srfpp_putp;
359 unsigned int srfpp_loaned;
360 mblk_t *srfpp_get;
361 unsigned int srfpp_count;
362 unsigned int srfpp_min;
363 /* Low water mark: Don't trim to below this */
364 unsigned int srfpp_lowat;
365 } sfxge_rx_fpp_t;
366
367 typedef struct sfxge_rx_flow_s sfxge_rx_flow_t;
368
369 struct sfxge_rx_flow_s {
370 uint32_t srf_tag;
371 /* in-order segment count */
372 unsigned int srf_count;
373 uint16_t srf_tci;
374 uint32_t srf_saddr;
375 uint32_t srf_daddr;
376 uint16_t srf_sport;
377 uint16_t srf_dport;
378 /* sequence number */
379 uint32_t srf_seq;
380 clock_t srf_lbolt;
381 mblk_t *srf_mp;
382 mblk_t **srf_mpp;
383 struct ether_header *srf_etherhp;
384 struct ip *srf_iphp;
385 struct tcphdr *srf_first_thp;
386 struct tcphdr *srf_last_thp;
387 size_t srf_len;
388 sfxge_rx_flow_t *srf_next;
389 };
390
391 #define SFXGE_MAX_FLOW 1024
392 #define SFXGE_SLOW_START 20
393
394 typedef enum sfxge_flush_state_e {
395 SFXGE_FLUSH_INACTIVE = 0,
396 SFXGE_FLUSH_DONE,
397 SFXGE_FLUSH_PENDING,
398 SFXGE_FLUSH_FAILED
399 } sfxge_flush_state_t;
400
401 typedef enum sfxge_rxq_state_e {
402 SFXGE_RXQ_UNINITIALIZED = 0,
403 SFXGE_RXQ_INITIALIZED,
404 SFXGE_RXQ_STARTED
405 } sfxge_rxq_state_t;
406
407
408 #define SFXGE_RX_BATCH 128
409 #define SFXGE_RX_NSTATS 8 /* note that *esballoc share one kstat */
410
411 struct sfxge_rxq_s {
412 union {
413 struct {
414 sfxge_t *__sr_sp;
415 unsigned int __sr_index;
416 efsys_mem_t __sr_mem;
417 unsigned int __sr_id;
418 unsigned int __sr_lowat;
419 unsigned int __sr_hiwat;
420 volatile timeout_id_t __sr_tid;
421 sfxge_rxq_state_t __sr_state;
422 } __sr_s1;
423 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
424 } __sr_u1;
425 union {
426 struct {
427 sfxge_rx_packet_t **__sr_srpp;
428 unsigned int __sr_added;
429 unsigned int __sr_pushed;
430 unsigned int __sr_pending;
431 unsigned int __sr_completed;
432 unsigned int __sr_loopback;
433 mblk_t *__sr_mp;
434 mblk_t **__sr_mpp;
435 sfxge_rx_flow_t *__sr_flow;
436 sfxge_rx_flow_t *__sr_srfp;
437 sfxge_rx_flow_t **__sr_srfpp;
438 clock_t __sr_rto;
439 } __sr_s2;
440 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
441 } __sr_u2;
442 union {
443 struct {
444 sfxge_rx_fpp_t __sr_fpp;
445 efx_rxq_t *__sr_erp;
446 volatile sfxge_flush_state_t __sr_flush;
447 kcondvar_t __sr_flush_kv;
448 kstat_t *__sr_ksp;
449 } __sr_s3;
450 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE];
451 } __sr_u3;
452 struct {
453 /* NB must match SFXGE_RX_NSTATS */
454 uint32_t srk_rx_pkt_mem_limit;
455 uint32_t srk_kcache_alloc_nomem;
456 uint32_t srk_dma_alloc_nomem;
457 uint32_t srk_dma_alloc_fail;
458 uint32_t srk_dma_bind_nomem;
459 uint32_t srk_dma_bind_fail;
460 uint32_t srk_desballoc_fail;
461 uint32_t srk_rxq_empty_discard;
462 } sr_kstat;
463 };
464
465 #define sr_sp __sr_u1.__sr_s1.__sr_sp
466 #define sr_index __sr_u1.__sr_s1.__sr_index
467 #define sr_mem __sr_u1.__sr_s1.__sr_mem
468 #define sr_id __sr_u1.__sr_s1.__sr_id
469 #define sr_mrh __sr_u1.__sr_s1.__sr_mrh
470 #define sr_lowat __sr_u1.__sr_s1.__sr_lowat
471 #define sr_hiwat __sr_u1.__sr_s1.__sr_hiwat
472 #define sr_tid __sr_u1.__sr_s1.__sr_tid
473 #define sr_state __sr_u1.__sr_s1.__sr_state
474
475 #define sr_srpp __sr_u2.__sr_s2.__sr_srpp
476 #define sr_added __sr_u2.__sr_s2.__sr_added
477 #define sr_pushed __sr_u2.__sr_s2.__sr_pushed
478 #define sr_pending __sr_u2.__sr_s2.__sr_pending
479 #define sr_completed __sr_u2.__sr_s2.__sr_completed
480 #define sr_loopback __sr_u2.__sr_s2.__sr_loopback
481 #define sr_mp __sr_u2.__sr_s2.__sr_mp
482 #define sr_mpp __sr_u2.__sr_s2.__sr_mpp
483 #define sr_flow __sr_u2.__sr_s2.__sr_flow
484 #define sr_srfp __sr_u2.__sr_s2.__sr_srfp
485 #define sr_srfpp __sr_u2.__sr_s2.__sr_srfpp
486 #define sr_rto __sr_u2.__sr_s2.__sr_rto
487
488 #define sr_fpp __sr_u3.__sr_s3.__sr_fpp
489 #define sr_erp __sr_u3.__sr_s3.__sr_erp
490 #define sr_flush __sr_u3.__sr_s3.__sr_flush
491 #define sr_flush_kv __sr_u3.__sr_s3.__sr_flush_kv
492 #define sr_ksp __sr_u3.__sr_s3.__sr_ksp
493
494 typedef struct sfxge_tx_packet_s sfxge_tx_packet_t;
495
496 /* Packet type from parsing transmit packet */
497 typedef enum sfxge_packet_type_e {
498 SFXGE_PACKET_TYPE_UNKNOWN = 0,
499 SFXGE_PACKET_TYPE_IPV4_TCP,
500 SFXGE_PACKET_TYPE_IPV4_UDP,
501 SFXGE_PACKET_TYPE_IPV4_SCTP,
502 SFXGE_PACKET_TYPE_IPV4_OTHER,
503 SFXGE_PACKET_NTYPES
504 } sfxge_packet_type_t;
505
506 struct sfxge_tx_packet_s {
507 sfxge_tx_packet_t *stp_next;
508 mblk_t *stp_mp;
509 struct ether_header *stp_etherhp;
510 struct ip *stp_iphp;
511 struct tcphdr *stp_thp;
512 size_t stp_off;
513 size_t stp_size;
514 size_t stp_mss;
515 uint32_t stp_dpl_put_len;
516 };
517
518 #define SFXGE_TX_FPP_MAX 64
519
520 typedef struct sfxge_tx_fpp_s {
521 sfxge_tx_packet_t *stf_stpp;
522 unsigned int stf_count;
523 } sfxge_tx_fpp_t;
524
525 typedef struct sfxge_tx_mapping_s sfxge_tx_mapping_t;
526
527 #define SFXGE_TX_MAPPING_NADDR (((1 << 16) >> 12) + 2)
528
529 struct sfxge_tx_mapping_s {
530 sfxge_tx_mapping_t *stm_next;
531 sfxge_t *stm_sp;
532 mblk_t *stm_mp;
533 ddi_dma_handle_t stm_dma_handle;
534 caddr_t stm_base;
535 size_t stm_size;
536 size_t stm_off;
537 uint64_t stm_addr[SFXGE_TX_MAPPING_NADDR];
538 };
539
540 typedef struct sfxge_tx_fmp_s {
541 sfxge_tx_mapping_t *stf_stmp;
542 unsigned int stf_count;
543 } sfxge_tx_fmp_t;
544
545 typedef struct sfxge_tx_buffer_s sfxge_tx_buffer_t;
546
547 struct sfxge_tx_buffer_s {
548 sfxge_tx_buffer_t *stb_next;
549 size_t stb_off;
550 efsys_mem_t stb_esm;
551 };
552
553 #define SFXGE_TX_BUFFER_SIZE 0x400
554 #define SFXGE_TX_HEADER_SIZE 0x100
555 #define SFXGE_TX_COPY_THRESHOLD 0x200
556
557 typedef struct sfxge_tx_fbp_s {
558 sfxge_tx_buffer_t *stf_stbp;
559 unsigned int stf_count;
560 } sfxge_tx_fbp_t;
561
562 typedef struct sfxge_tx_dpl_s {
563 uintptr_t std_put;
564 sfxge_tx_packet_t *std_get;
565 sfxge_tx_packet_t **std_getp;
566 unsigned int std_count; /* only get list count */
567 unsigned int get_pkt_limit;
568 unsigned int put_pkt_limit;
569 unsigned int get_full_count;
570 unsigned int put_full_count;
571 } sfxge_tx_dpl_t;
572
573 typedef enum sfxge_txq_state_e {
574 SFXGE_TXQ_UNINITIALIZED = 0,
575 SFXGE_TXQ_INITIALIZED,
576 SFXGE_TXQ_STARTED,
577 SFXGE_TXQ_FLUSH_PENDING,
578 SFXGE_TXQ_FLUSH_DONE,
579 SFXGE_TXQ_FLUSH_FAILED
580 } sfxge_txq_state_t;
581
582 typedef enum sfxge_txq_type_e {
583 SFXGE_TXQ_NON_CKSUM = 0,
584 SFXGE_TXQ_IP_CKSUM,
585 SFXGE_TXQ_IP_TCP_UDP_CKSUM,
586 SFXGE_TXQ_NTYPES
587 } sfxge_txq_type_t;
588
589 #define SFXGE_TXQ_UNBLOCK_LEVEL1 (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4)
590 #define SFXGE_TXQ_UNBLOCK_LEVEL2 0
591 #define SFXGE_TXQ_NOT_BLOCKED -1
592
593 #define SFXGE_TX_BATCH 64
594
595 struct sfxge_txq_s {
596 union {
597 struct {
598 sfxge_t *__st_sp;
599 unsigned int __st_index;
600 unsigned int __st_label;
601 sfxge_txq_type_t __st_type;
602 unsigned int __st_evq;
603 efsys_mem_t __st_mem;
604 unsigned int __st_id;
605 kstat_t *__st_ksp;
606 kstat_named_t *__st_stat;
607 sfxge_txq_state_t __st_state;
608 } __st_s1;
609 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 2];
610 } __st_u1;
611 union {
612 struct {
613 sfxge_tx_dpl_t __st_dpl;
614 } __st_s2;
615 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
616 } __st_u2;
617 union {
618 struct {
619 kmutex_t __st_lock;
620 /* mapping pool - sfxge_tx_mapping_t */
621 sfxge_tx_fmp_t __st_fmp;
622 /* buffer pool - sfxge_tx_buffer_t */
623 sfxge_tx_fbp_t __st_fbp;
624 /* packet pool - sfxge_tx_packet_t */
625 sfxge_tx_fpp_t __st_fpp;
626 efx_buffer_t *__st_eb;
627 unsigned int __st_n;
628 efx_txq_t *__st_etp;
629 sfxge_tx_mapping_t **__st_stmp;
630 sfxge_tx_buffer_t **__st_stbp;
631 mblk_t **__st_mp;
632 unsigned int __st_added;
633 unsigned int __st_reaped;
634 int __st_unblock;
635 } __st_s3;
636 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 3];
637 } __st_u3;
638 union {
639 struct {
640 sfxge_txq_t *__st_next;
641 unsigned int __st_pending;
642 unsigned int __st_completed;
643
644 } __st_s4;
645 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
646 } __st_u4;
647 };
648
649 #define st_sp __st_u1.__st_s1.__st_sp
650 #define st_index __st_u1.__st_s1.__st_index
651 #define st_label __st_u1.__st_s1.__st_label
652 #define st_type __st_u1.__st_s1.__st_type
653 #define st_evq __st_u1.__st_s1.__st_evq
654 #define st_mem __st_u1.__st_s1.__st_mem
655 #define st_id __st_u1.__st_s1.__st_id
656 #define st_ksp __st_u1.__st_s1.__st_ksp
657 #define st_stat __st_u1.__st_s1.__st_stat
658 #define st_state __st_u1.__st_s1.__st_state
659
660 #define st_dpl __st_u2.__st_s2.__st_dpl
661
662 #define st_lock __st_u3.__st_s3.__st_lock
663 #define st_fmp __st_u3.__st_s3.__st_fmp
664 #define st_fbp __st_u3.__st_s3.__st_fbp
665 #define st_fpp __st_u3.__st_s3.__st_fpp
666 #define st_eb __st_u3.__st_s3.__st_eb
667 #define st_n __st_u3.__st_s3.__st_n
668 #define st_etp __st_u3.__st_s3.__st_etp
669 #define st_stmp __st_u3.__st_s3.__st_stmp
670 #define st_stbp __st_u3.__st_s3.__st_stbp
671 #define st_mp __st_u3.__st_s3.__st_mp
672 #define st_added __st_u3.__st_s3.__st_added
673 #define st_reaped __st_u3.__st_s3.__st_reaped
674 #define st_unblock __st_u3.__st_s3.__st_unblock
675
676 #define st_next __st_u4.__st_s4.__st_next
677 #define st_pending __st_u4.__st_s4.__st_pending
678 #define st_completed __st_u4.__st_s4.__st_completed
679
680 typedef enum sfxge_rx_scale_state_e {
681 SFXGE_RX_SCALE_UNINITIALIZED = 0,
682 SFXGE_RX_SCALE_INITIALIZED,
683 SFXGE_RX_SCALE_STARTED
684 } sfxge_rx_scale_state_t;
685
686 #define SFXGE_RX_SCALE_MAX EFX_RSS_TBL_SIZE
687
688 typedef struct sfxge_rx_scale_s {
689 kmutex_t srs_lock;
690 unsigned int *srs_cpu;
691 unsigned int srs_tbl[SFXGE_RX_SCALE_MAX];
692 unsigned int srs_count;
693 kstat_t *srs_ksp;
694 sfxge_rx_scale_state_t srs_state;
695 } sfxge_rx_scale_t;
696
697
698 typedef enum sfxge_rx_coalesce_mode_e {
699 SFXGE_RX_COALESCE_OFF = 0,
700 SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
701 SFXGE_RX_COALESCE_ALLOW_PUSH = 2
702 } sfxge_rx_coalesce_mode_t;
703
704 typedef enum sfxge_vpd_type_e {
705 SFXGE_VPD_ID = 0,
706 SFXGE_VPD_PN = 1,
707 SFXGE_VPD_SN = 2,
708 SFXGE_VPD_EC = 3,
709 SFXGE_VPD_MN = 4,
710 SFXGE_VPD_VD = 5,
711 SFXGE_VPD_VE = 6,
712 SFXGE_VPD_MAX = 7,
713 } sfxge_vpd_type_t;
714
715 typedef struct sfxge_vpd_kstat_s {
716 kstat_t *svk_ksp;
717 kstat_named_t svk_stat[SFXGE_VPD_MAX];
718 efx_vpd_value_t *svk_vv;
719 } sfxge_vpd_kstat_t;
720
721 typedef struct sfxge_cfg_kstat_s {
722 struct {
723 kstat_named_t sck_mac;
724 kstat_named_t sck_version;
725 } kstat;
726 struct {
727 char sck_mac[64 + 1];
728 } buf;
729 } sfxge_cfg_kstat_t;
730
731 typedef enum sfxge_state_e {
732 SFXGE_UNINITIALIZED = 0,
733 SFXGE_INITIALIZED,
734 SFXGE_REGISTERED,
735 SFXGE_STARTING,
736 SFXGE_STARTED,
737 SFXGE_STOPPING
738 } sfxge_state_t;
739
740 typedef enum sfxge_hw_err_e {
741 SFXGE_HW_OK = 0,
742 SFXGE_HW_ERR,
743 } sfxge_hw_err_t;
744
745 typedef enum sfxge_action_on_hw_err_e {
746 SFXGE_RECOVER = 0,
747 SFXGE_INVISIBLE = 1,
748 SFXGE_LEAVE_DEAD = 2,
749 } sfxge_action_on_hw_err_t;
750
751 typedef char *sfxge_mac_priv_prop_t;
752
753 #define SFXGE_TOEPLITZ_KEY_LEN 40
754
755 struct sfxge_s {
756 kmutex_t s_state_lock;
757 sfxge_state_t s_state;
758 dev_info_t *s_dip;
759 ddi_taskq_t *s_tqp;
760 ddi_acc_handle_t s_pci_handle;
761 uint16_t s_pci_venid;
762 uint16_t s_pci_devid;
763 #if EFSYS_OPT_MCDI_LOGGING
764 unsigned int s_bus_addr;
765 #endif
766 efx_family_t s_family;
767 unsigned int s_pcie_nlanes;
768 unsigned int s_pcie_linkspeed;
769 kmutex_t s_nic_lock;
770 efsys_bar_t s_bar;
771 sfxge_intr_t s_intr;
772 sfxge_mac_t s_mac;
773 sfxge_mon_t s_mon;
774 sfxge_sram_t s_sram;
775 sfxge_mcdi_t s_mcdi;
776 kmem_cache_t *s_eq0c; /* eventQ 0 */
777 kmem_cache_t *s_eqXc; /* all other eventQs */
778 sfxge_evq_t *s_sep[SFXGE_RX_SCALE_MAX];
779 unsigned int s_ev_moderation;
780 kmem_cache_t *s_rqc;
781 sfxge_rxq_t *s_srp[SFXGE_RX_SCALE_MAX];
782 sfxge_rx_scale_t s_rx_scale;
783 size_t s_rx_prefix_size;
784 size_t s_rx_buffer_size;
785 size_t s_rx_buffer_align;
786 sfxge_rx_coalesce_mode_t s_rx_coalesce_mode;
787 int64_t s_rx_pkt_mem_max;
788 volatile uint64_t s_rx_pkt_mem_alloc;
789 kmem_cache_t *s_rpc;
790 kmem_cache_t *s_tqc;
791 unsigned int s_tx_scale_base[SFXGE_TXQ_NTYPES];
792 unsigned int s_tx_scale_max[SFXGE_TXQ_NTYPES];
793 int s_tx_qcount;
794 sfxge_txq_t *s_stp[SFXGE_RX_SCALE_MAX *
795 SFXGE_TXQ_NTYPES]; /* Sparse array */
796 kmem_cache_t *s_tpc;
797 int s_tx_flush_pending;
798 kmutex_t s_tx_flush_lock;
799 kcondvar_t s_tx_flush_kv;
800 kmem_cache_t *s_tbc;
801 kmem_cache_t *s_tmc;
802 efx_nic_t *s_enp;
803 sfxge_vpd_kstat_t s_vpd_kstat;
804 sfxge_cfg_kstat_t s_cfg_kstat;
805 kstat_t *s_cfg_ksp;
806 size_t s_mtu;
807 int s_rxq_poll_usec;
808 mac_callbacks_t s_mc;
809 mac_handle_t s_mh;
810 sfxge_mac_priv_prop_t *s_mac_priv_props;
811 int s_mac_priv_props_alloc;
812 volatile uint32_t s_nested_restarts;
813 uint32_t s_num_restarts;
814 uint32_t s_num_restarts_hw_err;
815 sfxge_hw_err_t s_hw_err;
816 sfxge_action_on_hw_err_t s_action_on_hw_err;
817 uint16_t s_rxq_size;
818 uint16_t s_evq0_size;
819 uint16_t s_evqX_size;
820 #if EFSYS_OPT_MCDI_LOGGING
821 int s_mcdi_logging;
822 #endif
823 const uint32_t *s_toeplitz_cache;
824 };
825
826 typedef struct sfxge_dma_buffer_attr_s {
827 dev_info_t *sdba_dip;
828 ddi_dma_attr_t *sdba_dattrp;
829 int (*sdba_callback) (caddr_t);
830 size_t sdba_length;
831 uint_t sdba_memflags;
832 ddi_device_acc_attr_t *sdba_devaccp;
833 uint_t sdba_bindflags;
834 int sdba_maxcookies;
835 boolean_t sdba_zeroinit;
836 } sfxge_dma_buffer_attr_t;
837
838 extern const char sfxge_ident[];
839 extern uint8_t sfxge_brdcst[];
840
841 extern kmutex_t sfxge_global_lock;
842
843 extern unsigned int *sfxge_cpu;
844
845 extern int sfxge_start(sfxge_t *, boolean_t);
846 extern void sfxge_stop(sfxge_t *);
847 extern void sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
848 extern int sfxge_restart_dispatch(sfxge_t *, uint_t,
849 sfxge_hw_err_t, const char *, uint32_t);
850
851 extern void sfxge_gld_link_update(sfxge_t *);
852 extern void sfxge_gld_mtu_update(sfxge_t *);
853 extern void sfxge_gld_rx_post(sfxge_t *, unsigned int,
854 mblk_t *);
855 extern void sfxge_gld_rx_push(sfxge_t *);
856 extern int sfxge_gld_register(sfxge_t *);
857 extern int sfxge_gld_unregister(sfxge_t *);
858
859 extern int sfxge_dma_buffer_create(efsys_mem_t *,
860 const sfxge_dma_buffer_attr_t *);
861 extern void sfxge_dma_buffer_destroy(efsys_mem_t *);
862
863 extern int sfxge_intr_init(sfxge_t *);
864 extern int sfxge_intr_start(sfxge_t *);
865 extern void sfxge_intr_stop(sfxge_t *);
866 extern void sfxge_intr_fini(sfxge_t *);
867 extern void sfxge_intr_fatal(sfxge_t *);
868
869 extern int sfxge_ev_init(sfxge_t *);
870 extern int sfxge_ev_start(sfxge_t *);
871 extern void sfxge_ev_moderation_get(sfxge_t *,
872 unsigned int *);
873 extern int sfxge_ev_moderation_set(sfxge_t *,
874 unsigned int);
875 extern int sfxge_ev_qmoderate(sfxge_t *, unsigned int,
876 unsigned int);
877 extern int sfxge_ev_qpoll(sfxge_t *, unsigned int);
878 extern int sfxge_ev_qprime(sfxge_t *, unsigned int);
879 extern void sfxge_ev_stop(sfxge_t *);
880 extern void sfxge_ev_fini(sfxge_t *);
881 extern int sfxge_ev_txlabel_alloc(sfxge_t *sp,
882 unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp);
883 extern int sfxge_ev_txlabel_free(sfxge_t *sp,
884 unsigned int evq, sfxge_txq_t *stp, unsigned int label);
885
886 extern int sfxge_mon_init(sfxge_t *);
887 extern int sfxge_mon_start(sfxge_t *);
888 extern void sfxge_mon_stop(sfxge_t *);
889 extern void sfxge_mon_fini(sfxge_t *);
890
891 extern int sfxge_mac_init(sfxge_t *);
892 extern int sfxge_mac_start(sfxge_t *, boolean_t);
893 extern void sfxge_mac_stat_get(sfxge_t *, unsigned int,
894 uint64_t *);
895 extern void sfxge_mac_link_check(sfxge_t *, boolean_t *);
896 extern void sfxge_mac_link_speed_get(sfxge_t *,
897 unsigned int *);
898 extern void sfxge_mac_link_duplex_get(sfxge_t *,
899 sfxge_link_duplex_t *);
900 extern void sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
901 extern int sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
902 extern int sfxge_mac_unicst_get(sfxge_t *,
903 sfxge_unicst_type_t, uint8_t *);
904 extern int sfxge_mac_unicst_set(sfxge_t *,
905 uint8_t *);
906 extern int sfxge_mac_promisc_set(sfxge_t *,
907 sfxge_promisc_type_t);
908 extern int sfxge_mac_multicst_add(sfxge_t *,
909 uint8_t const *addr);
910 extern int sfxge_mac_multicst_remove(sfxge_t *,
911 uint8_t const *addr);
912 extern void sfxge_mac_stop(sfxge_t *);
913 extern void sfxge_mac_fini(sfxge_t *);
914 extern void sfxge_mac_link_update(sfxge_t *sp,
915 efx_link_mode_t mode);
916
917 extern int sfxge_mcdi_init(sfxge_t *sp);
918 extern void sfxge_mcdi_fini(sfxge_t *sp);
919 extern int sfxge_mcdi_ioctl(sfxge_t *sp,
920 sfxge_mcdi_ioc_t *smip);
921 extern int sfxge_mcdi2_ioctl(sfxge_t *sp,
922 sfxge_mcdi2_ioc_t *smip);
923
924 extern int sfxge_phy_init(sfxge_t *);
925 extern void sfxge_phy_link_mode_get(sfxge_t *,
926 efx_link_mode_t *);
927 extern void sfxge_phy_fini(sfxge_t *);
928 extern int sfxge_phy_kstat_init(sfxge_t *sp);
929 extern void sfxge_phy_kstat_fini(sfxge_t *sp);
930 extern uint8_t sfxge_phy_lp_cap_test(sfxge_t *sp,
931 uint32_t field);
932 extern int sfxge_phy_cap_apply(sfxge_t *sp,
933 boolean_t use_default);
934 extern uint8_t sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
935 uint32_t field, boolean_t *mutablep);
936 extern int sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
937 int set);
938
939 extern int sfxge_rx_init(sfxge_t *);
940 extern int sfxge_rx_start(sfxge_t *);
941 extern void sfxge_rx_coalesce_mode_get(sfxge_t *,
942 sfxge_rx_coalesce_mode_t *);
943 extern int sfxge_rx_coalesce_mode_set(sfxge_t *,
944 sfxge_rx_coalesce_mode_t);
945 extern unsigned int sfxge_rx_scale_prop_get(sfxge_t *);
946 extern void sfxge_rx_scale_update(void *);
947 extern int sfxge_rx_scale_count_get(sfxge_t *,
948 unsigned int *);
949 extern int sfxge_rx_scale_count_set(sfxge_t *,
950 unsigned int);
951 extern void sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
952 extern void sfxge_rx_qflush_done(sfxge_rxq_t *);
953 extern void sfxge_rx_qflush_failed(sfxge_rxq_t *);
954 extern void sfxge_rx_qfpp_trim(sfxge_rxq_t *);
955 extern void sfxge_rx_stop(sfxge_t *);
956 extern unsigned int sfxge_rx_loaned(sfxge_t *);
957 extern void sfxge_rx_fini(sfxge_t *);
958
959 extern int sfxge_tx_init(sfxge_t *);
960 extern int sfxge_tx_start(sfxge_t *);
961 extern int sfxge_tx_packet_add(sfxge_t *, mblk_t *);
962 extern void sfxge_tx_qcomplete(sfxge_txq_t *);
963 extern void sfxge_tx_qflush_done(sfxge_txq_t *);
964 extern void sfxge_tx_stop(sfxge_t *);
965 extern void sfxge_tx_fini(sfxge_t *);
966 extern void sfxge_tx_qdpl_flush(sfxge_txq_t *stp);
967
968 extern void sfxge_sram_init(sfxge_t *);
969 extern int sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
970 uint32_t *);
971 extern int sfxge_sram_start(sfxge_t *);
972 extern int sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
973 efsys_mem_t *, size_t);
974 extern void sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
975 size_t);
976 extern void sfxge_sram_stop(sfxge_t *);
977 extern void sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
978 size_t);
979 extern void sfxge_sram_fini(sfxge_t *);
980
981 extern sfxge_packet_type_t sfxge_pkthdr_parse(mblk_t *,
982 struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *,
983 uint16_t *, uint16_t *);
984
985 extern int sfxge_toeplitz_hash_init(sfxge_t *);
986 extern void sfxge_toeplitz_hash_fini(sfxge_t *);
987 extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int,
988 uint8_t *, uint16_t, uint8_t *, uint16_t);
989
990 /*
991 * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection.
992 * To compute the same hash value as Siena/Huntington hardware, the inputs
993 * must be in big endian (network) byte order.
994 */
995 #define SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash) \
996 do { \
997 (_hash) = sfxge_toeplitz_hash(_sp, \
998 sizeof (struct in_addr), \
999 (uint8_t *)(_raddr), \
1000 (_rport), \
1001 (uint8_t *)(_laddr), \
1002 (_lport)); \
1003 _NOTE(CONSTANTCONDITION) \
1004 } while (B_FALSE)
1005
1006 /*
1007 * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection.
1008 * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers.
1009 * For other IPv4 non-TCP packets, use zero for the port numbers.
1010 */
1011 #define SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash) \
1012 SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash))
1013
1014
1015 extern int sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);
1016
1017 extern int sfxge_pci_init(sfxge_t *);
1018 extern void sfxge_pcie_check_link(sfxge_t *, unsigned int,
1019 unsigned int);
1020 extern void sfxge_pci_fini(sfxge_t *);
1021
1022 extern int sfxge_bar_init(sfxge_t *);
1023 extern void sfxge_bar_fini(sfxge_t *);
1024
1025 extern int sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);
1026
1027
1028 #endif /* _KERNEL */
1029
1030 #ifdef __cplusplus
1031 }
1032 #endif
1033
1034 #endif /* _SYS_SFXGE_H */