#### This file contains configuration parameters for the Solarflare
#### network adapter driver "sfxge"

################################################################################
#### HOW TO MAKE CHANGES TO THIS FILE TAKE EFFECT
################################################################################
#### In order for changes made to this file to be used by the driver either:
#### i) reboot
#### OR
#### ii) Unplumb all sfxge inet,inet6 interfaces or link aggregations: e.g.
####       $ ifconfig sfxge0 unplumb
####       $ ifconfig sfxge0 inet6 unplumb
####     Delete any link aggregations that setup (see dladm show-aggr):
####       $ dladm delete-aggr <key>
####     Update the driver config
####       $ update_drv sfxge
####     Note update_drv will unload then reload the driver, otherwise it warns:
####       "Cannot unload module: sfxge"
####     Then plumb interfaces and reconfigure IP addresses as required
#### 
################################################################################

################################################################################
#### PER PORT SETTINGS
################################################################################
#### Note that is possible to have different settings per network port
#### First using the file "/etc/path_to_inst" you can identify some information
####
####  $ grep sfxge /etc/path_to_inst
####   # "/pci@3a,0/pci8086,340c@5/pci1924,6201@0"   4 "sfxge"
####   # "/pci@3a,0/pci8086,340c@5/pci1924,6201@0,1" 5 "sfxge"
####   #  ^^^^^^^^^^^^^^^^^^^^^^^^             ^^^
####   #         parent                    unit-address
####
#### Using the information gathered above to create entries in this file
#### based on the example below; this tries to allocate 4 MSI-X interrupts
#### to port0 and 8 MSI-X interrupts to port 1
####
####  name="sfxge" parent="/pci@3a,0/pci8086,340c@5" unit-address="0"
####               rx_scale_count=4;
####  name="sfxge" parent="/pci@3a,0/pci8086,340c@5" unit-address="0,1"
####               rx_scale_count=8;
################################################################################

################################################################################
#### RECEIVE SIDE SCALING
################################################################################
#### Maximum number of RSS-channels to use per port (default 0, global max 32)
#### This spreads RX flows across CPUs using multiple MSI-X interrupts
####
#### - A value of 0 tries to allocate the same number of RSS-channels as logical
####   CPUs in order to provide the best spreading accross the entire system.
####
#### - actual number may be lower due to availability of MSI-X interrupts
####
#### - to successfully use >2 MSI-X interrupts per port you also need to use
####  "set ddi_msix_alloc_limit=8" (maximum) in /etc/system and reboot
####   OR temporarily adjust this limit using mdb and then reload the driver
####     # mdb -kw
####    > ddi_msix_alloc_limit/W 8
####    ddi_msix_alloc_limit:           0x2            =       0x8
####    > ddi_msix_alloc_limit/X
####    ddi_msix_alloc_limit:
####    ddi_msix_alloc_limit:           8
####
#### - Note that the total MSI-X interrupt vector assigned to all network
####     drivers (in IPL 6; interrupt priority level) is 32
####   See the "INTERUPT PRIORITY" section for a way overcome this
####
####  -  You can see the current allocation with this run as root:
####   $ echo "::interrupts -d" | mdb -k | egrep 'IRQ| 6 '
####     IRQ  Vector IPL Bus   Type  CPU Share APIC/INT# Driver Name(s)
####     93   0x60   6         MSI-X 2   1     -         sfxge#0
####     94   0x61   6         MSI-X 3   1     -         sfxge#0
####                ^^^
####
#### - If the sysadmin does not take care to partition the interupts between
####     the network ports then later ports could use a single legacy interrupt
####     i.e. no RSS, significantly reducing performance
####   See PER PORT SETTINGS ABOVE for how to apply rx_scale_count per port
####
#### - rx_scale_count can be manipulated dynamically via ndd but the number
####   of interrupts will stay the same; though only a subset will fire for RX. 
####   All interrupts will still see ~10 interrupts/sec
####
#### - note that the number of RSS-channels can be set to be greater than the
####   number of logical CPUs. This can be useful to prevent some types of RX
####   drops

rx_scale_count=0;

################################################################################
#### USING NON_DEFAULT INTERRUPT PRIORITY LEVEL (IPL)
################################################################################
#### To overcome the Solaris limitation of 32 MSI-X vectors per IPL
#### it is possible to set this driver to allocate interrupts in a different IPL
#### Please only do this if necessary
####
#### See http://download.oracle.com/docs/cd/E20815_01/html/E20816/gjtag.html
#### Verify with:
####   $ echo "::interrupts -d" | mdb -k | egrep 'IRQ|sfxge'
####     IRQ  Vector IPL Bus   Type  CPU Share APIC/INT# Driver Name(s)
####     93   0x60   5         MSI-X 2   1     -         sfxge#0
####     94   0x61   5         MSI-X 3   1     -         sfxge#0
####                ^^^
####
#### (default IPL 6; the document referenced recommends 5 or 4)

#interrupt-priorities=5;

################################################################################
#### RX DMA RING SIZE
################################################################################
#### Number of entries in the per receive queue hardware DMA ring
####
#### Increase this value if you see increasing values of "rx_nodesc_drop_cnt"
#### as reported by "kstat -m sfxge -c mac -s rx_nodesc_drop_cnt" and you think
#### the traffic is bursty
####
#### Increasing this value can increase driver memory usage and reduce the
#### cache-hit rates seen for RX buffers so the largest value is not
#### necessarily the best
####
#### (minimum 512)
#### (default 1024)
#### (maximum 4096)
#### The value must be a power of two else it will be ignored.

#rxq_size=1024;

################################################################################
#### LARGE RECEIVE OFFLOAD
################################################################################
#### Coalesce RX packets (aka Large Receive Offload)
####  0 => off (default)
####  1 => on
####  2 => on, respecting TCP PSH boundaries

#rx_coalesce_mode=1;

################################################################################
#### INTERRUPT MODERATION
################################################################################
#### Interrupt moderation in us (default 30, minimum 0)
#### Decreasing this reduces latency but increases interrupt rate and therefore
#### CPU usage which will decrease maximum bandwidth achievable
####
#### Decrease this value if you see increasing values of "rx_nodesc_drop_cnt"
#### as reported by "kstat -m sfxge -c mac -s rx_nodesc_drop_cnt" so that
#### the driver has more opportunities to refill the hardware RX descriptor
#### ring
####
#### (maximum 20000 us)
#### (default 30 us)

#intr_moderation=10;

################################################################################
#### MAXIMUM MTU
################################################################################
#### Maximum MTU of an sfxge interface (in bytes - excludes ethernet framing)
#### Increase this value to support Jumbo frames
####
#### NB increasing this value consumes more memory for all RX buffers
#### even for network interfaces not configured with high MTUs
####
#### MTUs over 3968 mean that >1 contiguous 4k page of memory are allocated
#### for all receive buffers. Contiguous pages of memory can be harder for
#### the OS to allocate when under memory pressue.
#### 
#### In Solaris 11 the MTU can be dynamically changed so you should not
#### need to alter the setting below. A non-default MTU can be persistently
#### configured depending on your network configuration:
#### 1) For manual mode or using NWAM with the DefaultFixed profile:
####    dladm set-linkprop -p mtu=<MTU> sfxge<n>
#### 2) If using NWAM using a custom profile please set the property link-mtu:
####    netcfg "select ncp myprofile; select ncu phys sfxge0; set link-mtu=9000"
####
#### (maximum 9000)
#### (default 1500)

#mtu=1500;

################################################################################
#### PER PORT MEMORY LIMIT FOR RX PACKET BUFFERS
################################################################################
#### Limit max memory for RX packets allocated per instance (port) in bytes
####
#### RX packets are passed up from the driver to the kernel TCP/IP stack
#### and freed after the data is delivered to the application socket
#### buffers. If the OS falls behind this allocation can grow.
####
#### This parameter can help make behaviour reasonable when approaching
#### an overload condition. Other ways to avoid overload would be RSS
####
#### The size of each bufer, number in use can be seen with
####    echo "::kmastat" | mdb -k
#### OR kstat -c kmem_cache -n sfxgeX_rx_packet_cache # replace X
####
#### (default unlimited when unset)
####
#### Note In Solarflare driver package versions 3.1.x.xxxx and earlier set
#### the RX buffering limit to 10773741824 (1GB) per interface in this file.
####
#### 64MB of RX buffering per interface is still oversized for nearly all
#### applications but provides a hard limit.

rx_pkt_mem_max=67108864;

################################################################################
#### PREALLOCATION OF RX PACKET BUFFERS
################################################################################
#### Number of rx packet buffers to allocate at start of an rxq and keep a
#### free packet pool of atleast this many.
####
#### Keeping a free packet pool of rx packet buffers means we do not need to
#### repeatedly allocate and map dma buffers.
####
#### (minimum 0 => off)
#### (default 0 => off)
#### (maximum is limited by available memory)

#rx_prealloc_pkt_buffers=512;

################################################################################
#### ACTION ON HARDWARE ERRORS
################################################################################
#### This parameter controls the action taken on a hardware error
#### which may be a PCIe error or the driver detecting unexpected behaviour
#### from the hardware
####
#### Currently this driver does not reports error into the Solaris fault
#### management architecture (but the PCIe root-port may do so)
####
#### 0 => recover the server adapter to a working state
#### 1 => do not advertise to the kernel that the link is down during the reset
#### 2 => reset the hardware, but do not attempt to use it again.
####      this is useful if you have a failover mechanism, and want to ensure
####      that this server adapter does not become the active link again
####      note that the interface will stay plumbed but will not pass traffic
####
#### All of the above log a message and increment a kstat counter viewable with:
####    kstat -m sfxge -c mon -s num_restarts_hw_err
####
#### (default 0 => recover)

action_on_hw_err=0;

