--- /dev/null	2013-08-22 18:57:33.000000000 -0400
+++ new/usr/src/uts/common/io/sfxge/sfxge.conf	2013-08-22 18:57:33.102871784 -0400
@@ -0,0 +1,248 @@
+#### This file contains configuration parameters for the Solarflare
+#### network adapter driver "sfxge"
+
+################################################################################
+#### HOW TO MAKE CHANGES TO THIS FILE TAKE EFFECT
+################################################################################
+#### In order for changes made to this file to be used by the driver either:
+#### i) reboot
+#### OR
+#### ii) Unplumb all sfxge inet,inet6 interfaces or link aggregations: e.g.
+####       $ ifconfig sfxge0 unplumb
+####       $ ifconfig sfxge0 inet6 unplumb
+####     Delete any link aggregations that setup (see dladm show-aggr):
+####       $ dladm delete-aggr <key>
+####     Update the driver config
+####       $ update_drv sfxge
+####     Note update_drv will unload then reload the driver, otherwise it warns:
+####       "Cannot unload module: sfxge"
+####     Then plumb interfaces and reconfigure IP addresses as required
+#### 
+################################################################################
+
+################################################################################
+#### PER PORT SETTINGS
+################################################################################
+#### Note that is possible to have different settings per network port
+#### First using the file "/etc/path_to_inst" you can identify some information
+####
+####  $ grep sfxge /etc/path_to_inst
+####   # "/pci@3a,0/pci8086,340c@5/pci1924,6201@0"   4 "sfxge"
+####   # "/pci@3a,0/pci8086,340c@5/pci1924,6201@0,1" 5 "sfxge"
+####   #  ^^^^^^^^^^^^^^^^^^^^^^^^             ^^^
+####   #         parent                    unit-address
+####
+#### Using the information gathered above to create entries in this file
+#### based on the example below; this tries to allocate 4 MSI-X interrupts
+#### to port0 and 8 MSI-X interrupts to port 1
+####
+####  name="sfxge" parent="/pci@3a,0/pci8086,340c@5" unit-address="0"
+####               rx_scale_count=4;
+####  name="sfxge" parent="/pci@3a,0/pci8086,340c@5" unit-address="0,1"
+####               rx_scale_count=8;
+################################################################################
+
+################################################################################
+#### RECEIVE SIDE SCALING
+################################################################################
+#### Maximum number of RSS-channels to use per port (default 0, global max 32)
+#### This spreads RX flows across CPUs using multiple MSI-X interrupts
+####
+#### - A value of 0 tries to allocate the same number of RSS-channels as logical
+####   CPUs in order to provide the best spreading accross the entire system.
+####
+#### - actual number may be lower due to availability of MSI-X interrupts
+####
+#### - to successfully use >2 MSI-X interrupts per port you also need to use
+####  "set ddi_msix_alloc_limit=8" (maximum) in /etc/system and reboot
+####   OR temporarily adjust this limit using mdb and then reload the driver
+####     # mdb -kw
+####    > ddi_msix_alloc_limit/W 8
+####    ddi_msix_alloc_limit:           0x2            =       0x8
+####    > ddi_msix_alloc_limit/X
+####    ddi_msix_alloc_limit:
+####    ddi_msix_alloc_limit:           8
+####
+#### - Note that the total MSI-X interrupt vector assigned to all network
+####     drivers (in IPL 6; interrupt priority level) is 32
+####   See the "INTERUPT PRIORITY" section for a way overcome this
+####
+####  -  You can see the current allocation with this run as root:
+####   $ echo "::interrupts -d" | mdb -k | egrep 'IRQ| 6 '
+####     IRQ  Vector IPL Bus   Type  CPU Share APIC/INT# Driver Name(s)
+####     93   0x60   6         MSI-X 2   1     -         sfxge#0
+####     94   0x61   6         MSI-X 3   1     -         sfxge#0
+####                ^^^
+####
+#### - If the sysadmin does not take care to partition the interupts between
+####     the network ports then later ports could use a single legacy interrupt
+####     i.e. no RSS, significantly reducing performance
+####   See PER PORT SETTINGS ABOVE for how to apply rx_scale_count per port
+####
+#### - rx_scale_count can be manipulated dynamically via ndd but the number
+####   of interrupts will stay the same; though only a subset will fire for RX. 
+####   All interrupts will still see ~10 interrupts/sec
+####
+#### - note that the number of RSS-channels can be set to be greater than the
+####   number of logical CPUs. This can be useful to prevent some types of RX
+####   drops
+
+rx_scale_count=0;
+
+################################################################################
+#### USING NON_DEFAULT INTERRUPT PRIORITY LEVEL (IPL)
+################################################################################
+#### To overcome the Solaris limitation of 32 MSI-X vectors per IPL
+#### it is possible to set this driver to allocate interrupts in a different IPL
+#### Please only do this if necessary
+####
+#### See http://download.oracle.com/docs/cd/E20815_01/html/E20816/gjtag.html
+#### Verify with:
+####   $ echo "::interrupts -d" | mdb -k | egrep 'IRQ|sfxge'
+####     IRQ  Vector IPL Bus   Type  CPU Share APIC/INT# Driver Name(s)
+####     93   0x60   5         MSI-X 2   1     -         sfxge#0
+####     94   0x61   5         MSI-X 3   1     -         sfxge#0
+####                ^^^
+####
+#### (default IPL 6; the document referenced recommends 5 or 4)
+
+#interrupt-priorities=5;
+
+################################################################################
+#### RX DMA RING SIZE
+################################################################################
+#### Number of entries in the per receive queue hardware DMA ring
+####
+#### Increase this value if you see increasing values of "rx_nodesc_drop_cnt"
+#### as reported by "kstat -m sfxge -c mac -s rx_nodesc_drop_cnt" and you think
+#### the traffic is bursty
+####
+#### Increasing this value can increase driver memory usage and reduce the
+#### cache-hit rates seen for RX buffers so the largest value is not
+#### necessarily the best
+####
+#### (minimum 512)
+#### (default 1024)
+#### (maximum 4096)
+#### The value must be a power of two else it will be ignored.
+
+#rxq_size=1024;
+
+################################################################################
+#### LARGE RECEIVE OFFLOAD
+################################################################################
+#### Coalesce RX packets (aka Large Receive Offload)
+####  0 => off (default)
+####  1 => on
+####  2 => on, respecting TCP PSH boundaries
+
+#rx_coalesce_mode=1;
+
+################################################################################
+#### INTERRUPT MODERATION
+################################################################################
+#### Interrupt moderation in us (default 30, minimum 0)
+#### Decreasing this reduces latency but increases interrupt rate and therefore
+#### CPU usage which will decrease maximum bandwidth achievable
+####
+#### Decrease this value if you see increasing values of "rx_nodesc_drop_cnt"
+#### as reported by "kstat -m sfxge -c mac -s rx_nodesc_drop_cnt" so that
+#### the driver has more opportunities to refill the hardware RX descriptor
+#### ring
+####
+#### (maximum 20000 us)
+#### (default 30 us)
+
+#intr_moderation=10;
+
+################################################################################
+#### MAXIMUM MTU
+################################################################################
+#### Maximum MTU of an sfxge interface (in bytes - excludes ethernet framing)
+#### Increase this value to support Jumbo frames
+####
+#### NB increasing this value consumes more memory for all RX buffers
+#### even for network interfaces not configured with high MTUs
+####
+#### MTUs over 3968 mean that >1 contiguous 4k page of memory are allocated
+#### for all receive buffers. Contiguous pages of memory can be harder for
+#### the OS to allocate when under memory pressue.
+#### 
+#### In Solaris 11 the MTU can be dynamically changed so you should not
+#### need to alter the setting below. A non-default MTU can be persistently
+#### configured depending on your network configuration:
+#### 1) For manual mode or using NWAM with the DefaultFixed profile:
+####    dladm set-linkprop -p mtu=<MTU> sfxge<n>
+#### 2) If using NWAM using a custom profile please set the property link-mtu:
+####    netcfg "select ncp myprofile; select ncu phys sfxge0; set link-mtu=9000"
+####
+#### (maximum 9000)
+#### (default 1500)
+
+#mtu=1500;
+
+################################################################################
+#### PER PORT MEMORY LIMIT FOR RX PACKET BUFFERS
+################################################################################
+#### Limit max memory for RX packets allocated per instance (port) in bytes
+####
+#### RX packets are passed up from the driver to the kernel TCP/IP stack
+#### and freed after the data is delivered to the application socket
+#### buffers. If the OS falls behind this allocation can grow.
+####
+#### This parameter can help make behaviour reasonable when approaching
+#### an overload condition. Other ways to avoid overload would be RSS
+####
+#### The size of each bufer, number in use can be seen with
+####    echo "::kmastat" | mdb -k
+#### OR kstat -c kmem_cache -n sfxgeX_rx_packet_cache # replace X
+####
+#### (default unlimited when unset)
+####
+#### Note In Solarflare driver package versions 3.1.x.xxxx and earlier set
+#### the RX buffering limit to 10773741824 (1GB) per interface in this file.
+####
+#### 64MB of RX buffering per interface is still oversized for nearly all
+#### applications but provides a hard limit.
+
+rx_pkt_mem_max=67108864;
+
+################################################################################
+#### PREALLOCATION OF RX PACKET BUFFERS
+################################################################################
+#### Number of rx packet buffers to allocate at start of an rxq and keep a
+#### free packet pool of atleast this many.
+####
+#### Keeping a free packet pool of rx packet buffers means we do not need to
+#### repeatedly allocate and map dma buffers.
+####
+#### (minimum 0 => off)
+#### (default 0 => off)
+#### (maximum is limited by available memory)
+
+#rx_prealloc_pkt_buffers=512;
+
+################################################################################
+#### ACTION ON HARDWARE ERRORS
+################################################################################
+#### This parameter controls the action taken on a hardware error
+#### which may be a PCIe error or the driver detecting unexpected behaviour
+#### from the hardware
+####
+#### Currently this driver does not reports error into the Solaris fault
+#### management architecture (but the PCIe root-port may do so)
+####
+#### 0 => recover the server adapter to a working state
+#### 1 => do not advertise to the kernel that the link is down during the reset
+#### 2 => reset the hardware, but do not attempt to use it again.
+####      this is useful if you have a failover mechanism, and want to ensure
+####      that this server adapter does not become the active link again
+####      note that the interface will stay plumbed but will not pass traffic
+####
+#### All of the above log a message and increment a kstat counter viewable with:
+####    kstat -m sfxge -c mon -s num_restarts_hw_err
+####
+#### (default 0 => recover)
+
+action_on_hw_err=0;
+
