2 * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet
5 * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/version.h>
37 #include <linux/module.h>
38 #include <linux/moduleparam.h>
39 #include <linux/init.h>
40 #include <linux/pci.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/netdevice.h>
43 #include <linux/etherdevice.h>
44 #include <linux/debugfs.h>
45 #include <linux/ethtool.h>
47 #include "t4vf_common.h"
48 #include "t4vf_defs.h"
50 #include "../cxgb4/t4_regs.h"
51 #include "../cxgb4/t4_msg.h"
54 * Generic information about the driver.
56 #define DRV_VERSION "1.0.0"
57 #define DRV_DESC "Chelsio T4 Virtual Function (VF) Network Driver"
65 * Default ethtool "message level" for adapters.
67 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
68 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
69 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
71 static int dflt_msg_enable = DFLT_MSG_ENABLE;
73 module_param(dflt_msg_enable, int, 0644);
74 MODULE_PARM_DESC(dflt_msg_enable,
75 "default adapter ethtool message level bitmap");
78 * The driver uses the best interrupt scheme available on a platform in the
79 * order MSI-X then MSI. This parameter determines which of these schemes the
80 * driver may consider as follows:
82 * msi = 2: choose from among MSI-X and MSI
83 * msi = 1: only consider MSI interrupts
85 * Note that unlike the Physical Function driver, this Virtual Function driver
86 * does _not_ support legacy INTx interrupts (this limitation is mandated by
87 * the PCI-E SR-IOV standard).
91 #define MSI_DEFAULT MSI_MSIX
93 static int msi = MSI_DEFAULT;
95 module_param(msi, int, 0644);
96 MODULE_PARM_DESC(msi, "whether to use MSI-X or MSI");
99 * Fundamental constants.
100 * ======================
104 MAX_TXQ_ENTRIES = 16384,
105 MAX_RSPQ_ENTRIES = 16384,
106 MAX_RX_BUFFERS = 16384,
108 MIN_TXQ_ENTRIES = 32,
109 MIN_RSPQ_ENTRIES = 128,
113 * For purposes of manipulating the Free List size we need to
114 * recognize that Free Lists are actually Egress Queues (the host
115 * produces free buffers which the hardware consumes), Egress Queues
116 * indices are all in units of Egress Context Units bytes, and free
117 * list entries are 64-bit PCI DMA addresses. And since the state of
118 * the Producer Index == the Consumer Index implies an EMPTY list, we
119 * always have at least one Egress Unit's worth of Free List entries
120 * unused. See sge.c for more details ...
122 EQ_UNIT = SGE_EQ_IDXSIZE,
123 FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
124 MIN_FL_RESID = FL_PER_EQ_UNIT,
128 * Global driver state.
129 * ====================
132 static struct dentry *cxgb4vf_debugfs_root;
135 * OS "Callback" functions.
136 * ========================
140 * The link status has changed on the indicated "port" (Virtual Interface).
142 void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok)
144 struct net_device *dev = adapter->port[pidx];
147 * If the port is disabled or the current recorded "link up"
148 * status matches the new status, just return.
150 if (!netif_running(dev) || link_ok == netif_carrier_ok(dev))
154 * Tell the OS that the link status has changed and print a short
155 * informative message on the console about the event.
160 const struct port_info *pi = netdev_priv(dev);
162 netif_carrier_on(dev);
164 switch (pi->link_cfg.speed) {
182 switch (pi->link_cfg.fc) {
191 case PAUSE_RX|PAUSE_TX:
200 printk(KERN_INFO "%s: link up, %s, full-duplex, %s PAUSE\n",
203 netif_carrier_off(dev);
204 printk(KERN_INFO "%s: link down\n", dev->name);
209 * Net device operations.
210 * ======================
214 * Record our new VLAN Group and enable/disable hardware VLAN Tag extraction
215 * based on whether the specified VLAN Group pointer is NULL or not.
217 static void cxgb4vf_vlan_rx_register(struct net_device *dev,
218 struct vlan_group *grp)
220 struct port_info *pi = netdev_priv(dev);
223 t4vf_set_rxmode(pi->adapter, pi->viid, -1, -1, -1, -1, grp != NULL, 0);
227 * Perform the MAC and PHY actions needed to enable a "port" (Virtual
230 static int link_start(struct net_device *dev)
233 struct port_info *pi = netdev_priv(dev);
236 * We do not set address filters and promiscuity here, the stack does
237 * that step explicitly.
239 ret = t4vf_set_rxmode(pi->adapter, pi->viid, dev->mtu, -1, -1, -1, -1,
242 ret = t4vf_change_mac(pi->adapter, pi->viid,
243 pi->xact_addr_filt, dev->dev_addr, true);
245 pi->xact_addr_filt = ret;
251 * We don't need to actually "start the link" itself since the
252 * firmware will do that for us when the first Virtual Interface
253 * is enabled on a port.
256 ret = t4vf_enable_vi(pi->adapter, pi->viid, true, true);
261 * Name the MSI-X interrupts.
263 static void name_msix_vecs(struct adapter *adapter)
265 int namelen = sizeof(adapter->msix_info[0].desc) - 1;
271 snprintf(adapter->msix_info[MSIX_FW].desc, namelen,
272 "%s-FWeventq", adapter->name);
273 adapter->msix_info[MSIX_FW].desc[namelen] = 0;
278 for_each_port(adapter, pidx) {
279 struct net_device *dev = adapter->port[pidx];
280 const struct port_info *pi = netdev_priv(dev);
283 for (qs = 0, msi = MSIX_NIQFLINT;
286 snprintf(adapter->msix_info[msi].desc, namelen,
287 "%s-%d", dev->name, qs);
288 adapter->msix_info[msi].desc[namelen] = 0;
294 * Request all of our MSI-X resources.
296 static int request_msix_queue_irqs(struct adapter *adapter)
298 struct sge *s = &adapter->sge;
304 err = request_irq(adapter->msix_info[MSIX_FW].vec, t4vf_sge_intr_msix,
305 0, adapter->msix_info[MSIX_FW].desc, &s->fw_evtq);
313 for_each_ethrxq(s, rxq) {
314 err = request_irq(adapter->msix_info[msi].vec,
315 t4vf_sge_intr_msix, 0,
316 adapter->msix_info[msi].desc,
317 &s->ethrxq[rxq].rspq);
326 free_irq(adapter->msix_info[--msi].vec, &s->ethrxq[rxq].rspq);
327 free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
332 * Free our MSI-X resources.
334 static void free_msix_queue_irqs(struct adapter *adapter)
336 struct sge *s = &adapter->sge;
339 free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
341 for_each_ethrxq(s, rxq)
342 free_irq(adapter->msix_info[msi++].vec,
343 &s->ethrxq[rxq].rspq);
347 * Turn on NAPI and start up interrupts on a response queue.
349 static void qenable(struct sge_rspq *rspq)
351 napi_enable(&rspq->napi);
354 * 0-increment the Going To Sleep register to start the timer and
357 t4_write_reg(rspq->adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
359 SEINTARM(rspq->intr_params) |
360 INGRESSQID(rspq->cntxt_id));
364 * Enable NAPI scheduling and interrupt generation for all Receive Queues.
366 static void enable_rx(struct adapter *adapter)
369 struct sge *s = &adapter->sge;
371 for_each_ethrxq(s, rxq)
372 qenable(&s->ethrxq[rxq].rspq);
373 qenable(&s->fw_evtq);
376 * The interrupt queue doesn't use NAPI so we do the 0-increment of
377 * its Going To Sleep register here to get it started.
379 if (adapter->flags & USING_MSI)
380 t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
382 SEINTARM(s->intrq.intr_params) |
383 INGRESSQID(s->intrq.cntxt_id));
388 * Wait until all NAPI handlers are descheduled.
390 static void quiesce_rx(struct adapter *adapter)
392 struct sge *s = &adapter->sge;
395 for_each_ethrxq(s, rxq)
396 napi_disable(&s->ethrxq[rxq].rspq.napi);
397 napi_disable(&s->fw_evtq.napi);
401 * Response queue handler for the firmware event queue.
403 static int fwevtq_handler(struct sge_rspq *rspq, const __be64 *rsp,
404 const struct pkt_gl *gl)
407 * Extract response opcode and get pointer to CPL message body.
409 struct adapter *adapter = rspq->adapter;
410 u8 opcode = ((const struct rss_header *)rsp)->opcode;
411 void *cpl = (void *)(rsp + 1);
416 * We've received an asynchronous message from the firmware.
418 const struct cpl_fw6_msg *fw_msg = cpl;
419 if (fw_msg->type == FW6_TYPE_CMD_RPL)
420 t4vf_handle_fw_rpl(adapter, fw_msg->data);
424 case CPL_SGE_EGR_UPDATE: {
426 * We've received an Egress Queue Status Update message. We
427 * get these, if the SGE is configured to send these when the
428 * firmware passes certain points in processing our TX
429 * Ethernet Queue or if we make an explicit request for one.
430 * We use these updates to determine when we may need to
431 * restart a TX Ethernet Queue which was stopped for lack of
432 * free TX Queue Descriptors ...
434 const struct cpl_sge_egr_update *p = (void *)cpl;
435 unsigned int qid = EGR_QID(be32_to_cpu(p->opcode_qid));
436 struct sge *s = &adapter->sge;
438 struct sge_eth_txq *txq;
442 * Perform sanity checking on the Queue ID to make sure it
443 * really refers to one of our TX Ethernet Egress Queues which
444 * is active and matches the queue's ID. None of these error
445 * conditions should ever happen so we may want to either make
446 * them fatal and/or conditionalized under DEBUG.
448 eq_idx = EQ_IDX(s, qid);
449 if (unlikely(eq_idx >= MAX_EGRQ)) {
450 dev_err(adapter->pdev_dev,
451 "Egress Update QID %d out of range\n", qid);
454 tq = s->egr_map[eq_idx];
455 if (unlikely(tq == NULL)) {
456 dev_err(adapter->pdev_dev,
457 "Egress Update QID %d TXQ=NULL\n", qid);
460 txq = container_of(tq, struct sge_eth_txq, q);
461 if (unlikely(tq->abs_id != qid)) {
462 dev_err(adapter->pdev_dev,
463 "Egress Update QID %d refers to TXQ %d\n",
469 * Restart a stopped TX Queue which has less than half of its
473 netif_tx_wake_queue(txq->txq);
478 dev_err(adapter->pdev_dev,
479 "unexpected CPL %#x on FW event queue\n", opcode);
486 * Allocate SGE TX/RX response queues. Determine how many sets of SGE queues
487 * to use and initializes them. We support multiple "Queue Sets" per port if
488 * we have MSI-X, otherwise just one queue set per port.
490 static int setup_sge_queues(struct adapter *adapter)
492 struct sge *s = &adapter->sge;
496 * Clear "Queue Set" Free List Starving and TX Queue Mapping Error
499 bitmap_zero(s->starving_fl, MAX_EGRQ);
502 * If we're using MSI interrupt mode we need to set up a "forwarded
503 * interrupt" queue which we'll set up with our MSI vector. The rest
504 * of the ingress queues will be set up to forward their interrupts to
505 * this queue ... This must be first since t4vf_sge_alloc_rxq() uses
506 * the intrq's queue ID as the interrupt forwarding queue for the
507 * subsequent calls ...
509 if (adapter->flags & USING_MSI) {
510 err = t4vf_sge_alloc_rxq(adapter, &s->intrq, false,
511 adapter->port[0], 0, NULL, NULL);
513 goto err_free_queues;
517 * Allocate our ingress queue for asynchronous firmware messages.
519 err = t4vf_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->port[0],
520 MSIX_FW, NULL, fwevtq_handler);
522 goto err_free_queues;
525 * Allocate each "port"'s initial Queue Sets. These can be changed
526 * later on ... up to the point where any interface on the adapter is
527 * brought up at which point lots of things get nailed down
530 msix = MSIX_NIQFLINT;
531 for_each_port(adapter, pidx) {
532 struct net_device *dev = adapter->port[pidx];
533 struct port_info *pi = netdev_priv(dev);
534 struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
535 struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
538 for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
539 err = t4vf_sge_alloc_rxq(adapter, &rxq->rspq, false,
541 &rxq->fl, t4vf_ethrx_handler);
543 goto err_free_queues;
545 err = t4vf_sge_alloc_eth_txq(adapter, txq, dev,
546 netdev_get_tx_queue(dev, qs),
547 s->fw_evtq.cntxt_id);
549 goto err_free_queues;
552 memset(&rxq->stats, 0, sizeof(rxq->stats));
557 * Create the reverse mappings for the queues.
559 s->egr_base = s->ethtxq[0].q.abs_id - s->ethtxq[0].q.cntxt_id;
560 s->ingr_base = s->ethrxq[0].rspq.abs_id - s->ethrxq[0].rspq.cntxt_id;
561 IQ_MAP(s, s->fw_evtq.abs_id) = &s->fw_evtq;
562 for_each_port(adapter, pidx) {
563 struct net_device *dev = adapter->port[pidx];
564 struct port_info *pi = netdev_priv(dev);
565 struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
566 struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
569 for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
570 IQ_MAP(s, rxq->rspq.abs_id) = &rxq->rspq;
571 EQ_MAP(s, txq->q.abs_id) = &txq->q;
574 * The FW_IQ_CMD doesn't return the Absolute Queue IDs
575 * for Free Lists but since all of the Egress Queues
576 * (including Free Lists) have Relative Queue IDs
577 * which are computed as Absolute - Base Queue ID, we
578 * can synthesize the Absolute Queue IDs for the Free
579 * Lists. This is useful for debugging purposes when
580 * we want to dump Queue Contexts via the PF Driver.
582 rxq->fl.abs_id = rxq->fl.cntxt_id + s->egr_base;
583 EQ_MAP(s, rxq->fl.abs_id) = &rxq->fl;
589 t4vf_free_sge_resources(adapter);
594 * Set up Receive Side Scaling (RSS) to distribute packets to multiple receive
595 * queues. We configure the RSS CPU lookup table to distribute to the number
596 * of HW receive queues, and the response queue lookup table to narrow that
597 * down to the response queues actually configured for each "port" (Virtual
598 * Interface). We always configure the RSS mapping for all ports since the
599 * mapping table has plenty of entries.
601 static int setup_rss(struct adapter *adapter)
605 for_each_port(adapter, pidx) {
606 struct port_info *pi = adap2pinfo(adapter, pidx);
607 struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
608 u16 rss[MAX_PORT_QSETS];
611 for (qs = 0; qs < pi->nqsets; qs++)
612 rss[qs] = rxq[qs].rspq.abs_id;
614 err = t4vf_config_rss_range(adapter, pi->viid,
615 0, pi->rss_size, rss, pi->nqsets);
620 * Perform Global RSS Mode-specific initialization.
622 switch (adapter->params.rss.mode) {
623 case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL:
625 * If Tunnel All Lookup isn't specified in the global
626 * RSS Configuration, then we need to specify a
627 * default Ingress Queue for any ingress packets which
628 * aren't hashed. We'll use our first ingress queue
631 if (!adapter->params.rss.u.basicvirtual.tnlalllookup) {
632 union rss_vi_config config;
633 err = t4vf_read_rss_vi_config(adapter,
638 config.basicvirtual.defaultq =
640 err = t4vf_write_rss_vi_config(adapter,
654 * Bring the adapter up. Called whenever we go from no "ports" open to having
655 * one open. This function performs the actions necessary to make an adapter
656 * operational, such as completing the initialization of HW modules, and
657 * enabling interrupts. Must be called with the rtnl lock held. (Note that
658 * this is called "cxgb_up" in the PF Driver.)
660 static int adapter_up(struct adapter *adapter)
665 * If this is the first time we've been called, perform basic
666 * adapter setup. Once we've done this, many of our adapter
667 * parameters can no longer be changed ...
669 if ((adapter->flags & FULL_INIT_DONE) == 0) {
670 err = setup_sge_queues(adapter);
673 err = setup_rss(adapter);
675 t4vf_free_sge_resources(adapter);
679 if (adapter->flags & USING_MSIX)
680 name_msix_vecs(adapter);
681 adapter->flags |= FULL_INIT_DONE;
685 * Acquire our interrupt resources. We only support MSI-X and MSI.
687 BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
688 if (adapter->flags & USING_MSIX)
689 err = request_msix_queue_irqs(adapter);
691 err = request_irq(adapter->pdev->irq,
692 t4vf_intr_handler(adapter), 0,
693 adapter->name, adapter);
695 dev_err(adapter->pdev_dev, "request_irq failed, err %d\n",
701 * Enable NAPI ingress processing and return success.
704 t4vf_sge_start(adapter);
709 * Bring the adapter down. Called whenever the last "port" (Virtual
710 * Interface) closed. (Note that this routine is called "cxgb_down" in the PF
713 static void adapter_down(struct adapter *adapter)
716 * Free interrupt resources.
718 if (adapter->flags & USING_MSIX)
719 free_msix_queue_irqs(adapter);
721 free_irq(adapter->pdev->irq, adapter);
724 * Wait for NAPI handlers to finish.
730 * Start up a net device.
732 static int cxgb4vf_open(struct net_device *dev)
735 struct port_info *pi = netdev_priv(dev);
736 struct adapter *adapter = pi->adapter;
739 * If this is the first interface that we're opening on the "adapter",
740 * bring the "adapter" up now.
742 if (adapter->open_device_map == 0) {
743 err = adapter_up(adapter);
749 * Note that this interface is up and start everything up ...
751 dev->real_num_tx_queues = pi->nqsets;
752 set_bit(pi->port_id, &adapter->open_device_map);
754 netif_tx_start_all_queues(dev);
759 * Shut down a net device. This routine is called "cxgb_close" in the PF
762 static int cxgb4vf_stop(struct net_device *dev)
765 struct port_info *pi = netdev_priv(dev);
766 struct adapter *adapter = pi->adapter;
768 netif_tx_stop_all_queues(dev);
769 netif_carrier_off(dev);
770 ret = t4vf_enable_vi(adapter, pi->viid, false, false);
771 pi->link_cfg.link_ok = 0;
773 clear_bit(pi->port_id, &adapter->open_device_map);
774 if (adapter->open_device_map == 0)
775 adapter_down(adapter);
780 * Translate our basic statistics into the standard "ifconfig" statistics.
782 static struct net_device_stats *cxgb4vf_get_stats(struct net_device *dev)
784 struct t4vf_port_stats stats;
785 struct port_info *pi = netdev2pinfo(dev);
786 struct adapter *adapter = pi->adapter;
787 struct net_device_stats *ns = &dev->stats;
790 spin_lock(&adapter->stats_lock);
791 err = t4vf_get_port_stats(adapter, pi->pidx, &stats);
792 spin_unlock(&adapter->stats_lock);
794 memset(ns, 0, sizeof(*ns));
798 ns->tx_bytes = (stats.tx_bcast_bytes + stats.tx_mcast_bytes +
799 stats.tx_ucast_bytes + stats.tx_offload_bytes);
800 ns->tx_packets = (stats.tx_bcast_frames + stats.tx_mcast_frames +
801 stats.tx_ucast_frames + stats.tx_offload_frames);
802 ns->rx_bytes = (stats.rx_bcast_bytes + stats.rx_mcast_bytes +
803 stats.rx_ucast_bytes);
804 ns->rx_packets = (stats.rx_bcast_frames + stats.rx_mcast_frames +
805 stats.rx_ucast_frames);
806 ns->multicast = stats.rx_mcast_frames;
807 ns->tx_errors = stats.tx_drop_frames;
808 ns->rx_errors = stats.rx_err_frames;
814 * Collect up to maxaddrs worth of a netdevice's unicast addresses into an
815 * array of addrss pointers and return the number collected.
817 static inline int collect_netdev_uc_list_addrs(const struct net_device *dev,
819 unsigned int maxaddrs)
821 unsigned int naddr = 0;
822 const struct netdev_hw_addr *ha;
824 for_each_dev_addr(dev, ha) {
825 addr[naddr++] = ha->addr;
826 if (naddr >= maxaddrs)
833 * Collect up to maxaddrs worth of a netdevice's multicast addresses into an
834 * array of addrss pointers and return the number collected.
836 static inline int collect_netdev_mc_list_addrs(const struct net_device *dev,
838 unsigned int maxaddrs)
840 unsigned int naddr = 0;
841 const struct netdev_hw_addr *ha;
843 netdev_for_each_mc_addr(ha, dev) {
844 addr[naddr++] = ha->addr;
845 if (naddr >= maxaddrs)
852 * Configure the exact and hash address filters to handle a port's multicast
853 * and secondary unicast MAC addresses.
855 static int set_addr_filters(const struct net_device *dev, bool sleep)
863 const struct port_info *pi = netdev_priv(dev);
865 /* first do the secondary unicast addresses */
866 naddr = collect_netdev_uc_list_addrs(dev, addr, ARRAY_SIZE(addr));
868 ret = t4vf_alloc_mac_filt(pi->adapter, pi->viid, free,
869 naddr, addr, filt_idx, &uhash, sleep);
876 /* next set up the multicast addresses */
877 naddr = collect_netdev_mc_list_addrs(dev, addr, ARRAY_SIZE(addr));
879 ret = t4vf_alloc_mac_filt(pi->adapter, pi->viid, free,
880 naddr, addr, filt_idx, &mhash, sleep);
885 return t4vf_set_addr_hash(pi->adapter, pi->viid, uhash != 0,
886 uhash | mhash, sleep);
890 * Set RX properties of a port, such as promiscruity, address filters, and MTU.
891 * If @mtu is -1 it is left unchanged.
893 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
896 struct port_info *pi = netdev_priv(dev);
898 ret = set_addr_filters(dev, sleep_ok);
900 ret = t4vf_set_rxmode(pi->adapter, pi->viid, -1,
901 (dev->flags & IFF_PROMISC) != 0,
902 (dev->flags & IFF_ALLMULTI) != 0,
908 * Set the current receive modes on the device.
910 static void cxgb4vf_set_rxmode(struct net_device *dev)
912 /* unfortunately we can't return errors to the stack */
913 set_rxmode(dev, -1, false);
917 * Find the entry in the interrupt holdoff timer value array which comes
918 * closest to the specified interrupt holdoff value.
920 static int closest_timer(const struct sge *s, int us)
922 int i, timer_idx = 0, min_delta = INT_MAX;
924 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
925 int delta = us - s->timer_val[i];
928 if (delta < min_delta) {
936 static int closest_thres(const struct sge *s, int thres)
938 int i, delta, pktcnt_idx = 0, min_delta = INT_MAX;
940 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
941 delta = thres - s->counter_val[i];
944 if (delta < min_delta) {
953 * Return a queue's interrupt hold-off time in us. 0 means no timer.
955 static unsigned int qtimer_val(const struct adapter *adapter,
956 const struct sge_rspq *rspq)
958 unsigned int timer_idx = QINTR_TIMER_IDX_GET(rspq->intr_params);
960 return timer_idx < SGE_NTIMERS
961 ? adapter->sge.timer_val[timer_idx]
966 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
967 * @adapter: the adapter
968 * @rspq: the RX response queue
969 * @us: the hold-off time in us, or 0 to disable timer
970 * @cnt: the hold-off packet count, or 0 to disable counter
972 * Sets an RX response queue's interrupt hold-off time and packet count.
973 * At least one of the two needs to be enabled for the queue to generate
976 static int set_rxq_intr_params(struct adapter *adapter, struct sge_rspq *rspq,
977 unsigned int us, unsigned int cnt)
979 unsigned int timer_idx;
982 * If both the interrupt holdoff timer and count are specified as
983 * zero, default to a holdoff count of 1 ...
989 * If an interrupt holdoff count has been specified, then find the
990 * closest configured holdoff count and use that. If the response
991 * queue has already been created, then update its queue context
998 pktcnt_idx = closest_thres(&adapter->sge, cnt);
999 if (rspq->desc && rspq->pktcnt_idx != pktcnt_idx) {
1000 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1002 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1003 FW_PARAMS_PARAM_YZ(rspq->cntxt_id);
1004 err = t4vf_set_params(adapter, 1, &v, &pktcnt_idx);
1008 rspq->pktcnt_idx = pktcnt_idx;
1012 * Compute the closest holdoff timer index from the supplied holdoff
1015 timer_idx = (us == 0
1016 ? SGE_TIMER_RSTRT_CNTR
1017 : closest_timer(&adapter->sge, us));
1020 * Update the response queue's interrupt coalescing parameters and
1023 rspq->intr_params = (QINTR_TIMER_IDX(timer_idx) |
1024 (cnt > 0 ? QINTR_CNT_EN : 0));
1029 * Return a version number to identify the type of adapter. The scheme is:
1030 * - bits 0..9: chip version
1031 * - bits 10..15: chip revision
1033 static inline unsigned int mk_adap_vers(const struct adapter *adapter)
1036 * Chip version 4, revision 0x3f (cxgb4vf).
1038 return 4 | (0x3f << 10);
1042 * Execute the specified ioctl command.
1044 static int cxgb4vf_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1050 * The VF Driver doesn't have access to any of the other
1051 * common Ethernet device ioctl()'s (like reading/writing
1052 * PHY registers, etc.
1063 * Change the device's MTU.
1065 static int cxgb4vf_change_mtu(struct net_device *dev, int new_mtu)
1068 struct port_info *pi = netdev_priv(dev);
1070 /* accommodate SACK */
1074 ret = t4vf_set_rxmode(pi->adapter, pi->viid, new_mtu,
1075 -1, -1, -1, -1, true);
1082 * Change the devices MAC address.
1084 static int cxgb4vf_set_mac_addr(struct net_device *dev, void *_addr)
1087 struct sockaddr *addr = _addr;
1088 struct port_info *pi = netdev_priv(dev);
1090 if (!is_valid_ether_addr(addr->sa_data))
1093 ret = t4vf_change_mac(pi->adapter, pi->viid, pi->xact_addr_filt,
1094 addr->sa_data, true);
1098 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1099 pi->xact_addr_filt = ret;
1104 * Return a TX Queue on which to send the specified skb.
1106 static u16 cxgb4vf_select_queue(struct net_device *dev, struct sk_buff *skb)
1109 * XXX For now just use the default hash but we probably want to
1110 * XXX look at other possibilities ...
1112 return skb_tx_hash(dev, skb);
1115 #ifdef CONFIG_NET_POLL_CONTROLLER
1117 * Poll all of our receive queues. This is called outside of normal interrupt
1120 static void cxgb4vf_poll_controller(struct net_device *dev)
1122 struct port_info *pi = netdev_priv(dev);
1123 struct adapter *adapter = pi->adapter;
1125 if (adapter->flags & USING_MSIX) {
1126 struct sge_eth_rxq *rxq;
1129 rxq = &adapter->sge.ethrxq[pi->first_qset];
1130 for (nqsets = pi->nqsets; nqsets; nqsets--) {
1131 t4vf_sge_intr_msix(0, &rxq->rspq);
1135 t4vf_intr_handler(adapter)(0, adapter);
1140 * Ethtool operations.
1141 * ===================
1143 * Note that we don't support any ethtool operations which change the physical
1144 * state of the port to which we're linked.
1148 * Return current port link settings.
1150 static int cxgb4vf_get_settings(struct net_device *dev,
1151 struct ethtool_cmd *cmd)
1153 const struct port_info *pi = netdev_priv(dev);
1155 cmd->supported = pi->link_cfg.supported;
1156 cmd->advertising = pi->link_cfg.advertising;
1157 cmd->speed = netif_carrier_ok(dev) ? pi->link_cfg.speed : -1;
1158 cmd->duplex = DUPLEX_FULL;
1160 cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1161 cmd->phy_address = pi->port_id;
1162 cmd->transceiver = XCVR_EXTERNAL;
1163 cmd->autoneg = pi->link_cfg.autoneg;
1170 * Return our driver information.
1172 static void cxgb4vf_get_drvinfo(struct net_device *dev,
1173 struct ethtool_drvinfo *drvinfo)
1175 struct adapter *adapter = netdev2adap(dev);
1177 strcpy(drvinfo->driver, KBUILD_MODNAME);
1178 strcpy(drvinfo->version, DRV_VERSION);
1179 strcpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)));
1180 snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
1181 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1182 FW_HDR_FW_VER_MAJOR_GET(adapter->params.dev.fwrev),
1183 FW_HDR_FW_VER_MINOR_GET(adapter->params.dev.fwrev),
1184 FW_HDR_FW_VER_MICRO_GET(adapter->params.dev.fwrev),
1185 FW_HDR_FW_VER_BUILD_GET(adapter->params.dev.fwrev),
1186 FW_HDR_FW_VER_MAJOR_GET(adapter->params.dev.tprev),
1187 FW_HDR_FW_VER_MINOR_GET(adapter->params.dev.tprev),
1188 FW_HDR_FW_VER_MICRO_GET(adapter->params.dev.tprev),
1189 FW_HDR_FW_VER_BUILD_GET(adapter->params.dev.tprev));
1193 * Return current adapter message level.
1195 static u32 cxgb4vf_get_msglevel(struct net_device *dev)
1197 return netdev2adap(dev)->msg_enable;
1201 * Set current adapter message level.
1203 static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel)
1205 netdev2adap(dev)->msg_enable = msglevel;
1209 * Return the device's current Queue Set ring size parameters along with the
1210 * allowed maximum values. Since ethtool doesn't understand the concept of
1211 * multi-queue devices, we just return the current values associated with the
1214 static void cxgb4vf_get_ringparam(struct net_device *dev,
1215 struct ethtool_ringparam *rp)
1217 const struct port_info *pi = netdev_priv(dev);
1218 const struct sge *s = &pi->adapter->sge;
1220 rp->rx_max_pending = MAX_RX_BUFFERS;
1221 rp->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1222 rp->rx_jumbo_max_pending = 0;
1223 rp->tx_max_pending = MAX_TXQ_ENTRIES;
1225 rp->rx_pending = s->ethrxq[pi->first_qset].fl.size - MIN_FL_RESID;
1226 rp->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1227 rp->rx_jumbo_pending = 0;
1228 rp->tx_pending = s->ethtxq[pi->first_qset].q.size;
1232 * Set the Queue Set ring size parameters for the device. Again, since
1233 * ethtool doesn't allow for the concept of multiple queues per device, we'll
1234 * apply these new values across all of the Queue Sets associated with the
1235 * device -- after vetting them of course!
1237 static int cxgb4vf_set_ringparam(struct net_device *dev,
1238 struct ethtool_ringparam *rp)
1240 const struct port_info *pi = netdev_priv(dev);
1241 struct adapter *adapter = pi->adapter;
1242 struct sge *s = &adapter->sge;
1245 if (rp->rx_pending > MAX_RX_BUFFERS ||
1246 rp->rx_jumbo_pending ||
1247 rp->tx_pending > MAX_TXQ_ENTRIES ||
1248 rp->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1249 rp->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1250 rp->rx_pending < MIN_FL_ENTRIES ||
1251 rp->tx_pending < MIN_TXQ_ENTRIES)
1254 if (adapter->flags & FULL_INIT_DONE)
1257 for (qs = pi->first_qset; qs < pi->first_qset + pi->nqsets; qs++) {
1258 s->ethrxq[qs].fl.size = rp->rx_pending + MIN_FL_RESID;
1259 s->ethrxq[qs].rspq.size = rp->rx_mini_pending;
1260 s->ethtxq[qs].q.size = rp->tx_pending;
1266 * Return the interrupt holdoff timer and count for the first Queue Set on the
1267 * device. Our extension ioctl() (the cxgbtool interface) allows the
1268 * interrupt holdoff timer to be read on all of the device's Queue Sets.
1270 static int cxgb4vf_get_coalesce(struct net_device *dev,
1271 struct ethtool_coalesce *coalesce)
1273 const struct port_info *pi = netdev_priv(dev);
1274 const struct adapter *adapter = pi->adapter;
1275 const struct sge_rspq *rspq = &adapter->sge.ethrxq[pi->first_qset].rspq;
1277 coalesce->rx_coalesce_usecs = qtimer_val(adapter, rspq);
1278 coalesce->rx_max_coalesced_frames =
1279 ((rspq->intr_params & QINTR_CNT_EN)
1280 ? adapter->sge.counter_val[rspq->pktcnt_idx]
1286 * Set the RX interrupt holdoff timer and count for the first Queue Set on the
1287 * interface. Our extension ioctl() (the cxgbtool interface) allows us to set
1288 * the interrupt holdoff timer on any of the device's Queue Sets.
1290 static int cxgb4vf_set_coalesce(struct net_device *dev,
1291 struct ethtool_coalesce *coalesce)
1293 const struct port_info *pi = netdev_priv(dev);
1294 struct adapter *adapter = pi->adapter;
1296 return set_rxq_intr_params(adapter,
1297 &adapter->sge.ethrxq[pi->first_qset].rspq,
1298 coalesce->rx_coalesce_usecs,
1299 coalesce->rx_max_coalesced_frames);
1303 * Report current port link pause parameter settings.
1305 static void cxgb4vf_get_pauseparam(struct net_device *dev,
1306 struct ethtool_pauseparam *pauseparam)
1308 struct port_info *pi = netdev_priv(dev);
1310 pauseparam->autoneg = (pi->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1311 pauseparam->rx_pause = (pi->link_cfg.fc & PAUSE_RX) != 0;
1312 pauseparam->tx_pause = (pi->link_cfg.fc & PAUSE_TX) != 0;
1316 * Return whether RX Checksum Offloading is currently enabled for the device.
1318 static u32 cxgb4vf_get_rx_csum(struct net_device *dev)
1320 struct port_info *pi = netdev_priv(dev);
1322 return (pi->rx_offload & RX_CSO) != 0;
1326 * Turn RX Checksum Offloading on or off for the device.
1328 static int cxgb4vf_set_rx_csum(struct net_device *dev, u32 csum)
1330 struct port_info *pi = netdev_priv(dev);
1333 pi->rx_offload |= RX_CSO;
1335 pi->rx_offload &= ~RX_CSO;
1340 * Identify the port by blinking the port's LED.
1342 static int cxgb4vf_phys_id(struct net_device *dev, u32 id)
1344 struct port_info *pi = netdev_priv(dev);
1346 return t4vf_identify_port(pi->adapter, pi->viid, 5);
1350 * Port stats maintained per queue of the port.
1352 struct queue_port_stats {
1361 * Strings for the ETH_SS_STATS statistics set ("ethtool -S"). Note that
1362 * these need to match the order of statistics returned by
1363 * t4vf_get_port_stats().
1365 static const char stats_strings[][ETH_GSTRING_LEN] = {
1367 * These must match the layout of the t4vf_port_stats structure.
1369 "TxBroadcastBytes ",
1370 "TxBroadcastFrames ",
1371 "TxMulticastBytes ",
1372 "TxMulticastFrames ",
1378 "RxBroadcastBytes ",
1379 "RxBroadcastFrames ",
1380 "RxMulticastBytes ",
1381 "RxMulticastFrames ",
1387 * These are accumulated per-queue statistics and must match the
1388 * order of the fields in the queue_port_stats structure.
1398 * Return the number of statistics in the specified statistics set.
1400 static int cxgb4vf_get_sset_count(struct net_device *dev, int sset)
1404 return ARRAY_SIZE(stats_strings);
1412 * Return the strings for the specified statistics set.
1414 static void cxgb4vf_get_strings(struct net_device *dev,
1420 memcpy(data, stats_strings, sizeof(stats_strings));
1426 * Small utility routine to accumulate queue statistics across the queues of
1429 static void collect_sge_port_stats(const struct adapter *adapter,
1430 const struct port_info *pi,
1431 struct queue_port_stats *stats)
1433 const struct sge_eth_txq *txq = &adapter->sge.ethtxq[pi->first_qset];
1434 const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
1437 memset(stats, 0, sizeof(*stats));
1438 for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
1439 stats->tso += txq->tso;
1440 stats->tx_csum += txq->tx_cso;
1441 stats->rx_csum += rxq->stats.rx_cso;
1442 stats->vlan_ex += rxq->stats.vlan_ex;
1443 stats->vlan_ins += txq->vlan_ins;
1448 * Return the ETH_SS_STATS statistics set.
1450 static void cxgb4vf_get_ethtool_stats(struct net_device *dev,
1451 struct ethtool_stats *stats,
1454 struct port_info *pi = netdev2pinfo(dev);
1455 struct adapter *adapter = pi->adapter;
1456 int err = t4vf_get_port_stats(adapter, pi->pidx,
1457 (struct t4vf_port_stats *)data);
1459 memset(data, 0, sizeof(struct t4vf_port_stats));
1461 data += sizeof(struct t4vf_port_stats) / sizeof(u64);
1462 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1466 * Return the size of our register map.
1468 static int cxgb4vf_get_regs_len(struct net_device *dev)
1470 return T4VF_REGMAP_SIZE;
1474 * Dump a block of registers, start to end inclusive, into a buffer.
1476 static void reg_block_dump(struct adapter *adapter, void *regbuf,
1477 unsigned int start, unsigned int end)
1479 u32 *bp = regbuf + start - T4VF_REGMAP_START;
1481 for ( ; start <= end; start += sizeof(u32)) {
1483 * Avoid reading the Mailbox Control register since that
1484 * can trigger a Mailbox Ownership Arbitration cycle and
1485 * interfere with communication with the firmware.
1487 if (start == T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL)
1490 *bp++ = t4_read_reg(adapter, start);
1495 * Copy our entire register map into the provided buffer.
1497 static void cxgb4vf_get_regs(struct net_device *dev,
1498 struct ethtool_regs *regs,
1501 struct adapter *adapter = netdev2adap(dev);
1503 regs->version = mk_adap_vers(adapter);
1506 * Fill in register buffer with our register map.
1508 memset(regbuf, 0, T4VF_REGMAP_SIZE);
1510 reg_block_dump(adapter, regbuf,
1511 T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_FIRST,
1512 T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_LAST);
1513 reg_block_dump(adapter, regbuf,
1514 T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_FIRST,
1515 T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_LAST);
1516 reg_block_dump(adapter, regbuf,
1517 T4VF_PL_BASE_ADDR + T4VF_MOD_MAP_PL_FIRST,
1518 T4VF_PL_BASE_ADDR + T4VF_MOD_MAP_PL_LAST);
1519 reg_block_dump(adapter, regbuf,
1520 T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_FIRST,
1521 T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_LAST);
1523 reg_block_dump(adapter, regbuf,
1524 T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_FIRST,
1525 T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_LAST);
1529 * Report current Wake On LAN settings.
1531 static void cxgb4vf_get_wol(struct net_device *dev,
1532 struct ethtool_wolinfo *wol)
1536 memset(&wol->sopass, 0, sizeof(wol->sopass));
1540 * Set TCP Segmentation Offloading feature capabilities.
1542 static int cxgb4vf_set_tso(struct net_device *dev, u32 tso)
1545 dev->features |= NETIF_F_TSO | NETIF_F_TSO6;
1547 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
1551 static struct ethtool_ops cxgb4vf_ethtool_ops = {
1552 .get_settings = cxgb4vf_get_settings,
1553 .get_drvinfo = cxgb4vf_get_drvinfo,
1554 .get_msglevel = cxgb4vf_get_msglevel,
1555 .set_msglevel = cxgb4vf_set_msglevel,
1556 .get_ringparam = cxgb4vf_get_ringparam,
1557 .set_ringparam = cxgb4vf_set_ringparam,
1558 .get_coalesce = cxgb4vf_get_coalesce,
1559 .set_coalesce = cxgb4vf_set_coalesce,
1560 .get_pauseparam = cxgb4vf_get_pauseparam,
1561 .get_rx_csum = cxgb4vf_get_rx_csum,
1562 .set_rx_csum = cxgb4vf_set_rx_csum,
1563 .set_tx_csum = ethtool_op_set_tx_ipv6_csum,
1564 .set_sg = ethtool_op_set_sg,
1565 .get_link = ethtool_op_get_link,
1566 .get_strings = cxgb4vf_get_strings,
1567 .phys_id = cxgb4vf_phys_id,
1568 .get_sset_count = cxgb4vf_get_sset_count,
1569 .get_ethtool_stats = cxgb4vf_get_ethtool_stats,
1570 .get_regs_len = cxgb4vf_get_regs_len,
1571 .get_regs = cxgb4vf_get_regs,
1572 .get_wol = cxgb4vf_get_wol,
1573 .set_tso = cxgb4vf_set_tso,
1577 * /sys/kernel/debug/cxgb4vf support code and data.
1578 * ================================================
1582 * Show SGE Queue Set information. We display QPL Queues Sets per line.
1586 static int sge_qinfo_show(struct seq_file *seq, void *v)
1588 struct adapter *adapter = seq->private;
1589 int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
1590 int qs, r = (uintptr_t)v - 1;
1593 seq_putc(seq, '\n');
1595 #define S3(fmt_spec, s, v) \
1597 seq_printf(seq, "%-12s", s); \
1598 for (qs = 0; qs < n; ++qs) \
1599 seq_printf(seq, " %16" fmt_spec, v); \
1600 seq_putc(seq, '\n'); \
1602 #define S(s, v) S3("s", s, v)
1603 #define T(s, v) S3("u", s, txq[qs].v)
1604 #define R(s, v) S3("u", s, rxq[qs].v)
1606 if (r < eth_entries) {
1607 const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
1608 const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
1609 int n = min(QPL, adapter->sge.ethqsets - QPL * r);
1611 S("QType:", "Ethernet");
1613 (rxq[qs].rspq.netdev
1614 ? rxq[qs].rspq.netdev->name
1617 (rxq[qs].rspq.netdev
1618 ? ((struct port_info *)
1619 netdev_priv(rxq[qs].rspq.netdev))->port_id
1621 T("TxQ ID:", q.abs_id);
1622 T("TxQ size:", q.size);
1623 T("TxQ inuse:", q.in_use);
1624 T("TxQ PIdx:", q.pidx);
1625 T("TxQ CIdx:", q.cidx);
1626 R("RspQ ID:", rspq.abs_id);
1627 R("RspQ size:", rspq.size);
1628 R("RspQE size:", rspq.iqe_len);
1629 S3("u", "Intr delay:", qtimer_val(adapter, &rxq[qs].rspq));
1630 S3("u", "Intr pktcnt:",
1631 adapter->sge.counter_val[rxq[qs].rspq.pktcnt_idx]);
1632 R("RspQ CIdx:", rspq.cidx);
1633 R("RspQ Gen:", rspq.gen);
1634 R("FL ID:", fl.abs_id);
1635 R("FL size:", fl.size - MIN_FL_RESID);
1636 R("FL avail:", fl.avail);
1637 R("FL PIdx:", fl.pidx);
1638 R("FL CIdx:", fl.cidx);
1644 const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
1646 seq_printf(seq, "%-12s %16s\n", "QType:", "FW event queue");
1647 seq_printf(seq, "%-12s %16u\n", "RspQ ID:", evtq->abs_id);
1648 seq_printf(seq, "%-12s %16u\n", "Intr delay:",
1649 qtimer_val(adapter, evtq));
1650 seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
1651 adapter->sge.counter_val[evtq->pktcnt_idx]);
1652 seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", evtq->cidx);
1653 seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", evtq->gen);
1654 } else if (r == 1) {
1655 const struct sge_rspq *intrq = &adapter->sge.intrq;
1657 seq_printf(seq, "%-12s %16s\n", "QType:", "Interrupt Queue");
1658 seq_printf(seq, "%-12s %16u\n", "RspQ ID:", intrq->abs_id);
1659 seq_printf(seq, "%-12s %16u\n", "Intr delay:",
1660 qtimer_val(adapter, intrq));
1661 seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
1662 adapter->sge.counter_val[intrq->pktcnt_idx]);
1663 seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", intrq->cidx);
1664 seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", intrq->gen);
1676 * Return the number of "entries" in our "file". We group the multi-Queue
1677 * sections with QPL Queue Sets per "entry". The sections of the output are:
1679 * Ethernet RX/TX Queue Sets
1680 * Firmware Event Queue
1681 * Forwarded Interrupt Queue (if in MSI mode)
1683 static int sge_queue_entries(const struct adapter *adapter)
1685 return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
1686 ((adapter->flags & USING_MSI) != 0);
1689 static void *sge_queue_start(struct seq_file *seq, loff_t *pos)
1691 int entries = sge_queue_entries(seq->private);
1693 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1696 static void sge_queue_stop(struct seq_file *seq, void *v)
1700 static void *sge_queue_next(struct seq_file *seq, void *v, loff_t *pos)
1702 int entries = sge_queue_entries(seq->private);
1705 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1708 static const struct seq_operations sge_qinfo_seq_ops = {
1709 .start = sge_queue_start,
1710 .next = sge_queue_next,
1711 .stop = sge_queue_stop,
1712 .show = sge_qinfo_show
1715 static int sge_qinfo_open(struct inode *inode, struct file *file)
1717 int res = seq_open(file, &sge_qinfo_seq_ops);
1720 struct seq_file *seq = file->private_data;
1721 seq->private = inode->i_private;
1726 static const struct file_operations sge_qinfo_debugfs_fops = {
1727 .owner = THIS_MODULE,
1728 .open = sge_qinfo_open,
1730 .llseek = seq_lseek,
1731 .release = seq_release,
1735 * Show SGE Queue Set statistics. We display QPL Queues Sets per line.
1739 static int sge_qstats_show(struct seq_file *seq, void *v)
1741 struct adapter *adapter = seq->private;
1742 int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
1743 int qs, r = (uintptr_t)v - 1;
1746 seq_putc(seq, '\n');
1748 #define S3(fmt, s, v) \
1750 seq_printf(seq, "%-16s", s); \
1751 for (qs = 0; qs < n; ++qs) \
1752 seq_printf(seq, " %8" fmt, v); \
1753 seq_putc(seq, '\n'); \
1755 #define S(s, v) S3("s", s, v)
1757 #define T3(fmt, s, v) S3(fmt, s, txq[qs].v)
1758 #define T(s, v) T3("lu", s, v)
1760 #define R3(fmt, s, v) S3(fmt, s, rxq[qs].v)
1761 #define R(s, v) R3("lu", s, v)
1763 if (r < eth_entries) {
1764 const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
1765 const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
1766 int n = min(QPL, adapter->sge.ethqsets - QPL * r);
1768 S("QType:", "Ethernet");
1770 (rxq[qs].rspq.netdev
1771 ? rxq[qs].rspq.netdev->name
1773 R3("u", "RspQNullInts:", rspq.unhandled_irqs);
1774 R("RxPackets:", stats.pkts);
1775 R("RxCSO:", stats.rx_cso);
1776 R("VLANxtract:", stats.vlan_ex);
1777 R("LROmerged:", stats.lro_merged);
1778 R("LROpackets:", stats.lro_pkts);
1779 R("RxDrops:", stats.rx_drops);
1781 T("TxCSO:", tx_cso);
1782 T("VLANins:", vlan_ins);
1783 T("TxQFull:", q.stops);
1784 T("TxQRestarts:", q.restarts);
1785 T("TxMapErr:", mapping_err);
1786 R("FLAllocErr:", fl.alloc_failed);
1787 R("FLLrgAlcErr:", fl.large_alloc_failed);
1788 R("FLStarving:", fl.starving);
1794 const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
1796 seq_printf(seq, "%-8s %16s\n", "QType:", "FW event queue");
1797 seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
1798 evtq->unhandled_irqs);
1799 seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", evtq->cidx);
1800 seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", evtq->gen);
1801 } else if (r == 1) {
1802 const struct sge_rspq *intrq = &adapter->sge.intrq;
1804 seq_printf(seq, "%-8s %16s\n", "QType:", "Interrupt Queue");
1805 seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
1806 intrq->unhandled_irqs);
1807 seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", intrq->cidx);
1808 seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", intrq->gen);
1822 * Return the number of "entries" in our "file". We group the multi-Queue
1823 * sections with QPL Queue Sets per "entry". The sections of the output are:
1825 * Ethernet RX/TX Queue Sets
1826 * Firmware Event Queue
1827 * Forwarded Interrupt Queue (if in MSI mode)
1829 static int sge_qstats_entries(const struct adapter *adapter)
1831 return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
1832 ((adapter->flags & USING_MSI) != 0);
1835 static void *sge_qstats_start(struct seq_file *seq, loff_t *pos)
1837 int entries = sge_qstats_entries(seq->private);
1839 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1842 static void sge_qstats_stop(struct seq_file *seq, void *v)
1846 static void *sge_qstats_next(struct seq_file *seq, void *v, loff_t *pos)
1848 int entries = sge_qstats_entries(seq->private);
1851 return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
1854 static const struct seq_operations sge_qstats_seq_ops = {
1855 .start = sge_qstats_start,
1856 .next = sge_qstats_next,
1857 .stop = sge_qstats_stop,
1858 .show = sge_qstats_show
1861 static int sge_qstats_open(struct inode *inode, struct file *file)
1863 int res = seq_open(file, &sge_qstats_seq_ops);
1866 struct seq_file *seq = file->private_data;
1867 seq->private = inode->i_private;
1872 static const struct file_operations sge_qstats_proc_fops = {
1873 .owner = THIS_MODULE,
1874 .open = sge_qstats_open,
1876 .llseek = seq_lseek,
1877 .release = seq_release,
1881 * Show PCI-E SR-IOV Virtual Function Resource Limits.
1883 static int resources_show(struct seq_file *seq, void *v)
1885 struct adapter *adapter = seq->private;
1886 struct vf_resources *vfres = &adapter->params.vfres;
1888 #define S(desc, fmt, var) \
1889 seq_printf(seq, "%-60s " fmt "\n", \
1890 desc " (" #var "):", vfres->var)
1892 S("Virtual Interfaces", "%d", nvi);
1893 S("Egress Queues", "%d", neq);
1894 S("Ethernet Control", "%d", nethctrl);
1895 S("Ingress Queues/w Free Lists/Interrupts", "%d", niqflint);
1896 S("Ingress Queues", "%d", niq);
1897 S("Traffic Class", "%d", tc);
1898 S("Port Access Rights Mask", "%#x", pmask);
1899 S("MAC Address Filters", "%d", nexactf);
1900 S("Firmware Command Read Capabilities", "%#x", r_caps);
1901 S("Firmware Command Write/Execute Capabilities", "%#x", wx_caps);
1908 static int resources_open(struct inode *inode, struct file *file)
1910 return single_open(file, resources_show, inode->i_private);
1913 static const struct file_operations resources_proc_fops = {
1914 .owner = THIS_MODULE,
1915 .open = resources_open,
1917 .llseek = seq_lseek,
1918 .release = single_release,
1922 * Show Virtual Interfaces.
1924 static int interfaces_show(struct seq_file *seq, void *v)
1926 if (v == SEQ_START_TOKEN) {
1927 seq_puts(seq, "Interface Port VIID\n");
1929 struct adapter *adapter = seq->private;
1930 int pidx = (uintptr_t)v - 2;
1931 struct net_device *dev = adapter->port[pidx];
1932 struct port_info *pi = netdev_priv(dev);
1934 seq_printf(seq, "%9s %4d %#5x\n",
1935 dev->name, pi->port_id, pi->viid);
1940 static inline void *interfaces_get_idx(struct adapter *adapter, loff_t pos)
1942 return pos <= adapter->params.nports
1943 ? (void *)(uintptr_t)(pos + 1)
1947 static void *interfaces_start(struct seq_file *seq, loff_t *pos)
1950 ? interfaces_get_idx(seq->private, *pos)
1954 static void *interfaces_next(struct seq_file *seq, void *v, loff_t *pos)
1957 return interfaces_get_idx(seq->private, *pos);
1960 static void interfaces_stop(struct seq_file *seq, void *v)
1964 static const struct seq_operations interfaces_seq_ops = {
1965 .start = interfaces_start,
1966 .next = interfaces_next,
1967 .stop = interfaces_stop,
1968 .show = interfaces_show
1971 static int interfaces_open(struct inode *inode, struct file *file)
1973 int res = seq_open(file, &interfaces_seq_ops);
1976 struct seq_file *seq = file->private_data;
1977 seq->private = inode->i_private;
1982 static const struct file_operations interfaces_proc_fops = {
1983 .owner = THIS_MODULE,
1984 .open = interfaces_open,
1986 .llseek = seq_lseek,
1987 .release = seq_release,
1991 * /sys/kernel/debugfs/cxgb4vf/ files list.
1993 struct cxgb4vf_debugfs_entry {
1994 const char *name; /* name of debugfs node */
1995 mode_t mode; /* file system mode */
1996 const struct file_operations *fops;
1999 static struct cxgb4vf_debugfs_entry debugfs_files[] = {
2000 { "sge_qinfo", S_IRUGO, &sge_qinfo_debugfs_fops },
2001 { "sge_qstats", S_IRUGO, &sge_qstats_proc_fops },
2002 { "resources", S_IRUGO, &resources_proc_fops },
2003 { "interfaces", S_IRUGO, &interfaces_proc_fops },
2007 * Module and device initialization and cleanup code.
2008 * ==================================================
2012 * Set up out /sys/kernel/debug/cxgb4vf sub-nodes. We assume that the
2013 * directory (debugfs_root) has already been set up.
2015 static int __devinit setup_debugfs(struct adapter *adapter)
2019 BUG_ON(adapter->debugfs_root == NULL);
2022 * Debugfs support is best effort.
2024 for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
2025 (void)debugfs_create_file(debugfs_files[i].name,
2026 debugfs_files[i].mode,
2027 adapter->debugfs_root,
2029 debugfs_files[i].fops);
2035 * Tear down the /sys/kernel/debug/cxgb4vf sub-nodes created above. We leave
2036 * it to our caller to tear down the directory (debugfs_root).
2038 static void __devexit cleanup_debugfs(struct adapter *adapter)
2040 BUG_ON(adapter->debugfs_root == NULL);
2043 * Unlike our sister routine cleanup_proc(), we don't need to remove
2044 * individual entries because a call will be made to
2045 * debugfs_remove_recursive(). We just need to clean up any ancillary
2052 * Perform early "adapter" initialization. This is where we discover what
2053 * adapter parameters we're going to be using and initialize basic adapter
2056 static int adap_init0(struct adapter *adapter)
2058 struct vf_resources *vfres = &adapter->params.vfres;
2059 struct sge_params *sge_params = &adapter->params.sge;
2060 struct sge *s = &adapter->sge;
2061 unsigned int ethqsets;
2065 * Wait for the device to become ready before proceeding ...
2067 err = t4vf_wait_dev_ready(adapter);
2069 dev_err(adapter->pdev_dev, "device didn't become ready:"
2075 * Grab basic operational parameters. These will predominantly have
2076 * been set up by the Physical Function Driver or will be hard coded
2077 * into the adapter. We just have to live with them ... Note that
2078 * we _must_ get our VPD parameters before our SGE parameters because
2079 * we need to know the adapter's core clock from the VPD in order to
2080 * properly decode the SGE Timer Values.
2082 err = t4vf_get_dev_params(adapter);
2084 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2085 " device parameters: err=%d\n", err);
2088 err = t4vf_get_vpd_params(adapter);
2090 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2091 " VPD parameters: err=%d\n", err);
2094 err = t4vf_get_sge_params(adapter);
2096 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2097 " SGE parameters: err=%d\n", err);
2100 err = t4vf_get_rss_glb_config(adapter);
2102 dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2103 " RSS parameters: err=%d\n", err);
2106 if (adapter->params.rss.mode !=
2107 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
2108 dev_err(adapter->pdev_dev, "unable to operate with global RSS"
2109 " mode %d\n", adapter->params.rss.mode);
2112 err = t4vf_sge_init(adapter);
2114 dev_err(adapter->pdev_dev, "unable to use adapter parameters:"
2120 * Retrieve our RX interrupt holdoff timer values and counter
2121 * threshold values from the SGE parameters.
2123 s->timer_val[0] = core_ticks_to_us(adapter,
2124 TIMERVALUE0_GET(sge_params->sge_timer_value_0_and_1));
2125 s->timer_val[1] = core_ticks_to_us(adapter,
2126 TIMERVALUE1_GET(sge_params->sge_timer_value_0_and_1));
2127 s->timer_val[2] = core_ticks_to_us(adapter,
2128 TIMERVALUE0_GET(sge_params->sge_timer_value_2_and_3));
2129 s->timer_val[3] = core_ticks_to_us(adapter,
2130 TIMERVALUE1_GET(sge_params->sge_timer_value_2_and_3));
2131 s->timer_val[4] = core_ticks_to_us(adapter,
2132 TIMERVALUE0_GET(sge_params->sge_timer_value_4_and_5));
2133 s->timer_val[5] = core_ticks_to_us(adapter,
2134 TIMERVALUE1_GET(sge_params->sge_timer_value_4_and_5));
2137 THRESHOLD_0_GET(sge_params->sge_ingress_rx_threshold);
2139 THRESHOLD_1_GET(sge_params->sge_ingress_rx_threshold);
2141 THRESHOLD_2_GET(sge_params->sge_ingress_rx_threshold);
2143 THRESHOLD_3_GET(sge_params->sge_ingress_rx_threshold);
2146 * Grab our Virtual Interface resource allocation, extract the
2147 * features that we're interested in and do a bit of sanity testing on
2150 err = t4vf_get_vfres(adapter);
2152 dev_err(adapter->pdev_dev, "unable to get virtual interface"
2153 " resources: err=%d\n", err);
2158 * The number of "ports" which we support is equal to the number of
2159 * Virtual Interfaces with which we've been provisioned.
2161 adapter->params.nports = vfres->nvi;
2162 if (adapter->params.nports > MAX_NPORTS) {
2163 dev_warn(adapter->pdev_dev, "only using %d of %d allowed"
2164 " virtual interfaces\n", MAX_NPORTS,
2165 adapter->params.nports);
2166 adapter->params.nports = MAX_NPORTS;
2170 * We need to reserve a number of the ingress queues with Free List
2171 * and Interrupt capabilities for special interrupt purposes (like
2172 * asynchronous firmware messages, or forwarded interrupts if we're
2173 * using MSI). The rest of the FL/Intr-capable ingress queues will be
2174 * matched up one-for-one with Ethernet/Control egress queues in order
2175 * to form "Queue Sets" which will be aportioned between the "ports".
2176 * For each Queue Set, we'll need the ability to allocate two Egress
2177 * Contexts -- one for the Ingress Queue Free List and one for the TX
2180 ethqsets = vfres->niqflint - INGQ_EXTRAS;
2181 if (vfres->nethctrl != ethqsets) {
2182 dev_warn(adapter->pdev_dev, "unequal number of [available]"
2183 " ingress/egress queues (%d/%d); using minimum for"
2184 " number of Queue Sets\n", ethqsets, vfres->nethctrl);
2185 ethqsets = min(vfres->nethctrl, ethqsets);
2187 if (vfres->neq < ethqsets*2) {
2188 dev_warn(adapter->pdev_dev, "Not enough Egress Contexts (%d)"
2189 " to support Queue Sets (%d); reducing allowed Queue"
2190 " Sets\n", vfres->neq, ethqsets);
2191 ethqsets = vfres->neq/2;
2193 if (ethqsets > MAX_ETH_QSETS) {
2194 dev_warn(adapter->pdev_dev, "only using %d of %d allowed Queue"
2195 " Sets\n", MAX_ETH_QSETS, adapter->sge.max_ethqsets);
2196 ethqsets = MAX_ETH_QSETS;
2198 if (vfres->niq != 0 || vfres->neq > ethqsets*2) {
2199 dev_warn(adapter->pdev_dev, "unused resources niq/neq (%d/%d)"
2200 " ignored\n", vfres->niq, vfres->neq - ethqsets*2);
2202 adapter->sge.max_ethqsets = ethqsets;
2205 * Check for various parameter sanity issues. Most checks simply
2206 * result in us using fewer resources than our provissioning but we
2207 * do need at least one "port" with which to work ...
2209 if (adapter->sge.max_ethqsets < adapter->params.nports) {
2210 dev_warn(adapter->pdev_dev, "only using %d of %d available"
2211 " virtual interfaces (too few Queue Sets)\n",
2212 adapter->sge.max_ethqsets, adapter->params.nports);
2213 adapter->params.nports = adapter->sge.max_ethqsets;
2215 if (adapter->params.nports == 0) {
2216 dev_err(adapter->pdev_dev, "no virtual interfaces configured/"
2223 static inline void init_rspq(struct sge_rspq *rspq, u8 timer_idx,
2224 u8 pkt_cnt_idx, unsigned int size,
2225 unsigned int iqe_size)
2227 rspq->intr_params = (QINTR_TIMER_IDX(timer_idx) |
2228 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0));
2229 rspq->pktcnt_idx = (pkt_cnt_idx < SGE_NCOUNTERS
2232 rspq->iqe_len = iqe_size;
2237 * Perform default configuration of DMA queues depending on the number and
2238 * type of ports we found and the number of available CPUs. Most settings can
2239 * be modified by the admin via ethtool and cxgbtool prior to the adapter
2240 * being brought up for the first time.
2242 static void __devinit cfg_queues(struct adapter *adapter)
2244 struct sge *s = &adapter->sge;
2245 int q10g, n10g, qidx, pidx, qs;
2248 * We should not be called till we know how many Queue Sets we can
2249 * support. In particular, this means that we need to know what kind
2250 * of interrupts we'll be using ...
2252 BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
2255 * Count the number of 10GbE Virtual Interfaces that we have.
2258 for_each_port(adapter, pidx)
2259 n10g += is_10g_port(&adap2pinfo(adapter, pidx)->link_cfg);
2262 * We default to 1 queue per non-10G port and up to # of cores queues
2268 int n1g = (adapter->params.nports - n10g);
2269 q10g = (adapter->sge.max_ethqsets - n1g) / n10g;
2270 if (q10g > num_online_cpus())
2271 q10g = num_online_cpus();
2275 * Allocate the "Queue Sets" to the various Virtual Interfaces.
2276 * The layout will be established in setup_sge_queues() when the
2277 * adapter is brough up for the first time.
2280 for_each_port(adapter, pidx) {
2281 struct port_info *pi = adap2pinfo(adapter, pidx);
2283 pi->first_qset = qidx;
2284 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
2290 * Set up default Queue Set parameters ... Start off with the
2291 * shortest interrupt holdoff timer.
2293 for (qs = 0; qs < s->max_ethqsets; qs++) {
2294 struct sge_eth_rxq *rxq = &s->ethrxq[qs];
2295 struct sge_eth_txq *txq = &s->ethtxq[qs];
2297 init_rspq(&rxq->rspq, 0, 0, 1024, L1_CACHE_BYTES);
2303 * The firmware event queue is used for link state changes and
2304 * notifications of TX DMA completions.
2306 init_rspq(&s->fw_evtq, SGE_TIMER_RSTRT_CNTR, 0, 512,
2310 * The forwarded interrupt queue is used when we're in MSI interrupt
2311 * mode. In this mode all interrupts associated with RX queues will
2312 * be forwarded to a single queue which we'll associate with our MSI
2313 * interrupt vector. The messages dropped in the forwarded interrupt
2314 * queue will indicate which ingress queue needs servicing ... This
2315 * queue needs to be large enough to accommodate all of the ingress
2316 * queues which are forwarding their interrupt (+1 to prevent the PIDX
2317 * from equalling the CIDX if every ingress queue has an outstanding
2318 * interrupt). The queue doesn't need to be any larger because no
2319 * ingress queue will ever have more than one outstanding interrupt at
2322 init_rspq(&s->intrq, SGE_TIMER_RSTRT_CNTR, 0, MSIX_ENTRIES + 1,
2327 * Reduce the number of Ethernet queues across all ports to at most n.
2328 * n provides at least one queue per port.
2330 static void __devinit reduce_ethqs(struct adapter *adapter, int n)
2333 struct port_info *pi;
2336 * While we have too many active Ether Queue Sets, interate across the
2337 * "ports" and reduce their individual Queue Set allocations.
2339 BUG_ON(n < adapter->params.nports);
2340 while (n < adapter->sge.ethqsets)
2341 for_each_port(adapter, i) {
2342 pi = adap2pinfo(adapter, i);
2343 if (pi->nqsets > 1) {
2345 adapter->sge.ethqsets--;
2346 if (adapter->sge.ethqsets <= n)
2352 * Reassign the starting Queue Sets for each of the "ports" ...
2355 for_each_port(adapter, i) {
2356 pi = adap2pinfo(adapter, i);
2363 * We need to grab enough MSI-X vectors to cover our interrupt needs. Ideally
2364 * we get a separate MSI-X vector for every "Queue Set" plus any extras we
2365 * need. Minimally we need one for every Virtual Interface plus those needed
2366 * for our "extras". Note that this process may lower the maximum number of
2367 * allowed Queue Sets ...
2369 static int __devinit enable_msix(struct adapter *adapter)
2371 int i, err, want, need;
2372 struct msix_entry entries[MSIX_ENTRIES];
2373 struct sge *s = &adapter->sge;
2375 for (i = 0; i < MSIX_ENTRIES; ++i)
2376 entries[i].entry = i;
2379 * We _want_ enough MSI-X interrupts to cover all of our "Queue Sets"
2380 * plus those needed for our "extras" (for example, the firmware
2381 * message queue). We _need_ at least one "Queue Set" per Virtual
2382 * Interface plus those needed for our "extras". So now we get to see
2383 * if the song is right ...
2385 want = s->max_ethqsets + MSIX_EXTRAS;
2386 need = adapter->params.nports + MSIX_EXTRAS;
2387 while ((err = pci_enable_msix(adapter->pdev, entries, want)) >= need)
2391 int nqsets = want - MSIX_EXTRAS;
2392 if (nqsets < s->max_ethqsets) {
2393 dev_warn(adapter->pdev_dev, "only enough MSI-X vectors"
2394 " for %d Queue Sets\n", nqsets);
2395 s->max_ethqsets = nqsets;
2396 if (nqsets < s->ethqsets)
2397 reduce_ethqs(adapter, nqsets);
2399 for (i = 0; i < want; ++i)
2400 adapter->msix_info[i].vec = entries[i].vector;
2401 } else if (err > 0) {
2402 pci_disable_msix(adapter->pdev);
2403 dev_info(adapter->pdev_dev, "only %d MSI-X vectors left,"
2404 " not using MSI-X\n", err);
2409 #ifdef HAVE_NET_DEVICE_OPS
2410 static const struct net_device_ops cxgb4vf_netdev_ops = {
2411 .ndo_open = cxgb4vf_open,
2412 .ndo_stop = cxgb4vf_stop,
2413 .ndo_start_xmit = t4vf_eth_xmit,
2414 .ndo_get_stats = cxgb4vf_get_stats,
2415 .ndo_set_rx_mode = cxgb4vf_set_rxmode,
2416 .ndo_set_mac_address = cxgb4vf_set_mac_addr,
2417 .ndo_select_queue = cxgb4vf_select_queue,
2418 .ndo_validate_addr = eth_validate_addr,
2419 .ndo_do_ioctl = cxgb4vf_do_ioctl,
2420 .ndo_change_mtu = cxgb4vf_change_mtu,
2421 .ndo_vlan_rx_register = cxgb4vf_vlan_rx_register,
2422 #ifdef CONFIG_NET_POLL_CONTROLLER
2423 .ndo_poll_controller = cxgb4vf_poll_controller,
2429 * "Probe" a device: initialize a device and construct all kernel and driver
2430 * state needed to manage the device. This routine is called "init_one" in
2433 static int __devinit cxgb4vf_pci_probe(struct pci_dev *pdev,
2434 const struct pci_device_id *ent)
2436 static int version_printed;
2441 struct adapter *adapter;
2442 struct port_info *pi;
2443 struct net_device *netdev;
2446 * Vet our module parameters.
2448 if (msi != MSI_MSIX && msi != MSI_MSI) {
2449 dev_err(&pdev->dev, "bad module parameter msi=%d; must be %d"
2450 " (MSI-X or MSI) or %d (MSI)\n", msi, MSI_MSIX,
2457 * Print our driver banner the first time we're called to initialize a
2460 if (version_printed == 0) {
2461 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2462 version_printed = 1;
2467 * Initialize generic PCI device state.
2469 err = pci_enable_device(pdev);
2471 dev_err(&pdev->dev, "cannot enable PCI device\n");
2476 * Reserve PCI resources for the device. If we can't get them some
2477 * other driver may have already claimed the device ...
2479 err = pci_request_regions(pdev, KBUILD_MODNAME);
2481 dev_err(&pdev->dev, "cannot obtain PCI resources\n");
2482 goto err_disable_device;
2486 * Set up our DMA mask: try for 64-bit address masking first and
2487 * fall back to 32-bit if we can't get 64 bits ...
2489 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2491 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2493 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for"
2494 " coherent allocations\n");
2495 goto err_release_regions;
2499 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2501 dev_err(&pdev->dev, "no usable DMA configuration\n");
2502 goto err_release_regions;
2508 * Enable bus mastering for the device ...
2510 pci_set_master(pdev);
2513 * Allocate our adapter data structure and attach it to the device.
2515 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2518 goto err_release_regions;
2520 pci_set_drvdata(pdev, adapter);
2521 adapter->pdev = pdev;
2522 adapter->pdev_dev = &pdev->dev;
2525 * Initialize SMP data synchronization resources.
2527 spin_lock_init(&adapter->stats_lock);
2530 * Map our I/O registers in BAR0.
2532 adapter->regs = pci_ioremap_bar(pdev, 0);
2533 if (!adapter->regs) {
2534 dev_err(&pdev->dev, "cannot map device registers\n");
2536 goto err_free_adapter;
2540 * Initialize adapter level features.
2542 adapter->name = pci_name(pdev);
2543 adapter->msg_enable = dflt_msg_enable;
2544 err = adap_init0(adapter);
2549 * Allocate our "adapter ports" and stitch everything together.
2551 pmask = adapter->params.vfres.pmask;
2552 for_each_port(adapter, pidx) {
2556 * We simplistically allocate our virtual interfaces
2557 * sequentially across the port numbers to which we have
2558 * access rights. This should be configurable in some manner
2563 port_id = ffs(pmask) - 1;
2564 pmask &= ~(1 << port_id);
2565 viid = t4vf_alloc_vi(adapter, port_id);
2567 dev_err(&pdev->dev, "cannot allocate VI for port %d:"
2568 " err=%d\n", port_id, viid);
2574 * Allocate our network device and stitch things together.
2576 netdev = alloc_etherdev_mq(sizeof(struct port_info),
2578 if (netdev == NULL) {
2579 dev_err(&pdev->dev, "cannot allocate netdev for"
2580 " port %d\n", port_id);
2581 t4vf_free_vi(adapter, viid);
2585 adapter->port[pidx] = netdev;
2586 SET_NETDEV_DEV(netdev, &pdev->dev);
2587 pi = netdev_priv(netdev);
2588 pi->adapter = adapter;
2590 pi->port_id = port_id;
2594 * Initialize the starting state of our "port" and register
2597 pi->xact_addr_filt = -1;
2598 pi->rx_offload = RX_CSO;
2599 netif_carrier_off(netdev);
2600 netif_tx_stop_all_queues(netdev);
2601 netdev->irq = pdev->irq;
2603 netdev->features = (NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
2604 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2605 NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX |
2608 netdev->features |= NETIF_F_HIGHDMA;
2609 netdev->vlan_features =
2611 ~(NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX));
2613 #ifdef HAVE_NET_DEVICE_OPS
2614 netdev->netdev_ops = &cxgb4vf_netdev_ops;
2616 netdev->vlan_rx_register = cxgb4vf_vlan_rx_register;
2617 netdev->open = cxgb4vf_open;
2618 netdev->stop = cxgb4vf_stop;
2619 netdev->hard_start_xmit = t4vf_eth_xmit;
2620 netdev->get_stats = cxgb4vf_get_stats;
2621 netdev->set_rx_mode = cxgb4vf_set_rxmode;
2622 netdev->do_ioctl = cxgb4vf_do_ioctl;
2623 netdev->change_mtu = cxgb4vf_change_mtu;
2624 netdev->set_mac_address = cxgb4vf_set_mac_addr;
2625 netdev->select_queue = cxgb4vf_select_queue;
2626 #ifdef CONFIG_NET_POLL_CONTROLLER
2627 netdev->poll_controller = cxgb4vf_poll_controller;
2630 SET_ETHTOOL_OPS(netdev, &cxgb4vf_ethtool_ops);
2633 * Initialize the hardware/software state for the port.
2635 err = t4vf_port_init(adapter, pidx);
2637 dev_err(&pdev->dev, "cannot initialize port %d\n",
2644 * The "card" is now ready to go. If any errors occur during device
2645 * registration we do not fail the whole "card" but rather proceed
2646 * only with the ports we manage to register successfully. However we
2647 * must register at least one net device.
2649 for_each_port(adapter, pidx) {
2650 netdev = adapter->port[pidx];
2654 err = register_netdev(netdev);
2656 dev_warn(&pdev->dev, "cannot register net device %s,"
2657 " skipping\n", netdev->name);
2661 set_bit(pidx, &adapter->registered_device_map);
2663 if (adapter->registered_device_map == 0) {
2664 dev_err(&pdev->dev, "could not register any net devices\n");
2669 * Set up our debugfs entries.
2671 if (cxgb4vf_debugfs_root) {
2672 adapter->debugfs_root =
2673 debugfs_create_dir(pci_name(pdev),
2674 cxgb4vf_debugfs_root);
2675 if (adapter->debugfs_root == NULL)
2676 dev_warn(&pdev->dev, "could not create debugfs"
2679 setup_debugfs(adapter);
2683 * See what interrupts we'll be using. If we've been configured to
2684 * use MSI-X interrupts, try to enable them but fall back to using
2685 * MSI interrupts if we can't enable MSI-X interrupts. If we can't
2686 * get MSI interrupts we bail with the error.
2688 if (msi == MSI_MSIX && enable_msix(adapter) == 0)
2689 adapter->flags |= USING_MSIX;
2691 err = pci_enable_msi(pdev);
2693 dev_err(&pdev->dev, "Unable to allocate %s interrupts;"
2695 msi == MSI_MSIX ? "MSI-X or MSI" : "MSI", err);
2696 goto err_free_debugfs;
2698 adapter->flags |= USING_MSI;
2702 * Now that we know how many "ports" we have and what their types are,
2703 * and how many Queue Sets we can support, we can configure our queue
2706 cfg_queues(adapter);
2709 * Print a short notice on the existance and configuration of the new
2710 * VF network device ...
2712 for_each_port(adapter, pidx) {
2713 dev_info(adapter->pdev_dev, "%s: Chelsio VF NIC PCIe %s\n",
2714 adapter->port[pidx]->name,
2715 (adapter->flags & USING_MSIX) ? "MSI-X" :
2716 (adapter->flags & USING_MSI) ? "MSI" : "");
2725 * Error recovery and exit code. Unwind state that's been created
2726 * so far and return the error.
2730 if (adapter->debugfs_root) {
2731 cleanup_debugfs(adapter);
2732 debugfs_remove_recursive(adapter->debugfs_root);
2736 for_each_port(adapter, pidx) {
2737 netdev = adapter->port[pidx];
2740 pi = netdev_priv(netdev);
2741 t4vf_free_vi(adapter, pi->viid);
2742 if (test_bit(pidx, &adapter->registered_device_map))
2743 unregister_netdev(netdev);
2744 free_netdev(netdev);
2748 iounmap(adapter->regs);
2752 pci_set_drvdata(pdev, NULL);
2754 err_release_regions:
2755 pci_release_regions(pdev);
2756 pci_set_drvdata(pdev, NULL);
2757 pci_clear_master(pdev);
2760 pci_disable_device(pdev);
2767 * "Remove" a device: tear down all kernel and driver state created in the
2768 * "probe" routine and quiesce the device (disable interrupts, etc.). (Note
2769 * that this is called "remove_one" in the PF Driver.)
2771 static void __devexit cxgb4vf_pci_remove(struct pci_dev *pdev)
2773 struct adapter *adapter = pci_get_drvdata(pdev);
2776 * Tear down driver state associated with device.
2782 * Stop all of our activity. Unregister network port,
2783 * disable interrupts, etc.
2785 for_each_port(adapter, pidx)
2786 if (test_bit(pidx, &adapter->registered_device_map))
2787 unregister_netdev(adapter->port[pidx]);
2788 t4vf_sge_stop(adapter);
2789 if (adapter->flags & USING_MSIX) {
2790 pci_disable_msix(adapter->pdev);
2791 adapter->flags &= ~USING_MSIX;
2792 } else if (adapter->flags & USING_MSI) {
2793 pci_disable_msi(adapter->pdev);
2794 adapter->flags &= ~USING_MSI;
2798 * Tear down our debugfs entries.
2800 if (adapter->debugfs_root) {
2801 cleanup_debugfs(adapter);
2802 debugfs_remove_recursive(adapter->debugfs_root);
2806 * Free all of the various resources which we've acquired ...
2808 t4vf_free_sge_resources(adapter);
2809 for_each_port(adapter, pidx) {
2810 struct net_device *netdev = adapter->port[pidx];
2811 struct port_info *pi;
2816 pi = netdev_priv(netdev);
2817 t4vf_free_vi(adapter, pi->viid);
2818 free_netdev(netdev);
2820 iounmap(adapter->regs);
2822 pci_set_drvdata(pdev, NULL);
2826 * Disable the device and release its PCI resources.
2828 pci_disable_device(pdev);
2829 pci_clear_master(pdev);
2830 pci_release_regions(pdev);
2834 * PCI Device registration data structures.
2836 #define CH_DEVICE(devid, idx) \
2837 { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
2839 static struct pci_device_id cxgb4vf_pci_tbl[] = {
2840 CH_DEVICE(0xb000, 0), /* PE10K FPGA */
2841 CH_DEVICE(0x4800, 0), /* T440-dbg */
2842 CH_DEVICE(0x4801, 0), /* T420-cr */
2843 CH_DEVICE(0x4802, 0), /* T422-cr */
2847 MODULE_DESCRIPTION(DRV_DESC);
2848 MODULE_AUTHOR("Chelsio Communications");
2849 MODULE_LICENSE("Dual BSD/GPL");
2850 MODULE_VERSION(DRV_VERSION);
2851 MODULE_DEVICE_TABLE(pci, cxgb4vf_pci_tbl);
2853 static struct pci_driver cxgb4vf_driver = {
2854 .name = KBUILD_MODNAME,
2855 .id_table = cxgb4vf_pci_tbl,
2856 .probe = cxgb4vf_pci_probe,
2857 .remove = __devexit_p(cxgb4vf_pci_remove),
2861 * Initialize global driver state.
2863 static int __init cxgb4vf_module_init(void)
2867 /* Debugfs support is optional, just warn if this fails */
2868 cxgb4vf_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
2869 if (!cxgb4vf_debugfs_root)
2870 printk(KERN_WARNING KBUILD_MODNAME ": could not create"
2871 " debugfs entry, continuing\n");
2873 ret = pci_register_driver(&cxgb4vf_driver);
2875 debugfs_remove(cxgb4vf_debugfs_root);
2880 * Tear down global driver state.
2882 static void __exit cxgb4vf_module_exit(void)
2884 pci_unregister_driver(&cxgb4vf_driver);
2885 debugfs_remove(cxgb4vf_debugfs_root);
2888 module_init(cxgb4vf_module_init);
2889 module_exit(cxgb4vf_module_exit);