drivers/net/ethernet/aquantia/atlantic/aq_nic.c

   1 /*
   2  * aQuantia Corporation Network Driver
   3  * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  */
   9
  10 /* File aq_nic.c: Definition of common code for NIC. */
  11
  12 #include "aq_nic.h"
  13 #include "aq_ring.h"
  14 #include "aq_vec.h"
  15 #include "aq_hw.h"
  16 #include "aq_pci_func.h"
  17 #include "aq_nic_internal.h"
  18
  19 #include <linux/moduleparam.h>
  20 #include <linux/netdevice.h>
  21 #include <linux/etherdevice.h>
  22 #include <linux/timer.h>
  23 #include <linux/cpu.h>
  24 #include <linux/ip.h>
  25 #include <linux/tcp.h>
  26 #include <net/ip.h>
  27
  28 static unsigned int aq_itr = AQ_CFG_INTERRUPT_MODERATION_AUTO;
  29 module_param_named(aq_itr, aq_itr, uint, 0644);
  30 MODULE_PARM_DESC(aq_itr, "Interrupt throttling mode");
  31
  32 static unsigned int aq_itr_tx;
  33 module_param_named(aq_itr_tx, aq_itr_tx, uint, 0644);
  34 MODULE_PARM_DESC(aq_itr_tx, "TX interrupt throttle rate");
  35
  36 static unsigned int aq_itr_rx;
  37 module_param_named(aq_itr_rx, aq_itr_rx, uint, 0644);
  38 MODULE_PARM_DESC(aq_itr_rx, "RX interrupt throttle rate");
  39
  40 static void aq_nic_update_ndev_stats(struct aq_nic_s *self);
  41
  42 static void aq_nic_rss_init(struct aq_nic_s *self, unsigned int num_rss_queues)
  43 {
  44         struct aq_nic_cfg_s *cfg = &self->aq_nic_cfg;
  45         struct aq_rss_parameters *rss_params = &cfg->aq_rss;
  46         int i = 0;
  47
  48         static u8 rss_key[40] = {
  49                 0x1e, 0xad, 0x71, 0x87, 0x65, 0xfc, 0x26, 0x7d,
  50                 0x0d, 0x45, 0x67, 0x74, 0xcd, 0x06, 0x1a, 0x18,
  51                 0xb6, 0xc1, 0xf0, 0xc7, 0xbb, 0x18, 0xbe, 0xf8,
  52                 0x19, 0x13, 0x4b, 0xa9, 0xd0, 0x3e, 0xfe, 0x70,
  53                 0x25, 0x03, 0xab, 0x50, 0x6a, 0x8b, 0x82, 0x0c
  54         };
  55
  56         rss_params->hash_secret_key_size = sizeof(rss_key);
  57         memcpy(rss_params->hash_secret_key, rss_key, sizeof(rss_key));
  58         rss_params->indirection_table_size = AQ_CFG_RSS_INDIRECTION_TABLE_MAX;
  59
  60         for (i = rss_params->indirection_table_size; i--;)
  61                 rss_params->indirection_table[i] = i & (num_rss_queues - 1);
  62 }
  63
  64 /* Fills aq_nic_cfg with valid defaults */
  65 static void aq_nic_cfg_init_defaults(struct aq_nic_s *self)
  66 {
  67         struct aq_nic_cfg_s *cfg = &self->aq_nic_cfg;
  68
  69         cfg->aq_hw_caps = &self->aq_hw_caps;
  70
  71         cfg->vecs = AQ_CFG_VECS_DEF;
  72         cfg->tcs = AQ_CFG_TCS_DEF;
  73
  74         cfg->rxds = AQ_CFG_RXDS_DEF;
  75         cfg->txds = AQ_CFG_TXDS_DEF;
  76
  77         cfg->is_polling = AQ_CFG_IS_POLLING_DEF;
  78
  79         cfg->itr = aq_itr;
  80         cfg->tx_itr = aq_itr_tx;
  81         cfg->rx_itr = aq_itr_rx;
  82
  83         cfg->is_rss = AQ_CFG_IS_RSS_DEF;
  84         cfg->num_rss_queues = AQ_CFG_NUM_RSS_QUEUES_DEF;
  85         cfg->aq_rss.base_cpu_number = AQ_CFG_RSS_BASE_CPU_NUM_DEF;
  86         cfg->flow_control = AQ_CFG_FC_MODE;
  87
  88         cfg->mtu = AQ_CFG_MTU_DEF;
  89         cfg->link_speed_msk = AQ_CFG_SPEED_MSK;
  90         cfg->is_autoneg = AQ_CFG_IS_AUTONEG_DEF;
  91
  92         cfg->is_lro = AQ_CFG_IS_LRO_DEF;
  93
  94         cfg->vlan_id = 0U;
  95
  96         aq_nic_rss_init(self, cfg->num_rss_queues);
  97 }
  98
  99 /* Checks hw_caps and 'corrects' aq_nic_cfg in runtime */
 100 int aq_nic_cfg_start(struct aq_nic_s *self)
 101 {
 102         struct aq_nic_cfg_s *cfg = &self->aq_nic_cfg;
 103
 104         /*descriptors */
 105         cfg->rxds = min(cfg->rxds, cfg->aq_hw_caps->rxds);
 106         cfg->txds = min(cfg->txds, cfg->aq_hw_caps->txds);
 107
 108         /*rss rings */
 109         cfg->vecs = min(cfg->vecs, cfg->aq_hw_caps->vecs);
 110         cfg->vecs = min(cfg->vecs, num_online_cpus());
 111         /* cfg->vecs should be power of 2 for RSS */
 112         if (cfg->vecs >= 8U)
 113                 cfg->vecs = 8U;
 114         else if (cfg->vecs >= 4U)
 115                 cfg->vecs = 4U;
 116         else if (cfg->vecs >= 2U)
 117                 cfg->vecs = 2U;
 118         else
 119                 cfg->vecs = 1U;
 120
 121         cfg->num_rss_queues = min(cfg->vecs, AQ_CFG_NUM_RSS_QUEUES_DEF);
 122
 123         cfg->irq_type = aq_pci_func_get_irq_type(self->aq_pci_func);
 124
 125         if ((cfg->irq_type == AQ_HW_IRQ_LEGACY) ||
 126             (self->aq_hw_caps.vecs == 1U) ||
 127             (cfg->vecs == 1U)) {
 128                 cfg->is_rss = 0U;
 129                 cfg->vecs = 1U;
 130         }
 131
 132         cfg->link_speed_msk &= self->aq_hw_caps.link_speed_msk;
 133         cfg->hw_features = self->aq_hw_caps.hw_features;
 134         return 0;
 135 }
 136
 137 static int aq_nic_update_link_status(struct aq_nic_s *self)
 138 {
 139         int err = self->aq_hw_ops.hw_get_link_status(self->aq_hw);
 140
 141         if (err)
 142                 return err;
 143
 144         if (self->link_status.mbps != self->aq_hw->aq_link_status.mbps) {
 145                 pr_info("%s: link change old %d new %d\n",
 146                         AQ_CFG_DRV_NAME, self->link_status.mbps,
 147                         self->aq_hw->aq_link_status.mbps);
 148                 aq_nic_update_interrupt_moderation_settings(self);
 149         }
 150
 151         self->link_status = self->aq_hw->aq_link_status;
 152         if (!netif_carrier_ok(self->ndev) && self->link_status.mbps) {
 153                 aq_utils_obj_set(&self->header.flags,
 154                                  AQ_NIC_FLAG_STARTED);
 155                 aq_utils_obj_clear(&self->header.flags,
 156                                    AQ_NIC_LINK_DOWN);
 157                 netif_carrier_on(self->ndev);
 158                 netif_tx_wake_all_queues(self->ndev);
 159         }
 160         if (netif_carrier_ok(self->ndev) && !self->link_status.mbps) {
 161                 netif_carrier_off(self->ndev);
 162                 netif_tx_disable(self->ndev);
 163                 aq_utils_obj_set(&self->header.flags, AQ_NIC_LINK_DOWN);
 164         }
 165         return 0;
 166 }
 167
 168 static void aq_nic_service_timer_cb(struct timer_list *t)
 169 {
 170         struct aq_nic_s *self = from_timer(self, t, service_timer);
 171         int ctimer = AQ_CFG_SERVICE_TIMER_INTERVAL;
 172         int err = 0;
 173
 174         if (aq_utils_obj_test(&self->header.flags, AQ_NIC_FLAGS_IS_NOT_READY))
 175                 goto err_exit;
 176
 177         err = aq_nic_update_link_status(self);
 178         if (err)
 179                 goto err_exit;
 180
 181         if (self->aq_hw_ops.hw_update_stats)
 182                 self->aq_hw_ops.hw_update_stats(self->aq_hw);
 183
 184         aq_nic_update_ndev_stats(self);
 185
 186         /* If no link - use faster timer rate to detect link up asap */
 187         if (!netif_carrier_ok(self->ndev))
 188                 ctimer = max(ctimer / 2, 1);
 189
 190 err_exit:
 191         mod_timer(&self->service_timer, jiffies + ctimer);
 192 }
 193
 194 static void aq_nic_polling_timer_cb(struct timer_list *t)
 195 {
 196         struct aq_nic_s *self = from_timer(self, t, polling_timer);
 197         struct aq_vec_s *aq_vec = NULL;
 198         unsigned int i = 0U;
 199
 200         for (i = 0U, aq_vec = self->aq_vec[0];
 201                 self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i])
 202                 aq_vec_isr(i, (void *)aq_vec);
 203
 204         mod_timer(&self->polling_timer, jiffies +
 205                 AQ_CFG_POLLING_TIMER_INTERVAL);
 206 }
 207
 208 static struct net_device *aq_nic_ndev_alloc(void)
 209 {
 210         return alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_CFG_VECS_MAX);
 211 }
 212
 213 struct aq_nic_s *aq_nic_alloc_cold(const struct net_device_ops *ndev_ops,
 214                                    const struct ethtool_ops *et_ops,
 215                                    struct pci_dev *pdev,
 216                                    struct aq_pci_func_s *aq_pci_func,
 217                                    unsigned int port,
 218                                    const struct aq_hw_ops *aq_hw_ops)
 219 {
 220         struct net_device *ndev = NULL;
 221         struct aq_nic_s *self = NULL;
 222         int err = 0;
 223
 224         ndev = aq_nic_ndev_alloc();
 225         if (!ndev) {
 226                 err = -ENOMEM;
 227                 goto err_exit;
 228         }
 229
 230         self = netdev_priv(ndev);
 231
 232         ndev->netdev_ops = ndev_ops;
 233         ndev->ethtool_ops = et_ops;
 234
 235         SET_NETDEV_DEV(ndev, &pdev->dev);
 236
 237         ndev->if_port = port;
 238         self->ndev = ndev;
 239
 240         self->aq_pci_func = aq_pci_func;
 241
 242         self->aq_hw_ops = *aq_hw_ops;
 243         self->port = (u8)port;
 244
 245         self->aq_hw = self->aq_hw_ops.create(aq_pci_func, self->port,
 246                                                 &self->aq_hw_ops);
 247         err = self->aq_hw_ops.get_hw_caps(self->aq_hw, &self->aq_hw_caps,
 248                                           pdev->device, pdev->subsystem_device);
 249         if (err < 0)
 250                 goto err_exit;
 251
 252         aq_nic_cfg_init_defaults(self);
 253
 254 err_exit:
 255         if (err < 0) {
 256                 aq_nic_free_hot_resources(self);
 257                 self = NULL;
 258         }
 259         return self;
 260 }
 261
 262 int aq_nic_ndev_register(struct aq_nic_s *self)
 263 {
 264         int err = 0;
 265
 266         if (!self->ndev) {
 267                 err = -EINVAL;
 268                 goto err_exit;
 269         }
 270         err = self->aq_hw_ops.hw_get_mac_permanent(self->aq_hw,
 271                             self->aq_nic_cfg.aq_hw_caps,
 272                             self->ndev->dev_addr);
 273         if (err < 0)
 274                 goto err_exit;
 275
 276 #if defined(AQ_CFG_MAC_ADDR_PERMANENT)
 277         {
 278                 static u8 mac_addr_permanent[] = AQ_CFG_MAC_ADDR_PERMANENT;
 279
 280                 ether_addr_copy(self->ndev->dev_addr, mac_addr_permanent);
 281         }
 282 #endif
 283
 284         netif_carrier_off(self->ndev);
 285
 286         netif_tx_disable(self->ndev);
 287
 288         err = register_netdev(self->ndev);
 289         if (err < 0)
 290                 goto err_exit;
 291
 292 err_exit:
 293         return err;
 294 }
 295
 296 int aq_nic_ndev_init(struct aq_nic_s *self)
 297 {
 298         struct aq_hw_caps_s *aq_hw_caps = self->aq_nic_cfg.aq_hw_caps;
 299         struct aq_nic_cfg_s *aq_nic_cfg = &self->aq_nic_cfg;
 300
 301         self->ndev->hw_features |= aq_hw_caps->hw_features;
 302         self->ndev->features = aq_hw_caps->hw_features;
 303         self->ndev->priv_flags = aq_hw_caps->hw_priv_flags;
 304         self->ndev->mtu = aq_nic_cfg->mtu - ETH_HLEN;
 305         self->ndev->max_mtu = self->aq_hw_caps.mtu - ETH_FCS_LEN - ETH_HLEN;
 306
 307         return 0;
 308 }
 309
 310 void aq_nic_ndev_free(struct aq_nic_s *self)
 311 {
 312         if (!self->ndev)
 313                 goto err_exit;
 314
 315         if (self->ndev->reg_state == NETREG_REGISTERED)
 316                 unregister_netdev(self->ndev);
 317
 318         if (self->aq_hw)
 319                 self->aq_hw_ops.destroy(self->aq_hw);
 320
 321         free_netdev(self->ndev);
 322
 323 err_exit:;
 324 }
 325
 326 struct aq_nic_s *aq_nic_alloc_hot(struct net_device *ndev)
 327 {
 328         struct aq_nic_s *self = NULL;
 329         int err = 0;
 330
 331         if (!ndev) {
 332                 err = -EINVAL;
 333                 goto err_exit;
 334         }
 335         self = netdev_priv(ndev);
 336
 337         if (!self) {
 338                 err = -EINVAL;
 339                 goto err_exit;
 340         }
 341         if (netif_running(ndev))
 342                 netif_tx_disable(ndev);
 343         netif_carrier_off(self->ndev);
 344
 345         for (self->aq_vecs = 0; self->aq_vecs < self->aq_nic_cfg.vecs;
 346                 self->aq_vecs++) {
 347                 self->aq_vec[self->aq_vecs] =
 348                     aq_vec_alloc(self, self->aq_vecs, &self->aq_nic_cfg);
 349                 if (!self->aq_vec[self->aq_vecs]) {
 350                         err = -ENOMEM;
 351                         goto err_exit;
 352                 }
 353         }
 354
 355 err_exit:
 356         if (err < 0) {
 357                 aq_nic_free_hot_resources(self);
 358                 self = NULL;
 359         }
 360         return self;
 361 }
 362
 363 void aq_nic_set_tx_ring(struct aq_nic_s *self, unsigned int idx,
 364                         struct aq_ring_s *ring)
 365 {
 366         self->aq_ring_tx[idx] = ring;
 367 }
 368
 369 struct device *aq_nic_get_dev(struct aq_nic_s *self)
 370 {
 371         return self->ndev->dev.parent;
 372 }
 373
 374 struct net_device *aq_nic_get_ndev(struct aq_nic_s *self)
 375 {
 376         return self->ndev;
 377 }
 378
 379 int aq_nic_init(struct aq_nic_s *self)
 380 {
 381         struct aq_vec_s *aq_vec = NULL;
 382         int err = 0;
 383         unsigned int i = 0U;
 384
 385         self->power_state = AQ_HW_POWER_STATE_D0;
 386         err = self->aq_hw_ops.hw_reset(self->aq_hw);
 387         if (err < 0)
 388                 goto err_exit;
 389
 390         err = self->aq_hw_ops.hw_init(self->aq_hw, &self->aq_nic_cfg,
 391                             aq_nic_get_ndev(self)->dev_addr);
 392         if (err < 0)
 393                 goto err_exit;
 394
 395         for (i = 0U, aq_vec = self->aq_vec[0];
 396                 self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i])
 397                 aq_vec_init(aq_vec, &self->aq_hw_ops, self->aq_hw);
 398
 399 err_exit:
 400         return err;
 401 }
 402
 403 int aq_nic_start(struct aq_nic_s *self)
 404 {
 405         struct aq_vec_s *aq_vec = NULL;
 406         int err = 0;
 407         unsigned int i = 0U;
 408
 409         err = self->aq_hw_ops.hw_multicast_list_set(self->aq_hw,
 410                                                     self->mc_list.ar,
 411                                                     self->mc_list.count);
 412         if (err < 0)
 413                 goto err_exit;
 414
 415         err = self->aq_hw_ops.hw_packet_filter_set(self->aq_hw,
 416                                                    self->packet_filter);
 417         if (err < 0)
 418                 goto err_exit;
 419
 420         for (i = 0U, aq_vec = self->aq_vec[0];
 421                 self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
 422                 err = aq_vec_start(aq_vec);
 423                 if (err < 0)
 424                         goto err_exit;
 425         }
 426
 427         err = self->aq_hw_ops.hw_start(self->aq_hw);
 428         if (err < 0)
 429                 goto err_exit;
 430
 431         err = aq_nic_update_interrupt_moderation_settings(self);
 432         if (err)
 433                 goto err_exit;
 434         timer_setup(&self->service_timer, aq_nic_service_timer_cb, 0);
 435         mod_timer(&self->service_timer, jiffies +
 436                         AQ_CFG_SERVICE_TIMER_INTERVAL);
 437
 438         if (self->aq_nic_cfg.is_polling) {
 439                 timer_setup(&self->polling_timer, aq_nic_polling_timer_cb, 0);
 440                 mod_timer(&self->polling_timer, jiffies +
 441                           AQ_CFG_POLLING_TIMER_INTERVAL);
 442         } else {
 443                 for (i = 0U, aq_vec = self->aq_vec[0];
 444                         self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
 445                         err = aq_pci_func_alloc_irq(self->aq_pci_func, i,
 446                                                     self->ndev->name, aq_vec,
 447                                         aq_vec_get_affinity_mask(aq_vec));
 448                         if (err < 0)
 449                                 goto err_exit;
 450                 }
 451
 452                 err = self->aq_hw_ops.hw_irq_enable(self->aq_hw,
 453                                     AQ_CFG_IRQ_MASK);
 454                 if (err < 0)
 455                         goto err_exit;
 456         }
 457
 458         err = netif_set_real_num_tx_queues(self->ndev, self->aq_vecs);
 459         if (err < 0)
 460                 goto err_exit;
 461
 462         err = netif_set_real_num_rx_queues(self->ndev, self->aq_vecs);
 463         if (err < 0)
 464                 goto err_exit;
 465
 466         netif_tx_start_all_queues(self->ndev);
 467
 468 err_exit:
 469         return err;
 470 }
 471
 472 static unsigned int aq_nic_map_skb(struct aq_nic_s *self,
 473                                    struct sk_buff *skb,
 474                                    struct aq_ring_s *ring)
 475 {
 476         unsigned int ret = 0U;
 477         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
 478         unsigned int frag_count = 0U;
 479         unsigned int dx = ring->sw_tail;
 480         struct aq_ring_buff_s *first = NULL;
 481         struct aq_ring_buff_s *dx_buff = &ring->buff_ring[dx];
 482
 483         if (unlikely(skb_is_gso(skb))) {
 484                 dx_buff->flags = 0U;
 485                 dx_buff->len_pkt = skb->len;
 486                 dx_buff->len_l2 = ETH_HLEN;
 487                 dx_buff->len_l3 = ip_hdrlen(skb);
 488                 dx_buff->len_l4 = tcp_hdrlen(skb);
 489                 dx_buff->mss = skb_shinfo(skb)->gso_size;
 490                 dx_buff->is_txc = 1U;
 491                 dx_buff->eop_index = 0xffffU;
 492
 493                 dx_buff->is_ipv6 =
 494                         (ip_hdr(skb)->version == 6) ? 1U : 0U;
 495
 496                 dx = aq_ring_next_dx(ring, dx);
 497                 dx_buff = &ring->buff_ring[dx];
 498                 ++ret;
 499         }
 500
 501         dx_buff->flags = 0U;
 502         dx_buff->len = skb_headlen(skb);
 503         dx_buff->pa = dma_map_single(aq_nic_get_dev(self),
 504                                      skb->data,
 505                                      dx_buff->len,
 506                                      DMA_TO_DEVICE);
 507
 508         if (unlikely(dma_mapping_error(aq_nic_get_dev(self), dx_buff->pa)))
 509                 goto exit;
 510
 511         first = dx_buff;
 512         dx_buff->len_pkt = skb->len;
 513         dx_buff->is_sop = 1U;
 514         dx_buff->is_mapped = 1U;
 515         ++ret;
 516
 517         if (skb->ip_summed == CHECKSUM_PARTIAL) {
 518                 dx_buff->is_ip_cso = (htons(ETH_P_IP) == skb->protocol) ?
 519                         1U : 0U;
 520
 521                 if (ip_hdr(skb)->version == 4) {
 522                         dx_buff->is_tcp_cso =
 523                                 (ip_hdr(skb)->protocol == IPPROTO_TCP) ?
 524                                         1U : 0U;
 525                         dx_buff->is_udp_cso =
 526                                 (ip_hdr(skb)->protocol == IPPROTO_UDP) ?
 527                                         1U : 0U;
 528                 } else if (ip_hdr(skb)->version == 6) {
 529                         dx_buff->is_tcp_cso =
 530                                 (ipv6_hdr(skb)->nexthdr == NEXTHDR_TCP) ?
 531                                         1U : 0U;
 532                         dx_buff->is_udp_cso =
 533                                 (ipv6_hdr(skb)->nexthdr == NEXTHDR_UDP) ?
 534                                         1U : 0U;
 535                 }
 536         }
 537
 538         for (; nr_frags--; ++frag_count) {
 539                 unsigned int frag_len = 0U;
 540                 unsigned int buff_offset = 0U;
 541                 unsigned int buff_size = 0U;
 542                 dma_addr_t frag_pa;
 543                 skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_count];
 544
 545                 frag_len = skb_frag_size(frag);
 546
 547                 while (frag_len) {
 548                         if (frag_len > AQ_CFG_TX_FRAME_MAX)
 549                                 buff_size = AQ_CFG_TX_FRAME_MAX;
 550                         else
 551                                 buff_size = frag_len;
 552
 553                         frag_pa = skb_frag_dma_map(aq_nic_get_dev(self),
 554                                                    frag,
 555                                                    buff_offset,
 556                                                    buff_size,
 557                                                    DMA_TO_DEVICE);
 558
 559                         if (unlikely(dma_mapping_error(aq_nic_get_dev(self),
 560                                                        frag_pa)))
 561                                 goto mapping_error;
 562
 563                         dx = aq_ring_next_dx(ring, dx);
 564                         dx_buff = &ring->buff_ring[dx];
 565
 566                         dx_buff->flags = 0U;
 567                         dx_buff->len = buff_size;
 568                         dx_buff->pa = frag_pa;
 569                         dx_buff->is_mapped = 1U;
 570                         dx_buff->eop_index = 0xffffU;
 571
 572                         frag_len -= buff_size;
 573                         buff_offset += buff_size;
 574
 575                         ++ret;
 576                 }
 577         }
 578
 579         first->eop_index = dx;
 580         dx_buff->is_eop = 1U;
 581         dx_buff->skb = skb;
 582         goto exit;
 583
 584 mapping_error:
 585         for (dx = ring->sw_tail;
 586              ret > 0;
 587              --ret, dx = aq_ring_next_dx(ring, dx)) {
 588                 dx_buff = &ring->buff_ring[dx];
 589
 590                 if (!dx_buff->is_txc && dx_buff->pa) {
 591                         if (unlikely(dx_buff->is_sop)) {
 592                                 dma_unmap_single(aq_nic_get_dev(self),
 593                                                  dx_buff->pa,
 594                                                  dx_buff->len,
 595                                                  DMA_TO_DEVICE);
 596                         } else {
 597                                 dma_unmap_page(aq_nic_get_dev(self),
 598                                                dx_buff->pa,
 599                                                dx_buff->len,
 600                                                DMA_TO_DEVICE);
 601                         }
 602                 }
 603         }
 604
 605 exit:
 606         return ret;
 607 }
 608
 609 int aq_nic_xmit(struct aq_nic_s *self, struct sk_buff *skb)
 610 {
 611         struct aq_ring_s *ring = NULL;
 612         unsigned int frags = 0U;
 613         unsigned int vec = skb->queue_mapping % self->aq_nic_cfg.vecs;
 614         unsigned int tc = 0U;
 615         int err = NETDEV_TX_OK;
 616
 617         frags = skb_shinfo(skb)->nr_frags + 1;
 618
 619         ring = self->aq_ring_tx[AQ_NIC_TCVEC2RING(self, tc, vec)];
 620
 621         if (frags > AQ_CFG_SKB_FRAGS_MAX) {
 622                 dev_kfree_skb_any(skb);
 623                 goto err_exit;
 624         }
 625
 626         aq_ring_update_queue_state(ring);
 627
 628         /* Above status update may stop the queue. Check this. */
 629         if (__netif_subqueue_stopped(self->ndev, ring->idx)) {
 630                 err = NETDEV_TX_BUSY;
 631                 goto err_exit;
 632         }
 633
 634         frags = aq_nic_map_skb(self, skb, ring);
 635
 636         if (likely(frags)) {
 637                 err = self->aq_hw_ops.hw_ring_tx_xmit(self->aq_hw,
 638                                                       ring,
 639                                                       frags);
 640                 if (err >= 0) {
 641                         ++ring->stats.tx.packets;
 642                         ring->stats.tx.bytes += skb->len;
 643                 }
 644         } else {
 645                 err = NETDEV_TX_BUSY;
 646         }
 647
 648 err_exit:
 649         return err;
 650 }
 651
 652 int aq_nic_update_interrupt_moderation_settings(struct aq_nic_s *self)
 653 {
 654         return self->aq_hw_ops.hw_interrupt_moderation_set(self->aq_hw);
 655 }
 656
 657 int aq_nic_set_packet_filter(struct aq_nic_s *self, unsigned int flags)
 658 {
 659         int err = 0;
 660
 661         err = self->aq_hw_ops.hw_packet_filter_set(self->aq_hw, flags);
 662         if (err < 0)
 663                 goto err_exit;
 664
 665         self->packet_filter = flags;
 666
 667 err_exit:
 668         return err;
 669 }
 670
 671 int aq_nic_set_multicast_list(struct aq_nic_s *self, struct net_device *ndev)
 672 {
 673         struct netdev_hw_addr *ha = NULL;
 674         unsigned int i = 0U;
 675
 676         self->mc_list.count = 0U;
 677
 678         netdev_for_each_mc_addr(ha, ndev) {
 679                 ether_addr_copy(self->mc_list.ar[i++], ha->addr);
 680                 ++self->mc_list.count;
 681
 682                 if (i >= AQ_CFG_MULTICAST_ADDRESS_MAX)
 683                         break;
 684         }
 685
 686         if (i >= AQ_CFG_MULTICAST_ADDRESS_MAX) {
 687                 /* Number of filters is too big: atlantic does not support this.
 688                  * Force all multi filter to support this.
 689                  * With this we disable all UC filters and setup "all pass"
 690                  * multicast mask
 691                  */
 692                 self->packet_filter |= IFF_ALLMULTI;
 693                 self->aq_hw->aq_nic_cfg->mc_list_count = 0;
 694                 return self->aq_hw_ops.hw_packet_filter_set(self->aq_hw,
 695                                                         self->packet_filter);
 696         } else {
 697                 return self->aq_hw_ops.hw_multicast_list_set(self->aq_hw,
 698                                                     self->mc_list.ar,
 699                                                     self->mc_list.count);
 700         }
 701 }
 702
 703 int aq_nic_set_mtu(struct aq_nic_s *self, int new_mtu)
 704 {
 705         self->aq_nic_cfg.mtu = new_mtu;
 706
 707         return 0;
 708 }
 709
 710 int aq_nic_set_mac(struct aq_nic_s *self, struct net_device *ndev)
 711 {
 712         return self->aq_hw_ops.hw_set_mac_address(self->aq_hw, ndev->dev_addr);
 713 }
 714
 715 unsigned int aq_nic_get_link_speed(struct aq_nic_s *self)
 716 {
 717         return self->link_status.mbps;
 718 }
 719
 720 int aq_nic_get_regs(struct aq_nic_s *self, struct ethtool_regs *regs, void *p)
 721 {
 722         u32 *regs_buff = p;
 723         int err = 0;
 724
 725         regs->version = 1;
 726
 727         err = self->aq_hw_ops.hw_get_regs(self->aq_hw,
 728                                           &self->aq_hw_caps, regs_buff);
 729         if (err < 0)
 730                 goto err_exit;
 731
 732 err_exit:
 733         return err;
 734 }
 735
 736 int aq_nic_get_regs_count(struct aq_nic_s *self)
 737 {
 738         return self->aq_hw_caps.mac_regs_count;
 739 }
 740
 741 void aq_nic_get_stats(struct aq_nic_s *self, u64 *data)
 742 {
 743         unsigned int i = 0U;
 744         unsigned int count = 0U;
 745         struct aq_vec_s *aq_vec = NULL;
 746         struct aq_stats_s *stats = self->aq_hw_ops.hw_get_hw_stats(self->aq_hw);
 747
 748         if (!stats)
 749                 goto err_exit;
 750
 751         data[i] = stats->uprc + stats->mprc + stats->bprc;
 752         data[++i] = stats->uprc;
 753         data[++i] = stats->mprc;
 754         data[++i] = stats->bprc;
 755         data[++i] = stats->erpt;
 756         data[++i] = stats->uptc + stats->mptc + stats->bptc;
 757         data[++i] = stats->uptc;
 758         data[++i] = stats->mptc;
 759         data[++i] = stats->bptc;
 760         data[++i] = stats->ubrc;
 761         data[++i] = stats->ubtc;
 762         data[++i] = stats->mbrc;
 763         data[++i] = stats->mbtc;
 764         data[++i] = stats->bbrc;
 765         data[++i] = stats->bbtc;
 766         data[++i] = stats->ubrc + stats->mbrc + stats->bbrc;
 767         data[++i] = stats->ubtc + stats->mbtc + stats->bbtc;
 768         data[++i] = stats->dma_pkt_rc;
 769         data[++i] = stats->dma_pkt_tc;
 770         data[++i] = stats->dma_oct_rc;
 771         data[++i] = stats->dma_oct_tc;
 772         data[++i] = stats->dpc;
 773
 774         i++;
 775
 776         data += i;
 777         count = 0U;
 778
 779         for (i = 0U, aq_vec = self->aq_vec[0];
 780                 aq_vec && self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
 781                 data += count;
 782                 aq_vec_get_sw_stats(aq_vec, data, &count);
 783         }
 784
 785 err_exit:;
 786 }
 787
 788 static void aq_nic_update_ndev_stats(struct aq_nic_s *self)
 789 {
 790         struct net_device *ndev = self->ndev;
 791         struct aq_stats_s *stats = self->aq_hw_ops.hw_get_hw_stats(self->aq_hw);
 792
 793         ndev->stats.rx_packets = stats->uprc + stats->mprc + stats->bprc;
 794         ndev->stats.rx_bytes = stats->ubrc + stats->mbrc + stats->bbrc;
 795         ndev->stats.rx_errors = stats->erpr;
 796         ndev->stats.tx_packets = stats->uptc + stats->mptc + stats->bptc;
 797         ndev->stats.tx_bytes = stats->ubtc + stats->mbtc + stats->bbtc;
 798         ndev->stats.tx_errors = stats->erpt;
 799         ndev->stats.multicast = stats->mprc;
 800 }
 801
 802 void aq_nic_get_link_ksettings(struct aq_nic_s *self,
 803                                struct ethtool_link_ksettings *cmd)
 804 {
 805         cmd->base.port = PORT_TP;
 806         /* This driver supports only 10G capable adapters, so DUPLEX_FULL */
 807         cmd->base.duplex = DUPLEX_FULL;
 808         cmd->base.autoneg = self->aq_nic_cfg.is_autoneg;
 809
 810         ethtool_link_ksettings_zero_link_mode(cmd, supported);
 811
 812         if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_10G)
 813                 ethtool_link_ksettings_add_link_mode(cmd, supported,
 814                                                      10000baseT_Full);
 815
 816         if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_5G)
 817                 ethtool_link_ksettings_add_link_mode(cmd, supported,
 818                                                      5000baseT_Full);
 819
 820         if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_2GS)
 821                 ethtool_link_ksettings_add_link_mode(cmd, supported,
 822                                                      2500baseT_Full);
 823
 824         if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_1G)
 825                 ethtool_link_ksettings_add_link_mode(cmd, supported,
 826                                                      1000baseT_Full);
 827
 828         if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_100M)
 829                 ethtool_link_ksettings_add_link_mode(cmd, supported,
 830                                                      100baseT_Full);
 831
 832         if (self->aq_hw_caps.flow_control)
 833                 ethtool_link_ksettings_add_link_mode(cmd, supported,
 834                                                      Pause);
 835
 836         ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg);
 837         ethtool_link_ksettings_add_link_mode(cmd, supported, TP);
 838
 839         ethtool_link_ksettings_zero_link_mode(cmd, advertising);
 840
 841         if (self->aq_nic_cfg.is_autoneg)
 842                 ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg);
 843
 844         if (self->aq_nic_cfg.link_speed_msk  & AQ_NIC_RATE_10G)
 845                 ethtool_link_ksettings_add_link_mode(cmd, advertising,
 846                                                      10000baseT_Full);
 847
 848         if (self->aq_nic_cfg.link_speed_msk  & AQ_NIC_RATE_5G)
 849                 ethtool_link_ksettings_add_link_mode(cmd, advertising,
 850                                                      5000baseT_Full);
 851
 852         if (self->aq_nic_cfg.link_speed_msk  & AQ_NIC_RATE_2GS)
 853                 ethtool_link_ksettings_add_link_mode(cmd, advertising,
 854                                                      2500baseT_Full);
 855
 856         if (self->aq_nic_cfg.link_speed_msk  & AQ_NIC_RATE_1G)
 857                 ethtool_link_ksettings_add_link_mode(cmd, advertising,
 858                                                      1000baseT_Full);
 859
 860         if (self->aq_nic_cfg.link_speed_msk  & AQ_NIC_RATE_100M)
 861                 ethtool_link_ksettings_add_link_mode(cmd, advertising,
 862                                                      100baseT_Full);
 863
 864         if (self->aq_nic_cfg.flow_control)
 865                 ethtool_link_ksettings_add_link_mode(cmd, advertising,
 866                                                      Pause);
 867
 868         ethtool_link_ksettings_add_link_mode(cmd, advertising, TP);
 869 }
 870
 871 int aq_nic_set_link_ksettings(struct aq_nic_s *self,
 872                               const struct ethtool_link_ksettings *cmd)
 873 {
 874         u32 speed = 0U;
 875         u32 rate = 0U;
 876         int err = 0;
 877
 878         if (cmd->base.autoneg == AUTONEG_ENABLE) {
 879                 rate = self->aq_hw_caps.link_speed_msk;
 880                 self->aq_nic_cfg.is_autoneg = true;
 881         } else {
 882                 speed = cmd->base.speed;
 883
 884                 switch (speed) {
 885                 case SPEED_100:
 886                         rate = AQ_NIC_RATE_100M;
 887                         break;
 888
 889                 case SPEED_1000:
 890                         rate = AQ_NIC_RATE_1G;
 891                         break;
 892
 893                 case SPEED_2500:
 894                         rate = AQ_NIC_RATE_2GS;
 895                         break;
 896
 897                 case SPEED_5000:
 898                         rate = AQ_NIC_RATE_5G;
 899                         break;
 900
 901                 case SPEED_10000:
 902                         rate = AQ_NIC_RATE_10G;
 903                         break;
 904
 905                 default:
 906                         err = -1;
 907                         goto err_exit;
 908                 break;
 909                 }
 910                 if (!(self->aq_hw_caps.link_speed_msk & rate)) {
 911                         err = -1;
 912                         goto err_exit;
 913                 }
 914
 915                 self->aq_nic_cfg.is_autoneg = false;
 916         }
 917
 918         err = self->aq_hw_ops.hw_set_link_speed(self->aq_hw, rate);
 919         if (err < 0)
 920                 goto err_exit;
 921
 922         self->aq_nic_cfg.link_speed_msk = rate;
 923
 924 err_exit:
 925         return err;
 926 }
 927
 928 struct aq_nic_cfg_s *aq_nic_get_cfg(struct aq_nic_s *self)
 929 {
 930         return &self->aq_nic_cfg;
 931 }
 932
 933 u32 aq_nic_get_fw_version(struct aq_nic_s *self)
 934 {
 935         u32 fw_version = 0U;
 936
 937         self->aq_hw_ops.hw_get_fw_version(self->aq_hw, &fw_version);
 938
 939         return fw_version;
 940 }
 941
 942 int aq_nic_stop(struct aq_nic_s *self)
 943 {
 944         struct aq_vec_s *aq_vec = NULL;
 945         unsigned int i = 0U;
 946
 947         netif_tx_disable(self->ndev);
 948         netif_carrier_off(self->ndev);
 949
 950         del_timer_sync(&self->service_timer);
 951
 952         self->aq_hw_ops.hw_irq_disable(self->aq_hw, AQ_CFG_IRQ_MASK);
 953
 954         if (self->aq_nic_cfg.is_polling)
 955                 del_timer_sync(&self->polling_timer);
 956         else
 957                 aq_pci_func_free_irqs(self->aq_pci_func);
 958
 959         for (i = 0U, aq_vec = self->aq_vec[0];
 960                 self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i])
 961                 aq_vec_stop(aq_vec);
 962
 963         return self->aq_hw_ops.hw_stop(self->aq_hw);
 964 }
 965
 966 void aq_nic_deinit(struct aq_nic_s *self)
 967 {
 968         struct aq_vec_s *aq_vec = NULL;
 969         unsigned int i = 0U;
 970
 971         if (!self)
 972                 goto err_exit;
 973
 974         for (i = 0U, aq_vec = self->aq_vec[0];
 975                 self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i])
 976                 aq_vec_deinit(aq_vec);
 977
 978         if (self->power_state == AQ_HW_POWER_STATE_D0) {
 979                 (void)self->aq_hw_ops.hw_deinit(self->aq_hw);
 980         } else {
 981                 (void)self->aq_hw_ops.hw_set_power(self->aq_hw,
 982                                                    self->power_state);
 983         }
 984
 985 err_exit:;
 986 }
 987
 988 void aq_nic_free_hot_resources(struct aq_nic_s *self)
 989 {
 990         unsigned int i = 0U;
 991
 992         if (!self)
 993                 goto err_exit;
 994
 995         for (i = AQ_DIMOF(self->aq_vec); i--;) {
 996                 if (self->aq_vec[i]) {
 997                         aq_vec_free(self->aq_vec[i]);
 998                         self->aq_vec[i] = NULL;
 999                 }
1000         }
1001
1002 err_exit:;
1003 }
1004
1005 int aq_nic_change_pm_state(struct aq_nic_s *self, pm_message_t *pm_msg)
1006 {
1007         int err = 0;
1008
1009         if (!netif_running(self->ndev)) {
1010                 err = 0;
1011                 goto out;
1012         }
1013         rtnl_lock();
1014         if (pm_msg->event & PM_EVENT_SLEEP || pm_msg->event & PM_EVENT_FREEZE) {
1015                 self->power_state = AQ_HW_POWER_STATE_D3;
1016                 netif_device_detach(self->ndev);
1017                 netif_tx_stop_all_queues(self->ndev);
1018
1019                 err = aq_nic_stop(self);
1020                 if (err < 0)
1021                         goto err_exit;
1022
1023                 aq_nic_deinit(self);
1024         } else {
1025                 err = aq_nic_init(self);
1026                 if (err < 0)
1027                         goto err_exit;
1028
1029                 err = aq_nic_start(self);
1030                 if (err < 0)
1031                         goto err_exit;
1032
1033                 netif_device_attach(self->ndev);
1034                 netif_tx_start_all_queues(self->ndev);
1035         }
1036
1037 err_exit:
1038         rtnl_unlock();
1039 out:
1040         return err;
1041 }