drivers/net/ethernet/chelsio/cxgb4/t4_hw.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #include <linux/delay.h>
  36 #include "cxgb4.h"
  37 #include "t4_regs.h"
  38 #include "t4_values.h"
  39 #include "t4fw_api.h"
  40 #include "t4fw_version.h"
  41
  42 /**
  43  *      t4_wait_op_done_val - wait until an operation is completed
  44  *      @adapter: the adapter performing the operation
  45  *      @reg: the register to check for completion
  46  *      @mask: a single-bit field within @reg that indicates completion
  47  *      @polarity: the value of the field when the operation is completed
  48  *      @attempts: number of check iterations
  49  *      @delay: delay in usecs between iterations
  50  *      @valp: where to store the value of the register at completion time
  51  *
  52  *      Wait until an operation is completed by checking a bit in a register
  53  *      up to @attempts times.  If @valp is not NULL the value of the register
  54  *      at the time it indicated completion is stored there.  Returns 0 if the
  55  *      operation completes and -EAGAIN otherwise.
  56  */
  57 static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
  58                                int polarity, int attempts, int delay, u32 *valp)
  59 {
  60         while (1) {
  61                 u32 val = t4_read_reg(adapter, reg);
  62
  63                 if (!!(val & mask) == polarity) {
  64                         if (valp)
  65                                 *valp = val;
  66                         return 0;
  67                 }
  68                 if (--attempts == 0)
  69                         return -EAGAIN;
  70                 if (delay)
  71                         udelay(delay);
  72         }
  73 }
  74
  75 static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
  76                                   int polarity, int attempts, int delay)
  77 {
  78         return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
  79                                    delay, NULL);
  80 }
  81
  82 /**
  83  *      t4_set_reg_field - set a register field to a value
  84  *      @adapter: the adapter to program
  85  *      @addr: the register address
  86  *      @mask: specifies the portion of the register to modify
  87  *      @val: the new value for the register field
  88  *
  89  *      Sets a register field specified by the supplied mask to the
  90  *      given value.
  91  */
  92 void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
  93                       u32 val)
  94 {
  95         u32 v = t4_read_reg(adapter, addr) & ~mask;
  96
  97         t4_write_reg(adapter, addr, v | val);
  98         (void) t4_read_reg(adapter, addr);      /* flush */
  99 }
 100
 101 /**
 102  *      t4_read_indirect - read indirectly addressed registers
 103  *      @adap: the adapter
 104  *      @addr_reg: register holding the indirect address
 105  *      @data_reg: register holding the value of the indirect register
 106  *      @vals: where the read register values are stored
 107  *      @nregs: how many indirect registers to read
 108  *      @start_idx: index of first indirect register to read
 109  *
 110  *      Reads registers that are accessed indirectly through an address/data
 111  *      register pair.
 112  */
 113 void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
 114                              unsigned int data_reg, u32 *vals,
 115                              unsigned int nregs, unsigned int start_idx)
 116 {
 117         while (nregs--) {
 118                 t4_write_reg(adap, addr_reg, start_idx);
 119                 *vals++ = t4_read_reg(adap, data_reg);
 120                 start_idx++;
 121         }
 122 }
 123
 124 /**
 125  *      t4_write_indirect - write indirectly addressed registers
 126  *      @adap: the adapter
 127  *      @addr_reg: register holding the indirect addresses
 128  *      @data_reg: register holding the value for the indirect registers
 129  *      @vals: values to write
 130  *      @nregs: how many indirect registers to write
 131  *      @start_idx: address of first indirect register to write
 132  *
 133  *      Writes a sequential block of registers that are accessed indirectly
 134  *      through an address/data register pair.
 135  */
 136 void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 137                        unsigned int data_reg, const u32 *vals,
 138                        unsigned int nregs, unsigned int start_idx)
 139 {
 140         while (nregs--) {
 141                 t4_write_reg(adap, addr_reg, start_idx++);
 142                 t4_write_reg(adap, data_reg, *vals++);
 143         }
 144 }
 145
 146 /*
 147  * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
 148  * mechanism.  This guarantees that we get the real value even if we're
 149  * operating within a Virtual Machine and the Hypervisor is trapping our
 150  * Configuration Space accesses.
 151  */
 152 void t4_hw_pci_read_cfg4(struct adapter *adap, int reg, u32 *val)
 153 {
 154         u32 req = FUNCTION_V(adap->pf) | REGISTER_V(reg);
 155
 156         if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
 157                 req |= ENABLE_F;
 158         else
 159                 req |= T6_ENABLE_F;
 160
 161         if (is_t4(adap->params.chip))
 162                 req |= LOCALCFG_F;
 163
 164         t4_write_reg(adap, PCIE_CFG_SPACE_REQ_A, req);
 165         *val = t4_read_reg(adap, PCIE_CFG_SPACE_DATA_A);
 166
 167         /* Reset ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a
 168          * Configuration Space read.  (None of the other fields matter when
 169          * ENABLE is 0 so a simple register write is easier than a
 170          * read-modify-write via t4_set_reg_field().)
 171          */
 172         t4_write_reg(adap, PCIE_CFG_SPACE_REQ_A, 0);
 173 }
 174
 175 /*
 176  * t4_report_fw_error - report firmware error
 177  * @adap: the adapter
 178  *
 179  * The adapter firmware can indicate error conditions to the host.
 180  * If the firmware has indicated an error, print out the reason for
 181  * the firmware error.
 182  */
 183 static void t4_report_fw_error(struct adapter *adap)
 184 {
 185         static const char *const reason[] = {
 186                 "Crash",                        /* PCIE_FW_EVAL_CRASH */
 187                 "During Device Preparation",    /* PCIE_FW_EVAL_PREP */
 188                 "During Device Configuration",  /* PCIE_FW_EVAL_CONF */
 189                 "During Device Initialization", /* PCIE_FW_EVAL_INIT */
 190                 "Unexpected Event",             /* PCIE_FW_EVAL_UNEXPECTEDEVENT */
 191                 "Insufficient Airflow",         /* PCIE_FW_EVAL_OVERHEAT */
 192                 "Device Shutdown",              /* PCIE_FW_EVAL_DEVICESHUTDOWN */
 193                 "Reserved",                     /* reserved */
 194         };
 195         u32 pcie_fw;
 196
 197         pcie_fw = t4_read_reg(adap, PCIE_FW_A);
 198         if (pcie_fw & PCIE_FW_ERR_F) {
 199                 dev_err(adap->pdev_dev, "Firmware reports adapter error: %s\n",
 200                         reason[PCIE_FW_EVAL_G(pcie_fw)]);
 201                 adap->flags &= ~FW_OK;
 202         }
 203 }
 204
 205 /*
 206  * Get the reply to a mailbox command and store it in @rpl in big-endian order.
 207  */
 208 static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
 209                          u32 mbox_addr)
 210 {
 211         for ( ; nflit; nflit--, mbox_addr += 8)
 212                 *rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
 213 }
 214
 215 /*
 216  * Handle a FW assertion reported in a mailbox.
 217  */
 218 static void fw_asrt(struct adapter *adap, u32 mbox_addr)
 219 {
 220         struct fw_debug_cmd asrt;
 221
 222         get_mbox_rpl(adap, (__be64 *)&asrt, sizeof(asrt) / 8, mbox_addr);
 223         dev_alert(adap->pdev_dev,
 224                   "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
 225                   asrt.u.assert.filename_0_7, be32_to_cpu(asrt.u.assert.line),
 226                   be32_to_cpu(asrt.u.assert.x), be32_to_cpu(asrt.u.assert.y));
 227 }
 228
 229 /**
 230  *      t4_record_mbox - record a Firmware Mailbox Command/Reply in the log
 231  *      @adapter: the adapter
 232  *      @cmd: the Firmware Mailbox Command or Reply
 233  *      @size: command length in bytes
 234  *      @access: the time (ms) needed to access the Firmware Mailbox
 235  *      @execute: the time (ms) the command spent being executed
 236  */
 237 static void t4_record_mbox(struct adapter *adapter,
 238                            const __be64 *cmd, unsigned int size,
 239                            int access, int execute)
 240 {
 241         struct mbox_cmd_log *log = adapter->mbox_log;
 242         struct mbox_cmd *entry;
 243         int i;
 244
 245         entry = mbox_cmd_log_entry(log, log->cursor++);
 246         if (log->cursor == log->size)
 247                 log->cursor = 0;
 248
 249         for (i = 0; i < size / 8; i++)
 250                 entry->cmd[i] = be64_to_cpu(cmd[i]);
 251         while (i < MBOX_LEN / 8)
 252                 entry->cmd[i++] = 0;
 253         entry->timestamp = jiffies;
 254         entry->seqno = log->seqno++;
 255         entry->access = access;
 256         entry->execute = execute;
 257 }
 258
 259 /**
 260  *      t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox
 261  *      @adap: the adapter
 262  *      @mbox: index of the mailbox to use
 263  *      @cmd: the command to write
 264  *      @size: command length in bytes
 265  *      @rpl: where to optionally store the reply
 266  *      @sleep_ok: if true we may sleep while awaiting command completion
 267  *      @timeout: time to wait for command to finish before timing out
 268  *
 269  *      Sends the given command to FW through the selected mailbox and waits
 270  *      for the FW to execute the command.  If @rpl is not %NULL it is used to
 271  *      store the FW's reply to the command.  The command and its optional
 272  *      reply are of the same length.  FW can take up to %FW_CMD_MAX_TIMEOUT ms
 273  *      to respond.  @sleep_ok determines whether we may sleep while awaiting
 274  *      the response.  If sleeping is allowed we use progressive backoff
 275  *      otherwise we spin.
 276  *
 277  *      The return value is 0 on success or a negative errno on failure.  A
 278  *      failure can happen either because we are not able to execute the
 279  *      command or FW executes it but signals an error.  In the latter case
 280  *      the return value is the error code indicated by FW (negated).
 281  */
 282 int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 283                             int size, void *rpl, bool sleep_ok, int timeout)
 284 {
 285         static const int delay[] = {
 286                 1, 1, 3, 5, 10, 10, 20, 50, 100, 200
 287         };
 288
 289         struct mbox_list entry;
 290         u16 access = 0;
 291         u16 execute = 0;
 292         u32 v;
 293         u64 res;
 294         int i, ms, delay_idx, ret;
 295         const __be64 *p = cmd;
 296         u32 data_reg = PF_REG(mbox, CIM_PF_MAILBOX_DATA_A);
 297         u32 ctl_reg = PF_REG(mbox, CIM_PF_MAILBOX_CTRL_A);
 298         __be64 cmd_rpl[MBOX_LEN / 8];
 299         u32 pcie_fw;
 300
 301         if ((size & 15) || size > MBOX_LEN)
 302                 return -EINVAL;
 303
 304         /*
 305          * If the device is off-line, as in EEH, commands will time out.
 306          * Fail them early so we don't waste time waiting.
 307          */
 308         if (adap->pdev->error_state != pci_channel_io_normal)
 309                 return -EIO;
 310
 311         /* If we have a negative timeout, that implies that we can't sleep. */
 312         if (timeout < 0) {
 313                 sleep_ok = false;
 314                 timeout = -timeout;
 315         }
 316
 317         /* Queue ourselves onto the mailbox access list.  When our entry is at
 318          * the front of the list, we have rights to access the mailbox.  So we
 319          * wait [for a while] till we're at the front [or bail out with an
 320          * EBUSY] ...
 321          */
 322         spin_lock_bh(&adap->mbox_lock);
 323         list_add_tail(&entry.list, &adap->mlist.list);
 324         spin_unlock_bh(&adap->mbox_lock);
 325
 326         delay_idx = 0;
 327         ms = delay[0];
 328
 329         for (i = 0; ; i += ms) {
 330                 /* If we've waited too long, return a busy indication.  This
 331                  * really ought to be based on our initial position in the
 332                  * mailbox access list but this is a start.  We very rearely
 333                  * contend on access to the mailbox ...
 334                  */
 335                 pcie_fw = t4_read_reg(adap, PCIE_FW_A);
 336                 if (i > FW_CMD_MAX_TIMEOUT || (pcie_fw & PCIE_FW_ERR_F)) {
 337                         spin_lock_bh(&adap->mbox_lock);
 338                         list_del(&entry.list);
 339                         spin_unlock_bh(&adap->mbox_lock);
 340                         ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -EBUSY;
 341                         t4_record_mbox(adap, cmd, size, access, ret);
 342                         return ret;
 343                 }
 344
 345                 /* If we're at the head, break out and start the mailbox
 346                  * protocol.
 347                  */
 348                 if (list_first_entry(&adap->mlist.list, struct mbox_list,
 349                                      list) == &entry)
 350                         break;
 351
 352                 /* Delay for a bit before checking again ... */
 353                 if (sleep_ok) {
 354                         ms = delay[delay_idx];  /* last element may repeat */
 355                         if (delay_idx < ARRAY_SIZE(delay) - 1)
 356                                 delay_idx++;
 357                         msleep(ms);
 358                 } else {
 359                         mdelay(ms);
 360                 }
 361         }
 362
 363         /* Loop trying to get ownership of the mailbox.  Return an error
 364          * if we can't gain ownership.
 365          */
 366         v = MBOWNER_G(t4_read_reg(adap, ctl_reg));
 367         for (i = 0; v == MBOX_OWNER_NONE && i < 3; i++)
 368                 v = MBOWNER_G(t4_read_reg(adap, ctl_reg));
 369         if (v != MBOX_OWNER_DRV) {
 370                 spin_lock_bh(&adap->mbox_lock);
 371                 list_del(&entry.list);
 372                 spin_unlock_bh(&adap->mbox_lock);
 373                 ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT;
 374                 t4_record_mbox(adap, cmd, size, access, ret);
 375                 return ret;
 376         }
 377
 378         /* Copy in the new mailbox command and send it on its way ... */
 379         t4_record_mbox(adap, cmd, size, access, 0);
 380         for (i = 0; i < size; i += 8)
 381                 t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p++));
 382
 383         t4_write_reg(adap, ctl_reg, MBMSGVALID_F | MBOWNER_V(MBOX_OWNER_FW));
 384         t4_read_reg(adap, ctl_reg);          /* flush write */
 385
 386         delay_idx = 0;
 387         ms = delay[0];
 388
 389         for (i = 0;
 390              !((pcie_fw = t4_read_reg(adap, PCIE_FW_A)) & PCIE_FW_ERR_F) &&
 391              i < timeout;
 392              i += ms) {
 393                 if (sleep_ok) {
 394                         ms = delay[delay_idx];  /* last element may repeat */
 395                         if (delay_idx < ARRAY_SIZE(delay) - 1)
 396                                 delay_idx++;
 397                         msleep(ms);
 398                 } else
 399                         mdelay(ms);
 400
 401                 v = t4_read_reg(adap, ctl_reg);
 402                 if (MBOWNER_G(v) == MBOX_OWNER_DRV) {
 403                         if (!(v & MBMSGVALID_F)) {
 404                                 t4_write_reg(adap, ctl_reg, 0);
 405                                 continue;
 406                         }
 407
 408                         get_mbox_rpl(adap, cmd_rpl, MBOX_LEN / 8, data_reg);
 409                         res = be64_to_cpu(cmd_rpl[0]);
 410
 411                         if (FW_CMD_OP_G(res >> 32) == FW_DEBUG_CMD) {
 412                                 fw_asrt(adap, data_reg);
 413                                 res = FW_CMD_RETVAL_V(EIO);
 414                         } else if (rpl) {
 415                                 memcpy(rpl, cmd_rpl, size);
 416                         }
 417
 418                         t4_write_reg(adap, ctl_reg, 0);
 419
 420                         execute = i + ms;
 421                         t4_record_mbox(adap, cmd_rpl,
 422                                        MBOX_LEN, access, execute);
 423                         spin_lock_bh(&adap->mbox_lock);
 424                         list_del(&entry.list);
 425                         spin_unlock_bh(&adap->mbox_lock);
 426                         return -FW_CMD_RETVAL_G((int)res);
 427                 }
 428         }
 429
 430         ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -ETIMEDOUT;
 431         t4_record_mbox(adap, cmd, size, access, ret);
 432         dev_err(adap->pdev_dev, "command %#x in mailbox %d timed out\n",
 433                 *(const u8 *)cmd, mbox);
 434         t4_report_fw_error(adap);
 435         spin_lock_bh(&adap->mbox_lock);
 436         list_del(&entry.list);
 437         spin_unlock_bh(&adap->mbox_lock);
 438         t4_fatal_err(adap);
 439         return ret;
 440 }
 441
 442 int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
 443                     void *rpl, bool sleep_ok)
 444 {
 445         return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl, sleep_ok,
 446                                        FW_CMD_MAX_TIMEOUT);
 447 }
 448
 449 static int t4_edc_err_read(struct adapter *adap, int idx)
 450 {
 451         u32 edc_ecc_err_addr_reg;
 452         u32 rdata_reg;
 453
 454         if (is_t4(adap->params.chip)) {
 455                 CH_WARN(adap, "%s: T4 NOT supported.\n", __func__);
 456                 return 0;
 457         }
 458         if (idx != 0 && idx != 1) {
 459                 CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx);
 460                 return 0;
 461         }
 462
 463         edc_ecc_err_addr_reg = EDC_T5_REG(EDC_H_ECC_ERR_ADDR_A, idx);
 464         rdata_reg = EDC_T5_REG(EDC_H_BIST_STATUS_RDATA_A, idx);
 465
 466         CH_WARN(adap,
 467                 "edc%d err addr 0x%x: 0x%x.\n",
 468                 idx, edc_ecc_err_addr_reg,
 469                 t4_read_reg(adap, edc_ecc_err_addr_reg));
 470         CH_WARN(adap,
 471                 "bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
 472                 rdata_reg,
 473                 (unsigned long long)t4_read_reg64(adap, rdata_reg),
 474                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 8),
 475                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 16),
 476                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 24),
 477                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 32),
 478                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 40),
 479                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 48),
 480                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 56),
 481                 (unsigned long long)t4_read_reg64(adap, rdata_reg + 64));
 482
 483         return 0;
 484 }
 485
 486 /**
 487  * t4_memory_rw_init - Get memory window relative offset, base, and size.
 488  * @adap: the adapter
 489  * @win: PCI-E Memory Window to use
 490  * @mtype: memory type: MEM_EDC0, MEM_EDC1, MEM_HMA or MEM_MC
 491  * @mem_off: memory relative offset with respect to @mtype.
 492  * @mem_base: configured memory base address.
 493  * @mem_aperture: configured memory window aperture.
 494  *
 495  * Get the configured memory window's relative offset, base, and size.
 496  */
 497 int t4_memory_rw_init(struct adapter *adap, int win, int mtype, u32 *mem_off,
 498                       u32 *mem_base, u32 *mem_aperture)
 499 {
 500         u32 edc_size, mc_size, mem_reg;
 501
 502         /* Offset into the region of memory which is being accessed
 503          * MEM_EDC0 = 0
 504          * MEM_EDC1 = 1
 505          * MEM_MC   = 2 -- MEM_MC for chips with only 1 memory controller
 506          * MEM_MC1  = 3 -- for chips with 2 memory controllers (e.g. T5)
 507          * MEM_HMA  = 4
 508          */
 509         edc_size  = EDRAM0_SIZE_G(t4_read_reg(adap, MA_EDRAM0_BAR_A));
 510         if (mtype == MEM_HMA) {
 511                 *mem_off = 2 * (edc_size * 1024 * 1024);
 512         } else if (mtype != MEM_MC1) {
 513                 *mem_off = (mtype * (edc_size * 1024 * 1024));
 514         } else {
 515                 mc_size = EXT_MEM0_SIZE_G(t4_read_reg(adap,
 516                                                       MA_EXT_MEMORY0_BAR_A));
 517                 *mem_off = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024;
 518         }
 519
 520         /* Each PCI-E Memory Window is programmed with a window size -- or
 521          * "aperture" -- which controls the granularity of its mapping onto
 522          * adapter memory.  We need to grab that aperture in order to know
 523          * how to use the specified window.  The window is also programmed
 524          * with the base address of the Memory Window in BAR0's address
 525          * space.  For T4 this is an absolute PCI-E Bus Address.  For T5
 526          * the address is relative to BAR0.
 527          */
 528         mem_reg = t4_read_reg(adap,
 529                               PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A,
 530                                                   win));
 531         /* a dead adapter will return 0xffffffff for PIO reads */
 532         if (mem_reg == 0xffffffff)
 533                 return -ENXIO;
 534
 535         *mem_aperture = 1 << (WINDOW_G(mem_reg) + WINDOW_SHIFT_X);
 536         *mem_base = PCIEOFST_G(mem_reg) << PCIEOFST_SHIFT_X;
 537         if (is_t4(adap->params.chip))
 538                 *mem_base -= adap->t4_bar0;
 539
 540         return 0;
 541 }
 542
 543 /**
 544  * t4_memory_update_win - Move memory window to specified address.
 545  * @adap: the adapter
 546  * @win: PCI-E Memory Window to use
 547  * @addr: location to move.
 548  *
 549  * Move memory window to specified address.
 550  */
 551 void t4_memory_update_win(struct adapter *adap, int win, u32 addr)
 552 {
 553         t4_write_reg(adap,
 554                      PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, win),
 555                      addr);
 556         /* Read it back to ensure that changes propagate before we
 557          * attempt to use the new value.
 558          */
 559         t4_read_reg(adap,
 560                     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, win));
 561 }
 562
 563 /**
 564  * t4_memory_rw_residual - Read/Write residual data.
 565  * @adap: the adapter
 566  * @off: relative offset within residual to start read/write.
 567  * @addr: address within indicated memory type.
 568  * @buf: host memory buffer
 569  * @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
 570  *
 571  * Read/Write residual data less than 32-bits.
 572  */
 573 void t4_memory_rw_residual(struct adapter *adap, u32 off, u32 addr, u8 *buf,
 574                            int dir)
 575 {
 576         union {
 577                 u32 word;
 578                 char byte[4];
 579         } last;
 580         unsigned char *bp;
 581         int i;
 582
 583         if (dir == T4_MEMORY_READ) {
 584                 last.word = le32_to_cpu((__force __le32)
 585                                         t4_read_reg(adap, addr));
 586                 for (bp = (unsigned char *)buf, i = off; i < 4; i++)
 587                         bp[i] = last.byte[i];
 588         } else {
 589                 last.word = *buf;
 590                 for (i = off; i < 4; i++)
 591                         last.byte[i] = 0;
 592                 t4_write_reg(adap, addr,
 593                              (__force u32)cpu_to_le32(last.word));
 594         }
 595 }
 596
 597 /**
 598  *      t4_memory_rw - read/write EDC 0, EDC 1 or MC via PCIE memory window
 599  *      @adap: the adapter
 600  *      @win: PCI-E Memory Window to use
 601  *      @mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC
 602  *      @addr: address within indicated memory type
 603  *      @len: amount of memory to transfer
 604  *      @hbuf: host memory buffer
 605  *      @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
 606  *
 607  *      Reads/writes an [almost] arbitrary memory region in the firmware: the
 608  *      firmware memory address and host buffer must be aligned on 32-bit
 609  *      boudaries; the length may be arbitrary.  The memory is transferred as
 610  *      a raw byte sequence from/to the firmware's memory.  If this memory
 611  *      contains data structures which contain multi-byte integers, it's the
 612  *      caller's responsibility to perform appropriate byte order conversions.
 613  */
 614 int t4_memory_rw(struct adapter *adap, int win, int mtype, u32 addr,
 615                  u32 len, void *hbuf, int dir)
 616 {
 617         u32 pos, offset, resid, memoffset;
 618         u32 win_pf, mem_aperture, mem_base;
 619         u32 *buf;
 620         int ret;
 621
 622         /* Argument sanity checks ...
 623          */
 624         if (addr & 0x3 || (uintptr_t)hbuf & 0x3)
 625                 return -EINVAL;
 626         buf = (u32 *)hbuf;
 627
 628         /* It's convenient to be able to handle lengths which aren't a
 629          * multiple of 32-bits because we often end up transferring files to
 630          * the firmware.  So we'll handle that by normalizing the length here
 631          * and then handling any residual transfer at the end.
 632          */
 633         resid = len & 0x3;
 634         len -= resid;
 635
 636         ret = t4_memory_rw_init(adap, win, mtype, &memoffset, &mem_base,
 637                                 &mem_aperture);
 638         if (ret)
 639                 return ret;
 640
 641         /* Determine the PCIE_MEM_ACCESS_OFFSET */
 642         addr = addr + memoffset;
 643
 644         win_pf = is_t4(adap->params.chip) ? 0 : PFNUM_V(adap->pf);
 645
 646         /* Calculate our initial PCI-E Memory Window Position and Offset into
 647          * that Window.
 648          */
 649         pos = addr & ~(mem_aperture - 1);
 650         offset = addr - pos;
 651
 652         /* Set up initial PCI-E Memory Window to cover the start of our
 653          * transfer.
 654          */
 655         t4_memory_update_win(adap, win, pos | win_pf);
 656
 657         /* Transfer data to/from the adapter as long as there's an integral
 658          * number of 32-bit transfers to complete.
 659          *
 660          * A note on Endianness issues:
 661          *
 662          * The "register" reads and writes below from/to the PCI-E Memory
 663          * Window invoke the standard adapter Big-Endian to PCI-E Link
 664          * Little-Endian "swizzel."  As a result, if we have the following
 665          * data in adapter memory:
 666          *
 667          *     Memory:  ... | b0 | b1 | b2 | b3 | ...
 668          *     Address:      i+0  i+1  i+2  i+3
 669          *
 670          * Then a read of the adapter memory via the PCI-E Memory Window
 671          * will yield:
 672          *
 673          *     x = readl(i)
 674          *         31                  0
 675          *         [ b3 | b2 | b1 | b0 ]
 676          *
 677          * If this value is stored into local memory on a Little-Endian system
 678          * it will show up correctly in local memory as:
 679          *
 680          *     ( ..., b0, b1, b2, b3, ... )
 681          *
 682          * But on a Big-Endian system, the store will show up in memory
 683          * incorrectly swizzled as:
 684          *
 685          *     ( ..., b3, b2, b1, b0, ... )
 686          *
 687          * So we need to account for this in the reads and writes to the
 688          * PCI-E Memory Window below by undoing the register read/write
 689          * swizzels.
 690          */
 691         while (len > 0) {
 692                 if (dir == T4_MEMORY_READ)
 693                         *buf++ = le32_to_cpu((__force __le32)t4_read_reg(adap,
 694                                                 mem_base + offset));
 695                 else
 696                         t4_write_reg(adap, mem_base + offset,
 697                                      (__force u32)cpu_to_le32(*buf++));
 698                 offset += sizeof(__be32);
 699                 len -= sizeof(__be32);
 700
 701                 /* If we've reached the end of our current window aperture,
 702                  * move the PCI-E Memory Window on to the next.  Note that
 703                  * doing this here after "len" may be 0 allows us to set up
 704                  * the PCI-E Memory Window for a possible final residual
 705                  * transfer below ...
 706                  */
 707                 if (offset == mem_aperture) {
 708                         pos += mem_aperture;
 709                         offset = 0;
 710                         t4_memory_update_win(adap, win, pos | win_pf);
 711                 }
 712         }
 713
 714         /* If the original transfer had a length which wasn't a multiple of
 715          * 32-bits, now's where we need to finish off the transfer of the
 716          * residual amount.  The PCI-E Memory Window has already been moved
 717          * above (if necessary) to cover this final transfer.
 718          */
 719         if (resid)
 720                 t4_memory_rw_residual(adap, resid, mem_base + offset,
 721                                       (u8 *)buf, dir);
 722
 723         return 0;
 724 }
 725
 726 /* Return the specified PCI-E Configuration Space register from our Physical
 727  * Function.  We try first via a Firmware LDST Command since we prefer to let
 728  * the firmware own all of these registers, but if that fails we go for it
 729  * directly ourselves.
 730  */
 731 u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
 732 {
 733         u32 val, ldst_addrspace;
 734
 735         /* If fw_attach != 0, construct and send the Firmware LDST Command to
 736          * retrieve the specified PCI-E Configuration Space register.
 737          */
 738         struct fw_ldst_cmd ldst_cmd;
 739         int ret;
 740
 741         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
 742         ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE);
 743         ldst_cmd.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
 744                                                FW_CMD_REQUEST_F |
 745                                                FW_CMD_READ_F |
 746                                                ldst_addrspace);
 747         ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
 748         ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
 749         ldst_cmd.u.pcie.ctrl_to_fn =
 750                 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->pf));
 751         ldst_cmd.u.pcie.r = reg;
 752
 753         /* If the LDST Command succeeds, return the result, otherwise
 754          * fall through to reading it directly ourselves ...
 755          */
 756         ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
 757                          &ldst_cmd);
 758         if (ret == 0)
 759                 val = be32_to_cpu(ldst_cmd.u.pcie.data[0]);
 760         else
 761                 /* Read the desired Configuration Space register via the PCI-E
 762                  * Backdoor mechanism.
 763                  */
 764                 t4_hw_pci_read_cfg4(adap, reg, &val);
 765         return val;
 766 }
 767
 768 /* Get the window based on base passed to it.
 769  * Window aperture is currently unhandled, but there is no use case for it
 770  * right now
 771  */
 772 static u32 t4_get_window(struct adapter *adap, u32 pci_base, u64 pci_mask,
 773                          u32 memwin_base)
 774 {
 775         u32 ret;
 776
 777         if (is_t4(adap->params.chip)) {
 778                 u32 bar0;
 779
 780                 /* Truncation intentional: we only read the bottom 32-bits of
 781                  * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
 782                  * mechanism to read BAR0 instead of using
 783                  * pci_resource_start() because we could be operating from
 784                  * within a Virtual Machine which is trapping our accesses to
 785                  * our Configuration Space and we need to set up the PCI-E
 786                  * Memory Window decoders with the actual addresses which will
 787                  * be coming across the PCI-E link.
 788                  */
 789                 bar0 = t4_read_pcie_cfg4(adap, pci_base);
 790                 bar0 &= pci_mask;
 791                 adap->t4_bar0 = bar0;
 792
 793                 ret = bar0 + memwin_base;
 794         } else {
 795                 /* For T5, only relative offset inside the PCIe BAR is passed */
 796                 ret = memwin_base;
 797         }
 798         return ret;
 799 }
 800
 801 /* Get the default utility window (win0) used by everyone */
 802 u32 t4_get_util_window(struct adapter *adap)
 803 {
 804         return t4_get_window(adap, PCI_BASE_ADDRESS_0,
 805                              PCI_BASE_ADDRESS_MEM_MASK, MEMWIN0_BASE);
 806 }
 807
 808 /* Set up memory window for accessing adapter memory ranges.  (Read
 809  * back MA register to ensure that changes propagate before we attempt
 810  * to use the new values.)
 811  */
 812 void t4_setup_memwin(struct adapter *adap, u32 memwin_base, u32 window)
 813 {
 814         t4_write_reg(adap,
 815                      PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, window),
 816                      memwin_base | BIR_V(0) |
 817                      WINDOW_V(ilog2(MEMWIN0_APERTURE) - WINDOW_SHIFT_X));
 818         t4_read_reg(adap,
 819                     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, window));
 820 }
 821
 822 /**
 823  *      t4_get_regs_len - return the size of the chips register set
 824  *      @adapter: the adapter
 825  *
 826  *      Returns the size of the chip's BAR0 register space.
 827  */
 828 unsigned int t4_get_regs_len(struct adapter *adapter)
 829 {
 830         unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
 831
 832         switch (chip_version) {
 833         case CHELSIO_T4:
 834                 return T4_REGMAP_SIZE;
 835
 836         case CHELSIO_T5:
 837         case CHELSIO_T6:
 838                 return T5_REGMAP_SIZE;
 839         }
 840
 841         dev_err(adapter->pdev_dev,
 842                 "Unsupported chip version %d\n", chip_version);
 843         return 0;
 844 }
 845
 846 /**
 847  *      t4_get_regs - read chip registers into provided buffer
 848  *      @adap: the adapter
 849  *      @buf: register buffer
 850  *      @buf_size: size (in bytes) of register buffer
 851  *
 852  *      If the provided register buffer isn't large enough for the chip's
 853  *      full register range, the register dump will be truncated to the
 854  *      register buffer's size.
 855  */
 856 void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 857 {
 858         static const unsigned int t4_reg_ranges[] = {
 859                 0x1008, 0x1108,
 860                 0x1180, 0x1184,
 861                 0x1190, 0x1194,
 862                 0x11a0, 0x11a4,
 863                 0x11b0, 0x11b4,
 864                 0x11fc, 0x123c,
 865                 0x1300, 0x173c,
 866                 0x1800, 0x18fc,
 867                 0x3000, 0x30d8,
 868                 0x30e0, 0x30e4,
 869                 0x30ec, 0x5910,
 870                 0x5920, 0x5924,
 871                 0x5960, 0x5960,
 872                 0x5968, 0x5968,
 873                 0x5970, 0x5970,
 874                 0x5978, 0x5978,
 875                 0x5980, 0x5980,
 876                 0x5988, 0x5988,
 877                 0x5990, 0x5990,
 878                 0x5998, 0x5998,
 879                 0x59a0, 0x59d4,
 880                 0x5a00, 0x5ae0,
 881                 0x5ae8, 0x5ae8,
 882                 0x5af0, 0x5af0,
 883                 0x5af8, 0x5af8,
 884                 0x6000, 0x6098,
 885                 0x6100, 0x6150,
 886                 0x6200, 0x6208,
 887                 0x6240, 0x6248,
 888                 0x6280, 0x62b0,
 889                 0x62c0, 0x6338,
 890                 0x6370, 0x638c,
 891                 0x6400, 0x643c,
 892                 0x6500, 0x6524,
 893                 0x6a00, 0x6a04,
 894                 0x6a14, 0x6a38,
 895                 0x6a60, 0x6a70,
 896                 0x6a78, 0x6a78,
 897                 0x6b00, 0x6b0c,
 898                 0x6b1c, 0x6b84,
 899                 0x6bf0, 0x6bf8,
 900                 0x6c00, 0x6c0c,
 901                 0x6c1c, 0x6c84,
 902                 0x6cf0, 0x6cf8,
 903                 0x6d00, 0x6d0c,
 904                 0x6d1c, 0x6d84,
 905                 0x6df0, 0x6df8,
 906                 0x6e00, 0x6e0c,
 907                 0x6e1c, 0x6e84,
 908                 0x6ef0, 0x6ef8,
 909                 0x6f00, 0x6f0c,
 910                 0x6f1c, 0x6f84,
 911                 0x6ff0, 0x6ff8,
 912                 0x7000, 0x700c,
 913                 0x701c, 0x7084,
 914                 0x70f0, 0x70f8,
 915                 0x7100, 0x710c,
 916                 0x711c, 0x7184,
 917                 0x71f0, 0x71f8,
 918                 0x7200, 0x720c,
 919                 0x721c, 0x7284,
 920                 0x72f0, 0x72f8,
 921                 0x7300, 0x730c,
 922                 0x731c, 0x7384,
 923                 0x73f0, 0x73f8,
 924                 0x7400, 0x7450,
 925                 0x7500, 0x7530,
 926                 0x7600, 0x760c,
 927                 0x7614, 0x761c,
 928                 0x7680, 0x76cc,
 929                 0x7700, 0x7798,
 930                 0x77c0, 0x77fc,
 931                 0x7900, 0x79fc,
 932                 0x7b00, 0x7b58,
 933                 0x7b60, 0x7b84,
 934                 0x7b8c, 0x7c38,
 935                 0x7d00, 0x7d38,
 936                 0x7d40, 0x7d80,
 937                 0x7d8c, 0x7ddc,
 938                 0x7de4, 0x7e04,
 939                 0x7e10, 0x7e1c,
 940                 0x7e24, 0x7e38,
 941                 0x7e40, 0x7e44,
 942                 0x7e4c, 0x7e78,
 943                 0x7e80, 0x7ea4,
 944                 0x7eac, 0x7edc,
 945                 0x7ee8, 0x7efc,
 946                 0x8dc0, 0x8e04,
 947                 0x8e10, 0x8e1c,
 948                 0x8e30, 0x8e78,
 949                 0x8ea0, 0x8eb8,
 950                 0x8ec0, 0x8f6c,
 951                 0x8fc0, 0x9008,
 952                 0x9010, 0x9058,
 953                 0x9060, 0x9060,
 954                 0x9068, 0x9074,
 955                 0x90fc, 0x90fc,
 956                 0x9400, 0x9408,
 957                 0x9410, 0x9458,
 958                 0x9600, 0x9600,
 959                 0x9608, 0x9638,
 960                 0x9640, 0x96bc,
 961                 0x9800, 0x9808,
 962                 0x9820, 0x983c,
 963                 0x9850, 0x9864,
 964                 0x9c00, 0x9c6c,
 965                 0x9c80, 0x9cec,
 966                 0x9d00, 0x9d6c,
 967                 0x9d80, 0x9dec,
 968                 0x9e00, 0x9e6c,
 969                 0x9e80, 0x9eec,
 970                 0x9f00, 0x9f6c,
 971                 0x9f80, 0x9fec,
 972                 0xd004, 0xd004,
 973                 0xd010, 0xd03c,
 974                 0xdfc0, 0xdfe0,
 975                 0xe000, 0xea7c,
 976                 0xf000, 0x11110,
 977                 0x11118, 0x11190,
 978                 0x19040, 0x1906c,
 979                 0x19078, 0x19080,
 980                 0x1908c, 0x190e4,
 981                 0x190f0, 0x190f8,
 982                 0x19100, 0x19110,
 983                 0x19120, 0x19124,
 984                 0x19150, 0x19194,
 985                 0x1919c, 0x191b0,
 986                 0x191d0, 0x191e8,
 987                 0x19238, 0x1924c,
 988                 0x193f8, 0x1943c,
 989                 0x1944c, 0x19474,
 990                 0x19490, 0x194e0,
 991                 0x194f0, 0x194f8,
 992                 0x19800, 0x19c08,
 993                 0x19c10, 0x19c90,
 994                 0x19ca0, 0x19ce4,
 995                 0x19cf0, 0x19d40,
 996                 0x19d50, 0x19d94,
 997                 0x19da0, 0x19de8,
 998                 0x19df0, 0x19e40,
 999                 0x19e50, 0x19e90,
1000                 0x19ea0, 0x19f4c,
1001                 0x1a000, 0x1a004,
1002                 0x1a010, 0x1a06c,
1003                 0x1a0b0, 0x1a0e4,
1004                 0x1a0ec, 0x1a0f4,
1005                 0x1a100, 0x1a108,
1006                 0x1a114, 0x1a120,
1007                 0x1a128, 0x1a130,
1008                 0x1a138, 0x1a138,
1009                 0x1a190, 0x1a1c4,
1010                 0x1a1fc, 0x1a1fc,
1011                 0x1e040, 0x1e04c,
1012                 0x1e284, 0x1e28c,
1013                 0x1e2c0, 0x1e2c0,
1014                 0x1e2e0, 0x1e2e0,
1015                 0x1e300, 0x1e384,
1016                 0x1e3c0, 0x1e3c8,
1017                 0x1e440, 0x1e44c,
1018                 0x1e684, 0x1e68c,
1019                 0x1e6c0, 0x1e6c0,
1020                 0x1e6e0, 0x1e6e0,
1021                 0x1e700, 0x1e784,
1022                 0x1e7c0, 0x1e7c8,
1023                 0x1e840, 0x1e84c,
1024                 0x1ea84, 0x1ea8c,
1025                 0x1eac0, 0x1eac0,
1026                 0x1eae0, 0x1eae0,
1027                 0x1eb00, 0x1eb84,
1028                 0x1ebc0, 0x1ebc8,
1029                 0x1ec40, 0x1ec4c,
1030                 0x1ee84, 0x1ee8c,
1031                 0x1eec0, 0x1eec0,
1032                 0x1eee0, 0x1eee0,
1033                 0x1ef00, 0x1ef84,
1034                 0x1efc0, 0x1efc8,
1035                 0x1f040, 0x1f04c,
1036                 0x1f284, 0x1f28c,
1037                 0x1f2c0, 0x1f2c0,
1038                 0x1f2e0, 0x1f2e0,
1039                 0x1f300, 0x1f384,
1040                 0x1f3c0, 0x1f3c8,
1041                 0x1f440, 0x1f44c,
1042                 0x1f684, 0x1f68c,
1043                 0x1f6c0, 0x1f6c0,
1044                 0x1f6e0, 0x1f6e0,
1045                 0x1f700, 0x1f784,
1046                 0x1f7c0, 0x1f7c8,
1047                 0x1f840, 0x1f84c,
1048                 0x1fa84, 0x1fa8c,
1049                 0x1fac0, 0x1fac0,
1050                 0x1fae0, 0x1fae0,
1051                 0x1fb00, 0x1fb84,
1052                 0x1fbc0, 0x1fbc8,
1053                 0x1fc40, 0x1fc4c,
1054                 0x1fe84, 0x1fe8c,
1055                 0x1fec0, 0x1fec0,
1056                 0x1fee0, 0x1fee0,
1057                 0x1ff00, 0x1ff84,
1058                 0x1ffc0, 0x1ffc8,
1059                 0x20000, 0x2002c,
1060                 0x20100, 0x2013c,
1061                 0x20190, 0x201a0,
1062                 0x201a8, 0x201b8,
1063                 0x201c4, 0x201c8,
1064                 0x20200, 0x20318,
1065                 0x20400, 0x204b4,
1066                 0x204c0, 0x20528,
1067                 0x20540, 0x20614,
1068                 0x21000, 0x21040,
1069                 0x2104c, 0x21060,
1070                 0x210c0, 0x210ec,
1071                 0x21200, 0x21268,
1072                 0x21270, 0x21284,
1073                 0x212fc, 0x21388,
1074                 0x21400, 0x21404,
1075                 0x21500, 0x21500,
1076                 0x21510, 0x21518,
1077                 0x2152c, 0x21530,
1078                 0x2153c, 0x2153c,
1079                 0x21550, 0x21554,
1080                 0x21600, 0x21600,
1081                 0x21608, 0x2161c,
1082                 0x21624, 0x21628,
1083                 0x21630, 0x21634,
1084                 0x2163c, 0x2163c,
1085                 0x21700, 0x2171c,
1086                 0x21780, 0x2178c,
1087                 0x21800, 0x21818,
1088                 0x21820, 0x21828,
1089                 0x21830, 0x21848,
1090                 0x21850, 0x21854,
1091                 0x21860, 0x21868,
1092                 0x21870, 0x21870,
1093                 0x21878, 0x21898,
1094                 0x218a0, 0x218a8,
1095                 0x218b0, 0x218c8,
1096                 0x218d0, 0x218d4,
1097                 0x218e0, 0x218e8,
1098                 0x218f0, 0x218f0,
1099                 0x218f8, 0x21a18,
1100                 0x21a20, 0x21a28,
1101                 0x21a30, 0x21a48,
1102                 0x21a50, 0x21a54,
1103                 0x21a60, 0x21a68,
1104                 0x21a70, 0x21a70,
1105                 0x21a78, 0x21a98,
1106                 0x21aa0, 0x21aa8,
1107                 0x21ab0, 0x21ac8,
1108                 0x21ad0, 0x21ad4,
1109                 0x21ae0, 0x21ae8,
1110                 0x21af0, 0x21af0,
1111                 0x21af8, 0x21c18,
1112                 0x21c20, 0x21c20,
1113                 0x21c28, 0x21c30,
1114                 0x21c38, 0x21c38,
1115                 0x21c80, 0x21c98,
1116                 0x21ca0, 0x21ca8,
1117                 0x21cb0, 0x21cc8,
1118                 0x21cd0, 0x21cd4,
1119                 0x21ce0, 0x21ce8,
1120                 0x21cf0, 0x21cf0,
1121                 0x21cf8, 0x21d7c,
1122                 0x21e00, 0x21e04,
1123                 0x22000, 0x2202c,
1124                 0x22100, 0x2213c,
1125                 0x22190, 0x221a0,
1126                 0x221a8, 0x221b8,
1127                 0x221c4, 0x221c8,
1128                 0x22200, 0x22318,
1129                 0x22400, 0x224b4,
1130                 0x224c0, 0x22528,
1131                 0x22540, 0x22614,
1132                 0x23000, 0x23040,
1133                 0x2304c, 0x23060,
1134                 0x230c0, 0x230ec,
1135                 0x23200, 0x23268,
1136                 0x23270, 0x23284,
1137                 0x232fc, 0x23388,
1138                 0x23400, 0x23404,
1139                 0x23500, 0x23500,
1140                 0x23510, 0x23518,
1141                 0x2352c, 0x23530,
1142                 0x2353c, 0x2353c,
1143                 0x23550, 0x23554,
1144                 0x23600, 0x23600,
1145                 0x23608, 0x2361c,
1146                 0x23624, 0x23628,
1147                 0x23630, 0x23634,
1148                 0x2363c, 0x2363c,
1149                 0x23700, 0x2371c,
1150                 0x23780, 0x2378c,
1151                 0x23800, 0x23818,
1152                 0x23820, 0x23828,
1153                 0x23830, 0x23848,
1154                 0x23850, 0x23854,
1155                 0x23860, 0x23868,
1156                 0x23870, 0x23870,
1157                 0x23878, 0x23898,
1158                 0x238a0, 0x238a8,
1159                 0x238b0, 0x238c8,
1160                 0x238d0, 0x238d4,
1161                 0x238e0, 0x238e8,
1162                 0x238f0, 0x238f0,
1163                 0x238f8, 0x23a18,
1164                 0x23a20, 0x23a28,
1165                 0x23a30, 0x23a48,
1166                 0x23a50, 0x23a54,
1167                 0x23a60, 0x23a68,
1168                 0x23a70, 0x23a70,
1169                 0x23a78, 0x23a98,
1170                 0x23aa0, 0x23aa8,
1171                 0x23ab0, 0x23ac8,
1172                 0x23ad0, 0x23ad4,
1173                 0x23ae0, 0x23ae8,
1174                 0x23af0, 0x23af0,
1175                 0x23af8, 0x23c18,
1176                 0x23c20, 0x23c20,
1177                 0x23c28, 0x23c30,
1178                 0x23c38, 0x23c38,
1179                 0x23c80, 0x23c98,
1180                 0x23ca0, 0x23ca8,
1181                 0x23cb0, 0x23cc8,
1182                 0x23cd0, 0x23cd4,
1183                 0x23ce0, 0x23ce8,
1184                 0x23cf0, 0x23cf0,
1185                 0x23cf8, 0x23d7c,
1186                 0x23e00, 0x23e04,
1187                 0x24000, 0x2402c,
1188                 0x24100, 0x2413c,
1189                 0x24190, 0x241a0,
1190                 0x241a8, 0x241b8,
1191                 0x241c4, 0x241c8,
1192                 0x24200, 0x24318,
1193                 0x24400, 0x244b4,
1194                 0x244c0, 0x24528,
1195                 0x24540, 0x24614,
1196                 0x25000, 0x25040,
1197                 0x2504c, 0x25060,
1198                 0x250c0, 0x250ec,
1199                 0x25200, 0x25268,
1200                 0x25270, 0x25284,
1201                 0x252fc, 0x25388,
1202                 0x25400, 0x25404,
1203                 0x25500, 0x25500,
1204                 0x25510, 0x25518,
1205                 0x2552c, 0x25530,
1206                 0x2553c, 0x2553c,
1207                 0x25550, 0x25554,
1208                 0x25600, 0x25600,
1209                 0x25608, 0x2561c,
1210                 0x25624, 0x25628,
1211                 0x25630, 0x25634,
1212                 0x2563c, 0x2563c,
1213                 0x25700, 0x2571c,
1214                 0x25780, 0x2578c,
1215                 0x25800, 0x25818,
1216                 0x25820, 0x25828,
1217                 0x25830, 0x25848,
1218                 0x25850, 0x25854,
1219                 0x25860, 0x25868,
1220                 0x25870, 0x25870,
1221                 0x25878, 0x25898,
1222                 0x258a0, 0x258a8,
1223                 0x258b0, 0x258c8,
1224                 0x258d0, 0x258d4,
1225                 0x258e0, 0x258e8,
1226                 0x258f0, 0x258f0,
1227                 0x258f8, 0x25a18,
1228                 0x25a20, 0x25a28,
1229                 0x25a30, 0x25a48,
1230                 0x25a50, 0x25a54,
1231                 0x25a60, 0x25a68,
1232                 0x25a70, 0x25a70,
1233                 0x25a78, 0x25a98,
1234                 0x25aa0, 0x25aa8,
1235                 0x25ab0, 0x25ac8,
1236                 0x25ad0, 0x25ad4,
1237                 0x25ae0, 0x25ae8,
1238                 0x25af0, 0x25af0,
1239                 0x25af8, 0x25c18,
1240                 0x25c20, 0x25c20,
1241                 0x25c28, 0x25c30,
1242                 0x25c38, 0x25c38,
1243                 0x25c80, 0x25c98,
1244                 0x25ca0, 0x25ca8,
1245                 0x25cb0, 0x25cc8,
1246                 0x25cd0, 0x25cd4,
1247                 0x25ce0, 0x25ce8,
1248                 0x25cf0, 0x25cf0,
1249                 0x25cf8, 0x25d7c,
1250                 0x25e00, 0x25e04,
1251                 0x26000, 0x2602c,
1252                 0x26100, 0x2613c,
1253                 0x26190, 0x261a0,
1254                 0x261a8, 0x261b8,
1255                 0x261c4, 0x261c8,
1256                 0x26200, 0x26318,
1257                 0x26400, 0x264b4,
1258                 0x264c0, 0x26528,
1259                 0x26540, 0x26614,
1260                 0x27000, 0x27040,
1261                 0x2704c, 0x27060,
1262                 0x270c0, 0x270ec,
1263                 0x27200, 0x27268,
1264                 0x27270, 0x27284,
1265                 0x272fc, 0x27388,
1266                 0x27400, 0x27404,
1267                 0x27500, 0x27500,
1268                 0x27510, 0x27518,
1269                 0x2752c, 0x27530,
1270                 0x2753c, 0x2753c,
1271                 0x27550, 0x27554,
1272                 0x27600, 0x27600,
1273                 0x27608, 0x2761c,
1274                 0x27624, 0x27628,
1275                 0x27630, 0x27634,
1276                 0x2763c, 0x2763c,
1277                 0x27700, 0x2771c,
1278                 0x27780, 0x2778c,
1279                 0x27800, 0x27818,
1280                 0x27820, 0x27828,
1281                 0x27830, 0x27848,
1282                 0x27850, 0x27854,
1283                 0x27860, 0x27868,
1284                 0x27870, 0x27870,
1285                 0x27878, 0x27898,
1286                 0x278a0, 0x278a8,
1287                 0x278b0, 0x278c8,
1288                 0x278d0, 0x278d4,
1289                 0x278e0, 0x278e8,
1290                 0x278f0, 0x278f0,
1291                 0x278f8, 0x27a18,
1292                 0x27a20, 0x27a28,
1293                 0x27a30, 0x27a48,
1294                 0x27a50, 0x27a54,
1295                 0x27a60, 0x27a68,
1296                 0x27a70, 0x27a70,
1297                 0x27a78, 0x27a98,
1298                 0x27aa0, 0x27aa8,
1299                 0x27ab0, 0x27ac8,
1300                 0x27ad0, 0x27ad4,
1301                 0x27ae0, 0x27ae8,
1302                 0x27af0, 0x27af0,
1303                 0x27af8, 0x27c18,
1304                 0x27c20, 0x27c20,
1305                 0x27c28, 0x27c30,
1306                 0x27c38, 0x27c38,
1307                 0x27c80, 0x27c98,
1308                 0x27ca0, 0x27ca8,
1309                 0x27cb0, 0x27cc8,
1310                 0x27cd0, 0x27cd4,
1311                 0x27ce0, 0x27ce8,
1312                 0x27cf0, 0x27cf0,
1313                 0x27cf8, 0x27d7c,
1314                 0x27e00, 0x27e04,
1315         };
1316
1317         static const unsigned int t5_reg_ranges[] = {
1318                 0x1008, 0x10c0,
1319                 0x10cc, 0x10f8,
1320                 0x1100, 0x1100,
1321                 0x110c, 0x1148,
1322                 0x1180, 0x1184,
1323                 0x1190, 0x1194,
1324                 0x11a0, 0x11a4,
1325                 0x11b0, 0x11b4,
1326                 0x11fc, 0x123c,
1327                 0x1280, 0x173c,
1328                 0x1800, 0x18fc,
1329                 0x3000, 0x3028,
1330                 0x3060, 0x30b0,
1331                 0x30b8, 0x30d8,
1332                 0x30e0, 0x30fc,
1333                 0x3140, 0x357c,
1334                 0x35a8, 0x35cc,
1335                 0x35ec, 0x35ec,
1336                 0x3600, 0x5624,
1337                 0x56cc, 0x56ec,
1338                 0x56f4, 0x5720,
1339                 0x5728, 0x575c,
1340                 0x580c, 0x5814,
1341                 0x5890, 0x589c,
1342                 0x58a4, 0x58ac,
1343                 0x58b8, 0x58bc,
1344                 0x5940, 0x59c8,
1345                 0x59d0, 0x59dc,
1346                 0x59fc, 0x5a18,
1347                 0x5a60, 0x5a70,
1348                 0x5a80, 0x5a9c,
1349                 0x5b94, 0x5bfc,
1350                 0x6000, 0x6020,
1351                 0x6028, 0x6040,
1352                 0x6058, 0x609c,
1353                 0x60a8, 0x614c,
1354                 0x7700, 0x7798,
1355                 0x77c0, 0x78fc,
1356                 0x7b00, 0x7b58,
1357                 0x7b60, 0x7b84,
1358                 0x7b8c, 0x7c54,
1359                 0x7d00, 0x7d38,
1360                 0x7d40, 0x7d80,
1361                 0x7d8c, 0x7ddc,
1362                 0x7de4, 0x7e04,
1363                 0x7e10, 0x7e1c,
1364                 0x7e24, 0x7e38,
1365                 0x7e40, 0x7e44,
1366                 0x7e4c, 0x7e78,
1367                 0x7e80, 0x7edc,
1368                 0x7ee8, 0x7efc,
1369                 0x8dc0, 0x8de0,
1370                 0x8df8, 0x8e04,
1371                 0x8e10, 0x8e84,
1372                 0x8ea0, 0x8f84,
1373                 0x8fc0, 0x9058,
1374                 0x9060, 0x9060,
1375                 0x9068, 0x90f8,
1376                 0x9400, 0x9408,
1377                 0x9410, 0x9470,
1378                 0x9600, 0x9600,
1379                 0x9608, 0x9638,
1380                 0x9640, 0x96f4,
1381                 0x9800, 0x9808,
1382                 0x9820, 0x983c,
1383                 0x9850, 0x9864,
1384                 0x9c00, 0x9c6c,
1385                 0x9c80, 0x9cec,
1386                 0x9d00, 0x9d6c,
1387                 0x9d80, 0x9dec,
1388                 0x9e00, 0x9e6c,
1389                 0x9e80, 0x9eec,
1390                 0x9f00, 0x9f6c,
1391                 0x9f80, 0xa020,
1392                 0xd004, 0xd004,
1393                 0xd010, 0xd03c,
1394                 0xdfc0, 0xdfe0,
1395                 0xe000, 0x1106c,
1396                 0x11074, 0x11088,
1397                 0x1109c, 0x1117c,
1398                 0x11190, 0x11204,
1399                 0x19040, 0x1906c,
1400                 0x19078, 0x19080,
1401                 0x1908c, 0x190e8,
1402                 0x190f0, 0x190f8,
1403                 0x19100, 0x19110,
1404                 0x19120, 0x19124,
1405                 0x19150, 0x19194,
1406                 0x1919c, 0x191b0,
1407                 0x191d0, 0x191e8,
1408                 0x19238, 0x19290,
1409                 0x193f8, 0x19428,
1410                 0x19430, 0x19444,
1411                 0x1944c, 0x1946c,
1412                 0x19474, 0x19474,
1413                 0x19490, 0x194cc,
1414                 0x194f0, 0x194f8,
1415                 0x19c00, 0x19c08,
1416                 0x19c10, 0x19c60,
1417                 0x19c94, 0x19ce4,
1418                 0x19cf0, 0x19d40,
1419                 0x19d50, 0x19d94,
1420                 0x19da0, 0x19de8,
1421                 0x19df0, 0x19e10,
1422                 0x19e50, 0x19e90,
1423                 0x19ea0, 0x19f24,
1424                 0x19f34, 0x19f34,
1425                 0x19f40, 0x19f50,
1426                 0x19f90, 0x19fb4,
1427                 0x19fc4, 0x19fe4,
1428                 0x1a000, 0x1a004,
1429                 0x1a010, 0x1a06c,
1430                 0x1a0b0, 0x1a0e4,
1431                 0x1a0ec, 0x1a0f8,
1432                 0x1a100, 0x1a108,
1433                 0x1a114, 0x1a120,
1434                 0x1a128, 0x1a130,
1435                 0x1a138, 0x1a138,
1436                 0x1a190, 0x1a1c4,
1437                 0x1a1fc, 0x1a1fc,
1438                 0x1e008, 0x1e00c,
1439                 0x1e040, 0x1e044,
1440                 0x1e04c, 0x1e04c,
1441                 0x1e284, 0x1e290,
1442                 0x1e2c0, 0x1e2c0,
1443                 0x1e2e0, 0x1e2e0,
1444                 0x1e300, 0x1e384,
1445                 0x1e3c0, 0x1e3c8,
1446                 0x1e408, 0x1e40c,
1447                 0x1e440, 0x1e444,
1448                 0x1e44c, 0x1e44c,
1449                 0x1e684, 0x1e690,
1450                 0x1e6c0, 0x1e6c0,
1451                 0x1e6e0, 0x1e6e0,
1452                 0x1e700, 0x1e784,
1453                 0x1e7c0, 0x1e7c8,
1454                 0x1e808, 0x1e80c,
1455                 0x1e840, 0x1e844,
1456                 0x1e84c, 0x1e84c,
1457                 0x1ea84, 0x1ea90,
1458                 0x1eac0, 0x1eac0,
1459                 0x1eae0, 0x1eae0,
1460                 0x1eb00, 0x1eb84,
1461                 0x1ebc0, 0x1ebc8,
1462                 0x1ec08, 0x1ec0c,
1463                 0x1ec40, 0x1ec44,
1464                 0x1ec4c, 0x1ec4c,
1465                 0x1ee84, 0x1ee90,
1466                 0x1eec0, 0x1eec0,
1467                 0x1eee0, 0x1eee0,
1468                 0x1ef00, 0x1ef84,
1469                 0x1efc0, 0x1efc8,
1470                 0x1f008, 0x1f00c,
1471                 0x1f040, 0x1f044,
1472                 0x1f04c, 0x1f04c,
1473                 0x1f284, 0x1f290,
1474                 0x1f2c0, 0x1f2c0,
1475                 0x1f2e0, 0x1f2e0,
1476                 0x1f300, 0x1f384,
1477                 0x1f3c0, 0x1f3c8,
1478                 0x1f408, 0x1f40c,
1479                 0x1f440, 0x1f444,
1480                 0x1f44c, 0x1f44c,
1481                 0x1f684, 0x1f690,
1482                 0x1f6c0, 0x1f6c0,
1483                 0x1f6e0, 0x1f6e0,
1484                 0x1f700, 0x1f784,
1485                 0x1f7c0, 0x1f7c8,
1486                 0x1f808, 0x1f80c,
1487                 0x1f840, 0x1f844,
1488                 0x1f84c, 0x1f84c,
1489                 0x1fa84, 0x1fa90,
1490                 0x1fac0, 0x1fac0,
1491                 0x1fae0, 0x1fae0,
1492                 0x1fb00, 0x1fb84,
1493                 0x1fbc0, 0x1fbc8,
1494                 0x1fc08, 0x1fc0c,
1495                 0x1fc40, 0x1fc44,
1496                 0x1fc4c, 0x1fc4c,
1497                 0x1fe84, 0x1fe90,
1498                 0x1fec0, 0x1fec0,
1499                 0x1fee0, 0x1fee0,
1500                 0x1ff00, 0x1ff84,
1501                 0x1ffc0, 0x1ffc8,
1502                 0x30000, 0x30030,
1503                 0x30100, 0x30144,
1504                 0x30190, 0x301a0,
1505                 0x301a8, 0x301b8,
1506                 0x301c4, 0x301c8,
1507                 0x301d0, 0x301d0,
1508                 0x30200, 0x30318,
1509                 0x30400, 0x304b4,
1510                 0x304c0, 0x3052c,
1511                 0x30540, 0x3061c,
1512                 0x30800, 0x30828,
1513                 0x30834, 0x30834,
1514                 0x308c0, 0x30908,
1515                 0x30910, 0x309ac,
1516                 0x30a00, 0x30a14,
1517                 0x30a1c, 0x30a2c,
1518                 0x30a44, 0x30a50,
1519                 0x30a74, 0x30a74,
1520                 0x30a7c, 0x30afc,
1521                 0x30b08, 0x30c24,
1522                 0x30d00, 0x30d00,
1523                 0x30d08, 0x30d14,
1524                 0x30d1c, 0x30d20,
1525                 0x30d3c, 0x30d3c,
1526                 0x30d48, 0x30d50,
1527                 0x31200, 0x3120c,
1528                 0x31220, 0x31220,
1529                 0x31240, 0x31240,
1530                 0x31600, 0x3160c,
1531                 0x31a00, 0x31a1c,
1532                 0x31e00, 0x31e20,
1533                 0x31e38, 0x31e3c,
1534                 0x31e80, 0x31e80,
1535                 0x31e88, 0x31ea8,
1536                 0x31eb0, 0x31eb4,
1537                 0x31ec8, 0x31ed4,
1538                 0x31fb8, 0x32004,
1539                 0x32200, 0x32200,
1540                 0x32208, 0x32240,
1541                 0x32248, 0x32280,
1542                 0x32288, 0x322c0,
1543                 0x322c8, 0x322fc,
1544                 0x32600, 0x32630,
1545                 0x32a00, 0x32abc,
1546                 0x32b00, 0x32b10,
1547                 0x32b20, 0x32b30,
1548                 0x32b40, 0x32b50,
1549                 0x32b60, 0x32b70,
1550                 0x33000, 0x33028,
1551                 0x33030, 0x33048,
1552                 0x33060, 0x33068,
1553                 0x33070, 0x3309c,
1554                 0x330f0, 0x33128,
1555                 0x33130, 0x33148,
1556                 0x33160, 0x33168,
1557                 0x33170, 0x3319c,
1558                 0x331f0, 0x33238,
1559                 0x33240, 0x33240,
1560                 0x33248, 0x33250,
1561                 0x3325c, 0x33264,
1562                 0x33270, 0x332b8,
1563                 0x332c0, 0x332e4,
1564                 0x332f8, 0x33338,
1565                 0x33340, 0x33340,
1566                 0x33348, 0x33350,
1567                 0x3335c, 0x33364,
1568                 0x33370, 0x333b8,
1569                 0x333c0, 0x333e4,
1570                 0x333f8, 0x33428,
1571                 0x33430, 0x33448,
1572                 0x33460, 0x33468,
1573                 0x33470, 0x3349c,
1574                 0x334f0, 0x33528,
1575                 0x33530, 0x33548,
1576                 0x33560, 0x33568,
1577                 0x33570, 0x3359c,
1578                 0x335f0, 0x33638,
1579                 0x33640, 0x33640,
1580                 0x33648, 0x33650,
1581                 0x3365c, 0x33664,
1582                 0x33670, 0x336b8,
1583                 0x336c0, 0x336e4,
1584                 0x336f8, 0x33738,
1585                 0x33740, 0x33740,
1586                 0x33748, 0x33750,
1587                 0x3375c, 0x33764,
1588                 0x33770, 0x337b8,
1589                 0x337c0, 0x337e4,
1590                 0x337f8, 0x337fc,
1591                 0x33814, 0x33814,
1592                 0x3382c, 0x3382c,
1593                 0x33880, 0x3388c,
1594                 0x338e8, 0x338ec,
1595                 0x33900, 0x33928,
1596                 0x33930, 0x33948,
1597                 0x33960, 0x33968,
1598                 0x33970, 0x3399c,
1599                 0x339f0, 0x33a38,
1600                 0x33a40, 0x33a40,
1601                 0x33a48, 0x33a50,
1602                 0x33a5c, 0x33a64,
1603                 0x33a70, 0x33ab8,
1604                 0x33ac0, 0x33ae4,
1605                 0x33af8, 0x33b10,
1606                 0x33b28, 0x33b28,
1607                 0x33b3c, 0x33b50,
1608                 0x33bf0, 0x33c10,
1609                 0x33c28, 0x33c28,
1610                 0x33c3c, 0x33c50,
1611                 0x33cf0, 0x33cfc,
1612                 0x34000, 0x34030,
1613                 0x34100, 0x34144,
1614                 0x34190, 0x341a0,
1615                 0x341a8, 0x341b8,
1616                 0x341c4, 0x341c8,
1617                 0x341d0, 0x341d0,
1618                 0x34200, 0x34318,
1619                 0x34400, 0x344b4,
1620                 0x344c0, 0x3452c,
1621                 0x34540, 0x3461c,
1622                 0x34800, 0x34828,
1623                 0x34834, 0x34834,
1624                 0x348c0, 0x34908,
1625                 0x34910, 0x349ac,
1626                 0x34a00, 0x34a14,
1627                 0x34a1c, 0x34a2c,
1628                 0x34a44, 0x34a50,
1629                 0x34a74, 0x34a74,
1630                 0x34a7c, 0x34afc,
1631                 0x34b08, 0x34c24,
1632                 0x34d00, 0x34d00,
1633                 0x34d08, 0x34d14,
1634                 0x34d1c, 0x34d20,
1635                 0x34d3c, 0x34d3c,
1636                 0x34d48, 0x34d50,
1637                 0x35200, 0x3520c,
1638                 0x35220, 0x35220,
1639                 0x35240, 0x35240,
1640                 0x35600, 0x3560c,
1641                 0x35a00, 0x35a1c,
1642                 0x35e00, 0x35e20,
1643                 0x35e38, 0x35e3c,
1644                 0x35e80, 0x35e80,
1645                 0x35e88, 0x35ea8,
1646                 0x35eb0, 0x35eb4,
1647                 0x35ec8, 0x35ed4,
1648                 0x35fb8, 0x36004,
1649                 0x36200, 0x36200,
1650                 0x36208, 0x36240,
1651                 0x36248, 0x36280,
1652                 0x36288, 0x362c0,
1653                 0x362c8, 0x362fc,
1654                 0x36600, 0x36630,
1655                 0x36a00, 0x36abc,
1656                 0x36b00, 0x36b10,
1657                 0x36b20, 0x36b30,
1658                 0x36b40, 0x36b50,
1659                 0x36b60, 0x36b70,
1660                 0x37000, 0x37028,
1661                 0x37030, 0x37048,
1662                 0x37060, 0x37068,
1663                 0x37070, 0x3709c,
1664                 0x370f0, 0x37128,
1665                 0x37130, 0x37148,
1666                 0x37160, 0x37168,
1667                 0x37170, 0x3719c,
1668                 0x371f0, 0x37238,
1669                 0x37240, 0x37240,
1670                 0x37248, 0x37250,
1671                 0x3725c, 0x37264,
1672                 0x37270, 0x372b8,
1673                 0x372c0, 0x372e4,
1674                 0x372f8, 0x37338,
1675                 0x37340, 0x37340,
1676                 0x37348, 0x37350,
1677                 0x3735c, 0x37364,
1678                 0x37370, 0x373b8,
1679                 0x373c0, 0x373e4,
1680                 0x373f8, 0x37428,
1681                 0x37430, 0x37448,
1682                 0x37460, 0x37468,
1683                 0x37470, 0x3749c,
1684                 0x374f0, 0x37528,
1685                 0x37530, 0x37548,
1686                 0x37560, 0x37568,
1687                 0x37570, 0x3759c,
1688                 0x375f0, 0x37638,
1689                 0x37640, 0x37640,
1690                 0x37648, 0x37650,
1691                 0x3765c, 0x37664,
1692                 0x37670, 0x376b8,
1693                 0x376c0, 0x376e4,
1694                 0x376f8, 0x37738,
1695                 0x37740, 0x37740,
1696                 0x37748, 0x37750,
1697                 0x3775c, 0x37764,
1698                 0x37770, 0x377b8,
1699                 0x377c0, 0x377e4,
1700                 0x377f8, 0x377fc,
1701                 0x37814, 0x37814,
1702                 0x3782c, 0x3782c,
1703                 0x37880, 0x3788c,
1704                 0x378e8, 0x378ec,
1705                 0x37900, 0x37928,
1706                 0x37930, 0x37948,
1707                 0x37960, 0x37968,
1708                 0x37970, 0x3799c,
1709                 0x379f0, 0x37a38,
1710                 0x37a40, 0x37a40,
1711                 0x37a48, 0x37a50,
1712                 0x37a5c, 0x37a64,
1713                 0x37a70, 0x37ab8,
1714                 0x37ac0, 0x37ae4,
1715                 0x37af8, 0x37b10,
1716                 0x37b28, 0x37b28,
1717                 0x37b3c, 0x37b50,
1718                 0x37bf0, 0x37c10,
1719                 0x37c28, 0x37c28,
1720                 0x37c3c, 0x37c50,
1721                 0x37cf0, 0x37cfc,
1722                 0x38000, 0x38030,
1723                 0x38100, 0x38144,
1724                 0x38190, 0x381a0,
1725                 0x381a8, 0x381b8,
1726                 0x381c4, 0x381c8,
1727                 0x381d0, 0x381d0,
1728                 0x38200, 0x38318,
1729                 0x38400, 0x384b4,
1730                 0x384c0, 0x3852c,
1731                 0x38540, 0x3861c,
1732                 0x38800, 0x38828,
1733                 0x38834, 0x38834,
1734                 0x388c0, 0x38908,
1735                 0x38910, 0x389ac,
1736                 0x38a00, 0x38a14,
1737                 0x38a1c, 0x38a2c,
1738                 0x38a44, 0x38a50,
1739                 0x38a74, 0x38a74,
1740                 0x38a7c, 0x38afc,
1741                 0x38b08, 0x38c24,
1742                 0x38d00, 0x38d00,
1743                 0x38d08, 0x38d14,
1744                 0x38d1c, 0x38d20,
1745                 0x38d3c, 0x38d3c,
1746                 0x38d48, 0x38d50,
1747                 0x39200, 0x3920c,
1748                 0x39220, 0x39220,
1749                 0x39240, 0x39240,
1750                 0x39600, 0x3960c,
1751                 0x39a00, 0x39a1c,
1752                 0x39e00, 0x39e20,
1753                 0x39e38, 0x39e3c,
1754                 0x39e80, 0x39e80,
1755                 0x39e88, 0x39ea8,
1756                 0x39eb0, 0x39eb4,
1757                 0x39ec8, 0x39ed4,
1758                 0x39fb8, 0x3a004,
1759                 0x3a200, 0x3a200,
1760                 0x3a208, 0x3a240,
1761                 0x3a248, 0x3a280,
1762                 0x3a288, 0x3a2c0,
1763                 0x3a2c8, 0x3a2fc,
1764                 0x3a600, 0x3a630,
1765                 0x3aa00, 0x3aabc,
1766                 0x3ab00, 0x3ab10,
1767                 0x3ab20, 0x3ab30,
1768                 0x3ab40, 0x3ab50,
1769                 0x3ab60, 0x3ab70,
1770                 0x3b000, 0x3b028,
1771                 0x3b030, 0x3b048,
1772                 0x3b060, 0x3b068,
1773                 0x3b070, 0x3b09c,
1774                 0x3b0f0, 0x3b128,
1775                 0x3b130, 0x3b148,
1776                 0x3b160, 0x3b168,
1777                 0x3b170, 0x3b19c,
1778                 0x3b1f0, 0x3b238,
1779                 0x3b240, 0x3b240,
1780                 0x3b248, 0x3b250,
1781                 0x3b25c, 0x3b264,
1782                 0x3b270, 0x3b2b8,
1783                 0x3b2c0, 0x3b2e4,
1784                 0x3b2f8, 0x3b338,
1785                 0x3b340, 0x3b340,
1786                 0x3b348, 0x3b350,
1787                 0x3b35c, 0x3b364,
1788                 0x3b370, 0x3b3b8,
1789                 0x3b3c0, 0x3b3e4,
1790                 0x3b3f8, 0x3b428,
1791                 0x3b430, 0x3b448,
1792                 0x3b460, 0x3b468,
1793                 0x3b470, 0x3b49c,
1794                 0x3b4f0, 0x3b528,
1795                 0x3b530, 0x3b548,
1796                 0x3b560, 0x3b568,
1797                 0x3b570, 0x3b59c,
1798                 0x3b5f0, 0x3b638,
1799                 0x3b640, 0x3b640,
1800                 0x3b648, 0x3b650,
1801                 0x3b65c, 0x3b664,
1802                 0x3b670, 0x3b6b8,
1803                 0x3b6c0, 0x3b6e4,
1804                 0x3b6f8, 0x3b738,
1805                 0x3b740, 0x3b740,
1806                 0x3b748, 0x3b750,
1807                 0x3b75c, 0x3b764,
1808                 0x3b770, 0x3b7b8,
1809                 0x3b7c0, 0x3b7e4,
1810                 0x3b7f8, 0x3b7fc,
1811                 0x3b814, 0x3b814,
1812                 0x3b82c, 0x3b82c,
1813                 0x3b880, 0x3b88c,
1814                 0x3b8e8, 0x3b8ec,
1815                 0x3b900, 0x3b928,
1816                 0x3b930, 0x3b948,
1817                 0x3b960, 0x3b968,
1818                 0x3b970, 0x3b99c,
1819                 0x3b9f0, 0x3ba38,
1820                 0x3ba40, 0x3ba40,
1821                 0x3ba48, 0x3ba50,
1822                 0x3ba5c, 0x3ba64,
1823                 0x3ba70, 0x3bab8,
1824                 0x3bac0, 0x3bae4,
1825                 0x3baf8, 0x3bb10,
1826                 0x3bb28, 0x3bb28,
1827                 0x3bb3c, 0x3bb50,
1828                 0x3bbf0, 0x3bc10,
1829                 0x3bc28, 0x3bc28,
1830                 0x3bc3c, 0x3bc50,
1831                 0x3bcf0, 0x3bcfc,
1832                 0x3c000, 0x3c030,
1833                 0x3c100, 0x3c144,
1834                 0x3c190, 0x3c1a0,
1835                 0x3c1a8, 0x3c1b8,
1836                 0x3c1c4, 0x3c1c8,
1837                 0x3c1d0, 0x3c1d0,
1838                 0x3c200, 0x3c318,
1839                 0x3c400, 0x3c4b4,
1840                 0x3c4c0, 0x3c52c,
1841                 0x3c540, 0x3c61c,
1842                 0x3c800, 0x3c828,
1843                 0x3c834, 0x3c834,
1844                 0x3c8c0, 0x3c908,
1845                 0x3c910, 0x3c9ac,
1846                 0x3ca00, 0x3ca14,
1847                 0x3ca1c, 0x3ca2c,
1848                 0x3ca44, 0x3ca50,
1849                 0x3ca74, 0x3ca74,
1850                 0x3ca7c, 0x3cafc,
1851                 0x3cb08, 0x3cc24,
1852                 0x3cd00, 0x3cd00,
1853                 0x3cd08, 0x3cd14,
1854                 0x3cd1c, 0x3cd20,
1855                 0x3cd3c, 0x3cd3c,
1856                 0x3cd48, 0x3cd50,
1857                 0x3d200, 0x3d20c,
1858                 0x3d220, 0x3d220,
1859                 0x3d240, 0x3d240,
1860                 0x3d600, 0x3d60c,
1861                 0x3da00, 0x3da1c,
1862                 0x3de00, 0x3de20,
1863                 0x3de38, 0x3de3c,
1864                 0x3de80, 0x3de80,
1865                 0x3de88, 0x3dea8,
1866                 0x3deb0, 0x3deb4,
1867                 0x3dec8, 0x3ded4,
1868                 0x3dfb8, 0x3e004,
1869                 0x3e200, 0x3e200,
1870                 0x3e208, 0x3e240,
1871                 0x3e248, 0x3e280,
1872                 0x3e288, 0x3e2c0,
1873                 0x3e2c8, 0x3e2fc,
1874                 0x3e600, 0x3e630,
1875                 0x3ea00, 0x3eabc,
1876                 0x3eb00, 0x3eb10,
1877                 0x3eb20, 0x3eb30,
1878                 0x3eb40, 0x3eb50,
1879                 0x3eb60, 0x3eb70,
1880                 0x3f000, 0x3f028,
1881                 0x3f030, 0x3f048,
1882                 0x3f060, 0x3f068,
1883                 0x3f070, 0x3f09c,
1884                 0x3f0f0, 0x3f128,
1885                 0x3f130, 0x3f148,
1886                 0x3f160, 0x3f168,
1887                 0x3f170, 0x3f19c,
1888                 0x3f1f0, 0x3f238,
1889                 0x3f240, 0x3f240,
1890                 0x3f248, 0x3f250,
1891                 0x3f25c, 0x3f264,
1892                 0x3f270, 0x3f2b8,
1893                 0x3f2c0, 0x3f2e4,
1894                 0x3f2f8, 0x3f338,
1895                 0x3f340, 0x3f340,
1896                 0x3f348, 0x3f350,
1897                 0x3f35c, 0x3f364,
1898                 0x3f370, 0x3f3b8,
1899                 0x3f3c0, 0x3f3e4,
1900                 0x3f3f8, 0x3f428,
1901                 0x3f430, 0x3f448,
1902                 0x3f460, 0x3f468,
1903                 0x3f470, 0x3f49c,
1904                 0x3f4f0, 0x3f528,
1905                 0x3f530, 0x3f548,
1906                 0x3f560, 0x3f568,
1907                 0x3f570, 0x3f59c,
1908                 0x3f5f0, 0x3f638,
1909                 0x3f640, 0x3f640,
1910                 0x3f648, 0x3f650,
1911                 0x3f65c, 0x3f664,
1912                 0x3f670, 0x3f6b8,
1913                 0x3f6c0, 0x3f6e4,
1914                 0x3f6f8, 0x3f738,
1915                 0x3f740, 0x3f740,
1916                 0x3f748, 0x3f750,
1917                 0x3f75c, 0x3f764,
1918                 0x3f770, 0x3f7b8,
1919                 0x3f7c0, 0x3f7e4,
1920                 0x3f7f8, 0x3f7fc,
1921                 0x3f814, 0x3f814,
1922                 0x3f82c, 0x3f82c,
1923                 0x3f880, 0x3f88c,
1924                 0x3f8e8, 0x3f8ec,
1925                 0x3f900, 0x3f928,
1926                 0x3f930, 0x3f948,
1927                 0x3f960, 0x3f968,
1928                 0x3f970, 0x3f99c,
1929                 0x3f9f0, 0x3fa38,
1930                 0x3fa40, 0x3fa40,
1931                 0x3fa48, 0x3fa50,
1932                 0x3fa5c, 0x3fa64,
1933                 0x3fa70, 0x3fab8,
1934                 0x3fac0, 0x3fae4,
1935                 0x3faf8, 0x3fb10,
1936                 0x3fb28, 0x3fb28,
1937                 0x3fb3c, 0x3fb50,
1938                 0x3fbf0, 0x3fc10,
1939                 0x3fc28, 0x3fc28,
1940                 0x3fc3c, 0x3fc50,
1941                 0x3fcf0, 0x3fcfc,
1942                 0x40000, 0x4000c,
1943                 0x40040, 0x40050,
1944                 0x40060, 0x40068,
1945                 0x4007c, 0x4008c,
1946                 0x40094, 0x400b0,
1947                 0x400c0, 0x40144,
1948                 0x40180, 0x4018c,
1949                 0x40200, 0x40254,
1950                 0x40260, 0x40264,
1951                 0x40270, 0x40288,
1952                 0x40290, 0x40298,
1953                 0x402ac, 0x402c8,
1954                 0x402d0, 0x402e0,
1955                 0x402f0, 0x402f0,
1956                 0x40300, 0x4033c,
1957                 0x403f8, 0x403fc,
1958                 0x41304, 0x413c4,
1959                 0x41400, 0x4140c,
1960                 0x41414, 0x4141c,
1961                 0x41480, 0x414d0,
1962                 0x44000, 0x44054,
1963                 0x4405c, 0x44078,
1964                 0x440c0, 0x44174,
1965                 0x44180, 0x441ac,
1966                 0x441b4, 0x441b8,
1967                 0x441c0, 0x44254,
1968                 0x4425c, 0x44278,
1969                 0x442c0, 0x44374,
1970                 0x44380, 0x443ac,
1971                 0x443b4, 0x443b8,
1972                 0x443c0, 0x44454,
1973                 0x4445c, 0x44478,
1974                 0x444c0, 0x44574,
1975                 0x44580, 0x445ac,
1976                 0x445b4, 0x445b8,
1977                 0x445c0, 0x44654,
1978                 0x4465c, 0x44678,
1979                 0x446c0, 0x44774,
1980                 0x44780, 0x447ac,
1981                 0x447b4, 0x447b8,
1982                 0x447c0, 0x44854,
1983                 0x4485c, 0x44878,
1984                 0x448c0, 0x44974,
1985                 0x44980, 0x449ac,
1986                 0x449b4, 0x449b8,
1987                 0x449c0, 0x449fc,
1988                 0x45000, 0x45004,
1989                 0x45010, 0x45030,
1990                 0x45040, 0x45060,
1991                 0x45068, 0x45068,
1992                 0x45080, 0x45084,
1993                 0x450a0, 0x450b0,
1994                 0x45200, 0x45204,
1995                 0x45210, 0x45230,
1996                 0x45240, 0x45260,
1997                 0x45268, 0x45268,
1998                 0x45280, 0x45284,
1999                 0x452a0, 0x452b0,
2000                 0x460c0, 0x460e4,
2001                 0x47000, 0x4703c,
2002                 0x47044, 0x4708c,
2003                 0x47200, 0x47250,
2004                 0x47400, 0x47408,
2005                 0x47414, 0x47420,
2006                 0x47600, 0x47618,
2007                 0x47800, 0x47814,
2008                 0x48000, 0x4800c,
2009                 0x48040, 0x48050,
2010                 0x48060, 0x48068,
2011                 0x4807c, 0x4808c,
2012                 0x48094, 0x480b0,
2013                 0x480c0, 0x48144,
2014                 0x48180, 0x4818c,
2015                 0x48200, 0x48254,
2016                 0x48260, 0x48264,
2017                 0x48270, 0x48288,
2018                 0x48290, 0x48298,
2019                 0x482ac, 0x482c8,
2020                 0x482d0, 0x482e0,
2021                 0x482f0, 0x482f0,
2022                 0x48300, 0x4833c,
2023                 0x483f8, 0x483fc,
2024                 0x49304, 0x493c4,
2025                 0x49400, 0x4940c,
2026                 0x49414, 0x4941c,
2027                 0x49480, 0x494d0,
2028                 0x4c000, 0x4c054,
2029                 0x4c05c, 0x4c078,
2030                 0x4c0c0, 0x4c174,
2031                 0x4c180, 0x4c1ac,
2032                 0x4c1b4, 0x4c1b8,
2033                 0x4c1c0, 0x4c254,
2034                 0x4c25c, 0x4c278,
2035                 0x4c2c0, 0x4c374,
2036                 0x4c380, 0x4c3ac,
2037                 0x4c3b4, 0x4c3b8,
2038                 0x4c3c0, 0x4c454,
2039                 0x4c45c, 0x4c478,
2040                 0x4c4c0, 0x4c574,
2041                 0x4c580, 0x4c5ac,
2042                 0x4c5b4, 0x4c5b8,
2043                 0x4c5c0, 0x4c654,
2044                 0x4c65c, 0x4c678,
2045                 0x4c6c0, 0x4c774,
2046                 0x4c780, 0x4c7ac,
2047                 0x4c7b4, 0x4c7b8,
2048                 0x4c7c0, 0x4c854,
2049                 0x4c85c, 0x4c878,
2050                 0x4c8c0, 0x4c974,
2051                 0x4c980, 0x4c9ac,
2052                 0x4c9b4, 0x4c9b8,
2053                 0x4c9c0, 0x4c9fc,
2054                 0x4d000, 0x4d004,
2055                 0x4d010, 0x4d030,
2056                 0x4d040, 0x4d060,
2057                 0x4d068, 0x4d068,
2058                 0x4d080, 0x4d084,
2059                 0x4d0a0, 0x4d0b0,
2060                 0x4d200, 0x4d204,
2061                 0x4d210, 0x4d230,
2062                 0x4d240, 0x4d260,
2063                 0x4d268, 0x4d268,
2064                 0x4d280, 0x4d284,
2065                 0x4d2a0, 0x4d2b0,
2066                 0x4e0c0, 0x4e0e4,
2067                 0x4f000, 0x4f03c,
2068                 0x4f044, 0x4f08c,
2069                 0x4f200, 0x4f250,
2070                 0x4f400, 0x4f408,
2071                 0x4f414, 0x4f420,
2072                 0x4f600, 0x4f618,
2073                 0x4f800, 0x4f814,
2074                 0x50000, 0x50084,
2075                 0x50090, 0x500cc,
2076                 0x50400, 0x50400,
2077                 0x50800, 0x50884,
2078                 0x50890, 0x508cc,
2079                 0x50c00, 0x50c00,
2080                 0x51000, 0x5101c,
2081                 0x51300, 0x51308,
2082         };
2083
2084         static const unsigned int t6_reg_ranges[] = {
2085                 0x1008, 0x101c,
2086                 0x1024, 0x10a8,
2087                 0x10b4, 0x10f8,
2088                 0x1100, 0x1114,
2089                 0x111c, 0x112c,
2090                 0x1138, 0x113c,
2091                 0x1144, 0x114c,
2092                 0x1180, 0x1184,
2093                 0x1190, 0x1194,
2094                 0x11a0, 0x11a4,
2095                 0x11b0, 0x11b4,
2096                 0x11fc, 0x1274,
2097                 0x1280, 0x133c,
2098                 0x1800, 0x18fc,
2099                 0x3000, 0x302c,
2100                 0x3060, 0x30b0,
2101                 0x30b8, 0x30d8,
2102                 0x30e0, 0x30fc,
2103                 0x3140, 0x357c,
2104                 0x35a8, 0x35cc,
2105                 0x35ec, 0x35ec,
2106                 0x3600, 0x5624,
2107                 0x56cc, 0x56ec,
2108                 0x56f4, 0x5720,
2109                 0x5728, 0x575c,
2110                 0x580c, 0x5814,
2111                 0x5890, 0x589c,
2112                 0x58a4, 0x58ac,
2113                 0x58b8, 0x58bc,
2114                 0x5940, 0x595c,
2115                 0x5980, 0x598c,
2116                 0x59b0, 0x59c8,
2117                 0x59d0, 0x59dc,
2118                 0x59fc, 0x5a18,
2119                 0x5a60, 0x5a6c,
2120                 0x5a80, 0x5a8c,
2121                 0x5a94, 0x5a9c,
2122                 0x5b94, 0x5bfc,
2123                 0x5c10, 0x5e48,
2124                 0x5e50, 0x5e94,
2125                 0x5ea0, 0x5eb0,
2126                 0x5ec0, 0x5ec0,
2127                 0x5ec8, 0x5ed0,
2128                 0x5ee0, 0x5ee0,
2129                 0x5ef0, 0x5ef0,
2130                 0x5f00, 0x5f00,
2131                 0x6000, 0x6020,
2132                 0x6028, 0x6040,
2133                 0x6058, 0x609c,
2134                 0x60a8, 0x619c,
2135                 0x7700, 0x7798,
2136                 0x77c0, 0x7880,
2137                 0x78cc, 0x78fc,
2138                 0x7b00, 0x7b58,
2139                 0x7b60, 0x7b84,
2140                 0x7b8c, 0x7c54,
2141                 0x7d00, 0x7d38,
2142                 0x7d40, 0x7d84,
2143                 0x7d8c, 0x7ddc,
2144                 0x7de4, 0x7e04,
2145                 0x7e10, 0x7e1c,
2146                 0x7e24, 0x7e38,
2147                 0x7e40, 0x7e44,
2148                 0x7e4c, 0x7e78,
2149                 0x7e80, 0x7edc,
2150                 0x7ee8, 0x7efc,
2151                 0x8dc0, 0x8de4,
2152                 0x8df8, 0x8e04,
2153                 0x8e10, 0x8e84,
2154                 0x8ea0, 0x8f88,
2155                 0x8fb8, 0x9058,
2156                 0x9060, 0x9060,
2157                 0x9068, 0x90f8,
2158                 0x9100, 0x9124,
2159                 0x9400, 0x9470,
2160                 0x9600, 0x9600,
2161                 0x9608, 0x9638,
2162                 0x9640, 0x9704,
2163                 0x9710, 0x971c,
2164                 0x9800, 0x9808,
2165                 0x9820, 0x983c,
2166                 0x9850, 0x9864,
2167                 0x9c00, 0x9c6c,
2168                 0x9c80, 0x9cec,
2169                 0x9d00, 0x9d6c,
2170                 0x9d80, 0x9dec,
2171                 0x9e00, 0x9e6c,
2172                 0x9e80, 0x9eec,
2173                 0x9f00, 0x9f6c,
2174                 0x9f80, 0xa020,
2175                 0xd004, 0xd03c,
2176                 0xd100, 0xd118,
2177                 0xd200, 0xd214,
2178                 0xd220, 0xd234,
2179                 0xd240, 0xd254,
2180                 0xd260, 0xd274,
2181                 0xd280, 0xd294,
2182                 0xd2a0, 0xd2b4,
2183                 0xd2c0, 0xd2d4,
2184                 0xd2e0, 0xd2f4,
2185                 0xd300, 0xd31c,
2186                 0xdfc0, 0xdfe0,
2187                 0xe000, 0xf008,
2188                 0xf010, 0xf018,
2189                 0xf020, 0xf028,
2190                 0x11000, 0x11014,
2191                 0x11048, 0x1106c,
2192                 0x11074, 0x11088,
2193                 0x11098, 0x11120,
2194                 0x1112c, 0x1117c,
2195                 0x11190, 0x112e0,
2196                 0x11300, 0x1130c,
2197                 0x12000, 0x1206c,
2198                 0x19040, 0x1906c,
2199                 0x19078, 0x19080,
2200                 0x1908c, 0x190e8,
2201                 0x190f0, 0x190f8,
2202                 0x19100, 0x19110,
2203                 0x19120, 0x19124,
2204                 0x19150, 0x19194,
2205                 0x1919c, 0x191b0,
2206                 0x191d0, 0x191e8,
2207                 0x19238, 0x19290,
2208                 0x192a4, 0x192b0,
2209                 0x192bc, 0x192bc,
2210                 0x19348, 0x1934c,
2211                 0x193f8, 0x19418,
2212                 0x19420, 0x19428,
2213                 0x19430, 0x19444,
2214                 0x1944c, 0x1946c,
2215                 0x19474, 0x19474,
2216                 0x19490, 0x194cc,
2217                 0x194f0, 0x194f8,
2218                 0x19c00, 0x19c48,
2219                 0x19c50, 0x19c80,
2220                 0x19c94, 0x19c98,
2221                 0x19ca0, 0x19cbc,
2222                 0x19ce4, 0x19ce4,
2223                 0x19cf0, 0x19cf8,
2224                 0x19d00, 0x19d28,
2225                 0x19d50, 0x19d78,
2226                 0x19d94, 0x19d98,
2227                 0x19da0, 0x19dc8,
2228                 0x19df0, 0x19e10,
2229                 0x19e50, 0x19e6c,
2230                 0x19ea0, 0x19ebc,
2231                 0x19ec4, 0x19ef4,
2232                 0x19f04, 0x19f2c,
2233                 0x19f34, 0x19f34,
2234                 0x19f40, 0x19f50,
2235                 0x19f90, 0x19fac,
2236                 0x19fc4, 0x19fc8,
2237                 0x19fd0, 0x19fe4,
2238                 0x1a000, 0x1a004,
2239                 0x1a010, 0x1a06c,
2240                 0x1a0b0, 0x1a0e4,
2241                 0x1a0ec, 0x1a0f8,
2242                 0x1a100, 0x1a108,
2243                 0x1a114, 0x1a120,
2244                 0x1a128, 0x1a130,
2245                 0x1a138, 0x1a138,
2246                 0x1a190, 0x1a1c4,
2247                 0x1a1fc, 0x1a1fc,
2248                 0x1e008, 0x1e00c,
2249                 0x1e040, 0x1e044,
2250                 0x1e04c, 0x1e04c,
2251                 0x1e284, 0x1e290,
2252                 0x1e2c0, 0x1e2c0,
2253                 0x1e2e0, 0x1e2e0,
2254                 0x1e300, 0x1e384,
2255                 0x1e3c0, 0x1e3c8,
2256                 0x1e408, 0x1e40c,
2257                 0x1e440, 0x1e444,
2258                 0x1e44c, 0x1e44c,
2259                 0x1e684, 0x1e690,
2260                 0x1e6c0, 0x1e6c0,
2261                 0x1e6e0, 0x1e6e0,
2262                 0x1e700, 0x1e784,
2263                 0x1e7c0, 0x1e7c8,
2264                 0x1e808, 0x1e80c,
2265                 0x1e840, 0x1e844,
2266                 0x1e84c, 0x1e84c,
2267                 0x1ea84, 0x1ea90,
2268                 0x1eac0, 0x1eac0,
2269                 0x1eae0, 0x1eae0,
2270                 0x1eb00, 0x1eb84,
2271                 0x1ebc0, 0x1ebc8,
2272                 0x1ec08, 0x1ec0c,
2273                 0x1ec40, 0x1ec44,
2274                 0x1ec4c, 0x1ec4c,
2275                 0x1ee84, 0x1ee90,
2276                 0x1eec0, 0x1eec0,
2277                 0x1eee0, 0x1eee0,
2278                 0x1ef00, 0x1ef84,
2279                 0x1efc0, 0x1efc8,
2280                 0x1f008, 0x1f00c,
2281                 0x1f040, 0x1f044,
2282                 0x1f04c, 0x1f04c,
2283                 0x1f284, 0x1f290,
2284                 0x1f2c0, 0x1f2c0,
2285                 0x1f2e0, 0x1f2e0,
2286                 0x1f300, 0x1f384,
2287                 0x1f3c0, 0x1f3c8,
2288                 0x1f408, 0x1f40c,
2289                 0x1f440, 0x1f444,
2290                 0x1f44c, 0x1f44c,
2291                 0x1f684, 0x1f690,
2292                 0x1f6c0, 0x1f6c0,
2293                 0x1f6e0, 0x1f6e0,
2294                 0x1f700, 0x1f784,
2295                 0x1f7c0, 0x1f7c8,
2296                 0x1f808, 0x1f80c,
2297                 0x1f840, 0x1f844,
2298                 0x1f84c, 0x1f84c,
2299                 0x1fa84, 0x1fa90,
2300                 0x1fac0, 0x1fac0,
2301                 0x1fae0, 0x1fae0,
2302                 0x1fb00, 0x1fb84,
2303                 0x1fbc0, 0x1fbc8,
2304                 0x1fc08, 0x1fc0c,
2305                 0x1fc40, 0x1fc44,
2306                 0x1fc4c, 0x1fc4c,
2307                 0x1fe84, 0x1fe90,
2308                 0x1fec0, 0x1fec0,
2309                 0x1fee0, 0x1fee0,
2310                 0x1ff00, 0x1ff84,
2311                 0x1ffc0, 0x1ffc8,
2312                 0x30000, 0x30030,
2313                 0x30100, 0x30168,
2314                 0x30190, 0x301a0,
2315                 0x301a8, 0x301b8,
2316                 0x301c4, 0x301c8,
2317                 0x301d0, 0x301d0,
2318                 0x30200, 0x30320,
2319                 0x30400, 0x304b4,
2320                 0x304c0, 0x3052c,
2321                 0x30540, 0x3061c,
2322                 0x30800, 0x308a0,
2323                 0x308c0, 0x30908,
2324                 0x30910, 0x309b8,
2325                 0x30a00, 0x30a04,
2326                 0x30a0c, 0x30a14,
2327                 0x30a1c, 0x30a2c,
2328                 0x30a44, 0x30a50,
2329                 0x30a74, 0x30a74,
2330                 0x30a7c, 0x30afc,
2331                 0x30b08, 0x30c24,
2332                 0x30d00, 0x30d14,
2333                 0x30d1c, 0x30d3c,
2334                 0x30d44, 0x30d4c,
2335                 0x30d54, 0x30d74,
2336                 0x30d7c, 0x30d7c,
2337                 0x30de0, 0x30de0,
2338                 0x30e00, 0x30ed4,
2339                 0x30f00, 0x30fa4,
2340                 0x30fc0, 0x30fc4,
2341                 0x31000, 0x31004,
2342                 0x31080, 0x310fc,
2343                 0x31208, 0x31220,
2344                 0x3123c, 0x31254,
2345                 0x31300, 0x31300,
2346                 0x31308, 0x3131c,
2347                 0x31338, 0x3133c,
2348                 0x31380, 0x31380,
2349                 0x31388, 0x313a8,
2350                 0x313b4, 0x313b4,
2351                 0x31400, 0x31420,
2352                 0x31438, 0x3143c,
2353                 0x31480, 0x31480,
2354                 0x314a8, 0x314a8,
2355                 0x314b0, 0x314b4,
2356                 0x314c8, 0x314d4,
2357                 0x31a40, 0x31a4c,
2358                 0x31af0, 0x31b20,
2359                 0x31b38, 0x31b3c,
2360                 0x31b80, 0x31b80,
2361                 0x31ba8, 0x31ba8,
2362                 0x31bb0, 0x31bb4,
2363                 0x31bc8, 0x31bd4,
2364                 0x32140, 0x3218c,
2365                 0x321f0, 0x321f4,
2366                 0x32200, 0x32200,
2367                 0x32218, 0x32218,
2368                 0x32400, 0x32400,
2369                 0x32408, 0x3241c,
2370                 0x32618, 0x32620,
2371                 0x32664, 0x32664,
2372                 0x326a8, 0x326a8,
2373                 0x326ec, 0x326ec,
2374                 0x32a00, 0x32abc,
2375                 0x32b00, 0x32b18,
2376                 0x32b20, 0x32b38,
2377                 0x32b40, 0x32b58,
2378                 0x32b60, 0x32b78,
2379                 0x32c00, 0x32c00,
2380                 0x32c08, 0x32c3c,
2381                 0x33000, 0x3302c,
2382                 0x33034, 0x33050,
2383                 0x33058, 0x33058,
2384                 0x33060, 0x3308c,
2385                 0x3309c, 0x330ac,
2386                 0x330c0, 0x330c0,
2387                 0x330c8, 0x330d0,
2388                 0x330d8, 0x330e0,
2389                 0x330ec, 0x3312c,
2390                 0x33134, 0x33150,
2391                 0x33158, 0x33158,
2392                 0x33160, 0x3318c,
2393                 0x3319c, 0x331ac,
2394                 0x331c0, 0x331c0,
2395                 0x331c8, 0x331d0,
2396                 0x331d8, 0x331e0,
2397                 0x331ec, 0x33290,
2398                 0x33298, 0x332c4,
2399                 0x332e4, 0x33390,
2400                 0x33398, 0x333c4,
2401                 0x333e4, 0x3342c,
2402                 0x33434, 0x33450,
2403                 0x33458, 0x33458,
2404                 0x33460, 0x3348c,
2405                 0x3349c, 0x334ac,
2406                 0x334c0, 0x334c0,
2407                 0x334c8, 0x334d0,
2408                 0x334d8, 0x334e0,
2409                 0x334ec, 0x3352c,
2410                 0x33534, 0x33550,
2411                 0x33558, 0x33558,
2412                 0x33560, 0x3358c,
2413                 0x3359c, 0x335ac,
2414                 0x335c0, 0x335c0,
2415                 0x335c8, 0x335d0,
2416                 0x335d8, 0x335e0,
2417                 0x335ec, 0x33690,
2418                 0x33698, 0x336c4,
2419                 0x336e4, 0x33790,
2420                 0x33798, 0x337c4,
2421                 0x337e4, 0x337fc,
2422                 0x33814, 0x33814,
2423                 0x33854, 0x33868,
2424                 0x33880, 0x3388c,
2425                 0x338c0, 0x338d0,
2426                 0x338e8, 0x338ec,
2427                 0x33900, 0x3392c,
2428                 0x33934, 0x33950,
2429                 0x33958, 0x33958,
2430                 0x33960, 0x3398c,
2431                 0x3399c, 0x339ac,
2432                 0x339c0, 0x339c0,
2433                 0x339c8, 0x339d0,
2434                 0x339d8, 0x339e0,
2435                 0x339ec, 0x33a90,
2436                 0x33a98, 0x33ac4,
2437                 0x33ae4, 0x33b10,
2438                 0x33b24, 0x33b28,
2439                 0x33b38, 0x33b50,
2440                 0x33bf0, 0x33c10,
2441                 0x33c24, 0x33c28,
2442                 0x33c38, 0x33c50,
2443                 0x33cf0, 0x33cfc,
2444                 0x34000, 0x34030,
2445                 0x34100, 0x34168,
2446                 0x34190, 0x341a0,
2447                 0x341a8, 0x341b8,
2448                 0x341c4, 0x341c8,
2449                 0x341d0, 0x341d0,
2450                 0x34200, 0x34320,
2451                 0x34400, 0x344b4,
2452                 0x344c0, 0x3452c,
2453                 0x34540, 0x3461c,
2454                 0x34800, 0x348a0,
2455                 0x348c0, 0x34908,
2456                 0x34910, 0x349b8,
2457                 0x34a00, 0x34a04,
2458                 0x34a0c, 0x34a14,
2459                 0x34a1c, 0x34a2c,
2460                 0x34a44, 0x34a50,
2461                 0x34a74, 0x34a74,
2462                 0x34a7c, 0x34afc,
2463                 0x34b08, 0x34c24,
2464                 0x34d00, 0x34d14,
2465                 0x34d1c, 0x34d3c,
2466                 0x34d44, 0x34d4c,
2467                 0x34d54, 0x34d74,
2468                 0x34d7c, 0x34d7c,
2469                 0x34de0, 0x34de0,
2470                 0x34e00, 0x34ed4,
2471                 0x34f00, 0x34fa4,
2472                 0x34fc0, 0x34fc4,
2473                 0x35000, 0x35004,
2474                 0x35080, 0x350fc,
2475                 0x35208, 0x35220,
2476                 0x3523c, 0x35254,
2477                 0x35300, 0x35300,
2478                 0x35308, 0x3531c,
2479                 0x35338, 0x3533c,
2480                 0x35380, 0x35380,
2481                 0x35388, 0x353a8,
2482                 0x353b4, 0x353b4,
2483                 0x35400, 0x35420,
2484                 0x35438, 0x3543c,
2485                 0x35480, 0x35480,
2486                 0x354a8, 0x354a8,
2487                 0x354b0, 0x354b4,
2488                 0x354c8, 0x354d4,
2489                 0x35a40, 0x35a4c,
2490                 0x35af0, 0x35b20,
2491                 0x35b38, 0x35b3c,
2492                 0x35b80, 0x35b80,
2493                 0x35ba8, 0x35ba8,
2494                 0x35bb0, 0x35bb4,
2495                 0x35bc8, 0x35bd4,
2496                 0x36140, 0x3618c,
2497                 0x361f0, 0x361f4,
2498                 0x36200, 0x36200,
2499                 0x36218, 0x36218,
2500                 0x36400, 0x36400,
2501                 0x36408, 0x3641c,
2502                 0x36618, 0x36620,
2503                 0x36664, 0x36664,
2504                 0x366a8, 0x366a8,
2505                 0x366ec, 0x366ec,
2506                 0x36a00, 0x36abc,
2507                 0x36b00, 0x36b18,
2508                 0x36b20, 0x36b38,
2509                 0x36b40, 0x36b58,
2510                 0x36b60, 0x36b78,
2511                 0x36c00, 0x36c00,
2512                 0x36c08, 0x36c3c,
2513                 0x37000, 0x3702c,
2514                 0x37034, 0x37050,
2515                 0x37058, 0x37058,
2516                 0x37060, 0x3708c,
2517                 0x3709c, 0x370ac,
2518                 0x370c0, 0x370c0,
2519                 0x370c8, 0x370d0,
2520                 0x370d8, 0x370e0,
2521                 0x370ec, 0x3712c,
2522                 0x37134, 0x37150,
2523                 0x37158, 0x37158,
2524                 0x37160, 0x3718c,
2525                 0x3719c, 0x371ac,
2526                 0x371c0, 0x371c0,
2527                 0x371c8, 0x371d0,
2528                 0x371d8, 0x371e0,
2529                 0x371ec, 0x37290,
2530                 0x37298, 0x372c4,
2531                 0x372e4, 0x37390,
2532                 0x37398, 0x373c4,
2533                 0x373e4, 0x3742c,
2534                 0x37434, 0x37450,
2535                 0x37458, 0x37458,
2536                 0x37460, 0x3748c,
2537                 0x3749c, 0x374ac,
2538                 0x374c0, 0x374c0,
2539                 0x374c8, 0x374d0,
2540                 0x374d8, 0x374e0,
2541                 0x374ec, 0x3752c,
2542                 0x37534, 0x37550,
2543                 0x37558, 0x37558,
2544                 0x37560, 0x3758c,
2545                 0x3759c, 0x375ac,
2546                 0x375c0, 0x375c0,
2547                 0x375c8, 0x375d0,
2548                 0x375d8, 0x375e0,
2549                 0x375ec, 0x37690,
2550                 0x37698, 0x376c4,
2551                 0x376e4, 0x37790,
2552                 0x37798, 0x377c4,
2553                 0x377e4, 0x377fc,
2554                 0x37814, 0x37814,
2555                 0x37854, 0x37868,
2556                 0x37880, 0x3788c,
2557                 0x378c0, 0x378d0,
2558                 0x378e8, 0x378ec,
2559                 0x37900, 0x3792c,
2560                 0x37934, 0x37950,
2561                 0x37958, 0x37958,
2562                 0x37960, 0x3798c,
2563                 0x3799c, 0x379ac,
2564                 0x379c0, 0x379c0,
2565                 0x379c8, 0x379d0,
2566                 0x379d8, 0x379e0,
2567                 0x379ec, 0x37a90,
2568                 0x37a98, 0x37ac4,
2569                 0x37ae4, 0x37b10,
2570                 0x37b24, 0x37b28,
2571                 0x37b38, 0x37b50,
2572                 0x37bf0, 0x37c10,
2573                 0x37c24, 0x37c28,
2574                 0x37c38, 0x37c50,
2575                 0x37cf0, 0x37cfc,
2576                 0x40040, 0x40040,
2577                 0x40080, 0x40084,
2578                 0x40100, 0x40100,
2579                 0x40140, 0x401bc,
2580                 0x40200, 0x40214,
2581                 0x40228, 0x40228,
2582                 0x40240, 0x40258,
2583                 0x40280, 0x40280,
2584                 0x40304, 0x40304,
2585                 0x40330, 0x4033c,
2586                 0x41304, 0x413c8,
2587                 0x413d0, 0x413dc,
2588                 0x413f0, 0x413f0,
2589                 0x41400, 0x4140c,
2590                 0x41414, 0x4141c,
2591                 0x41480, 0x414d0,
2592                 0x44000, 0x4407c,
2593                 0x440c0, 0x441ac,
2594                 0x441b4, 0x4427c,
2595                 0x442c0, 0x443ac,
2596                 0x443b4, 0x4447c,
2597                 0x444c0, 0x445ac,
2598                 0x445b4, 0x4467c,
2599                 0x446c0, 0x447ac,
2600                 0x447b4, 0x4487c,
2601                 0x448c0, 0x449ac,
2602                 0x449b4, 0x44a7c,
2603                 0x44ac0, 0x44bac,
2604                 0x44bb4, 0x44c7c,
2605                 0x44cc0, 0x44dac,
2606                 0x44db4, 0x44e7c,
2607                 0x44ec0, 0x44fac,
2608                 0x44fb4, 0x4507c,
2609                 0x450c0, 0x451ac,
2610                 0x451b4, 0x451fc,
2611                 0x45800, 0x45804,
2612                 0x45810, 0x45830,
2613                 0x45840, 0x45860,
2614                 0x45868, 0x45868,
2615                 0x45880, 0x45884,
2616                 0x458a0, 0x458b0,
2617                 0x45a00, 0x45a04,
2618                 0x45a10, 0x45a30,
2619                 0x45a40, 0x45a60,
2620                 0x45a68, 0x45a68,
2621                 0x45a80, 0x45a84,
2622                 0x45aa0, 0x45ab0,
2623                 0x460c0, 0x460e4,
2624                 0x47000, 0x4703c,
2625                 0x47044, 0x4708c,
2626                 0x47200, 0x47250,
2627                 0x47400, 0x47408,
2628                 0x47414, 0x47420,
2629                 0x47600, 0x47618,
2630                 0x47800, 0x47814,
2631                 0x47820, 0x4782c,
2632                 0x50000, 0x50084,
2633                 0x50090, 0x500cc,
2634                 0x50300, 0x50384,
2635                 0x50400, 0x50400,
2636                 0x50800, 0x50884,
2637                 0x50890, 0x508cc,
2638                 0x50b00, 0x50b84,
2639                 0x50c00, 0x50c00,
2640                 0x51000, 0x51020,
2641                 0x51028, 0x510b0,
2642                 0x51300, 0x51324,
2643         };
2644
2645         u32 *buf_end = (u32 *)((char *)buf + buf_size);
2646         const unsigned int *reg_ranges;
2647         int reg_ranges_size, range;
2648         unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
2649
2650         /* Select the right set of register ranges to dump depending on the
2651          * adapter chip type.
2652          */
2653         switch (chip_version) {
2654         case CHELSIO_T4:
2655                 reg_ranges = t4_reg_ranges;
2656                 reg_ranges_size = ARRAY_SIZE(t4_reg_ranges);
2657                 break;
2658
2659         case CHELSIO_T5:
2660                 reg_ranges = t5_reg_ranges;
2661                 reg_ranges_size = ARRAY_SIZE(t5_reg_ranges);
2662                 break;
2663
2664         case CHELSIO_T6:
2665                 reg_ranges = t6_reg_ranges;
2666                 reg_ranges_size = ARRAY_SIZE(t6_reg_ranges);
2667                 break;
2668
2669         default:
2670                 dev_err(adap->pdev_dev,
2671                         "Unsupported chip version %d\n", chip_version);
2672                 return;
2673         }
2674
2675         /* Clear the register buffer and insert the appropriate register
2676          * values selected by the above register ranges.
2677          */
2678         memset(buf, 0, buf_size);
2679         for (range = 0; range < reg_ranges_size; range += 2) {
2680                 unsigned int reg = reg_ranges[range];
2681                 unsigned int last_reg = reg_ranges[range + 1];
2682                 u32 *bufp = (u32 *)((char *)buf + reg);
2683
2684                 /* Iterate across the register range filling in the register
2685                  * buffer but don't write past the end of the register buffer.
2686                  */
2687                 while (reg <= last_reg && bufp < buf_end) {
2688                         *bufp++ = t4_read_reg(adap, reg);
2689                         reg += sizeof(u32);
2690                 }
2691         }
2692 }
2693
2694 #define EEPROM_STAT_ADDR   0x7bfc
2695 #define VPD_BASE           0x400
2696 #define VPD_BASE_OLD       0
2697 #define VPD_LEN            1024
2698 #define CHELSIO_VPD_UNIQUE_ID 0x82
2699
2700 /**
2701  * t4_eeprom_ptov - translate a physical EEPROM address to virtual
2702  * @phys_addr: the physical EEPROM address
2703  * @fn: the PCI function number
2704  * @sz: size of function-specific area
2705  *
2706  * Translate a physical EEPROM address to virtual.  The first 1K is
2707  * accessed through virtual addresses starting at 31K, the rest is
2708  * accessed through virtual addresses starting at 0.
2709  *
2710  * The mapping is as follows:
2711  * [0..1K) -> [31K..32K)
2712  * [1K..1K+A) -> [31K-A..31K)
2713  * [1K+A..ES) -> [0..ES-A-1K)
2714  *
2715  * where A = @fn * @sz, and ES = EEPROM size.
2716  */
2717 int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2718 {
2719         fn *= sz;
2720         if (phys_addr < 1024)
2721                 return phys_addr + (31 << 10);
2722         if (phys_addr < 1024 + fn)
2723                 return 31744 - fn + phys_addr - 1024;
2724         if (phys_addr < EEPROMSIZE)
2725                 return phys_addr - 1024 - fn;
2726         return -EINVAL;
2727 }
2728
2729 /**
2730  *      t4_seeprom_wp - enable/disable EEPROM write protection
2731  *      @adapter: the adapter
2732  *      @enable: whether to enable or disable write protection
2733  *
2734  *      Enables or disables write protection on the serial EEPROM.
2735  */
2736 int t4_seeprom_wp(struct adapter *adapter, bool enable)
2737 {
2738         unsigned int v = enable ? 0xc : 0;
2739         int ret = pci_write_vpd(adapter->pdev, EEPROM_STAT_ADDR, 4, &v);
2740         return ret < 0 ? ret : 0;
2741 }
2742
2743 /**
2744  *      t4_get_raw_vpd_params - read VPD parameters from VPD EEPROM
2745  *      @adapter: adapter to read
2746  *      @p: where to store the parameters
2747  *
2748  *      Reads card parameters stored in VPD EEPROM.
2749  */
2750 int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p)
2751 {
2752         int i, ret = 0, addr;
2753         int ec, sn, pn, na;
2754         u8 *vpd, csum;
2755         unsigned int vpdr_len, kw_offset, id_len;
2756
2757         vpd = vmalloc(VPD_LEN);
2758         if (!vpd)
2759                 return -ENOMEM;
2760
2761         /* Card information normally starts at VPD_BASE but early cards had
2762          * it at 0.
2763          */
2764         ret = pci_read_vpd(adapter->pdev, VPD_BASE, sizeof(u32), vpd);
2765         if (ret < 0)
2766                 goto out;
2767
2768         /* The VPD shall have a unique identifier specified by the PCI SIG.
2769          * For chelsio adapters, the identifier is 0x82. The first byte of a VPD
2770          * shall be CHELSIO_VPD_UNIQUE_ID (0x82). The VPD programming software
2771          * is expected to automatically put this entry at the
2772          * beginning of the VPD.
2773          */
2774         addr = *vpd == CHELSIO_VPD_UNIQUE_ID ? VPD_BASE : VPD_BASE_OLD;
2775
2776         ret = pci_read_vpd(adapter->pdev, addr, VPD_LEN, vpd);
2777         if (ret < 0)
2778                 goto out;
2779
2780         if (vpd[0] != PCI_VPD_LRDT_ID_STRING) {
2781                 dev_err(adapter->pdev_dev, "missing VPD ID string\n");
2782                 ret = -EINVAL;
2783                 goto out;
2784         }
2785
2786         id_len = pci_vpd_lrdt_size(vpd);
2787         if (id_len > ID_LEN)
2788                 id_len = ID_LEN;
2789
2790         i = pci_vpd_find_tag(vpd, 0, VPD_LEN, PCI_VPD_LRDT_RO_DATA);
2791         if (i < 0) {
2792                 dev_err(adapter->pdev_dev, "missing VPD-R section\n");
2793                 ret = -EINVAL;
2794                 goto out;
2795         }
2796
2797         vpdr_len = pci_vpd_lrdt_size(&vpd[i]);
2798         kw_offset = i + PCI_VPD_LRDT_TAG_SIZE;
2799         if (vpdr_len + kw_offset > VPD_LEN) {
2800                 dev_err(adapter->pdev_dev, "bad VPD-R length %u\n", vpdr_len);
2801                 ret = -EINVAL;
2802                 goto out;
2803         }
2804
2805 #define FIND_VPD_KW(var, name) do { \
2806         var = pci_vpd_find_info_keyword(vpd, kw_offset, vpdr_len, name); \
2807         if (var < 0) { \
2808                 dev_err(adapter->pdev_dev, "missing VPD keyword " name "\n"); \
2809                 ret = -EINVAL; \
2810                 goto out; \
2811         } \
2812         var += PCI_VPD_INFO_FLD_HDR_SIZE; \
2813 } while (0)
2814
2815         FIND_VPD_KW(i, "RV");
2816         for (csum = 0; i >= 0; i--)
2817                 csum += vpd[i];
2818
2819         if (csum) {
2820                 dev_err(adapter->pdev_dev,
2821                         "corrupted VPD EEPROM, actual csum %u\n", csum);
2822                 ret = -EINVAL;
2823                 goto out;
2824         }
2825
2826         FIND_VPD_KW(ec, "EC");
2827         FIND_VPD_KW(sn, "SN");
2828         FIND_VPD_KW(pn, "PN");
2829         FIND_VPD_KW(na, "NA");
2830 #undef FIND_VPD_KW
2831
2832         memcpy(p->id, vpd + PCI_VPD_LRDT_TAG_SIZE, id_len);
2833         strim(p->id);
2834         memcpy(p->ec, vpd + ec, EC_LEN);
2835         strim(p->ec);
2836         i = pci_vpd_info_field_size(vpd + sn - PCI_VPD_INFO_FLD_HDR_SIZE);
2837         memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN));
2838         strim(p->sn);
2839         i = pci_vpd_info_field_size(vpd + pn - PCI_VPD_INFO_FLD_HDR_SIZE);
2840         memcpy(p->pn, vpd + pn, min(i, PN_LEN));
2841         strim(p->pn);
2842         memcpy(p->na, vpd + na, min(i, MACADDR_LEN));
2843         strim((char *)p->na);
2844
2845 out:
2846         vfree(vpd);
2847         return ret < 0 ? ret : 0;
2848 }
2849
2850 /**
2851  *      t4_get_vpd_params - read VPD parameters & retrieve Core Clock
2852  *      @adapter: adapter to read
2853  *      @p: where to store the parameters
2854  *
2855  *      Reads card parameters stored in VPD EEPROM and retrieves the Core
2856  *      Clock.  This can only be called after a connection to the firmware
2857  *      is established.
2858  */
2859 int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p)
2860 {
2861         u32 cclk_param, cclk_val;
2862         int ret;
2863
2864         /* Grab the raw VPD parameters.
2865          */
2866         ret = t4_get_raw_vpd_params(adapter, p);
2867         if (ret)
2868                 return ret;
2869
2870         /* Ask firmware for the Core Clock since it knows how to translate the
2871          * Reference Clock ('V2') VPD field into a Core Clock value ...
2872          */
2873         cclk_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
2874                       FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CCLK));
2875         ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
2876                               1, &cclk_param, &cclk_val);
2877
2878         if (ret)
2879                 return ret;
2880         p->cclk = cclk_val;
2881
2882         return 0;
2883 }
2884
2885 /* serial flash and firmware constants */
2886 enum {
2887         SF_ATTEMPTS = 10,             /* max retries for SF operations */
2888
2889         /* flash command opcodes */
2890         SF_PROG_PAGE    = 2,          /* program page */
2891         SF_WR_DISABLE   = 4,          /* disable writes */
2892         SF_RD_STATUS    = 5,          /* read status register */
2893         SF_WR_ENABLE    = 6,          /* enable writes */
2894         SF_RD_DATA_FAST = 0xb,        /* read flash */
2895         SF_RD_ID        = 0x9f,       /* read ID */
2896         SF_ERASE_SECTOR = 0xd8,       /* erase sector */
2897 };
2898
2899 /**
2900  *      sf1_read - read data from the serial flash
2901  *      @adapter: the adapter
2902  *      @byte_cnt: number of bytes to read
2903  *      @cont: whether another operation will be chained
2904  *      @lock: whether to lock SF for PL access only
2905  *      @valp: where to store the read data
2906  *
2907  *      Reads up to 4 bytes of data from the serial flash.  The location of
2908  *      the read needs to be specified prior to calling this by issuing the
2909  *      appropriate commands to the serial flash.
2910  */
2911 static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont,
2912                     int lock, u32 *valp)
2913 {
2914         int ret;
2915
2916         if (!byte_cnt || byte_cnt > 4)
2917                 return -EINVAL;
2918         if (t4_read_reg(adapter, SF_OP_A) & SF_BUSY_F)
2919                 return -EBUSY;
2920         t4_write_reg(adapter, SF_OP_A, SF_LOCK_V(lock) |
2921                      SF_CONT_V(cont) | BYTECNT_V(byte_cnt - 1));
2922         ret = t4_wait_op_done(adapter, SF_OP_A, SF_BUSY_F, 0, SF_ATTEMPTS, 5);
2923         if (!ret)
2924                 *valp = t4_read_reg(adapter, SF_DATA_A);
2925         return ret;
2926 }
2927
2928 /**
2929  *      sf1_write - write data to the serial flash
2930  *      @adapter: the adapter
2931  *      @byte_cnt: number of bytes to write
2932  *      @cont: whether another operation will be chained
2933  *      @lock: whether to lock SF for PL access only
2934  *      @val: value to write
2935  *
2936  *      Writes up to 4 bytes of data to the serial flash.  The location of
2937  *      the write needs to be specified prior to calling this by issuing the
2938  *      appropriate commands to the serial flash.
2939  */
2940 static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont,
2941                      int lock, u32 val)
2942 {
2943         if (!byte_cnt || byte_cnt > 4)
2944                 return -EINVAL;
2945         if (t4_read_reg(adapter, SF_OP_A) & SF_BUSY_F)
2946                 return -EBUSY;
2947         t4_write_reg(adapter, SF_DATA_A, val);
2948         t4_write_reg(adapter, SF_OP_A, SF_LOCK_V(lock) |
2949                      SF_CONT_V(cont) | BYTECNT_V(byte_cnt - 1) | OP_V(1));
2950         return t4_wait_op_done(adapter, SF_OP_A, SF_BUSY_F, 0, SF_ATTEMPTS, 5);
2951 }
2952
2953 /**
2954  *      flash_wait_op - wait for a flash operation to complete
2955  *      @adapter: the adapter
2956  *      @attempts: max number of polls of the status register
2957  *      @delay: delay between polls in ms
2958  *
2959  *      Wait for a flash operation to complete by polling the status register.
2960  */
2961 static int flash_wait_op(struct adapter *adapter, int attempts, int delay)
2962 {
2963         int ret;
2964         u32 status;
2965
2966         while (1) {
2967                 if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 ||
2968                     (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0)
2969                         return ret;
2970                 if (!(status & 1))
2971                         return 0;
2972                 if (--attempts == 0)
2973                         return -EAGAIN;
2974                 if (delay)
2975                         msleep(delay);
2976         }
2977 }
2978
2979 /**
2980  *      t4_read_flash - read words from serial flash
2981  *      @adapter: the adapter
2982  *      @addr: the start address for the read
2983  *      @nwords: how many 32-bit words to read
2984  *      @data: where to store the read data
2985  *      @byte_oriented: whether to store data as bytes or as words
2986  *
2987  *      Read the specified number of 32-bit words from the serial flash.
2988  *      If @byte_oriented is set the read data is stored as a byte array
2989  *      (i.e., big-endian), otherwise as 32-bit words in the platform's
2990  *      natural endianness.
2991  */
2992 int t4_read_flash(struct adapter *adapter, unsigned int addr,
2993                   unsigned int nwords, u32 *data, int byte_oriented)
2994 {
2995         int ret;
2996
2997         if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3))
2998                 return -EINVAL;
2999
3000         addr = swab32(addr) | SF_RD_DATA_FAST;
3001
3002         if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 ||
3003             (ret = sf1_read(adapter, 1, 1, 0, data)) != 0)
3004                 return ret;
3005
3006         for ( ; nwords; nwords--, data++) {
3007                 ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data);
3008                 if (nwords == 1)
3009                         t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
3010                 if (ret)
3011                         return ret;
3012                 if (byte_oriented)
3013                         *data = (__force __u32)(cpu_to_be32(*data));
3014         }
3015         return 0;
3016 }
3017
3018 /**
3019  *      t4_write_flash - write up to a page of data to the serial flash
3020  *      @adapter: the adapter
3021  *      @addr: the start address to write
3022  *      @n: length of data to write in bytes
3023  *      @data: the data to write
3024  *
3025  *      Writes up to a page of data (256 bytes) to the serial flash starting
3026  *      at the given address.  All the data must be written to the same page.
3027  */
3028 static int t4_write_flash(struct adapter *adapter, unsigned int addr,
3029                           unsigned int n, const u8 *data)
3030 {
3031         int ret;
3032         u32 buf[64];
3033         unsigned int i, c, left, val, offset = addr & 0xff;
3034
3035         if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE)
3036                 return -EINVAL;
3037
3038         val = swab32(addr) | SF_PROG_PAGE;
3039
3040         if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
3041             (ret = sf1_write(adapter, 4, 1, 1, val)) != 0)
3042                 goto unlock;
3043
3044         for (left = n; left; left -= c) {
3045                 c = min(left, 4U);
3046                 for (val = 0, i = 0; i < c; ++i)
3047                         val = (val << 8) + *data++;
3048
3049                 ret = sf1_write(adapter, c, c != left, 1, val);
3050                 if (ret)
3051                         goto unlock;
3052         }
3053         ret = flash_wait_op(adapter, 8, 1);
3054         if (ret)
3055                 goto unlock;
3056
3057         t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
3058
3059         /* Read the page to verify the write succeeded */
3060         ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
3061         if (ret)
3062                 return ret;
3063
3064         if (memcmp(data - n, (u8 *)buf + offset, n)) {
3065                 dev_err(adapter->pdev_dev,
3066                         "failed to correctly write the flash page at %#x\n",
3067                         addr);
3068                 return -EIO;
3069         }
3070         return 0;
3071
3072 unlock:
3073         t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
3074         return ret;
3075 }
3076
3077 /**
3078  *      t4_get_fw_version - read the firmware version
3079  *      @adapter: the adapter
3080  *      @vers: where to place the version
3081  *
3082  *      Reads the FW version from flash.
3083  */
3084 int t4_get_fw_version(struct adapter *adapter, u32 *vers)
3085 {
3086         return t4_read_flash(adapter, FLASH_FW_START +
3087                              offsetof(struct fw_hdr, fw_ver), 1,
3088                              vers, 0);
3089 }
3090
3091 /**
3092  *      t4_get_bs_version - read the firmware bootstrap version
3093  *      @adapter: the adapter
3094  *      @vers: where to place the version
3095  *
3096  *      Reads the FW Bootstrap version from flash.
3097  */
3098 int t4_get_bs_version(struct adapter *adapter, u32 *vers)
3099 {
3100         return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START +
3101                              offsetof(struct fw_hdr, fw_ver), 1,
3102                              vers, 0);
3103 }
3104
3105 /**
3106  *      t4_get_tp_version - read the TP microcode version
3107  *      @adapter: the adapter
3108  *      @vers: where to place the version
3109  *
3110  *      Reads the TP microcode version from flash.
3111  */
3112 int t4_get_tp_version(struct adapter *adapter, u32 *vers)
3113 {
3114         return t4_read_flash(adapter, FLASH_FW_START +
3115                              offsetof(struct fw_hdr, tp_microcode_ver),
3116                              1, vers, 0);
3117 }
3118
3119 /**
3120  *      t4_get_exprom_version - return the Expansion ROM version (if any)
3121  *      @adapter: the adapter
3122  *      @vers: where to place the version
3123  *
3124  *      Reads the Expansion ROM header from FLASH and returns the version
3125  *      number (if present) through the @vers return value pointer.  We return
3126  *      this in the Firmware Version Format since it's convenient.  Return
3127  *      0 on success, -ENOENT if no Expansion ROM is present.
3128  */
3129 int t4_get_exprom_version(struct adapter *adap, u32 *vers)
3130 {
3131         struct exprom_header {
3132                 unsigned char hdr_arr[16];      /* must start with 0x55aa */
3133                 unsigned char hdr_ver[4];       /* Expansion ROM version */
3134         } *hdr;
3135         u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header),
3136                                            sizeof(u32))];
3137         int ret;
3138
3139         ret = t4_read_flash(adap, FLASH_EXP_ROM_START,
3140                             ARRAY_SIZE(exprom_header_buf), exprom_header_buf,
3141                             0);
3142         if (ret)
3143                 return ret;
3144
3145         hdr = (struct exprom_header *)exprom_header_buf;
3146         if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa)
3147                 return -ENOENT;
3148
3149         *vers = (FW_HDR_FW_VER_MAJOR_V(hdr->hdr_ver[0]) |
3150                  FW_HDR_FW_VER_MINOR_V(hdr->hdr_ver[1]) |
3151                  FW_HDR_FW_VER_MICRO_V(hdr->hdr_ver[2]) |
3152                  FW_HDR_FW_VER_BUILD_V(hdr->hdr_ver[3]));
3153         return 0;
3154 }
3155
3156 /**
3157  *      t4_get_vpd_version - return the VPD version
3158  *      @adapter: the adapter
3159  *      @vers: where to place the version
3160  *
3161  *      Reads the VPD via the Firmware interface (thus this can only be called
3162  *      once we're ready to issue Firmware commands).  The format of the
3163  *      VPD version is adapter specific.  Returns 0 on success, an error on
3164  *      failure.
3165  *
3166  *      Note that early versions of the Firmware didn't include the ability
3167  *      to retrieve the VPD version, so we zero-out the return-value parameter
3168  *      in that case to avoid leaving it with garbage in it.
3169  *
3170  *      Also note that the Firmware will return its cached copy of the VPD
3171  *      Revision ID, not the actual Revision ID as written in the Serial
3172  *      EEPROM.  This is only an issue if a new VPD has been written and the
3173  *      Firmware/Chip haven't yet gone through a RESET sequence.  So it's best
3174  *      to defer calling this routine till after a FW_RESET_CMD has been issued
3175  *      if the Host Driver will be performing a full adapter initialization.
3176  */
3177 int t4_get_vpd_version(struct adapter *adapter, u32 *vers)
3178 {
3179         u32 vpdrev_param;
3180         int ret;
3181
3182         vpdrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3183                         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_VPDREV));
3184         ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3185                               1, &vpdrev_param, vers);
3186         if (ret)
3187                 *vers = 0;
3188         return ret;
3189 }
3190
3191 /**
3192  *      t4_get_scfg_version - return the Serial Configuration version
3193  *      @adapter: the adapter
3194  *      @vers: where to place the version
3195  *
3196  *      Reads the Serial Configuration Version via the Firmware interface
3197  *      (thus this can only be called once we're ready to issue Firmware
3198  *      commands).  The format of the Serial Configuration version is
3199  *      adapter specific.  Returns 0 on success, an error on failure.
3200  *
3201  *      Note that early versions of the Firmware didn't include the ability
3202  *      to retrieve the Serial Configuration version, so we zero-out the
3203  *      return-value parameter in that case to avoid leaving it with
3204  *      garbage in it.
3205  *
3206  *      Also note that the Firmware will return its cached copy of the Serial
3207  *      Initialization Revision ID, not the actual Revision ID as written in
3208  *      the Serial EEPROM.  This is only an issue if a new VPD has been written
3209  *      and the Firmware/Chip haven't yet gone through a RESET sequence.  So
3210  *      it's best to defer calling this routine till after a FW_RESET_CMD has
3211  *      been issued if the Host Driver will be performing a full adapter
3212  *      initialization.
3213  */
3214 int t4_get_scfg_version(struct adapter *adapter, u32 *vers)
3215 {
3216         u32 scfgrev_param;
3217         int ret;
3218
3219         scfgrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3220                          FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_SCFGREV));
3221         ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3222                               1, &scfgrev_param, vers);
3223         if (ret)
3224                 *vers = 0;
3225         return ret;
3226 }
3227
3228 /**
3229  *      t4_get_version_info - extract various chip/firmware version information
3230  *      @adapter: the adapter
3231  *
3232  *      Reads various chip/firmware version numbers and stores them into the
3233  *      adapter Adapter Parameters structure.  If any of the efforts fails
3234  *      the first failure will be returned, but all of the version numbers
3235  *      will be read.
3236  */
3237 int t4_get_version_info(struct adapter *adapter)
3238 {
3239         int ret = 0;
3240
3241         #define FIRST_RET(__getvinfo) \
3242         do { \
3243                 int __ret = __getvinfo; \
3244                 if (__ret && !ret) \
3245                         ret = __ret; \
3246         } while (0)
3247
3248         FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers));
3249         FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers));
3250         FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers));
3251         FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers));
3252         FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers));
3253         FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers));
3254
3255         #undef FIRST_RET
3256         return ret;
3257 }
3258
3259 /**
3260  *      t4_dump_version_info - dump all of the adapter configuration IDs
3261  *      @adapter: the adapter
3262  *
3263  *      Dumps all of the various bits of adapter configuration version/revision
3264  *      IDs information.  This is typically called at some point after
3265  *      t4_get_version_info() has been called.
3266  */
3267 void t4_dump_version_info(struct adapter *adapter)
3268 {
3269         /* Device information */
3270         dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n",
3271                  adapter->params.vpd.id,
3272                  CHELSIO_CHIP_RELEASE(adapter->params.chip));
3273         dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n",
3274                  adapter->params.vpd.sn, adapter->params.vpd.pn);
3275
3276         /* Firmware Version */
3277         if (!adapter->params.fw_vers)
3278                 dev_warn(adapter->pdev_dev, "No firmware loaded\n");
3279         else
3280                 dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n",
3281                          FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
3282                          FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
3283                          FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
3284                          FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers));
3285
3286         /* Bootstrap Firmware Version. (Some adapters don't have Bootstrap
3287          * Firmware, so dev_info() is more appropriate here.)
3288          */
3289         if (!adapter->params.bs_vers)
3290                 dev_info(adapter->pdev_dev, "No bootstrap loaded\n");
3291         else
3292                 dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n",
3293                          FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers),
3294                          FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers),
3295                          FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers),
3296                          FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers));
3297
3298         /* TP Microcode Version */
3299         if (!adapter->params.tp_vers)
3300                 dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n");
3301         else
3302                 dev_info(adapter->pdev_dev,
3303                          "TP Microcode version: %u.%u.%u.%u\n",
3304                          FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
3305                          FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
3306                          FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
3307                          FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
3308
3309         /* Expansion ROM version */
3310         if (!adapter->params.er_vers)
3311                 dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n");
3312         else
3313                 dev_info(adapter->pdev_dev,
3314                          "Expansion ROM version: %u.%u.%u.%u\n",
3315                          FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers),
3316                          FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers),
3317                          FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers),
3318                          FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers));
3319
3320         /* Serial Configuration version */
3321         dev_info(adapter->pdev_dev, "Serial Configuration version: %#x\n",
3322                  adapter->params.scfg_vers);
3323
3324         /* VPD Version */
3325         dev_info(adapter->pdev_dev, "VPD version: %#x\n",
3326                  adapter->params.vpd_vers);
3327 }
3328
3329 /**
3330  *      t4_check_fw_version - check if the FW is supported with this driver
3331  *      @adap: the adapter
3332  *
3333  *      Checks if an adapter's FW is compatible with the driver.  Returns 0
3334  *      if there's exact match, a negative error if the version could not be
3335  *      read or there's a major version mismatch
3336  */
3337 int t4_check_fw_version(struct adapter *adap)
3338 {
3339         int i, ret, major, minor, micro;
3340         int exp_major, exp_minor, exp_micro;
3341         unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
3342
3343         ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3344         /* Try multiple times before returning error */
3345         for (i = 0; (ret == -EBUSY || ret == -EAGAIN) && i < 3; i++)
3346                 ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3347
3348         if (ret)
3349                 return ret;
3350
3351         major = FW_HDR_FW_VER_MAJOR_G(adap->params.fw_vers);
3352         minor = FW_HDR_FW_VER_MINOR_G(adap->params.fw_vers);
3353         micro = FW_HDR_FW_VER_MICRO_G(adap->params.fw_vers);
3354
3355         switch (chip_version) {
3356         case CHELSIO_T4:
3357                 exp_major = T4FW_MIN_VERSION_MAJOR;
3358                 exp_minor = T4FW_MIN_VERSION_MINOR;
3359                 exp_micro = T4FW_MIN_VERSION_MICRO;
3360                 break;
3361         case CHELSIO_T5:
3362                 exp_major = T5FW_MIN_VERSION_MAJOR;
3363                 exp_minor = T5FW_MIN_VERSION_MINOR;
3364                 exp_micro = T5FW_MIN_VERSION_MICRO;
3365                 break;
3366         case CHELSIO_T6:
3367                 exp_major = T6FW_MIN_VERSION_MAJOR;
3368                 exp_minor = T6FW_MIN_VERSION_MINOR;
3369                 exp_micro = T6FW_MIN_VERSION_MICRO;
3370                 break;
3371         default:
3372                 dev_err(adap->pdev_dev, "Unsupported chip type, %x\n",
3373                         adap->chip);
3374                 return -EINVAL;
3375         }
3376
3377         if (major < exp_major || (major == exp_major && minor < exp_minor) ||
3378             (major == exp_major && minor == exp_minor && micro < exp_micro)) {
3379                 dev_err(adap->pdev_dev,
3380                         "Card has firmware version %u.%u.%u, minimum "
3381                         "supported firmware is %u.%u.%u.\n", major, minor,
3382                         micro, exp_major, exp_minor, exp_micro);
3383                 return -EFAULT;
3384         }
3385         return 0;
3386 }
3387
3388 /* Is the given firmware API compatible with the one the driver was compiled
3389  * with?
3390  */
3391 static int fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
3392 {
3393
3394         /* short circuit if it's the exact same firmware version */
3395         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
3396                 return 1;
3397
3398 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
3399         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
3400             SAME_INTF(ri) && SAME_INTF(iscsi) && SAME_INTF(fcoe))
3401                 return 1;
3402 #undef SAME_INTF
3403
3404         return 0;
3405 }
3406
3407 /* The firmware in the filesystem is usable, but should it be installed?
3408  * This routine explains itself in detail if it indicates the filesystem
3409  * firmware should be installed.
3410  */
3411 static int should_install_fs_fw(struct adapter *adap, int card_fw_usable,
3412                                 int k, int c)
3413 {
3414         const char *reason;
3415
3416         if (!card_fw_usable) {
3417                 reason = "incompatible or unusable";
3418                 goto install;
3419         }
3420
3421         if (k > c) {
3422                 reason = "older than the version supported with this driver";
3423                 goto install;
3424         }
3425
3426         return 0;
3427
3428 install:
3429         dev_err(adap->pdev_dev, "firmware on card (%u.%u.%u.%u) is %s, "
3430                 "installing firmware %u.%u.%u.%u on card.\n",
3431                 FW_HDR_FW_VER_MAJOR_G(c), FW_HDR_FW_VER_MINOR_G(c),
3432                 FW_HDR_FW_VER_MICRO_G(c), FW_HDR_FW_VER_BUILD_G(c), reason,
3433                 FW_HDR_FW_VER_MAJOR_G(k), FW_HDR_FW_VER_MINOR_G(k),
3434                 FW_HDR_FW_VER_MICRO_G(k), FW_HDR_FW_VER_BUILD_G(k));
3435
3436         return 1;
3437 }
3438
3439 int t4_prep_fw(struct adapter *adap, struct fw_info *fw_info,
3440                const u8 *fw_data, unsigned int fw_size,
3441                struct fw_hdr *card_fw, enum dev_state state,
3442                int *reset)
3443 {
3444         int ret, card_fw_usable, fs_fw_usable;
3445         const struct fw_hdr *fs_fw;
3446         const struct fw_hdr *drv_fw;
3447
3448         drv_fw = &fw_info->fw_hdr;
3449
3450         /* Read the header of the firmware on the card */
3451         ret = -t4_read_flash(adap, FLASH_FW_START,
3452                             sizeof(*card_fw) / sizeof(uint32_t),
3453                             (uint32_t *)card_fw, 1);
3454         if (ret == 0) {
3455                 card_fw_usable = fw_compatible(drv_fw, (const void *)card_fw);
3456         } else {
3457                 dev_err(adap->pdev_dev,
3458                         "Unable to read card's firmware header: %d\n", ret);
3459                 card_fw_usable = 0;
3460         }
3461
3462         if (fw_data != NULL) {
3463                 fs_fw = (const void *)fw_data;
3464                 fs_fw_usable = fw_compatible(drv_fw, fs_fw);
3465         } else {
3466                 fs_fw = NULL;
3467                 fs_fw_usable = 0;
3468         }
3469
3470         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
3471             (!fs_fw_usable || fs_fw->fw_ver == drv_fw->fw_ver)) {
3472                 /* Common case: the firmware on the card is an exact match and
3473                  * the filesystem one is an exact match too, or the filesystem
3474                  * one is absent/incompatible.
3475                  */
3476         } else if (fs_fw_usable && state == DEV_STATE_UNINIT &&
3477                    should_install_fs_fw(adap, card_fw_usable,
3478                                         be32_to_cpu(fs_fw->fw_ver),
3479                                         be32_to_cpu(card_fw->fw_ver))) {
3480                 ret = -t4_fw_upgrade(adap, adap->mbox, fw_data,
3481                                      fw_size, 0);
3482                 if (ret != 0) {
3483                         dev_err(adap->pdev_dev,
3484                                 "failed to install firmware: %d\n", ret);
3485                         goto bye;
3486                 }
3487
3488                 /* Installed successfully, update the cached header too. */
3489                 *card_fw = *fs_fw;
3490                 card_fw_usable = 1;
3491                 *reset = 0;     /* already reset as part of load_fw */
3492         }
3493
3494         if (!card_fw_usable) {
3495                 uint32_t d, c, k;
3496
3497                 d = be32_to_cpu(drv_fw->fw_ver);
3498                 c = be32_to_cpu(card_fw->fw_ver);
3499                 k = fs_fw ? be32_to_cpu(fs_fw->fw_ver) : 0;
3500
3501                 dev_err(adap->pdev_dev, "Cannot find a usable firmware: "
3502                         "chip state %d, "
3503                         "driver compiled with %d.%d.%d.%d, "
3504                         "card has %d.%d.%d.%d, filesystem has %d.%d.%d.%d\n",
3505                         state,
3506                         FW_HDR_FW_VER_MAJOR_G(d), FW_HDR_FW_VER_MINOR_G(d),
3507                         FW_HDR_FW_VER_MICRO_G(d), FW_HDR_FW_VER_BUILD_G(d),
3508                         FW_HDR_FW_VER_MAJOR_G(c), FW_HDR_FW_VER_MINOR_G(c),
3509                         FW_HDR_FW_VER_MICRO_G(c), FW_HDR_FW_VER_BUILD_G(c),
3510                         FW_HDR_FW_VER_MAJOR_G(k), FW_HDR_FW_VER_MINOR_G(k),
3511                         FW_HDR_FW_VER_MICRO_G(k), FW_HDR_FW_VER_BUILD_G(k));
3512                 ret = EINVAL;
3513                 goto bye;
3514         }
3515
3516         /* We're using whatever's on the card and it's known to be good. */
3517         adap->params.fw_vers = be32_to_cpu(card_fw->fw_ver);
3518         adap->params.tp_vers = be32_to_cpu(card_fw->tp_microcode_ver);
3519
3520 bye:
3521         return ret;
3522 }
3523
3524 /**
3525  *      t4_flash_erase_sectors - erase a range of flash sectors
3526  *      @adapter: the adapter
3527  *      @start: the first sector to erase
3528  *      @end: the last sector to erase
3529  *
3530  *      Erases the sectors in the given inclusive range.
3531  */
3532 static int t4_flash_erase_sectors(struct adapter *adapter, int start, int end)
3533 {
3534         int ret = 0;
3535
3536         if (end >= adapter->params.sf_nsec)
3537                 return -EINVAL;
3538
3539         while (start <= end) {
3540                 if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
3541                     (ret = sf1_write(adapter, 4, 0, 1,
3542                                      SF_ERASE_SECTOR | (start << 8))) != 0 ||
3543                     (ret = flash_wait_op(adapter, 14, 500)) != 0) {
3544                         dev_err(adapter->pdev_dev,
3545                                 "erase of flash sector %d failed, error %d\n",
3546                                 start, ret);
3547                         break;
3548                 }
3549                 start++;
3550         }
3551         t4_write_reg(adapter, SF_OP_A, 0);    /* unlock SF */
3552         return ret;
3553 }
3554
3555 /**
3556  *      t4_flash_cfg_addr - return the address of the flash configuration file
3557  *      @adapter: the adapter
3558  *
3559  *      Return the address within the flash where the Firmware Configuration
3560  *      File is stored.
3561  */
3562 unsigned int t4_flash_cfg_addr(struct adapter *adapter)
3563 {
3564         if (adapter->params.sf_size == 0x100000)
3565                 return FLASH_FPGA_CFG_START;
3566         else
3567                 return FLASH_CFG_START;
3568 }
3569
3570 /* Return TRUE if the specified firmware matches the adapter.  I.e. T4
3571  * firmware for T4 adapters, T5 firmware for T5 adapters, etc.  We go ahead
3572  * and emit an error message for mismatched firmware to save our caller the
3573  * effort ...
3574  */
3575 static bool t4_fw_matches_chip(const struct adapter *adap,
3576                                const struct fw_hdr *hdr)
3577 {
3578         /* The expression below will return FALSE for any unsupported adapter
3579          * which will keep us "honest" in the future ...
3580          */
3581         if ((is_t4(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T4) ||
3582             (is_t5(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T5) ||
3583             (is_t6(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T6))
3584                 return true;
3585
3586         dev_err(adap->pdev_dev,
3587                 "FW image (%d) is not suitable for this adapter (%d)\n",
3588                 hdr->chip, CHELSIO_CHIP_VERSION(adap->params.chip));
3589         return false;
3590 }
3591
3592 /**
3593  *      t4_load_fw - download firmware
3594  *      @adap: the adapter
3595  *      @fw_data: the firmware image to write
3596  *      @size: image size
3597  *
3598  *      Write the supplied firmware image to the card's serial flash.
3599  */
3600 int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
3601 {
3602         u32 csum;
3603         int ret, addr;
3604         unsigned int i;
3605         u8 first_page[SF_PAGE_SIZE];
3606         const __be32 *p = (const __be32 *)fw_data;
3607         const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data;
3608         unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
3609         unsigned int fw_start_sec = FLASH_FW_START_SEC;
3610         unsigned int fw_size = FLASH_FW_MAX_SIZE;
3611         unsigned int fw_start = FLASH_FW_START;
3612
3613         if (!size) {
3614                 dev_err(adap->pdev_dev, "FW image has no data\n");
3615                 return -EINVAL;
3616         }
3617         if (size & 511) {
3618                 dev_err(adap->pdev_dev,
3619                         "FW image size not multiple of 512 bytes\n");
3620                 return -EINVAL;
3621         }
3622         if ((unsigned int)be16_to_cpu(hdr->len512) * 512 != size) {
3623                 dev_err(adap->pdev_dev,
3624                         "FW image size differs from size in FW header\n");
3625                 return -EINVAL;
3626         }
3627         if (size > fw_size) {
3628                 dev_err(adap->pdev_dev, "FW image too large, max is %u bytes\n",
3629                         fw_size);
3630                 return -EFBIG;
3631         }
3632         if (!t4_fw_matches_chip(adap, hdr))
3633                 return -EINVAL;
3634
3635         for (csum = 0, i = 0; i < size / sizeof(csum); i++)
3636                 csum += be32_to_cpu(p[i]);
3637
3638         if (csum != 0xffffffff) {
3639                 dev_err(adap->pdev_dev,
3640                         "corrupted firmware image, checksum %#x\n", csum);
3641                 return -EINVAL;
3642         }
3643
3644         i = DIV_ROUND_UP(size, sf_sec_size);        /* # of sectors spanned */
3645         ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1);
3646         if (ret)
3647                 goto out;
3648
3649         /*
3650          * We write the correct version at the end so the driver can see a bad
3651          * version if the FW write fails.  Start by writing a copy of the
3652          * first page with a bad version.
3653          */
3654         memcpy(first_page, fw_data, SF_PAGE_SIZE);
3655         ((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff);
3656         ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page);
3657         if (ret)
3658                 goto out;
3659
3660         addr = fw_start;
3661         for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
3662                 addr += SF_PAGE_SIZE;
3663                 fw_data += SF_PAGE_SIZE;
3664                 ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data);
3665                 if (ret)
3666                         goto out;
3667         }
3668
3669         ret = t4_write_flash(adap,
3670                              fw_start + offsetof(struct fw_hdr, fw_ver),
3671                              sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver);
3672 out:
3673         if (ret)
3674                 dev_err(adap->pdev_dev, "firmware download failed, error %d\n",
3675                         ret);
3676         else
3677                 ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3678         return ret;
3679 }
3680
3681 /**
3682  *      t4_phy_fw_ver - return current PHY firmware version
3683  *      @adap: the adapter
3684  *      @phy_fw_ver: return value buffer for PHY firmware version
3685  *
3686  *      Returns the current version of external PHY firmware on the
3687  *      adapter.
3688  */
3689 int t4_phy_fw_ver(struct adapter *adap, int *phy_fw_ver)
3690 {
3691         u32 param, val;
3692         int ret;
3693
3694         param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3695                  FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) |
3696                  FW_PARAMS_PARAM_Y_V(adap->params.portvec) |
3697                  FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_VERSION));
3698         ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
3699                               &param, &val);
3700         if (ret < 0)
3701                 return ret;
3702         *phy_fw_ver = val;
3703         return 0;
3704 }
3705
3706 /**
3707  *      t4_load_phy_fw - download port PHY firmware
3708  *      @adap: the adapter
3709  *      @win: the PCI-E Memory Window index to use for t4_memory_rw()
3710  *      @win_lock: the lock to use to guard the memory copy
3711  *      @phy_fw_version: function to check PHY firmware versions
3712  *      @phy_fw_data: the PHY firmware image to write
3713  *      @phy_fw_size: image size
3714  *
3715  *      Transfer the specified PHY firmware to the adapter.  If a non-NULL
3716  *      @phy_fw_version is supplied, then it will be used to determine if
3717  *      it's necessary to perform the transfer by comparing the version
3718  *      of any existing adapter PHY firmware with that of the passed in
3719  *      PHY firmware image.  If @win_lock is non-NULL then it will be used
3720  *      around the call to t4_memory_rw() which transfers the PHY firmware
3721  *      to the adapter.
3722  *
3723  *      A negative error number will be returned if an error occurs.  If
3724  *      version number support is available and there's no need to upgrade
3725  *      the firmware, 0 will be returned.  If firmware is successfully
3726  *      transferred to the adapter, 1 will be retured.
3727  *
3728  *      NOTE: some adapters only have local RAM to store the PHY firmware.  As
3729  *      a result, a RESET of the adapter would cause that RAM to lose its
3730  *      contents.  Thus, loading PHY firmware on such adapters must happen
3731  *      after any FW_RESET_CMDs ...
3732  */
3733 int t4_load_phy_fw(struct adapter *adap,
3734                    int win, spinlock_t *win_lock,
3735                    int (*phy_fw_version)(const u8 *, size_t),
3736                    const u8 *phy_fw_data, size_t phy_fw_size)
3737 {
3738         unsigned long mtype = 0, maddr = 0;
3739         u32 param, val;
3740         int cur_phy_fw_ver = 0, new_phy_fw_vers = 0;
3741         int ret;
3742
3743         /* If we have version number support, then check to see if the adapter
3744          * already has up-to-date PHY firmware loaded.
3745          */
3746          if (phy_fw_version) {
3747                 new_phy_fw_vers = phy_fw_version(phy_fw_data, phy_fw_size);
3748                 ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver);
3749                 if (ret < 0)
3750                         return ret;
3751
3752                 if (cur_phy_fw_ver >= new_phy_fw_vers) {
3753                         CH_WARN(adap, "PHY Firmware already up-to-date, "
3754                                 "version %#x\n", cur_phy_fw_ver);
3755                         return 0;
3756                 }
3757         }
3758
3759         /* Ask the firmware where it wants us to copy the PHY firmware image.
3760          * The size of the file requires a special version of the READ coommand
3761          * which will pass the file size via the values field in PARAMS_CMD and
3762          * retrieve the return value from firmware and place it in the same
3763          * buffer values
3764          */
3765         param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3766                  FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) |
3767                  FW_PARAMS_PARAM_Y_V(adap->params.portvec) |
3768                  FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD));
3769         val = phy_fw_size;
3770         ret = t4_query_params_rw(adap, adap->mbox, adap->pf, 0, 1,
3771                                  &param, &val, 1, true);
3772         if (ret < 0)
3773                 return ret;
3774         mtype = val >> 8;
3775         maddr = (val & 0xff) << 16;
3776
3777         /* Copy the supplied PHY Firmware image to the adapter memory location
3778          * allocated by the adapter firmware.
3779          */
3780         if (win_lock)
3781                 spin_lock_bh(win_lock);
3782         ret = t4_memory_rw(adap, win, mtype, maddr,
3783                            phy_fw_size, (__be32 *)phy_fw_data,
3784                            T4_MEMORY_WRITE);
3785         if (win_lock)
3786                 spin_unlock_bh(win_lock);
3787         if (ret)
3788                 return ret;
3789
3790         /* Tell the firmware that the PHY firmware image has been written to
3791          * RAM and it can now start copying it over to the PHYs.  The chip
3792          * firmware will RESET the affected PHYs as part of this operation
3793          * leaving them running the new PHY firmware image.
3794          */
3795         param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3796                  FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) |
3797                  FW_PARAMS_PARAM_Y_V(adap->params.portvec) |
3798                  FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD));
3799         ret = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1,
3800                                     &param, &val, 30000);
3801
3802         /* If we have version number support, then check to see that the new
3803          * firmware got loaded properly.
3804          */
3805         if (phy_fw_version) {
3806                 ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver);
3807                 if (ret < 0)
3808                         return ret;
3809
3810                 if (cur_phy_fw_ver != new_phy_fw_vers) {
3811                         CH_WARN(adap, "PHY Firmware did not update: "
3812                                 "version on adapter %#x, "
3813                                 "version flashed %#x\n",
3814                                 cur_phy_fw_ver, new_phy_fw_vers);
3815                         return -ENXIO;
3816                 }
3817         }
3818
3819         return 1;
3820 }
3821
3822 /**
3823  *      t4_fwcache - firmware cache operation
3824  *      @adap: the adapter
3825  *      @op  : the operation (flush or flush and invalidate)
3826  */
3827 int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op)
3828 {
3829         struct fw_params_cmd c;
3830
3831         memset(&c, 0, sizeof(c));
3832         c.op_to_vfn =
3833                 cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) |
3834                             FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
3835                             FW_PARAMS_CMD_PFN_V(adap->pf) |
3836                             FW_PARAMS_CMD_VFN_V(0));
3837         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
3838         c.param[0].mnem =
3839                 cpu_to_be32(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
3840                             FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_FWCACHE));
3841         c.param[0].val = (__force __be32)op;
3842
3843         return t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), NULL);
3844 }
3845
3846 void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp,
3847                         unsigned int *pif_req_wrptr,
3848                         unsigned int *pif_rsp_wrptr)
3849 {
3850         int i, j;
3851         u32 cfg, val, req, rsp;
3852
3853         cfg = t4_read_reg(adap, CIM_DEBUGCFG_A);
3854         if (cfg & LADBGEN_F)
3855                 t4_write_reg(adap, CIM_DEBUGCFG_A, cfg ^ LADBGEN_F);
3856
3857         val = t4_read_reg(adap, CIM_DEBUGSTS_A);
3858         req = POLADBGWRPTR_G(val);
3859         rsp = PILADBGWRPTR_G(val);
3860         if (pif_req_wrptr)
3861                 *pif_req_wrptr = req;
3862         if (pif_rsp_wrptr)
3863                 *pif_rsp_wrptr = rsp;
3864
3865         for (i = 0; i < CIM_PIFLA_SIZE; i++) {
3866                 for (j = 0; j < 6; j++) {
3867                         t4_write_reg(adap, CIM_DEBUGCFG_A, POLADBGRDPTR_V(req) |
3868                                      PILADBGRDPTR_V(rsp));
3869                         *pif_req++ = t4_read_reg(adap, CIM_PO_LA_DEBUGDATA_A);
3870                         *pif_rsp++ = t4_read_reg(adap, CIM_PI_LA_DEBUGDATA_A);
3871                         req++;
3872                         rsp++;
3873                 }
3874                 req = (req + 2) & POLADBGRDPTR_M;
3875                 rsp = (rsp + 2) & PILADBGRDPTR_M;
3876         }
3877         t4_write_reg(adap, CIM_DEBUGCFG_A, cfg);
3878 }
3879
3880 void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp)
3881 {
3882         u32 cfg;
3883         int i, j, idx;
3884
3885         cfg = t4_read_reg(adap, CIM_DEBUGCFG_A);
3886         if (cfg & LADBGEN_F)
3887                 t4_write_reg(adap, CIM_DEBUGCFG_A, cfg ^ LADBGEN_F);
3888
3889         for (i = 0; i < CIM_MALA_SIZE; i++) {
3890                 for (j = 0; j < 5; j++) {
3891                         idx = 8 * i + j;
3892                         t4_write_reg(adap, CIM_DEBUGCFG_A, POLADBGRDPTR_V(idx) |
3893                                      PILADBGRDPTR_V(idx));
3894                         *ma_req++ = t4_read_reg(adap, CIM_PO_LA_MADEBUGDATA_A);
3895                         *ma_rsp++ = t4_read_reg(adap, CIM_PI_LA_MADEBUGDATA_A);
3896                 }
3897         }
3898         t4_write_reg(adap, CIM_DEBUGCFG_A, cfg);
3899 }
3900
3901 void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
3902 {
3903         unsigned int i, j;
3904
3905         for (i = 0; i < 8; i++) {
3906                 u32 *p = la_buf + i;
3907
3908                 t4_write_reg(adap, ULP_RX_LA_CTL_A, i);
3909                 j = t4_read_reg(adap, ULP_RX_LA_WRPTR_A);
3910                 t4_write_reg(adap, ULP_RX_LA_RDPTR_A, j);
3911                 for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8)
3912                         *p = t4_read_reg(adap, ULP_RX_LA_RDDATA_A);
3913         }
3914 }
3915
3916 #define ADVERT_MASK (FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) | \
3917                      FW_PORT_CAP32_ANEG)
3918
3919 /**
3920  *      fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits
3921  *      @caps16: a 16-bit Port Capabilities value
3922  *
3923  *      Returns the equivalent 32-bit Port Capabilities value.
3924  */
3925 static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
3926 {
3927         fw_port_cap32_t caps32 = 0;
3928
3929         #define CAP16_TO_CAP32(__cap) \
3930                 do { \
3931                         if (caps16 & FW_PORT_CAP_##__cap) \
3932                                 caps32 |= FW_PORT_CAP32_##__cap; \
3933                 } while (0)
3934
3935         CAP16_TO_CAP32(SPEED_100M);
3936         CAP16_TO_CAP32(SPEED_1G);
3937         CAP16_TO_CAP32(SPEED_25G);
3938         CAP16_TO_CAP32(SPEED_10G);
3939         CAP16_TO_CAP32(SPEED_40G);
3940         CAP16_TO_CAP32(SPEED_100G);
3941         CAP16_TO_CAP32(FC_RX);
3942         CAP16_TO_CAP32(FC_TX);
3943         CAP16_TO_CAP32(ANEG);
3944         CAP16_TO_CAP32(MDIX);
3945         CAP16_TO_CAP32(MDIAUTO);
3946         CAP16_TO_CAP32(FEC_RS);
3947         CAP16_TO_CAP32(FEC_BASER_RS);
3948         CAP16_TO_CAP32(802_3_PAUSE);
3949         CAP16_TO_CAP32(802_3_ASM_DIR);
3950
3951         #undef CAP16_TO_CAP32
3952
3953         return caps32;
3954 }
3955
3956 /**
3957  *      fwcaps32_to_caps16 - convert 32-bit Port Capabilities to 16-bits
3958  *      @caps32: a 32-bit Port Capabilities value
3959  *
3960  *      Returns the equivalent 16-bit Port Capabilities value.  Note that
3961  *      not all 32-bit Port Capabilities can be represented in the 16-bit
3962  *      Port Capabilities and some fields/values may not make it.
3963  */
3964 static fw_port_cap16_t fwcaps32_to_caps16(fw_port_cap32_t caps32)
3965 {
3966         fw_port_cap16_t caps16 = 0;
3967
3968         #define CAP32_TO_CAP16(__cap) \
3969                 do { \
3970                         if (caps32 & FW_PORT_CAP32_##__cap) \
3971                                 caps16 |= FW_PORT_CAP_##__cap; \
3972                 } while (0)
3973
3974         CAP32_TO_CAP16(SPEED_100M);
3975         CAP32_TO_CAP16(SPEED_1G);
3976         CAP32_TO_CAP16(SPEED_10G);
3977         CAP32_TO_CAP16(SPEED_25G);
3978         CAP32_TO_CAP16(SPEED_40G);
3979         CAP32_TO_CAP16(SPEED_100G);
3980         CAP32_TO_CAP16(FC_RX);
3981         CAP32_TO_CAP16(FC_TX);
3982         CAP32_TO_CAP16(802_3_PAUSE);
3983         CAP32_TO_CAP16(802_3_ASM_DIR);
3984         CAP32_TO_CAP16(ANEG);
3985         CAP32_TO_CAP16(MDIX);
3986         CAP32_TO_CAP16(MDIAUTO);
3987         CAP32_TO_CAP16(FEC_RS);
3988         CAP32_TO_CAP16(FEC_BASER_RS);
3989
3990         #undef CAP32_TO_CAP16
3991
3992         return caps16;
3993 }
3994
3995 /* Translate Firmware Port Capabilities Pause specification to Common Code */
3996 static inline enum cc_pause fwcap_to_cc_pause(fw_port_cap32_t fw_pause)
3997 {
3998         enum cc_pause cc_pause = 0;
3999
4000         if (fw_pause & FW_PORT_CAP32_FC_RX)
4001                 cc_pause |= PAUSE_RX;
4002         if (fw_pause & FW_PORT_CAP32_FC_TX)
4003                 cc_pause |= PAUSE_TX;
4004
4005         return cc_pause;
4006 }
4007
4008 /* Translate Common Code Pause specification into Firmware Port Capabilities */
4009 static inline fw_port_cap32_t cc_to_fwcap_pause(enum cc_pause cc_pause)
4010 {
4011         fw_port_cap32_t fw_pause = 0;
4012
4013         if (cc_pause & PAUSE_RX)
4014                 fw_pause |= FW_PORT_CAP32_FC_RX;
4015         if (cc_pause & PAUSE_TX)
4016                 fw_pause |= FW_PORT_CAP32_FC_TX;
4017
4018         return fw_pause;
4019 }
4020
4021 /* Translate Firmware Forward Error Correction specification to Common Code */
4022 static inline enum cc_fec fwcap_to_cc_fec(fw_port_cap32_t fw_fec)
4023 {
4024         enum cc_fec cc_fec = 0;
4025
4026         if (fw_fec & FW_PORT_CAP32_FEC_RS)
4027                 cc_fec |= FEC_RS;
4028         if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
4029                 cc_fec |= FEC_BASER_RS;
4030
4031         return cc_fec;
4032 }
4033
4034 /* Translate Common Code Forward Error Correction specification to Firmware */
4035 static inline fw_port_cap32_t cc_to_fwcap_fec(enum cc_fec cc_fec)
4036 {
4037         fw_port_cap32_t fw_fec = 0;
4038
4039         if (cc_fec & FEC_RS)
4040                 fw_fec |= FW_PORT_CAP32_FEC_RS;
4041         if (cc_fec & FEC_BASER_RS)
4042                 fw_fec |= FW_PORT_CAP32_FEC_BASER_RS;
4043
4044         return fw_fec;
4045 }
4046
4047 /**
4048  *      t4_link_l1cfg - apply link configuration to MAC/PHY
4049  *      @adapter: the adapter
4050  *      @mbox: the Firmware Mailbox to use
4051  *      @port: the Port ID
4052  *      @lc: the Port's Link Configuration
4053  *
4054  *      Set up a port's MAC and PHY according to a desired link configuration.
4055  *      - If the PHY can auto-negotiate first decide what to advertise, then
4056  *        enable/disable auto-negotiation as desired, and reset.
4057  *      - If the PHY does not auto-negotiate just reset it.
4058  *      - If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
4059  *        otherwise do it later based on the outcome of auto-negotiation.
4060  */
4061 int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
4062                   unsigned int port, struct link_config *lc)
4063 {
4064         unsigned int fw_caps = adapter->params.fw_caps_support;
4065         struct fw_port_cmd cmd;
4066         unsigned int fw_mdi = FW_PORT_CAP32_MDI_V(FW_PORT_CAP32_MDI_AUTO);
4067         fw_port_cap32_t fw_fc, cc_fec, fw_fec, rcap;
4068
4069         /* Convert driver coding of Pause Frame Flow Control settings into the
4070          * Firmware's API.
4071          */
4072         fw_fc = cc_to_fwcap_pause(lc->requested_fc);
4073
4074         /* Convert Common Code Forward Error Control settings into the
4075          * Firmware's API.  If the current Requested FEC has "Automatic"
4076          * (IEEE 802.3) specified, then we use whatever the Firmware
4077          * sent us as part of it's IEEE 802.3-based interpratation of
4078          * the Transceiver Module EPROM FEC parameters.  Otherwise we
4079          * use whatever is in the current Requested FEC settings.
4080          */
4081         if (lc->requested_fec & FEC_AUTO)
4082                 cc_fec = fwcap_to_cc_fec(lc->def_acaps);
4083         else
4084                 cc_fec = lc->requested_fec;
4085         fw_fec = cc_to_fwcap_fec(cc_fec);
4086
4087         /* Figure out what our Requested Port Capabilities are going to be.
4088          */
4089         if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
4090                 rcap = (lc->pcaps & ADVERT_MASK) | fw_fc | fw_fec;
4091                 lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
4092                 lc->fec = cc_fec;
4093         } else if (lc->autoneg == AUTONEG_DISABLE) {
4094                 rcap = lc->speed_caps | fw_fc | fw_fec | fw_mdi;
4095                 lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
4096                 lc->fec = cc_fec;
4097         } else {
4098                 rcap = lc->acaps | fw_fc | fw_fec | fw_mdi;
4099         }
4100
4101         /* And send that on to the Firmware ...
4102          */
4103         memset(&cmd, 0, sizeof(cmd));
4104         cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
4105                                        FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
4106                                        FW_PORT_CMD_PORTID_V(port));
4107         cmd.action_to_len16 =
4108                 cpu_to_be32(FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
4109                                                  ? FW_PORT_ACTION_L1_CFG
4110                                                  : FW_PORT_ACTION_L1_CFG32) |
4111                             FW_LEN16(cmd));
4112         if (fw_caps == FW_CAPS16)
4113                 cmd.u.l1cfg.rcap = cpu_to_be32(fwcaps32_to_caps16(rcap));
4114         else
4115                 cmd.u.l1cfg32.rcap32 = cpu_to_be32(rcap);
4116         return t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
4117 }
4118
4119 /**
4120  *      t4_restart_aneg - restart autonegotiation
4121  *      @adap: the adapter
4122  *      @mbox: mbox to use for the FW command
4123  *      @port: the port id
4124  *
4125  *      Restarts autonegotiation for the selected port.
4126  */
4127 int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port)
4128 {
4129         struct fw_port_cmd c;
4130
4131         memset(&c, 0, sizeof(c));
4132         c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
4133                                      FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
4134                                      FW_PORT_CMD_PORTID_V(port));
4135         c.action_to_len16 =
4136                 cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_L1_CFG) |
4137                             FW_LEN16(c));
4138         c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP32_ANEG);
4139         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
4140 }
4141
4142 typedef void (*int_handler_t)(struct adapter *adap);
4143
4144 struct intr_info {
4145         unsigned int mask;       /* bits to check in interrupt status */
4146         const char *msg;         /* message to print or NULL */
4147         short stat_idx;          /* stat counter to increment or -1 */
4148         unsigned short fatal;    /* whether the condition reported is fatal */
4149         int_handler_t int_handler; /* platform-specific int handler */
4150 };
4151
4152 /**
4153  *      t4_handle_intr_status - table driven interrupt handler
4154  *      @adapter: the adapter that generated the interrupt
4155  *      @reg: the interrupt status register to process
4156  *      @acts: table of interrupt actions
4157  *
4158  *      A table driven interrupt handler that applies a set of masks to an
4159  *      interrupt status word and performs the corresponding actions if the
4160  *      interrupts described by the mask have occurred.  The actions include
4161  *      optionally emitting a warning or alert message.  The table is terminated
4162  *      by an entry specifying mask 0.  Returns the number of fatal interrupt
4163  *      conditions.
4164  */
4165 static int t4_handle_intr_status(struct adapter *adapter, unsigned int reg,
4166                                  const struct intr_info *acts)
4167 {
4168         int fatal = 0;
4169         unsigned int mask = 0;
4170         unsigned int status = t4_read_reg(adapter, reg);
4171
4172         for ( ; acts->mask; ++acts) {
4173                 if (!(status & acts->mask))
4174                         continue;
4175                 if (acts->fatal) {
4176                         fatal++;
4177                         dev_alert(adapter->pdev_dev, "%s (0x%x)\n", acts->msg,
4178                                   status & acts->mask);
4179                 } else if (acts->msg && printk_ratelimit())
4180                         dev_warn(adapter->pdev_dev, "%s (0x%x)\n", acts->msg,
4181                                  status & acts->mask);
4182                 if (acts->int_handler)
4183                         acts->int_handler(adapter);
4184                 mask |= acts->mask;
4185         }
4186         status &= mask;
4187         if (status)                           /* clear processed interrupts */
4188                 t4_write_reg(adapter, reg, status);
4189         return fatal;
4190 }
4191
4192 /*
4193  * Interrupt handler for the PCIE module.
4194  */
4195 static void pcie_intr_handler(struct adapter *adapter)
4196 {
4197         static const struct intr_info sysbus_intr_info[] = {
4198                 { RNPP_F, "RXNP array parity error", -1, 1 },
4199                 { RPCP_F, "RXPC array parity error", -1, 1 },
4200                 { RCIP_F, "RXCIF array parity error", -1, 1 },
4201                 { RCCP_F, "Rx completions control array parity error", -1, 1 },
4202                 { RFTP_F, "RXFT array parity error", -1, 1 },
4203                 { 0 }
4204         };
4205         static const struct intr_info pcie_port_intr_info[] = {
4206                 { TPCP_F, "TXPC array parity error", -1, 1 },
4207                 { TNPP_F, "TXNP array parity error", -1, 1 },
4208                 { TFTP_F, "TXFT array parity error", -1, 1 },
4209                 { TCAP_F, "TXCA array parity error", -1, 1 },
4210                 { TCIP_F, "TXCIF array parity error", -1, 1 },
4211                 { RCAP_F, "RXCA array parity error", -1, 1 },
4212                 { OTDD_F, "outbound request TLP discarded", -1, 1 },
4213                 { RDPE_F, "Rx data parity error", -1, 1 },
4214                 { TDUE_F, "Tx uncorrectable data error", -1, 1 },
4215                 { 0 }
4216         };
4217         static const struct intr_info pcie_intr_info[] = {
4218                 { MSIADDRLPERR_F, "MSI AddrL parity error", -1, 1 },
4219                 { MSIADDRHPERR_F, "MSI AddrH parity error", -1, 1 },
4220                 { MSIDATAPERR_F, "MSI data parity error", -1, 1 },
4221                 { MSIXADDRLPERR_F, "MSI-X AddrL parity error", -1, 1 },
4222                 { MSIXADDRHPERR_F, "MSI-X AddrH parity error", -1, 1 },
4223                 { MSIXDATAPERR_F, "MSI-X data parity error", -1, 1 },
4224                 { MSIXDIPERR_F, "MSI-X DI parity error", -1, 1 },
4225                 { PIOCPLPERR_F, "PCI PIO completion FIFO parity error", -1, 1 },
4226                 { PIOREQPERR_F, "PCI PIO request FIFO parity error", -1, 1 },
4227                 { TARTAGPERR_F, "PCI PCI target tag FIFO parity error", -1, 1 },
4228                 { CCNTPERR_F, "PCI CMD channel count parity error", -1, 1 },
4229                 { CREQPERR_F, "PCI CMD channel request parity error", -1, 1 },
4230                 { CRSPPERR_F, "PCI CMD channel response parity error", -1, 1 },
4231                 { DCNTPERR_F, "PCI DMA channel count parity error", -1, 1 },
4232                 { DREQPERR_F, "PCI DMA channel request parity error", -1, 1 },
4233                 { DRSPPERR_F, "PCI DMA channel response parity error", -1, 1 },
4234                 { HCNTPERR_F, "PCI HMA channel count parity error", -1, 1 },
4235                 { HREQPERR_F, "PCI HMA channel request parity error", -1, 1 },
4236                 { HRSPPERR_F, "PCI HMA channel response parity error", -1, 1 },
4237                 { CFGSNPPERR_F, "PCI config snoop FIFO parity error", -1, 1 },
4238                 { FIDPERR_F, "PCI FID parity error", -1, 1 },
4239                 { INTXCLRPERR_F, "PCI INTx clear parity error", -1, 1 },
4240                 { MATAGPERR_F, "PCI MA tag parity error", -1, 1 },
4241                 { PIOTAGPERR_F, "PCI PIO tag parity error", -1, 1 },
4242                 { RXCPLPERR_F, "PCI Rx completion parity error", -1, 1 },
4243                 { RXWRPERR_F, "PCI Rx write parity error", -1, 1 },
4244                 { RPLPERR_F, "PCI replay buffer parity error", -1, 1 },
4245                 { PCIESINT_F, "PCI core secondary fault", -1, 1 },
4246                 { PCIEPINT_F, "PCI core primary fault", -1, 1 },
4247                 { UNXSPLCPLERR_F, "PCI unexpected split completion error",
4248                   -1, 0 },
4249                 { 0 }
4250         };
4251
4252         static struct intr_info t5_pcie_intr_info[] = {
4253                 { MSTGRPPERR_F, "Master Response Read Queue parity error",
4254                   -1, 1 },
4255                 { MSTTIMEOUTPERR_F, "Master Timeout FIFO parity error", -1, 1 },
4256                 { MSIXSTIPERR_F, "MSI-X STI SRAM parity error", -1, 1 },
4257                 { MSIXADDRLPERR_F, "MSI-X AddrL parity error", -1, 1 },
4258                 { MSIXADDRHPERR_F, "MSI-X AddrH parity error", -1, 1 },
4259                 { MSIXDATAPERR_F, "MSI-X data parity error", -1, 1 },
4260                 { MSIXDIPERR_F, "MSI-X DI parity error", -1, 1 },
4261                 { PIOCPLGRPPERR_F, "PCI PIO completion Group FIFO parity error",
4262                   -1, 1 },
4263                 { PIOREQGRPPERR_F, "PCI PIO request Group FIFO parity error",
4264                   -1, 1 },
4265                 { TARTAGPERR_F, "PCI PCI target tag FIFO parity error", -1, 1 },
4266                 { MSTTAGQPERR_F, "PCI master tag queue parity error", -1, 1 },
4267                 { CREQPERR_F, "PCI CMD channel request parity error", -1, 1 },
4268                 { CRSPPERR_F, "PCI CMD channel response parity error", -1, 1 },
4269                 { DREQWRPERR_F, "PCI DMA channel write request parity error",
4270                   -1, 1 },
4271                 { DREQPERR_F, "PCI DMA channel request parity error", -1, 1 },
4272                 { DRSPPERR_F, "PCI DMA channel response parity error", -1, 1 },
4273                 { HREQWRPERR_F, "PCI HMA channel count parity error", -1, 1 },
4274                 { HREQPERR_F, "PCI HMA channel request parity error", -1, 1 },
4275                 { HRSPPERR_F, "PCI HMA channel response parity error", -1, 1 },
4276                 { CFGSNPPERR_F, "PCI config snoop FIFO parity error", -1, 1 },
4277                 { FIDPERR_F, "PCI FID parity error", -1, 1 },
4278                 { VFIDPERR_F, "PCI INTx clear parity error", -1, 1 },
4279                 { MAGRPPERR_F, "PCI MA group FIFO parity error", -1, 1 },
4280                 { PIOTAGPERR_F, "PCI PIO tag parity error", -1, 1 },
4281                 { IPRXHDRGRPPERR_F, "PCI IP Rx header group parity error",
4282                   -1, 1 },
4283                 { IPRXDATAGRPPERR_F, "PCI IP Rx data group parity error",
4284                   -1, 1 },
4285                 { RPLPERR_F, "PCI IP replay buffer parity error", -1, 1 },
4286                 { IPSOTPERR_F, "PCI IP SOT buffer parity error", -1, 1 },
4287                 { TRGT1GRPPERR_F, "PCI TRGT1 group FIFOs parity error", -1, 1 },
4288                 { READRSPERR_F, "Outbound read error", -1, 0 },
4289                 { 0 }
4290         };
4291
4292         int fat;
4293
4294         if (is_t4(adapter->params.chip))
4295                 fat = t4_handle_intr_status(adapter,
4296                                 PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS_A,
4297                                 sysbus_intr_info) +
4298                         t4_handle_intr_status(adapter,
4299                                         PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS_A,
4300                                         pcie_port_intr_info) +
4301                         t4_handle_intr_status(adapter, PCIE_INT_CAUSE_A,
4302                                               pcie_intr_info);
4303         else
4304                 fat = t4_handle_intr_status(adapter, PCIE_INT_CAUSE_A,
4305                                             t5_pcie_intr_info);
4306
4307         if (fat)
4308                 t4_fatal_err(adapter);
4309 }
4310
4311 /*
4312  * TP interrupt handler.
4313  */
4314 static void tp_intr_handler(struct adapter *adapter)
4315 {
4316         static const struct intr_info tp_intr_info[] = {
4317                 { 0x3fffffff, "TP parity error", -1, 1 },
4318                 { FLMTXFLSTEMPTY_F, "TP out of Tx pages", -1, 1 },
4319                 { 0 }
4320         };
4321
4322         if (t4_handle_intr_status(adapter, TP_INT_CAUSE_A, tp_intr_info))
4323                 t4_fatal_err(adapter);
4324 }
4325
4326 /*
4327  * SGE interrupt handler.
4328  */
4329 static void sge_intr_handler(struct adapter *adapter)
4330 {
4331         u64 v;
4332         u32 err;
4333
4334         static const struct intr_info sge_intr_info[] = {
4335                 { ERR_CPL_EXCEED_IQE_SIZE_F,
4336                   "SGE received CPL exceeding IQE size", -1, 1 },
4337                 { ERR_INVALID_CIDX_INC_F,
4338                   "SGE GTS CIDX increment too large", -1, 0 },
4339                 { ERR_CPL_OPCODE_0_F, "SGE received 0-length CPL", -1, 0 },
4340                 { DBFIFO_LP_INT_F, NULL, -1, 0, t4_db_full },
4341                 { ERR_DATA_CPL_ON_HIGH_QID1_F | ERR_DATA_CPL_ON_HIGH_QID0_F,
4342                   "SGE IQID > 1023 received CPL for FL", -1, 0 },
4343                 { ERR_BAD_DB_PIDX3_F, "SGE DBP 3 pidx increment too large", -1,
4344                   0 },
4345                 { ERR_BAD_DB_PIDX2_F, "SGE DBP 2 pidx increment too large", -1,
4346                   0 },
4347                 { ERR_BAD_DB_PIDX1_F, "SGE DBP 1 pidx increment too large", -1,
4348                   0 },
4349                 { ERR_BAD_DB_PIDX0_F, "SGE DBP 0 pidx increment too large", -1,
4350                   0 },
4351                 { ERR_ING_CTXT_PRIO_F,
4352                   "SGE too many priority ingress contexts", -1, 0 },
4353                 { INGRESS_SIZE_ERR_F, "SGE illegal ingress QID", -1, 0 },
4354                 { EGRESS_SIZE_ERR_F, "SGE illegal egress QID", -1, 0 },
4355                 { 0 }
4356         };
4357
4358         static struct intr_info t4t5_sge_intr_info[] = {
4359                 { ERR_DROPPED_DB_F, NULL, -1, 0, t4_db_dropped },
4360                 { DBFIFO_HP_INT_F, NULL, -1, 0, t4_db_full },
4361                 { ERR_EGR_CTXT_PRIO_F,
4362                   "SGE too many priority egress contexts", -1, 0 },
4363                 { 0 }
4364         };
4365
4366         v = (u64)t4_read_reg(adapter, SGE_INT_CAUSE1_A) |
4367                 ((u64)t4_read_reg(adapter, SGE_INT_CAUSE2_A) << 32);
4368         if (v) {
4369                 dev_alert(adapter->pdev_dev, "SGE parity error (%#llx)\n",
4370                                 (unsigned long long)v);
4371                 t4_write_reg(adapter, SGE_INT_CAUSE1_A, v);
4372                 t4_write_reg(adapter, SGE_INT_CAUSE2_A, v >> 32);
4373         }
4374
4375         v |= t4_handle_intr_status(adapter, SGE_INT_CAUSE3_A, sge_intr_info);
4376         if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
4377                 v |= t4_handle_intr_status(adapter, SGE_INT_CAUSE3_A,
4378                                            t4t5_sge_intr_info);
4379
4380         err = t4_read_reg(adapter, SGE_ERROR_STATS_A);
4381         if (err & ERROR_QID_VALID_F) {
4382                 dev_err(adapter->pdev_dev, "SGE error for queue %u\n",
4383                         ERROR_QID_G(err));
4384                 if (err & UNCAPTURED_ERROR_F)
4385                         dev_err(adapter->pdev_dev,
4386                                 "SGE UNCAPTURED_ERROR set (clearing)\n");
4387                 t4_write_reg(adapter, SGE_ERROR_STATS_A, ERROR_QID_VALID_F |
4388                              UNCAPTURED_ERROR_F);
4389         }
4390
4391         if (v != 0)
4392                 t4_fatal_err(adapter);
4393 }
4394
4395 #define CIM_OBQ_INTR (OBQULP0PARERR_F | OBQULP1PARERR_F | OBQULP2PARERR_F |\
4396                       OBQULP3PARERR_F | OBQSGEPARERR_F | OBQNCSIPARERR_F)
4397 #define CIM_IBQ_INTR (IBQTP0PARERR_F | IBQTP1PARERR_F | IBQULPPARERR_F |\
4398                       IBQSGEHIPARERR_F | IBQSGELOPARERR_F | IBQNCSIPARERR_F)
4399
4400 /*
4401  * CIM interrupt handler.
4402  */
4403 static void cim_intr_handler(struct adapter *adapter)
4404 {
4405         static const struct intr_info cim_intr_info[] = {
4406                 { PREFDROPINT_F, "CIM control register prefetch drop", -1, 1 },
4407                 { CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 },
4408                 { CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 },
4409                 { MBUPPARERR_F, "CIM mailbox uP parity error", -1, 1 },
4410                 { MBHOSTPARERR_F, "CIM mailbox host parity error", -1, 1 },
4411                 { TIEQINPARERRINT_F, "CIM TIEQ outgoing parity error", -1, 1 },
4412                 { TIEQOUTPARERRINT_F, "CIM TIEQ incoming parity error", -1, 1 },
4413                 { TIMER0INT_F, "CIM TIMER0 interrupt", -1, 1 },
4414                 { 0 }
4415         };
4416         static const struct intr_info cim_upintr_info[] = {
4417                 { RSVDSPACEINT_F, "CIM reserved space access", -1, 1 },
4418                 { ILLTRANSINT_F, "CIM illegal transaction", -1, 1 },
4419                 { ILLWRINT_F, "CIM illegal write", -1, 1 },
4420                 { ILLRDINT_F, "CIM illegal read", -1, 1 },
4421                 { ILLRDBEINT_F, "CIM illegal read BE", -1, 1 },
4422                 { ILLWRBEINT_F, "CIM illegal write BE", -1, 1 },
4423                 { SGLRDBOOTINT_F, "CIM single read from boot space", -1, 1 },
4424                 { SGLWRBOOTINT_F, "CIM single write to boot space", -1, 1 },
4425                 { BLKWRBOOTINT_F, "CIM block write to boot space", -1, 1 },
4426                 { SGLRDFLASHINT_F, "CIM single read from flash space", -1, 1 },
4427                 { SGLWRFLASHINT_F, "CIM single write to flash space", -1, 1 },
4428                 { BLKWRFLASHINT_F, "CIM block write to flash space", -1, 1 },
4429                 { SGLRDEEPROMINT_F, "CIM single EEPROM read", -1, 1 },
4430                 { SGLWREEPROMINT_F, "CIM single EEPROM write", -1, 1 },
4431                 { BLKRDEEPROMINT_F, "CIM block EEPROM read", -1, 1 },
4432                 { BLKWREEPROMINT_F, "CIM block EEPROM write", -1, 1 },
4433                 { SGLRDCTLINT_F, "CIM single read from CTL space", -1, 1 },
4434                 { SGLWRCTLINT_F, "CIM single write to CTL space", -1, 1 },
4435                 { BLKRDCTLINT_F, "CIM block read from CTL space", -1, 1 },
4436                 { BLKWRCTLINT_F, "CIM block write to CTL space", -1, 1 },
4437                 { SGLRDPLINT_F, "CIM single read from PL space", -1, 1 },
4438                 { SGLWRPLINT_F, "CIM single write to PL space", -1, 1 },
4439                 { BLKRDPLINT_F, "CIM block read from PL space", -1, 1 },
4440                 { BLKWRPLINT_F, "CIM block write to PL space", -1, 1 },
4441                 { REQOVRLOOKUPINT_F, "CIM request FIFO overwrite", -1, 1 },
4442                 { RSPOVRLOOKUPINT_F, "CIM response FIFO overwrite", -1, 1 },
4443                 { TIMEOUTINT_F, "CIM PIF timeout", -1, 1 },
4444                 { TIMEOUTMAINT_F, "CIM PIF MA timeout", -1, 1 },
4445                 { 0 }
4446         };
4447
4448         u32 val, fw_err;
4449         int fat;
4450
4451         fw_err = t4_read_reg(adapter, PCIE_FW_A);
4452         if (fw_err & PCIE_FW_ERR_F)
4453                 t4_report_fw_error(adapter);
4454
4455         /* When the Firmware detects an internal error which normally
4456          * wouldn't raise a Host Interrupt, it forces a CIM Timer0 interrupt
4457          * in order to make sure the Host sees the Firmware Crash.  So
4458          * if we have a Timer0 interrupt and don't see a Firmware Crash,
4459          * ignore the Timer0 interrupt.
4460          */
4461
4462         val = t4_read_reg(adapter, CIM_HOST_INT_CAUSE_A);
4463         if (val & TIMER0INT_F)
4464                 if (!(fw_err & PCIE_FW_ERR_F) ||
4465                     (PCIE_FW_EVAL_G(fw_err) != PCIE_FW_EVAL_CRASH))
4466                         t4_write_reg(adapter, CIM_HOST_INT_CAUSE_A,
4467                                      TIMER0INT_F);
4468
4469         fat = t4_handle_intr_status(adapter, CIM_HOST_INT_CAUSE_A,
4470                                     cim_intr_info) +
4471               t4_handle_intr_status(adapter, CIM_HOST_UPACC_INT_CAUSE_A,
4472                                     cim_upintr_info);
4473         if (fat)
4474                 t4_fatal_err(adapter);
4475 }
4476
4477 /*
4478  * ULP RX interrupt handler.
4479  */
4480 static void ulprx_intr_handler(struct adapter *adapter)
4481 {
4482         static const struct intr_info ulprx_intr_info[] = {
4483                 { 0x1800000, "ULPRX context error", -1, 1 },
4484                 { 0x7fffff, "ULPRX parity error", -1, 1 },
4485                 { 0 }
4486         };
4487
4488         if (t4_handle_intr_status(adapter, ULP_RX_INT_CAUSE_A, ulprx_intr_info))
4489                 t4_fatal_err(adapter);
4490 }
4491
4492 /*
4493  * ULP TX interrupt handler.
4494  */
4495 static void ulptx_intr_handler(struct adapter *adapter)
4496 {
4497         static const struct intr_info ulptx_intr_info[] = {
4498                 { PBL_BOUND_ERR_CH3_F, "ULPTX channel 3 PBL out of bounds", -1,
4499                   0 },
4500                 { PBL_BOUND_ERR_CH2_F, "ULPTX channel 2 PBL out of bounds", -1,
4501                   0 },
4502                 { PBL_BOUND_ERR_CH1_F, "ULPTX channel 1 PBL out of bounds", -1,
4503                   0 },
4504                 { PBL_BOUND_ERR_CH0_F, "ULPTX channel 0 PBL out of bounds", -1,
4505                   0 },
4506                 { 0xfffffff, "ULPTX parity error", -1, 1 },
4507                 { 0 }
4508         };
4509
4510         if (t4_handle_intr_status(adapter, ULP_TX_INT_CAUSE_A, ulptx_intr_info))
4511                 t4_fatal_err(adapter);
4512 }
4513
4514 /*
4515  * PM TX interrupt handler.
4516  */
4517 static void pmtx_intr_handler(struct adapter *adapter)
4518 {
4519         static const struct intr_info pmtx_intr_info[] = {
4520                 { PCMD_LEN_OVFL0_F, "PMTX channel 0 pcmd too large", -1, 1 },
4521                 { PCMD_LEN_OVFL1_F, "PMTX channel 1 pcmd too large", -1, 1 },
4522                 { PCMD_LEN_OVFL2_F, "PMTX channel 2 pcmd too large", -1, 1 },
4523                 { ZERO_C_CMD_ERROR_F, "PMTX 0-length pcmd", -1, 1 },
4524                 { PMTX_FRAMING_ERROR_F, "PMTX framing error", -1, 1 },
4525                 { OESPI_PAR_ERROR_F, "PMTX oespi parity error", -1, 1 },
4526                 { DB_OPTIONS_PAR_ERROR_F, "PMTX db_options parity error",
4527                   -1, 1 },
4528                 { ICSPI_PAR_ERROR_F, "PMTX icspi parity error", -1, 1 },
4529                 { PMTX_C_PCMD_PAR_ERROR_F, "PMTX c_pcmd parity error", -1, 1},
4530                 { 0 }
4531         };
4532
4533         if (t4_handle_intr_status(adapter, PM_TX_INT_CAUSE_A, pmtx_intr_info))
4534                 t4_fatal_err(adapter);
4535 }
4536
4537 /*
4538  * PM RX interrupt handler.
4539  */
4540 static void pmrx_intr_handler(struct adapter *adapter)
4541 {
4542         static const struct intr_info pmrx_intr_info[] = {
4543                 { ZERO_E_CMD_ERROR_F, "PMRX 0-length pcmd", -1, 1 },
4544                 { PMRX_FRAMING_ERROR_F, "PMRX framing error", -1, 1 },
4545                 { OCSPI_PAR_ERROR_F, "PMRX ocspi parity error", -1, 1 },
4546                 { DB_OPTIONS_PAR_ERROR_F, "PMRX db_options parity error",
4547                   -1, 1 },
4548                 { IESPI_PAR_ERROR_F, "PMRX iespi parity error", -1, 1 },
4549                 { PMRX_E_PCMD_PAR_ERROR_F, "PMRX e_pcmd parity error", -1, 1},
4550                 { 0 }
4551         };
4552
4553         if (t4_handle_intr_status(adapter, PM_RX_INT_CAUSE_A, pmrx_intr_info))
4554                 t4_fatal_err(adapter);
4555 }
4556
4557 /*
4558  * CPL switch interrupt handler.
4559  */
4560 static void cplsw_intr_handler(struct adapter *adapter)
4561 {
4562         static const struct intr_info cplsw_intr_info[] = {
4563                 { CIM_OP_MAP_PERR_F, "CPLSW CIM op_map parity error", -1, 1 },
4564                 { CIM_OVFL_ERROR_F, "CPLSW CIM overflow", -1, 1 },
4565                 { TP_FRAMING_ERROR_F, "CPLSW TP framing error", -1, 1 },
4566                 { SGE_FRAMING_ERROR_F, "CPLSW SGE framing error", -1, 1 },
4567                 { CIM_FRAMING_ERROR_F, "CPLSW CIM framing error", -1, 1 },
4568                 { ZERO_SWITCH_ERROR_F, "CPLSW no-switch error", -1, 1 },
4569                 { 0 }
4570         };
4571
4572         if (t4_handle_intr_status(adapter, CPL_INTR_CAUSE_A, cplsw_intr_info))
4573                 t4_fatal_err(adapter);
4574 }
4575
4576 /*
4577  * LE interrupt handler.
4578  */
4579 static void le_intr_handler(struct adapter *adap)
4580 {
4581         enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
4582         static const struct intr_info le_intr_info[] = {
4583                 { LIPMISS_F, "LE LIP miss", -1, 0 },
4584                 { LIP0_F, "LE 0 LIP error", -1, 0 },
4585                 { PARITYERR_F, "LE parity error", -1, 1 },
4586                 { UNKNOWNCMD_F, "LE unknown command", -1, 1 },
4587                 { REQQPARERR_F, "LE request queue parity error", -1, 1 },
4588                 { 0 }
4589         };
4590
4591         static struct intr_info t6_le_intr_info[] = {
4592                 { T6_LIPMISS_F, "LE LIP miss", -1, 0 },
4593                 { T6_LIP0_F, "LE 0 LIP error", -1, 0 },
4594                 { TCAMINTPERR_F, "LE parity error", -1, 1 },
4595                 { T6_UNKNOWNCMD_F, "LE unknown command", -1, 1 },
4596                 { SSRAMINTPERR_F, "LE request queue parity error", -1, 1 },
4597                 { 0 }
4598         };
4599
4600         if (t4_handle_intr_status(adap, LE_DB_INT_CAUSE_A,
4601                                   (chip <= CHELSIO_T5) ?
4602                                   le_intr_info : t6_le_intr_info))
4603                 t4_fatal_err(adap);
4604 }
4605
4606 /*
4607  * MPS interrupt handler.
4608  */
4609 static void mps_intr_handler(struct adapter *adapter)
4610 {
4611         static const struct intr_info mps_rx_intr_info[] = {
4612                 { 0xffffff, "MPS Rx parity error", -1, 1 },
4613                 { 0 }
4614         };
4615         static const struct intr_info mps_tx_intr_info[] = {
4616                 { TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 },
4617                 { NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 },
4618                 { TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error",
4619                   -1, 1 },
4620                 { TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error",
4621                   -1, 1 },
4622                 { BUBBLE_F, "MPS Tx underflow", -1, 1 },
4623                 { SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 },
4624                 { FRMERR_F, "MPS Tx framing error", -1, 1 },
4625                 { 0 }
4626         };
4627         static const struct intr_info t6_mps_tx_intr_info[] = {
4628                 { TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 },
4629                 { NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 },
4630                 { TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error",
4631                   -1, 1 },
4632                 { TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error",
4633                   -1, 1 },
4634                 /* MPS Tx Bubble is normal for T6 */
4635                 { SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 },
4636                 { FRMERR_F, "MPS Tx framing error", -1, 1 },
4637                 { 0 }
4638         };
4639         static const struct intr_info mps_trc_intr_info[] = {
4640                 { FILTMEM_V(FILTMEM_M), "MPS TRC filter parity error", -1, 1 },
4641                 { PKTFIFO_V(PKTFIFO_M), "MPS TRC packet FIFO parity error",
4642                   -1, 1 },
4643                 { MISCPERR_F, "MPS TRC misc parity error", -1, 1 },
4644                 { 0 }
4645         };
4646         static const struct intr_info mps_stat_sram_intr_info[] = {
4647                 { 0x1fffff, "MPS statistics SRAM parity error", -1, 1 },
4648                 { 0 }
4649         };
4650         static const struct intr_info mps_stat_tx_intr_info[] = {
4651                 { 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 },
4652                 { 0 }
4653         };
4654         static const struct intr_info mps_stat_rx_intr_info[] = {
4655                 { 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 },
4656                 { 0 }
4657         };
4658         static const struct intr_info mps_cls_intr_info[] = {
4659                 { MATCHSRAM_F, "MPS match SRAM parity error", -1, 1 },
4660                 { MATCHTCAM_F, "MPS match TCAM parity error", -1, 1 },
4661                 { HASHSRAM_F, "MPS hash SRAM parity error", -1, 1 },
4662                 { 0 }
4663         };
4664
4665         int fat;
4666
4667         fat = t4_handle_intr_status(adapter, MPS_RX_PERR_INT_CAUSE_A,
4668                                     mps_rx_intr_info) +
4669               t4_handle_intr_status(adapter, MPS_TX_INT_CAUSE_A,
4670                                     is_t6(adapter->params.chip)
4671                                     ? t6_mps_tx_intr_info
4672                                     : mps_tx_intr_info) +
4673               t4_handle_intr_status(adapter, MPS_TRC_INT_CAUSE_A,
4674                                     mps_trc_intr_info) +
4675               t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_SRAM_A,
4676                                     mps_stat_sram_intr_info) +
4677               t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_TX_FIFO_A,
4678                                     mps_stat_tx_intr_info) +
4679               t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_RX_FIFO_A,
4680                                     mps_stat_rx_intr_info) +
4681               t4_handle_intr_status(adapter, MPS_CLS_INT_CAUSE_A,
4682                                     mps_cls_intr_info);
4683
4684         t4_write_reg(adapter, MPS_INT_CAUSE_A, 0);
4685         t4_read_reg(adapter, MPS_INT_CAUSE_A);                    /* flush */
4686         if (fat)
4687                 t4_fatal_err(adapter);
4688 }
4689
4690 #define MEM_INT_MASK (PERR_INT_CAUSE_F | ECC_CE_INT_CAUSE_F | \
4691                       ECC_UE_INT_CAUSE_F)
4692
4693 /*
4694  * EDC/MC interrupt handler.
4695  */
4696 static void mem_intr_handler(struct adapter *adapter, int idx)
4697 {
4698         static const char name[4][7] = { "EDC0", "EDC1", "MC/MC0", "MC1" };
4699
4700         unsigned int addr, cnt_addr, v;
4701
4702         if (idx <= MEM_EDC1) {
4703                 addr = EDC_REG(EDC_INT_CAUSE_A, idx);
4704                 cnt_addr = EDC_REG(EDC_ECC_STATUS_A, idx);
4705         } else if (idx == MEM_MC) {
4706                 if (is_t4(adapter->params.chip)) {
4707                         addr = MC_INT_CAUSE_A;
4708                         cnt_addr = MC_ECC_STATUS_A;
4709                 } else {
4710                         addr = MC_P_INT_CAUSE_A;
4711                         cnt_addr = MC_P_ECC_STATUS_A;
4712                 }
4713         } else {
4714                 addr = MC_REG(MC_P_INT_CAUSE_A, 1);
4715                 cnt_addr = MC_REG(MC_P_ECC_STATUS_A, 1);
4716         }
4717
4718         v = t4_read_reg(adapter, addr) & MEM_INT_MASK;
4719         if (v & PERR_INT_CAUSE_F)
4720                 dev_alert(adapter->pdev_dev, "%s FIFO parity error\n",
4721                           name[idx]);
4722         if (v & ECC_CE_INT_CAUSE_F) {
4723                 u32 cnt = ECC_CECNT_G(t4_read_reg(adapter, cnt_addr));
4724
4725                 t4_edc_err_read(adapter, idx);
4726
4727                 t4_write_reg(adapter, cnt_addr, ECC_CECNT_V(ECC_CECNT_M));
4728                 if (printk_ratelimit())
4729                         dev_warn(adapter->pdev_dev,
4730                                  "%u %s correctable ECC data error%s\n",
4731                                  cnt, name[idx], cnt > 1 ? "s" : "");
4732         }
4733         if (v & ECC_UE_INT_CAUSE_F)
4734                 dev_alert(adapter->pdev_dev,
4735                           "%s uncorrectable ECC data error\n", name[idx]);
4736
4737         t4_write_reg(adapter, addr, v);
4738         if (v & (PERR_INT_CAUSE_F | ECC_UE_INT_CAUSE_F))
4739                 t4_fatal_err(adapter);
4740 }
4741
4742 /*
4743  * MA interrupt handler.
4744  */
4745 static void ma_intr_handler(struct adapter *adap)
4746 {
4747         u32 v, status = t4_read_reg(adap, MA_INT_CAUSE_A);
4748
4749         if (status & MEM_PERR_INT_CAUSE_F) {
4750                 dev_alert(adap->pdev_dev,
4751                           "MA parity error, parity status %#x\n",
4752                           t4_read_reg(adap, MA_PARITY_ERROR_STATUS1_A));
4753                 if (is_t5(adap->params.chip))
4754                         dev_alert(adap->pdev_dev,
4755                                   "MA parity error, parity status %#x\n",
4756                                   t4_read_reg(adap,
4757                                               MA_PARITY_ERROR_STATUS2_A));
4758         }
4759         if (status & MEM_WRAP_INT_CAUSE_F) {
4760                 v = t4_read_reg(adap, MA_INT_WRAP_STATUS_A);
4761                 dev_alert(adap->pdev_dev, "MA address wrap-around error by "
4762                           "client %u to address %#x\n",
4763                           MEM_WRAP_CLIENT_NUM_G(v),
4764                           MEM_WRAP_ADDRESS_G(v) << 4);
4765         }
4766         t4_write_reg(adap, MA_INT_CAUSE_A, status);
4767         t4_fatal_err(adap);
4768 }
4769
4770 /*
4771  * SMB interrupt handler.
4772  */
4773 static void smb_intr_handler(struct adapter *adap)
4774 {
4775         static const struct intr_info smb_intr_info[] = {
4776                 { MSTTXFIFOPARINT_F, "SMB master Tx FIFO parity error", -1, 1 },
4777                 { MSTRXFIFOPARINT_F, "SMB master Rx FIFO parity error", -1, 1 },
4778                 { SLVFIFOPARINT_F, "SMB slave FIFO parity error", -1, 1 },
4779                 { 0 }
4780         };
4781
4782         if (t4_handle_intr_status(adap, SMB_INT_CAUSE_A, smb_intr_info))
4783                 t4_fatal_err(adap);
4784 }
4785
4786 /*
4787  * NC-SI interrupt handler.
4788  */
4789 static void ncsi_intr_handler(struct adapter *adap)
4790 {
4791         static const struct intr_info ncsi_intr_info[] = {
4792                 { CIM_DM_PRTY_ERR_F, "NC-SI CIM parity error", -1, 1 },
4793                 { MPS_DM_PRTY_ERR_F, "NC-SI MPS parity error", -1, 1 },
4794                 { TXFIFO_PRTY_ERR_F, "NC-SI Tx FIFO parity error", -1, 1 },
4795                 { RXFIFO_PRTY_ERR_F, "NC-SI Rx FIFO parity error", -1, 1 },
4796                 { 0 }
4797         };
4798
4799         if (t4_handle_intr_status(adap, NCSI_INT_CAUSE_A, ncsi_intr_info))
4800                 t4_fatal_err(adap);
4801 }
4802
4803 /*
4804  * XGMAC interrupt handler.
4805  */
4806 static void xgmac_intr_handler(struct adapter *adap, int port)
4807 {
4808         u32 v, int_cause_reg;
4809
4810         if (is_t4(adap->params.chip))
4811                 int_cause_reg = PORT_REG(port, XGMAC_PORT_INT_CAUSE_A);
4812         else
4813                 int_cause_reg = T5_PORT_REG(port, MAC_PORT_INT_CAUSE_A);
4814
4815         v = t4_read_reg(adap, int_cause_reg);
4816
4817         v &= TXFIFO_PRTY_ERR_F | RXFIFO_PRTY_ERR_F;
4818         if (!v)
4819                 return;
4820
4821         if (v & TXFIFO_PRTY_ERR_F)
4822                 dev_alert(adap->pdev_dev, "XGMAC %d Tx FIFO parity error\n",
4823                           port);
4824         if (v & RXFIFO_PRTY_ERR_F)
4825                 dev_alert(adap->pdev_dev, "XGMAC %d Rx FIFO parity error\n",
4826                           port);
4827         t4_write_reg(adap, PORT_REG(port, XGMAC_PORT_INT_CAUSE_A), v);
4828         t4_fatal_err(adap);
4829 }
4830
4831 /*
4832  * PL interrupt handler.
4833  */
4834 static void pl_intr_handler(struct adapter *adap)
4835 {
4836         static const struct intr_info pl_intr_info[] = {
4837                 { FATALPERR_F, "T4 fatal parity error", -1, 1 },
4838                 { PERRVFID_F, "PL VFID_MAP parity error", -1, 1 },
4839                 { 0 }
4840         };
4841
4842         if (t4_handle_intr_status(adap, PL_PL_INT_CAUSE_A, pl_intr_info))
4843                 t4_fatal_err(adap);
4844 }
4845
4846 #define PF_INTR_MASK (PFSW_F)
4847 #define GLBL_INTR_MASK (CIM_F | MPS_F | PL_F | PCIE_F | MC_F | EDC0_F | \
4848                 EDC1_F | LE_F | TP_F | MA_F | PM_TX_F | PM_RX_F | ULP_RX_F | \
4849                 CPL_SWITCH_F | SGE_F | ULP_TX_F | SF_F)
4850
4851 /**
4852  *      t4_slow_intr_handler - control path interrupt handler
4853  *      @adapter: the adapter
4854  *
4855  *      T4 interrupt handler for non-data global interrupt events, e.g., errors.
4856  *      The designation 'slow' is because it involves register reads, while
4857  *      data interrupts typically don't involve any MMIOs.
4858  */
4859 int t4_slow_intr_handler(struct adapter *adapter)
4860 {
4861         u32 cause = t4_read_reg(adapter, PL_INT_CAUSE_A);
4862
4863         if (!(cause & GLBL_INTR_MASK))
4864                 return 0;
4865         if (cause & CIM_F)
4866                 cim_intr_handler(adapter);
4867         if (cause & MPS_F)
4868                 mps_intr_handler(adapter);
4869         if (cause & NCSI_F)
4870                 ncsi_intr_handler(adapter);
4871         if (cause & PL_F)
4872                 pl_intr_handler(adapter);
4873         if (cause & SMB_F)
4874                 smb_intr_handler(adapter);
4875         if (cause & XGMAC0_F)
4876                 xgmac_intr_handler(adapter, 0);
4877         if (cause & XGMAC1_F)
4878                 xgmac_intr_handler(adapter, 1);
4879         if (cause & XGMAC_KR0_F)
4880                 xgmac_intr_handler(adapter, 2);
4881         if (cause & XGMAC_KR1_F)
4882                 xgmac_intr_handler(adapter, 3);
4883         if (cause & PCIE_F)
4884                 pcie_intr_handler(adapter);
4885         if (cause & MC_F)
4886                 mem_intr_handler(adapter, MEM_MC);
4887         if (is_t5(adapter->params.chip) && (cause & MC1_F))
4888                 mem_intr_handler(adapter, MEM_MC1);
4889         if (cause & EDC0_F)
4890                 mem_intr_handler(adapter, MEM_EDC0);
4891         if (cause & EDC1_F)
4892                 mem_intr_handler(adapter, MEM_EDC1);
4893         if (cause & LE_F)
4894                 le_intr_handler(adapter);
4895         if (cause & TP_F)
4896                 tp_intr_handler(adapter);
4897         if (cause & MA_F)
4898                 ma_intr_handler(adapter);
4899         if (cause & PM_TX_F)
4900                 pmtx_intr_handler(adapter);
4901         if (cause & PM_RX_F)
4902                 pmrx_intr_handler(adapter);
4903         if (cause & ULP_RX_F)
4904                 ulprx_intr_handler(adapter);
4905         if (cause & CPL_SWITCH_F)
4906                 cplsw_intr_handler(adapter);
4907         if (cause & SGE_F)
4908                 sge_intr_handler(adapter);
4909         if (cause & ULP_TX_F)
4910                 ulptx_intr_handler(adapter);
4911
4912         /* Clear the interrupts just processed for which we are the master. */
4913         t4_write_reg(adapter, PL_INT_CAUSE_A, cause & GLBL_INTR_MASK);
4914         (void)t4_read_reg(adapter, PL_INT_CAUSE_A); /* flush */
4915         return 1;
4916 }
4917
4918 /**
4919  *      t4_intr_enable - enable interrupts
4920  *      @adapter: the adapter whose interrupts should be enabled
4921  *
4922  *      Enable PF-specific interrupts for the calling function and the top-level
4923  *      interrupt concentrator for global interrupts.  Interrupts are already
4924  *      enabled at each module, here we just enable the roots of the interrupt
4925  *      hierarchies.
4926  *
4927  *      Note: this function should be called only when the driver manages
4928  *      non PF-specific interrupts from the various HW modules.  Only one PCI
4929  *      function at a time should be doing this.
4930  */
4931 void t4_intr_enable(struct adapter *adapter)
4932 {
4933         u32 val = 0;
4934         u32 whoami = t4_read_reg(adapter, PL_WHOAMI_A);
4935         u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ?
4936                         SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
4937
4938         if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
4939                 val = ERR_DROPPED_DB_F | ERR_EGR_CTXT_PRIO_F | DBFIFO_HP_INT_F;
4940         t4_write_reg(adapter, SGE_INT_ENABLE3_A, ERR_CPL_EXCEED_IQE_SIZE_F |
4941                      ERR_INVALID_CIDX_INC_F | ERR_CPL_OPCODE_0_F |
4942                      ERR_DATA_CPL_ON_HIGH_QID1_F | INGRESS_SIZE_ERR_F |
4943                      ERR_DATA_CPL_ON_HIGH_QID0_F | ERR_BAD_DB_PIDX3_F |
4944                      ERR_BAD_DB_PIDX2_F | ERR_BAD_DB_PIDX1_F |
4945                      ERR_BAD_DB_PIDX0_F | ERR_ING_CTXT_PRIO_F |
4946                      DBFIFO_LP_INT_F | EGRESS_SIZE_ERR_F | val);
4947         t4_write_reg(adapter, MYPF_REG(PL_PF_INT_ENABLE_A), PF_INTR_MASK);
4948         t4_set_reg_field(adapter, PL_INT_MAP0_A, 0, 1 << pf);
4949 }
4950
4951 /**
4952  *      t4_intr_disable - disable interrupts
4953  *      @adapter: the adapter whose interrupts should be disabled
4954  *
4955  *      Disable interrupts.  We only disable the top-level interrupt
4956  *      concentrators.  The caller must be a PCI function managing global
4957  *      interrupts.
4958  */
4959 void t4_intr_disable(struct adapter *adapter)
4960 {
4961         u32 whoami, pf;
4962
4963         if (pci_channel_offline(adapter->pdev))
4964                 return;
4965
4966         whoami = t4_read_reg(adapter, PL_WHOAMI_A);
4967         pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ?
4968                         SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
4969
4970         t4_write_reg(adapter, MYPF_REG(PL_PF_INT_ENABLE_A), 0);
4971         t4_set_reg_field(adapter, PL_INT_MAP0_A, 1 << pf, 0);
4972 }
4973
4974 unsigned int t4_chip_rss_size(struct adapter *adap)
4975 {
4976         if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
4977                 return RSS_NENTRIES;
4978         else
4979                 return T6_RSS_NENTRIES;
4980 }
4981
4982 /**
4983  *      t4_config_rss_range - configure a portion of the RSS mapping table
4984  *      @adapter: the adapter
4985  *      @mbox: mbox to use for the FW command
4986  *      @viid: virtual interface whose RSS subtable is to be written
4987  *      @start: start entry in the table to write
4988  *      @n: how many table entries to write
4989  *      @rspq: values for the response queue lookup table
4990  *      @nrspq: number of values in @rspq
4991  *
4992  *      Programs the selected part of the VI's RSS mapping table with the
4993  *      provided values.  If @nrspq < @n the supplied values are used repeatedly
4994  *      until the full table range is populated.
4995  *
4996  *      The caller must ensure the values in @rspq are in the range allowed for
4997  *      @viid.
4998  */
4999 int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
5000                         int start, int n, const u16 *rspq, unsigned int nrspq)
5001 {
5002         int ret;
5003         const u16 *rsp = rspq;
5004         const u16 *rsp_end = rspq + nrspq;
5005         struct fw_rss_ind_tbl_cmd cmd;
5006
5007         memset(&cmd, 0, sizeof(cmd));
5008         cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_IND_TBL_CMD) |
5009                                FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
5010                                FW_RSS_IND_TBL_CMD_VIID_V(viid));
5011         cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
5012
5013         /* each fw_rss_ind_tbl_cmd takes up to 32 entries */
5014         while (n > 0) {
5015                 int nq = min(n, 32);
5016                 __be32 *qp = &cmd.iq0_to_iq2;
5017
5018                 cmd.niqid = cpu_to_be16(nq);
5019                 cmd.startidx = cpu_to_be16(start);
5020
5021                 start += nq;
5022                 n -= nq;
5023
5024                 while (nq > 0) {
5025                         unsigned int v;
5026
5027                         v = FW_RSS_IND_TBL_CMD_IQ0_V(*rsp);
5028                         if (++rsp >= rsp_end)
5029                                 rsp = rspq;
5030                         v |= FW_RSS_IND_TBL_CMD_IQ1_V(*rsp);
5031                         if (++rsp >= rsp_end)
5032                                 rsp = rspq;
5033                         v |= FW_RSS_IND_TBL_CMD_IQ2_V(*rsp);
5034                         if (++rsp >= rsp_end)
5035                                 rsp = rspq;
5036
5037                         *qp++ = cpu_to_be32(v);
5038                         nq -= 3;
5039                 }
5040
5041                 ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
5042                 if (ret)
5043                         return ret;
5044         }
5045         return 0;
5046 }
5047
5048 /**
5049  *      t4_config_glbl_rss - configure the global RSS mode
5050  *      @adapter: the adapter
5051  *      @mbox: mbox to use for the FW command
5052  *      @mode: global RSS mode
5053  *      @flags: mode-specific flags
5054  *
5055  *      Sets the global RSS mode.
5056  */
5057 int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
5058                        unsigned int flags)
5059 {
5060         struct fw_rss_glb_config_cmd c;
5061
5062         memset(&c, 0, sizeof(c));
5063         c.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_RSS_GLB_CONFIG_CMD) |
5064                                     FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
5065         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
5066         if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) {
5067                 c.u.manual.mode_pkd =
5068                         cpu_to_be32(FW_RSS_GLB_CONFIG_CMD_MODE_V(mode));
5069         } else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
5070                 c.u.basicvirtual.mode_pkd =
5071                         cpu_to_be32(FW_RSS_GLB_CONFIG_CMD_MODE_V(mode));
5072                 c.u.basicvirtual.synmapen_to_hashtoeplitz = cpu_to_be32(flags);
5073         } else
5074                 return -EINVAL;
5075         return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
5076 }
5077
5078 /**
5079  *      t4_config_vi_rss - configure per VI RSS settings
5080  *      @adapter: the adapter
5081  *      @mbox: mbox to use for the FW command
5082  *      @viid: the VI id
5083  *      @flags: RSS flags
5084  *      @defq: id of the default RSS queue for the VI.
5085  *
5086  *      Configures VI-specific RSS properties.
5087  */
5088 int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
5089                      unsigned int flags, unsigned int defq)
5090 {
5091         struct fw_rss_vi_config_cmd c;
5092
5093         memset(&c, 0, sizeof(c));
5094         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) |
5095                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
5096                                    FW_RSS_VI_CONFIG_CMD_VIID_V(viid));
5097         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
5098         c.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(flags |
5099                                         FW_RSS_VI_CONFIG_CMD_DEFAULTQ_V(defq));
5100         return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
5101 }
5102
5103 /* Read an RSS table row */
5104 static int rd_rss_row(struct adapter *adap, int row, u32 *val)
5105 {
5106         t4_write_reg(adap, TP_RSS_LKP_TABLE_A, 0xfff00000 | row);
5107         return t4_wait_op_done_val(adap, TP_RSS_LKP_TABLE_A, LKPTBLROWVLD_F, 1,
5108                                    5, 0, val);
5109 }
5110
5111 /**
5112  *      t4_read_rss - read the contents of the RSS mapping table
5113  *      @adapter: the adapter
5114  *      @map: holds the contents of the RSS mapping table
5115  *
5116  *      Reads the contents of the RSS hash->queue mapping table.
5117  */
5118 int t4_read_rss(struct adapter *adapter, u16 *map)
5119 {
5120         int i, ret, nentries;
5121         u32 val;
5122
5123         nentries = t4_chip_rss_size(adapter);
5124         for (i = 0; i < nentries / 2; ++i) {
5125                 ret = rd_rss_row(adapter, i, &val);
5126                 if (ret)
5127                         return ret;
5128                 *map++ = LKPTBLQUEUE0_G(val);
5129                 *map++ = LKPTBLQUEUE1_G(val);
5130         }
5131         return 0;
5132 }
5133
5134 static unsigned int t4_use_ldst(struct adapter *adap)
5135 {
5136         return (adap->flags & FW_OK) && !adap->use_bd;
5137 }
5138
5139 /**
5140  * t4_tp_fw_ldst_rw - Access TP indirect register through LDST
5141  * @adap: the adapter
5142  * @cmd: TP fw ldst address space type
5143  * @vals: where the indirect register values are stored/written
5144  * @nregs: how many indirect registers to read/write
5145  * @start_idx: index of first indirect register to read/write
5146  * @rw: Read (1) or Write (0)
5147  * @sleep_ok: if true we may sleep while awaiting command completion
5148  *
5149  * Access TP indirect registers through LDST
5150  */
5151 static int t4_tp_fw_ldst_rw(struct adapter *adap, int cmd, u32 *vals,
5152                             unsigned int nregs, unsigned int start_index,
5153                             unsigned int rw, bool sleep_ok)
5154 {
5155         int ret = 0;
5156         unsigned int i;
5157         struct fw_ldst_cmd c;
5158
5159         for (i = 0; i < nregs; i++) {
5160                 memset(&c, 0, sizeof(c));
5161                 c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
5162                                                 FW_CMD_REQUEST_F |
5163                                                 (rw ? FW_CMD_READ_F :
5164                                                       FW_CMD_WRITE_F) |
5165                                                 FW_LDST_CMD_ADDRSPACE_V(cmd));
5166                 c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
5167
5168                 c.u.addrval.addr = cpu_to_be32(start_index + i);
5169                 c.u.addrval.val  = rw ? 0 : cpu_to_be32(vals[i]);
5170                 ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c,
5171                                       sleep_ok);
5172                 if (ret)
5173                         return ret;
5174
5175                 if (rw)
5176                         vals[i] = be32_to_cpu(c.u.addrval.val);
5177         }
5178         return 0;
5179 }
5180
5181 /**
5182  * t4_tp_indirect_rw - Read/Write TP indirect register through LDST or backdoor
5183  * @adap: the adapter
5184  * @reg_addr: Address Register
5185  * @reg_data: Data register
5186  * @buff: where the indirect register values are stored/written
5187  * @nregs: how many indirect registers to read/write
5188  * @start_index: index of first indirect register to read/write
5189  * @rw: READ(1) or WRITE(0)
5190  * @sleep_ok: if true we may sleep while awaiting command completion
5191  *
5192  * Read/Write TP indirect registers through LDST if possible.
5193  * Else, use backdoor access
5194  **/
5195 static void t4_tp_indirect_rw(struct adapter *adap, u32 reg_addr, u32 reg_data,
5196                               u32 *buff, u32 nregs, u32 start_index, int rw,
5197                               bool sleep_ok)
5198 {
5199         int rc = -EINVAL;
5200         int cmd;
5201
5202         switch (reg_addr) {
5203         case TP_PIO_ADDR_A:
5204                 cmd = FW_LDST_ADDRSPC_TP_PIO;
5205                 break;
5206         case TP_TM_PIO_ADDR_A:
5207                 cmd = FW_LDST_ADDRSPC_TP_TM_PIO;
5208                 break;
5209         case TP_MIB_INDEX_A:
5210                 cmd = FW_LDST_ADDRSPC_TP_MIB;
5211                 break;
5212         default:
5213                 goto indirect_access;
5214         }
5215
5216         if (t4_use_ldst(adap))
5217                 rc = t4_tp_fw_ldst_rw(adap, cmd, buff, nregs, start_index, rw,
5218                                       sleep_ok);
5219
5220 indirect_access:
5221
5222         if (rc) {
5223                 if (rw)
5224                         t4_read_indirect(adap, reg_addr, reg_data, buff, nregs,
5225                                          start_index);
5226                 else
5227                         t4_write_indirect(adap, reg_addr, reg_data, buff, nregs,
5228                                           start_index);
5229         }
5230 }
5231
5232 /**
5233  * t4_tp_pio_read - Read TP PIO registers
5234  * @adap: the adapter
5235  * @buff: where the indirect register values are written
5236  * @nregs: how many indirect registers to read
5237  * @start_index: index of first indirect register to read
5238  * @sleep_ok: if true we may sleep while awaiting command completion
5239  *
5240  * Read TP PIO Registers
5241  **/
5242 void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs,
5243                     u32 start_index, bool sleep_ok)
5244 {
5245         t4_tp_indirect_rw(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, buff, nregs,
5246                           start_index, 1, sleep_ok);
5247 }
5248
5249 /**
5250  * t4_tp_pio_write - Write TP PIO registers
5251  * @adap: the adapter
5252  * @buff: where the indirect register values are stored
5253  * @nregs: how many indirect registers to write
5254  * @start_index: index of first indirect register to write
5255  * @sleep_ok: if true we may sleep while awaiting command completion
5256  *
5257  * Write TP PIO Registers
5258  **/
5259 static void t4_tp_pio_write(struct adapter *adap, u32 *buff, u32 nregs,
5260                             u32 start_index, bool sleep_ok)
5261 {
5262         t4_tp_indirect_rw(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, buff, nregs,
5263                           start_index, 0, sleep_ok);
5264 }
5265
5266 /**
5267  * t4_tp_tm_pio_read - Read TP TM PIO registers
5268  * @adap: the adapter
5269  * @buff: where the indirect register values are written
5270  * @nregs: how many indirect registers to read
5271  * @start_index: index of first indirect register to read
5272  * @sleep_ok: if true we may sleep while awaiting command completion
5273  *
5274  * Read TP TM PIO Registers
5275  **/
5276 void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs,
5277                        u32 start_index, bool sleep_ok)
5278 {
5279         t4_tp_indirect_rw(adap, TP_TM_PIO_ADDR_A, TP_TM_PIO_DATA_A, buff,
5280                           nregs, start_index, 1, sleep_ok);
5281 }
5282
5283 /**
5284  * t4_tp_mib_read - Read TP MIB registers
5285  * @adap: the adapter
5286  * @buff: where the indirect register values are written
5287  * @nregs: how many indirect registers to read
5288  * @start_index: index of first indirect register to read
5289  * @sleep_ok: if true we may sleep while awaiting command completion
5290  *
5291  * Read TP MIB Registers
5292  **/
5293 void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, u32 start_index,
5294                     bool sleep_ok)
5295 {
5296         t4_tp_indirect_rw(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, buff, nregs,
5297                           start_index, 1, sleep_ok);
5298 }
5299
5300 /**
5301  *      t4_read_rss_key - read the global RSS key
5302  *      @adap: the adapter
5303  *      @key: 10-entry array holding the 320-bit RSS key
5304  *      @sleep_ok: if true we may sleep while awaiting command completion
5305  *
5306  *      Reads the global 320-bit RSS key.
5307  */
5308 void t4_read_rss_key(struct adapter *adap, u32 *key, bool sleep_ok)
5309 {
5310         t4_tp_pio_read(adap, key, 10, TP_RSS_SECRET_KEY0_A, sleep_ok);
5311 }
5312
5313 /**
5314  *      t4_write_rss_key - program one of the RSS keys
5315  *      @adap: the adapter
5316  *      @key: 10-entry array holding the 320-bit RSS key
5317  *      @idx: which RSS key to write
5318  *      @sleep_ok: if true we may sleep while awaiting command completion
5319  *
5320  *      Writes one of the RSS keys with the given 320-bit value.  If @idx is
5321  *      0..15 the corresponding entry in the RSS key table is written,
5322  *      otherwise the global RSS key is written.
5323  */
5324 void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx,
5325                       bool sleep_ok)
5326 {
5327         u8 rss_key_addr_cnt = 16;
5328         u32 vrt = t4_read_reg(adap, TP_RSS_CONFIG_VRT_A);
5329
5330         /* T6 and later: for KeyMode 3 (per-vf and per-vf scramble),
5331          * allows access to key addresses 16-63 by using KeyWrAddrX
5332          * as index[5:4](upper 2) into key table
5333          */
5334         if ((CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) &&
5335             (vrt & KEYEXTEND_F) && (KEYMODE_G(vrt) == 3))
5336                 rss_key_addr_cnt = 32;
5337
5338         t4_tp_pio_write(adap, (void *)key, 10, TP_RSS_SECRET_KEY0_A, sleep_ok);
5339
5340         if (idx >= 0 && idx < rss_key_addr_cnt) {
5341                 if (rss_key_addr_cnt > 16)
5342                         t4_write_reg(adap, TP_RSS_CONFIG_VRT_A,
5343                                      KEYWRADDRX_V(idx >> 4) |
5344                                      T6_VFWRADDR_V(idx) | KEYWREN_F);
5345                 else
5346                         t4_write_reg(adap, TP_RSS_CONFIG_VRT_A,
5347                                      KEYWRADDR_V(idx) | KEYWREN_F);
5348         }
5349 }
5350
5351 /**
5352  *      t4_read_rss_pf_config - read PF RSS Configuration Table
5353  *      @adapter: the adapter
5354  *      @index: the entry in the PF RSS table to read
5355  *      @valp: where to store the returned value
5356  *      @sleep_ok: if true we may sleep while awaiting command completion
5357  *
5358  *      Reads the PF RSS Configuration Table at the specified index and returns
5359  *      the value found there.
5360  */
5361 void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index,
5362                            u32 *valp, bool sleep_ok)
5363 {
5364         t4_tp_pio_read(adapter, valp, 1, TP_RSS_PF0_CONFIG_A + index, sleep_ok);
5365 }
5366
5367 /**
5368  *      t4_read_rss_vf_config - read VF RSS Configuration Table
5369  *      @adapter: the adapter
5370  *      @index: the entry in the VF RSS table to read
5371  *      @vfl: where to store the returned VFL
5372  *      @vfh: where to store the returned VFH
5373  *      @sleep_ok: if true we may sleep while awaiting command completion
5374  *
5375  *      Reads the VF RSS Configuration Table at the specified index and returns
5376  *      the (VFL, VFH) values found there.
5377  */
5378 void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
5379                            u32 *vfl, u32 *vfh, bool sleep_ok)
5380 {
5381         u32 vrt, mask, data;
5382
5383         if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) {
5384                 mask = VFWRADDR_V(VFWRADDR_M);
5385                 data = VFWRADDR_V(index);
5386         } else {
5387                  mask =  T6_VFWRADDR_V(T6_VFWRADDR_M);
5388                  data = T6_VFWRADDR_V(index);
5389         }
5390
5391         /* Request that the index'th VF Table values be read into VFL/VFH.
5392          */
5393         vrt = t4_read_reg(adapter, TP_RSS_CONFIG_VRT_A);
5394         vrt &= ~(VFRDRG_F | VFWREN_F | KEYWREN_F | mask);
5395         vrt |= data | VFRDEN_F;
5396         t4_write_reg(adapter, TP_RSS_CONFIG_VRT_A, vrt);
5397
5398         /* Grab the VFL/VFH values ...
5399          */
5400         t4_tp_pio_read(adapter, vfl, 1, TP_RSS_VFL_CONFIG_A, sleep_ok);
5401         t4_tp_pio_read(adapter, vfh, 1, TP_RSS_VFH_CONFIG_A, sleep_ok);
5402 }
5403
5404 /**
5405  *      t4_read_rss_pf_map - read PF RSS Map
5406  *      @adapter: the adapter
5407  *      @sleep_ok: if true we may sleep while awaiting command completion
5408  *
5409  *      Reads the PF RSS Map register and returns its value.
5410  */
5411 u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok)
5412 {
5413         u32 pfmap;
5414
5415         t4_tp_pio_read(adapter, &pfmap, 1, TP_RSS_PF_MAP_A, sleep_ok);
5416         return pfmap;
5417 }
5418
5419 /**
5420  *      t4_read_rss_pf_mask - read PF RSS Mask
5421  *      @adapter: the adapter
5422  *      @sleep_ok: if true we may sleep while awaiting command completion
5423  *
5424  *      Reads the PF RSS Mask register and returns its value.
5425  */
5426 u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok)
5427 {
5428         u32 pfmask;
5429
5430         t4_tp_pio_read(adapter, &pfmask, 1, TP_RSS_PF_MSK_A, sleep_ok);
5431         return pfmask;
5432 }
5433
5434 /**
5435  *      t4_tp_get_tcp_stats - read TP's TCP MIB counters
5436  *      @adap: the adapter
5437  *      @v4: holds the TCP/IP counter values
5438  *      @v6: holds the TCP/IPv6 counter values
5439  *      @sleep_ok: if true we may sleep while awaiting command completion
5440  *
5441  *      Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters.
5442  *      Either @v4 or @v6 may be %NULL to skip the corresponding stats.
5443  */
5444 void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4,
5445                          struct tp_tcp_stats *v6, bool sleep_ok)
5446 {
5447         u32 val[TP_MIB_TCP_RXT_SEG_LO_A - TP_MIB_TCP_OUT_RST_A + 1];
5448
5449 #define STAT_IDX(x) ((TP_MIB_TCP_##x##_A) - TP_MIB_TCP_OUT_RST_A)
5450 #define STAT(x)     val[STAT_IDX(x)]
5451 #define STAT64(x)   (((u64)STAT(x##_HI) << 32) | STAT(x##_LO))
5452
5453         if (v4) {
5454                 t4_tp_mib_read(adap, val, ARRAY_SIZE(val),
5455                                TP_MIB_TCP_OUT_RST_A, sleep_ok);
5456                 v4->tcp_out_rsts = STAT(OUT_RST);
5457                 v4->tcp_in_segs  = STAT64(IN_SEG);
5458                 v4->tcp_out_segs = STAT64(OUT_SEG);
5459                 v4->tcp_retrans_segs = STAT64(RXT_SEG);
5460         }
5461         if (v6) {
5462                 t4_tp_mib_read(adap, val, ARRAY_SIZE(val),
5463                                TP_MIB_TCP_V6OUT_RST_A, sleep_ok);
5464                 v6->tcp_out_rsts = STAT(OUT_RST);
5465                 v6->tcp_in_segs  = STAT64(IN_SEG);
5466                 v6->tcp_out_segs = STAT64(OUT_SEG);
5467                 v6->tcp_retrans_segs = STAT64(RXT_SEG);
5468         }
5469 #undef STAT64
5470 #undef STAT
5471 #undef STAT_IDX
5472 }
5473
5474 /**
5475  *      t4_tp_get_err_stats - read TP's error MIB counters
5476  *      @adap: the adapter
5477  *      @st: holds the counter values
5478  *      @sleep_ok: if true we may sleep while awaiting command completion
5479  *
5480  *      Returns the values of TP's error counters.
5481  */
5482 void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st,
5483                          bool sleep_ok)
5484 {
5485         int nchan = adap->params.arch.nchan;
5486
5487         t4_tp_mib_read(adap, st->mac_in_errs, nchan, TP_MIB_MAC_IN_ERR_0_A,
5488                        sleep_ok);
5489         t4_tp_mib_read(adap, st->hdr_in_errs, nchan, TP_MIB_HDR_IN_ERR_0_A,
5490                        sleep_ok);
5491         t4_tp_mib_read(adap, st->tcp_in_errs, nchan, TP_MIB_TCP_IN_ERR_0_A,
5492                        sleep_ok);
5493         t4_tp_mib_read(adap, st->tnl_cong_drops, nchan,
5494                        TP_MIB_TNL_CNG_DROP_0_A, sleep_ok);
5495         t4_tp_mib_read(adap, st->ofld_chan_drops, nchan,
5496                        TP_MIB_OFD_CHN_DROP_0_A, sleep_ok);
5497         t4_tp_mib_read(adap, st->tnl_tx_drops, nchan, TP_MIB_TNL_DROP_0_A,
5498                        sleep_ok);
5499         t4_tp_mib_read(adap, st->ofld_vlan_drops, nchan,
5500                        TP_MIB_OFD_VLN_DROP_0_A, sleep_ok);
5501         t4_tp_mib_read(adap, st->tcp6_in_errs, nchan,
5502                        TP_MIB_TCP_V6IN_ERR_0_A, sleep_ok);
5503         t4_tp_mib_read(adap, &st->ofld_no_neigh, 2, TP_MIB_OFD_ARP_DROP_A,
5504                        sleep_ok);
5505 }
5506
5507 /**
5508  *      t4_tp_get_cpl_stats - read TP's CPL MIB counters
5509  *      @adap: the adapter
5510  *      @st: holds the counter values
5511  *      @sleep_ok: if true we may sleep while awaiting command completion
5512  *
5513  *      Returns the values of TP's CPL counters.
5514  */
5515 void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st,
5516                          bool sleep_ok)
5517 {
5518         int nchan = adap->params.arch.nchan;
5519
5520         t4_tp_mib_read(adap, st->req, nchan, TP_MIB_CPL_IN_REQ_0_A, sleep_ok);
5521
5522         t4_tp_mib_read(adap, st->rsp, nchan, TP_MIB_CPL_OUT_RSP_0_A, sleep_ok);
5523 }
5524
5525 /**
5526  *      t4_tp_get_rdma_stats - read TP's RDMA MIB counters
5527  *      @adap: the adapter
5528  *      @st: holds the counter values
5529  *      @sleep_ok: if true we may sleep while awaiting command completion
5530  *
5531  *      Returns the values of TP's RDMA counters.
5532  */
5533 void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st,
5534                           bool sleep_ok)
5535 {
5536         t4_tp_mib_read(adap, &st->rqe_dfr_pkt, 2, TP_MIB_RQE_DFR_PKT_A,
5537                        sleep_ok);
5538 }
5539
5540 /**
5541  *      t4_get_fcoe_stats - read TP's FCoE MIB counters for a port
5542  *      @adap: the adapter
5543  *      @idx: the port index
5544  *      @st: holds the counter values
5545  *      @sleep_ok: if true we may sleep while awaiting command completion
5546  *
5547  *      Returns the values of TP's FCoE counters for the selected port.
5548  */
5549 void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx,
5550                        struct tp_fcoe_stats *st, bool sleep_ok)
5551 {
5552         u32 val[2];
5553
5554         t4_tp_mib_read(adap, &st->frames_ddp, 1, TP_MIB_FCOE_DDP_0_A + idx,
5555                        sleep_ok);
5556
5557         t4_tp_mib_read(adap, &st->frames_drop, 1,
5558                        TP_MIB_FCOE_DROP_0_A + idx, sleep_ok);
5559
5560         t4_tp_mib_read(adap, val, 2, TP_MIB_FCOE_BYTE_0_HI_A + 2 * idx,
5561                        sleep_ok);
5562
5563         st->octets_ddp = ((u64)val[0] << 32) | val[1];
5564 }
5565
5566 /**
5567  *      t4_get_usm_stats - read TP's non-TCP DDP MIB counters
5568  *      @adap: the adapter
5569  *      @st: holds the counter values
5570  *      @sleep_ok: if true we may sleep while awaiting command completion
5571  *
5572  *      Returns the values of TP's counters for non-TCP directly-placed packets.
5573  */
5574 void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st,
5575                       bool sleep_ok)
5576 {
5577         u32 val[4];
5578
5579         t4_tp_mib_read(adap, val, 4, TP_MIB_USM_PKTS_A, sleep_ok);
5580         st->frames = val[0];
5581         st->drops = val[1];
5582         st->octets = ((u64)val[2] << 32) | val[3];
5583 }
5584
5585 /**
5586  *      t4_read_mtu_tbl - returns the values in the HW path MTU table
5587  *      @adap: the adapter
5588  *      @mtus: where to store the MTU values
5589  *      @mtu_log: where to store the MTU base-2 log (may be %NULL)
5590  *
5591  *      Reads the HW path MTU table.
5592  */
5593 void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log)
5594 {
5595         u32 v;
5596         int i;
5597
5598         for (i = 0; i < NMTUS; ++i) {
5599                 t4_write_reg(adap, TP_MTU_TABLE_A,
5600                              MTUINDEX_V(0xff) | MTUVALUE_V(i));
5601                 v = t4_read_reg(adap, TP_MTU_TABLE_A);
5602                 mtus[i] = MTUVALUE_G(v);
5603                 if (mtu_log)
5604                         mtu_log[i] = MTUWIDTH_G(v);
5605         }
5606 }
5607
5608 /**
5609  *      t4_read_cong_tbl - reads the congestion control table
5610  *      @adap: the adapter
5611  *      @incr: where to store the alpha values
5612  *
5613  *      Reads the additive increments programmed into the HW congestion
5614  *      control table.
5615  */
5616 void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN])
5617 {
5618         unsigned int mtu, w;
5619
5620         for (mtu = 0; mtu < NMTUS; ++mtu)
5621                 for (w = 0; w < NCCTRL_WIN; ++w) {
5622                         t4_write_reg(adap, TP_CCTRL_TABLE_A,
5623                                      ROWINDEX_V(0xffff) | (mtu << 5) | w);
5624                         incr[mtu][w] = (u16)t4_read_reg(adap,
5625                                                 TP_CCTRL_TABLE_A) & 0x1fff;
5626                 }
5627 }
5628
5629 /**
5630  *      t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register
5631  *      @adap: the adapter
5632  *      @addr: the indirect TP register address
5633  *      @mask: specifies the field within the register to modify
5634  *      @val: new value for the field
5635  *
5636  *      Sets a field of an indirect TP register to the given value.
5637  */
5638 void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
5639                             unsigned int mask, unsigned int val)
5640 {
5641         t4_write_reg(adap, TP_PIO_ADDR_A, addr);
5642         val |= t4_read_reg(adap, TP_PIO_DATA_A) & ~mask;
5643         t4_write_reg(adap, TP_PIO_DATA_A, val);
5644 }
5645
5646 /**
5647  *      init_cong_ctrl - initialize congestion control parameters
5648  *      @a: the alpha values for congestion control
5649  *      @b: the beta values for congestion control
5650  *
5651  *      Initialize the congestion control parameters.
5652  */
5653 static void init_cong_ctrl(unsigned short *a, unsigned short *b)
5654 {
5655         a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1;
5656         a[9] = 2;
5657         a[10] = 3;
5658         a[11] = 4;
5659         a[12] = 5;
5660         a[13] = 6;
5661         a[14] = 7;
5662         a[15] = 8;
5663         a[16] = 9;
5664         a[17] = 10;
5665         a[18] = 14;
5666         a[19] = 17;
5667         a[20] = 21;
5668         a[21] = 25;
5669         a[22] = 30;
5670         a[23] = 35;
5671         a[24] = 45;
5672         a[25] = 60;
5673         a[26] = 80;
5674         a[27] = 100;
5675         a[28] = 200;
5676         a[29] = 300;
5677         a[30] = 400;
5678         a[31] = 500;
5679
5680         b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0;
5681         b[9] = b[10] = 1;
5682         b[11] = b[12] = 2;
5683         b[13] = b[14] = b[15] = b[16] = 3;
5684         b[17] = b[18] = b[19] = b[20] = b[21] = 4;
5685         b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5;
5686         b[28] = b[29] = 6;
5687         b[30] = b[31] = 7;
5688 }
5689
5690 /* The minimum additive increment value for the congestion control table */
5691 #define CC_MIN_INCR 2U
5692
5693 /**
5694  *      t4_load_mtus - write the MTU and congestion control HW tables
5695  *      @adap: the adapter
5696  *      @mtus: the values for the MTU table
5697  *      @alpha: the values for the congestion control alpha parameter
5698  *      @beta: the values for the congestion control beta parameter
5699  *
5700  *      Write the HW MTU table with the supplied MTUs and the high-speed
5701  *      congestion control table with the supplied alpha, beta, and MTUs.
5702  *      We write the two tables together because the additive increments
5703  *      depend on the MTUs.
5704  */
5705 void t4_load_mtus(struct adapter *adap, const unsigned short *mtus,
5706                   const unsigned short *alpha, const unsigned short *beta)
5707 {
5708         static const unsigned int avg_pkts[NCCTRL_WIN] = {
5709                 2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640,
5710                 896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480,
5711                 28672, 40960, 57344, 81920, 114688, 163840, 229376
5712         };
5713
5714         unsigned int i, w;
5715
5716         for (i = 0; i < NMTUS; ++i) {
5717                 unsigned int mtu = mtus[i];
5718                 unsigned int log2 = fls(mtu);
5719
5720                 if (!(mtu & ((1 << log2) >> 2)))     /* round */
5721                         log2--;
5722                 t4_write_reg(adap, TP_MTU_TABLE_A, MTUINDEX_V(i) |
5723                              MTUWIDTH_V(log2) | MTUVALUE_V(mtu));
5724
5725                 for (w = 0; w < NCCTRL_WIN; ++w) {
5726                         unsigned int inc;
5727
5728                         inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w],
5729                                   CC_MIN_INCR);
5730
5731                         t4_write_reg(adap, TP_CCTRL_TABLE_A, (i << 21) |
5732                                      (w << 16) | (beta[w] << 13) | inc);
5733                 }
5734         }
5735 }
5736
5737 /* Calculates a rate in bytes/s given the number of 256-byte units per 4K core
5738  * clocks.  The formula is
5739  *
5740  * bytes/s = bytes256 * 256 * ClkFreq / 4096
5741  *
5742  * which is equivalent to
5743  *
5744  * bytes/s = 62.5 * bytes256 * ClkFreq_ms
5745  */
5746 static u64 chan_rate(struct adapter *adap, unsigned int bytes256)
5747 {
5748         u64 v = bytes256 * adap->params.vpd.cclk;
5749
5750         return v * 62 + v / 2;
5751 }
5752
5753 /**
5754  *      t4_get_chan_txrate - get the current per channel Tx rates
5755  *      @adap: the adapter
5756  *      @nic_rate: rates for NIC traffic
5757  *      @ofld_rate: rates for offloaded traffic
5758  *
5759  *      Return the current Tx rates in bytes/s for NIC and offloaded traffic
5760  *      for each channel.
5761  */
5762 void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate)
5763 {
5764         u32 v;
5765
5766         v = t4_read_reg(adap, TP_TX_TRATE_A);
5767         nic_rate[0] = chan_rate(adap, TNLRATE0_G(v));
5768         nic_rate[1] = chan_rate(adap, TNLRATE1_G(v));
5769         if (adap->params.arch.nchan == NCHAN) {
5770                 nic_rate[2] = chan_rate(adap, TNLRATE2_G(v));
5771                 nic_rate[3] = chan_rate(adap, TNLRATE3_G(v));
5772         }
5773
5774         v = t4_read_reg(adap, TP_TX_ORATE_A);
5775         ofld_rate[0] = chan_rate(adap, OFDRATE0_G(v));
5776         ofld_rate[1] = chan_rate(adap, OFDRATE1_G(v));
5777         if (adap->params.arch.nchan == NCHAN) {
5778                 ofld_rate[2] = chan_rate(adap, OFDRATE2_G(v));
5779                 ofld_rate[3] = chan_rate(adap, OFDRATE3_G(v));
5780         }
5781 }
5782
5783 /**
5784  *      t4_set_trace_filter - configure one of the tracing filters
5785  *      @adap: the adapter
5786  *      @tp: the desired trace filter parameters
5787  *      @idx: which filter to configure
5788  *      @enable: whether to enable or disable the filter
5789  *
5790  *      Configures one of the tracing filters available in HW.  If @enable is
5791  *      %0 @tp is not examined and may be %NULL. The user is responsible to
5792  *      set the single/multiple trace mode by writing to MPS_TRC_CFG_A register
5793  */
5794 int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp,
5795                         int idx, int enable)
5796 {
5797         int i, ofst = idx * 4;
5798         u32 data_reg, mask_reg, cfg;
5799         u32 multitrc = TRCMULTIFILTER_F;
5800
5801         if (!enable) {
5802                 t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, 0);
5803                 return 0;
5804         }
5805
5806         cfg = t4_read_reg(adap, MPS_TRC_CFG_A);
5807         if (cfg & TRCMULTIFILTER_F) {
5808                 /* If multiple tracers are enabled, then maximum
5809                  * capture size is 2.5KB (FIFO size of a single channel)
5810                  * minus 2 flits for CPL_TRACE_PKT header.
5811                  */
5812                 if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8)))
5813                         return -EINVAL;
5814         } else {
5815                 /* If multiple tracers are disabled, to avoid deadlocks
5816                  * maximum packet capture size of 9600 bytes is recommended.
5817                  * Also in this mode, only trace0 can be enabled and running.
5818                  */
5819                 multitrc = 0;
5820                 if (tp->snap_len > 9600 || idx)
5821                         return -EINVAL;
5822         }
5823
5824         if (tp->port > (is_t4(adap->params.chip) ? 11 : 19) || tp->invert > 1 ||
5825             tp->skip_len > TFLENGTH_M || tp->skip_ofst > TFOFFSET_M ||
5826             tp->min_len > TFMINPKTSIZE_M)
5827                 return -EINVAL;
5828
5829         /* stop the tracer we'll be changing */
5830         t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, 0);
5831
5832         idx *= (MPS_TRC_FILTER1_MATCH_A - MPS_TRC_FILTER0_MATCH_A);
5833         data_reg = MPS_TRC_FILTER0_MATCH_A + idx;
5834         mask_reg = MPS_TRC_FILTER0_DONT_CARE_A + idx;
5835
5836         for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
5837                 t4_write_reg(adap, data_reg, tp->data[i]);
5838                 t4_write_reg(adap, mask_reg, ~tp->mask[i]);
5839         }
5840         t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_B_A + ofst,
5841                      TFCAPTUREMAX_V(tp->snap_len) |
5842                      TFMINPKTSIZE_V(tp->min_len));
5843         t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst,
5844                      TFOFFSET_V(tp->skip_ofst) | TFLENGTH_V(tp->skip_len) |
5845                      (is_t4(adap->params.chip) ?
5846                      TFPORT_V(tp->port) | TFEN_F | TFINVERTMATCH_V(tp->invert) :
5847                      T5_TFPORT_V(tp->port) | T5_TFEN_F |
5848                      T5_TFINVERTMATCH_V(tp->invert)));
5849
5850         return 0;
5851 }
5852
5853 /**
5854  *      t4_get_trace_filter - query one of the tracing filters
5855  *      @adap: the adapter
5856  *      @tp: the current trace filter parameters
5857  *      @idx: which trace filter to query
5858  *      @enabled: non-zero if the filter is enabled
5859  *
5860  *      Returns the current settings of one of the HW tracing filters.
5861  */
5862 void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx,
5863                          int *enabled)
5864 {
5865         u32 ctla, ctlb;
5866         int i, ofst = idx * 4;
5867         u32 data_reg, mask_reg;
5868
5869         ctla = t4_read_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst);
5870         ctlb = t4_read_reg(adap, MPS_TRC_FILTER_MATCH_CTL_B_A + ofst);
5871
5872         if (is_t4(adap->params.chip)) {
5873                 *enabled = !!(ctla & TFEN_F);
5874                 tp->port =  TFPORT_G(ctla);
5875                 tp->invert = !!(ctla & TFINVERTMATCH_F);
5876         } else {
5877                 *enabled = !!(ctla & T5_TFEN_F);
5878                 tp->port = T5_TFPORT_G(ctla);
5879                 tp->invert = !!(ctla & T5_TFINVERTMATCH_F);
5880         }
5881         tp->snap_len = TFCAPTUREMAX_G(ctlb);
5882         tp->min_len = TFMINPKTSIZE_G(ctlb);
5883         tp->skip_ofst = TFOFFSET_G(ctla);
5884         tp->skip_len = TFLENGTH_G(ctla);
5885
5886         ofst = (MPS_TRC_FILTER1_MATCH_A - MPS_TRC_FILTER0_MATCH_A) * idx;
5887         data_reg = MPS_TRC_FILTER0_MATCH_A + ofst;
5888         mask_reg = MPS_TRC_FILTER0_DONT_CARE_A + ofst;
5889
5890         for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
5891                 tp->mask[i] = ~t4_read_reg(adap, mask_reg);
5892                 tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i];
5893         }
5894 }
5895
5896 /**
5897  *      t4_pmtx_get_stats - returns the HW stats from PMTX
5898  *      @adap: the adapter
5899  *      @cnt: where to store the count statistics
5900  *      @cycles: where to store the cycle statistics
5901  *
5902  *      Returns performance statistics from PMTX.
5903  */
5904 void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
5905 {
5906         int i;
5907         u32 data[2];
5908
5909         for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) {
5910                 t4_write_reg(adap, PM_TX_STAT_CONFIG_A, i + 1);
5911                 cnt[i] = t4_read_reg(adap, PM_TX_STAT_COUNT_A);
5912                 if (is_t4(adap->params.chip)) {
5913                         cycles[i] = t4_read_reg64(adap, PM_TX_STAT_LSB_A);
5914                 } else {
5915                         t4_read_indirect(adap, PM_TX_DBG_CTRL_A,
5916                                          PM_TX_DBG_DATA_A, data, 2,
5917                                          PM_TX_DBG_STAT_MSB_A);
5918                         cycles[i] = (((u64)data[0] << 32) | data[1]);
5919                 }
5920         }
5921 }
5922
5923 /**
5924  *      t4_pmrx_get_stats - returns the HW stats from PMRX
5925  *      @adap: the adapter
5926  *      @cnt: where to store the count statistics
5927  *      @cycles: where to store the cycle statistics
5928  *
5929  *      Returns performance statistics from PMRX.
5930  */
5931 void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
5932 {
5933         int i;
5934         u32 data[2];
5935
5936         for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) {
5937                 t4_write_reg(adap, PM_RX_STAT_CONFIG_A, i + 1);
5938                 cnt[i] = t4_read_reg(adap, PM_RX_STAT_COUNT_A);
5939                 if (is_t4(adap->params.chip)) {
5940                         cycles[i] = t4_read_reg64(adap, PM_RX_STAT_LSB_A);
5941                 } else {
5942                         t4_read_indirect(adap, PM_RX_DBG_CTRL_A,
5943                                          PM_RX_DBG_DATA_A, data, 2,
5944                                          PM_RX_DBG_STAT_MSB_A);
5945                         cycles[i] = (((u64)data[0] << 32) | data[1]);
5946                 }
5947         }
5948 }
5949
5950 /**
5951  *      compute_mps_bg_map - compute the MPS Buffer Group Map for a Port
5952  *      @adap: the adapter
5953  *      @pidx: the port index
5954  *
5955  *      Computes and returns a bitmap indicating which MPS buffer groups are
5956  *      associated with the given Port.  Bit i is set if buffer group i is
5957  *      used by the Port.
5958  */
5959 static inline unsigned int compute_mps_bg_map(struct adapter *adapter,
5960                                               int pidx)
5961 {
5962         unsigned int chip_version, nports;
5963
5964         chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
5965         nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A));
5966
5967         switch (chip_version) {
5968         case CHELSIO_T4:
5969         case CHELSIO_T5:
5970                 switch (nports) {
5971                 case 1: return 0xf;
5972                 case 2: return 3 << (2 * pidx);
5973                 case 4: return 1 << pidx;
5974                 }
5975                 break;
5976
5977         case CHELSIO_T6:
5978                 switch (nports) {
5979                 case 2: return 1 << (2 * pidx);
5980                 }
5981                 break;
5982         }
5983
5984         dev_err(adapter->pdev_dev, "Need MPS Buffer Group Map for Chip %0x, Nports %d\n",
5985                 chip_version, nports);
5986
5987         return 0;
5988 }
5989
5990 /**
5991  *      t4_get_mps_bg_map - return the buffer groups associated with a port
5992  *      @adapter: the adapter
5993  *      @pidx: the port index
5994  *
5995  *      Returns a bitmap indicating which MPS buffer groups are associated
5996  *      with the given Port.  Bit i is set if buffer group i is used by the
5997  *      Port.
5998  */
5999 unsigned int t4_get_mps_bg_map(struct adapter *adapter, int pidx)
6000 {
6001         u8 *mps_bg_map;
6002         unsigned int nports;
6003
6004         nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A));
6005         if (pidx >= nports) {
6006                 CH_WARN(adapter, "MPS Port Index %d >= Nports %d\n",
6007                         pidx, nports);
6008                 return 0;
6009         }
6010
6011         /* If we've already retrieved/computed this, just return the result.
6012          */
6013         mps_bg_map = adapter->params.mps_bg_map;
6014         if (mps_bg_map[pidx])
6015                 return mps_bg_map[pidx];
6016
6017         /* Newer Firmware can tell us what the MPS Buffer Group Map is.
6018          * If we're talking to such Firmware, let it tell us.  If the new
6019          * API isn't supported, revert back to old hardcoded way.  The value
6020          * obtained from Firmware is encoded in below format:
6021          *
6022          * val = (( MPSBGMAP[Port 3] << 24 ) |
6023          *        ( MPSBGMAP[Port 2] << 16 ) |
6024          *        ( MPSBGMAP[Port 1] <<  8 ) |
6025          *        ( MPSBGMAP[Port 0] <<  0 ))
6026          */
6027         if (adapter->flags & FW_OK) {
6028                 u32 param, val;
6029                 int ret;
6030
6031                 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
6032                          FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_MPSBGMAP));
6033                 ret = t4_query_params_ns(adapter, adapter->mbox, adapter->pf,
6034                                          0, 1, &param, &val);
6035                 if (!ret) {
6036                         int p;
6037
6038                         /* Store the BG Map for all of the Ports in order to
6039                          * avoid more calls to the Firmware in the future.
6040                          */
6041                         for (p = 0; p < MAX_NPORTS; p++, val >>= 8)
6042                                 mps_bg_map[p] = val & 0xff;
6043
6044                         return mps_bg_map[pidx];
6045                 }
6046         }
6047
6048         /* Either we're not talking to the Firmware or we're dealing with
6049          * older Firmware which doesn't support the new API to get the MPS
6050          * Buffer Group Map.  Fall back to computing it ourselves.
6051          */
6052         mps_bg_map[pidx] = compute_mps_bg_map(adapter, pidx);
6053         return mps_bg_map[pidx];
6054 }
6055
6056 /**
6057  *      t4_get_tp_ch_map - return TP ingress channels associated with a port
6058  *      @adapter: the adapter
6059  *      @pidx: the port index
6060  *
6061  *      Returns a bitmap indicating which TP Ingress Channels are associated
6062  *      with a given Port.  Bit i is set if TP Ingress Channel i is used by
6063  *      the Port.
6064  */
6065 unsigned int t4_get_tp_ch_map(struct adapter *adap, int pidx)
6066 {
6067         unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
6068         unsigned int nports = 1 << NUMPORTS_G(t4_read_reg(adap, MPS_CMN_CTL_A));
6069
6070         if (pidx >= nports) {
6071                 dev_warn(adap->pdev_dev, "TP Port Index %d >= Nports %d\n",
6072                          pidx, nports);
6073                 return 0;
6074         }
6075
6076         switch (chip_version) {
6077         case CHELSIO_T4:
6078         case CHELSIO_T5:
6079                 /* Note that this happens to be the same values as the MPS
6080                  * Buffer Group Map for these Chips.  But we replicate the code
6081                  * here because they're really separate concepts.
6082                  */
6083                 switch (nports) {
6084                 case 1: return 0xf;
6085                 case 2: return 3 << (2 * pidx);
6086                 case 4: return 1 << pidx;
6087                 }
6088                 break;
6089
6090         case CHELSIO_T6:
6091                 switch (nports) {
6092                 case 1:
6093                 case 2: return 1 << pidx;
6094                 }
6095                 break;
6096         }
6097
6098         dev_err(adap->pdev_dev, "Need TP Channel Map for Chip %0x, Nports %d\n",
6099                 chip_version, nports);
6100         return 0;
6101 }
6102
6103 /**
6104  *      t4_get_port_type_description - return Port Type string description
6105  *      @port_type: firmware Port Type enumeration
6106  */
6107 const char *t4_get_port_type_description(enum fw_port_type port_type)
6108 {
6109         static const char *const port_type_description[] = {
6110                 "Fiber_XFI",
6111                 "Fiber_XAUI",
6112                 "BT_SGMII",
6113                 "BT_XFI",
6114                 "BT_XAUI",
6115                 "KX4",
6116                 "CX4",
6117                 "KX",
6118                 "KR",
6119                 "SFP",
6120                 "BP_AP",
6121                 "BP4_AP",
6122                 "QSFP_10G",
6123                 "QSA",
6124                 "QSFP",
6125                 "BP40_BA",
6126                 "KR4_100G",
6127                 "CR4_QSFP",
6128                 "CR_QSFP",
6129                 "CR2_QSFP",
6130                 "SFP28",
6131                 "KR_SFP28",
6132                 "KR_XLAUI"
6133         };
6134
6135         if (port_type < ARRAY_SIZE(port_type_description))
6136                 return port_type_description[port_type];
6137         return "UNKNOWN";
6138 }
6139
6140 /**
6141  *      t4_get_port_stats_offset - collect port stats relative to a previous
6142  *                                 snapshot
6143  *      @adap: The adapter
6144  *      @idx: The port
6145  *      @stats: Current stats to fill
6146  *      @offset: Previous stats snapshot
6147  */
6148 void t4_get_port_stats_offset(struct adapter *adap, int idx,
6149                               struct port_stats *stats,
6150                               struct port_stats *offset)
6151 {
6152         u64 *s, *o;
6153         int i;
6154
6155         t4_get_port_stats(adap, idx, stats);
6156         for (i = 0, s = (u64 *)stats, o = (u64 *)offset;
6157                         i < (sizeof(struct port_stats) / sizeof(u64));
6158                         i++, s++, o++)
6159                 *s -= *o;
6160 }
6161
6162 /**
6163  *      t4_get_port_stats - collect port statistics
6164  *      @adap: the adapter
6165  *      @idx: the port index
6166  *      @p: the stats structure to fill
6167  *
6168  *      Collect statistics related to the given port from HW.
6169  */
6170 void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
6171 {
6172         u32 bgmap = t4_get_mps_bg_map(adap, idx);
6173         u32 stat_ctl = t4_read_reg(adap, MPS_STAT_CTL_A);
6174
6175 #define GET_STAT(name) \
6176         t4_read_reg64(adap, \
6177         (is_t4(adap->params.chip) ? PORT_REG(idx, MPS_PORT_STAT_##name##_L) : \
6178         T5_PORT_REG(idx, MPS_PORT_STAT_##name##_L)))
6179 #define GET_STAT_COM(name) t4_read_reg64(adap, MPS_STAT_##name##_L)
6180
6181         p->tx_octets           = GET_STAT(TX_PORT_BYTES);
6182         p->tx_frames           = GET_STAT(TX_PORT_FRAMES);
6183         p->tx_bcast_frames     = GET_STAT(TX_PORT_BCAST);
6184         p->tx_mcast_frames     = GET_STAT(TX_PORT_MCAST);
6185         p->tx_ucast_frames     = GET_STAT(TX_PORT_UCAST);
6186         p->tx_error_frames     = GET_STAT(TX_PORT_ERROR);
6187         p->tx_frames_64        = GET_STAT(TX_PORT_64B);
6188         p->tx_frames_65_127    = GET_STAT(TX_PORT_65B_127B);
6189         p->tx_frames_128_255   = GET_STAT(TX_PORT_128B_255B);
6190         p->tx_frames_256_511   = GET_STAT(TX_PORT_256B_511B);
6191         p->tx_frames_512_1023  = GET_STAT(TX_PORT_512B_1023B);
6192         p->tx_frames_1024_1518 = GET_STAT(TX_PORT_1024B_1518B);
6193         p->tx_frames_1519_max  = GET_STAT(TX_PORT_1519B_MAX);
6194         p->tx_drop             = GET_STAT(TX_PORT_DROP);
6195         p->tx_pause            = GET_STAT(TX_PORT_PAUSE);
6196         p->tx_ppp0             = GET_STAT(TX_PORT_PPP0);
6197         p->tx_ppp1             = GET_STAT(TX_PORT_PPP1);
6198         p->tx_ppp2             = GET_STAT(TX_PORT_PPP2);
6199         p->tx_ppp3             = GET_STAT(TX_PORT_PPP3);
6200         p->tx_ppp4             = GET_STAT(TX_PORT_PPP4);
6201         p->tx_ppp5             = GET_STAT(TX_PORT_PPP5);
6202         p->tx_ppp6             = GET_STAT(TX_PORT_PPP6);
6203         p->tx_ppp7             = GET_STAT(TX_PORT_PPP7);
6204
6205         if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) {
6206                 if (stat_ctl & COUNTPAUSESTATTX_F)
6207                         p->tx_frames_64 -= p->tx_pause;
6208                 if (stat_ctl & COUNTPAUSEMCTX_F)
6209                         p->tx_mcast_frames -= p->tx_pause;
6210         }
6211         p->rx_octets           = GET_STAT(RX_PORT_BYTES);
6212         p->rx_frames           = GET_STAT(RX_PORT_FRAMES);
6213         p->rx_bcast_frames     = GET_STAT(RX_PORT_BCAST);
6214         p->rx_mcast_frames     = GET_STAT(RX_PORT_MCAST);
6215         p->rx_ucast_frames     = GET_STAT(RX_PORT_UCAST);
6216         p->rx_too_long         = GET_STAT(RX_PORT_MTU_ERROR);
6217         p->rx_jabber           = GET_STAT(RX_PORT_MTU_CRC_ERROR);
6218         p->rx_fcs_err          = GET_STAT(RX_PORT_CRC_ERROR);
6219         p->rx_len_err          = GET_STAT(RX_PORT_LEN_ERROR);
6220         p->rx_symbol_err       = GET_STAT(RX_PORT_SYM_ERROR);
6221         p->rx_runt             = GET_STAT(RX_PORT_LESS_64B);
6222         p->rx_frames_64        = GET_STAT(RX_PORT_64B);
6223         p->rx_frames_65_127    = GET_STAT(RX_PORT_65B_127B);
6224         p->rx_frames_128_255   = GET_STAT(RX_PORT_128B_255B);
6225         p->rx_frames_256_511   = GET_STAT(RX_PORT_256B_511B);
6226         p->rx_frames_512_1023  = GET_STAT(RX_PORT_512B_1023B);
6227         p->rx_frames_1024_1518 = GET_STAT(RX_PORT_1024B_1518B);
6228         p->rx_frames_1519_max  = GET_STAT(RX_PORT_1519B_MAX);
6229         p->rx_pause            = GET_STAT(RX_PORT_PAUSE);
6230         p->rx_ppp0             = GET_STAT(RX_PORT_PPP0);
6231         p->rx_ppp1             = GET_STAT(RX_PORT_PPP1);
6232         p->rx_ppp2             = GET_STAT(RX_PORT_PPP2);
6233         p->rx_ppp3             = GET_STAT(RX_PORT_PPP3);
6234         p->rx_ppp4             = GET_STAT(RX_PORT_PPP4);
6235         p->rx_ppp5             = GET_STAT(RX_PORT_PPP5);
6236         p->rx_ppp6             = GET_STAT(RX_PORT_PPP6);
6237         p->rx_ppp7             = GET_STAT(RX_PORT_PPP7);
6238
6239         if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) {
6240                 if (stat_ctl & COUNTPAUSESTATRX_F)
6241                         p->rx_frames_64 -= p->rx_pause;
6242                 if (stat_ctl & COUNTPAUSEMCRX_F)
6243                         p->rx_mcast_frames -= p->rx_pause;
6244         }
6245
6246         p->rx_ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0;
6247         p->rx_ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0;
6248         p->rx_ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0;
6249         p->rx_ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0;
6250         p->rx_trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0;
6251         p->rx_trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0;
6252         p->rx_trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0;
6253         p->rx_trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0;
6254
6255 #undef GET_STAT
6256 #undef GET_STAT_COM
6257 }
6258
6259 /**
6260  *      t4_get_lb_stats - collect loopback port statistics
6261  *      @adap: the adapter
6262  *      @idx: the loopback port index
6263  *      @p: the stats structure to fill
6264  *
6265  *      Return HW statistics for the given loopback port.
6266  */
6267 void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p)
6268 {
6269         u32 bgmap = t4_get_mps_bg_map(adap, idx);
6270
6271 #define GET_STAT(name) \
6272         t4_read_reg64(adap, \
6273         (is_t4(adap->params.chip) ? \
6274         PORT_REG(idx, MPS_PORT_STAT_LB_PORT_##name##_L) : \
6275         T5_PORT_REG(idx, MPS_PORT_STAT_LB_PORT_##name##_L)))
6276 #define GET_STAT_COM(name) t4_read_reg64(adap, MPS_STAT_##name##_L)
6277
6278         p->octets           = GET_STAT(BYTES);
6279         p->frames           = GET_STAT(FRAMES);
6280         p->bcast_frames     = GET_STAT(BCAST);
6281         p->mcast_frames     = GET_STAT(MCAST);
6282         p->ucast_frames     = GET_STAT(UCAST);
6283         p->error_frames     = GET_STAT(ERROR);
6284
6285         p->frames_64        = GET_STAT(64B);
6286         p->frames_65_127    = GET_STAT(65B_127B);
6287         p->frames_128_255   = GET_STAT(128B_255B);
6288         p->frames_256_511   = GET_STAT(256B_511B);
6289         p->frames_512_1023  = GET_STAT(512B_1023B);
6290         p->frames_1024_1518 = GET_STAT(1024B_1518B);
6291         p->frames_1519_max  = GET_STAT(1519B_MAX);
6292         p->drop             = GET_STAT(DROP_FRAMES);
6293
6294         p->ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_DROP_FRAME) : 0;
6295         p->ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_DROP_FRAME) : 0;
6296         p->ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_DROP_FRAME) : 0;
6297         p->ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_DROP_FRAME) : 0;
6298         p->trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_TRUNC_FRAME) : 0;
6299         p->trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_TRUNC_FRAME) : 0;
6300         p->trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_TRUNC_FRAME) : 0;
6301         p->trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_TRUNC_FRAME) : 0;
6302
6303 #undef GET_STAT
6304 #undef GET_STAT_COM
6305 }
6306
6307 /*     t4_mk_filtdelwr - create a delete filter WR
6308  *     @ftid: the filter ID
6309  *     @wr: the filter work request to populate
6310  *     @qid: ingress queue to receive the delete notification
6311  *
6312  *     Creates a filter work request to delete the supplied filter.  If @qid is
6313  *     negative the delete notification is suppressed.
6314  */
6315 void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid)
6316 {
6317         memset(wr, 0, sizeof(*wr));
6318         wr->op_pkd = cpu_to_be32(FW_WR_OP_V(FW_FILTER_WR));
6319         wr->len16_pkd = cpu_to_be32(FW_WR_LEN16_V(sizeof(*wr) / 16));
6320         wr->tid_to_iq = cpu_to_be32(FW_FILTER_WR_TID_V(ftid) |
6321                                     FW_FILTER_WR_NOREPLY_V(qid < 0));
6322         wr->del_filter_to_l2tix = cpu_to_be32(FW_FILTER_WR_DEL_FILTER_F);
6323         if (qid >= 0)
6324                 wr->rx_chan_rx_rpl_iq =
6325                         cpu_to_be16(FW_FILTER_WR_RX_RPL_IQ_V(qid));
6326 }
6327
6328 #define INIT_CMD(var, cmd, rd_wr) do { \
6329         (var).op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_##cmd##_CMD) | \
6330                                         FW_CMD_REQUEST_F | \
6331                                         FW_CMD_##rd_wr##_F); \
6332         (var).retval_len16 = cpu_to_be32(FW_LEN16(var)); \
6333 } while (0)
6334
6335 int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox,
6336                           u32 addr, u32 val)
6337 {
6338         u32 ldst_addrspace;
6339         struct fw_ldst_cmd c;
6340
6341         memset(&c, 0, sizeof(c));
6342         ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FIRMWARE);
6343         c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6344                                         FW_CMD_REQUEST_F |
6345                                         FW_CMD_WRITE_F |
6346                                         ldst_addrspace);
6347         c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6348         c.u.addrval.addr = cpu_to_be32(addr);
6349         c.u.addrval.val = cpu_to_be32(val);
6350
6351         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6352 }
6353
6354 /**
6355  *      t4_mdio_rd - read a PHY register through MDIO
6356  *      @adap: the adapter
6357  *      @mbox: mailbox to use for the FW command
6358  *      @phy_addr: the PHY address
6359  *      @mmd: the PHY MMD to access (0 for clause 22 PHYs)
6360  *      @reg: the register to read
6361  *      @valp: where to store the value
6362  *
6363  *      Issues a FW command through the given mailbox to read a PHY register.
6364  */
6365 int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
6366                unsigned int mmd, unsigned int reg, u16 *valp)
6367 {
6368         int ret;
6369         u32 ldst_addrspace;
6370         struct fw_ldst_cmd c;
6371
6372         memset(&c, 0, sizeof(c));
6373         ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MDIO);
6374         c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6375                                         FW_CMD_REQUEST_F | FW_CMD_READ_F |
6376                                         ldst_addrspace);
6377         c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6378         c.u.mdio.paddr_mmd = cpu_to_be16(FW_LDST_CMD_PADDR_V(phy_addr) |
6379                                          FW_LDST_CMD_MMD_V(mmd));
6380         c.u.mdio.raddr = cpu_to_be16(reg);
6381
6382         ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
6383         if (ret == 0)
6384                 *valp = be16_to_cpu(c.u.mdio.rval);
6385         return ret;
6386 }
6387
6388 /**
6389  *      t4_mdio_wr - write a PHY register through MDIO
6390  *      @adap: the adapter
6391  *      @mbox: mailbox to use for the FW command
6392  *      @phy_addr: the PHY address
6393  *      @mmd: the PHY MMD to access (0 for clause 22 PHYs)
6394  *      @reg: the register to write
6395  *      @valp: value to write
6396  *
6397  *      Issues a FW command through the given mailbox to write a PHY register.
6398  */
6399 int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr,
6400                unsigned int mmd, unsigned int reg, u16 val)
6401 {
6402         u32 ldst_addrspace;
6403         struct fw_ldst_cmd c;
6404
6405         memset(&c, 0, sizeof(c));
6406         ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MDIO);
6407         c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6408                                         FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
6409                                         ldst_addrspace);
6410         c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6411         c.u.mdio.paddr_mmd = cpu_to_be16(FW_LDST_CMD_PADDR_V(phy_addr) |
6412                                          FW_LDST_CMD_MMD_V(mmd));
6413         c.u.mdio.raddr = cpu_to_be16(reg);
6414         c.u.mdio.rval = cpu_to_be16(val);
6415
6416         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6417 }
6418
6419 /**
6420  *      t4_sge_decode_idma_state - decode the idma state
6421  *      @adap: the adapter
6422  *      @state: the state idma is stuck in
6423  */
6424 void t4_sge_decode_idma_state(struct adapter *adapter, int state)
6425 {
6426         static const char * const t4_decode[] = {
6427                 "IDMA_IDLE",
6428                 "IDMA_PUSH_MORE_CPL_FIFO",
6429                 "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO",
6430                 "Not used",
6431                 "IDMA_PHYSADDR_SEND_PCIEHDR",
6432                 "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST",
6433                 "IDMA_PHYSADDR_SEND_PAYLOAD",
6434                 "IDMA_SEND_FIFO_TO_IMSG",
6435                 "IDMA_FL_REQ_DATA_FL_PREP",
6436                 "IDMA_FL_REQ_DATA_FL",
6437                 "IDMA_FL_DROP",
6438                 "IDMA_FL_H_REQ_HEADER_FL",
6439                 "IDMA_FL_H_SEND_PCIEHDR",
6440                 "IDMA_FL_H_PUSH_CPL_FIFO",
6441                 "IDMA_FL_H_SEND_CPL",
6442                 "IDMA_FL_H_SEND_IP_HDR_FIRST",
6443                 "IDMA_FL_H_SEND_IP_HDR",
6444                 "IDMA_FL_H_REQ_NEXT_HEADER_FL",
6445                 "IDMA_FL_H_SEND_NEXT_PCIEHDR",
6446                 "IDMA_FL_H_SEND_IP_HDR_PADDING",
6447                 "IDMA_FL_D_SEND_PCIEHDR",
6448                 "IDMA_FL_D_SEND_CPL_AND_IP_HDR",
6449                 "IDMA_FL_D_REQ_NEXT_DATA_FL",
6450                 "IDMA_FL_SEND_PCIEHDR",
6451                 "IDMA_FL_PUSH_CPL_FIFO",
6452                 "IDMA_FL_SEND_CPL",
6453                 "IDMA_FL_SEND_PAYLOAD_FIRST",
6454                 "IDMA_FL_SEND_PAYLOAD",
6455                 "IDMA_FL_REQ_NEXT_DATA_FL",
6456                 "IDMA_FL_SEND_NEXT_PCIEHDR",
6457                 "IDMA_FL_SEND_PADDING",
6458                 "IDMA_FL_SEND_COMPLETION_TO_IMSG",
6459                 "IDMA_FL_SEND_FIFO_TO_IMSG",
6460                 "IDMA_FL_REQ_DATAFL_DONE",
6461                 "IDMA_FL_REQ_HEADERFL_DONE",
6462         };
6463         static const char * const t5_decode[] = {
6464                 "IDMA_IDLE",
6465                 "IDMA_ALMOST_IDLE",
6466                 "IDMA_PUSH_MORE_CPL_FIFO",
6467                 "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO",
6468                 "IDMA_SGEFLRFLUSH_SEND_PCIEHDR",
6469                 "IDMA_PHYSADDR_SEND_PCIEHDR",
6470                 "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST",
6471                 "IDMA_PHYSADDR_SEND_PAYLOAD",
6472                 "IDMA_SEND_FIFO_TO_IMSG",
6473                 "IDMA_FL_REQ_DATA_FL",
6474                 "IDMA_FL_DROP",
6475                 "IDMA_FL_DROP_SEND_INC",
6476                 "IDMA_FL_H_REQ_HEADER_FL",
6477                 "IDMA_FL_H_SEND_PCIEHDR",
6478                 "IDMA_FL_H_PUSH_CPL_FIFO",
6479                 "IDMA_FL_H_SEND_CPL",
6480                 "IDMA_FL_H_SEND_IP_HDR_FIRST",
6481                 "IDMA_FL_H_SEND_IP_HDR",
6482                 "IDMA_FL_H_REQ_NEXT_HEADER_FL",
6483                 "IDMA_FL_H_SEND_NEXT_PCIEHDR",
6484                 "IDMA_FL_H_SEND_IP_HDR_PADDING",
6485                 "IDMA_FL_D_SEND_PCIEHDR",
6486                 "IDMA_FL_D_SEND_CPL_AND_IP_HDR",
6487                 "IDMA_FL_D_REQ_NEXT_DATA_FL",
6488                 "IDMA_FL_SEND_PCIEHDR",
6489                 "IDMA_FL_PUSH_CPL_FIFO",
6490                 "IDMA_FL_SEND_CPL",
6491                 "IDMA_FL_SEND_PAYLOAD_FIRST",
6492                 "IDMA_FL_SEND_PAYLOAD",
6493                 "IDMA_FL_REQ_NEXT_DATA_FL",
6494                 "IDMA_FL_SEND_NEXT_PCIEHDR",
6495                 "IDMA_FL_SEND_PADDING",
6496                 "IDMA_FL_SEND_COMPLETION_TO_IMSG",
6497         };
6498         static const char * const t6_decode[] = {
6499                 "IDMA_IDLE",
6500                 "IDMA_PUSH_MORE_CPL_FIFO",
6501                 "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO",
6502                 "IDMA_SGEFLRFLUSH_SEND_PCIEHDR",
6503                 "IDMA_PHYSADDR_SEND_PCIEHDR",
6504                 "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST",
6505                 "IDMA_PHYSADDR_SEND_PAYLOAD",
6506                 "IDMA_FL_REQ_DATA_FL",
6507                 "IDMA_FL_DROP",
6508                 "IDMA_FL_DROP_SEND_INC",
6509                 "IDMA_FL_H_REQ_HEADER_FL",
6510                 "IDMA_FL_H_SEND_PCIEHDR",
6511                 "IDMA_FL_H_PUSH_CPL_FIFO",
6512                 "IDMA_FL_H_SEND_CPL",
6513                 "IDMA_FL_H_SEND_IP_HDR_FIRST",
6514                 "IDMA_FL_H_SEND_IP_HDR",
6515                 "IDMA_FL_H_REQ_NEXT_HEADER_FL",
6516                 "IDMA_FL_H_SEND_NEXT_PCIEHDR",
6517                 "IDMA_FL_H_SEND_IP_HDR_PADDING",
6518                 "IDMA_FL_D_SEND_PCIEHDR",
6519                 "IDMA_FL_D_SEND_CPL_AND_IP_HDR",
6520                 "IDMA_FL_D_REQ_NEXT_DATA_FL",
6521                 "IDMA_FL_SEND_PCIEHDR",
6522                 "IDMA_FL_PUSH_CPL_FIFO",
6523                 "IDMA_FL_SEND_CPL",
6524                 "IDMA_FL_SEND_PAYLOAD_FIRST",
6525                 "IDMA_FL_SEND_PAYLOAD",
6526                 "IDMA_FL_REQ_NEXT_DATA_FL",
6527                 "IDMA_FL_SEND_NEXT_PCIEHDR",
6528                 "IDMA_FL_SEND_PADDING",
6529                 "IDMA_FL_SEND_COMPLETION_TO_IMSG",
6530         };
6531         static const u32 sge_regs[] = {
6532                 SGE_DEBUG_DATA_LOW_INDEX_2_A,
6533                 SGE_DEBUG_DATA_LOW_INDEX_3_A,
6534                 SGE_DEBUG_DATA_HIGH_INDEX_10_A,
6535         };
6536         const char **sge_idma_decode;
6537         int sge_idma_decode_nstates;
6538         int i;
6539         unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
6540
6541         /* Select the right set of decode strings to dump depending on the
6542          * adapter chip type.
6543          */
6544         switch (chip_version) {
6545         case CHELSIO_T4:
6546                 sge_idma_decode = (const char **)t4_decode;
6547                 sge_idma_decode_nstates = ARRAY_SIZE(t4_decode);
6548                 break;
6549
6550         case CHELSIO_T5:
6551                 sge_idma_decode = (const char **)t5_decode;
6552                 sge_idma_decode_nstates = ARRAY_SIZE(t5_decode);
6553                 break;
6554
6555         case CHELSIO_T6:
6556                 sge_idma_decode = (const char **)t6_decode;
6557                 sge_idma_decode_nstates = ARRAY_SIZE(t6_decode);
6558                 break;
6559
6560         default:
6561                 dev_err(adapter->pdev_dev,
6562                         "Unsupported chip version %d\n", chip_version);
6563                 return;
6564         }
6565
6566         if (is_t4(adapter->params.chip)) {
6567                 sge_idma_decode = (const char **)t4_decode;
6568                 sge_idma_decode_nstates = ARRAY_SIZE(t4_decode);
6569         } else {
6570                 sge_idma_decode = (const char **)t5_decode;
6571                 sge_idma_decode_nstates = ARRAY_SIZE(t5_decode);
6572         }
6573
6574         if (state < sge_idma_decode_nstates)
6575                 CH_WARN(adapter, "idma state %s\n", sge_idma_decode[state]);
6576         else
6577                 CH_WARN(adapter, "idma state %d unknown\n", state);
6578
6579         for (i = 0; i < ARRAY_SIZE(sge_regs); i++)
6580                 CH_WARN(adapter, "SGE register %#x value %#x\n",
6581                         sge_regs[i], t4_read_reg(adapter, sge_regs[i]));
6582 }
6583
6584 /**
6585  *      t4_sge_ctxt_flush - flush the SGE context cache
6586  *      @adap: the adapter
6587  *      @mbox: mailbox to use for the FW command
6588  *      @ctx_type: Egress or Ingress
6589  *
6590  *      Issues a FW command through the given mailbox to flush the
6591  *      SGE context cache.
6592  */
6593 int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox, int ctxt_type)
6594 {
6595         int ret;
6596         u32 ldst_addrspace;
6597         struct fw_ldst_cmd c;
6598
6599         memset(&c, 0, sizeof(c));
6600         ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(ctxt_type == CTXT_EGRESS ?
6601                                                  FW_LDST_ADDRSPC_SGE_EGRC :
6602                                                  FW_LDST_ADDRSPC_SGE_INGC);
6603         c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
6604                                         FW_CMD_REQUEST_F | FW_CMD_READ_F |
6605                                         ldst_addrspace);
6606         c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6607         c.u.idctxt.msg_ctxtflush = cpu_to_be32(FW_LDST_CMD_CTXTFLUSH_F);
6608
6609         ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
6610         return ret;
6611 }
6612
6613 /**
6614  *      t4_fw_hello - establish communication with FW
6615  *      @adap: the adapter
6616  *      @mbox: mailbox to use for the FW command
6617  *      @evt_mbox: mailbox to receive async FW events
6618  *      @master: specifies the caller's willingness to be the device master
6619  *      @state: returns the current device state (if non-NULL)
6620  *
6621  *      Issues a command to establish communication with FW.  Returns either
6622  *      an error (negative integer) or the mailbox of the Master PF.
6623  */
6624 int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox,
6625                 enum dev_master master, enum dev_state *state)
6626 {
6627         int ret;
6628         struct fw_hello_cmd c;
6629         u32 v;
6630         unsigned int master_mbox;
6631         int retries = FW_CMD_HELLO_RETRIES;
6632
6633 retry:
6634         memset(&c, 0, sizeof(c));
6635         INIT_CMD(c, HELLO, WRITE);
6636         c.err_to_clearinit = cpu_to_be32(
6637                 FW_HELLO_CMD_MASTERDIS_V(master == MASTER_CANT) |
6638                 FW_HELLO_CMD_MASTERFORCE_V(master == MASTER_MUST) |
6639                 FW_HELLO_CMD_MBMASTER_V(master == MASTER_MUST ?
6640                                         mbox : FW_HELLO_CMD_MBMASTER_M) |
6641                 FW_HELLO_CMD_MBASYNCNOT_V(evt_mbox) |
6642                 FW_HELLO_CMD_STAGE_V(fw_hello_cmd_stage_os) |
6643                 FW_HELLO_CMD_CLEARINIT_F);
6644
6645         /*
6646          * Issue the HELLO command to the firmware.  If it's not successful
6647          * but indicates that we got a "busy" or "timeout" condition, retry
6648          * the HELLO until we exhaust our retry limit.  If we do exceed our
6649          * retry limit, check to see if the firmware left us any error
6650          * information and report that if so.
6651          */
6652         ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
6653         if (ret < 0) {
6654                 if ((ret == -EBUSY || ret == -ETIMEDOUT) && retries-- > 0)
6655                         goto retry;
6656                 if (t4_read_reg(adap, PCIE_FW_A) & PCIE_FW_ERR_F)
6657                         t4_report_fw_error(adap);
6658                 return ret;
6659         }
6660
6661         v = be32_to_cpu(c.err_to_clearinit);
6662         master_mbox = FW_HELLO_CMD_MBMASTER_G(v);
6663         if (state) {
6664                 if (v & FW_HELLO_CMD_ERR_F)
6665                         *state = DEV_STATE_ERR;
6666                 else if (v & FW_HELLO_CMD_INIT_F)
6667                         *state = DEV_STATE_INIT;
6668                 else
6669                         *state = DEV_STATE_UNINIT;
6670         }
6671
6672         /*
6673          * If we're not the Master PF then we need to wait around for the
6674          * Master PF Driver to finish setting up the adapter.
6675          *
6676          * Note that we also do this wait if we're a non-Master-capable PF and
6677          * there is no current Master PF; a Master PF may show up momentarily
6678          * and we wouldn't want to fail pointlessly.  (This can happen when an
6679          * OS loads lots of different drivers rapidly at the same time).  In
6680          * this case, the Master PF returned by the firmware will be
6681          * PCIE_FW_MASTER_M so the test below will work ...
6682          */
6683         if ((v & (FW_HELLO_CMD_ERR_F|FW_HELLO_CMD_INIT_F)) == 0 &&
6684             master_mbox != mbox) {
6685                 int waiting = FW_CMD_HELLO_TIMEOUT;
6686
6687                 /*
6688                  * Wait for the firmware to either indicate an error or
6689                  * initialized state.  If we see either of these we bail out
6690                  * and report the issue to the caller.  If we exhaust the
6691                  * "hello timeout" and we haven't exhausted our retries, try
6692                  * again.  Otherwise bail with a timeout error.
6693                  */
6694                 for (;;) {
6695                         u32 pcie_fw;
6696
6697                         msleep(50);
6698                         waiting -= 50;
6699
6700                         /*
6701                          * If neither Error nor Initialialized are indicated
6702                          * by the firmware keep waiting till we exaust our
6703                          * timeout ... and then retry if we haven't exhausted
6704                          * our retries ...
6705                          */
6706                         pcie_fw = t4_read_reg(adap, PCIE_FW_A);
6707                         if (!(pcie_fw & (PCIE_FW_ERR_F|PCIE_FW_INIT_F))) {
6708                                 if (waiting <= 0) {
6709                                         if (retries-- > 0)
6710                                                 goto retry;
6711
6712                                         return -ETIMEDOUT;
6713                                 }
6714                                 continue;
6715                         }
6716
6717                         /*
6718                          * We either have an Error or Initialized condition
6719                          * report errors preferentially.
6720                          */
6721                         if (state) {
6722                                 if (pcie_fw & PCIE_FW_ERR_F)
6723                                         *state = DEV_STATE_ERR;
6724                                 else if (pcie_fw & PCIE_FW_INIT_F)
6725                                         *state = DEV_STATE_INIT;
6726                         }
6727
6728                         /*
6729                          * If we arrived before a Master PF was selected and
6730                          * there's not a valid Master PF, grab its identity
6731                          * for our caller.
6732                          */
6733                         if (master_mbox == PCIE_FW_MASTER_M &&
6734                             (pcie_fw & PCIE_FW_MASTER_VLD_F))
6735                                 master_mbox = PCIE_FW_MASTER_G(pcie_fw);
6736                         break;
6737                 }
6738         }
6739
6740         return master_mbox;
6741 }
6742
6743 /**
6744  *      t4_fw_bye - end communication with FW
6745  *      @adap: the adapter
6746  *      @mbox: mailbox to use for the FW command
6747  *
6748  *      Issues a command to terminate communication with FW.
6749  */
6750 int t4_fw_bye(struct adapter *adap, unsigned int mbox)
6751 {
6752         struct fw_bye_cmd c;
6753
6754         memset(&c, 0, sizeof(c));
6755         INIT_CMD(c, BYE, WRITE);
6756         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6757 }
6758
6759 /**
6760  *      t4_init_cmd - ask FW to initialize the device
6761  *      @adap: the adapter
6762  *      @mbox: mailbox to use for the FW command
6763  *
6764  *      Issues a command to FW to partially initialize the device.  This
6765  *      performs initialization that generally doesn't depend on user input.
6766  */
6767 int t4_early_init(struct adapter *adap, unsigned int mbox)
6768 {
6769         struct fw_initialize_cmd c;
6770
6771         memset(&c, 0, sizeof(c));
6772         INIT_CMD(c, INITIALIZE, WRITE);
6773         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6774 }
6775
6776 /**
6777  *      t4_fw_reset - issue a reset to FW
6778  *      @adap: the adapter
6779  *      @mbox: mailbox to use for the FW command
6780  *      @reset: specifies the type of reset to perform
6781  *
6782  *      Issues a reset command of the specified type to FW.
6783  */
6784 int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset)
6785 {
6786         struct fw_reset_cmd c;
6787
6788         memset(&c, 0, sizeof(c));
6789         INIT_CMD(c, RESET, WRITE);
6790         c.val = cpu_to_be32(reset);
6791         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6792 }
6793
6794 /**
6795  *      t4_fw_halt - issue a reset/halt to FW and put uP into RESET
6796  *      @adap: the adapter
6797  *      @mbox: mailbox to use for the FW RESET command (if desired)
6798  *      @force: force uP into RESET even if FW RESET command fails
6799  *
6800  *      Issues a RESET command to firmware (if desired) with a HALT indication
6801  *      and then puts the microprocessor into RESET state.  The RESET command
6802  *      will only be issued if a legitimate mailbox is provided (mbox <=
6803  *      PCIE_FW_MASTER_M).
6804  *
6805  *      This is generally used in order for the host to safely manipulate the
6806  *      adapter without fear of conflicting with whatever the firmware might
6807  *      be doing.  The only way out of this state is to RESTART the firmware
6808  *      ...
6809  */
6810 static int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force)
6811 {
6812         int ret = 0;
6813
6814         /*
6815          * If a legitimate mailbox is provided, issue a RESET command
6816          * with a HALT indication.
6817          */
6818         if (mbox <= PCIE_FW_MASTER_M) {
6819                 struct fw_reset_cmd c;
6820
6821                 memset(&c, 0, sizeof(c));
6822                 INIT_CMD(c, RESET, WRITE);
6823                 c.val = cpu_to_be32(PIORST_F | PIORSTMODE_F);
6824                 c.halt_pkd = cpu_to_be32(FW_RESET_CMD_HALT_F);
6825                 ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
6826         }
6827
6828         /*
6829          * Normally we won't complete the operation if the firmware RESET
6830          * command fails but if our caller insists we'll go ahead and put the
6831          * uP into RESET.  This can be useful if the firmware is hung or even
6832          * missing ...  We'll have to take the risk of putting the uP into
6833          * RESET without the cooperation of firmware in that case.
6834          *
6835          * We also force the firmware's HALT flag to be on in case we bypassed
6836          * the firmware RESET command above or we're dealing with old firmware
6837          * which doesn't have the HALT capability.  This will serve as a flag
6838          * for the incoming firmware to know that it's coming out of a HALT
6839          * rather than a RESET ... if it's new enough to understand that ...
6840          */
6841         if (ret == 0 || force) {
6842                 t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, UPCRST_F);
6843                 t4_set_reg_field(adap, PCIE_FW_A, PCIE_FW_HALT_F,
6844                                  PCIE_FW_HALT_F);
6845         }
6846
6847         /*
6848          * And we always return the result of the firmware RESET command
6849          * even when we force the uP into RESET ...
6850          */
6851         return ret;
6852 }
6853
6854 /**
6855  *      t4_fw_restart - restart the firmware by taking the uP out of RESET
6856  *      @adap: the adapter
6857  *      @reset: if we want to do a RESET to restart things
6858  *
6859  *      Restart firmware previously halted by t4_fw_halt().  On successful
6860  *      return the previous PF Master remains as the new PF Master and there
6861  *      is no need to issue a new HELLO command, etc.
6862  *
6863  *      We do this in two ways:
6864  *
6865  *       1. If we're dealing with newer firmware we'll simply want to take
6866  *          the chip's microprocessor out of RESET.  This will cause the
6867  *          firmware to start up from its start vector.  And then we'll loop
6868  *          until the firmware indicates it's started again (PCIE_FW.HALT
6869  *          reset to 0) or we timeout.
6870  *
6871  *       2. If we're dealing with older firmware then we'll need to RESET
6872  *          the chip since older firmware won't recognize the PCIE_FW.HALT
6873  *          flag and automatically RESET itself on startup.
6874  */
6875 static int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset)
6876 {
6877         if (reset) {
6878                 /*
6879                  * Since we're directing the RESET instead of the firmware
6880                  * doing it automatically, we need to clear the PCIE_FW.HALT
6881                  * bit.
6882                  */
6883                 t4_set_reg_field(adap, PCIE_FW_A, PCIE_FW_HALT_F, 0);
6884
6885                 /*
6886                  * If we've been given a valid mailbox, first try to get the
6887                  * firmware to do the RESET.  If that works, great and we can
6888                  * return success.  Otherwise, if we haven't been given a
6889                  * valid mailbox or the RESET command failed, fall back to
6890                  * hitting the chip with a hammer.
6891                  */
6892                 if (mbox <= PCIE_FW_MASTER_M) {
6893                         t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, 0);
6894                         msleep(100);
6895                         if (t4_fw_reset(adap, mbox,
6896                                         PIORST_F | PIORSTMODE_F) == 0)
6897                                 return 0;
6898                 }
6899
6900                 t4_write_reg(adap, PL_RST_A, PIORST_F | PIORSTMODE_F);
6901                 msleep(2000);
6902         } else {
6903                 int ms;
6904
6905                 t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, 0);
6906                 for (ms = 0; ms < FW_CMD_MAX_TIMEOUT; ) {
6907                         if (!(t4_read_reg(adap, PCIE_FW_A) & PCIE_FW_HALT_F))
6908                                 return 0;
6909                         msleep(100);
6910                         ms += 100;
6911                 }
6912                 return -ETIMEDOUT;
6913         }
6914         return 0;
6915 }
6916
6917 /**
6918  *      t4_fw_upgrade - perform all of the steps necessary to upgrade FW
6919  *      @adap: the adapter
6920  *      @mbox: mailbox to use for the FW RESET command (if desired)
6921  *      @fw_data: the firmware image to write
6922  *      @size: image size
6923  *      @force: force upgrade even if firmware doesn't cooperate
6924  *
6925  *      Perform all of the steps necessary for upgrading an adapter's
6926  *      firmware image.  Normally this requires the cooperation of the
6927  *      existing firmware in order to halt all existing activities
6928  *      but if an invalid mailbox token is passed in we skip that step
6929  *      (though we'll still put the adapter microprocessor into RESET in
6930  *      that case).
6931  *
6932  *      On successful return the new firmware will have been loaded and
6933  *      the adapter will have been fully RESET losing all previous setup
6934  *      state.  On unsuccessful return the adapter may be completely hosed ...
6935  *      positive errno indicates that the adapter is ~probably~ intact, a
6936  *      negative errno indicates that things are looking bad ...
6937  */
6938 int t4_fw_upgrade(struct adapter *adap, unsigned int mbox,
6939                   const u8 *fw_data, unsigned int size, int force)
6940 {
6941         const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data;
6942         int reset, ret;
6943
6944         if (!t4_fw_matches_chip(adap, fw_hdr))
6945                 return -EINVAL;
6946
6947         /* Disable FW_OK flag so that mbox commands with FW_OK flag set
6948          * wont be sent when we are flashing FW.
6949          */
6950         adap->flags &= ~FW_OK;
6951
6952         ret = t4_fw_halt(adap, mbox, force);
6953         if (ret < 0 && !force)
6954                 goto out;
6955
6956         ret = t4_load_fw(adap, fw_data, size);
6957         if (ret < 0)
6958                 goto out;
6959
6960         /*
6961          * If there was a Firmware Configuration File stored in FLASH,
6962          * there's a good chance that it won't be compatible with the new
6963          * Firmware.  In order to prevent difficult to diagnose adapter
6964          * initialization issues, we clear out the Firmware Configuration File
6965          * portion of the FLASH .  The user will need to re-FLASH a new
6966          * Firmware Configuration File which is compatible with the new
6967          * Firmware if that's desired.
6968          */
6969         (void)t4_load_cfg(adap, NULL, 0);
6970
6971         /*
6972          * Older versions of the firmware don't understand the new
6973          * PCIE_FW.HALT flag and so won't know to perform a RESET when they
6974          * restart.  So for newly loaded older firmware we'll have to do the
6975          * RESET for it so it starts up on a clean slate.  We can tell if
6976          * the newly loaded firmware will handle this right by checking
6977          * its header flags to see if it advertises the capability.
6978          */
6979         reset = ((be32_to_cpu(fw_hdr->flags) & FW_HDR_FLAGS_RESET_HALT) == 0);
6980         ret = t4_fw_restart(adap, mbox, reset);
6981
6982         /* Grab potentially new Firmware Device Log parameters so we can see
6983          * how healthy the new Firmware is.  It's okay to contact the new
6984          * Firmware for these parameters even though, as far as it's
6985          * concerned, we've never said "HELLO" to it ...
6986          */
6987         (void)t4_init_devlog_params(adap);
6988 out:
6989         adap->flags |= FW_OK;
6990         return ret;
6991 }
6992
6993 /**
6994  *      t4_fl_pkt_align - return the fl packet alignment
6995  *      @adap: the adapter
6996  *
6997  *      T4 has a single field to specify the packing and padding boundary.
6998  *      T5 onwards has separate fields for this and hence the alignment for
6999  *      next packet offset is maximum of these two.
7000  *
7001  */
7002 int t4_fl_pkt_align(struct adapter *adap)
7003 {
7004         u32 sge_control, sge_control2;
7005         unsigned int ingpadboundary, ingpackboundary, fl_align, ingpad_shift;
7006
7007         sge_control = t4_read_reg(adap, SGE_CONTROL_A);
7008
7009         /* T4 uses a single control field to specify both the PCIe Padding and
7010          * Packing Boundary.  T5 introduced the ability to specify these
7011          * separately.  The actual Ingress Packet Data alignment boundary
7012          * within Packed Buffer Mode is the maximum of these two
7013          * specifications.  (Note that it makes no real practical sense to
7014          * have the Pading Boudary be larger than the Packing Boundary but you
7015          * could set the chip up that way and, in fact, legacy T4 code would
7016          * end doing this because it would initialize the Padding Boundary and
7017          * leave the Packing Boundary initialized to 0 (16 bytes).)
7018          * Padding Boundary values in T6 starts from 8B,
7019          * where as it is 32B for T4 and T5.
7020          */
7021         if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
7022                 ingpad_shift = INGPADBOUNDARY_SHIFT_X;
7023         else
7024                 ingpad_shift = T6_INGPADBOUNDARY_SHIFT_X;
7025
7026         ingpadboundary = 1 << (INGPADBOUNDARY_G(sge_control) + ingpad_shift);
7027
7028         fl_align = ingpadboundary;
7029         if (!is_t4(adap->params.chip)) {
7030                 /* T5 has a weird interpretation of one of the PCIe Packing
7031                  * Boundary values.  No idea why ...
7032                  */
7033                 sge_control2 = t4_read_reg(adap, SGE_CONTROL2_A);
7034                 ingpackboundary = INGPACKBOUNDARY_G(sge_control2);
7035                 if (ingpackboundary == INGPACKBOUNDARY_16B_X)
7036                         ingpackboundary = 16;
7037                 else
7038                         ingpackboundary = 1 << (ingpackboundary +
7039                                                 INGPACKBOUNDARY_SHIFT_X);
7040
7041                 fl_align = max(ingpadboundary, ingpackboundary);
7042         }
7043         return fl_align;
7044 }
7045
7046 /**
7047  *      t4_fixup_host_params - fix up host-dependent parameters
7048  *      @adap: the adapter
7049  *      @page_size: the host's Base Page Size
7050  *      @cache_line_size: the host's Cache Line Size
7051  *
7052  *      Various registers in T4 contain values which are dependent on the
7053  *      host's Base Page and Cache Line Sizes.  This function will fix all of
7054  *      those registers with the appropriate values as passed in ...
7055  */
7056 int t4_fixup_host_params(struct adapter *adap, unsigned int page_size,
7057                          unsigned int cache_line_size)
7058 {
7059         unsigned int page_shift = fls(page_size) - 1;
7060         unsigned int sge_hps = page_shift - 10;
7061         unsigned int stat_len = cache_line_size > 64 ? 128 : 64;
7062         unsigned int fl_align = cache_line_size < 32 ? 32 : cache_line_size;
7063         unsigned int fl_align_log = fls(fl_align) - 1;
7064
7065         t4_write_reg(adap, SGE_HOST_PAGE_SIZE_A,
7066                      HOSTPAGESIZEPF0_V(sge_hps) |
7067                      HOSTPAGESIZEPF1_V(sge_hps) |
7068                      HOSTPAGESIZEPF2_V(sge_hps) |
7069                      HOSTPAGESIZEPF3_V(sge_hps) |
7070                      HOSTPAGESIZEPF4_V(sge_hps) |
7071                      HOSTPAGESIZEPF5_V(sge_hps) |
7072                      HOSTPAGESIZEPF6_V(sge_hps) |
7073                      HOSTPAGESIZEPF7_V(sge_hps));
7074
7075         if (is_t4(adap->params.chip)) {
7076                 t4_set_reg_field(adap, SGE_CONTROL_A,
7077                                  INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
7078                                  EGRSTATUSPAGESIZE_F,
7079                                  INGPADBOUNDARY_V(fl_align_log -
7080                                                   INGPADBOUNDARY_SHIFT_X) |
7081                                  EGRSTATUSPAGESIZE_V(stat_len != 64));
7082         } else {
7083                 unsigned int pack_align;
7084                 unsigned int ingpad, ingpack;
7085                 unsigned int pcie_cap;
7086
7087                 /* T5 introduced the separation of the Free List Padding and
7088                  * Packing Boundaries.  Thus, we can select a smaller Padding
7089                  * Boundary to avoid uselessly chewing up PCIe Link and Memory
7090                  * Bandwidth, and use a Packing Boundary which is large enough
7091                  * to avoid false sharing between CPUs, etc.
7092                  *
7093                  * For the PCI Link, the smaller the Padding Boundary the
7094                  * better.  For the Memory Controller, a smaller Padding
7095                  * Boundary is better until we cross under the Memory Line
7096                  * Size (the minimum unit of transfer to/from Memory).  If we
7097                  * have a Padding Boundary which is smaller than the Memory
7098                  * Line Size, that'll involve a Read-Modify-Write cycle on the
7099                  * Memory Controller which is never good.
7100                  */
7101
7102                 /* We want the Packing Boundary to be based on the Cache Line
7103                  * Size in order to help avoid False Sharing performance
7104                  * issues between CPUs, etc.  We also want the Packing
7105                  * Boundary to incorporate the PCI-E Maximum Payload Size.  We
7106                  * get best performance when the Packing Boundary is a
7107                  * multiple of the Maximum Payload Size.
7108                  */
7109                 pack_align = fl_align;
7110                 pcie_cap = pci_find_capability(adap->pdev, PCI_CAP_ID_EXP);
7111                 if (pcie_cap) {
7112                         unsigned int mps, mps_log;
7113                         u16 devctl;
7114
7115                         /* The PCIe Device Control Maximum Payload Size field
7116                          * [bits 7:5] encodes sizes as powers of 2 starting at
7117                          * 128 bytes.
7118                          */
7119                         pci_read_config_word(adap->pdev,
7120                                              pcie_cap + PCI_EXP_DEVCTL,
7121                                              &devctl);
7122                         mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7;
7123                         mps = 1 << mps_log;
7124                         if (mps > pack_align)
7125                                 pack_align = mps;
7126                 }
7127
7128                 /* N.B. T5/T6 have a crazy special interpretation of the "0"
7129                  * value for the Packing Boundary.  This corresponds to 16
7130                  * bytes instead of the expected 32 bytes.  So if we want 32
7131                  * bytes, the best we can really do is 64 bytes ...
7132                  */
7133                 if (pack_align <= 16) {
7134                         ingpack = INGPACKBOUNDARY_16B_X;
7135                         fl_align = 16;
7136                 } else if (pack_align == 32) {
7137                         ingpack = INGPACKBOUNDARY_64B_X;
7138                         fl_align = 64;
7139                 } else {
7140                         unsigned int pack_align_log = fls(pack_align) - 1;
7141
7142                         ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X;
7143                         fl_align = pack_align;
7144                 }
7145
7146                 /* Use the smallest Ingress Padding which isn't smaller than
7147                  * the Memory Controller Read/Write Size.  We'll take that as
7148                  * being 8 bytes since we don't know of any system with a
7149                  * wider Memory Controller Bus Width.
7150                  */
7151                 if (is_t5(adap->params.chip))
7152                         ingpad = INGPADBOUNDARY_32B_X;
7153                 else
7154                         ingpad = T6_INGPADBOUNDARY_8B_X;
7155
7156                 t4_set_reg_field(adap, SGE_CONTROL_A,
7157                                  INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
7158                                  EGRSTATUSPAGESIZE_F,
7159                                  INGPADBOUNDARY_V(ingpad) |
7160                                  EGRSTATUSPAGESIZE_V(stat_len != 64));
7161                 t4_set_reg_field(adap, SGE_CONTROL2_A,
7162                                  INGPACKBOUNDARY_V(INGPACKBOUNDARY_M),
7163                                  INGPACKBOUNDARY_V(ingpack));
7164         }
7165         /*
7166          * Adjust various SGE Free List Host Buffer Sizes.
7167          *
7168          * This is something of a crock since we're using fixed indices into
7169          * the array which are also known by the sge.c code and the T4
7170          * Firmware Configuration File.  We need to come up with a much better
7171          * approach to managing this array.  For now, the first four entries
7172          * are:
7173          *
7174          *   0: Host Page Size
7175          *   1: 64KB
7176          *   2: Buffer size corresponding to 1500 byte MTU (unpacked mode)
7177          *   3: Buffer size corresponding to 9000 byte MTU (unpacked mode)
7178          *
7179          * For the single-MTU buffers in unpacked mode we need to include
7180          * space for the SGE Control Packet Shift, 14 byte Ethernet header,
7181          * possible 4 byte VLAN tag, all rounded up to the next Ingress Packet
7182          * Padding boundary.  All of these are accommodated in the Factory
7183          * Default Firmware Configuration File but we need to adjust it for
7184          * this host's cache line size.
7185          */
7186         t4_write_reg(adap, SGE_FL_BUFFER_SIZE0_A, page_size);
7187         t4_write_reg(adap, SGE_FL_BUFFER_SIZE2_A,
7188                      (t4_read_reg(adap, SGE_FL_BUFFER_SIZE2_A) + fl_align-1)
7189                      & ~(fl_align-1));
7190         t4_write_reg(adap, SGE_FL_BUFFER_SIZE3_A,
7191                      (t4_read_reg(adap, SGE_FL_BUFFER_SIZE3_A) + fl_align-1)
7192                      & ~(fl_align-1));
7193
7194         t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(page_shift - 12));
7195
7196         return 0;
7197 }
7198
7199 /**
7200  *      t4_fw_initialize - ask FW to initialize the device
7201  *      @adap: the adapter
7202  *      @mbox: mailbox to use for the FW command
7203  *
7204  *      Issues a command to FW to partially initialize the device.  This
7205  *      performs initialization that generally doesn't depend on user input.
7206  */
7207 int t4_fw_initialize(struct adapter *adap, unsigned int mbox)
7208 {
7209         struct fw_initialize_cmd c;
7210
7211         memset(&c, 0, sizeof(c));
7212         INIT_CMD(c, INITIALIZE, WRITE);
7213         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7214 }
7215
7216 /**
7217  *      t4_query_params_rw - query FW or device parameters
7218  *      @adap: the adapter
7219  *      @mbox: mailbox to use for the FW command
7220  *      @pf: the PF
7221  *      @vf: the VF
7222  *      @nparams: the number of parameters
7223  *      @params: the parameter names
7224  *      @val: the parameter values
7225  *      @rw: Write and read flag
7226  *      @sleep_ok: if true, we may sleep awaiting mbox cmd completion
7227  *
7228  *      Reads the value of FW or device parameters.  Up to 7 parameters can be
7229  *      queried at once.
7230  */
7231 int t4_query_params_rw(struct adapter *adap, unsigned int mbox, unsigned int pf,
7232                        unsigned int vf, unsigned int nparams, const u32 *params,
7233                        u32 *val, int rw, bool sleep_ok)
7234 {
7235         int i, ret;
7236         struct fw_params_cmd c;
7237         __be32 *p = &c.param[0].mnem;
7238
7239         if (nparams > 7)
7240                 return -EINVAL;
7241
7242         memset(&c, 0, sizeof(c));
7243         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) |
7244                                   FW_CMD_REQUEST_F | FW_CMD_READ_F |
7245                                   FW_PARAMS_CMD_PFN_V(pf) |
7246                                   FW_PARAMS_CMD_VFN_V(vf));
7247         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
7248
7249         for (i = 0; i < nparams; i++) {
7250                 *p++ = cpu_to_be32(*params++);
7251                 if (rw)
7252                         *p = cpu_to_be32(*(val + i));
7253                 p++;
7254         }
7255
7256         ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
7257         if (ret == 0)
7258                 for (i = 0, p = &c.param[0].val; i < nparams; i++, p += 2)
7259                         *val++ = be32_to_cpu(*p);
7260         return ret;
7261 }
7262
7263 int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
7264                     unsigned int vf, unsigned int nparams, const u32 *params,
7265                     u32 *val)
7266 {
7267         return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0,
7268                                   true);
7269 }
7270
7271 int t4_query_params_ns(struct adapter *adap, unsigned int mbox, unsigned int pf,
7272                        unsigned int vf, unsigned int nparams, const u32 *params,
7273                        u32 *val)
7274 {
7275         return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0,
7276                                   false);
7277 }
7278
7279 /**
7280  *      t4_set_params_timeout - sets FW or device parameters
7281  *      @adap: the adapter
7282  *      @mbox: mailbox to use for the FW command
7283  *      @pf: the PF
7284  *      @vf: the VF
7285  *      @nparams: the number of parameters
7286  *      @params: the parameter names
7287  *      @val: the parameter values
7288  *      @timeout: the timeout time
7289  *
7290  *      Sets the value of FW or device parameters.  Up to 7 parameters can be
7291  *      specified at once.
7292  */
7293 int t4_set_params_timeout(struct adapter *adap, unsigned int mbox,
7294                           unsigned int pf, unsigned int vf,
7295                           unsigned int nparams, const u32 *params,
7296                           const u32 *val, int timeout)
7297 {
7298         struct fw_params_cmd c;
7299         __be32 *p = &c.param[0].mnem;
7300
7301         if (nparams > 7)
7302                 return -EINVAL;
7303
7304         memset(&c, 0, sizeof(c));
7305         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) |
7306                                   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7307                                   FW_PARAMS_CMD_PFN_V(pf) |
7308                                   FW_PARAMS_CMD_VFN_V(vf));
7309         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
7310
7311         while (nparams--) {
7312                 *p++ = cpu_to_be32(*params++);
7313                 *p++ = cpu_to_be32(*val++);
7314         }
7315
7316         return t4_wr_mbox_timeout(adap, mbox, &c, sizeof(c), NULL, timeout);
7317 }
7318
7319 /**
7320  *      t4_set_params - sets FW or device parameters
7321  *      @adap: the adapter
7322  *      @mbox: mailbox to use for the FW command
7323  *      @pf: the PF
7324  *      @vf: the VF
7325  *      @nparams: the number of parameters
7326  *      @params: the parameter names
7327  *      @val: the parameter values
7328  *
7329  *      Sets the value of FW or device parameters.  Up to 7 parameters can be
7330  *      specified at once.
7331  */
7332 int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
7333                   unsigned int vf, unsigned int nparams, const u32 *params,
7334                   const u32 *val)
7335 {
7336         return t4_set_params_timeout(adap, mbox, pf, vf, nparams, params, val,
7337                                      FW_CMD_MAX_TIMEOUT);
7338 }
7339
7340 /**
7341  *      t4_cfg_pfvf - configure PF/VF resource limits
7342  *      @adap: the adapter
7343  *      @mbox: mailbox to use for the FW command
7344  *      @pf: the PF being configured
7345  *      @vf: the VF being configured
7346  *      @txq: the max number of egress queues
7347  *      @txq_eth_ctrl: the max number of egress Ethernet or control queues
7348  *      @rxqi: the max number of interrupt-capable ingress queues
7349  *      @rxq: the max number of interruptless ingress queues
7350  *      @tc: the PCI traffic class
7351  *      @vi: the max number of virtual interfaces
7352  *      @cmask: the channel access rights mask for the PF/VF
7353  *      @pmask: the port access rights mask for the PF/VF
7354  *      @nexact: the maximum number of exact MPS filters
7355  *      @rcaps: read capabilities
7356  *      @wxcaps: write/execute capabilities
7357  *
7358  *      Configures resource limits and capabilities for a physical or virtual
7359  *      function.
7360  */
7361 int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf,
7362                 unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl,
7363                 unsigned int rxqi, unsigned int rxq, unsigned int tc,
7364                 unsigned int vi, unsigned int cmask, unsigned int pmask,
7365                 unsigned int nexact, unsigned int rcaps, unsigned int wxcaps)
7366 {
7367         struct fw_pfvf_cmd c;
7368
7369         memset(&c, 0, sizeof(c));
7370         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | FW_CMD_REQUEST_F |
7371                                   FW_CMD_WRITE_F | FW_PFVF_CMD_PFN_V(pf) |
7372                                   FW_PFVF_CMD_VFN_V(vf));
7373         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
7374         c.niqflint_niq = cpu_to_be32(FW_PFVF_CMD_NIQFLINT_V(rxqi) |
7375                                      FW_PFVF_CMD_NIQ_V(rxq));
7376         c.type_to_neq = cpu_to_be32(FW_PFVF_CMD_CMASK_V(cmask) |
7377                                     FW_PFVF_CMD_PMASK_V(pmask) |
7378                                     FW_PFVF_CMD_NEQ_V(txq));
7379         c.tc_to_nexactf = cpu_to_be32(FW_PFVF_CMD_TC_V(tc) |
7380                                       FW_PFVF_CMD_NVI_V(vi) |
7381                                       FW_PFVF_CMD_NEXACTF_V(nexact));
7382         c.r_caps_to_nethctrl = cpu_to_be32(FW_PFVF_CMD_R_CAPS_V(rcaps) |
7383                                         FW_PFVF_CMD_WX_CAPS_V(wxcaps) |
7384                                         FW_PFVF_CMD_NETHCTRL_V(txq_eth_ctrl));
7385         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7386 }
7387
7388 /**
7389  *      t4_alloc_vi - allocate a virtual interface
7390  *      @adap: the adapter
7391  *      @mbox: mailbox to use for the FW command
7392  *      @port: physical port associated with the VI
7393  *      @pf: the PF owning the VI
7394  *      @vf: the VF owning the VI
7395  *      @nmac: number of MAC addresses needed (1 to 5)
7396  *      @mac: the MAC addresses of the VI
7397  *      @rss_size: size of RSS table slice associated with this VI
7398  *
7399  *      Allocates a virtual interface for the given physical port.  If @mac is
7400  *      not %NULL it contains the MAC addresses of the VI as assigned by FW.
7401  *      @mac should be large enough to hold @nmac Ethernet addresses, they are
7402  *      stored consecutively so the space needed is @nmac * 6 bytes.
7403  *      Returns a negative error number or the non-negative VI id.
7404  */
7405 int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port,
7406                 unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac,
7407                 unsigned int *rss_size)
7408 {
7409         int ret;
7410         struct fw_vi_cmd c;
7411
7412         memset(&c, 0, sizeof(c));
7413         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | FW_CMD_REQUEST_F |
7414                                   FW_CMD_WRITE_F | FW_CMD_EXEC_F |
7415                                   FW_VI_CMD_PFN_V(pf) | FW_VI_CMD_VFN_V(vf));
7416         c.alloc_to_len16 = cpu_to_be32(FW_VI_CMD_ALLOC_F | FW_LEN16(c));
7417         c.portid_pkd = FW_VI_CMD_PORTID_V(port);
7418         c.nmac = nmac - 1;
7419
7420         ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
7421         if (ret)
7422                 return ret;
7423
7424         if (mac) {
7425                 memcpy(mac, c.mac, sizeof(c.mac));
7426                 switch (nmac) {
7427                 case 5:
7428                         memcpy(mac + 24, c.nmac3, sizeof(c.nmac3));
7429                 case 4:
7430                         memcpy(mac + 18, c.nmac2, sizeof(c.nmac2));
7431                 case 3:
7432                         memcpy(mac + 12, c.nmac1, sizeof(c.nmac1));
7433                 case 2:
7434                         memcpy(mac + 6,  c.nmac0, sizeof(c.nmac0));
7435                 }
7436         }
7437         if (rss_size)
7438                 *rss_size = FW_VI_CMD_RSSSIZE_G(be16_to_cpu(c.rsssize_pkd));
7439         return FW_VI_CMD_VIID_G(be16_to_cpu(c.type_viid));
7440 }
7441
7442 /**
7443  *      t4_free_vi - free a virtual interface
7444  *      @adap: the adapter
7445  *      @mbox: mailbox to use for the FW command
7446  *      @pf: the PF owning the VI
7447  *      @vf: the VF owning the VI
7448  *      @viid: virtual interface identifiler
7449  *
7450  *      Free a previously allocated virtual interface.
7451  */
7452 int t4_free_vi(struct adapter *adap, unsigned int mbox, unsigned int pf,
7453                unsigned int vf, unsigned int viid)
7454 {
7455         struct fw_vi_cmd c;
7456
7457         memset(&c, 0, sizeof(c));
7458         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) |
7459                                   FW_CMD_REQUEST_F |
7460                                   FW_CMD_EXEC_F |
7461                                   FW_VI_CMD_PFN_V(pf) |
7462                                   FW_VI_CMD_VFN_V(vf));
7463         c.alloc_to_len16 = cpu_to_be32(FW_VI_CMD_FREE_F | FW_LEN16(c));
7464         c.type_viid = cpu_to_be16(FW_VI_CMD_VIID_V(viid));
7465
7466         return t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
7467 }
7468
7469 /**
7470  *      t4_set_rxmode - set Rx properties of a virtual interface
7471  *      @adap: the adapter
7472  *      @mbox: mailbox to use for the FW command
7473  *      @viid: the VI id
7474  *      @mtu: the new MTU or -1
7475  *      @promisc: 1 to enable promiscuous mode, 0 to disable it, -1 no change
7476  *      @all_multi: 1 to enable all-multi mode, 0 to disable it, -1 no change
7477  *      @bcast: 1 to enable broadcast Rx, 0 to disable it, -1 no change
7478  *      @vlanex: 1 to enable HW VLAN extraction, 0 to disable it, -1 no change
7479  *      @sleep_ok: if true we may sleep while awaiting command completion
7480  *
7481  *      Sets Rx properties of a virtual interface.
7482  */
7483 int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
7484                   int mtu, int promisc, int all_multi, int bcast, int vlanex,
7485                   bool sleep_ok)
7486 {
7487         struct fw_vi_rxmode_cmd c;
7488
7489         /* convert to FW values */
7490         if (mtu < 0)
7491                 mtu = FW_RXMODE_MTU_NO_CHG;
7492         if (promisc < 0)
7493                 promisc = FW_VI_RXMODE_CMD_PROMISCEN_M;
7494         if (all_multi < 0)
7495                 all_multi = FW_VI_RXMODE_CMD_ALLMULTIEN_M;
7496         if (bcast < 0)
7497                 bcast = FW_VI_RXMODE_CMD_BROADCASTEN_M;
7498         if (vlanex < 0)
7499                 vlanex = FW_VI_RXMODE_CMD_VLANEXEN_M;
7500
7501         memset(&c, 0, sizeof(c));
7502         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_RXMODE_CMD) |
7503                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7504                                    FW_VI_RXMODE_CMD_VIID_V(viid));
7505         c.retval_len16 = cpu_to_be32(FW_LEN16(c));
7506         c.mtu_to_vlanexen =
7507                 cpu_to_be32(FW_VI_RXMODE_CMD_MTU_V(mtu) |
7508                             FW_VI_RXMODE_CMD_PROMISCEN_V(promisc) |
7509                             FW_VI_RXMODE_CMD_ALLMULTIEN_V(all_multi) |
7510                             FW_VI_RXMODE_CMD_BROADCASTEN_V(bcast) |
7511                             FW_VI_RXMODE_CMD_VLANEXEN_V(vlanex));
7512         return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
7513 }
7514
7515 /**
7516  *      t4_free_raw_mac_filt - Frees a raw mac entry in mps tcam
7517  *      @adap: the adapter
7518  *      @viid: the VI id
7519  *      @addr: the MAC address
7520  *      @mask: the mask
7521  *      @idx: index of the entry in mps tcam
7522  *      @lookup_type: MAC address for inner (1) or outer (0) header
7523  *      @port_id: the port index
7524  *      @sleep_ok: call is allowed to sleep
7525  *
7526  *      Removes the mac entry at the specified index using raw mac interface.
7527  *
7528  *      Returns a negative error number on failure.
7529  */
7530 int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
7531                          const u8 *addr, const u8 *mask, unsigned int idx,
7532                          u8 lookup_type, u8 port_id, bool sleep_ok)
7533 {
7534         struct fw_vi_mac_cmd c;
7535         struct fw_vi_mac_raw *p = &c.u.raw;
7536         u32 val;
7537
7538         memset(&c, 0, sizeof(c));
7539         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7540                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7541                                    FW_CMD_EXEC_V(0) |
7542                                    FW_VI_MAC_CMD_VIID_V(viid));
7543         val = FW_CMD_LEN16_V(1) |
7544               FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_RAW);
7545         c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) |
7546                                           FW_CMD_LEN16_V(val));
7547
7548         p->raw_idx_pkd = cpu_to_be32(FW_VI_MAC_CMD_RAW_IDX_V(idx) |
7549                                      FW_VI_MAC_ID_BASED_FREE);
7550
7551         /* Lookup Type. Outer header: 0, Inner header: 1 */
7552         p->data0_pkd = cpu_to_be32(DATALKPTYPE_V(lookup_type) |
7553                                    DATAPORTNUM_V(port_id));
7554         /* Lookup mask and port mask */
7555         p->data0m_pkd = cpu_to_be64(DATALKPTYPE_V(DATALKPTYPE_M) |
7556                                     DATAPORTNUM_V(DATAPORTNUM_M));
7557
7558         /* Copy the address and the mask */
7559         memcpy((u8 *)&p->data1[0] + 2, addr, ETH_ALEN);
7560         memcpy((u8 *)&p->data1m[0] + 2, mask, ETH_ALEN);
7561
7562         return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
7563 }
7564
7565 /**
7566  *      t4_alloc_raw_mac_filt - Adds a mac entry in mps tcam
7567  *      @adap: the adapter
7568  *      @viid: the VI id
7569  *      @mac: the MAC address
7570  *      @mask: the mask
7571  *      @idx: index at which to add this entry
7572  *      @port_id: the port index
7573  *      @lookup_type: MAC address for inner (1) or outer (0) header
7574  *      @sleep_ok: call is allowed to sleep
7575  *
7576  *      Adds the mac entry at the specified index using raw mac interface.
7577  *
7578  *      Returns a negative error number or the allocated index for this mac.
7579  */
7580 int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
7581                           const u8 *addr, const u8 *mask, unsigned int idx,
7582                           u8 lookup_type, u8 port_id, bool sleep_ok)
7583 {
7584         int ret = 0;
7585         struct fw_vi_mac_cmd c;
7586         struct fw_vi_mac_raw *p = &c.u.raw;
7587         u32 val;
7588
7589         memset(&c, 0, sizeof(c));
7590         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7591                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7592                                    FW_VI_MAC_CMD_VIID_V(viid));
7593         val = FW_CMD_LEN16_V(1) |
7594               FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_RAW);
7595         c.freemacs_to_len16 = cpu_to_be32(val);
7596
7597         /* Specify that this is an inner mac address */
7598         p->raw_idx_pkd = cpu_to_be32(FW_VI_MAC_CMD_RAW_IDX_V(idx));
7599
7600         /* Lookup Type. Outer header: 0, Inner header: 1 */
7601         p->data0_pkd = cpu_to_be32(DATALKPTYPE_V(lookup_type) |
7602                                    DATAPORTNUM_V(port_id));
7603         /* Lookup mask and port mask */
7604         p->data0m_pkd = cpu_to_be64(DATALKPTYPE_V(DATALKPTYPE_M) |
7605                                     DATAPORTNUM_V(DATAPORTNUM_M));
7606
7607         /* Copy the address and the mask */
7608         memcpy((u8 *)&p->data1[0] + 2, addr, ETH_ALEN);
7609         memcpy((u8 *)&p->data1m[0] + 2, mask, ETH_ALEN);
7610
7611         ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
7612         if (ret == 0) {
7613                 ret = FW_VI_MAC_CMD_RAW_IDX_G(be32_to_cpu(p->raw_idx_pkd));
7614                 if (ret != idx)
7615                         ret = -ENOMEM;
7616         }
7617
7618         return ret;
7619 }
7620
7621 /**
7622  *      t4_alloc_mac_filt - allocates exact-match filters for MAC addresses
7623  *      @adap: the adapter
7624  *      @mbox: mailbox to use for the FW command
7625  *      @viid: the VI id
7626  *      @free: if true any existing filters for this VI id are first removed
7627  *      @naddr: the number of MAC addresses to allocate filters for (up to 7)
7628  *      @addr: the MAC address(es)
7629  *      @idx: where to store the index of each allocated filter
7630  *      @hash: pointer to hash address filter bitmap
7631  *      @sleep_ok: call is allowed to sleep
7632  *
7633  *      Allocates an exact-match filter for each of the supplied addresses and
7634  *      sets it to the corresponding address.  If @idx is not %NULL it should
7635  *      have at least @naddr entries, each of which will be set to the index of
7636  *      the filter allocated for the corresponding MAC address.  If a filter
7637  *      could not be allocated for an address its index is set to 0xffff.
7638  *      If @hash is not %NULL addresses that fail to allocate an exact filter
7639  *      are hashed and update the hash filter bitmap pointed at by @hash.
7640  *
7641  *      Returns a negative error number or the number of filters allocated.
7642  */
7643 int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox,
7644                       unsigned int viid, bool free, unsigned int naddr,
7645                       const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok)
7646 {
7647         int offset, ret = 0;
7648         struct fw_vi_mac_cmd c;
7649         unsigned int nfilters = 0;
7650         unsigned int max_naddr = adap->params.arch.mps_tcam_size;
7651         unsigned int rem = naddr;
7652
7653         if (naddr > max_naddr)
7654                 return -EINVAL;
7655
7656         for (offset = 0; offset < naddr ; /**/) {
7657                 unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact) ?
7658                                          rem : ARRAY_SIZE(c.u.exact));
7659                 size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd,
7660                                                      u.exact[fw_naddr]), 16);
7661                 struct fw_vi_mac_exact *p;
7662                 int i;
7663
7664                 memset(&c, 0, sizeof(c));
7665                 c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7666                                            FW_CMD_REQUEST_F |
7667                                            FW_CMD_WRITE_F |
7668                                            FW_CMD_EXEC_V(free) |
7669                                            FW_VI_MAC_CMD_VIID_V(viid));
7670                 c.freemacs_to_len16 =
7671                         cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(free) |
7672                                     FW_CMD_LEN16_V(len16));
7673
7674                 for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
7675                         p->valid_to_idx =
7676                                 cpu_to_be16(FW_VI_MAC_CMD_VALID_F |
7677                                             FW_VI_MAC_CMD_IDX_V(
7678                                                     FW_VI_MAC_ADD_MAC));
7679                         memcpy(p->macaddr, addr[offset + i],
7680                                sizeof(p->macaddr));
7681                 }
7682
7683                 /* It's okay if we run out of space in our MAC address arena.
7684                  * Some of the addresses we submit may get stored so we need
7685                  * to run through the reply to see what the results were ...
7686                  */
7687                 ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
7688                 if (ret && ret != -FW_ENOMEM)
7689                         break;
7690
7691                 for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
7692                         u16 index = FW_VI_MAC_CMD_IDX_G(
7693                                         be16_to_cpu(p->valid_to_idx));
7694
7695                         if (idx)
7696                                 idx[offset + i] = (index >= max_naddr ?
7697                                                    0xffff : index);
7698                         if (index < max_naddr)
7699                                 nfilters++;
7700                         else if (hash)
7701                                 *hash |= (1ULL <<
7702                                           hash_mac_addr(addr[offset + i]));
7703                 }
7704
7705                 free = false;
7706                 offset += fw_naddr;
7707                 rem -= fw_naddr;
7708         }
7709
7710         if (ret == 0 || ret == -FW_ENOMEM)
7711                 ret = nfilters;
7712         return ret;
7713 }
7714
7715 /**
7716  *      t4_free_mac_filt - frees exact-match filters of given MAC addresses
7717  *      @adap: the adapter
7718  *      @mbox: mailbox to use for the FW command
7719  *      @viid: the VI id
7720  *      @naddr: the number of MAC addresses to allocate filters for (up to 7)
7721  *      @addr: the MAC address(es)
7722  *      @sleep_ok: call is allowed to sleep
7723  *
7724  *      Frees the exact-match filter for each of the supplied addresses
7725  *
7726  *      Returns a negative error number or the number of filters freed.
7727  */
7728 int t4_free_mac_filt(struct adapter *adap, unsigned int mbox,
7729                      unsigned int viid, unsigned int naddr,
7730                      const u8 **addr, bool sleep_ok)
7731 {
7732         int offset, ret = 0;
7733         struct fw_vi_mac_cmd c;
7734         unsigned int nfilters = 0;
7735         unsigned int max_naddr = is_t4(adap->params.chip) ?
7736                                        NUM_MPS_CLS_SRAM_L_INSTANCES :
7737                                        NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
7738         unsigned int rem = naddr;
7739
7740         if (naddr > max_naddr)
7741                 return -EINVAL;
7742
7743         for (offset = 0; offset < (int)naddr ; /**/) {
7744                 unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact)
7745                                          ? rem
7746                                          : ARRAY_SIZE(c.u.exact));
7747                 size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd,
7748                                                      u.exact[fw_naddr]), 16);
7749                 struct fw_vi_mac_exact *p;
7750                 int i;
7751
7752                 memset(&c, 0, sizeof(c));
7753                 c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7754                                      FW_CMD_REQUEST_F |
7755                                      FW_CMD_WRITE_F |
7756                                      FW_CMD_EXEC_V(0) |
7757                                      FW_VI_MAC_CMD_VIID_V(viid));
7758                 c.freemacs_to_len16 =
7759                                 cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) |
7760                                             FW_CMD_LEN16_V(len16));
7761
7762                 for (i = 0, p = c.u.exact; i < (int)fw_naddr; i++, p++) {
7763                         p->valid_to_idx = cpu_to_be16(
7764                                 FW_VI_MAC_CMD_VALID_F |
7765                                 FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_MAC_BASED_FREE));
7766                         memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr));
7767                 }
7768
7769                 ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok);
7770                 if (ret)
7771                         break;
7772
7773                 for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) {
7774                         u16 index = FW_VI_MAC_CMD_IDX_G(
7775                                                 be16_to_cpu(p->valid_to_idx));
7776
7777                         if (index < max_naddr)
7778                                 nfilters++;
7779                 }
7780
7781                 offset += fw_naddr;
7782                 rem -= fw_naddr;
7783         }
7784
7785         if (ret == 0)
7786                 ret = nfilters;
7787         return ret;
7788 }
7789
7790 /**
7791  *      t4_change_mac - modifies the exact-match filter for a MAC address
7792  *      @adap: the adapter
7793  *      @mbox: mailbox to use for the FW command
7794  *      @viid: the VI id
7795  *      @idx: index of existing filter for old value of MAC address, or -1
7796  *      @addr: the new MAC address value
7797  *      @persist: whether a new MAC allocation should be persistent
7798  *      @add_smt: if true also add the address to the HW SMT
7799  *
7800  *      Modifies an exact-match filter and sets it to the new MAC address.
7801  *      Note that in general it is not possible to modify the value of a given
7802  *      filter so the generic way to modify an address filter is to free the one
7803  *      being used by the old address value and allocate a new filter for the
7804  *      new address value.  @idx can be -1 if the address is a new addition.
7805  *
7806  *      Returns a negative error number or the index of the filter with the new
7807  *      MAC value.
7808  */
7809 int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid,
7810                   int idx, const u8 *addr, bool persist, bool add_smt)
7811 {
7812         int ret, mode;
7813         struct fw_vi_mac_cmd c;
7814         struct fw_vi_mac_exact *p = c.u.exact;
7815         unsigned int max_mac_addr = adap->params.arch.mps_tcam_size;
7816
7817         if (idx < 0)                             /* new allocation */
7818                 idx = persist ? FW_VI_MAC_ADD_PERSIST_MAC : FW_VI_MAC_ADD_MAC;
7819         mode = add_smt ? FW_VI_MAC_SMT_AND_MPSTCAM : FW_VI_MAC_MPS_TCAM_ENTRY;
7820
7821         memset(&c, 0, sizeof(c));
7822         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7823                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7824                                    FW_VI_MAC_CMD_VIID_V(viid));
7825         c.freemacs_to_len16 = cpu_to_be32(FW_CMD_LEN16_V(1));
7826         p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F |
7827                                       FW_VI_MAC_CMD_SMAC_RESULT_V(mode) |
7828                                       FW_VI_MAC_CMD_IDX_V(idx));
7829         memcpy(p->macaddr, addr, sizeof(p->macaddr));
7830
7831         ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
7832         if (ret == 0) {
7833                 ret = FW_VI_MAC_CMD_IDX_G(be16_to_cpu(p->valid_to_idx));
7834                 if (ret >= max_mac_addr)
7835                         ret = -ENOMEM;
7836         }
7837         return ret;
7838 }
7839
7840 /**
7841  *      t4_set_addr_hash - program the MAC inexact-match hash filter
7842  *      @adap: the adapter
7843  *      @mbox: mailbox to use for the FW command
7844  *      @viid: the VI id
7845  *      @ucast: whether the hash filter should also match unicast addresses
7846  *      @vec: the value to be written to the hash filter
7847  *      @sleep_ok: call is allowed to sleep
7848  *
7849  *      Sets the 64-bit inexact-match hash filter for a virtual interface.
7850  */
7851 int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid,
7852                      bool ucast, u64 vec, bool sleep_ok)
7853 {
7854         struct fw_vi_mac_cmd c;
7855
7856         memset(&c, 0, sizeof(c));
7857         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
7858                                    FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
7859                                    FW_VI_ENABLE_CMD_VIID_V(viid));
7860         c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_HASHVECEN_F |
7861                                           FW_VI_MAC_CMD_HASHUNIEN_V(ucast) |
7862                                           FW_CMD_LEN16_V(1));
7863         c.u.hash.hashvec = cpu_to_be64(vec);
7864         return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
7865 }
7866
7867 /**
7868  *      t4_enable_vi_params - enable/disable a virtual interface
7869  *      @adap: the adapter
7870  *      @mbox: mailbox to use for the FW command
7871  *      @viid: the VI id
7872  *      @rx_en: 1=enable Rx, 0=disable Rx
7873  *      @tx_en: 1=enable Tx, 0=disable Tx
7874  *      @dcb_en: 1=enable delivery of Data Center Bridging messages.
7875  *
7876  *      Enables/disables a virtual interface.  Note that setting DCB Enable
7877  *      only makes sense when enabling a Virtual Interface ...
7878  */
7879 int t4_enable_vi_params(struct adapter *adap, unsigned int mbox,
7880                         unsigned int viid, bool rx_en, bool tx_en, bool dcb_en)
7881 {
7882         struct fw_vi_enable_cmd c;
7883
7884         memset(&c, 0, sizeof(c));
7885         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) |
7886                                    FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7887                                    FW_VI_ENABLE_CMD_VIID_V(viid));
7888         c.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_IEN_V(rx_en) |
7889                                      FW_VI_ENABLE_CMD_EEN_V(tx_en) |
7890                                      FW_VI_ENABLE_CMD_DCB_INFO_V(dcb_en) |
7891                                      FW_LEN16(c));
7892         return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL);
7893 }
7894
7895 /**
7896  *      t4_enable_vi - enable/disable a virtual interface
7897  *      @adap: the adapter
7898  *      @mbox: mailbox to use for the FW command
7899  *      @viid: the VI id
7900  *      @rx_en: 1=enable Rx, 0=disable Rx
7901  *      @tx_en: 1=enable Tx, 0=disable Tx
7902  *
7903  *      Enables/disables a virtual interface.
7904  */
7905 int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
7906                  bool rx_en, bool tx_en)
7907 {
7908         return t4_enable_vi_params(adap, mbox, viid, rx_en, tx_en, 0);
7909 }
7910
7911 /**
7912  *      t4_identify_port - identify a VI's port by blinking its LED
7913  *      @adap: the adapter
7914  *      @mbox: mailbox to use for the FW command
7915  *      @viid: the VI id
7916  *      @nblinks: how many times to blink LED at 2.5 Hz
7917  *
7918  *      Identifies a VI's port by blinking its LED.
7919  */
7920 int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid,
7921                      unsigned int nblinks)
7922 {
7923         struct fw_vi_enable_cmd c;
7924
7925         memset(&c, 0, sizeof(c));
7926         c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) |
7927                                    FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
7928                                    FW_VI_ENABLE_CMD_VIID_V(viid));
7929         c.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_LED_F | FW_LEN16(c));
7930         c.blinkdur = cpu_to_be16(nblinks);
7931         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7932 }
7933
7934 /**
7935  *      t4_iq_stop - stop an ingress queue and its FLs
7936  *      @adap: the adapter
7937  *      @mbox: mailbox to use for the FW command
7938  *      @pf: the PF owning the queues
7939  *      @vf: the VF owning the queues
7940  *      @iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.)
7941  *      @iqid: ingress queue id
7942  *      @fl0id: FL0 queue id or 0xffff if no attached FL0
7943  *      @fl1id: FL1 queue id or 0xffff if no attached FL1
7944  *
7945  *      Stops an ingress queue and its associated FLs, if any.  This causes
7946  *      any current or future data/messages destined for these queues to be
7947  *      tossed.
7948  */
7949 int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf,
7950                unsigned int vf, unsigned int iqtype, unsigned int iqid,
7951                unsigned int fl0id, unsigned int fl1id)
7952 {
7953         struct fw_iq_cmd c;
7954
7955         memset(&c, 0, sizeof(c));
7956         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
7957                                   FW_CMD_EXEC_F | FW_IQ_CMD_PFN_V(pf) |
7958                                   FW_IQ_CMD_VFN_V(vf));
7959         c.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_IQSTOP_F | FW_LEN16(c));
7960         c.type_to_iqandstindex = cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype));
7961         c.iqid = cpu_to_be16(iqid);
7962         c.fl0id = cpu_to_be16(fl0id);
7963         c.fl1id = cpu_to_be16(fl1id);
7964         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7965 }
7966
7967 /**
7968  *      t4_iq_free - free an ingress queue and its FLs
7969  *      @adap: the adapter
7970  *      @mbox: mailbox to use for the FW command
7971  *      @pf: the PF owning the queues
7972  *      @vf: the VF owning the queues
7973  *      @iqtype: the ingress queue type
7974  *      @iqid: ingress queue id
7975  *      @fl0id: FL0 queue id or 0xffff if no attached FL0
7976  *      @fl1id: FL1 queue id or 0xffff if no attached FL1
7977  *
7978  *      Frees an ingress queue and its associated FLs, if any.
7979  */
7980 int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
7981                unsigned int vf, unsigned int iqtype, unsigned int iqid,
7982                unsigned int fl0id, unsigned int fl1id)
7983 {
7984         struct fw_iq_cmd c;
7985
7986         memset(&c, 0, sizeof(c));
7987         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
7988                                   FW_CMD_EXEC_F | FW_IQ_CMD_PFN_V(pf) |
7989                                   FW_IQ_CMD_VFN_V(vf));
7990         c.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_FREE_F | FW_LEN16(c));
7991         c.type_to_iqandstindex = cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype));
7992         c.iqid = cpu_to_be16(iqid);
7993         c.fl0id = cpu_to_be16(fl0id);
7994         c.fl1id = cpu_to_be16(fl1id);
7995         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
7996 }
7997
7998 /**
7999  *      t4_eth_eq_free - free an Ethernet egress queue
8000  *      @adap: the adapter
8001  *      @mbox: mailbox to use for the FW command
8002  *      @pf: the PF owning the queue
8003  *      @vf: the VF owning the queue
8004  *      @eqid: egress queue id
8005  *
8006  *      Frees an Ethernet egress queue.
8007  */
8008 int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
8009                    unsigned int vf, unsigned int eqid)
8010 {
8011         struct fw_eq_eth_cmd c;
8012
8013         memset(&c, 0, sizeof(c));
8014         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) |
8015                                   FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
8016                                   FW_EQ_ETH_CMD_PFN_V(pf) |
8017                                   FW_EQ_ETH_CMD_VFN_V(vf));
8018         c.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_FREE_F | FW_LEN16(c));
8019         c.eqid_pkd = cpu_to_be32(FW_EQ_ETH_CMD_EQID_V(eqid));
8020         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
8021 }
8022
8023 /**
8024  *      t4_ctrl_eq_free - free a control egress queue
8025  *      @adap: the adapter
8026  *      @mbox: mailbox to use for the FW command
8027  *      @pf: the PF owning the queue
8028  *      @vf: the VF owning the queue
8029  *      @eqid: egress queue id
8030  *
8031  *      Frees a control egress queue.
8032  */
8033 int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
8034                     unsigned int vf, unsigned int eqid)
8035 {
8036         struct fw_eq_ctrl_cmd c;
8037
8038         memset(&c, 0, sizeof(c));
8039         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_CTRL_CMD) |
8040                                   FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
8041                                   FW_EQ_CTRL_CMD_PFN_V(pf) |
8042                                   FW_EQ_CTRL_CMD_VFN_V(vf));
8043         c.alloc_to_len16 = cpu_to_be32(FW_EQ_CTRL_CMD_FREE_F | FW_LEN16(c));
8044         c.cmpliqid_eqid = cpu_to_be32(FW_EQ_CTRL_CMD_EQID_V(eqid));
8045         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
8046 }
8047
8048 /**
8049  *      t4_ofld_eq_free - free an offload egress queue
8050  *      @adap: the adapter
8051  *      @mbox: mailbox to use for the FW command
8052  *      @pf: the PF owning the queue
8053  *      @vf: the VF owning the queue
8054  *      @eqid: egress queue id
8055  *
8056  *      Frees a control egress queue.
8057  */
8058 int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
8059                     unsigned int vf, unsigned int eqid)
8060 {
8061         struct fw_eq_ofld_cmd c;
8062
8063         memset(&c, 0, sizeof(c));
8064         c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_OFLD_CMD) |
8065                                   FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
8066                                   FW_EQ_OFLD_CMD_PFN_V(pf) |
8067                                   FW_EQ_OFLD_CMD_VFN_V(vf));
8068         c.alloc_to_len16 = cpu_to_be32(FW_EQ_OFLD_CMD_FREE_F | FW_LEN16(c));
8069         c.eqid_pkd = cpu_to_be32(FW_EQ_OFLD_CMD_EQID_V(eqid));
8070         return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
8071 }
8072
8073 /**
8074  *      t4_link_down_rc_str - return a string for a Link Down Reason Code
8075  *      @adap: the adapter
8076  *      @link_down_rc: Link Down Reason Code
8077  *
8078  *      Returns a string representation of the Link Down Reason Code.
8079  */
8080 static const char *t4_link_down_rc_str(unsigned char link_down_rc)
8081 {
8082         static const char * const reason[] = {
8083                 "Link Down",
8084                 "Remote Fault",
8085                 "Auto-negotiation Failure",
8086                 "Reserved",
8087                 "Insufficient Airflow",
8088                 "Unable To Determine Reason",
8089                 "No RX Signal Detected",
8090                 "Reserved",
8091         };
8092
8093         if (link_down_rc >= ARRAY_SIZE(reason))
8094                 return "Bad Reason Code";
8095
8096         return reason[link_down_rc];
8097 }
8098
8099 /**
8100  * Return the highest speed set in the port capabilities, in Mb/s.
8101  */
8102 static unsigned int fwcap_to_speed(fw_port_cap32_t caps)
8103 {
8104         #define TEST_SPEED_RETURN(__caps_speed, __speed) \
8105                 do { \
8106                         if (caps & FW_PORT_CAP32_SPEED_##__caps_speed) \
8107                                 return __speed; \
8108                 } while (0)
8109
8110         TEST_SPEED_RETURN(400G, 400000);
8111         TEST_SPEED_RETURN(200G, 200000);
8112         TEST_SPEED_RETURN(100G, 100000);
8113         TEST_SPEED_RETURN(50G,   50000);
8114         TEST_SPEED_RETURN(40G,   40000);
8115         TEST_SPEED_RETURN(25G,   25000);
8116         TEST_SPEED_RETURN(10G,   10000);
8117         TEST_SPEED_RETURN(1G,     1000);
8118         TEST_SPEED_RETURN(100M,    100);
8119
8120         #undef TEST_SPEED_RETURN
8121
8122         return 0;
8123 }
8124
8125 /**
8126  *      fwcap_to_fwspeed - return highest speed in Port Capabilities
8127  *      @acaps: advertised Port Capabilities
8128  *
8129  *      Get the highest speed for the port from the advertised Port
8130  *      Capabilities.  It will be either the highest speed from the list of
8131  *      speeds or whatever user has set using ethtool.
8132  */
8133 static fw_port_cap32_t fwcap_to_fwspeed(fw_port_cap32_t acaps)
8134 {
8135         #define TEST_SPEED_RETURN(__caps_speed) \
8136                 do { \
8137                         if (acaps & FW_PORT_CAP32_SPEED_##__caps_speed) \
8138                                 return FW_PORT_CAP32_SPEED_##__caps_speed; \
8139                 } while (0)
8140
8141         TEST_SPEED_RETURN(400G);
8142         TEST_SPEED_RETURN(200G);
8143         TEST_SPEED_RETURN(100G);
8144         TEST_SPEED_RETURN(50G);
8145         TEST_SPEED_RETURN(40G);
8146         TEST_SPEED_RETURN(25G);
8147         TEST_SPEED_RETURN(10G);
8148         TEST_SPEED_RETURN(1G);
8149         TEST_SPEED_RETURN(100M);
8150
8151         #undef TEST_SPEED_RETURN
8152
8153         return 0;
8154 }
8155
8156 /**
8157  *      lstatus_to_fwcap - translate old lstatus to 32-bit Port Capabilities
8158  *      @lstatus: old FW_PORT_ACTION_GET_PORT_INFO lstatus value
8159  *
8160  *      Translates old FW_PORT_ACTION_GET_PORT_INFO lstatus field into new
8161  *      32-bit Port Capabilities value.
8162  */
8163 static fw_port_cap32_t lstatus_to_fwcap(u32 lstatus)
8164 {
8165         fw_port_cap32_t linkattr = 0;
8166
8167         /* Unfortunately the format of the Link Status in the old
8168          * 16-bit Port Information message isn't the same as the
8169          * 16-bit Port Capabilities bitfield used everywhere else ...
8170          */
8171         if (lstatus & FW_PORT_CMD_RXPAUSE_F)
8172                 linkattr |= FW_PORT_CAP32_FC_RX;
8173         if (lstatus & FW_PORT_CMD_TXPAUSE_F)
8174                 linkattr |= FW_PORT_CAP32_FC_TX;
8175         if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
8176                 linkattr |= FW_PORT_CAP32_SPEED_100M;
8177         if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
8178                 linkattr |= FW_PORT_CAP32_SPEED_1G;
8179         if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
8180                 linkattr |= FW_PORT_CAP32_SPEED_10G;
8181         if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
8182                 linkattr |= FW_PORT_CAP32_SPEED_25G;
8183         if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
8184                 linkattr |= FW_PORT_CAP32_SPEED_40G;
8185         if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
8186                 linkattr |= FW_PORT_CAP32_SPEED_100G;
8187
8188         return linkattr;
8189 }
8190
8191 /**
8192  *      t4_handle_get_port_info - process a FW reply message
8193  *      @pi: the port info
8194  *      @rpl: start of the FW message
8195  *
8196  *      Processes a GET_PORT_INFO FW reply message.
8197  */
8198 void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
8199 {
8200         const struct fw_port_cmd *cmd = (const void *)rpl;
8201         int action = FW_PORT_CMD_ACTION_G(be32_to_cpu(cmd->action_to_len16));
8202         struct adapter *adapter = pi->adapter;
8203         struct link_config *lc = &pi->link_cfg;
8204         int link_ok, linkdnrc;
8205         enum fw_port_type port_type;
8206         enum fw_port_module_type mod_type;
8207         unsigned int speed, fc, fec;
8208         fw_port_cap32_t pcaps, acaps, lpacaps, linkattr;
8209
8210         /* Extract the various fields from the Port Information message.
8211          */
8212         switch (action) {
8213         case FW_PORT_ACTION_GET_PORT_INFO: {
8214                 u32 lstatus = be32_to_cpu(cmd->u.info.lstatus_to_modtype);
8215
8216                 link_ok = (lstatus & FW_PORT_CMD_LSTATUS_F) != 0;
8217                 linkdnrc = FW_PORT_CMD_LINKDNRC_G(lstatus);
8218                 port_type = FW_PORT_CMD_PTYPE_G(lstatus);
8219                 mod_type = FW_PORT_CMD_MODTYPE_G(lstatus);
8220                 pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.pcap));
8221                 acaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.acap));
8222                 lpacaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.lpacap));
8223                 linkattr = lstatus_to_fwcap(lstatus);
8224                 break;
8225         }
8226
8227         case FW_PORT_ACTION_GET_PORT_INFO32: {
8228                 u32 lstatus32;
8229
8230                 lstatus32 = be32_to_cpu(cmd->u.info32.lstatus32_to_cbllen32);
8231                 link_ok = (lstatus32 & FW_PORT_CMD_LSTATUS32_F) != 0;
8232                 linkdnrc = FW_PORT_CMD_LINKDNRC32_G(lstatus32);
8233                 port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32);
8234                 mod_type = FW_PORT_CMD_MODTYPE32_G(lstatus32);
8235                 pcaps = be32_to_cpu(cmd->u.info32.pcaps32);
8236                 acaps = be32_to_cpu(cmd->u.info32.acaps32);
8237                 lpacaps = be32_to_cpu(cmd->u.info32.lpacaps32);
8238                 linkattr = be32_to_cpu(cmd->u.info32.linkattr32);
8239                 break;
8240         }
8241
8242         default:
8243                 dev_err(adapter->pdev_dev, "Handle Port Information: Bad Command/Action %#x\n",
8244                         be32_to_cpu(cmd->action_to_len16));
8245                 return;
8246         }
8247
8248         fec = fwcap_to_cc_fec(acaps);
8249         fc = fwcap_to_cc_pause(linkattr);
8250         speed = fwcap_to_speed(linkattr);
8251
8252         if (mod_type != pi->mod_type) {
8253                 /* With the newer SFP28 and QSFP28 Transceiver Module Types,
8254                  * various fundamental Port Capabilities which used to be
8255                  * immutable can now change radically.  We can now have
8256                  * Speeds, Auto-Negotiation, Forward Error Correction, etc.
8257                  * all change based on what Transceiver Module is inserted.
8258                  * So we need to record the Physical "Port" Capabilities on
8259                  * every Transceiver Module change.
8260                  */
8261                 lc->pcaps = pcaps;
8262
8263                 /* When a new Transceiver Module is inserted, the Firmware
8264                  * will examine its i2c EPROM to determine its type and
8265                  * general operating parameters including things like Forward
8266                  * Error Control, etc.  Various IEEE 802.3 standards dictate
8267                  * how to interpret these i2c values to determine default
8268                  * "sutomatic" settings.  We record these for future use when
8269                  * the user explicitly requests these standards-based values.
8270                  */
8271                 lc->def_acaps = acaps;
8272
8273                 /* Some versions of the early T6 Firmware "cheated" when
8274                  * handling different Transceiver Modules by changing the
8275                  * underlaying Port Type reported to the Host Drivers.  As
8276                  * such we need to capture whatever Port Type the Firmware
8277                  * sends us and record it in case it's different from what we
8278                  * were told earlier.  Unfortunately, since Firmware is
8279                  * forever, we'll need to keep this code here forever, but in
8280                  * later T6 Firmware it should just be an assignment of the
8281                  * same value already recorded.
8282                  */
8283                 pi->port_type = port_type;
8284
8285                 pi->mod_type = mod_type;
8286                 t4_os_portmod_changed(adapter, pi->port_id);
8287         }
8288
8289         if (link_ok != lc->link_ok || speed != lc->speed ||
8290             fc != lc->fc || fec != lc->fec) {   /* something changed */
8291                 if (!link_ok && lc->link_ok) {
8292                         lc->link_down_rc = linkdnrc;
8293                         dev_warn(adapter->pdev_dev, "Port %d link down, reason: %s\n",
8294                                  pi->tx_chan, t4_link_down_rc_str(linkdnrc));
8295                 }
8296                 lc->link_ok = link_ok;
8297                 lc->speed = speed;
8298                 lc->fc = fc;
8299                 lc->fec = fec;
8300
8301                 lc->lpacaps = lpacaps;
8302                 lc->acaps = acaps & ADVERT_MASK;
8303
8304                 if (lc->acaps & FW_PORT_CAP32_ANEG) {
8305                         lc->autoneg = AUTONEG_ENABLE;
8306                 } else {
8307                         /* When Autoneg is disabled, user needs to set
8308                          * single speed.
8309                          * Similar to cxgb4_ethtool.c: set_link_ksettings
8310                          */
8311                         lc->acaps = 0;
8312                         lc->speed_caps = fwcap_to_fwspeed(acaps);
8313                         lc->autoneg = AUTONEG_DISABLE;
8314                 }
8315
8316                 t4_os_link_changed(adapter, pi->port_id, link_ok);
8317         }
8318 }
8319
8320 /**
8321  *      t4_update_port_info - retrieve and update port information if changed
8322  *      @pi: the port_info
8323  *
8324  *      We issue a Get Port Information Command to the Firmware and, if
8325  *      successful, we check to see if anything is different from what we
8326  *      last recorded and update things accordingly.
8327  */
8328 int t4_update_port_info(struct port_info *pi)
8329 {
8330         unsigned int fw_caps = pi->adapter->params.fw_caps_support;
8331         struct fw_port_cmd port_cmd;
8332         int ret;
8333
8334         memset(&port_cmd, 0, sizeof(port_cmd));
8335         port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
8336                                             FW_CMD_REQUEST_F | FW_CMD_READ_F |
8337                                             FW_PORT_CMD_PORTID_V(pi->tx_chan));
8338         port_cmd.action_to_len16 = cpu_to_be32(
8339                 FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
8340                                      ? FW_PORT_ACTION_GET_PORT_INFO
8341                                      : FW_PORT_ACTION_GET_PORT_INFO32) |
8342                 FW_LEN16(port_cmd));
8343         ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox,
8344                          &port_cmd, sizeof(port_cmd), &port_cmd);
8345         if (ret)
8346                 return ret;
8347
8348         t4_handle_get_port_info(pi, (__be64 *)&port_cmd);
8349         return 0;
8350 }
8351
8352 /**
8353  *      t4_get_link_params - retrieve basic link parameters for given port
8354  *      @pi: the port
8355  *      @link_okp: value return pointer for link up/down
8356  *      @speedp: value return pointer for speed (Mb/s)
8357  *      @mtup: value return pointer for mtu
8358  *
8359  *      Retrieves basic link parameters for a port: link up/down, speed (Mb/s),
8360  *      and MTU for a specified port.  A negative error is returned on
8361  *      failure; 0 on success.
8362  */
8363 int t4_get_link_params(struct port_info *pi, unsigned int *link_okp,
8364                        unsigned int *speedp, unsigned int *mtup)
8365 {
8366         unsigned int fw_caps = pi->adapter->params.fw_caps_support;
8367         struct fw_port_cmd port_cmd;
8368         unsigned int action, link_ok, speed, mtu;
8369         fw_port_cap32_t linkattr;
8370         int ret;
8371
8372         memset(&port_cmd, 0, sizeof(port_cmd));
8373         port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
8374                                             FW_CMD_REQUEST_F | FW_CMD_READ_F |
8375                                             FW_PORT_CMD_PORTID_V(pi->tx_chan));
8376         action = (fw_caps == FW_CAPS16
8377                   ? FW_PORT_ACTION_GET_PORT_INFO
8378                   : FW_PORT_ACTION_GET_PORT_INFO32);
8379         port_cmd.action_to_len16 = cpu_to_be32(
8380                 FW_PORT_CMD_ACTION_V(action) |
8381                 FW_LEN16(port_cmd));
8382         ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox,
8383                          &port_cmd, sizeof(port_cmd), &port_cmd);
8384         if (ret)
8385                 return ret;
8386
8387         if (action == FW_PORT_ACTION_GET_PORT_INFO) {
8388                 u32 lstatus = be32_to_cpu(port_cmd.u.info.lstatus_to_modtype);
8389
8390                 link_ok = !!(lstatus & FW_PORT_CMD_LSTATUS_F);
8391                 linkattr = lstatus_to_fwcap(lstatus);
8392                 mtu = be16_to_cpu(port_cmd.u.info.mtu);
8393         } else {
8394                 u32 lstatus32 =
8395                            be32_to_cpu(port_cmd.u.info32.lstatus32_to_cbllen32);
8396
8397                 link_ok = !!(lstatus32 & FW_PORT_CMD_LSTATUS32_F);
8398                 linkattr = be32_to_cpu(port_cmd.u.info32.linkattr32);
8399                 mtu = FW_PORT_CMD_MTU32_G(
8400                         be32_to_cpu(port_cmd.u.info32.auxlinfo32_mtu32));
8401         }
8402         speed = fwcap_to_speed(linkattr);
8403
8404         *link_okp = link_ok;
8405         *speedp = fwcap_to_speed(linkattr);
8406         *mtup = mtu;
8407
8408         return 0;
8409 }
8410
8411 /**
8412  *      t4_handle_fw_rpl - process a FW reply message
8413  *      @adap: the adapter
8414  *      @rpl: start of the FW message
8415  *
8416  *      Processes a FW message, such as link state change messages.
8417  */
8418 int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl)
8419 {
8420         u8 opcode = *(const u8 *)rpl;
8421
8422         /* This might be a port command ... this simplifies the following
8423          * conditionals ...  We can get away with pre-dereferencing
8424          * action_to_len16 because it's in the first 16 bytes and all messages
8425          * will be at least that long.
8426          */
8427         const struct fw_port_cmd *p = (const void *)rpl;
8428         unsigned int action =
8429                 FW_PORT_CMD_ACTION_G(be32_to_cpu(p->action_to_len16));
8430
8431         if (opcode == FW_PORT_CMD &&
8432             (action == FW_PORT_ACTION_GET_PORT_INFO ||
8433              action == FW_PORT_ACTION_GET_PORT_INFO32)) {
8434                 int i;
8435                 int chan = FW_PORT_CMD_PORTID_G(be32_to_cpu(p->op_to_portid));
8436                 struct port_info *pi = NULL;
8437
8438                 for_each_port(adap, i) {
8439                         pi = adap2pinfo(adap, i);
8440                         if (pi->tx_chan == chan)
8441                                 break;
8442                 }
8443
8444                 t4_handle_get_port_info(pi, rpl);
8445         } else {
8446                 dev_warn(adap->pdev_dev, "Unknown firmware reply %d\n",
8447                          opcode);
8448                 return -EINVAL;
8449         }
8450         return 0;
8451 }
8452
8453 static void get_pci_mode(struct adapter *adapter, struct pci_params *p)
8454 {
8455         u16 val;
8456
8457         if (pci_is_pcie(adapter->pdev)) {
8458                 pcie_capability_read_word(adapter->pdev, PCI_EXP_LNKSTA, &val);
8459                 p->speed = val & PCI_EXP_LNKSTA_CLS;
8460                 p->width = (val & PCI_EXP_LNKSTA_NLW) >> 4;
8461         }
8462 }
8463
8464 /**
8465  *      init_link_config - initialize a link's SW state
8466  *      @lc: pointer to structure holding the link state
8467  *      @pcaps: link Port Capabilities
8468  *      @acaps: link current Advertised Port Capabilities
8469  *
8470  *      Initializes the SW state maintained for each link, including the link's
8471  *      capabilities and default speed/flow-control/autonegotiation settings.
8472  */
8473 static void init_link_config(struct link_config *lc, fw_port_cap32_t pcaps,
8474                              fw_port_cap32_t acaps)
8475 {
8476         lc->pcaps = pcaps;
8477         lc->def_acaps = acaps;
8478         lc->lpacaps = 0;
8479         lc->speed_caps = 0;
8480         lc->speed = 0;
8481         lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX;
8482
8483         /* For Forward Error Control, we default to whatever the Firmware
8484          * tells us the Link is currently advertising.
8485          */
8486         lc->requested_fec = FEC_AUTO;
8487         lc->fec = fwcap_to_cc_fec(lc->def_acaps);
8488
8489         if (lc->pcaps & FW_PORT_CAP32_ANEG) {
8490                 lc->acaps = lc->pcaps & ADVERT_MASK;
8491                 lc->autoneg = AUTONEG_ENABLE;
8492                 lc->requested_fc |= PAUSE_AUTONEG;
8493         } else {
8494                 lc->acaps = 0;
8495                 lc->autoneg = AUTONEG_DISABLE;
8496         }
8497 }
8498
8499 #define CIM_PF_NOACCESS 0xeeeeeeee
8500
8501 int t4_wait_dev_ready(void __iomem *regs)
8502 {
8503         u32 whoami;
8504
8505         whoami = readl(regs + PL_WHOAMI_A);
8506         if (whoami != 0xffffffff && whoami != CIM_PF_NOACCESS)
8507                 return 0;
8508
8509         msleep(500);
8510         whoami = readl(regs + PL_WHOAMI_A);
8511         return (whoami != 0xffffffff && whoami != CIM_PF_NOACCESS ? 0 : -EIO);
8512 }
8513
8514 struct flash_desc {
8515         u32 vendor_and_model_id;
8516         u32 size_mb;
8517 };
8518
8519 static int t4_get_flash_params(struct adapter *adap)
8520 {
8521         /* Table for non-Numonix supported flash parts.  Numonix parts are left
8522          * to the preexisting code.  All flash parts have 64KB sectors.
8523          */
8524         static struct flash_desc supported_flash[] = {
8525                 { 0x150201, 4 << 20 },       /* Spansion 4MB S25FL032P */
8526         };
8527
8528         unsigned int part, manufacturer;
8529         unsigned int density, size;
8530         u32 flashid = 0;
8531         int ret;
8532
8533         /* Issue a Read ID Command to the Flash part.  We decode supported
8534          * Flash parts and their sizes from this.  There's a newer Query
8535          * Command which can retrieve detailed geometry information but many
8536          * Flash parts don't support it.
8537          */
8538
8539         ret = sf1_write(adap, 1, 1, 0, SF_RD_ID);
8540         if (!ret)
8541                 ret = sf1_read(adap, 3, 0, 1, &flashid);
8542         t4_write_reg(adap, SF_OP_A, 0);                    /* unlock SF */
8543         if (ret)
8544                 return ret;
8545
8546         /* Check to see if it's one of our non-standard supported Flash parts.
8547          */
8548         for (part = 0; part < ARRAY_SIZE(supported_flash); part++)
8549                 if (supported_flash[part].vendor_and_model_id == flashid) {
8550                         adap->params.sf_size = supported_flash[part].size_mb;
8551                         adap->params.sf_nsec =
8552                                 adap->params.sf_size / SF_SEC_SIZE;
8553                         goto found;
8554                 }
8555
8556         /* Decode Flash part size.  The code below looks repetative with
8557          * common encodings, but that's not guaranteed in the JEDEC
8558          * specification for the Read JADEC ID command.  The only thing that
8559          * we're guaranteed by the JADEC specification is where the
8560          * Manufacturer ID is in the returned result.  After that each
8561          * Manufacturer ~could~ encode things completely differently.
8562          * Note, all Flash parts must have 64KB sectors.
8563          */
8564         manufacturer = flashid & 0xff;
8565         switch (manufacturer) {
8566         case 0x20: { /* Micron/Numonix */
8567                 /* This Density -> Size decoding table is taken from Micron
8568                  * Data Sheets.
8569                  */
8570                 density = (flashid >> 16) & 0xff;
8571                 switch (density) {
8572                 case 0x14: /* 1MB */
8573                         size = 1 << 20;
8574                         break;
8575                 case 0x15: /* 2MB */
8576                         size = 1 << 21;
8577                         break;
8578                 case 0x16: /* 4MB */
8579                         size = 1 << 22;
8580                         break;
8581                 case 0x17: /* 8MB */
8582                         size = 1 << 23;
8583                         break;
8584                 case 0x18: /* 16MB */
8585                         size = 1 << 24;
8586                         break;
8587                 case 0x19: /* 32MB */
8588                         size = 1 << 25;
8589                         break;
8590                 case 0x20: /* 64MB */
8591                         size = 1 << 26;
8592                         break;
8593                 case 0x21: /* 128MB */
8594                         size = 1 << 27;
8595                         break;
8596                 case 0x22: /* 256MB */
8597                         size = 1 << 28;
8598                         break;
8599
8600                 default:
8601                         dev_err(adap->pdev_dev, "Micron Flash Part has bad size, ID = %#x, Density code = %#x\n",
8602                                 flashid, density);
8603                         return -EINVAL;
8604                 }
8605                 break;
8606         }
8607         case 0x9d: { /* ISSI -- Integrated Silicon Solution, Inc. */
8608                 /* This Density -> Size decoding table is taken from ISSI
8609                  * Data Sheets.
8610                  */
8611                 density = (flashid >> 16) & 0xff;
8612                 switch (density) {
8613                 case 0x16: /* 32 MB */
8614                         size = 1 << 25;
8615                         break;
8616                 case 0x17: /* 64MB */
8617                         size = 1 << 26;
8618                         break;
8619                 default:
8620                         dev_err(adap->pdev_dev, "ISSI Flash Part has bad size, ID = %#x, Density code = %#x\n",
8621                                 flashid, density);
8622                         return -EINVAL;
8623                 }
8624                 break;
8625         }
8626         case 0xc2: { /* Macronix */
8627                 /* This Density -> Size decoding table is taken from Macronix
8628                  * Data Sheets.
8629                  */
8630                 density = (flashid >> 16) & 0xff;
8631                 switch (density) {
8632                 case 0x17: /* 8MB */
8633                         size = 1 << 23;
8634                         break;
8635                 case 0x18: /* 16MB */
8636                         size = 1 << 24;
8637                         break;
8638                 default:
8639                         dev_err(adap->pdev_dev, "Macronix Flash Part has bad size, ID = %#x, Density code = %#x\n",
8640                                 flashid, density);
8641                         return -EINVAL;
8642                 }
8643                 break;
8644         }
8645         case 0xef: { /* Winbond */
8646                 /* This Density -> Size decoding table is taken from Winbond
8647                  * Data Sheets.
8648                  */
8649                 density = (flashid >> 16) & 0xff;
8650                 switch (density) {
8651                 case 0x17: /* 8MB */
8652                         size = 1 << 23;
8653                         break;
8654                 case 0x18: /* 16MB */
8655                         size = 1 << 24;
8656                         break;
8657                 default:
8658                         dev_err(adap->pdev_dev, "Winbond Flash Part has bad size, ID = %#x, Density code = %#x\n",
8659                                 flashid, density);
8660                         return -EINVAL;
8661                 }
8662                 break;
8663         }
8664         default:
8665                 dev_err(adap->pdev_dev, "Unsupported Flash Part, ID = %#x\n",
8666                         flashid);
8667                 return -EINVAL;
8668         }
8669
8670         /* Store decoded Flash size and fall through into vetting code. */
8671         adap->params.sf_size = size;
8672         adap->params.sf_nsec = size / SF_SEC_SIZE;
8673
8674 found:
8675         if (adap->params.sf_size < FLASH_MIN_SIZE)
8676                 dev_warn(adap->pdev_dev, "WARNING: Flash Part ID %#x, size %#x < %#x\n",
8677                          flashid, adap->params.sf_size, FLASH_MIN_SIZE);
8678         return 0;
8679 }
8680
8681 /**
8682  *      t4_prep_adapter - prepare SW and HW for operation
8683  *      @adapter: the adapter
8684  *      @reset: if true perform a HW reset
8685  *
8686  *      Initialize adapter SW state for the various HW modules, set initial
8687  *      values for some adapter tunables, take PHYs out of reset, and
8688  *      initialize the MDIO interface.
8689  */
8690 int t4_prep_adapter(struct adapter *adapter)
8691 {
8692         int ret, ver;
8693         uint16_t device_id;
8694         u32 pl_rev;
8695
8696         get_pci_mode(adapter, &adapter->params.pci);
8697         pl_rev = REV_G(t4_read_reg(adapter, PL_REV_A));
8698
8699         ret = t4_get_flash_params(adapter);
8700         if (ret < 0) {
8701                 dev_err(adapter->pdev_dev, "error %d identifying flash\n", ret);
8702                 return ret;
8703         }
8704
8705         /* Retrieve adapter's device ID
8706          */
8707         pci_read_config_word(adapter->pdev, PCI_DEVICE_ID, &device_id);
8708         ver = device_id >> 12;
8709         adapter->params.chip = 0;
8710         switch (ver) {
8711         case CHELSIO_T4:
8712                 adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev);
8713                 adapter->params.arch.sge_fl_db = DBPRIO_F;
8714                 adapter->params.arch.mps_tcam_size =
8715                                  NUM_MPS_CLS_SRAM_L_INSTANCES;
8716                 adapter->params.arch.mps_rplc_size = 128;
8717                 adapter->params.arch.nchan = NCHAN;
8718                 adapter->params.arch.pm_stats_cnt = PM_NSTATS;
8719                 adapter->params.arch.vfcount = 128;
8720                 /* Congestion map is for 4 channels so that
8721                  * MPS can have 4 priority per port.
8722                  */
8723                 adapter->params.arch.cng_ch_bits_log = 2;
8724                 break;
8725         case CHELSIO_T5:
8726                 adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev);
8727                 adapter->params.arch.sge_fl_db = DBPRIO_F | DBTYPE_F;
8728                 adapter->params.arch.mps_tcam_size =
8729                                  NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
8730                 adapter->params.arch.mps_rplc_size = 128;
8731                 adapter->params.arch.nchan = NCHAN;
8732                 adapter->params.arch.pm_stats_cnt = PM_NSTATS;
8733                 adapter->params.arch.vfcount = 128;
8734                 adapter->params.arch.cng_ch_bits_log = 2;
8735                 break;
8736         case CHELSIO_T6:
8737                 adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev);
8738                 adapter->params.arch.sge_fl_db = 0;
8739                 adapter->params.arch.mps_tcam_size =
8740                                  NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
8741                 adapter->params.arch.mps_rplc_size = 256;
8742                 adapter->params.arch.nchan = 2;
8743                 adapter->params.arch.pm_stats_cnt = T6_PM_NSTATS;
8744                 adapter->params.arch.vfcount = 256;
8745                 /* Congestion map will be for 2 channels so that
8746                  * MPS can have 8 priority per port.
8747                  */
8748                 adapter->params.arch.cng_ch_bits_log = 3;
8749                 break;
8750         default:
8751                 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
8752                         device_id);
8753                 return -EINVAL;
8754         }
8755
8756         adapter->params.cim_la_size = CIMLA_SIZE;
8757         init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd);
8758
8759         /*
8760          * Default port for debugging in case we can't reach FW.
8761          */
8762         adapter->params.nports = 1;
8763         adapter->params.portvec = 1;
8764         adapter->params.vpd.cclk = 50000;
8765
8766         /* Set PCIe completion timeout to 4 seconds. */
8767         pcie_capability_clear_and_set_word(adapter->pdev, PCI_EXP_DEVCTL2,
8768                                            PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0xd);
8769         return 0;
8770 }
8771
8772 /**
8773  *      t4_shutdown_adapter - shut down adapter, host & wire
8774  *      @adapter: the adapter
8775  *
8776  *      Perform an emergency shutdown of the adapter and stop it from
8777  *      continuing any further communication on the ports or DMA to the
8778  *      host.  This is typically used when the adapter and/or firmware
8779  *      have crashed and we want to prevent any further accidental
8780  *      communication with the rest of the world.  This will also force
8781  *      the port Link Status to go down -- if register writes work --
8782  *      which should help our peers figure out that we're down.
8783  */
8784 int t4_shutdown_adapter(struct adapter *adapter)
8785 {
8786         int port;
8787
8788         t4_intr_disable(adapter);
8789         t4_write_reg(adapter, DBG_GPIO_EN_A, 0);
8790         for_each_port(adapter, port) {
8791                 u32 a_port_cfg = is_t4(adapter->params.chip) ?
8792                                        PORT_REG(port, XGMAC_PORT_CFG_A) :
8793                                        T5_PORT_REG(port, MAC_PORT_CFG_A);
8794
8795                 t4_write_reg(adapter, a_port_cfg,
8796                              t4_read_reg(adapter, a_port_cfg)
8797                              & ~SIGNAL_DET_V(1));
8798         }
8799         t4_set_reg_field(adapter, SGE_CONTROL_A, GLOBALENABLE_F, 0);
8800
8801         return 0;
8802 }
8803
8804 /**
8805  *      t4_bar2_sge_qregs - return BAR2 SGE Queue register information
8806  *      @adapter: the adapter
8807  *      @qid: the Queue ID
8808  *      @qtype: the Ingress or Egress type for @qid
8809  *      @user: true if this request is for a user mode queue
8810  *      @pbar2_qoffset: BAR2 Queue Offset
8811  *      @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
8812  *
8813  *      Returns the BAR2 SGE Queue Registers information associated with the
8814  *      indicated Absolute Queue ID.  These are passed back in return value
8815  *      pointers.  @qtype should be T4_BAR2_QTYPE_EGRESS for Egress Queue
8816  *      and T4_BAR2_QTYPE_INGRESS for Ingress Queues.
8817  *
8818  *      This may return an error which indicates that BAR2 SGE Queue
8819  *      registers aren't available.  If an error is not returned, then the
8820  *      following values are returned:
8821  *
8822  *        *@pbar2_qoffset: the BAR2 Offset of the @qid Registers
8823  *        *@pbar2_qid: the BAR2 SGE Queue ID or 0 of @qid
8824  *
8825  *      If the returned BAR2 Queue ID is 0, then BAR2 SGE registers which
8826  *      require the "Inferred Queue ID" ability may be used.  E.g. the
8827  *      Write Combining Doorbell Buffer. If the BAR2 Queue ID is not 0,
8828  *      then these "Inferred Queue ID" register may not be used.
8829  */
8830 int t4_bar2_sge_qregs(struct adapter *adapter,
8831                       unsigned int qid,
8832                       enum t4_bar2_qtype qtype,
8833                       int user,
8834                       u64 *pbar2_qoffset,
8835                       unsigned int *pbar2_qid)
8836 {
8837         unsigned int page_shift, page_size, qpp_shift, qpp_mask;
8838         u64 bar2_page_offset, bar2_qoffset;
8839         unsigned int bar2_qid, bar2_qid_offset, bar2_qinferred;
8840
8841         /* T4 doesn't support BAR2 SGE Queue registers for kernel mode queues */
8842         if (!user && is_t4(adapter->params.chip))
8843                 return -EINVAL;
8844
8845         /* Get our SGE Page Size parameters.
8846          */
8847         page_shift = adapter->params.sge.hps + 10;
8848         page_size = 1 << page_shift;
8849
8850         /* Get the right Queues per Page parameters for our Queue.
8851          */
8852         qpp_shift = (qtype == T4_BAR2_QTYPE_EGRESS
8853                      ? adapter->params.sge.eq_qpp
8854                      : adapter->params.sge.iq_qpp);
8855         qpp_mask = (1 << qpp_shift) - 1;
8856
8857         /*  Calculate the basics of the BAR2 SGE Queue register area:
8858          *  o The BAR2 page the Queue registers will be in.
8859          *  o The BAR2 Queue ID.
8860          *  o The BAR2 Queue ID Offset into the BAR2 page.
8861          */
8862         bar2_page_offset = ((u64)(qid >> qpp_shift) << page_shift);
8863         bar2_qid = qid & qpp_mask;
8864         bar2_qid_offset = bar2_qid * SGE_UDB_SIZE;
8865
8866         /* If the BAR2 Queue ID Offset is less than the Page Size, then the
8867          * hardware will infer the Absolute Queue ID simply from the writes to
8868          * the BAR2 Queue ID Offset within the BAR2 Page (and we need to use a
8869          * BAR2 Queue ID of 0 for those writes).  Otherwise, we'll simply
8870          * write to the first BAR2 SGE Queue Area within the BAR2 Page with
8871          * the BAR2 Queue ID and the hardware will infer the Absolute Queue ID
8872          * from the BAR2 Page and BAR2 Queue ID.
8873          *
8874          * One important censequence of this is that some BAR2 SGE registers
8875          * have a "Queue ID" field and we can write the BAR2 SGE Queue ID
8876          * there.  But other registers synthesize the SGE Queue ID purely
8877          * from the writes to the registers -- the Write Combined Doorbell
8878          * Buffer is a good example.  These BAR2 SGE Registers are only
8879          * available for those BAR2 SGE Register areas where the SGE Absolute
8880          * Queue ID can be inferred from simple writes.
8881          */
8882         bar2_qoffset = bar2_page_offset;
8883         bar2_qinferred = (bar2_qid_offset < page_size);
8884         if (bar2_qinferred) {
8885                 bar2_qoffset += bar2_qid_offset;
8886                 bar2_qid = 0;
8887         }
8888
8889         *pbar2_qoffset = bar2_qoffset;
8890         *pbar2_qid = bar2_qid;
8891         return 0;
8892 }
8893
8894 /**
8895  *      t4_init_devlog_params - initialize adapter->params.devlog
8896  *      @adap: the adapter
8897  *
8898  *      Initialize various fields of the adapter's Firmware Device Log
8899  *      Parameters structure.
8900  */
8901 int t4_init_devlog_params(struct adapter *adap)
8902 {
8903         struct devlog_params *dparams = &adap->params.devlog;
8904         u32 pf_dparams;
8905         unsigned int devlog_meminfo;
8906         struct fw_devlog_cmd devlog_cmd;
8907         int ret;
8908
8909         /* If we're dealing with newer firmware, the Device Log Paramerters
8910          * are stored in a designated register which allows us to access the
8911          * Device Log even if we can't talk to the firmware.
8912          */
8913         pf_dparams =
8914                 t4_read_reg(adap, PCIE_FW_REG(PCIE_FW_PF_A, PCIE_FW_PF_DEVLOG));
8915         if (pf_dparams) {
8916                 unsigned int nentries, nentries128;
8917
8918                 dparams->memtype = PCIE_FW_PF_DEVLOG_MEMTYPE_G(pf_dparams);
8919                 dparams->start = PCIE_FW_PF_DEVLOG_ADDR16_G(pf_dparams) << 4;
8920
8921                 nentries128 = PCIE_FW_PF_DEVLOG_NENTRIES128_G(pf_dparams);
8922                 nentries = (nentries128 + 1) * 128;
8923                 dparams->size = nentries * sizeof(struct fw_devlog_e);
8924
8925                 return 0;
8926         }
8927
8928         /* Otherwise, ask the firmware for it's Device Log Parameters.
8929          */
8930         memset(&devlog_cmd, 0, sizeof(devlog_cmd));
8931         devlog_cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_DEVLOG_CMD) |
8932                                              FW_CMD_REQUEST_F | FW_CMD_READ_F);
8933         devlog_cmd.retval_len16 = cpu_to_be32(FW_LEN16(devlog_cmd));
8934         ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
8935                          &devlog_cmd);
8936         if (ret)
8937                 return ret;
8938
8939         devlog_meminfo =
8940                 be32_to_cpu(devlog_cmd.memtype_devlog_memaddr16_devlog);
8941         dparams->memtype = FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
8942         dparams->start = FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
8943         dparams->size = be32_to_cpu(devlog_cmd.memsize_devlog);
8944
8945         return 0;
8946 }
8947
8948 /**
8949  *      t4_init_sge_params - initialize adap->params.sge
8950  *      @adapter: the adapter
8951  *
8952  *      Initialize various fields of the adapter's SGE Parameters structure.
8953  */
8954 int t4_init_sge_params(struct adapter *adapter)
8955 {
8956         struct sge_params *sge_params = &adapter->params.sge;
8957         u32 hps, qpp;
8958         unsigned int s_hps, s_qpp;
8959
8960         /* Extract the SGE Page Size for our PF.
8961          */
8962         hps = t4_read_reg(adapter, SGE_HOST_PAGE_SIZE_A);
8963         s_hps = (HOSTPAGESIZEPF0_S +
8964                  (HOSTPAGESIZEPF1_S - HOSTPAGESIZEPF0_S) * adapter->pf);
8965         sge_params->hps = ((hps >> s_hps) & HOSTPAGESIZEPF0_M);
8966
8967         /* Extract the SGE Egress and Ingess Queues Per Page for our PF.
8968          */
8969         s_qpp = (QUEUESPERPAGEPF0_S +
8970                 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * adapter->pf);
8971         qpp = t4_read_reg(adapter, SGE_EGRESS_QUEUES_PER_PAGE_PF_A);
8972         sge_params->eq_qpp = ((qpp >> s_qpp) & QUEUESPERPAGEPF0_M);
8973         qpp = t4_read_reg(adapter, SGE_INGRESS_QUEUES_PER_PAGE_PF_A);
8974         sge_params->iq_qpp = ((qpp >> s_qpp) & QUEUESPERPAGEPF0_M);
8975
8976         return 0;
8977 }
8978
8979 /**
8980  *      t4_init_tp_params - initialize adap->params.tp
8981  *      @adap: the adapter
8982  *      @sleep_ok: if true we may sleep while awaiting command completion
8983  *
8984  *      Initialize various fields of the adapter's TP Parameters structure.
8985  */
8986 int t4_init_tp_params(struct adapter *adap, bool sleep_ok)
8987 {
8988         int chan;
8989         u32 v;
8990
8991         v = t4_read_reg(adap, TP_TIMER_RESOLUTION_A);
8992         adap->params.tp.tre = TIMERRESOLUTION_G(v);
8993         adap->params.tp.dack_re = DELAYEDACKRESOLUTION_G(v);
8994
8995         /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
8996         for (chan = 0; chan < NCHAN; chan++)
8997                 adap->params.tp.tx_modq[chan] = chan;
8998
8999         /* Cache the adapter's Compressed Filter Mode and global Incress
9000          * Configuration.
9001          */
9002         t4_tp_pio_read(adap, &adap->params.tp.vlan_pri_map, 1,
9003                        TP_VLAN_PRI_MAP_A, sleep_ok);
9004         t4_tp_pio_read(adap, &adap->params.tp.ingress_config, 1,
9005                        TP_INGRESS_CONFIG_A, sleep_ok);
9006
9007         /* For T6, cache the adapter's compressed error vector
9008          * and passing outer header info for encapsulated packets.
9009          */
9010         if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) {
9011                 v = t4_read_reg(adap, TP_OUT_CONFIG_A);
9012                 adap->params.tp.rx_pkt_encap = (v & CRXPKTENC_F) ? 1 : 0;
9013         }
9014
9015         /* Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field
9016          * shift positions of several elements of the Compressed Filter Tuple
9017          * for this adapter which we need frequently ...
9018          */
9019         adap->params.tp.fcoe_shift = t4_filter_field_shift(adap, FCOE_F);
9020         adap->params.tp.port_shift = t4_filter_field_shift(adap, PORT_F);
9021         adap->params.tp.vnic_shift = t4_filter_field_shift(adap, VNIC_ID_F);
9022         adap->params.tp.vlan_shift = t4_filter_field_shift(adap, VLAN_F);
9023         adap->params.tp.tos_shift = t4_filter_field_shift(adap, TOS_F);
9024         adap->params.tp.protocol_shift = t4_filter_field_shift(adap,
9025                                                                PROTOCOL_F);
9026         adap->params.tp.ethertype_shift = t4_filter_field_shift(adap,
9027                                                                 ETHERTYPE_F);
9028         adap->params.tp.macmatch_shift = t4_filter_field_shift(adap,
9029                                                                MACMATCH_F);
9030         adap->params.tp.matchtype_shift = t4_filter_field_shift(adap,
9031                                                                 MPSHITTYPE_F);
9032         adap->params.tp.frag_shift = t4_filter_field_shift(adap,
9033                                                            FRAGMENTATION_F);
9034
9035         /* If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
9036          * represents the presence of an Outer VLAN instead of a VNIC ID.
9037          */
9038         if ((adap->params.tp.ingress_config & VNIC_F) == 0)
9039                 adap->params.tp.vnic_shift = -1;
9040
9041         v = t4_read_reg(adap, LE_3_DB_HASH_MASK_GEN_IPV4_T6_A);
9042         adap->params.tp.hash_filter_mask = v;
9043         v = t4_read_reg(adap, LE_4_DB_HASH_MASK_GEN_IPV4_T6_A);
9044         adap->params.tp.hash_filter_mask |= ((u64)v << 32);
9045         return 0;
9046 }
9047
9048 /**
9049  *      t4_filter_field_shift - calculate filter field shift
9050  *      @adap: the adapter
9051  *      @filter_sel: the desired field (from TP_VLAN_PRI_MAP bits)
9052  *
9053  *      Return the shift position of a filter field within the Compressed
9054  *      Filter Tuple.  The filter field is specified via its selection bit
9055  *      within TP_VLAN_PRI_MAL (filter mode).  E.g. F_VLAN.
9056  */
9057 int t4_filter_field_shift(const struct adapter *adap, int filter_sel)
9058 {
9059         unsigned int filter_mode = adap->params.tp.vlan_pri_map;
9060         unsigned int sel;
9061         int field_shift;
9062
9063         if ((filter_mode & filter_sel) == 0)
9064                 return -1;
9065
9066         for (sel = 1, field_shift = 0; sel < filter_sel; sel <<= 1) {
9067                 switch (filter_mode & sel) {
9068                 case FCOE_F:
9069                         field_shift += FT_FCOE_W;
9070                         break;
9071                 case PORT_F:
9072                         field_shift += FT_PORT_W;
9073                         break;
9074                 case VNIC_ID_F:
9075                         field_shift += FT_VNIC_ID_W;
9076                         break;
9077                 case VLAN_F:
9078                         field_shift += FT_VLAN_W;
9079                         break;
9080                 case TOS_F:
9081                         field_shift += FT_TOS_W;
9082                         break;
9083                 case PROTOCOL_F:
9084                         field_shift += FT_PROTOCOL_W;
9085                         break;
9086                 case ETHERTYPE_F:
9087                         field_shift += FT_ETHERTYPE_W;
9088                         break;
9089                 case MACMATCH_F:
9090                         field_shift += FT_MACMATCH_W;
9091                         break;
9092                 case MPSHITTYPE_F:
9093                         field_shift += FT_MPSHITTYPE_W;
9094                         break;
9095                 case FRAGMENTATION_F:
9096                         field_shift += FT_FRAGMENTATION_W;
9097                         break;
9098                 }
9099         }
9100         return field_shift;
9101 }
9102
9103 int t4_init_rss_mode(struct adapter *adap, int mbox)
9104 {
9105         int i, ret;
9106         struct fw_rss_vi_config_cmd rvc;
9107
9108         memset(&rvc, 0, sizeof(rvc));
9109
9110         for_each_port(adap, i) {
9111                 struct port_info *p = adap2pinfo(adap, i);
9112
9113                 rvc.op_to_viid =
9114                         cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) |
9115                                     FW_CMD_REQUEST_F | FW_CMD_READ_F |
9116                                     FW_RSS_VI_CONFIG_CMD_VIID_V(p->viid));
9117                 rvc.retval_len16 = cpu_to_be32(FW_LEN16(rvc));
9118                 ret = t4_wr_mbox(adap, mbox, &rvc, sizeof(rvc), &rvc);
9119                 if (ret)
9120                         return ret;
9121                 p->rss_mode = be32_to_cpu(rvc.u.basicvirtual.defaultq_to_udpen);
9122         }
9123         return 0;
9124 }
9125
9126 /**
9127  *      t4_init_portinfo - allocate a virtual interface and initialize port_info
9128  *      @pi: the port_info
9129  *      @mbox: mailbox to use for the FW command
9130  *      @port: physical port associated with the VI
9131  *      @pf: the PF owning the VI
9132  *      @vf: the VF owning the VI
9133  *      @mac: the MAC address of the VI
9134  *
9135  *      Allocates a virtual interface for the given physical port.  If @mac is
9136  *      not %NULL it contains the MAC address of the VI as assigned by FW.
9137  *      @mac should be large enough to hold an Ethernet address.
9138  *      Returns < 0 on error.
9139  */
9140 int t4_init_portinfo(struct port_info *pi, int mbox,
9141                      int port, int pf, int vf, u8 mac[])
9142 {
9143         struct adapter *adapter = pi->adapter;
9144         unsigned int fw_caps = adapter->params.fw_caps_support;
9145         struct fw_port_cmd cmd;
9146         unsigned int rss_size;
9147         enum fw_port_type port_type;
9148         int mdio_addr;
9149         fw_port_cap32_t pcaps, acaps;
9150         int ret;
9151
9152         /* If we haven't yet determined whether we're talking to Firmware
9153          * which knows the new 32-bit Port Capabilities, it's time to find
9154          * out now.  This will also tell new Firmware to send us Port Status
9155          * Updates using the new 32-bit Port Capabilities version of the
9156          * Port Information message.
9157          */
9158         if (fw_caps == FW_CAPS_UNKNOWN) {
9159                 u32 param, val;
9160
9161                 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
9162                          FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_PORT_CAPS32));
9163                 val = 1;
9164                 ret = t4_set_params(adapter, mbox, pf, vf, 1, &param, &val);
9165                 fw_caps = (ret == 0 ? FW_CAPS32 : FW_CAPS16);
9166                 adapter->params.fw_caps_support = fw_caps;
9167         }
9168
9169         memset(&cmd, 0, sizeof(cmd));
9170         cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
9171                                        FW_CMD_REQUEST_F | FW_CMD_READ_F |
9172                                        FW_PORT_CMD_PORTID_V(port));
9173         cmd.action_to_len16 = cpu_to_be32(
9174                 FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
9175                                      ? FW_PORT_ACTION_GET_PORT_INFO
9176                                      : FW_PORT_ACTION_GET_PORT_INFO32) |
9177                 FW_LEN16(cmd));
9178         ret = t4_wr_mbox(pi->adapter, mbox, &cmd, sizeof(cmd), &cmd);
9179         if (ret)
9180                 return ret;
9181
9182         /* Extract the various fields from the Port Information message.
9183          */
9184         if (fw_caps == FW_CAPS16) {
9185                 u32 lstatus = be32_to_cpu(cmd.u.info.lstatus_to_modtype);
9186
9187                 port_type = FW_PORT_CMD_PTYPE_G(lstatus);
9188                 mdio_addr = ((lstatus & FW_PORT_CMD_MDIOCAP_F)
9189                              ? FW_PORT_CMD_MDIOADDR_G(lstatus)
9190                              : -1);
9191                 pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd.u.info.pcap));
9192                 acaps = fwcaps16_to_caps32(be16_to_cpu(cmd.u.info.acap));
9193         } else {
9194                 u32 lstatus32 = be32_to_cpu(cmd.u.info32.lstatus32_to_cbllen32);
9195
9196                 port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32);
9197                 mdio_addr = ((lstatus32 & FW_PORT_CMD_MDIOCAP32_F)
9198                              ? FW_PORT_CMD_MDIOADDR32_G(lstatus32)
9199                              : -1);
9200                 pcaps = be32_to_cpu(cmd.u.info32.pcaps32);
9201                 acaps = be32_to_cpu(cmd.u.info32.acaps32);
9202         }
9203
9204         ret = t4_alloc_vi(pi->adapter, mbox, port, pf, vf, 1, mac, &rss_size);
9205         if (ret < 0)
9206                 return ret;
9207
9208         pi->viid = ret;
9209         pi->tx_chan = port;
9210         pi->lport = port;
9211         pi->rss_size = rss_size;
9212
9213         pi->port_type = port_type;
9214         pi->mdio_addr = mdio_addr;
9215         pi->mod_type = FW_PORT_MOD_TYPE_NA;
9216
9217         init_link_config(&pi->link_cfg, pcaps, acaps);
9218         return 0;
9219 }
9220
9221 int t4_port_init(struct adapter *adap, int mbox, int pf, int vf)
9222 {
9223         u8 addr[6];
9224         int ret, i, j = 0;
9225
9226         for_each_port(adap, i) {
9227                 struct port_info *pi = adap2pinfo(adap, i);
9228
9229                 while ((adap->params.portvec & (1 << j)) == 0)
9230                         j++;
9231
9232                 ret = t4_init_portinfo(pi, mbox, j, pf, vf, addr);
9233                 if (ret)
9234                         return ret;
9235
9236                 memcpy(adap->port[i]->dev_addr, addr, ETH_ALEN);
9237                 j++;
9238         }
9239         return 0;
9240 }
9241
9242 /**
9243  *      t4_read_cimq_cfg - read CIM queue configuration
9244  *      @adap: the adapter
9245  *      @base: holds the queue base addresses in bytes
9246  *      @size: holds the queue sizes in bytes
9247  *      @thres: holds the queue full thresholds in bytes
9248  *
9249  *      Returns the current configuration of the CIM queues, starting with
9250  *      the IBQs, then the OBQs.
9251  */
9252 void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres)
9253 {
9254         unsigned int i, v;
9255         int cim_num_obq = is_t4(adap->params.chip) ?
9256                                 CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
9257
9258         for (i = 0; i < CIM_NUM_IBQ; i++) {
9259                 t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, IBQSELECT_F |
9260                              QUENUMSELECT_V(i));
9261                 v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A);
9262                 /* value is in 256-byte units */
9263                 *base++ = CIMQBASE_G(v) * 256;
9264                 *size++ = CIMQSIZE_G(v) * 256;
9265                 *thres++ = QUEFULLTHRSH_G(v) * 8; /* 8-byte unit */
9266         }
9267         for (i = 0; i < cim_num_obq; i++) {
9268                 t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F |
9269                              QUENUMSELECT_V(i));
9270                 v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A);
9271                 /* value is in 256-byte units */
9272                 *base++ = CIMQBASE_G(v) * 256;
9273                 *size++ = CIMQSIZE_G(v) * 256;
9274         }
9275 }
9276
9277 /**
9278  *      t4_read_cim_ibq - read the contents of a CIM inbound queue
9279  *      @adap: the adapter
9280  *      @qid: the queue index
9281  *      @data: where to store the queue contents
9282  *      @n: capacity of @data in 32-bit words
9283  *
9284  *      Reads the contents of the selected CIM queue starting at address 0 up
9285  *      to the capacity of @data.  @n must be a multiple of 4.  Returns < 0 on
9286  *      error and the number of 32-bit words actually read on success.
9287  */
9288 int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n)
9289 {
9290         int i, err, attempts;
9291         unsigned int addr;
9292         const unsigned int nwords = CIM_IBQ_SIZE * 4;
9293
9294         if (qid > 5 || (n & 3))
9295                 return -EINVAL;
9296
9297         addr = qid * nwords;
9298         if (n > nwords)
9299                 n = nwords;
9300
9301         /* It might take 3-10ms before the IBQ debug read access is allowed.
9302          * Wait for 1 Sec with a delay of 1 usec.
9303          */
9304         attempts = 1000000;
9305
9306         for (i = 0; i < n; i++, addr++) {
9307                 t4_write_reg(adap, CIM_IBQ_DBG_CFG_A, IBQDBGADDR_V(addr) |
9308                              IBQDBGEN_F);
9309                 err = t4_wait_op_done(adap, CIM_IBQ_DBG_CFG_A, IBQDBGBUSY_F, 0,
9310                                       attempts, 1);
9311                 if (err)
9312                         return err;
9313                 *data++ = t4_read_reg(adap, CIM_IBQ_DBG_DATA_A);
9314         }
9315         t4_write_reg(adap, CIM_IBQ_DBG_CFG_A, 0);
9316         return i;
9317 }
9318
9319 /**
9320  *      t4_read_cim_obq - read the contents of a CIM outbound queue
9321  *      @adap: the adapter
9322  *      @qid: the queue index
9323  *      @data: where to store the queue contents
9324  *      @n: capacity of @data in 32-bit words
9325  *
9326  *      Reads the contents of the selected CIM queue starting at address 0 up
9327  *      to the capacity of @data.  @n must be a multiple of 4.  Returns < 0 on
9328  *      error and the number of 32-bit words actually read on success.
9329  */
9330 int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n)
9331 {
9332         int i, err;
9333         unsigned int addr, v, nwords;
9334         int cim_num_obq = is_t4(adap->params.chip) ?
9335                                 CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
9336
9337         if ((qid > (cim_num_obq - 1)) || (n & 3))
9338                 return -EINVAL;
9339
9340         t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F |
9341                      QUENUMSELECT_V(qid));
9342         v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A);
9343
9344         addr = CIMQBASE_G(v) * 64;    /* muliple of 256 -> muliple of 4 */
9345         nwords = CIMQSIZE_G(v) * 64;  /* same */
9346         if (n > nwords)
9347                 n = nwords;
9348
9349         for (i = 0; i < n; i++, addr++) {
9350                 t4_write_reg(adap, CIM_OBQ_DBG_CFG_A, OBQDBGADDR_V(addr) |
9351                              OBQDBGEN_F);
9352                 err = t4_wait_op_done(adap, CIM_OBQ_DBG_CFG_A, OBQDBGBUSY_F, 0,
9353                                       2, 1);
9354                 if (err)
9355                         return err;
9356                 *data++ = t4_read_reg(adap, CIM_OBQ_DBG_DATA_A);
9357         }
9358         t4_write_reg(adap, CIM_OBQ_DBG_CFG_A, 0);
9359         return i;
9360 }
9361
9362 /**
9363  *      t4_cim_read - read a block from CIM internal address space
9364  *      @adap: the adapter
9365  *      @addr: the start address within the CIM address space
9366  *      @n: number of words to read
9367  *      @valp: where to store the result
9368  *
9369  *      Reads a block of 4-byte words from the CIM intenal address space.
9370  */
9371 int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n,
9372                 unsigned int *valp)
9373 {
9374         int ret = 0;
9375
9376         if (t4_read_reg(adap, CIM_HOST_ACC_CTRL_A) & HOSTBUSY_F)
9377                 return -EBUSY;
9378
9379         for ( ; !ret && n--; addr += 4) {
9380                 t4_write_reg(adap, CIM_HOST_ACC_CTRL_A, addr);
9381                 ret = t4_wait_op_done(adap, CIM_HOST_ACC_CTRL_A, HOSTBUSY_F,
9382                                       0, 5, 2);
9383                 if (!ret)
9384                         *valp++ = t4_read_reg(adap, CIM_HOST_ACC_DATA_A);
9385         }
9386         return ret;
9387 }
9388
9389 /**
9390  *      t4_cim_write - write a block into CIM internal address space
9391  *      @adap: the adapter
9392  *      @addr: the start address within the CIM address space
9393  *      @n: number of words to write
9394  *      @valp: set of values to write
9395  *
9396  *      Writes a block of 4-byte words into the CIM intenal address space.
9397  */
9398 int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n,
9399                  const unsigned int *valp)
9400 {
9401         int ret = 0;
9402
9403         if (t4_read_reg(adap, CIM_HOST_ACC_CTRL_A) & HOSTBUSY_F)
9404                 return -EBUSY;
9405
9406         for ( ; !ret && n--; addr += 4) {
9407                 t4_write_reg(adap, CIM_HOST_ACC_DATA_A, *valp++);
9408                 t4_write_reg(adap, CIM_HOST_ACC_CTRL_A, addr | HOSTWRITE_F);
9409                 ret = t4_wait_op_done(adap, CIM_HOST_ACC_CTRL_A, HOSTBUSY_F,
9410                                       0, 5, 2);
9411         }
9412         return ret;
9413 }
9414
9415 static int t4_cim_write1(struct adapter *adap, unsigned int addr,
9416                          unsigned int val)
9417 {
9418         return t4_cim_write(adap, addr, 1, &val);
9419 }
9420
9421 /**
9422  *      t4_cim_read_la - read CIM LA capture buffer
9423  *      @adap: the adapter
9424  *      @la_buf: where to store the LA data
9425  *      @wrptr: the HW write pointer within the capture buffer
9426  *
9427  *      Reads the contents of the CIM LA buffer with the most recent entry at
9428  *      the end of the returned data and with the entry at @wrptr first.
9429  *      We try to leave the LA in the running state we find it in.
9430  */
9431 int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr)
9432 {
9433         int i, ret;
9434         unsigned int cfg, val, idx;
9435
9436         ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &cfg);
9437         if (ret)
9438                 return ret;
9439
9440         if (cfg & UPDBGLAEN_F) {        /* LA is running, freeze it */
9441                 ret = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A, 0);
9442                 if (ret)
9443                         return ret;
9444         }
9445
9446         ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &val);
9447         if (ret)
9448                 goto restart;
9449
9450         idx = UPDBGLAWRPTR_G(val);
9451         if (wrptr)
9452                 *wrptr = idx;
9453
9454         for (i = 0; i < adap->params.cim_la_size; i++) {
9455                 ret = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A,
9456                                     UPDBGLARDPTR_V(idx) | UPDBGLARDEN_F);
9457                 if (ret)
9458                         break;
9459                 ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &val);
9460                 if (ret)
9461                         break;
9462                 if (val & UPDBGLARDEN_F) {
9463                         ret = -ETIMEDOUT;
9464                         break;
9465                 }
9466                 ret = t4_cim_read(adap, UP_UP_DBG_LA_DATA_A, 1, &la_buf[i]);
9467                 if (ret)
9468                         break;
9469
9470                 /* Bits 0-3 of UpDbgLaRdPtr can be between 0000 to 1001 to
9471                  * identify the 32-bit portion of the full 312-bit data
9472                  */
9473                 if (is_t6(adap->params.chip) && (idx & 0xf) >= 9)
9474                         idx = (idx & 0xff0) + 0x10;
9475                 else
9476                         idx++;
9477                 /* address can't exceed 0xfff */
9478                 idx &= UPDBGLARDPTR_M;
9479         }
9480 restart:
9481         if (cfg & UPDBGLAEN_F) {
9482                 int r = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A,
9483                                       cfg & ~UPDBGLARDEN_F);
9484                 if (!ret)
9485                         ret = r;
9486         }
9487         return ret;
9488 }
9489
9490 /**
9491  *      t4_tp_read_la - read TP LA capture buffer
9492  *      @adap: the adapter
9493  *      @la_buf: where to store the LA data
9494  *      @wrptr: the HW write pointer within the capture buffer
9495  *
9496  *      Reads the contents of the TP LA buffer with the most recent entry at
9497  *      the end of the returned data and with the entry at @wrptr first.
9498  *      We leave the LA in the running state we find it in.
9499  */
9500 void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr)
9501 {
9502         bool last_incomplete;
9503         unsigned int i, cfg, val, idx;
9504
9505         cfg = t4_read_reg(adap, TP_DBG_LA_CONFIG_A) & 0xffff;
9506         if (cfg & DBGLAENABLE_F)                        /* freeze LA */
9507                 t4_write_reg(adap, TP_DBG_LA_CONFIG_A,
9508                              adap->params.tp.la_mask | (cfg ^ DBGLAENABLE_F));
9509
9510         val = t4_read_reg(adap, TP_DBG_LA_CONFIG_A);
9511         idx = DBGLAWPTR_G(val);
9512         last_incomplete = DBGLAMODE_G(val) >= 2 && (val & DBGLAWHLF_F) == 0;
9513         if (last_incomplete)
9514                 idx = (idx + 1) & DBGLARPTR_M;
9515         if (wrptr)
9516                 *wrptr = idx;
9517
9518         val &= 0xffff;
9519         val &= ~DBGLARPTR_V(DBGLARPTR_M);
9520         val |= adap->params.tp.la_mask;
9521
9522         for (i = 0; i < TPLA_SIZE; i++) {
9523                 t4_write_reg(adap, TP_DBG_LA_CONFIG_A, DBGLARPTR_V(idx) | val);
9524                 la_buf[i] = t4_read_reg64(adap, TP_DBG_LA_DATAL_A);
9525                 idx = (idx + 1) & DBGLARPTR_M;
9526         }
9527
9528         /* Wipe out last entry if it isn't valid */
9529         if (last_incomplete)
9530                 la_buf[TPLA_SIZE - 1] = ~0ULL;
9531
9532         if (cfg & DBGLAENABLE_F)                    /* restore running state */
9533                 t4_write_reg(adap, TP_DBG_LA_CONFIG_A,
9534                              cfg | adap->params.tp.la_mask);
9535 }
9536
9537 /* SGE Hung Ingress DMA Warning Threshold time and Warning Repeat Rate (in
9538  * seconds).  If we find one of the SGE Ingress DMA State Machines in the same
9539  * state for more than the Warning Threshold then we'll issue a warning about
9540  * a potential hang.  We'll repeat the warning as the SGE Ingress DMA Channel
9541  * appears to be hung every Warning Repeat second till the situation clears.
9542  * If the situation clears, we'll note that as well.
9543  */
9544 #define SGE_IDMA_WARN_THRESH 1
9545 #define SGE_IDMA_WARN_REPEAT 300
9546
9547 /**
9548  *      t4_idma_monitor_init - initialize SGE Ingress DMA Monitor
9549  *      @adapter: the adapter
9550  *      @idma: the adapter IDMA Monitor state
9551  *
9552  *      Initialize the state of an SGE Ingress DMA Monitor.
9553  */
9554 void t4_idma_monitor_init(struct adapter *adapter,
9555                           struct sge_idma_monitor_state *idma)
9556 {
9557         /* Initialize the state variables for detecting an SGE Ingress DMA
9558          * hang.  The SGE has internal counters which count up on each clock
9559          * tick whenever the SGE finds its Ingress DMA State Engines in the
9560          * same state they were on the previous clock tick.  The clock used is
9561          * the Core Clock so we have a limit on the maximum "time" they can
9562          * record; typically a very small number of seconds.  For instance,
9563          * with a 600MHz Core Clock, we can only count up to a bit more than
9564          * 7s.  So we'll synthesize a larger counter in order to not run the
9565          * risk of having the "timers" overflow and give us the flexibility to
9566          * maintain a Hung SGE State Machine of our own which operates across
9567          * a longer time frame.
9568          */
9569         idma->idma_1s_thresh = core_ticks_per_usec(adapter) * 1000000; /* 1s */
9570         idma->idma_stalled[0] = 0;
9571         idma->idma_stalled[1] = 0;
9572 }
9573
9574 /**
9575  *      t4_idma_monitor - monitor SGE Ingress DMA state
9576  *      @adapter: the adapter
9577  *      @idma: the adapter IDMA Monitor state
9578  *      @hz: number of ticks/second
9579  *      @ticks: number of ticks since the last IDMA Monitor call
9580  */
9581 void t4_idma_monitor(struct adapter *adapter,
9582                      struct sge_idma_monitor_state *idma,
9583                      int hz, int ticks)
9584 {
9585         int i, idma_same_state_cnt[2];
9586
9587          /* Read the SGE Debug Ingress DMA Same State Count registers.  These
9588           * are counters inside the SGE which count up on each clock when the
9589           * SGE finds its Ingress DMA State Engines in the same states they
9590           * were in the previous clock.  The counters will peg out at
9591           * 0xffffffff without wrapping around so once they pass the 1s
9592           * threshold they'll stay above that till the IDMA state changes.
9593           */
9594         t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 13);
9595         idma_same_state_cnt[0] = t4_read_reg(adapter, SGE_DEBUG_DATA_HIGH_A);
9596         idma_same_state_cnt[1] = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A);
9597
9598         for (i = 0; i < 2; i++) {
9599                 u32 debug0, debug11;
9600
9601                 /* If the Ingress DMA Same State Counter ("timer") is less
9602                  * than 1s, then we can reset our synthesized Stall Timer and
9603                  * continue.  If we have previously emitted warnings about a
9604                  * potential stalled Ingress Queue, issue a note indicating
9605                  * that the Ingress Queue has resumed forward progress.
9606                  */
9607                 if (idma_same_state_cnt[i] < idma->idma_1s_thresh) {
9608                         if (idma->idma_stalled[i] >= SGE_IDMA_WARN_THRESH * hz)
9609                                 dev_warn(adapter->pdev_dev, "SGE idma%d, queue %u, "
9610                                          "resumed after %d seconds\n",
9611                                          i, idma->idma_qid[i],
9612                                          idma->idma_stalled[i] / hz);
9613                         idma->idma_stalled[i] = 0;
9614                         continue;
9615                 }
9616
9617                 /* Synthesize an SGE Ingress DMA Same State Timer in the Hz
9618                  * domain.  The first time we get here it'll be because we
9619                  * passed the 1s Threshold; each additional time it'll be
9620                  * because the RX Timer Callback is being fired on its regular
9621                  * schedule.
9622                  *
9623                  * If the stall is below our Potential Hung Ingress Queue
9624                  * Warning Threshold, continue.
9625                  */
9626                 if (idma->idma_stalled[i] == 0) {
9627                         idma->idma_stalled[i] = hz;
9628                         idma->idma_warn[i] = 0;
9629                 } else {
9630                         idma->idma_stalled[i] += ticks;
9631                         idma->idma_warn[i] -= ticks;
9632                 }
9633
9634                 if (idma->idma_stalled[i] < SGE_IDMA_WARN_THRESH * hz)
9635                         continue;
9636
9637                 /* We'll issue a warning every SGE_IDMA_WARN_REPEAT seconds.
9638                  */
9639                 if (idma->idma_warn[i] > 0)
9640                         continue;
9641                 idma->idma_warn[i] = SGE_IDMA_WARN_REPEAT * hz;
9642
9643                 /* Read and save the SGE IDMA State and Queue ID information.
9644                  * We do this every time in case it changes across time ...
9645                  * can't be too careful ...
9646                  */
9647                 t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 0);
9648                 debug0 = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A);
9649                 idma->idma_state[i] = (debug0 >> (i * 9)) & 0x3f;
9650
9651                 t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 11);
9652                 debug11 = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A);
9653                 idma->idma_qid[i] = (debug11 >> (i * 16)) & 0xffff;
9654
9655                 dev_warn(adapter->pdev_dev, "SGE idma%u, queue %u, potentially stuck in "
9656                          "state %u for %d seconds (debug0=%#x, debug11=%#x)\n",
9657                          i, idma->idma_qid[i], idma->idma_state[i],
9658                          idma->idma_stalled[i] / hz,
9659                          debug0, debug11);
9660                 t4_sge_decode_idma_state(adapter, idma->idma_state[i]);
9661         }
9662 }
9663
9664 /**
9665  *      t4_load_cfg - download config file
9666  *      @adap: the adapter
9667  *      @cfg_data: the cfg text file to write
9668  *      @size: text file size
9669  *
9670  *      Write the supplied config text file to the card's serial flash.
9671  */
9672 int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
9673 {
9674         int ret, i, n, cfg_addr;
9675         unsigned int addr;
9676         unsigned int flash_cfg_start_sec;
9677         unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
9678
9679         cfg_addr = t4_flash_cfg_addr(adap);
9680         if (cfg_addr < 0)
9681                 return cfg_addr;
9682
9683         addr = cfg_addr;
9684         flash_cfg_start_sec = addr / SF_SEC_SIZE;
9685
9686         if (size > FLASH_CFG_MAX_SIZE) {
9687                 dev_err(adap->pdev_dev, "cfg file too large, max is %u bytes\n",
9688                         FLASH_CFG_MAX_SIZE);
9689                 return -EFBIG;
9690         }
9691
9692         i = DIV_ROUND_UP(FLASH_CFG_MAX_SIZE,    /* # of sectors spanned */
9693                          sf_sec_size);
9694         ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec,
9695                                      flash_cfg_start_sec + i - 1);
9696         /* If size == 0 then we're simply erasing the FLASH sectors associated
9697          * with the on-adapter Firmware Configuration File.
9698          */
9699         if (ret || size == 0)
9700                 goto out;
9701
9702         /* this will write to the flash up to SF_PAGE_SIZE at a time */
9703         for (i = 0; i < size; i += SF_PAGE_SIZE) {
9704                 if ((size - i) <  SF_PAGE_SIZE)
9705                         n = size - i;
9706                 else
9707                         n = SF_PAGE_SIZE;
9708                 ret = t4_write_flash(adap, addr, n, cfg_data);
9709                 if (ret)
9710                         goto out;
9711
9712                 addr += SF_PAGE_SIZE;
9713                 cfg_data += SF_PAGE_SIZE;
9714         }
9715
9716 out:
9717         if (ret)
9718                 dev_err(adap->pdev_dev, "config file %s failed %d\n",
9719                         (size == 0 ? "clear" : "download"), ret);
9720         return ret;
9721 }
9722
9723 /**
9724  *      t4_set_vf_mac - Set MAC address for the specified VF
9725  *      @adapter: The adapter
9726  *      @vf: one of the VFs instantiated by the specified PF
9727  *      @naddr: the number of MAC addresses
9728  *      @addr: the MAC address(es) to be set to the specified VF
9729  */
9730 int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf,
9731                       unsigned int naddr, u8 *addr)
9732 {
9733         struct fw_acl_mac_cmd cmd;
9734
9735         memset(&cmd, 0, sizeof(cmd));
9736         cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) |
9737                                     FW_CMD_REQUEST_F |
9738                                     FW_CMD_WRITE_F |
9739                                     FW_ACL_MAC_CMD_PFN_V(adapter->pf) |
9740                                     FW_ACL_MAC_CMD_VFN_V(vf));
9741
9742         /* Note: Do not enable the ACL */
9743         cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd));
9744         cmd.nmac = naddr;
9745
9746         switch (adapter->pf) {
9747         case 3:
9748                 memcpy(cmd.macaddr3, addr, sizeof(cmd.macaddr3));
9749                 break;
9750         case 2:
9751                 memcpy(cmd.macaddr2, addr, sizeof(cmd.macaddr2));
9752                 break;
9753         case 1:
9754                 memcpy(cmd.macaddr1, addr, sizeof(cmd.macaddr1));
9755                 break;
9756         case 0:
9757                 memcpy(cmd.macaddr0, addr, sizeof(cmd.macaddr0));
9758                 break;
9759         }
9760
9761         return t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &cmd);
9762 }
9763
9764 /**
9765  * t4_read_pace_tbl - read the pace table
9766  * @adap: the adapter
9767  * @pace_vals: holds the returned values
9768  *
9769  * Returns the values of TP's pace table in microseconds.
9770  */
9771 void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED])
9772 {
9773         unsigned int i, v;
9774
9775         for (i = 0; i < NTX_SCHED; i++) {
9776                 t4_write_reg(adap, TP_PACE_TABLE_A, 0xffff0000 + i);
9777                 v = t4_read_reg(adap, TP_PACE_TABLE_A);
9778                 pace_vals[i] = dack_ticks_to_usec(adap, v);
9779         }
9780 }
9781
9782 /**
9783  * t4_get_tx_sched - get the configuration of a Tx HW traffic scheduler
9784  * @adap: the adapter
9785  * @sched: the scheduler index
9786  * @kbps: the byte rate in Kbps
9787  * @ipg: the interpacket delay in tenths of nanoseconds
9788  * @sleep_ok: if true we may sleep while awaiting command completion
9789  *
9790  * Return the current configuration of a HW Tx scheduler.
9791  */
9792 void t4_get_tx_sched(struct adapter *adap, unsigned int sched,
9793                      unsigned int *kbps, unsigned int *ipg, bool sleep_ok)
9794 {
9795         unsigned int v, addr, bpt, cpt;
9796
9797         if (kbps) {
9798                 addr = TP_TX_MOD_Q1_Q0_RATE_LIMIT_A - sched / 2;
9799                 t4_tp_tm_pio_read(adap, &v, 1, addr, sleep_ok);
9800                 if (sched & 1)
9801                         v >>= 16;
9802                 bpt = (v >> 8) & 0xff;
9803                 cpt = v & 0xff;
9804                 if (!cpt) {
9805                         *kbps = 0;      /* scheduler disabled */
9806                 } else {
9807                         v = (adap->params.vpd.cclk * 1000) / cpt; /* ticks/s */
9808                         *kbps = (v * bpt) / 125;
9809                 }
9810         }
9811         if (ipg) {
9812                 addr = TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR_A - sched / 2;
9813                 t4_tp_tm_pio_read(adap, &v, 1, addr, sleep_ok);
9814                 if (sched & 1)
9815                         v >>= 16;
9816                 v &= 0xffff;
9817                 *ipg = (10000 * v) / core_ticks_per_usec(adap);
9818         }
9819 }
9820
9821 /* t4_sge_ctxt_rd - read an SGE context through FW
9822  * @adap: the adapter
9823  * @mbox: mailbox to use for the FW command
9824  * @cid: the context id
9825  * @ctype: the context type
9826  * @data: where to store the context data
9827  *
9828  * Issues a FW command through the given mailbox to read an SGE context.
9829  */
9830 int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid,
9831                    enum ctxt_type ctype, u32 *data)
9832 {
9833         struct fw_ldst_cmd c;
9834         int ret;
9835
9836         if (ctype == CTXT_FLM)
9837                 ret = FW_LDST_ADDRSPC_SGE_FLMC;
9838         else
9839                 ret = FW_LDST_ADDRSPC_SGE_CONMC;
9840
9841         memset(&c, 0, sizeof(c));
9842         c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
9843                                         FW_CMD_REQUEST_F | FW_CMD_READ_F |
9844                                         FW_LDST_CMD_ADDRSPACE_V(ret));
9845         c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
9846         c.u.idctxt.physid = cpu_to_be32(cid);
9847
9848         ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
9849         if (ret == 0) {
9850                 data[0] = be32_to_cpu(c.u.idctxt.ctxt_data0);
9851                 data[1] = be32_to_cpu(c.u.idctxt.ctxt_data1);
9852                 data[2] = be32_to_cpu(c.u.idctxt.ctxt_data2);
9853                 data[3] = be32_to_cpu(c.u.idctxt.ctxt_data3);
9854                 data[4] = be32_to_cpu(c.u.idctxt.ctxt_data4);
9855                 data[5] = be32_to_cpu(c.u.idctxt.ctxt_data5);
9856         }
9857         return ret;
9858 }
9859
9860 /**
9861  * t4_sge_ctxt_rd_bd - read an SGE context bypassing FW
9862  * @adap: the adapter
9863  * @cid: the context id
9864  * @ctype: the context type
9865  * @data: where to store the context data
9866  *
9867  * Reads an SGE context directly, bypassing FW.  This is only for
9868  * debugging when FW is unavailable.
9869  */
9870 int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid,
9871                       enum ctxt_type ctype, u32 *data)
9872 {
9873         int i, ret;
9874
9875         t4_write_reg(adap, SGE_CTXT_CMD_A, CTXTQID_V(cid) | CTXTTYPE_V(ctype));
9876         ret = t4_wait_op_done(adap, SGE_CTXT_CMD_A, BUSY_F, 0, 3, 1);
9877         if (!ret)
9878                 for (i = SGE_CTXT_DATA0_A; i <= SGE_CTXT_DATA5_A; i += 4)
9879                         *data++ = t4_read_reg(adap, i);
9880         return ret;
9881 }
9882
9883 int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
9884                     int rateunit, int ratemode, int channel, int class,
9885                     int minrate, int maxrate, int weight, int pktsize)
9886 {
9887         struct fw_sched_cmd cmd;
9888
9889         memset(&cmd, 0, sizeof(cmd));
9890         cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_SCHED_CMD) |
9891                                       FW_CMD_REQUEST_F |
9892                                       FW_CMD_WRITE_F);
9893         cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
9894
9895         cmd.u.params.sc = FW_SCHED_SC_PARAMS;
9896         cmd.u.params.type = type;
9897         cmd.u.params.level = level;
9898         cmd.u.params.mode = mode;
9899         cmd.u.params.ch = channel;
9900         cmd.u.params.cl = class;
9901         cmd.u.params.unit = rateunit;
9902         cmd.u.params.rate = ratemode;
9903         cmd.u.params.min = cpu_to_be32(minrate);
9904         cmd.u.params.max = cpu_to_be32(maxrate);
9905         cmd.u.params.weight = cpu_to_be16(weight);
9906         cmd.u.params.pktsize = cpu_to_be16(pktsize);
9907
9908         return t4_wr_mbox_meat(adapter, adapter->mbox, &cmd, sizeof(cmd),
9909                                NULL, 1);
9910 }
9911
9912 /**
9913  *      t4_i2c_rd - read I2C data from adapter
9914  *      @adap: the adapter
9915  *      @port: Port number if per-port device; <0 if not
9916  *      @devid: per-port device ID or absolute device ID
9917  *      @offset: byte offset into device I2C space
9918  *      @len: byte length of I2C space data
9919  *      @buf: buffer in which to return I2C data
9920  *
9921  *      Reads the I2C data from the indicated device and location.
9922  */
9923 int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port,
9924               unsigned int devid, unsigned int offset,
9925               unsigned int len, u8 *buf)
9926 {
9927         struct fw_ldst_cmd ldst_cmd, ldst_rpl;
9928         unsigned int i2c_max = sizeof(ldst_cmd.u.i2c.data);
9929         int ret = 0;
9930
9931         if (len > I2C_PAGE_SIZE)
9932                 return -EINVAL;
9933
9934         /* Dont allow reads that spans multiple pages */
9935         if (offset < I2C_PAGE_SIZE && offset + len > I2C_PAGE_SIZE)
9936                 return -EINVAL;
9937
9938         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
9939         ldst_cmd.op_to_addrspace =
9940                 cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
9941                             FW_CMD_REQUEST_F |
9942                             FW_CMD_READ_F |
9943                             FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_I2C));
9944         ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
9945         ldst_cmd.u.i2c.pid = (port < 0 ? 0xff : port);
9946         ldst_cmd.u.i2c.did = devid;
9947
9948         while (len > 0) {
9949                 unsigned int i2c_len = (len < i2c_max) ? len : i2c_max;
9950
9951                 ldst_cmd.u.i2c.boffset = offset;
9952                 ldst_cmd.u.i2c.blen = i2c_len;
9953
9954                 ret = t4_wr_mbox(adap, mbox, &ldst_cmd, sizeof(ldst_cmd),
9955                                  &ldst_rpl);
9956                 if (ret)
9957                         break;
9958
9959                 memcpy(buf, ldst_rpl.u.i2c.data, i2c_len);
9960                 offset += i2c_len;
9961                 buf += i2c_len;
9962                 len -= i2c_len;
9963         }
9964
9965         return ret;
9966 }
9967
9968 /**
9969  *      t4_set_vlan_acl - Set a VLAN id for the specified VF
9970  *      @adapter: the adapter
9971  *      @mbox: mailbox to use for the FW command
9972  *      @vf: one of the VFs instantiated by the specified PF
9973  *      @vlan: The vlanid to be set
9974  */
9975 int t4_set_vlan_acl(struct adapter *adap, unsigned int mbox, unsigned int vf,
9976                     u16 vlan)
9977 {
9978         struct fw_acl_vlan_cmd vlan_cmd;
9979         unsigned int enable;
9980
9981         enable = (vlan ? FW_ACL_VLAN_CMD_EN_F : 0);
9982         memset(&vlan_cmd, 0, sizeof(vlan_cmd));
9983         vlan_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_VLAN_CMD) |
9984                                          FW_CMD_REQUEST_F |
9985                                          FW_CMD_WRITE_F |
9986                                          FW_CMD_EXEC_F |
9987                                          FW_ACL_VLAN_CMD_PFN_V(adap->pf) |
9988                                          FW_ACL_VLAN_CMD_VFN_V(vf));
9989         vlan_cmd.en_to_len16 = cpu_to_be32(enable | FW_LEN16(vlan_cmd));
9990         /* Drop all packets that donot match vlan id */
9991         vlan_cmd.dropnovlan_fm = FW_ACL_VLAN_CMD_FM_F;
9992         if (enable != 0) {
9993                 vlan_cmd.nvlan = 1;
9994                 vlan_cmd.vlanid[0] = cpu_to_be16(vlan);
9995         }
9996
9997         return t4_wr_mbox(adap, adap->mbox, &vlan_cmd, sizeof(vlan_cmd), NULL);
9998 }