drivers/tty/serial/8250/8250_bcm7271.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2020, Broadcom */
   3 /*
   4  * 8250-core based driver for Broadcom ns16550a UARTs
   5  *
   6  * This driver uses the standard 8250 driver core but adds additional
   7  * optional features including the ability to use a baud rate clock
   8  * mux for more accurate high speed baud rate selection and also
   9  * an optional DMA engine.
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/tty.h>
  16 #include <linux/errno.h>
  17 #include <linux/device.h>
  18 #include <linux/io.h>
  19 #include <linux/of.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/tty_flip.h>
  22 #include <linux/delay.h>
  23 #include <linux/clk.h>
  24 #include <linux/debugfs.h>
  25 #include <linux/units.h>
  26
  27 #include "8250.h"
  28
  29 /* Register definitions for UART DMA block. Version 1.1 or later. */
  30 #define UDMA_ARB_RX             0x00
  31 #define UDMA_ARB_TX             0x04
  32 #define         UDMA_ARB_REQ                            0x00000001
  33 #define         UDMA_ARB_GRANT                          0x00000002
  34
  35 #define UDMA_RX_REVISION        0x00
  36 #define UDMA_RX_REVISION_REQUIRED                       0x00000101
  37 #define UDMA_RX_CTRL            0x04
  38 #define         UDMA_RX_CTRL_BUF_CLOSE_MODE             0x00010000
  39 #define         UDMA_RX_CTRL_MASK_WR_DONE               0x00008000
  40 #define         UDMA_RX_CTRL_ENDIAN_OVERRIDE            0x00004000
  41 #define         UDMA_RX_CTRL_ENDIAN                     0x00002000
  42 #define         UDMA_RX_CTRL_OE_IS_ERR                  0x00001000
  43 #define         UDMA_RX_CTRL_PE_IS_ERR                  0x00000800
  44 #define         UDMA_RX_CTRL_FE_IS_ERR                  0x00000400
  45 #define         UDMA_RX_CTRL_NUM_BUF_USED_MASK          0x000003c0
  46 #define         UDMA_RX_CTRL_NUM_BUF_USED_SHIFT 6
  47 #define         UDMA_RX_CTRL_BUF_CLOSE_CLK_SEL_SYS      0x00000020
  48 #define         UDMA_RX_CTRL_BUF_CLOSE_ENA              0x00000010
  49 #define         UDMA_RX_CTRL_TIMEOUT_CLK_SEL_SYS        0x00000008
  50 #define         UDMA_RX_CTRL_TIMEOUT_ENA                0x00000004
  51 #define         UDMA_RX_CTRL_ABORT                      0x00000002
  52 #define         UDMA_RX_CTRL_ENA                        0x00000001
  53 #define UDMA_RX_STATUS          0x08
  54 #define         UDMA_RX_STATUS_ACTIVE_BUF_MASK          0x0000000f
  55 #define UDMA_RX_TRANSFER_LEN    0x0c
  56 #define UDMA_RX_TRANSFER_TOTAL  0x10
  57 #define UDMA_RX_BUFFER_SIZE     0x14
  58 #define UDMA_RX_SRC_ADDR        0x18
  59 #define UDMA_RX_TIMEOUT         0x1c
  60 #define UDMA_RX_BUFFER_CLOSE    0x20
  61 #define UDMA_RX_BLOCKOUT_COUNTER 0x24
  62 #define UDMA_RX_BUF0_PTR_LO     0x28
  63 #define UDMA_RX_BUF0_PTR_HI     0x2c
  64 #define UDMA_RX_BUF0_STATUS     0x30
  65 #define         UDMA_RX_BUFX_STATUS_OVERRUN_ERR         0x00000010
  66 #define         UDMA_RX_BUFX_STATUS_FRAME_ERR           0x00000008
  67 #define         UDMA_RX_BUFX_STATUS_PARITY_ERR          0x00000004
  68 #define         UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED       0x00000002
  69 #define         UDMA_RX_BUFX_STATUS_DATA_RDY            0x00000001
  70 #define UDMA_RX_BUF0_DATA_LEN   0x34
  71 #define UDMA_RX_BUF1_PTR_LO     0x38
  72 #define UDMA_RX_BUF1_PTR_HI     0x3c
  73 #define UDMA_RX_BUF1_STATUS     0x40
  74 #define UDMA_RX_BUF1_DATA_LEN   0x44
  75
  76 #define UDMA_TX_REVISION        0x00
  77 #define UDMA_TX_REVISION_REQUIRED                       0x00000101
  78 #define UDMA_TX_CTRL            0x04
  79 #define         UDMA_TX_CTRL_ENDIAN_OVERRIDE            0x00000080
  80 #define         UDMA_TX_CTRL_ENDIAN                     0x00000040
  81 #define         UDMA_TX_CTRL_NUM_BUF_USED_MASK          0x00000030
  82 #define         UDMA_TX_CTRL_NUM_BUF_USED_1             0x00000010
  83 #define         UDMA_TX_CTRL_ABORT                      0x00000002
  84 #define         UDMA_TX_CTRL_ENA                        0x00000001
  85 #define UDMA_TX_DST_ADDR        0x08
  86 #define UDMA_TX_BLOCKOUT_COUNTER 0x10
  87 #define UDMA_TX_TRANSFER_LEN    0x14
  88 #define UDMA_TX_TRANSFER_TOTAL  0x18
  89 #define UDMA_TX_STATUS          0x20
  90 #define UDMA_TX_BUF0_PTR_LO     0x24
  91 #define UDMA_TX_BUF0_PTR_HI     0x28
  92 #define UDMA_TX_BUF0_STATUS     0x2c
  93 #define         UDMA_TX_BUFX_LAST                       0x00000002
  94 #define         UDMA_TX_BUFX_EMPTY                      0x00000001
  95 #define UDMA_TX_BUF0_DATA_LEN   0x30
  96 #define UDMA_TX_BUF0_DATA_SENT  0x34
  97 #define UDMA_TX_BUF1_PTR_LO     0x38
  98
  99 #define UDMA_INTR_STATUS        0x00
 100 #define         UDMA_INTR_ARB_TX_GRANT                  0x00040000
 101 #define         UDMA_INTR_ARB_RX_GRANT                  0x00020000
 102 #define         UDMA_INTR_TX_ALL_EMPTY                  0x00010000
 103 #define         UDMA_INTR_TX_EMPTY_BUF1                 0x00008000
 104 #define         UDMA_INTR_TX_EMPTY_BUF0                 0x00004000
 105 #define         UDMA_INTR_TX_ABORT                      0x00002000
 106 #define         UDMA_INTR_TX_DONE                       0x00001000
 107 #define         UDMA_INTR_RX_ERROR                      0x00000800
 108 #define         UDMA_INTR_RX_TIMEOUT                    0x00000400
 109 #define         UDMA_INTR_RX_READY_BUF7                 0x00000200
 110 #define         UDMA_INTR_RX_READY_BUF6                 0x00000100
 111 #define         UDMA_INTR_RX_READY_BUF5                 0x00000080
 112 #define         UDMA_INTR_RX_READY_BUF4                 0x00000040
 113 #define         UDMA_INTR_RX_READY_BUF3                 0x00000020
 114 #define         UDMA_INTR_RX_READY_BUF2                 0x00000010
 115 #define         UDMA_INTR_RX_READY_BUF1                 0x00000008
 116 #define         UDMA_INTR_RX_READY_BUF0                 0x00000004
 117 #define         UDMA_INTR_RX_READY_MASK                 0x000003fc
 118 #define         UDMA_INTR_RX_READY_SHIFT                2
 119 #define         UDMA_INTR_RX_ABORT                      0x00000002
 120 #define         UDMA_INTR_RX_DONE                       0x00000001
 121 #define UDMA_INTR_SET           0x04
 122 #define UDMA_INTR_CLEAR         0x08
 123 #define UDMA_INTR_MASK_STATUS   0x0c
 124 #define UDMA_INTR_MASK_SET      0x10
 125 #define UDMA_INTR_MASK_CLEAR    0x14
 126
 127
 128 #define UDMA_RX_INTERRUPTS ( \
 129         UDMA_INTR_RX_ERROR | \
 130         UDMA_INTR_RX_TIMEOUT | \
 131         UDMA_INTR_RX_READY_BUF0 | \
 132         UDMA_INTR_RX_READY_BUF1 | \
 133         UDMA_INTR_RX_READY_BUF2 | \
 134         UDMA_INTR_RX_READY_BUF3 | \
 135         UDMA_INTR_RX_READY_BUF4 | \
 136         UDMA_INTR_RX_READY_BUF5 | \
 137         UDMA_INTR_RX_READY_BUF6 | \
 138         UDMA_INTR_RX_READY_BUF7 | \
 139         UDMA_INTR_RX_ABORT | \
 140         UDMA_INTR_RX_DONE)
 141
 142 #define UDMA_RX_ERR_INTERRUPTS ( \
 143         UDMA_INTR_RX_ERROR | \
 144         UDMA_INTR_RX_TIMEOUT | \
 145         UDMA_INTR_RX_ABORT | \
 146         UDMA_INTR_RX_DONE)
 147
 148 #define UDMA_TX_INTERRUPTS ( \
 149         UDMA_INTR_TX_ABORT | \
 150         UDMA_INTR_TX_DONE)
 151
 152 #define UDMA_IS_RX_INTERRUPT(status) ((status) & UDMA_RX_INTERRUPTS)
 153 #define UDMA_IS_TX_INTERRUPT(status) ((status) & UDMA_TX_INTERRUPTS)
 154
 155
 156 /* Current devices have 8 sets of RX buffer registers */
 157 #define UDMA_RX_BUFS_COUNT      8
 158 #define UDMA_RX_BUFS_REG_OFFSET (UDMA_RX_BUF1_PTR_LO - UDMA_RX_BUF0_PTR_LO)
 159 #define UDMA_RX_BUFx_PTR_LO(x)  (UDMA_RX_BUF0_PTR_LO + \
 160                                  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 161 #define UDMA_RX_BUFx_PTR_HI(x)  (UDMA_RX_BUF0_PTR_HI + \
 162                                  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 163 #define UDMA_RX_BUFx_STATUS(x)  (UDMA_RX_BUF0_STATUS + \
 164                                  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 165 #define UDMA_RX_BUFx_DATA_LEN(x) (UDMA_RX_BUF0_DATA_LEN + \
 166                                   ((x) * UDMA_RX_BUFS_REG_OFFSET))
 167
 168 /* Current devices have 2 sets of TX buffer registers */
 169 #define UDMA_TX_BUFS_COUNT      2
 170 #define UDMA_TX_BUFS_REG_OFFSET (UDMA_TX_BUF1_PTR_LO - UDMA_TX_BUF0_PTR_LO)
 171 #define UDMA_TX_BUFx_PTR_LO(x)  (UDMA_TX_BUF0_PTR_LO + \
 172                                  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 173 #define UDMA_TX_BUFx_PTR_HI(x)  (UDMA_TX_BUF0_PTR_HI + \
 174                                  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 175 #define UDMA_TX_BUFx_STATUS(x)  (UDMA_TX_BUF0_STATUS + \
 176                                  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 177 #define UDMA_TX_BUFx_DATA_LEN(x) (UDMA_TX_BUF0_DATA_LEN + \
 178                                   ((x) * UDMA_TX_BUFS_REG_OFFSET))
 179 #define UDMA_TX_BUFx_DATA_SENT(x) (UDMA_TX_BUF0_DATA_SENT + \
 180                                    ((x) * UDMA_TX_BUFS_REG_OFFSET))
 181 #define REGS_8250 0
 182 #define REGS_DMA_RX 1
 183 #define REGS_DMA_TX 2
 184 #define REGS_DMA_ISR 3
 185 #define REGS_DMA_ARB 4
 186 #define REGS_MAX 5
 187
 188 #define TX_BUF_SIZE 4096
 189 #define RX_BUF_SIZE 4096
 190 #define RX_BUFS_COUNT 2
 191
 192 static const u32 brcmstb_rate_table[] = {
 193         81 * HZ_PER_MHZ,
 194         108 * HZ_PER_MHZ,
 195         64 * HZ_PER_MHZ,                /* Actually 64285715 for some chips */
 196         48 * HZ_PER_MHZ,
 197 };
 198
 199 static const u32 brcmstb_rate_table_7278[] = {
 200         81 * HZ_PER_MHZ,
 201         108 * HZ_PER_MHZ,
 202         0,
 203         48 * HZ_PER_MHZ,
 204 };
 205
 206 struct brcmuart_priv {
 207         int             line;
 208         struct clk      *baud_mux_clk;
 209         unsigned long   default_mux_rate;
 210         u32             real_rates[ARRAY_SIZE(brcmstb_rate_table)];
 211         const u32       *rate_table;
 212         ktime_t         char_wait;
 213         struct uart_port *up;
 214         struct hrtimer  hrt;
 215         bool            shutdown;
 216         bool            dma_enabled;
 217         struct uart_8250_dma dma;
 218         void __iomem    *regs[REGS_MAX];
 219         dma_addr_t      rx_addr;
 220         void            *rx_bufs;
 221         size_t          rx_size;
 222         int             rx_next_buf;
 223         dma_addr_t      tx_addr;
 224         void            *tx_buf;
 225         size_t          tx_size;
 226         bool            tx_running;
 227         bool            rx_running;
 228         struct dentry   *debugfs_dir;
 229
 230         /* stats exposed through debugfs */
 231         u64             dma_rx_partial_buf;
 232         u64             dma_rx_full_buf;
 233         u32             rx_bad_timeout_late_char;
 234         u32             rx_bad_timeout_no_char;
 235         u32             rx_missing_close_timeout;
 236         u32             rx_err;
 237         u32             rx_timeout;
 238         u32             rx_abort;
 239         u32             saved_mctrl;
 240 };
 241
 242 static struct dentry *brcmuart_debugfs_root;
 243
 244 /*
 245  * Register access routines
 246  */
 247 static u32 udma_readl(struct brcmuart_priv *priv,
 248                 int reg_type, int offset)
 249 {
 250         return readl(priv->regs[reg_type] + offset);
 251 }
 252
 253 static void udma_writel(struct brcmuart_priv *priv,
 254                         int reg_type, int offset, u32 value)
 255 {
 256         writel(value, priv->regs[reg_type] + offset);
 257 }
 258
 259 static void udma_set(struct brcmuart_priv *priv,
 260                 int reg_type, int offset, u32 bits)
 261 {
 262         void __iomem *reg = priv->regs[reg_type] + offset;
 263         u32 value;
 264
 265         value = readl(reg);
 266         value |= bits;
 267         writel(value, reg);
 268 }
 269
 270 static void udma_unset(struct brcmuart_priv *priv,
 271                 int reg_type, int offset, u32 bits)
 272 {
 273         void __iomem *reg = priv->regs[reg_type] + offset;
 274         u32 value;
 275
 276         value = readl(reg);
 277         value &= ~bits;
 278         writel(value, reg);
 279 }
 280
 281 /*
 282  * The UART DMA engine hardware can be used by multiple UARTS, but
 283  * only one at a time. Sharing is not currently supported so
 284  * the first UART to request the DMA engine will get it and any
 285  * subsequent requests by other UARTS will fail.
 286  */
 287 static int brcmuart_arbitration(struct brcmuart_priv *priv, bool acquire)
 288 {
 289         u32 rx_grant;
 290         u32 tx_grant;
 291         int waits;
 292         int ret = 0;
 293
 294         if (acquire) {
 295                 udma_set(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
 296                 udma_set(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
 297
 298                 waits = 1;
 299                 while (1) {
 300                         rx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_RX);
 301                         tx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_TX);
 302                         if (rx_grant & tx_grant & UDMA_ARB_GRANT)
 303                                 return 0;
 304                         if (waits-- == 0)
 305                                 break;
 306                         msleep(1);
 307                 }
 308                 ret = 1;
 309         }
 310
 311         udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
 312         udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
 313         return ret;
 314 }
 315
 316 static void brcmuart_init_dma_hardware(struct brcmuart_priv *priv)
 317 {
 318         u32 daddr;
 319         u32 value;
 320         int x;
 321
 322         /* Start with all interrupts disabled */
 323         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, 0xffffffff);
 324
 325         udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_SIZE, RX_BUF_SIZE);
 326
 327         /*
 328          * Setup buffer close to happen when 32 character times have
 329          * elapsed since the last character was received.
 330          */
 331         udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_CLOSE, 16*10*32);
 332         value = (RX_BUFS_COUNT << UDMA_RX_CTRL_NUM_BUF_USED_SHIFT)
 333                 | UDMA_RX_CTRL_BUF_CLOSE_MODE
 334                 | UDMA_RX_CTRL_BUF_CLOSE_ENA;
 335         udma_writel(priv, REGS_DMA_RX, UDMA_RX_CTRL, value);
 336
 337         udma_writel(priv, REGS_DMA_RX, UDMA_RX_BLOCKOUT_COUNTER, 0);
 338         daddr = priv->rx_addr;
 339         for (x = 0; x < RX_BUFS_COUNT; x++) {
 340
 341                 /* Set RX transfer length to 0 for unknown */
 342                 udma_writel(priv, REGS_DMA_RX, UDMA_RX_TRANSFER_LEN, 0);
 343
 344                 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_LO(x),
 345                             lower_32_bits(daddr));
 346                 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_HI(x),
 347                             upper_32_bits(daddr));
 348                 daddr += RX_BUF_SIZE;
 349         }
 350
 351         daddr = priv->tx_addr;
 352         udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_LO(0),
 353                     lower_32_bits(daddr));
 354         udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_HI(0),
 355                     upper_32_bits(daddr));
 356         udma_writel(priv, REGS_DMA_TX, UDMA_TX_CTRL,
 357                     UDMA_TX_CTRL_NUM_BUF_USED_1);
 358
 359         /* clear all interrupts then enable them */
 360         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, 0xffffffff);
 361         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
 362                 UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
 363
 364 }
 365
 366 static void start_rx_dma(struct uart_8250_port *p)
 367 {
 368         struct brcmuart_priv *priv = p->port.private_data;
 369         int x;
 370
 371         udma_unset(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
 372
 373         /* Clear the RX ready bit for all buffers */
 374         for (x = 0; x < RX_BUFS_COUNT; x++)
 375                 udma_unset(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(x),
 376                         UDMA_RX_BUFX_STATUS_DATA_RDY);
 377
 378         /* always start with buffer 0 */
 379         udma_unset(priv, REGS_DMA_RX, UDMA_RX_STATUS,
 380                    UDMA_RX_STATUS_ACTIVE_BUF_MASK);
 381         priv->rx_next_buf = 0;
 382
 383         udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
 384         priv->rx_running = true;
 385 }
 386
 387 static void stop_rx_dma(struct uart_8250_port *p)
 388 {
 389         struct brcmuart_priv *priv = p->port.private_data;
 390
 391         /* If RX is running, set the RX ABORT */
 392         if (priv->rx_running)
 393                 udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ABORT);
 394 }
 395
 396 static int stop_tx_dma(struct uart_8250_port *p)
 397 {
 398         struct brcmuart_priv *priv = p->port.private_data;
 399         u32 value;
 400
 401         /* If TX is running, set the TX ABORT */
 402         value = udma_readl(priv, REGS_DMA_TX, UDMA_TX_CTRL);
 403         if (value & UDMA_TX_CTRL_ENA)
 404                 udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ABORT);
 405         priv->tx_running = false;
 406         return 0;
 407 }
 408
 409 /*
 410  * NOTE: printk's in this routine will hang the system if this is
 411  * the console tty
 412  */
 413 static int brcmuart_tx_dma(struct uart_8250_port *p)
 414 {
 415         struct brcmuart_priv *priv = p->port.private_data;
 416         struct circ_buf *xmit = &p->port.state->xmit;
 417         u32 tx_size;
 418
 419         if (uart_tx_stopped(&p->port) || priv->tx_running ||
 420                 uart_circ_empty(xmit)) {
 421                 return 0;
 422         }
 423         tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 424
 425         priv->dma.tx_err = 0;
 426         memcpy(priv->tx_buf, &xmit->buf[xmit->tail], tx_size);
 427         uart_xmit_advance(&p->port, tx_size);
 428
 429         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 430                 uart_write_wakeup(&p->port);
 431
 432         udma_writel(priv, REGS_DMA_TX, UDMA_TX_TRANSFER_LEN, tx_size);
 433         udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUF0_DATA_LEN, tx_size);
 434         udma_unset(priv, REGS_DMA_TX, UDMA_TX_BUF0_STATUS, UDMA_TX_BUFX_EMPTY);
 435         udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ENA);
 436         priv->tx_running = true;
 437
 438         return 0;
 439 }
 440
 441 static void brcmuart_rx_buf_done_isr(struct uart_port *up, int index)
 442 {
 443         struct brcmuart_priv *priv = up->private_data;
 444         struct tty_port *tty_port = &up->state->port;
 445         u32 status;
 446         u32 length;
 447         u32 copied;
 448
 449         /* Make sure we're still in sync with the hardware */
 450         status = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(index));
 451         length = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_DATA_LEN(index));
 452
 453         if ((status & UDMA_RX_BUFX_STATUS_DATA_RDY) == 0) {
 454                 dev_err(up->dev, "RX done interrupt but DATA_RDY not found\n");
 455                 return;
 456         }
 457         if (status & (UDMA_RX_BUFX_STATUS_OVERRUN_ERR |
 458                       UDMA_RX_BUFX_STATUS_FRAME_ERR |
 459                       UDMA_RX_BUFX_STATUS_PARITY_ERR)) {
 460                 if (status & UDMA_RX_BUFX_STATUS_OVERRUN_ERR) {
 461                         up->icount.overrun++;
 462                         dev_warn(up->dev, "RX OVERRUN Error\n");
 463                 }
 464                 if (status & UDMA_RX_BUFX_STATUS_FRAME_ERR) {
 465                         up->icount.frame++;
 466                         dev_warn(up->dev, "RX FRAMING Error\n");
 467                 }
 468                 if (status & UDMA_RX_BUFX_STATUS_PARITY_ERR) {
 469                         up->icount.parity++;
 470                         dev_warn(up->dev, "RX PARITY Error\n");
 471                 }
 472         }
 473         copied = (u32)tty_insert_flip_string(
 474                 tty_port,
 475                 priv->rx_bufs + (index * RX_BUF_SIZE),
 476                 length);
 477         if (copied != length) {
 478                 dev_warn(up->dev, "Flip buffer overrun of %d bytes\n",
 479                          length - copied);
 480                 up->icount.overrun += length - copied;
 481         }
 482         up->icount.rx += length;
 483         if (status & UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED)
 484                 priv->dma_rx_partial_buf++;
 485         else if (length != RX_BUF_SIZE)
 486                 /*
 487                  * This is a bug in the controller that doesn't cause
 488                  * any problems but will be fixed in the future.
 489                  */
 490                 priv->rx_missing_close_timeout++;
 491         else
 492                 priv->dma_rx_full_buf++;
 493
 494         tty_flip_buffer_push(tty_port);
 495 }
 496
 497 static void brcmuart_rx_isr(struct uart_port *up, u32 rx_isr)
 498 {
 499         struct brcmuart_priv *priv = up->private_data;
 500         struct device *dev = up->dev;
 501         u32 rx_done_isr;
 502         u32 check_isr;
 503
 504         rx_done_isr = (rx_isr & UDMA_INTR_RX_READY_MASK);
 505         while (rx_done_isr) {
 506                 check_isr = UDMA_INTR_RX_READY_BUF0 << priv->rx_next_buf;
 507                 if (check_isr & rx_done_isr) {
 508                         brcmuart_rx_buf_done_isr(up, priv->rx_next_buf);
 509                 } else {
 510                         dev_err(dev,
 511                                 "RX buffer ready out of sequence, restarting RX DMA\n");
 512                         start_rx_dma(up_to_u8250p(up));
 513                         break;
 514                 }
 515                 if (rx_isr & UDMA_RX_ERR_INTERRUPTS) {
 516                         if (rx_isr & UDMA_INTR_RX_ERROR)
 517                                 priv->rx_err++;
 518                         if (rx_isr & UDMA_INTR_RX_TIMEOUT) {
 519                                 priv->rx_timeout++;
 520                                 dev_err(dev, "RX TIMEOUT Error\n");
 521                         }
 522                         if (rx_isr & UDMA_INTR_RX_ABORT)
 523                                 priv->rx_abort++;
 524                         priv->rx_running = false;
 525                 }
 526                 /* If not ABORT, re-enable RX buffer */
 527                 if (!(rx_isr & UDMA_INTR_RX_ABORT))
 528                         udma_unset(priv, REGS_DMA_RX,
 529                                    UDMA_RX_BUFx_STATUS(priv->rx_next_buf),
 530                                    UDMA_RX_BUFX_STATUS_DATA_RDY);
 531                 rx_done_isr &= ~check_isr;
 532                 priv->rx_next_buf++;
 533                 if (priv->rx_next_buf == RX_BUFS_COUNT)
 534                         priv->rx_next_buf = 0;
 535         }
 536 }
 537
 538 static void brcmuart_tx_isr(struct uart_port *up, u32 isr)
 539 {
 540         struct brcmuart_priv *priv = up->private_data;
 541         struct device *dev = up->dev;
 542         struct uart_8250_port *port_8250 = up_to_u8250p(up);
 543         struct circ_buf *xmit = &port_8250->port.state->xmit;
 544
 545         if (isr & UDMA_INTR_TX_ABORT) {
 546                 if (priv->tx_running)
 547                         dev_err(dev, "Unexpected TX_ABORT interrupt\n");
 548                 return;
 549         }
 550         priv->tx_running = false;
 551         if (!uart_circ_empty(xmit) && !uart_tx_stopped(up))
 552                 brcmuart_tx_dma(port_8250);
 553 }
 554
 555 static irqreturn_t brcmuart_isr(int irq, void *dev_id)
 556 {
 557         struct uart_port *up = dev_id;
 558         struct device *dev = up->dev;
 559         struct brcmuart_priv *priv = up->private_data;
 560         unsigned long flags;
 561         u32 interrupts;
 562         u32 rval;
 563         u32 tval;
 564
 565         interrupts = udma_readl(priv, REGS_DMA_ISR, UDMA_INTR_STATUS);
 566         if (interrupts == 0)
 567                 return IRQ_NONE;
 568
 569         uart_port_lock_irqsave(up, &flags);
 570
 571         /* Clear all interrupts */
 572         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, interrupts);
 573
 574         rval = UDMA_IS_RX_INTERRUPT(interrupts);
 575         if (rval)
 576                 brcmuart_rx_isr(up, rval);
 577         tval = UDMA_IS_TX_INTERRUPT(interrupts);
 578         if (tval)
 579                 brcmuart_tx_isr(up, tval);
 580         if ((rval | tval) == 0)
 581                 dev_warn(dev, "Spurious interrupt: 0x%x\n", interrupts);
 582
 583         uart_port_unlock_irqrestore(up, flags);
 584         return IRQ_HANDLED;
 585 }
 586
 587 static int brcmuart_startup(struct uart_port *port)
 588 {
 589         int res;
 590         struct uart_8250_port *up = up_to_u8250p(port);
 591         struct brcmuart_priv *priv = up->port.private_data;
 592
 593         priv->shutdown = false;
 594
 595         /*
 596          * prevent serial8250_do_startup() from allocating non-existent
 597          * DMA resources
 598          */
 599         up->dma = NULL;
 600
 601         res = serial8250_do_startup(port);
 602         if (!priv->dma_enabled)
 603                 return res;
 604         /*
 605          * Disable the Receive Data Interrupt because the DMA engine
 606          * will handle this.
 607          *
 608          * Synchronize UART_IER access against the console.
 609          */
 610         uart_port_lock_irq(port);
 611         up->ier &= ~UART_IER_RDI;
 612         serial_port_out(port, UART_IER, up->ier);
 613         uart_port_unlock_irq(port);
 614
 615         priv->tx_running = false;
 616         priv->dma.rx_dma = NULL;
 617         priv->dma.tx_dma = brcmuart_tx_dma;
 618         up->dma = &priv->dma;
 619
 620         brcmuart_init_dma_hardware(priv);
 621         start_rx_dma(up);
 622         return res;
 623 }
 624
 625 static void brcmuart_shutdown(struct uart_port *port)
 626 {
 627         struct uart_8250_port *up = up_to_u8250p(port);
 628         struct brcmuart_priv *priv = up->port.private_data;
 629         unsigned long flags;
 630
 631         uart_port_lock_irqsave(port, &flags);
 632         priv->shutdown = true;
 633         if (priv->dma_enabled) {
 634                 stop_rx_dma(up);
 635                 stop_tx_dma(up);
 636                 /* disable all interrupts */
 637                 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET,
 638                         UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
 639         }
 640
 641         /*
 642          * prevent serial8250_do_shutdown() from trying to free
 643          * DMA resources that we never alloc'd for this driver.
 644          */
 645         up->dma = NULL;
 646
 647         uart_port_unlock_irqrestore(port, flags);
 648         serial8250_do_shutdown(port);
 649 }
 650
 651 /*
 652  * Not all clocks run at the exact specified rate, so set each requested
 653  * rate and then get the actual rate.
 654  */
 655 static void init_real_clk_rates(struct device *dev, struct brcmuart_priv *priv)
 656 {
 657         int x;
 658         int rc;
 659
 660         priv->default_mux_rate = clk_get_rate(priv->baud_mux_clk);
 661         for (x = 0; x < ARRAY_SIZE(priv->real_rates); x++) {
 662                 if (priv->rate_table[x] == 0) {
 663                         priv->real_rates[x] = 0;
 664                         continue;
 665                 }
 666                 rc = clk_set_rate(priv->baud_mux_clk, priv->rate_table[x]);
 667                 if (rc) {
 668                         dev_err(dev, "Error selecting BAUD MUX clock for %u\n",
 669                                 priv->rate_table[x]);
 670                         priv->real_rates[x] = priv->rate_table[x];
 671                 } else {
 672                         priv->real_rates[x] = clk_get_rate(priv->baud_mux_clk);
 673                 }
 674         }
 675         clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
 676 }
 677
 678 static void set_clock_mux(struct uart_port *up, struct brcmuart_priv *priv,
 679                         u32 baud)
 680 {
 681         u32 percent;
 682         u32 best_percent = UINT_MAX;
 683         u32 quot;
 684         u32 best_quot = 1;
 685         u32 rate;
 686         int best_index = -1;
 687         u64 hires_rate;
 688         u64 hires_baud;
 689         u64 hires_err;
 690         int rc;
 691         int i;
 692         int real_baud;
 693
 694         /* If the Baud Mux Clock was not specified, just return */
 695         if (priv->baud_mux_clk == NULL)
 696                 return;
 697
 698         /* Find the closest match for specified baud */
 699         for (i = 0; i < ARRAY_SIZE(priv->real_rates); i++) {
 700                 if (priv->real_rates[i] == 0)
 701                         continue;
 702                 rate = priv->real_rates[i] / 16;
 703                 quot = DIV_ROUND_CLOSEST(rate, baud);
 704                 if (!quot)
 705                         continue;
 706
 707                 /* increase resolution to get xx.xx percent */
 708                 hires_rate = (u64)rate * 10000;
 709                 hires_baud = (u64)baud * 10000;
 710
 711                 hires_err = div_u64(hires_rate, (u64)quot);
 712
 713                 /* get the delta */
 714                 if (hires_err > hires_baud)
 715                         hires_err = (hires_err - hires_baud);
 716                 else
 717                         hires_err = (hires_baud - hires_err);
 718
 719                 percent = (unsigned long)DIV_ROUND_CLOSEST_ULL(hires_err, baud);
 720                 dev_dbg(up->dev,
 721                         "Baud rate: %u, MUX Clk: %u, Error: %u.%u%%\n",
 722                         baud, priv->real_rates[i], percent / 100,
 723                         percent % 100);
 724                 if (percent < best_percent) {
 725                         best_percent = percent;
 726                         best_index = i;
 727                         best_quot = quot;
 728                 }
 729         }
 730         if (best_index == -1) {
 731                 dev_err(up->dev, "Error, %d BAUD rate is too fast.\n", baud);
 732                 return;
 733         }
 734         rate = priv->real_rates[best_index];
 735         rc = clk_set_rate(priv->baud_mux_clk, rate);
 736         if (rc)
 737                 dev_err(up->dev, "Error selecting BAUD MUX clock\n");
 738
 739         /* Error over 3 percent will cause data errors */
 740         if (best_percent > 300)
 741                 dev_err(up->dev, "Error, baud: %d has %u.%u%% error\n",
 742                         baud, percent / 100, percent % 100);
 743
 744         real_baud = rate / 16 / best_quot;
 745         dev_dbg(up->dev, "Selecting BAUD MUX rate: %u\n", rate);
 746         dev_dbg(up->dev, "Requested baud: %u, Actual baud: %u\n",
 747                 baud, real_baud);
 748
 749         /* calc nanoseconds for 1.5 characters time at the given baud rate */
 750         i = NSEC_PER_SEC / real_baud / 10;
 751         i += (i / 2);
 752         priv->char_wait = ns_to_ktime(i);
 753
 754         up->uartclk = rate;
 755 }
 756
 757 static void brcmstb_set_termios(struct uart_port *up,
 758                                 struct ktermios *termios,
 759                                 const struct ktermios *old)
 760 {
 761         struct uart_8250_port *p8250 = up_to_u8250p(up);
 762         struct brcmuart_priv *priv = up->private_data;
 763
 764         if (priv->dma_enabled)
 765                 stop_rx_dma(p8250);
 766         set_clock_mux(up, priv, tty_termios_baud_rate(termios));
 767         serial8250_do_set_termios(up, termios, old);
 768         if (p8250->mcr & UART_MCR_AFE)
 769                 p8250->port.status |= UPSTAT_AUTOCTS;
 770         if (priv->dma_enabled)
 771                 start_rx_dma(p8250);
 772 }
 773
 774 static int brcmuart_handle_irq(struct uart_port *p)
 775 {
 776         unsigned int iir = serial_port_in(p, UART_IIR);
 777         struct brcmuart_priv *priv = p->private_data;
 778         struct uart_8250_port *up = up_to_u8250p(p);
 779         unsigned int status;
 780         unsigned long flags;
 781         unsigned int ier;
 782         unsigned int mcr;
 783         int handled = 0;
 784
 785         /*
 786          * There's a bug in some 8250 cores where we get a timeout
 787          * interrupt but there is no data ready.
 788          */
 789         if (((iir & UART_IIR_ID) == UART_IIR_RX_TIMEOUT) && !(priv->shutdown)) {
 790                 uart_port_lock_irqsave(p, &flags);
 791                 status = serial_port_in(p, UART_LSR);
 792                 if ((status & UART_LSR_DR) == 0) {
 793
 794                         ier = serial_port_in(p, UART_IER);
 795                         /*
 796                          * if Receive Data Interrupt is enabled and
 797                          * we're uing hardware flow control, deassert
 798                          * RTS and wait for any chars in the pipline to
 799                          * arrive and then check for DR again.
 800                          */
 801                         if ((ier & UART_IER_RDI) && (up->mcr & UART_MCR_AFE)) {
 802                                 ier &= ~(UART_IER_RLSI | UART_IER_RDI);
 803                                 serial_port_out(p, UART_IER, ier);
 804                                 mcr = serial_port_in(p, UART_MCR);
 805                                 mcr &= ~UART_MCR_RTS;
 806                                 serial_port_out(p, UART_MCR, mcr);
 807                                 hrtimer_start(&priv->hrt, priv->char_wait,
 808                                               HRTIMER_MODE_REL);
 809                         } else {
 810                                 serial_port_in(p, UART_RX);
 811                         }
 812
 813                         handled = 1;
 814                 }
 815                 uart_port_unlock_irqrestore(p, flags);
 816                 if (handled)
 817                         return 1;
 818         }
 819         return serial8250_handle_irq(p, iir);
 820 }
 821
 822 static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t)
 823 {
 824         struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt);
 825         struct uart_port *p = priv->up;
 826         struct uart_8250_port *up = up_to_u8250p(p);
 827         unsigned int status;
 828         unsigned long flags;
 829
 830         if (priv->shutdown)
 831                 return HRTIMER_NORESTART;
 832
 833         uart_port_lock_irqsave(p, &flags);
 834         status = serial_port_in(p, UART_LSR);
 835
 836         /*
 837          * If a character did not arrive after the timeout, clear the false
 838          * receive timeout.
 839          */
 840         if ((status & UART_LSR_DR) == 0) {
 841                 serial_port_in(p, UART_RX);
 842                 priv->rx_bad_timeout_no_char++;
 843         } else {
 844                 priv->rx_bad_timeout_late_char++;
 845         }
 846
 847         /* re-enable receive unless upper layer has disabled it */
 848         if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) ==
 849             (UART_IER_RLSI | UART_IER_RDI)) {
 850                 status = serial_port_in(p, UART_IER);
 851                 status |= (UART_IER_RLSI | UART_IER_RDI);
 852                 serial_port_out(p, UART_IER, status);
 853                 status = serial_port_in(p, UART_MCR);
 854                 status |= UART_MCR_RTS;
 855                 serial_port_out(p, UART_MCR, status);
 856         }
 857         uart_port_unlock_irqrestore(p, flags);
 858         return HRTIMER_NORESTART;
 859 }
 860
 861 static const struct of_device_id brcmuart_dt_ids[] = {
 862         {
 863                 .compatible = "brcm,bcm7278-uart",
 864                 .data = brcmstb_rate_table_7278,
 865         },
 866         {
 867                 .compatible = "brcm,bcm7271-uart",
 868                 .data = brcmstb_rate_table,
 869         },
 870         {},
 871 };
 872
 873 MODULE_DEVICE_TABLE(of, brcmuart_dt_ids);
 874
 875 static void brcmuart_free_bufs(struct device *dev, struct brcmuart_priv *priv)
 876 {
 877         if (priv->rx_bufs)
 878                 dma_free_coherent(dev, priv->rx_size, priv->rx_bufs,
 879                                   priv->rx_addr);
 880         if (priv->tx_buf)
 881                 dma_free_coherent(dev, priv->tx_size, priv->tx_buf,
 882                                   priv->tx_addr);
 883 }
 884
 885 static void brcmuart_throttle(struct uart_port *port)
 886 {
 887         struct brcmuart_priv *priv = port->private_data;
 888
 889         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, UDMA_RX_INTERRUPTS);
 890 }
 891
 892 static void brcmuart_unthrottle(struct uart_port *port)
 893 {
 894         struct brcmuart_priv *priv = port->private_data;
 895
 896         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
 897                     UDMA_RX_INTERRUPTS);
 898 }
 899
 900 static int debugfs_stats_show(struct seq_file *s, void *unused)
 901 {
 902         struct brcmuart_priv *priv = s->private;
 903
 904         seq_printf(s, "rx_err:\t\t\t\t%u\n",
 905                    priv->rx_err);
 906         seq_printf(s, "rx_timeout:\t\t\t%u\n",
 907                    priv->rx_timeout);
 908         seq_printf(s, "rx_abort:\t\t\t%u\n",
 909                    priv->rx_abort);
 910         seq_printf(s, "rx_bad_timeout_late_char:\t%u\n",
 911                    priv->rx_bad_timeout_late_char);
 912         seq_printf(s, "rx_bad_timeout_no_char:\t\t%u\n",
 913                    priv->rx_bad_timeout_no_char);
 914         seq_printf(s, "rx_missing_close_timeout:\t%u\n",
 915                    priv->rx_missing_close_timeout);
 916         if (priv->dma_enabled) {
 917                 seq_printf(s, "dma_rx_partial_buf:\t\t%llu\n",
 918                            priv->dma_rx_partial_buf);
 919                 seq_printf(s, "dma_rx_full_buf:\t\t%llu\n",
 920                            priv->dma_rx_full_buf);
 921         }
 922         return 0;
 923 }
 924 DEFINE_SHOW_ATTRIBUTE(debugfs_stats);
 925
 926 static void brcmuart_init_debugfs(struct brcmuart_priv *priv,
 927                                   const char *device)
 928 {
 929         priv->debugfs_dir = debugfs_create_dir(device, brcmuart_debugfs_root);
 930         debugfs_create_file("stats", 0444, priv->debugfs_dir, priv,
 931                             &debugfs_stats_fops);
 932 }
 933
 934
 935 static int brcmuart_probe(struct platform_device *pdev)
 936 {
 937         struct resource *regs;
 938         const struct of_device_id *of_id = NULL;
 939         struct uart_8250_port *new_port;
 940         struct device *dev = &pdev->dev;
 941         struct brcmuart_priv *priv;
 942         struct clk *baud_mux_clk;
 943         struct uart_8250_port up;
 944         void __iomem *membase = NULL;
 945         resource_size_t mapbase = 0;
 946         int ret;
 947         int x;
 948         int dma_irq;
 949         static const char * const reg_names[REGS_MAX] = {
 950                 "uart", "dma_rx", "dma_tx", "dma_intr2", "dma_arb"
 951         };
 952
 953         priv = devm_kzalloc(dev, sizeof(struct brcmuart_priv),
 954                         GFP_KERNEL);
 955         if (!priv)
 956                 return -ENOMEM;
 957
 958         of_id = of_match_node(brcmuart_dt_ids, dev->of_node);
 959         if (!of_id || !of_id->data)
 960                 priv->rate_table = brcmstb_rate_table;
 961         else
 962                 priv->rate_table = of_id->data;
 963
 964         for (x = 0; x < REGS_MAX; x++) {
 965                 regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 966                                                 reg_names[x]);
 967                 if (!regs)
 968                         break;
 969                 priv->regs[x] = devm_ioremap(dev, regs->start,
 970                                              resource_size(regs));
 971                 if (!priv->regs[x])
 972                         return -ENOMEM;
 973                 if (x == REGS_8250) {
 974                         mapbase = regs->start;
 975                         membase = priv->regs[x];
 976                 }
 977         }
 978
 979         /* We should have just the uart base registers or all the registers */
 980         if (x != 1 && x != REGS_MAX)
 981                 return dev_err_probe(dev, -EINVAL, "%s registers not specified\n",
 982                                      reg_names[x]);
 983
 984         /* if the DMA registers were specified, try to enable DMA */
 985         if (x > REGS_DMA_RX) {
 986                 if (brcmuart_arbitration(priv, 1) == 0) {
 987                         u32 txrev = 0;
 988                         u32 rxrev = 0;
 989
 990                         txrev = udma_readl(priv, REGS_DMA_RX, UDMA_RX_REVISION);
 991                         rxrev = udma_readl(priv, REGS_DMA_TX, UDMA_TX_REVISION);
 992                         if ((txrev >= UDMA_TX_REVISION_REQUIRED) &&
 993                                 (rxrev >= UDMA_RX_REVISION_REQUIRED)) {
 994
 995                                 /* Enable the use of the DMA hardware */
 996                                 priv->dma_enabled = true;
 997                         } else {
 998                                 brcmuart_arbitration(priv, 0);
 999                                 dev_err(dev,
1000                                         "Unsupported DMA Hardware Revision\n");
1001                         }
1002                 } else {
1003                         dev_err(dev,
1004                                 "Timeout arbitrating for UART DMA hardware\n");
1005                 }
1006         }
1007
1008         dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");
1009
1010         memset(&up, 0, sizeof(up));
1011         up.port.type = PORT_BCM7271;
1012         up.port.dev = dev;
1013         up.port.mapbase = mapbase;
1014         up.port.membase = membase;
1015         up.port.handle_irq = brcmuart_handle_irq;
1016         up.port.flags = UPF_BOOT_AUTOCONF | UPF_FIXED_PORT | UPF_FIXED_TYPE;
1017         up.port.private_data = priv;
1018
1019         ret = uart_read_port_properties(&up.port);
1020         if (ret)
1021                 goto release_dma;
1022
1023         up.port.regshift = 2;
1024         up.port.iotype = device_is_big_endian(dev) ? UPIO_MEM32BE : UPIO_MEM32;
1025
1026         /* See if a Baud clock has been specified */
1027         baud_mux_clk = devm_clk_get_optional_enabled(dev, "sw_baud");
1028         ret = PTR_ERR_OR_ZERO(baud_mux_clk);
1029         if (ret)
1030                 goto release_dma;
1031         if (baud_mux_clk) {
1032                 dev_dbg(dev, "BAUD MUX clock found\n");
1033
1034                 priv->baud_mux_clk = baud_mux_clk;
1035                 init_real_clk_rates(dev, priv);
1036                 up.port.uartclk = priv->default_mux_rate;
1037         } else {
1038                 dev_dbg(dev, "BAUD MUX clock not specified\n");
1039         }
1040
1041         /* setup HR timer */
1042         hrtimer_init(&priv->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1043         priv->hrt.function = brcmuart_hrtimer_func;
1044
1045         up.port.shutdown = brcmuart_shutdown;
1046         up.port.startup = brcmuart_startup;
1047         up.port.throttle = brcmuart_throttle;
1048         up.port.unthrottle = brcmuart_unthrottle;
1049         up.port.set_termios = brcmstb_set_termios;
1050
1051         if (priv->dma_enabled) {
1052                 priv->rx_size = RX_BUF_SIZE * RX_BUFS_COUNT;
1053                 priv->rx_bufs = dma_alloc_coherent(dev,
1054                                                    priv->rx_size,
1055                                                    &priv->rx_addr, GFP_KERNEL);
1056                 if (!priv->rx_bufs) {
1057                         ret = -ENOMEM;
1058                         goto err;
1059                 }
1060                 priv->tx_size = UART_XMIT_SIZE;
1061                 priv->tx_buf = dma_alloc_coherent(dev,
1062                                                   priv->tx_size,
1063                                                   &priv->tx_addr, GFP_KERNEL);
1064                 if (!priv->tx_buf) {
1065                         ret = -ENOMEM;
1066                         goto err;
1067                 }
1068         }
1069
1070         ret = serial8250_register_8250_port(&up);
1071         if (ret < 0) {
1072                 dev_err_probe(dev, ret, "unable to register 8250 port\n");
1073                 goto err;
1074         }
1075         priv->line = ret;
1076         new_port = serial8250_get_port(ret);
1077         priv->up = &new_port->port;
1078         if (priv->dma_enabled) {
1079                 dma_irq = platform_get_irq_byname(pdev,  "dma");
1080                 if (dma_irq < 0) {
1081                         ret = dev_err_probe(dev, dma_irq, "no IRQ resource info\n");
1082                         goto err1;
1083                 }
1084                 ret = devm_request_irq(dev, dma_irq, brcmuart_isr,
1085                                 IRQF_SHARED, "uart DMA irq", &new_port->port);
1086                 if (ret) {
1087                         dev_err_probe(dev, ret, "unable to register IRQ handler\n");
1088                         goto err1;
1089                 }
1090         }
1091         platform_set_drvdata(pdev, priv);
1092         brcmuart_init_debugfs(priv, dev_name(&pdev->dev));
1093         return 0;
1094
1095 err1:
1096         serial8250_unregister_port(priv->line);
1097 err:
1098         brcmuart_free_bufs(dev, priv);
1099 release_dma:
1100         if (priv->dma_enabled)
1101                 brcmuart_arbitration(priv, 0);
1102         return ret;
1103 }
1104
1105 static void brcmuart_remove(struct platform_device *pdev)
1106 {
1107         struct brcmuart_priv *priv = platform_get_drvdata(pdev);
1108
1109         debugfs_remove_recursive(priv->debugfs_dir);
1110         hrtimer_cancel(&priv->hrt);
1111         serial8250_unregister_port(priv->line);
1112         brcmuart_free_bufs(&pdev->dev, priv);
1113         if (priv->dma_enabled)
1114                 brcmuart_arbitration(priv, 0);
1115 }
1116
1117 static int __maybe_unused brcmuart_suspend(struct device *dev)
1118 {
1119         struct brcmuart_priv *priv = dev_get_drvdata(dev);
1120         struct uart_8250_port *up = serial8250_get_port(priv->line);
1121         struct uart_port *port = &up->port;
1122         unsigned long flags;
1123
1124         /*
1125          * This will prevent resume from enabling RTS before the
1126          *  baud rate has been restored.
1127          */
1128         uart_port_lock_irqsave(port, &flags);
1129         priv->saved_mctrl = port->mctrl;
1130         port->mctrl &= ~TIOCM_RTS;
1131         uart_port_unlock_irqrestore(port, flags);
1132
1133         serial8250_suspend_port(priv->line);
1134         clk_disable_unprepare(priv->baud_mux_clk);
1135
1136         return 0;
1137 }
1138
1139 static int __maybe_unused brcmuart_resume(struct device *dev)
1140 {
1141         struct brcmuart_priv *priv = dev_get_drvdata(dev);
1142         struct uart_8250_port *up = serial8250_get_port(priv->line);
1143         struct uart_port *port = &up->port;
1144         unsigned long flags;
1145         int ret;
1146
1147         ret = clk_prepare_enable(priv->baud_mux_clk);
1148         if (ret)
1149                 dev_err(dev, "Error enabling BAUD MUX clock\n");
1150
1151         /*
1152          * The hardware goes back to it's default after suspend
1153          * so get the "clk" back in sync.
1154          */
1155         ret = clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
1156         if (ret)
1157                 dev_err(dev, "Error restoring default BAUD MUX clock\n");
1158         if (priv->dma_enabled) {
1159                 if (brcmuart_arbitration(priv, 1)) {
1160                         dev_err(dev, "Timeout arbitrating for DMA hardware on resume\n");
1161                         return(-EBUSY);
1162                 }
1163                 brcmuart_init_dma_hardware(priv);
1164                 start_rx_dma(serial8250_get_port(priv->line));
1165         }
1166         serial8250_resume_port(priv->line);
1167
1168         if (priv->saved_mctrl & TIOCM_RTS) {
1169                 /* Restore RTS */
1170                 uart_port_lock_irqsave(port, &flags);
1171                 port->mctrl |= TIOCM_RTS;
1172                 port->ops->set_mctrl(port, port->mctrl);
1173                 uart_port_unlock_irqrestore(port, flags);
1174         }
1175
1176         return 0;
1177 }
1178
1179 static const struct dev_pm_ops brcmuart_dev_pm_ops = {
1180         SET_SYSTEM_SLEEP_PM_OPS(brcmuart_suspend, brcmuart_resume)
1181 };
1182
1183 static struct platform_driver brcmuart_platform_driver = {
1184         .driver = {
1185                 .name   = "bcm7271-uart",
1186                 .pm             = &brcmuart_dev_pm_ops,
1187                 .of_match_table = brcmuart_dt_ids,
1188         },
1189         .probe          = brcmuart_probe,
1190         .remove_new     = brcmuart_remove,
1191 };
1192
1193 static int __init brcmuart_init(void)
1194 {
1195         int ret;
1196
1197         brcmuart_debugfs_root = debugfs_create_dir(
1198                 brcmuart_platform_driver.driver.name, NULL);
1199         ret = platform_driver_register(&brcmuart_platform_driver);
1200         if (ret) {
1201                 debugfs_remove_recursive(brcmuart_debugfs_root);
1202                 return ret;
1203         }
1204
1205         return 0;
1206 }
1207 module_init(brcmuart_init);
1208
1209 static void __exit brcmuart_deinit(void)
1210 {
1211         platform_driver_unregister(&brcmuart_platform_driver);
1212         debugfs_remove_recursive(brcmuart_debugfs_root);
1213 }
1214 module_exit(brcmuart_deinit);
1215
1216 MODULE_AUTHOR("Al Cooper");
1217 MODULE_DESCRIPTION("Broadcom NS16550A compatible serial port driver");
1218 MODULE_LICENSE("GPL v2");