drivers/staging/gasket/apex_driver.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for the Apex chip.
   4  *
   5  * Copyright (C) 2018 Google, Inc.
   6  */
   7
   8 #include <linux/compiler.h>
   9 #include <linux/delay.h>
  10 #include <linux/device.h>
  11 #include <linux/fs.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/module.h>
  15 #include <linux/moduleparam.h>
  16 #include <linux/pci.h>
  17 #include <linux/printk.h>
  18 #include <linux/sched.h>
  19 #include <linux/uaccess.h>
  20
  21 #include "apex.h"
  22
  23 #include "gasket_core.h"
  24 #include "gasket_interrupt.h"
  25 #include "gasket_page_table.h"
  26 #include "gasket_sysfs.h"
  27
  28 /* Constants */
  29 #define APEX_DEVICE_NAME "Apex"
  30 #define APEX_DRIVER_VERSION "1.0"
  31
  32 /* CSRs are in BAR 2. */
  33 #define APEX_BAR_INDEX 2
  34
  35 #define APEX_PCI_VENDOR_ID 0x1ac1
  36 #define APEX_PCI_DEVICE_ID 0x089a
  37
  38 /* Bar Offsets. */
  39 #define APEX_BAR_OFFSET 0
  40 #define APEX_CM_OFFSET 0x1000000
  41
  42 /* The sizes of each Apex BAR 2. */
  43 #define APEX_BAR_BYTES 0x100000
  44 #define APEX_CH_MEM_BYTES (PAGE_SIZE * MAX_NUM_COHERENT_PAGES)
  45
  46 /* The number of user-mappable memory ranges in BAR2 of a Apex chip. */
  47 #define NUM_REGIONS 3
  48
  49 /* The number of nodes in a Apex chip. */
  50 #define NUM_NODES 1
  51
  52 /*
  53  * The total number of entries in the page table. Should match the value read
  54  * from the register APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE.
  55  */
  56 #define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192
  57
  58 #define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */
  59
  60 /* Check reset 120 times */
  61 #define APEX_RESET_RETRY 120
  62 /* Wait 100 ms between checks. Total 12 sec wait maximum. */
  63 #define APEX_RESET_DELAY 100
  64
  65 /* Enumeration of the supported sysfs entries. */
  66 enum sysfs_attribute_type {
  67         ATTR_KERNEL_HIB_PAGE_TABLE_SIZE,
  68         ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE,
  69         ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES,
  70 };
  71
  72 /*
  73  * Register offsets into BAR2 memory.
  74  * Only values necessary for driver implementation are defined.
  75  */
  76 enum apex_bar2_regs {
  77         APEX_BAR2_REG_SCU_BASE = 0x1A300,
  78         APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE = 0x46000,
  79         APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE = 0x46008,
  80         APEX_BAR2_REG_KERNEL_HIB_TRANSLATION_ENABLE = 0x46010,
  81         APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL = 0x46018,
  82         APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL = 0x46020,
  83         APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL = 0x46028,
  84         APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL = 0x46030,
  85         APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL = 0x46038,
  86         APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL = 0x46040,
  87         APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL = 0x46048,
  88         APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE = 0x46050,
  89         APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE_MASK = 0x46058,
  90         APEX_BAR2_REG_KERNEL_HIB_STATUS_BLOCK_DELAY = 0x46060,
  91         APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY0 = 0x46068,
  92         APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY1 = 0x46070,
  93         APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT = 0x46078,
  94         APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT = 0x46080,
  95         APEX_BAR2_REG_KERNEL_WIRE_INT_PENDING_BIT_ARRAY = 0x48778,
  96         APEX_BAR2_REG_KERNEL_WIRE_INT_MASK_ARRAY = 0x48780,
  97         APEX_BAR2_REG_USER_HIB_DMA_PAUSE = 0x486D8,
  98         APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0,
  99         APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000,
 100         APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000,
 101
 102         /* Error registers - Used mostly for debug */
 103         APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0,
 104         APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS = 0x41a0,
 105 };
 106
 107 /* Addresses for packed registers. */
 108 #define APEX_BAR2_REG_AXI_QUIESCE (APEX_BAR2_REG_SCU_BASE + 0x2C)
 109 #define APEX_BAR2_REG_GCB_CLOCK_GATE (APEX_BAR2_REG_SCU_BASE + 0x14)
 110 #define APEX_BAR2_REG_SCU_0 (APEX_BAR2_REG_SCU_BASE + 0xc)
 111 #define APEX_BAR2_REG_SCU_1 (APEX_BAR2_REG_SCU_BASE + 0x10)
 112 #define APEX_BAR2_REG_SCU_2 (APEX_BAR2_REG_SCU_BASE + 0x14)
 113 #define APEX_BAR2_REG_SCU_3 (APEX_BAR2_REG_SCU_BASE + 0x18)
 114 #define APEX_BAR2_REG_SCU_4 (APEX_BAR2_REG_SCU_BASE + 0x1c)
 115 #define APEX_BAR2_REG_SCU_5 (APEX_BAR2_REG_SCU_BASE + 0x20)
 116
 117 #define SCU3_RG_PWR_STATE_OVR_BIT_OFFSET 26
 118 #define SCU3_RG_PWR_STATE_OVR_MASK_WIDTH 2
 119 #define SCU3_CUR_RST_GCB_BIT_MASK 0x10
 120 #define SCU2_RG_RST_GCB_BIT_MASK 0xc
 121
 122 /* Configuration for page table. */
 123 static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = {
 124         {
 125                 .id = 0,
 126                 .mode = GASKET_PAGE_TABLE_MODE_NORMAL,
 127                 .total_entries = APEX_PAGE_TABLE_TOTAL_ENTRIES,
 128                 .base_reg = APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE,
 129                 .extended_reg = APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE,
 130                 .extended_bit = APEX_EXTENDED_SHIFT,
 131         },
 132 };
 133
 134 /* The regions in the BAR2 space that can be mapped into user space. */
 135 static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = {
 136         { 0x40000, 0x1000 },
 137         { 0x44000, 0x1000 },
 138         { 0x48000, 0x1000 },
 139 };
 140
 141 /* Gasket device interrupts enums must be dense (i.e., no empty slots). */
 142 enum apex_interrupt {
 143         APEX_INTERRUPT_INSTR_QUEUE = 0,
 144         APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1,
 145         APEX_INTERRUPT_PARAM_QUEUE = 2,
 146         APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3,
 147         APEX_INTERRUPT_SC_HOST_0 = 4,
 148         APEX_INTERRUPT_SC_HOST_1 = 5,
 149         APEX_INTERRUPT_SC_HOST_2 = 6,
 150         APEX_INTERRUPT_SC_HOST_3 = 7,
 151         APEX_INTERRUPT_TOP_LEVEL_0 = 8,
 152         APEX_INTERRUPT_TOP_LEVEL_1 = 9,
 153         APEX_INTERRUPT_TOP_LEVEL_2 = 10,
 154         APEX_INTERRUPT_TOP_LEVEL_3 = 11,
 155         APEX_INTERRUPT_FATAL_ERR = 12,
 156         APEX_INTERRUPT_COUNT = 13,
 157 };
 158
 159 /* Interrupt descriptors for Apex */
 160 static struct gasket_interrupt_desc apex_interrupts[] = {
 161         {
 162                 APEX_INTERRUPT_INSTR_QUEUE,
 163                 APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL,
 164                 UNPACKED,
 165         },
 166         {
 167                 APEX_INTERRUPT_INPUT_ACTV_QUEUE,
 168                 APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL,
 169                 UNPACKED
 170         },
 171         {
 172                 APEX_INTERRUPT_PARAM_QUEUE,
 173                 APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL,
 174                 UNPACKED
 175         },
 176         {
 177                 APEX_INTERRUPT_OUTPUT_ACTV_QUEUE,
 178                 APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL,
 179                 UNPACKED
 180         },
 181         {
 182                 APEX_INTERRUPT_SC_HOST_0,
 183                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 184                 PACK_0
 185         },
 186         {
 187                 APEX_INTERRUPT_SC_HOST_1,
 188                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 189                 PACK_1
 190         },
 191         {
 192                 APEX_INTERRUPT_SC_HOST_2,
 193                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 194                 PACK_2
 195         },
 196         {
 197                 APEX_INTERRUPT_SC_HOST_3,
 198                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 199                 PACK_3
 200         },
 201         {
 202                 APEX_INTERRUPT_TOP_LEVEL_0,
 203                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 204                 PACK_0
 205         },
 206         {
 207                 APEX_INTERRUPT_TOP_LEVEL_1,
 208                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 209                 PACK_1
 210         },
 211         {
 212                 APEX_INTERRUPT_TOP_LEVEL_2,
 213                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 214                 PACK_2
 215         },
 216         {
 217                 APEX_INTERRUPT_TOP_LEVEL_3,
 218                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 219                 PACK_3
 220         },
 221         {
 222                 APEX_INTERRUPT_FATAL_ERR,
 223                 APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL,
 224                 UNPACKED
 225         },
 226 };
 227
 228 /* Allows device to enter power save upon driver close(). */
 229 static int allow_power_save = 1;
 230
 231 /* Allows SW based clock gating. */
 232 static int allow_sw_clock_gating;
 233
 234 /* Allows HW based clock gating. */
 235 /* Note: this is not mutual exclusive with SW clock gating. */
 236 static int allow_hw_clock_gating = 1;
 237
 238 /* Act as if only GCB is instantiated. */
 239 static int bypass_top_level;
 240
 241 module_param(allow_power_save, int, 0644);
 242 module_param(allow_sw_clock_gating, int, 0644);
 243 module_param(allow_hw_clock_gating, int, 0644);
 244 module_param(bypass_top_level, int, 0644);
 245
 246 /* Check the device status registers and return device status ALIVE or DEAD. */
 247 static int apex_get_status(struct gasket_dev *gasket_dev)
 248 {
 249         /* TODO: Check device status. */
 250         return GASKET_STATUS_ALIVE;
 251 }
 252
 253 /* Enter GCB reset state. */
 254 static int apex_enter_reset(struct gasket_dev *gasket_dev)
 255 {
 256         if (bypass_top_level)
 257                 return 0;
 258
 259         /*
 260          * Software reset:
 261          * Enable sleep mode
 262          *  - Software force GCB idle
 263          *    - Enable GCB idle
 264          */
 265         gasket_read_modify_write_64(gasket_dev, APEX_BAR_INDEX,
 266                                     APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER,
 267                                     0x0, 1, 32);
 268
 269         /*    - Initiate DMA pause */
 270         gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX,
 271                             APEX_BAR2_REG_USER_HIB_DMA_PAUSE);
 272
 273         /*    - Wait for DMA pause complete. */
 274         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 275                                         APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1,
 276                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 277                 dev_err(gasket_dev->dev,
 278                         "DMAs did not quiesce within timeout (%d ms)\n",
 279                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 280                 return -ETIMEDOUT;
 281         }
 282
 283         /*  - Enable GCB reset (0x1 to rg_rst_gcb) */
 284         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 285                                     APEX_BAR2_REG_SCU_2, 0x1, 2, 2);
 286
 287         /*  - Enable GCB clock Gate (0x1 to rg_gated_gcb) */
 288         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 289                                     APEX_BAR2_REG_SCU_2, 0x1, 2, 18);
 290
 291         /*  - Enable GCB memory shut down (0x3 to rg_force_ram_sd) */
 292         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 293                                     APEX_BAR2_REG_SCU_3, 0x3, 2, 14);
 294
 295         /*    - Wait for RAM shutdown. */
 296         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 297                                         APEX_BAR2_REG_SCU_3, BIT(6), BIT(6),
 298                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 299                 dev_err(gasket_dev->dev,
 300                         "RAM did not shut down within timeout (%d ms)\n",
 301                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 302                 return -ETIMEDOUT;
 303         }
 304
 305         return 0;
 306 }
 307
 308 /* Quit GCB reset state. */
 309 static int apex_quit_reset(struct gasket_dev *gasket_dev)
 310 {
 311         u32 val0, val1;
 312
 313         if (bypass_top_level)
 314                 return 0;
 315
 316         /*
 317          * Disable sleep mode:
 318          *  - Disable GCB memory shut down:
 319          *    - b00: Not forced (HW controlled)
 320          *    - b1x: Force disable
 321          */
 322         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 323                                     APEX_BAR2_REG_SCU_3, 0x0, 2, 14);
 324
 325         /*
 326          *  - Disable software clock gate:
 327          *    - b00: Not forced (HW controlled)
 328          *    - b1x: Force disable
 329          */
 330         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 331                                     APEX_BAR2_REG_SCU_2, 0x0, 2, 18);
 332
 333         /*
 334          *  - Disable GCB reset (rg_rst_gcb):
 335          *    - b00: Not forced (HW controlled)
 336          *    - b1x: Force disable = Force not Reset
 337          */
 338         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 339                                     APEX_BAR2_REG_SCU_2, 0x2, 2, 2);
 340
 341         /*    - Wait for RAM enable. */
 342         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 343                                         APEX_BAR2_REG_SCU_3, BIT(6), 0,
 344                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 345                 dev_err(gasket_dev->dev,
 346                         "RAM did not enable within timeout (%d ms)\n",
 347                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 348                 return -ETIMEDOUT;
 349         }
 350
 351         /*    - Wait for Reset complete. */
 352         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 353                                         APEX_BAR2_REG_SCU_3,
 354                                         SCU3_CUR_RST_GCB_BIT_MASK, 0,
 355                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 356                 dev_err(gasket_dev->dev,
 357                         "GCB did not leave reset within timeout (%d ms)\n",
 358                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 359                 return -ETIMEDOUT;
 360         }
 361
 362         if (!allow_hw_clock_gating) {
 363                 val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 364                                           APEX_BAR2_REG_SCU_3);
 365                 /* Inactive and Sleep mode are disabled. */
 366                 gasket_read_modify_write_32(gasket_dev,
 367                                             APEX_BAR_INDEX,
 368                                             APEX_BAR2_REG_SCU_3, 0x3,
 369                                             SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
 370                                             SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
 371                 val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 372                                           APEX_BAR2_REG_SCU_3);
 373                 dev_dbg(gasket_dev->dev,
 374                         "Disallow HW clock gating 0x%x -> 0x%x\n", val0, val1);
 375         } else {
 376                 val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 377                                           APEX_BAR2_REG_SCU_3);
 378                 /* Inactive mode enabled - Sleep mode disabled. */
 379                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 380                                             APEX_BAR2_REG_SCU_3, 2,
 381                                             SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
 382                                             SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
 383                 val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 384                                           APEX_BAR2_REG_SCU_3);
 385                 dev_dbg(gasket_dev->dev, "Allow HW clock gating 0x%x -> 0x%x\n",
 386                         val0, val1);
 387         }
 388
 389         return 0;
 390 }
 391
 392 /* Reset the Apex hardware. Called on final close via device_close_cb. */
 393 static int apex_device_cleanup(struct gasket_dev *gasket_dev)
 394 {
 395         u64 scalar_error;
 396         u64 hib_error;
 397         int ret = 0;
 398
 399         hib_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 400                                        APEX_BAR2_REG_USER_HIB_ERROR_STATUS);
 401         scalar_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 402                                           APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS);
 403
 404         dev_dbg(gasket_dev->dev,
 405                 "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n",
 406                 __func__, gasket_dev, hib_error, scalar_error);
 407
 408         if (allow_power_save)
 409                 ret = apex_enter_reset(gasket_dev);
 410
 411         return ret;
 412 }
 413
 414 /* Determine if GCB is in reset state. */
 415 static bool is_gcb_in_reset(struct gasket_dev *gasket_dev)
 416 {
 417         u32 val = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 418                                      APEX_BAR2_REG_SCU_3);
 419
 420         /* Masks rg_rst_gcb bit of SCU_CTRL_2 */
 421         return (val & SCU3_CUR_RST_GCB_BIT_MASK);
 422 }
 423
 424 /* Reset the hardware, then quit reset.  Called on device open. */
 425 static int apex_reset(struct gasket_dev *gasket_dev)
 426 {
 427         int ret;
 428
 429         if (bypass_top_level)
 430                 return 0;
 431
 432         if (!is_gcb_in_reset(gasket_dev)) {
 433                 /* We are not in reset - toggle the reset bit so as to force
 434                  * re-init of custom block
 435                  */
 436                 dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__);
 437
 438                 ret = apex_enter_reset(gasket_dev);
 439                 if (ret)
 440                         return ret;
 441         }
 442         return apex_quit_reset(gasket_dev);
 443 }
 444
 445 /*
 446  * Check permissions for Apex ioctls.
 447  * Returns true if the current user may execute this ioctl, and false otherwise.
 448  */
 449 static bool apex_ioctl_check_permissions(struct file *filp, uint cmd)
 450 {
 451         return !!(filp->f_mode & FMODE_WRITE);
 452 }
 453
 454 /* Gates or un-gates Apex clock. */
 455 static long apex_clock_gating(struct gasket_dev *gasket_dev,
 456                               struct apex_gate_clock_ioctl __user *argp)
 457 {
 458         struct apex_gate_clock_ioctl ibuf;
 459
 460         if (bypass_top_level || !allow_sw_clock_gating)
 461                 return 0;
 462
 463         if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
 464                 return -EFAULT;
 465
 466         dev_dbg(gasket_dev->dev, "%s %llu\n", __func__, ibuf.enable);
 467
 468         if (ibuf.enable) {
 469                 /* Quiesce AXI, gate GCB clock. */
 470                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 471                                             APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1,
 472                                             16);
 473                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 474                                             APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1,
 475                                             2, 18);
 476         } else {
 477                 /* Un-gate GCB clock, un-quiesce AXI. */
 478                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 479                                             APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0,
 480                                             2, 18);
 481                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 482                                             APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1,
 483                                             16);
 484         }
 485         return 0;
 486 }
 487
 488 /* Apex-specific ioctl handler. */
 489 static long apex_ioctl(struct file *filp, uint cmd, void __user *argp)
 490 {
 491         struct gasket_dev *gasket_dev = filp->private_data;
 492
 493         if (!apex_ioctl_check_permissions(filp, cmd))
 494                 return -EPERM;
 495
 496         switch (cmd) {
 497         case APEX_IOCTL_GATE_CLOCK:
 498                 return apex_clock_gating(gasket_dev, argp);
 499         default:
 500                 return -ENOTTY; /* unknown command */
 501         }
 502 }
 503
 504 /* Display driver sysfs entries. */
 505 static ssize_t sysfs_show(struct device *device, struct device_attribute *attr,
 506                           char *buf)
 507 {
 508         int ret;
 509         struct gasket_dev *gasket_dev;
 510         struct gasket_sysfs_attribute *gasket_attr;
 511         enum sysfs_attribute_type type;
 512
 513         gasket_dev = gasket_sysfs_get_device_data(device);
 514         if (!gasket_dev) {
 515                 dev_err(device, "No Apex device sysfs mapping found\n");
 516                 return -ENODEV;
 517         }
 518
 519         gasket_attr = gasket_sysfs_get_attr(device, attr);
 520         if (!gasket_attr) {
 521                 dev_err(device, "No Apex device sysfs attr data found\n");
 522                 gasket_sysfs_put_device_data(device, gasket_dev);
 523                 return -ENODEV;
 524         }
 525
 526         type = (enum sysfs_attribute_type)gasket_attr->data.attr_type;
 527         switch (type) {
 528         case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE:
 529                 ret = scnprintf(buf, PAGE_SIZE, "%u\n",
 530                                 gasket_page_table_num_entries(
 531                                         gasket_dev->page_table[0]));
 532                 break;
 533         case ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE:
 534                 ret = scnprintf(buf, PAGE_SIZE, "%u\n",
 535                                 gasket_page_table_num_simple_entries(
 536                                         gasket_dev->page_table[0]));
 537                 break;
 538         case ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES:
 539                 ret = scnprintf(buf, PAGE_SIZE, "%u\n",
 540                                 gasket_page_table_num_active_pages(
 541                                         gasket_dev->page_table[0]));
 542                 break;
 543         default:
 544                 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
 545                         attr->attr.name);
 546                 ret = 0;
 547                 break;
 548         }
 549
 550         gasket_sysfs_put_attr(device, gasket_attr);
 551         gasket_sysfs_put_device_data(device, gasket_dev);
 552         return ret;
 553 }
 554
 555 static struct gasket_sysfs_attribute apex_sysfs_attrs[] = {
 556         GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show,
 557                         ATTR_KERNEL_HIB_PAGE_TABLE_SIZE),
 558         GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show,
 559                         ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE),
 560         GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show,
 561                         ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES),
 562         GASKET_END_OF_ATTR_ARRAY
 563 };
 564
 565 /* On device open, perform a core reinit reset. */
 566 static int apex_device_open_cb(struct gasket_dev *gasket_dev)
 567 {
 568         return gasket_reset_nolock(gasket_dev);
 569 }
 570
 571 static const struct pci_device_id apex_pci_ids[] = {
 572         { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 }
 573 };
 574
 575 static void apex_pci_fixup_class(struct pci_dev *pdev)
 576 {
 577         pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class;
 578 }
 579 DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID,
 580                                PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class);
 581
 582 static int apex_pci_probe(struct pci_dev *pci_dev,
 583                           const struct pci_device_id *id)
 584 {
 585         int ret;
 586         ulong page_table_ready, msix_table_ready;
 587         int retries = 0;
 588         struct gasket_dev *gasket_dev;
 589
 590         ret = pci_enable_device(pci_dev);
 591         if (ret) {
 592                 dev_err(&pci_dev->dev, "error enabling PCI device\n");
 593                 return ret;
 594         }
 595
 596         pci_set_master(pci_dev);
 597
 598         ret = gasket_pci_add_device(pci_dev, &gasket_dev);
 599         if (ret) {
 600                 dev_err(&pci_dev->dev, "error adding gasket device\n");
 601                 pci_disable_device(pci_dev);
 602                 return ret;
 603         }
 604
 605         pci_set_drvdata(pci_dev, gasket_dev);
 606         apex_reset(gasket_dev);
 607
 608         while (retries < APEX_RESET_RETRY) {
 609                 page_table_ready =
 610                         gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 611                                            APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT);
 612                 msix_table_ready =
 613                         gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 614                                            APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT);
 615                 if (page_table_ready && msix_table_ready)
 616                         break;
 617                 schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY));
 618                 retries++;
 619         }
 620
 621         if (retries == APEX_RESET_RETRY) {
 622                 if (!page_table_ready)
 623                         dev_err(gasket_dev->dev, "Page table init timed out\n");
 624                 if (!msix_table_ready)
 625                         dev_err(gasket_dev->dev, "MSI-X table init timed out\n");
 626                 ret = -ETIMEDOUT;
 627                 goto remove_device;
 628         }
 629
 630         ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device,
 631                                           apex_sysfs_attrs);
 632         if (ret)
 633                 dev_err(&pci_dev->dev, "error creating device sysfs entries\n");
 634
 635         ret = gasket_enable_device(gasket_dev);
 636         if (ret) {
 637                 dev_err(&pci_dev->dev, "error enabling gasket device\n");
 638                 goto remove_device;
 639         }
 640
 641         /* Place device in low power mode until opened */
 642         if (allow_power_save)
 643                 apex_enter_reset(gasket_dev);
 644
 645         return 0;
 646
 647 remove_device:
 648         gasket_pci_remove_device(pci_dev);
 649         pci_disable_device(pci_dev);
 650         return ret;
 651 }
 652
 653 static void apex_pci_remove(struct pci_dev *pci_dev)
 654 {
 655         struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev);
 656
 657         gasket_disable_device(gasket_dev);
 658         gasket_pci_remove_device(pci_dev);
 659         pci_disable_device(pci_dev);
 660 }
 661
 662 static struct gasket_driver_desc apex_desc = {
 663         .name = "apex",
 664         .driver_version = APEX_DRIVER_VERSION,
 665         .major = 120,
 666         .minor = 0,
 667         .module = THIS_MODULE,
 668         .pci_id_table = apex_pci_ids,
 669
 670         .num_page_tables = NUM_NODES,
 671         .page_table_bar_index = APEX_BAR_INDEX,
 672         .page_table_configs = apex_page_table_configs,
 673         .page_table_extended_bit = APEX_EXTENDED_SHIFT,
 674
 675         .bar_descriptions = {
 676                 GASKET_UNUSED_BAR,
 677                 GASKET_UNUSED_BAR,
 678                 { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET,
 679                         NUM_REGIONS, mappable_regions, PCI_BAR },
 680                 GASKET_UNUSED_BAR,
 681                 GASKET_UNUSED_BAR,
 682                 GASKET_UNUSED_BAR,
 683         },
 684         .coherent_buffer_description = {
 685                 APEX_CH_MEM_BYTES,
 686                 (VM_WRITE | VM_READ),
 687                 APEX_CM_OFFSET,
 688         },
 689         .interrupt_type = PCI_MSIX,
 690         .interrupt_bar_index = APEX_BAR_INDEX,
 691         .num_interrupts = APEX_INTERRUPT_COUNT,
 692         .interrupts = apex_interrupts,
 693         .interrupt_pack_width = 7,
 694
 695         .device_open_cb = apex_device_open_cb,
 696         .device_close_cb = apex_device_cleanup,
 697
 698         .ioctl_handler_cb = apex_ioctl,
 699         .device_status_cb = apex_get_status,
 700         .hardware_revision_cb = NULL,
 701         .device_reset_cb = apex_reset,
 702 };
 703
 704 static struct pci_driver apex_pci_driver = {
 705         .name = "apex",
 706         .probe = apex_pci_probe,
 707         .remove = apex_pci_remove,
 708         .id_table = apex_pci_ids,
 709 };
 710
 711 static int __init apex_init(void)
 712 {
 713         int ret;
 714
 715         ret = gasket_register_device(&apex_desc);
 716         if (ret)
 717                 return ret;
 718         ret = pci_register_driver(&apex_pci_driver);
 719         if (ret)
 720                 gasket_unregister_device(&apex_desc);
 721         return ret;
 722 }
 723
 724 static void apex_exit(void)
 725 {
 726         pci_unregister_driver(&apex_pci_driver);
 727         gasket_unregister_device(&apex_desc);
 728 }
 729 MODULE_DESCRIPTION("Google Apex driver");
 730 MODULE_VERSION(APEX_DRIVER_VERSION);
 731 MODULE_LICENSE("GPL v2");
 732 MODULE_AUTHOR("John Joseph <jnjoseph@google.com>");
 733 MODULE_DEVICE_TABLE(pci, apex_pci_ids);
 734 module_init(apex_init);
 735 module_exit(apex_exit);