1 // SPDX-License-Identifier: GPL-2.0-only
3 * Intel MIC Platform Software Stack (MPSS)
5 * Copyright(c) 2013 Intel Corporation.
7 * Intel MIC Host driver.
10 #include <linux/pci.h>
11 #include <linux/sched.h>
12 #include <linux/firmware.h>
13 #include <linux/delay.h>
15 #include "../common/mic_dev.h"
16 #include "mic_device.h"
20 static const u16 mic_x100_intr_init[] = {
21 MIC_X100_DOORBELL_IDX_START,
22 MIC_X100_DMA_IDX_START,
23 MIC_X100_ERR_IDX_START,
24 MIC_X100_NUM_DOORBELL,
30 * mic_x100_write_spad - write to the scratchpad register
31 * @mdev: pointer to mic_device instance
32 * @idx: index to the scratchpad register, 0 based
33 * @val: the data value to put into the register
35 * This function allows writing of a 32bit value to the indexed scratchpad
41 mic_x100_write_spad(struct mic_device *mdev, unsigned int idx, u32 val)
43 dev_dbg(&mdev->pdev->dev, "Writing 0x%x to scratch pad index %d\n",
45 mic_mmio_write(&mdev->mmio, val,
46 MIC_X100_SBOX_BASE_ADDRESS +
47 MIC_X100_SBOX_SPAD0 + idx * 4);
51 * mic_x100_read_spad - read from the scratchpad register
52 * @mdev: pointer to mic_device instance
53 * @idx: index to scratchpad register, 0 based
55 * This function allows reading of the 32bit scratchpad register.
57 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
60 mic_x100_read_spad(struct mic_device *mdev, unsigned int idx)
62 u32 val = mic_mmio_read(&mdev->mmio,
63 MIC_X100_SBOX_BASE_ADDRESS +
64 MIC_X100_SBOX_SPAD0 + idx * 4);
66 dev_dbg(&mdev->pdev->dev,
67 "Reading 0x%x from scratch pad index %d\n", val, idx);
72 * mic_x100_enable_interrupts - Enable interrupts.
73 * @mdev: pointer to mic_device instance
75 static void mic_x100_enable_interrupts(struct mic_device *mdev)
78 struct mic_mw *mw = &mdev->mmio;
79 u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0;
80 u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0;
82 reg = mic_mmio_read(mw, sice0);
83 reg |= MIC_X100_SBOX_DBR_BITS(0xf) | MIC_X100_SBOX_DMA_BITS(0xff);
84 mic_mmio_write(mw, reg, sice0);
87 * Enable auto-clear when enabling interrupts. Applicable only for
88 * MSI-x. Legacy and MSI mode cannot have auto-clear enabled.
90 if (mdev->irq_info.num_vectors > 1) {
91 reg = mic_mmio_read(mw, siac0);
92 reg |= MIC_X100_SBOX_DBR_BITS(0xf) |
93 MIC_X100_SBOX_DMA_BITS(0xff);
94 mic_mmio_write(mw, reg, siac0);
99 * mic_x100_disable_interrupts - Disable interrupts.
100 * @mdev: pointer to mic_device instance
102 static void mic_x100_disable_interrupts(struct mic_device *mdev)
105 struct mic_mw *mw = &mdev->mmio;
106 u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0;
107 u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0;
108 u32 sicc0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICC0;
110 reg = mic_mmio_read(mw, sice0);
111 mic_mmio_write(mw, reg, sicc0);
113 if (mdev->irq_info.num_vectors > 1) {
114 reg = mic_mmio_read(mw, siac0);
115 reg &= ~(MIC_X100_SBOX_DBR_BITS(0xf) |
116 MIC_X100_SBOX_DMA_BITS(0xff));
117 mic_mmio_write(mw, reg, siac0);
122 * mic_x100_send_sbox_intr - Send an MIC_X100_SBOX interrupt to MIC.
123 * @mdev: pointer to mic_device instance
124 * @doorbell: doorbell number
126 static void mic_x100_send_sbox_intr(struct mic_device *mdev,
129 struct mic_mw *mw = &mdev->mmio;
130 u64 apic_icr_offset = MIC_X100_SBOX_APICICR0 + doorbell * 8;
131 u32 apicicr_low = mic_mmio_read(mw, MIC_X100_SBOX_BASE_ADDRESS +
134 /* for MIC we need to make sure we "hit" the send_icr bit (13) */
135 apicicr_low = (apicicr_low | (1 << 13));
137 /* Ensure that the interrupt is ordered w.r.t. previous stores. */
139 mic_mmio_write(mw, apicicr_low,
140 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset);
144 * mic_x100_send_rdmasr_intr - Send an RDMASR interrupt to MIC.
145 * @mdev: pointer to mic_device instance
146 * @doorbell: doorbell number
148 static void mic_x100_send_rdmasr_intr(struct mic_device *mdev,
151 int rdmasr_offset = MIC_X100_SBOX_RDMASR0 + (doorbell << 2);
152 /* Ensure that the interrupt is ordered w.r.t. previous stores. */
154 mic_mmio_write(&mdev->mmio, 0,
155 MIC_X100_SBOX_BASE_ADDRESS + rdmasr_offset);
159 * __mic_x100_send_intr - Send interrupt to MIC.
160 * @mdev: pointer to mic_device instance
161 * @doorbell: doorbell number.
163 static void mic_x100_send_intr(struct mic_device *mdev, int doorbell)
166 if (doorbell < MIC_X100_NUM_SBOX_IRQ) {
167 mic_x100_send_sbox_intr(mdev, doorbell);
169 rdmasr_db = doorbell - MIC_X100_NUM_SBOX_IRQ;
170 mic_x100_send_rdmasr_intr(mdev, rdmasr_db);
175 * mic_x100_ack_interrupt - Read the interrupt sources register and
176 * clear it. This function will be called in the MSI/INTx case.
177 * @mdev: Pointer to mic_device instance.
179 * Returns: bitmask of interrupt sources triggered.
181 static u32 mic_x100_ack_interrupt(struct mic_device *mdev)
183 u32 sicr0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICR0;
184 u32 reg = mic_mmio_read(&mdev->mmio, sicr0);
185 mic_mmio_write(&mdev->mmio, reg, sicr0);
190 * mic_x100_intr_workarounds - These hardware specific workarounds are
191 * to be invoked everytime an interrupt is handled.
192 * @mdev: Pointer to mic_device instance.
196 static void mic_x100_intr_workarounds(struct mic_device *mdev)
198 struct mic_mw *mw = &mdev->mmio;
200 /* Clear pending bit array. */
201 if (MIC_A0_STEP == mdev->stepping)
202 mic_mmio_write(mw, 1, MIC_X100_SBOX_BASE_ADDRESS +
203 MIC_X100_SBOX_MSIXPBACR);
205 if (mdev->stepping >= MIC_B0_STEP)
206 mdev->intr_ops->enable_interrupts(mdev);
210 * mic_x100_hw_intr_init - Initialize h/w specific interrupt
212 * @mdev: pointer to mic_device instance
214 static void mic_x100_hw_intr_init(struct mic_device *mdev)
216 mdev->intr_info = (struct mic_intr_info *)mic_x100_intr_init;
220 * mic_x100_read_msi_to_src_map - read from the MSI mapping registers
221 * @mdev: pointer to mic_device instance
222 * @idx: index to the mapping register, 0 based
224 * This function allows reading of the 32bit MSI mapping register.
226 * RETURNS: The value in the register.
229 mic_x100_read_msi_to_src_map(struct mic_device *mdev, int idx)
231 return mic_mmio_read(&mdev->mmio,
232 MIC_X100_SBOX_BASE_ADDRESS +
233 MIC_X100_SBOX_MXAR0 + idx * 4);
237 * mic_x100_program_msi_to_src_map - program the MSI mapping registers
238 * @mdev: pointer to mic_device instance
239 * @idx: index to the mapping register, 0 based
240 * @offset: The bit offset in the register that needs to be updated.
241 * @set: boolean specifying if the bit in the specified offset needs
242 * to be set or cleared.
247 mic_x100_program_msi_to_src_map(struct mic_device *mdev,
248 int idx, int offset, bool set)
251 struct mic_mw *mw = &mdev->mmio;
252 u32 mxar = MIC_X100_SBOX_BASE_ADDRESS +
253 MIC_X100_SBOX_MXAR0 + idx * 4;
255 reg = mic_mmio_read(mw, mxar);
257 __set_bit(offset, ®);
259 __clear_bit(offset, ®);
260 mic_mmio_write(mw, reg, mxar);
264 * mic_x100_reset_fw_ready - Reset Firmware ready status field.
265 * @mdev: pointer to mic_device instance
267 static void mic_x100_reset_fw_ready(struct mic_device *mdev)
269 mdev->ops->write_spad(mdev, MIC_X100_DOWNLOAD_INFO, 0);
273 * mic_x100_is_fw_ready - Check if firmware is ready.
274 * @mdev: pointer to mic_device instance
276 static bool mic_x100_is_fw_ready(struct mic_device *mdev)
278 u32 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
279 return MIC_X100_SPAD2_DOWNLOAD_STATUS(scratch2) ? true : false;
283 * mic_x100_get_apic_id - Get bootstrap APIC ID.
284 * @mdev: pointer to mic_device instance
286 static u32 mic_x100_get_apic_id(struct mic_device *mdev)
290 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
291 return MIC_X100_SPAD2_APIC_ID(scratch2);
295 * mic_x100_send_firmware_intr - Send an interrupt to the firmware on MIC.
296 * @mdev: pointer to mic_device instance
298 static void mic_x100_send_firmware_intr(struct mic_device *mdev)
301 u64 apic_icr_offset = MIC_X100_SBOX_APICICR7;
302 int vector = MIC_X100_BSP_INTERRUPT_VECTOR;
303 struct mic_mw *mw = &mdev->mmio;
306 * For MIC we need to make sure we "hit"
307 * the send_icr bit (13).
309 apicicr_low = (vector | (1 << 13));
311 mic_mmio_write(mw, mic_x100_get_apic_id(mdev),
312 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset + 4);
314 /* Ensure that the interrupt is ordered w.r.t. previous stores. */
316 mic_mmio_write(mw, apicicr_low,
317 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset);
321 * mic_x100_hw_reset - Reset the MIC device.
322 * @mdev: pointer to mic_device instance
324 static void mic_x100_hw_reset(struct mic_device *mdev)
327 u32 rgcr = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_RGCR;
328 struct mic_mw *mw = &mdev->mmio;
330 /* Ensure that the reset is ordered w.r.t. previous loads and stores */
333 reset_reg = mic_mmio_read(mw, rgcr);
335 mic_mmio_write(mw, reset_reg, rgcr);
337 * It seems we really want to delay at least 1 second
338 * after touching reset to prevent a lot of problems.
344 * mic_x100_load_command_line - Load command line to MIC.
345 * @mdev: pointer to mic_device instance
346 * @fw: the firmware image
348 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
351 mic_x100_load_command_line(struct mic_device *mdev, const struct firmware *fw)
356 void __iomem *cmd_line_va = mdev->aper.va + mdev->bootaddr + fw->size;
357 #define CMDLINE_SIZE 2048
359 boot_mem = mdev->aper.len >> 20;
360 buf = kzalloc(CMDLINE_SIZE, GFP_KERNEL);
364 len += scnprintf(buf, CMDLINE_SIZE - len,
365 " mem=%dM", boot_mem);
366 if (mdev->cosm_dev->cmdline)
367 scnprintf(buf + len, CMDLINE_SIZE - len, " %s",
368 mdev->cosm_dev->cmdline);
369 memcpy_toio(cmd_line_va, buf, strlen(buf) + 1);
375 * mic_x100_load_ramdisk - Load ramdisk to MIC.
376 * @mdev: pointer to mic_device instance
378 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
381 mic_x100_load_ramdisk(struct mic_device *mdev)
383 const struct firmware *fw;
385 struct boot_params __iomem *bp = mdev->aper.va + mdev->bootaddr;
387 rc = request_firmware(&fw, mdev->cosm_dev->ramdisk, &mdev->pdev->dev);
389 dev_err(&mdev->pdev->dev,
390 "ramdisk request_firmware failed: %d %s\n",
391 rc, mdev->cosm_dev->ramdisk);
395 * Typically the bootaddr for card OS is 64M
396 * so copy over the ramdisk @ 128M.
398 memcpy_toio(mdev->aper.va + (mdev->bootaddr << 1), fw->data, fw->size);
399 iowrite32(mdev->bootaddr << 1, &bp->hdr.ramdisk_image);
400 iowrite32(fw->size, &bp->hdr.ramdisk_size);
401 release_firmware(fw);
407 * mic_x100_get_boot_addr - Get MIC boot address.
408 * @mdev: pointer to mic_device instance
410 * This function is called during firmware load to determine
411 * the address at which the OS should be downloaded in card
413 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
416 mic_x100_get_boot_addr(struct mic_device *mdev)
418 u32 scratch2, boot_addr;
421 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
422 boot_addr = MIC_X100_SPAD2_DOWNLOAD_ADDR(scratch2);
423 dev_dbg(&mdev->pdev->dev, "%s %d boot_addr 0x%x\n",
424 __func__, __LINE__, boot_addr);
425 if (boot_addr > (1 << 31)) {
426 dev_err(&mdev->pdev->dev,
427 "incorrect bootaddr 0x%x\n",
432 mdev->bootaddr = boot_addr;
438 * mic_x100_load_firmware - Load firmware to MIC.
439 * @mdev: pointer to mic_device instance
440 * @buf: buffer containing boot string including firmware/ramdisk path.
442 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
445 mic_x100_load_firmware(struct mic_device *mdev, const char *buf)
448 const struct firmware *fw;
450 rc = mic_x100_get_boot_addr(mdev);
454 rc = request_firmware(&fw, mdev->cosm_dev->firmware, &mdev->pdev->dev);
456 dev_err(&mdev->pdev->dev,
457 "ramdisk request_firmware failed: %d %s\n",
458 rc, mdev->cosm_dev->firmware);
461 if (mdev->bootaddr > mdev->aper.len - fw->size) {
463 dev_err(&mdev->pdev->dev, "%s %d rc %d bootaddr 0x%x\n",
464 __func__, __LINE__, rc, mdev->bootaddr);
467 memcpy_toio(mdev->aper.va + mdev->bootaddr, fw->data, fw->size);
468 mdev->ops->write_spad(mdev, MIC_X100_FW_SIZE, fw->size);
469 if (!strcmp(mdev->cosm_dev->bootmode, "flash")) {
471 dev_err(&mdev->pdev->dev, "%s %d rc %d\n",
472 __func__, __LINE__, rc);
475 /* load command line */
476 rc = mic_x100_load_command_line(mdev, fw);
478 dev_err(&mdev->pdev->dev, "%s %d rc %d\n",
479 __func__, __LINE__, rc);
482 release_firmware(fw);
484 if (mdev->cosm_dev->ramdisk)
485 rc = mic_x100_load_ramdisk(mdev);
490 release_firmware(fw);
495 * mic_x100_get_postcode - Get postcode status from firmware.
496 * @mdev: pointer to mic_device instance
500 static u32 mic_x100_get_postcode(struct mic_device *mdev)
502 return mic_mmio_read(&mdev->mmio, MIC_X100_POSTCODE);
506 * mic_x100_smpt_set - Update an SMPT entry with a DMA address.
507 * @mdev: pointer to mic_device instance
508 * @dma_addr: DMA address to use
509 * @index: entry to write to
514 mic_x100_smpt_set(struct mic_device *mdev, dma_addr_t dma_addr, u8 index)
516 #define SNOOP_ON (0 << 0)
517 #define SNOOP_OFF (1 << 0)
519 * Sbox Smpt Reg Bits:
520 * Bits 31:2 Host address
524 #define BUILD_SMPT(NO_SNOOP, HOST_ADDR) \
525 (u32)(((HOST_ADDR) << 2) | ((NO_SNOOP) & 0x01))
527 uint32_t smpt_reg_val = BUILD_SMPT(SNOOP_ON,
528 dma_addr >> mdev->smpt->info.page_shift);
529 mic_mmio_write(&mdev->mmio, smpt_reg_val,
530 MIC_X100_SBOX_BASE_ADDRESS +
531 MIC_X100_SBOX_SMPT00 + (4 * index));
535 * mic_x100_smpt_hw_init - Initialize SMPT X100 specific fields.
536 * @mdev: pointer to mic_device instance
540 static void mic_x100_smpt_hw_init(struct mic_device *mdev)
542 struct mic_smpt_hw_info *info = &mdev->smpt->info;
545 info->page_shift = 34;
546 info->page_size = (1ULL << info->page_shift);
547 info->base = 0x8000000000ULL;
550 struct mic_smpt_ops mic_x100_smpt_ops = {
551 .init = mic_x100_smpt_hw_init,
552 .set = mic_x100_smpt_set,
555 static bool mic_x100_dma_filter(struct dma_chan *chan, void *param)
557 if (chan->device->dev->parent == (struct device *)param)
562 struct mic_hw_ops mic_x100_ops = {
563 .aper_bar = MIC_X100_APER_BAR,
564 .mmio_bar = MIC_X100_MMIO_BAR,
565 .read_spad = mic_x100_read_spad,
566 .write_spad = mic_x100_write_spad,
567 .send_intr = mic_x100_send_intr,
568 .ack_interrupt = mic_x100_ack_interrupt,
569 .intr_workarounds = mic_x100_intr_workarounds,
570 .reset = mic_x100_hw_reset,
571 .reset_fw_ready = mic_x100_reset_fw_ready,
572 .is_fw_ready = mic_x100_is_fw_ready,
573 .send_firmware_intr = mic_x100_send_firmware_intr,
574 .load_mic_fw = mic_x100_load_firmware,
575 .get_postcode = mic_x100_get_postcode,
576 .dma_filter = mic_x100_dma_filter,
579 struct mic_hw_intr_ops mic_x100_intr_ops = {
580 .intr_init = mic_x100_hw_intr_init,
581 .enable_interrupts = mic_x100_enable_interrupts,
582 .disable_interrupts = mic_x100_disable_interrupts,
583 .program_msi_to_src_map = mic_x100_program_msi_to_src_map,
584 .read_msi_to_src_map = mic_x100_read_msi_to_src_map,