1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/errno.h>
7 #include <linux/interrupt.h>
8 #include <linux/pagemap.h>
9 #include <linux/dma-mapping.h>
11 #include <linux/platform_device.h>
12 #include <linux/uaccess.h>
15 #include <soc/bcm2835/raspberrypi-firmware.h>
17 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
19 #include "vchiq_arm.h"
20 #include "vchiq_connected.h"
21 #include "vchiq_pagelist.h"
23 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
25 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
26 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1
31 struct vchiq_2835_state {
33 struct vchiq_arm_state arm_state;
36 struct vchiq_pagelist_info {
37 struct pagelist *pagelist;
38 size_t pagelist_buffer_size;
40 enum dma_data_direction dma_dir;
41 unsigned int num_pages;
42 unsigned int pages_need_release;
44 struct scatterlist *scatterlist;
45 unsigned int scatterlist_mapped;
48 static void __iomem *g_regs;
49 /* This value is the size of the L2 cache lines as understood by the
50 * VPU firmware, which determines the required alignment of the
51 * offsets/sizes in pagelists.
53 * Modern VPU firmware looks for a DT "cache-line-size" property in
54 * the VCHIQ node and will overwrite it with the actual L2 cache size,
55 * which the kernel must then respect. That property was rejected
56 * upstream, so we have to use the VPU firmware's compatibility value
59 static unsigned int g_cache_line_size = 32;
60 static unsigned int g_fragments_size;
61 static char *g_fragments_base;
62 static char *g_free_fragments;
63 static struct semaphore g_free_fragments_sema;
64 static struct device *g_dev;
66 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
69 vchiq_doorbell_irq(int irq, void *dev_id);
71 static struct vchiq_pagelist_info *
72 create_pagelist(char __user *buf, size_t count, unsigned short type);
75 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
78 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
80 struct device *dev = &pdev->dev;
81 struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
82 struct rpi_firmware *fw = drvdata->fw;
83 struct vchiq_slot_zero *vchiq_slot_zero;
87 int slot_mem_size, frag_mem_size;
91 * VCHI messages between the CPU and firmware use
92 * 32-bit bus addresses.
94 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
99 g_cache_line_size = drvdata->cache_line_size;
100 g_fragments_size = 2 * g_cache_line_size;
102 /* Allocate space for the channels in coherent memory */
103 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
104 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
106 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
107 &slot_phys, GFP_KERNEL);
109 dev_err(dev, "could not allocate DMA memory\n");
113 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
115 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
116 if (!vchiq_slot_zero)
119 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
120 (int)slot_phys + slot_mem_size;
121 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
124 g_fragments_base = (char *)slot_mem + slot_mem_size;
126 g_free_fragments = g_fragments_base;
127 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
128 *(char **)&g_fragments_base[i*g_fragments_size] =
129 &g_fragments_base[(i + 1)*g_fragments_size];
131 *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
132 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
134 if (vchiq_init_state(state, vchiq_slot_zero) != VCHIQ_SUCCESS)
137 g_regs = devm_platform_ioremap_resource(pdev, 0);
139 return PTR_ERR(g_regs);
141 irq = platform_get_irq(pdev, 0);
145 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
146 "VCHIQ doorbell", state);
148 dev_err(dev, "failed to register irq=%d\n", irq);
152 /* Send the base address of the slots to VideoCore */
153 channelbase = slot_phys;
154 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
155 &channelbase, sizeof(channelbase));
156 if (err || channelbase) {
157 dev_err(dev, "failed to set channelbase\n");
158 return err ? : -ENXIO;
162 vchiq_log_info(vchiq_arm_log_level,
163 "vchiq_init - done (slots %pK, phys %pad)",
164 vchiq_slot_zero, &slot_phys);
166 vchiq_call_connected_callbacks();
172 vchiq_platform_init_state(struct vchiq_state *state)
174 enum vchiq_status status = VCHIQ_SUCCESS;
175 struct vchiq_2835_state *platform_state;
177 state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
178 if (!state->platform_state)
181 platform_state = (struct vchiq_2835_state *)state->platform_state;
183 platform_state->inited = 1;
184 status = vchiq_arm_init_state(state, &platform_state->arm_state);
186 if (status != VCHIQ_SUCCESS)
187 platform_state->inited = 0;
192 struct vchiq_arm_state*
193 vchiq_platform_get_arm_state(struct vchiq_state *state)
195 struct vchiq_2835_state *platform_state;
197 platform_state = (struct vchiq_2835_state *)state->platform_state;
199 WARN_ON_ONCE(!platform_state->inited);
201 return &platform_state->arm_state;
205 remote_event_signal(struct remote_event *event)
211 dsb(sy); /* data barrier operation */
214 writel(0, g_regs + BELL2); /* trigger vc interrupt */
218 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
221 struct vchiq_pagelist_info *pagelistinfo;
223 pagelistinfo = create_pagelist((char __user *)offset, size,
224 (dir == VCHIQ_BULK_RECEIVE)
231 bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr;
234 * Store the pagelistinfo address in remote_data,
235 * which isn't used by the slave.
237 bulk->remote_data = pagelistinfo;
239 return VCHIQ_SUCCESS;
243 vchiq_complete_bulk(struct vchiq_bulk *bulk)
245 if (bulk && bulk->remote_data && bulk->actual)
246 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
250 int vchiq_dump_platform_state(void *dump_context)
255 len = snprintf(buf, sizeof(buf),
256 " Platform: 2835 (VC master)");
257 return vchiq_dump(dump_context, buf, len + 1);
261 vchiq_platform_suspend(struct vchiq_state *state)
267 vchiq_platform_resume(struct vchiq_state *state)
269 return VCHIQ_SUCCESS;
273 vchiq_platform_paused(struct vchiq_state *state)
278 vchiq_platform_resumed(struct vchiq_state *state)
283 vchiq_platform_videocore_wanted(struct vchiq_state *state)
285 return 1; // autosuspend not supported - videocore always wanted
289 vchiq_platform_use_suspend_timer(void)
294 vchiq_dump_platform_use_state(struct vchiq_state *state)
296 vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
299 vchiq_platform_handle_timeout(struct vchiq_state *state)
308 vchiq_doorbell_irq(int irq, void *dev_id)
310 struct vchiq_state *state = dev_id;
311 irqreturn_t ret = IRQ_NONE;
314 /* Read (and clear) the doorbell */
315 status = readl(g_regs + BELL0);
317 if (status & 0x4) { /* Was the doorbell rung? */
318 remote_event_pollall(state);
326 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
328 if (pagelistinfo->scatterlist_mapped) {
329 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
330 pagelistinfo->num_pages, pagelistinfo->dma_dir);
333 if (pagelistinfo->pages_need_release) {
336 for (i = 0; i < pagelistinfo->num_pages; i++)
337 put_page(pagelistinfo->pages[i]);
340 dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
341 pagelistinfo->pagelist, pagelistinfo->dma_addr);
344 /* There is a potential problem with partial cache lines (pages?)
345 * at the ends of the block when reading. If the CPU accessed anything in
346 * the same line (page?) then it may have pulled old data into the cache,
347 * obscuring the new data underneath. We can solve this by transferring the
348 * partial cache lines separately, and allowing the ARM to copy into the
352 static struct vchiq_pagelist_info *
353 create_pagelist(char __user *buf, size_t count, unsigned short type)
355 struct pagelist *pagelist;
356 struct vchiq_pagelist_info *pagelistinfo;
359 unsigned int num_pages, offset, i, k;
361 size_t pagelist_size;
362 struct scatterlist *scatterlist, *sg;
366 if (count >= INT_MAX - PAGE_SIZE)
369 offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1));
370 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
372 if (num_pages > (SIZE_MAX - sizeof(struct pagelist) -
373 sizeof(struct vchiq_pagelist_info)) /
374 (sizeof(u32) + sizeof(pages[0]) +
375 sizeof(struct scatterlist)))
378 pagelist_size = sizeof(struct pagelist) +
379 (num_pages * sizeof(u32)) +
380 (num_pages * sizeof(pages[0]) +
381 (num_pages * sizeof(struct scatterlist))) +
382 sizeof(struct vchiq_pagelist_info);
384 /* Allocate enough storage to hold the page pointers and the page
387 pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
390 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
395 addrs = pagelist->addrs;
396 pages = (struct page **)(addrs + num_pages);
397 scatterlist = (struct scatterlist *)(pages + num_pages);
398 pagelistinfo = (struct vchiq_pagelist_info *)
399 (scatterlist + num_pages);
401 pagelist->length = count;
402 pagelist->type = type;
403 pagelist->offset = offset;
405 /* Populate the fields of the pagelistinfo structure */
406 pagelistinfo->pagelist = pagelist;
407 pagelistinfo->pagelist_buffer_size = pagelist_size;
408 pagelistinfo->dma_addr = dma_addr;
409 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ?
410 DMA_TO_DEVICE : DMA_FROM_DEVICE;
411 pagelistinfo->num_pages = num_pages;
412 pagelistinfo->pages_need_release = 0;
413 pagelistinfo->pages = pages;
414 pagelistinfo->scatterlist = scatterlist;
415 pagelistinfo->scatterlist_mapped = 0;
417 if (is_vmalloc_addr(buf)) {
418 unsigned long length = count;
419 unsigned int off = offset;
421 for (actual_pages = 0; actual_pages < num_pages;
423 struct page *pg = vmalloc_to_page(buf + (actual_pages *
425 size_t bytes = PAGE_SIZE - off;
428 cleanup_pagelistinfo(pagelistinfo);
434 pages[actual_pages] = pg;
438 /* do not try and release vmalloc pages */
440 actual_pages = get_user_pages_fast(
441 (unsigned long)buf & PAGE_MASK,
443 type == PAGELIST_READ,
446 if (actual_pages != num_pages) {
447 vchiq_log_info(vchiq_arm_log_level,
448 "%s - only %d/%d pages locked",
449 __func__, actual_pages, num_pages);
451 /* This is probably due to the process being killed */
452 while (actual_pages > 0) {
454 put_page(pages[actual_pages]);
456 cleanup_pagelistinfo(pagelistinfo);
459 /* release user pages */
460 pagelistinfo->pages_need_release = 1;
464 * Initialize the scatterlist so that the magic cookie
465 * is filled if debugging is enabled
467 sg_init_table(scatterlist, num_pages);
468 /* Now set the pages for each scatterlist */
469 for (i = 0; i < num_pages; i++) {
470 unsigned int len = PAGE_SIZE - offset;
474 sg_set_page(scatterlist + i, pages[i], len, offset);
479 dma_buffers = dma_map_sg(g_dev,
482 pagelistinfo->dma_dir);
484 if (dma_buffers == 0) {
485 cleanup_pagelistinfo(pagelistinfo);
489 pagelistinfo->scatterlist_mapped = 1;
491 /* Combine adjacent blocks for performance */
493 for_each_sg(scatterlist, sg, dma_buffers, i) {
494 u32 len = sg_dma_len(sg);
495 u32 addr = sg_dma_address(sg);
497 /* Note: addrs is the address + page_count - 1
498 * The firmware expects blocks after the first to be page-
499 * aligned and a multiple of the page size
502 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
503 WARN_ON(i && (addr & ~PAGE_MASK));
505 ((addrs[k - 1] & PAGE_MASK) +
506 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
507 == (addr & PAGE_MASK))
508 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
510 addrs[k++] = (addr & PAGE_MASK) |
511 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
514 /* Partial cache lines (fragments) require special measures */
515 if ((type == PAGELIST_READ) &&
516 ((pagelist->offset & (g_cache_line_size - 1)) ||
517 ((pagelist->offset + pagelist->length) &
518 (g_cache_line_size - 1)))) {
521 if (down_interruptible(&g_free_fragments_sema)) {
522 cleanup_pagelistinfo(pagelistinfo);
526 WARN_ON(!g_free_fragments);
528 down(&g_free_fragments_mutex);
529 fragments = g_free_fragments;
531 g_free_fragments = *(char **) g_free_fragments;
532 up(&g_free_fragments_mutex);
533 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
534 (fragments - g_fragments_base) / g_fragments_size;
541 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
544 struct pagelist *pagelist = pagelistinfo->pagelist;
545 struct page **pages = pagelistinfo->pages;
546 unsigned int num_pages = pagelistinfo->num_pages;
548 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
549 __func__, pagelistinfo->pagelist, actual);
552 * NOTE: dma_unmap_sg must be called before the
553 * cpu can touch any of the data/pages.
555 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
556 pagelistinfo->num_pages, pagelistinfo->dma_dir);
557 pagelistinfo->scatterlist_mapped = 0;
559 /* Deal with any partial cache lines (fragments) */
560 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
561 char *fragments = g_fragments_base +
562 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
564 int head_bytes, tail_bytes;
566 head_bytes = (g_cache_line_size - pagelist->offset) &
567 (g_cache_line_size - 1);
568 tail_bytes = (pagelist->offset + actual) &
569 (g_cache_line_size - 1);
571 if ((actual >= 0) && (head_bytes != 0)) {
572 if (head_bytes > actual)
575 memcpy((char *)kmap(pages[0]) +
581 if ((actual >= 0) && (head_bytes < actual) &&
583 memcpy((char *)kmap(pages[num_pages - 1]) +
584 ((pagelist->offset + actual) &
585 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
586 fragments + g_cache_line_size,
588 kunmap(pages[num_pages - 1]);
591 down(&g_free_fragments_mutex);
592 *(char **)fragments = g_free_fragments;
593 g_free_fragments = fragments;
594 up(&g_free_fragments_mutex);
595 up(&g_free_fragments_sema);
598 /* Need to mark all the pages dirty. */
599 if (pagelist->type != PAGELIST_WRITE &&
600 pagelistinfo->pages_need_release) {
603 for (i = 0; i < num_pages; i++)
604 set_page_dirty(pages[i]);
607 cleanup_pagelistinfo(pagelistinfo);