2 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions, and the following disclaimer,
9 * without modification.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The names of the above-listed copyright holders may not be used
14 * to endorse or promote products derived from this software without
15 * specific prior written permission.
17 * ALTERNATIVELY, this software may be distributed under the terms of the
18 * GNU General Public License ("GPL") version 2, as published by the Free
19 * Software Foundation.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
23 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <linux/kernel.h>
35 #include <linux/types.h>
36 #include <linux/errno.h>
37 #include <linux/interrupt.h>
38 #include <linux/pagemap.h>
39 #include <linux/dma-mapping.h>
41 #include <linux/platform_device.h>
42 #include <linux/uaccess.h>
45 #include <soc/bcm2835/raspberrypi-firmware.h>
47 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
49 #include "vchiq_arm.h"
50 #include "vchiq_connected.h"
51 #include "vchiq_killable.h"
52 #include "vchiq_pagelist.h"
54 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
56 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
57 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1
62 typedef struct vchiq_2835_state_struct {
64 VCHIQ_ARM_STATE_T arm_state;
65 } VCHIQ_2835_ARM_STATE_T;
67 struct vchiq_pagelist_info {
69 size_t pagelist_buffer_size;
71 enum dma_data_direction dma_dir;
72 unsigned int num_pages;
73 unsigned int pages_need_release;
75 struct scatterlist *scatterlist;
76 unsigned int scatterlist_mapped;
79 static void __iomem *g_regs;
80 static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE);
81 static unsigned int g_fragments_size;
82 static char *g_fragments_base;
83 static char *g_free_fragments;
84 static struct semaphore g_free_fragments_sema;
85 static struct device *g_dev;
87 extern int vchiq_arm_log_level;
89 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
92 vchiq_doorbell_irq(int irq, void *dev_id);
94 static struct vchiq_pagelist_info *
95 create_pagelist(char __user *buf, size_t count, unsigned short type,
96 struct task_struct *task);
99 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
102 int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
104 struct device *dev = &pdev->dev;
105 struct rpi_firmware *fw = platform_get_drvdata(pdev);
106 VCHIQ_SLOT_ZERO_T *vchiq_slot_zero;
107 struct resource *res;
109 dma_addr_t slot_phys;
111 int slot_mem_size, frag_mem_size;
115 * VCHI messages between the CPU and firmware use
116 * 32-bit bus addresses.
118 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
123 err = of_property_read_u32(dev->of_node, "cache-line-size",
127 dev_err(dev, "Missing cache-line-size property\n");
131 g_fragments_size = 2 * g_cache_line_size;
133 /* Allocate space for the channels in coherent memory */
134 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
135 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
137 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
138 &slot_phys, GFP_KERNEL);
140 dev_err(dev, "could not allocate DMA memory\n");
144 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
146 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
147 if (!vchiq_slot_zero)
150 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
151 (int)slot_phys + slot_mem_size;
152 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
155 g_fragments_base = (char *)slot_mem + slot_mem_size;
157 g_free_fragments = g_fragments_base;
158 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
159 *(char **)&g_fragments_base[i*g_fragments_size] =
160 &g_fragments_base[(i + 1)*g_fragments_size];
162 *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
163 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
165 if (vchiq_init_state(state, vchiq_slot_zero, 0) != VCHIQ_SUCCESS)
168 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
169 g_regs = devm_ioremap_resource(&pdev->dev, res);
171 return PTR_ERR(g_regs);
173 irq = platform_get_irq(pdev, 0);
175 dev_err(dev, "failed to get IRQ\n");
179 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
180 "VCHIQ doorbell", state);
182 dev_err(dev, "failed to register irq=%d\n", irq);
186 /* Send the base address of the slots to VideoCore */
187 channelbase = slot_phys;
188 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
189 &channelbase, sizeof(channelbase));
190 if (err || channelbase) {
191 dev_err(dev, "failed to set channelbase\n");
192 return err ? : -ENXIO;
196 vchiq_log_info(vchiq_arm_log_level,
197 "vchiq_init - done (slots %pK, phys %pad)",
198 vchiq_slot_zero, &slot_phys);
200 vchiq_call_connected_callbacks();
206 vchiq_platform_init_state(VCHIQ_STATE_T *state)
208 VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
210 state->platform_state = kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T), GFP_KERNEL);
211 ((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->inited = 1;
212 status = vchiq_arm_init_state(state, &((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->arm_state);
213 if (status != VCHIQ_SUCCESS)
215 ((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->inited = 0;
221 vchiq_platform_get_arm_state(VCHIQ_STATE_T *state)
223 if (!((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->inited)
227 return &((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->arm_state;
231 remote_event_signal(REMOTE_EVENT_T *event)
237 dsb(sy); /* data barrier operation */
240 writel(0, g_regs + BELL2); /* trigger vc interrupt */
244 vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle,
245 void *offset, int size, int dir)
247 struct vchiq_pagelist_info *pagelistinfo;
249 WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID);
251 pagelistinfo = create_pagelist((char __user *)offset, size,
252 (dir == VCHIQ_BULK_RECEIVE)
260 bulk->handle = memhandle;
261 bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr;
264 * Store the pagelistinfo address in remote_data,
265 * which isn't used by the slave.
267 bulk->remote_data = pagelistinfo;
269 return VCHIQ_SUCCESS;
273 vchiq_complete_bulk(VCHIQ_BULK_T *bulk)
275 if (bulk && bulk->remote_data && bulk->actual)
276 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
281 vchiq_transfer_bulk(VCHIQ_BULK_T *bulk)
284 * This should only be called on the master (VideoCore) side, but
285 * provide an implementation to avoid the need for ifdefery.
291 vchiq_dump_platform_state(void *dump_context)
296 len = snprintf(buf, sizeof(buf),
297 " Platform: 2835 (VC master)");
298 vchiq_dump(dump_context, buf, len + 1);
302 vchiq_platform_suspend(VCHIQ_STATE_T *state)
308 vchiq_platform_resume(VCHIQ_STATE_T *state)
310 return VCHIQ_SUCCESS;
314 vchiq_platform_paused(VCHIQ_STATE_T *state)
319 vchiq_platform_resumed(VCHIQ_STATE_T *state)
324 vchiq_platform_videocore_wanted(VCHIQ_STATE_T *state)
326 return 1; // autosuspend not supported - videocore always wanted
330 vchiq_platform_use_suspend_timer(void)
335 vchiq_dump_platform_use_state(VCHIQ_STATE_T *state)
337 vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
340 vchiq_platform_handle_timeout(VCHIQ_STATE_T *state)
349 vchiq_doorbell_irq(int irq, void *dev_id)
351 VCHIQ_STATE_T *state = dev_id;
352 irqreturn_t ret = IRQ_NONE;
355 /* Read (and clear) the doorbell */
356 status = readl(g_regs + BELL0);
358 if (status & 0x4) { /* Was the doorbell rung? */
359 remote_event_pollall(state);
367 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
369 if (pagelistinfo->scatterlist_mapped) {
370 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
371 pagelistinfo->num_pages, pagelistinfo->dma_dir);
374 if (pagelistinfo->pages_need_release) {
377 for (i = 0; i < pagelistinfo->num_pages; i++)
378 put_page(pagelistinfo->pages[i]);
381 dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
382 pagelistinfo->pagelist, pagelistinfo->dma_addr);
385 /* There is a potential problem with partial cache lines (pages?)
386 ** at the ends of the block when reading. If the CPU accessed anything in
387 ** the same line (page?) then it may have pulled old data into the cache,
388 ** obscuring the new data underneath. We can solve this by transferring the
389 ** partial cache lines separately, and allowing the ARM to copy into the
393 static struct vchiq_pagelist_info *
394 create_pagelist(char __user *buf, size_t count, unsigned short type,
395 struct task_struct *task)
397 PAGELIST_T *pagelist;
398 struct vchiq_pagelist_info *pagelistinfo;
401 unsigned int num_pages, offset, i, k;
403 size_t pagelist_size;
404 struct scatterlist *scatterlist, *sg;
408 offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1));
409 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
411 pagelist_size = sizeof(PAGELIST_T) +
412 (num_pages * sizeof(u32)) +
413 (num_pages * sizeof(pages[0]) +
414 (num_pages * sizeof(struct scatterlist))) +
415 sizeof(struct vchiq_pagelist_info);
417 /* Allocate enough storage to hold the page pointers and the page
420 pagelist = dma_zalloc_coherent(g_dev,
425 vchiq_log_trace(vchiq_arm_log_level, "create_pagelist - %pK",
430 addrs = pagelist->addrs;
431 pages = (struct page **)(addrs + num_pages);
432 scatterlist = (struct scatterlist *)(pages + num_pages);
433 pagelistinfo = (struct vchiq_pagelist_info *)
434 (scatterlist + num_pages);
436 pagelist->length = count;
437 pagelist->type = type;
438 pagelist->offset = offset;
440 /* Populate the fields of the pagelistinfo structure */
441 pagelistinfo->pagelist = pagelist;
442 pagelistinfo->pagelist_buffer_size = pagelist_size;
443 pagelistinfo->dma_addr = dma_addr;
444 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ?
445 DMA_TO_DEVICE : DMA_FROM_DEVICE;
446 pagelistinfo->num_pages = num_pages;
447 pagelistinfo->pages_need_release = 0;
448 pagelistinfo->pages = pages;
449 pagelistinfo->scatterlist = scatterlist;
450 pagelistinfo->scatterlist_mapped = 0;
452 if (is_vmalloc_addr(buf)) {
453 unsigned long length = count;
454 unsigned int off = offset;
456 for (actual_pages = 0; actual_pages < num_pages;
458 struct page *pg = vmalloc_to_page(buf + (actual_pages *
460 size_t bytes = PAGE_SIZE - off;
463 cleanup_pagelistinfo(pagelistinfo);
469 pages[actual_pages] = pg;
473 /* do not try and release vmalloc pages */
475 down_read(&task->mm->mmap_sem);
476 actual_pages = get_user_pages(
477 (unsigned long)buf & PAGE_MASK,
479 (type == PAGELIST_READ) ? FOLL_WRITE : 0,
482 up_read(&task->mm->mmap_sem);
484 if (actual_pages != num_pages) {
485 vchiq_log_info(vchiq_arm_log_level,
486 "create_pagelist - only %d/%d pages locked",
490 /* This is probably due to the process being killed */
491 while (actual_pages > 0)
494 put_page(pages[actual_pages]);
496 cleanup_pagelistinfo(pagelistinfo);
499 /* release user pages */
500 pagelistinfo->pages_need_release = 1;
504 * Initialize the scatterlist so that the magic cookie
505 * is filled if debugging is enabled
507 sg_init_table(scatterlist, num_pages);
508 /* Now set the pages for each scatterlist */
509 for (i = 0; i < num_pages; i++) {
510 unsigned int len = PAGE_SIZE - offset;
514 sg_set_page(scatterlist + i, pages[i], len, offset);
519 dma_buffers = dma_map_sg(g_dev,
522 pagelistinfo->dma_dir);
524 if (dma_buffers == 0) {
525 cleanup_pagelistinfo(pagelistinfo);
529 pagelistinfo->scatterlist_mapped = 1;
531 /* Combine adjacent blocks for performance */
533 for_each_sg(scatterlist, sg, dma_buffers, i) {
534 u32 len = sg_dma_len(sg);
535 u32 addr = sg_dma_address(sg);
537 /* Note: addrs is the address + page_count - 1
538 * The firmware expects blocks after the first to be page-
539 * aligned and a multiple of the page size
542 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
543 WARN_ON(i && (addr & ~PAGE_MASK));
545 ((addrs[k - 1] & PAGE_MASK) +
546 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
547 == (addr & PAGE_MASK))
548 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
550 addrs[k++] = (addr & PAGE_MASK) |
551 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
554 /* Partial cache lines (fragments) require special measures */
555 if ((type == PAGELIST_READ) &&
556 ((pagelist->offset & (g_cache_line_size - 1)) ||
557 ((pagelist->offset + pagelist->length) &
558 (g_cache_line_size - 1)))) {
561 if (down_interruptible(&g_free_fragments_sema) != 0) {
562 cleanup_pagelistinfo(pagelistinfo);
566 WARN_ON(g_free_fragments == NULL);
568 down(&g_free_fragments_mutex);
569 fragments = g_free_fragments;
570 WARN_ON(fragments == NULL);
571 g_free_fragments = *(char **) g_free_fragments;
572 up(&g_free_fragments_mutex);
573 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
574 (fragments - g_fragments_base) / g_fragments_size;
581 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
584 PAGELIST_T *pagelist = pagelistinfo->pagelist;
585 struct page **pages = pagelistinfo->pages;
586 unsigned int num_pages = pagelistinfo->num_pages;
588 vchiq_log_trace(vchiq_arm_log_level, "free_pagelist - %pK, %d",
589 pagelistinfo->pagelist, actual);
592 * NOTE: dma_unmap_sg must be called before the
593 * cpu can touch any of the data/pages.
595 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
596 pagelistinfo->num_pages, pagelistinfo->dma_dir);
597 pagelistinfo->scatterlist_mapped = 0;
599 /* Deal with any partial cache lines (fragments) */
600 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
601 char *fragments = g_fragments_base +
602 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
604 int head_bytes, tail_bytes;
606 head_bytes = (g_cache_line_size - pagelist->offset) &
607 (g_cache_line_size - 1);
608 tail_bytes = (pagelist->offset + actual) &
609 (g_cache_line_size - 1);
611 if ((actual >= 0) && (head_bytes != 0)) {
612 if (head_bytes > actual)
615 memcpy((char *)page_address(pages[0]) +
620 if ((actual >= 0) && (head_bytes < actual) &&
622 memcpy((char *)page_address(pages[num_pages - 1]) +
623 ((pagelist->offset + actual) &
624 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
625 fragments + g_cache_line_size,
629 down(&g_free_fragments_mutex);
630 *(char **)fragments = g_free_fragments;
631 g_free_fragments = fragments;
632 up(&g_free_fragments_mutex);
633 up(&g_free_fragments_sema);
636 /* Need to mark all the pages dirty. */
637 if (pagelist->type != PAGELIST_WRITE &&
638 pagelistinfo->pages_need_release) {
641 for (i = 0; i < num_pages; i++)
642 set_page_dirty(pages[i]);
645 cleanup_pagelistinfo(pagelistinfo);