#include <linux/scatterlist.h>
#include <linux/log2.h>
#include <linux/dma-mapping.h>
+#include <linux/iommu-helper.h>
#include <asm/io.h>
#include <asm/hwrpb.h>
#endif
#define DEBUG_NODIRECT 0
-#define DEBUG_FORCEDAC 0
#define ISA_DMA_MASK 0x00ffffff
#ifdef CONFIG_DISCONTIGMEM
- if (!NODE_DATA(nid) ||
- (NULL == (arena = alloc_bootmem_node(NODE_DATA(nid),
- sizeof(*arena))))) {
- printk("%s: couldn't allocate arena from node %d\n"
- " falling back to system-wide allocation\n",
- __FUNCTION__, nid);
- arena = alloc_bootmem(sizeof(*arena));
- }
-
- if (!NODE_DATA(nid) ||
- (NULL == (arena->ptes = __alloc_bootmem_node(NODE_DATA(nid),
- mem_size,
- align,
- 0)))) {
- printk("%s: couldn't allocate arena ptes from node %d\n"
- " falling back to system-wide allocation\n",
- __FUNCTION__, nid);
- arena->ptes = __alloc_bootmem(mem_size, align, 0);
- }
+ arena = alloc_bootmem_node(NODE_DATA(nid), sizeof(*arena));
+ if (!NODE_DATA(nid) || !arena) {
+ printk("%s: couldn't allocate arena from node %d\n"
+ " falling back to system-wide allocation\n",
+ __func__, nid);
+ arena = alloc_bootmem(sizeof(*arena));
+ }
+
+ arena->ptes = __alloc_bootmem_node(NODE_DATA(nid), mem_size, align, 0);
+ if (!NODE_DATA(nid) || !arena->ptes) {
+ printk("%s: couldn't allocate arena ptes from node %d\n"
+ " falling back to system-wide allocation\n",
+ __func__, nid);
+ arena->ptes = __alloc_bootmem(mem_size, align, 0);
+ }
#else /* CONFIG_DISCONTIGMEM */
/* Must be called with the arena lock held */
static long
-iommu_arena_find_pages(struct pci_iommu_arena *arena, long n, long mask)
+iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
+ long n, long mask)
{
unsigned long *ptes;
long i, p, nent;
+ int pass = 0;
+ unsigned long base;
+ unsigned long boundary_size;
+
+ base = arena->dma_base >> PAGE_SHIFT;
+ if (dev) {
+ boundary_size = dma_get_seg_boundary(dev) + 1;
+ boundary_size >>= PAGE_SHIFT;
+ } else {
+ boundary_size = 1UL << (32 - PAGE_SHIFT);
+ }
/* Search forward for the first mask-aligned sequence of N free ptes */
ptes = arena->ptes;
nent = arena->size >> PAGE_SHIFT;
- p = (arena->next_entry + mask) & ~mask;
+ p = ALIGN(arena->next_entry, mask + 1);
i = 0;
+
+again:
while (i < n && p+i < nent) {
+ if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) {
+ p = ALIGN(p + 1, mask + 1);
+ goto again;
+ }
+
if (ptes[p+i])
- p = (p + i + 1 + mask) & ~mask, i = 0;
+ p = ALIGN(p + i + 1, mask + 1), i = 0;
else
i = i + 1;
}
if (i < n) {
- /* Reached the end. Flush the TLB and restart the
- search from the beginning. */
- alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
-
- p = 0, i = 0;
- while (i < n && p+i < nent) {
- if (ptes[p+i])
- p = (p + i + 1 + mask) & ~mask, i = 0;
- else
- i = i + 1;
- }
-
- if (i < n)
+ if (pass < 1) {
+ /*
+ * Reached the end. Flush the TLB and restart
+ * the search from the beginning.
+ */
+ alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
+
+ pass++;
+ p = 0;
+ i = 0;
+ goto again;
+ } else
return -1;
}
}
static long
-iommu_arena_alloc(struct pci_iommu_arena *arena, long n, unsigned int align)
+iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
+ unsigned int align)
{
unsigned long flags;
unsigned long *ptes;
/* Search for N empty ptes */
ptes = arena->ptes;
mask = max(align, arena->align_entry) - 1;
- p = iommu_arena_find_pages(arena, n, mask);
+ p = iommu_arena_find_pages(dev, arena, n, mask);
if (p < 0) {
spin_unlock_irqrestore(&arena->lock, flags);
return -1;
unsigned long paddr;
dma_addr_t ret;
unsigned int align = 0;
+ struct device *dev = pdev ? &pdev->dev : NULL;
paddr = __pa(cpu_addr);
/* Force allocation to 64KB boundary for ISA bridges. */
if (pdev && pdev == isa_bridge)
align = 8;
- dma_ofs = iommu_arena_alloc(arena, npages, align);
+ dma_ofs = iommu_arena_alloc(dev, arena, npages, align);
if (dma_ofs < 0) {
printk(KERN_WARNING "pci_map_single failed: "
"could not allocate dma page tables\n");
else DMA_ADDRP is undefined. */
void *
-pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp)
+__pci_alloc_consistent(struct pci_dev *pdev, size_t size,
+ dma_addr_t *dma_addrp, gfp_t gfp)
{
void *cpu_addr;
long order = get_order(size);
- gfp_t gfp = GFP_ATOMIC;
+
+ gfp &= ~GFP_DMA;
try_again:
cpu_addr = (void *)__get_free_pages(gfp, order);
return cpu_addr;
}
-EXPORT_SYMBOL(pci_alloc_consistent);
+EXPORT_SYMBOL(__pci_alloc_consistent);
/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must
be values that were returned from pci_alloc_consistent. SIZE must
paddr &= ~PAGE_MASK;
npages = calc_npages(paddr + size);
- dma_ofs = iommu_arena_alloc(arena, npages, 0);
+ dma_ofs = iommu_arena_alloc(dev, arena, npages, 0);
if (dma_ofs < 0) {
/* If we attempted a direct map above but failed, die. */
if (leader->dma_address == 0)
/* Search for N empty ptes. */
ptes = arena->ptes;
- p = iommu_arena_find_pages(arena, pg_count, align_mask);
+ p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask);
if (p < 0) {
spin_unlock_irqrestore(&arena->lock, flags);
return -1;
git.samba.org / sfrench / cifs-2.6.git / blobdiff