drivers/gpu/drm/gma500/mmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**************************************************************************
   3  * Copyright (c) 2007, Intel Corporation.
   4  *
   5  **************************************************************************/
   6 #include <drm/drmP.h>
   7 #include "psb_drv.h"
   8 #include "psb_reg.h"
   9 #include "mmu.h"
  10
  11 /*
  12  * Code for the SGX MMU:
  13  */
  14
  15 /*
  16  * clflush on one processor only:
  17  * clflush should apparently flush the cache line on all processors in an
  18  * SMP system.
  19  */
  20
  21 /*
  22  * kmap atomic:
  23  * The usage of the slots must be completely encapsulated within a spinlock, and
  24  * no other functions that may be using the locks for other purposed may be
  25  * called from within the locked region.
  26  * Since the slots are per processor, this will guarantee that we are the only
  27  * user.
  28  */
  29
  30 /*
  31  * TODO: Inserting ptes from an interrupt handler:
  32  * This may be desirable for some SGX functionality where the GPU can fault in
  33  * needed pages. For that, we need to make an atomic insert_pages function, that
  34  * may fail.
  35  * If it fails, the caller need to insert the page using a workqueue function,
  36  * but on average it should be fast.
  37  */
  38
  39 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
  40 {
  41         return (offset >> PSB_PTE_SHIFT) & 0x3FF;
  42 }
  43
  44 static inline uint32_t psb_mmu_pd_index(uint32_t offset)
  45 {
  46         return offset >> PSB_PDE_SHIFT;
  47 }
  48
  49 #if defined(CONFIG_X86)
  50 static inline void psb_clflush(void *addr)
  51 {
  52         __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
  53 }
  54
  55 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
  56 {
  57         if (!driver->has_clflush)
  58                 return;
  59
  60         mb();
  61         psb_clflush(addr);
  62         mb();
  63 }
  64 #else
  65
  66 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
  67 {;
  68 }
  69
  70 #endif
  71
  72 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
  73 {
  74         struct drm_device *dev = driver->dev;
  75         struct drm_psb_private *dev_priv = dev->dev_private;
  76
  77         if (atomic_read(&driver->needs_tlbflush) || force) {
  78                 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
  79                 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
  80
  81                 /* Make sure data cache is turned off before enabling it */
  82                 wmb();
  83                 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
  84                 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
  85                 if (driver->msvdx_mmu_invaldc)
  86                         atomic_set(driver->msvdx_mmu_invaldc, 1);
  87         }
  88         atomic_set(&driver->needs_tlbflush, 0);
  89 }
  90
  91 #if 0
  92 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
  93 {
  94         down_write(&driver->sem);
  95         psb_mmu_flush_pd_locked(driver, force);
  96         up_write(&driver->sem);
  97 }
  98 #endif
  99
 100 void psb_mmu_flush(struct psb_mmu_driver *driver)
 101 {
 102         struct drm_device *dev = driver->dev;
 103         struct drm_psb_private *dev_priv = dev->dev_private;
 104         uint32_t val;
 105
 106         down_write(&driver->sem);
 107         val = PSB_RSGX32(PSB_CR_BIF_CTRL);
 108         if (atomic_read(&driver->needs_tlbflush))
 109                 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
 110         else
 111                 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
 112
 113         /* Make sure data cache is turned off and MMU is flushed before
 114            restoring bank interface control register */
 115         wmb();
 116         PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
 117                    PSB_CR_BIF_CTRL);
 118         (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
 119
 120         atomic_set(&driver->needs_tlbflush, 0);
 121         if (driver->msvdx_mmu_invaldc)
 122                 atomic_set(driver->msvdx_mmu_invaldc, 1);
 123         up_write(&driver->sem);
 124 }
 125
 126 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
 127 {
 128         struct drm_device *dev = pd->driver->dev;
 129         struct drm_psb_private *dev_priv = dev->dev_private;
 130         uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
 131                           PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
 132
 133         down_write(&pd->driver->sem);
 134         PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
 135         wmb();
 136         psb_mmu_flush_pd_locked(pd->driver, 1);
 137         pd->hw_context = hw_context;
 138         up_write(&pd->driver->sem);
 139
 140 }
 141
 142 static inline unsigned long psb_pd_addr_end(unsigned long addr,
 143                                             unsigned long end)
 144 {
 145         addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
 146         return (addr < end) ? addr : end;
 147 }
 148
 149 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
 150 {
 151         uint32_t mask = PSB_PTE_VALID;
 152
 153         if (type & PSB_MMU_CACHED_MEMORY)
 154                 mask |= PSB_PTE_CACHED;
 155         if (type & PSB_MMU_RO_MEMORY)
 156                 mask |= PSB_PTE_RO;
 157         if (type & PSB_MMU_WO_MEMORY)
 158                 mask |= PSB_PTE_WO;
 159
 160         return (pfn << PAGE_SHIFT) | mask;
 161 }
 162
 163 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
 164                                     int trap_pagefaults, int invalid_type)
 165 {
 166         struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
 167         uint32_t *v;
 168         int i;
 169
 170         if (!pd)
 171                 return NULL;
 172
 173         pd->p = alloc_page(GFP_DMA32);
 174         if (!pd->p)
 175                 goto out_err1;
 176         pd->dummy_pt = alloc_page(GFP_DMA32);
 177         if (!pd->dummy_pt)
 178                 goto out_err2;
 179         pd->dummy_page = alloc_page(GFP_DMA32);
 180         if (!pd->dummy_page)
 181                 goto out_err3;
 182
 183         if (!trap_pagefaults) {
 184                 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
 185                                                    invalid_type);
 186                 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
 187                                                    invalid_type);
 188         } else {
 189                 pd->invalid_pde = 0;
 190                 pd->invalid_pte = 0;
 191         }
 192
 193         v = kmap(pd->dummy_pt);
 194         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 195                 v[i] = pd->invalid_pte;
 196
 197         kunmap(pd->dummy_pt);
 198
 199         v = kmap(pd->p);
 200         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 201                 v[i] = pd->invalid_pde;
 202
 203         kunmap(pd->p);
 204
 205         clear_page(kmap(pd->dummy_page));
 206         kunmap(pd->dummy_page);
 207
 208         pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
 209         if (!pd->tables)
 210                 goto out_err4;
 211
 212         pd->hw_context = -1;
 213         pd->pd_mask = PSB_PTE_VALID;
 214         pd->driver = driver;
 215
 216         return pd;
 217
 218 out_err4:
 219         __free_page(pd->dummy_page);
 220 out_err3:
 221         __free_page(pd->dummy_pt);
 222 out_err2:
 223         __free_page(pd->p);
 224 out_err1:
 225         kfree(pd);
 226         return NULL;
 227 }
 228
 229 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
 230 {
 231         __free_page(pt->p);
 232         kfree(pt);
 233 }
 234
 235 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
 236 {
 237         struct psb_mmu_driver *driver = pd->driver;
 238         struct drm_device *dev = driver->dev;
 239         struct drm_psb_private *dev_priv = dev->dev_private;
 240         struct psb_mmu_pt *pt;
 241         int i;
 242
 243         down_write(&driver->sem);
 244         if (pd->hw_context != -1) {
 245                 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
 246                 psb_mmu_flush_pd_locked(driver, 1);
 247         }
 248
 249         /* Should take the spinlock here, but we don't need to do that
 250            since we have the semaphore in write mode. */
 251
 252         for (i = 0; i < 1024; ++i) {
 253                 pt = pd->tables[i];
 254                 if (pt)
 255                         psb_mmu_free_pt(pt);
 256         }
 257
 258         vfree(pd->tables);
 259         __free_page(pd->dummy_page);
 260         __free_page(pd->dummy_pt);
 261         __free_page(pd->p);
 262         kfree(pd);
 263         up_write(&driver->sem);
 264 }
 265
 266 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
 267 {
 268         struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
 269         void *v;
 270         uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
 271         uint32_t clflush_count = PAGE_SIZE / clflush_add;
 272         spinlock_t *lock = &pd->driver->lock;
 273         uint8_t *clf;
 274         uint32_t *ptes;
 275         int i;
 276
 277         if (!pt)
 278                 return NULL;
 279
 280         pt->p = alloc_page(GFP_DMA32);
 281         if (!pt->p) {
 282                 kfree(pt);
 283                 return NULL;
 284         }
 285
 286         spin_lock(lock);
 287
 288         v = kmap_atomic(pt->p);
 289         clf = (uint8_t *) v;
 290         ptes = (uint32_t *) v;
 291         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 292                 *ptes++ = pd->invalid_pte;
 293
 294 #if defined(CONFIG_X86)
 295         if (pd->driver->has_clflush && pd->hw_context != -1) {
 296                 mb();
 297                 for (i = 0; i < clflush_count; ++i) {
 298                         psb_clflush(clf);
 299                         clf += clflush_add;
 300                 }
 301                 mb();
 302         }
 303 #endif
 304         kunmap_atomic(v);
 305         spin_unlock(lock);
 306
 307         pt->count = 0;
 308         pt->pd = pd;
 309         pt->index = 0;
 310
 311         return pt;
 312 }
 313
 314 struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
 315                                              unsigned long addr)
 316 {
 317         uint32_t index = psb_mmu_pd_index(addr);
 318         struct psb_mmu_pt *pt;
 319         uint32_t *v;
 320         spinlock_t *lock = &pd->driver->lock;
 321
 322         spin_lock(lock);
 323         pt = pd->tables[index];
 324         while (!pt) {
 325                 spin_unlock(lock);
 326                 pt = psb_mmu_alloc_pt(pd);
 327                 if (!pt)
 328                         return NULL;
 329                 spin_lock(lock);
 330
 331                 if (pd->tables[index]) {
 332                         spin_unlock(lock);
 333                         psb_mmu_free_pt(pt);
 334                         spin_lock(lock);
 335                         pt = pd->tables[index];
 336                         continue;
 337                 }
 338
 339                 v = kmap_atomic(pd->p);
 340                 pd->tables[index] = pt;
 341                 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
 342                 pt->index = index;
 343                 kunmap_atomic((void *) v);
 344
 345                 if (pd->hw_context != -1) {
 346                         psb_mmu_clflush(pd->driver, (void *)&v[index]);
 347                         atomic_set(&pd->driver->needs_tlbflush, 1);
 348                 }
 349         }
 350         pt->v = kmap_atomic(pt->p);
 351         return pt;
 352 }
 353
 354 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
 355                                               unsigned long addr)
 356 {
 357         uint32_t index = psb_mmu_pd_index(addr);
 358         struct psb_mmu_pt *pt;
 359         spinlock_t *lock = &pd->driver->lock;
 360
 361         spin_lock(lock);
 362         pt = pd->tables[index];
 363         if (!pt) {
 364                 spin_unlock(lock);
 365                 return NULL;
 366         }
 367         pt->v = kmap_atomic(pt->p);
 368         return pt;
 369 }
 370
 371 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
 372 {
 373         struct psb_mmu_pd *pd = pt->pd;
 374         uint32_t *v;
 375
 376         kunmap_atomic(pt->v);
 377         if (pt->count == 0) {
 378                 v = kmap_atomic(pd->p);
 379                 v[pt->index] = pd->invalid_pde;
 380                 pd->tables[pt->index] = NULL;
 381
 382                 if (pd->hw_context != -1) {
 383                         psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
 384                         atomic_set(&pd->driver->needs_tlbflush, 1);
 385                 }
 386                 kunmap_atomic(v);
 387                 spin_unlock(&pd->driver->lock);
 388                 psb_mmu_free_pt(pt);
 389                 return;
 390         }
 391         spin_unlock(&pd->driver->lock);
 392 }
 393
 394 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
 395                                    uint32_t pte)
 396 {
 397         pt->v[psb_mmu_pt_index(addr)] = pte;
 398 }
 399
 400 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
 401                                           unsigned long addr)
 402 {
 403         pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
 404 }
 405
 406 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
 407 {
 408         struct psb_mmu_pd *pd;
 409
 410         down_read(&driver->sem);
 411         pd = driver->default_pd;
 412         up_read(&driver->sem);
 413
 414         return pd;
 415 }
 416
 417 /* Returns the physical address of the PD shared by sgx/msvdx */
 418 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
 419 {
 420         struct psb_mmu_pd *pd;
 421
 422         pd = psb_mmu_get_default_pd(driver);
 423         return page_to_pfn(pd->p) << PAGE_SHIFT;
 424 }
 425
 426 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
 427 {
 428         struct drm_device *dev = driver->dev;
 429         struct drm_psb_private *dev_priv = dev->dev_private;
 430
 431         PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
 432         psb_mmu_free_pagedir(driver->default_pd);
 433         kfree(driver);
 434 }
 435
 436 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
 437                                            int trap_pagefaults,
 438                                            int invalid_type,
 439                                            atomic_t *msvdx_mmu_invaldc)
 440 {
 441         struct psb_mmu_driver *driver;
 442         struct drm_psb_private *dev_priv = dev->dev_private;
 443
 444         driver = kmalloc(sizeof(*driver), GFP_KERNEL);
 445
 446         if (!driver)
 447                 return NULL;
 448
 449         driver->dev = dev;
 450         driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
 451                                               invalid_type);
 452         if (!driver->default_pd)
 453                 goto out_err1;
 454
 455         spin_lock_init(&driver->lock);
 456         init_rwsem(&driver->sem);
 457         down_write(&driver->sem);
 458         atomic_set(&driver->needs_tlbflush, 1);
 459         driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
 460
 461         driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
 462         PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
 463                    PSB_CR_BIF_CTRL);
 464         PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
 465                    PSB_CR_BIF_CTRL);
 466
 467         driver->has_clflush = 0;
 468
 469 #if defined(CONFIG_X86)
 470         if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
 471                 uint32_t tfms, misc, cap0, cap4, clflush_size;
 472
 473                 /*
 474                  * clflush size is determined at kernel setup for x86_64 but not
 475                  * for i386. We have to do it here.
 476                  */
 477
 478                 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
 479                 clflush_size = ((misc >> 8) & 0xff) * 8;
 480                 driver->has_clflush = 1;
 481                 driver->clflush_add =
 482                     PAGE_SIZE * clflush_size / sizeof(uint32_t);
 483                 driver->clflush_mask = driver->clflush_add - 1;
 484                 driver->clflush_mask = ~driver->clflush_mask;
 485         }
 486 #endif
 487
 488         up_write(&driver->sem);
 489         return driver;
 490
 491 out_err1:
 492         kfree(driver);
 493         return NULL;
 494 }
 495
 496 #if defined(CONFIG_X86)
 497 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 498                                uint32_t num_pages, uint32_t desired_tile_stride,
 499                                uint32_t hw_tile_stride)
 500 {
 501         struct psb_mmu_pt *pt;
 502         uint32_t rows = 1;
 503         uint32_t i;
 504         unsigned long addr;
 505         unsigned long end;
 506         unsigned long next;
 507         unsigned long add;
 508         unsigned long row_add;
 509         unsigned long clflush_add = pd->driver->clflush_add;
 510         unsigned long clflush_mask = pd->driver->clflush_mask;
 511
 512         if (!pd->driver->has_clflush)
 513                 return;
 514
 515         if (hw_tile_stride)
 516                 rows = num_pages / desired_tile_stride;
 517         else
 518                 desired_tile_stride = num_pages;
 519
 520         add = desired_tile_stride << PAGE_SHIFT;
 521         row_add = hw_tile_stride << PAGE_SHIFT;
 522         mb();
 523         for (i = 0; i < rows; ++i) {
 524
 525                 addr = address;
 526                 end = addr + add;
 527
 528                 do {
 529                         next = psb_pd_addr_end(addr, end);
 530                         pt = psb_mmu_pt_map_lock(pd, addr);
 531                         if (!pt)
 532                                 continue;
 533                         do {
 534                                 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
 535                         } while (addr += clflush_add,
 536                                  (addr & clflush_mask) < next);
 537
 538                         psb_mmu_pt_unmap_unlock(pt);
 539                 } while (addr = next, next != end);
 540                 address += row_add;
 541         }
 542         mb();
 543 }
 544 #else
 545 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 546                                uint32_t num_pages, uint32_t desired_tile_stride,
 547                                uint32_t hw_tile_stride)
 548 {
 549         drm_ttm_cache_flush();
 550 }
 551 #endif
 552
 553 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
 554                                  unsigned long address, uint32_t num_pages)
 555 {
 556         struct psb_mmu_pt *pt;
 557         unsigned long addr;
 558         unsigned long end;
 559         unsigned long next;
 560         unsigned long f_address = address;
 561
 562         down_read(&pd->driver->sem);
 563
 564         addr = address;
 565         end = addr + (num_pages << PAGE_SHIFT);
 566
 567         do {
 568                 next = psb_pd_addr_end(addr, end);
 569                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 570                 if (!pt)
 571                         goto out;
 572                 do {
 573                         psb_mmu_invalidate_pte(pt, addr);
 574                         --pt->count;
 575                 } while (addr += PAGE_SIZE, addr < next);
 576                 psb_mmu_pt_unmap_unlock(pt);
 577
 578         } while (addr = next, next != end);
 579
 580 out:
 581         if (pd->hw_context != -1)
 582                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 583
 584         up_read(&pd->driver->sem);
 585
 586         if (pd->hw_context != -1)
 587                 psb_mmu_flush(pd->driver);
 588
 589         return;
 590 }
 591
 592 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
 593                           uint32_t num_pages, uint32_t desired_tile_stride,
 594                           uint32_t hw_tile_stride)
 595 {
 596         struct psb_mmu_pt *pt;
 597         uint32_t rows = 1;
 598         uint32_t i;
 599         unsigned long addr;
 600         unsigned long end;
 601         unsigned long next;
 602         unsigned long add;
 603         unsigned long row_add;
 604         unsigned long f_address = address;
 605
 606         if (hw_tile_stride)
 607                 rows = num_pages / desired_tile_stride;
 608         else
 609                 desired_tile_stride = num_pages;
 610
 611         add = desired_tile_stride << PAGE_SHIFT;
 612         row_add = hw_tile_stride << PAGE_SHIFT;
 613
 614         down_read(&pd->driver->sem);
 615
 616         /* Make sure we only need to flush this processor's cache */
 617
 618         for (i = 0; i < rows; ++i) {
 619
 620                 addr = address;
 621                 end = addr + add;
 622
 623                 do {
 624                         next = psb_pd_addr_end(addr, end);
 625                         pt = psb_mmu_pt_map_lock(pd, addr);
 626                         if (!pt)
 627                                 continue;
 628                         do {
 629                                 psb_mmu_invalidate_pte(pt, addr);
 630                                 --pt->count;
 631
 632                         } while (addr += PAGE_SIZE, addr < next);
 633                         psb_mmu_pt_unmap_unlock(pt);
 634
 635                 } while (addr = next, next != end);
 636                 address += row_add;
 637         }
 638         if (pd->hw_context != -1)
 639                 psb_mmu_flush_ptes(pd, f_address, num_pages,
 640                                    desired_tile_stride, hw_tile_stride);
 641
 642         up_read(&pd->driver->sem);
 643
 644         if (pd->hw_context != -1)
 645                 psb_mmu_flush(pd->driver);
 646 }
 647
 648 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
 649                                 unsigned long address, uint32_t num_pages,
 650                                 int type)
 651 {
 652         struct psb_mmu_pt *pt;
 653         uint32_t pte;
 654         unsigned long addr;
 655         unsigned long end;
 656         unsigned long next;
 657         unsigned long f_address = address;
 658         int ret = -ENOMEM;
 659
 660         down_read(&pd->driver->sem);
 661
 662         addr = address;
 663         end = addr + (num_pages << PAGE_SHIFT);
 664
 665         do {
 666                 next = psb_pd_addr_end(addr, end);
 667                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 668                 if (!pt) {
 669                         ret = -ENOMEM;
 670                         goto out;
 671                 }
 672                 do {
 673                         pte = psb_mmu_mask_pte(start_pfn++, type);
 674                         psb_mmu_set_pte(pt, addr, pte);
 675                         pt->count++;
 676                 } while (addr += PAGE_SIZE, addr < next);
 677                 psb_mmu_pt_unmap_unlock(pt);
 678
 679         } while (addr = next, next != end);
 680         ret = 0;
 681
 682 out:
 683         if (pd->hw_context != -1)
 684                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 685
 686         up_read(&pd->driver->sem);
 687
 688         if (pd->hw_context != -1)
 689                 psb_mmu_flush(pd->driver);
 690
 691         return 0;
 692 }
 693
 694 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
 695                          unsigned long address, uint32_t num_pages,
 696                          uint32_t desired_tile_stride, uint32_t hw_tile_stride,
 697                          int type)
 698 {
 699         struct psb_mmu_pt *pt;
 700         uint32_t rows = 1;
 701         uint32_t i;
 702         uint32_t pte;
 703         unsigned long addr;
 704         unsigned long end;
 705         unsigned long next;
 706         unsigned long add;
 707         unsigned long row_add;
 708         unsigned long f_address = address;
 709         int ret = -ENOMEM;
 710
 711         if (hw_tile_stride) {
 712                 if (num_pages % desired_tile_stride != 0)
 713                         return -EINVAL;
 714                 rows = num_pages / desired_tile_stride;
 715         } else {
 716                 desired_tile_stride = num_pages;
 717         }
 718
 719         add = desired_tile_stride << PAGE_SHIFT;
 720         row_add = hw_tile_stride << PAGE_SHIFT;
 721
 722         down_read(&pd->driver->sem);
 723
 724         for (i = 0; i < rows; ++i) {
 725
 726                 addr = address;
 727                 end = addr + add;
 728
 729                 do {
 730                         next = psb_pd_addr_end(addr, end);
 731                         pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 732                         if (!pt)
 733                                 goto out;
 734                         do {
 735                                 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
 736                                                        type);
 737                                 psb_mmu_set_pte(pt, addr, pte);
 738                                 pt->count++;
 739                         } while (addr += PAGE_SIZE, addr < next);
 740                         psb_mmu_pt_unmap_unlock(pt);
 741
 742                 } while (addr = next, next != end);
 743
 744                 address += row_add;
 745         }
 746
 747         ret = 0;
 748 out:
 749         if (pd->hw_context != -1)
 750                 psb_mmu_flush_ptes(pd, f_address, num_pages,
 751                                    desired_tile_stride, hw_tile_stride);
 752
 753         up_read(&pd->driver->sem);
 754
 755         if (pd->hw_context != -1)
 756                 psb_mmu_flush(pd->driver);
 757
 758         return ret;
 759 }
 760
 761 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
 762                            unsigned long *pfn)
 763 {
 764         int ret;
 765         struct psb_mmu_pt *pt;
 766         uint32_t tmp;
 767         spinlock_t *lock = &pd->driver->lock;
 768
 769         down_read(&pd->driver->sem);
 770         pt = psb_mmu_pt_map_lock(pd, virtual);
 771         if (!pt) {
 772                 uint32_t *v;
 773
 774                 spin_lock(lock);
 775                 v = kmap_atomic(pd->p);
 776                 tmp = v[psb_mmu_pd_index(virtual)];
 777                 kunmap_atomic(v);
 778                 spin_unlock(lock);
 779
 780                 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
 781                     !(pd->invalid_pte & PSB_PTE_VALID)) {
 782                         ret = -EINVAL;
 783                         goto out;
 784                 }
 785                 ret = 0;
 786                 *pfn = pd->invalid_pte >> PAGE_SHIFT;
 787                 goto out;
 788         }
 789         tmp = pt->v[psb_mmu_pt_index(virtual)];
 790         if (!(tmp & PSB_PTE_VALID)) {
 791                 ret = -EINVAL;
 792         } else {
 793                 ret = 0;
 794                 *pfn = tmp >> PAGE_SHIFT;
 795         }
 796         psb_mmu_pt_unmap_unlock(pt);
 797 out:
 798         up_read(&pd->driver->sem);
 799         return ret;
 800 }