drivers/gpu/drm/ttm/ttm_page_alloc.c

   1 /*
   2  * Copyright (c) Red Hat Inc.
   3
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors: Dave Airlie <airlied@redhat.com>
  24  *          Jerome Glisse <jglisse@redhat.com>
  25  *          Pauli Nieminen <suokkos@gmail.com>
  26  */
  27
  28 /* simple list based uncached page pool
  29  * - Pool collects resently freed pages for reuse
  30  * - Use page->lru to keep a free list
  31  * - doesn't track currently in use pages
  32  */
  33 #include <linux/list.h>
  34 #include <linux/spinlock.h>
  35 #include <linux/highmem.h>
  36 #include <linux/mm_types.h>
  37 #include <linux/module.h>
  38 #include <linux/mm.h>
  39
  40 #include <asm/atomic.h>
  41 #include <asm/agp.h>
  42
  43 #include "ttm/ttm_bo_driver.h"
  44 #include "ttm/ttm_page_alloc.h"
  45
  46
  47 #define NUM_PAGES_TO_ALLOC              (PAGE_SIZE/sizeof(struct page *))
  48 #define SMALL_ALLOCATION                16
  49 #define FREE_ALL_PAGES                  (~0U)
  50 /* times are in msecs */
  51 #define PAGE_FREE_INTERVAL              1000
  52
  53 /**
  54  * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
  55  *
  56  * @lock: Protects the shared pool from concurrnet access. Must be used with
  57  * irqsave/irqrestore variants because pool allocator maybe called from
  58  * delayed work.
  59  * @fill_lock: Prevent concurrent calls to fill.
  60  * @list: Pool of free uc/wc pages for fast reuse.
  61  * @gfp_flags: Flags to pass for alloc_page.
  62  * @npages: Number of pages in pool.
  63  */
  64 struct ttm_page_pool {
  65         spinlock_t              lock;
  66         bool                    fill_lock;
  67         struct list_head        list;
  68         int                     gfp_flags;
  69         unsigned                npages;
  70         char                    *name;
  71         unsigned long           nfrees;
  72         unsigned long           nrefills;
  73 };
  74
  75 /**
  76  * Limits for the pool. They are handled without locks because only place where
  77  * they may change is in sysfs store. They won't have immediate effect anyway
  78  * so forcing serialiazation to access them is pointless.
  79  */
  80
  81 struct ttm_pool_opts {
  82         unsigned        alloc_size;
  83         unsigned        max_size;
  84         unsigned        small;
  85 };
  86
  87 #define NUM_POOLS 4
  88
  89 /**
  90  * struct ttm_pool_manager - Holds memory pools for fst allocation
  91  *
  92  * Manager is read only object for pool code so it doesn't need locking.
  93  *
  94  * @free_interval: minimum number of jiffies between freeing pages from pool.
  95  * @page_alloc_inited: reference counting for pool allocation.
  96  * @work: Work that is used to shrink the pool. Work is only run when there is
  97  * some pages to free.
  98  * @small_allocation: Limit in number of pages what is small allocation.
  99  *
 100  * @pools: All pool objects in use.
 101  **/
 102 struct ttm_pool_manager {
 103         struct kobject          kobj;
 104         struct shrinker         mm_shrink;
 105         atomic_t                page_alloc_inited;
 106         struct ttm_pool_opts    options;
 107
 108         union {
 109                 struct ttm_page_pool    pools[NUM_POOLS];
 110                 struct {
 111                         struct ttm_page_pool    wc_pool;
 112                         struct ttm_page_pool    uc_pool;
 113                         struct ttm_page_pool    wc_pool_dma32;
 114                         struct ttm_page_pool    uc_pool_dma32;
 115                 } ;
 116         };
 117 };
 118
 119 static struct attribute ttm_page_pool_max = {
 120         .name = "pool_max_size",
 121         .mode = S_IRUGO | S_IWUSR
 122 };
 123 static struct attribute ttm_page_pool_small = {
 124         .name = "pool_small_allocation",
 125         .mode = S_IRUGO | S_IWUSR
 126 };
 127 static struct attribute ttm_page_pool_alloc_size = {
 128         .name = "pool_allocation_size",
 129         .mode = S_IRUGO | S_IWUSR
 130 };
 131
 132 static struct attribute *ttm_pool_attrs[] = {
 133         &ttm_page_pool_max,
 134         &ttm_page_pool_small,
 135         &ttm_page_pool_alloc_size,
 136         NULL
 137 };
 138
 139 static void ttm_pool_kobj_release(struct kobject *kobj)
 140 {
 141         struct ttm_pool_manager *m =
 142                 container_of(kobj, struct ttm_pool_manager, kobj);
 143         (void)m;
 144 }
 145
 146 static ssize_t ttm_pool_store(struct kobject *kobj,
 147                 struct attribute *attr, const char *buffer, size_t size)
 148 {
 149         struct ttm_pool_manager *m =
 150                 container_of(kobj, struct ttm_pool_manager, kobj);
 151         int chars;
 152         unsigned val;
 153         chars = sscanf(buffer, "%u", &val);
 154         if (chars == 0)
 155                 return size;
 156
 157         /* Convert kb to number of pages */
 158         val = val / (PAGE_SIZE >> 10);
 159
 160         if (attr == &ttm_page_pool_max)
 161                 m->options.max_size = val;
 162         else if (attr == &ttm_page_pool_small)
 163                 m->options.small = val;
 164         else if (attr == &ttm_page_pool_alloc_size) {
 165                 if (val > NUM_PAGES_TO_ALLOC*8) {
 166                         printk(KERN_ERR "[ttm] Setting allocation size to %lu "
 167                                         "is not allowed. Recomended size is "
 168                                         "%lu\n",
 169                                         NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
 170                                         NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 171                         return size;
 172                 } else if (val > NUM_PAGES_TO_ALLOC) {
 173                         printk(KERN_WARNING "[ttm] Setting allocation size to "
 174                                         "larger than %lu is not recomended.\n",
 175                                         NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 176                 }
 177                 m->options.alloc_size = val;
 178         }
 179
 180         return size;
 181 }
 182
 183 static ssize_t ttm_pool_show(struct kobject *kobj,
 184                 struct attribute *attr, char *buffer)
 185 {
 186         struct ttm_pool_manager *m =
 187                 container_of(kobj, struct ttm_pool_manager, kobj);
 188         unsigned val = 0;
 189
 190         if (attr == &ttm_page_pool_max)
 191                 val = m->options.max_size;
 192         else if (attr == &ttm_page_pool_small)
 193                 val = m->options.small;
 194         else if (attr == &ttm_page_pool_alloc_size)
 195                 val = m->options.alloc_size;
 196
 197         val = val * (PAGE_SIZE >> 10);
 198
 199         return snprintf(buffer, PAGE_SIZE, "%u\n", val);
 200 }
 201
 202 static const struct sysfs_ops ttm_pool_sysfs_ops = {
 203         .show = &ttm_pool_show,
 204         .store = &ttm_pool_store,
 205 };
 206
 207 static struct kobj_type ttm_pool_kobj_type = {
 208         .release = &ttm_pool_kobj_release,
 209         .sysfs_ops = &ttm_pool_sysfs_ops,
 210         .default_attrs = ttm_pool_attrs,
 211 };
 212
 213 static struct ttm_pool_manager _manager = {
 214         .page_alloc_inited      = ATOMIC_INIT(0)
 215 };
 216
 217 #ifndef CONFIG_X86
 218 static int set_pages_array_wb(struct page **pages, int addrinarray)
 219 {
 220 #ifdef TTM_HAS_AGP
 221         int i;
 222
 223         for (i = 0; i < addrinarray; i++)
 224                 unmap_page_from_agp(pages[i]);
 225 #endif
 226         return 0;
 227 }
 228
 229 static int set_pages_array_wc(struct page **pages, int addrinarray)
 230 {
 231 #ifdef TTM_HAS_AGP
 232         int i;
 233
 234         for (i = 0; i < addrinarray; i++)
 235                 map_page_into_agp(pages[i]);
 236 #endif
 237         return 0;
 238 }
 239
 240 static int set_pages_array_uc(struct page **pages, int addrinarray)
 241 {
 242 #ifdef TTM_HAS_AGP
 243         int i;
 244
 245         for (i = 0; i < addrinarray; i++)
 246                 map_page_into_agp(pages[i]);
 247 #endif
 248         return 0;
 249 }
 250 #endif
 251
 252 /**
 253  * Select the right pool or requested caching state and ttm flags. */
 254 static struct ttm_page_pool *ttm_get_pool(int flags,
 255                 enum ttm_caching_state cstate)
 256 {
 257         int pool_index;
 258
 259         if (cstate == tt_cached)
 260                 return NULL;
 261
 262         if (cstate == tt_wc)
 263                 pool_index = 0x0;
 264         else
 265                 pool_index = 0x1;
 266
 267         if (flags & TTM_PAGE_FLAG_DMA32)
 268                 pool_index |= 0x2;
 269
 270         return &_manager.pools[pool_index];
 271 }
 272
 273 /* set memory back to wb and free the pages. */
 274 static void ttm_pages_put(struct page *pages[], unsigned npages)
 275 {
 276         unsigned i;
 277         if (set_pages_array_wb(pages, npages))
 278                 printk(KERN_ERR "[ttm] Failed to set %d pages to wb!\n",
 279                                 npages);
 280         for (i = 0; i < npages; ++i)
 281                 __free_page(pages[i]);
 282 }
 283
 284 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
 285                 unsigned freed_pages)
 286 {
 287         pool->npages -= freed_pages;
 288         pool->nfrees += freed_pages;
 289 }
 290
 291 /**
 292  * Free pages from pool.
 293  *
 294  * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
 295  * number of pages in one go.
 296  *
 297  * @pool: to free the pages from
 298  * @free_all: If set to true will free all pages in pool
 299  **/
 300 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
 301 {
 302         unsigned long irq_flags;
 303         struct page *p;
 304         struct page **pages_to_free;
 305         unsigned freed_pages = 0,
 306                  npages_to_free = nr_free;
 307
 308         if (NUM_PAGES_TO_ALLOC < nr_free)
 309                 npages_to_free = NUM_PAGES_TO_ALLOC;
 310
 311         pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
 312                         GFP_KERNEL);
 313         if (!pages_to_free) {
 314                 printk(KERN_ERR "Failed to allocate memory for pool free operation.\n");
 315                 return 0;
 316         }
 317
 318 restart:
 319         spin_lock_irqsave(&pool->lock, irq_flags);
 320
 321         list_for_each_entry_reverse(p, &pool->list, lru) {
 322                 if (freed_pages >= npages_to_free)
 323                         break;
 324
 325                 pages_to_free[freed_pages++] = p;
 326                 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
 327                 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
 328                         /* remove range of pages from the pool */
 329                         __list_del(p->lru.prev, &pool->list);
 330
 331                         ttm_pool_update_free_locked(pool, freed_pages);
 332                         /**
 333                          * Because changing page caching is costly
 334                          * we unlock the pool to prevent stalling.
 335                          */
 336                         spin_unlock_irqrestore(&pool->lock, irq_flags);
 337
 338                         ttm_pages_put(pages_to_free, freed_pages);
 339                         if (likely(nr_free != FREE_ALL_PAGES))
 340                                 nr_free -= freed_pages;
 341
 342                         if (NUM_PAGES_TO_ALLOC >= nr_free)
 343                                 npages_to_free = nr_free;
 344                         else
 345                                 npages_to_free = NUM_PAGES_TO_ALLOC;
 346
 347                         freed_pages = 0;
 348
 349                         /* free all so restart the processing */
 350                         if (nr_free)
 351                                 goto restart;
 352
 353                         /* Not allowed to fall tough or break because
 354                          * following context is inside spinlock while we are
 355                          * outside here.
 356                          */
 357                         goto out;
 358
 359                 }
 360         }
 361
 362         /* remove range of pages from the pool */
 363         if (freed_pages) {
 364                 __list_del(&p->lru, &pool->list);
 365
 366                 ttm_pool_update_free_locked(pool, freed_pages);
 367                 nr_free -= freed_pages;
 368         }
 369
 370         spin_unlock_irqrestore(&pool->lock, irq_flags);
 371
 372         if (freed_pages)
 373                 ttm_pages_put(pages_to_free, freed_pages);
 374 out:
 375         kfree(pages_to_free);
 376         return nr_free;
 377 }
 378
 379 /* Get good estimation how many pages are free in pools */
 380 static int ttm_pool_get_num_unused_pages(void)
 381 {
 382         unsigned i;
 383         int total = 0;
 384         for (i = 0; i < NUM_POOLS; ++i)
 385                 total += _manager.pools[i].npages;
 386
 387         return total;
 388 }
 389
 390 /**
 391  * Calback for mm to request pool to reduce number of page held.
 392  */
 393 static int ttm_pool_mm_shrink(int shrink_pages, gfp_t gfp_mask)
 394 {
 395         static atomic_t start_pool = ATOMIC_INIT(0);
 396         unsigned i;
 397         unsigned pool_offset = atomic_add_return(1, &start_pool);
 398         struct ttm_page_pool *pool;
 399
 400         pool_offset = pool_offset % NUM_POOLS;
 401         /* select start pool in round robin fashion */
 402         for (i = 0; i < NUM_POOLS; ++i) {
 403                 unsigned nr_free = shrink_pages;
 404                 if (shrink_pages == 0)
 405                         break;
 406                 pool = &_manager.pools[(i + pool_offset)%NUM_POOLS];
 407                 shrink_pages = ttm_page_pool_free(pool, nr_free);
 408         }
 409         /* return estimated number of unused pages in pool */
 410         return ttm_pool_get_num_unused_pages();
 411 }
 412
 413 static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
 414 {
 415         manager->mm_shrink.shrink = &ttm_pool_mm_shrink;
 416         manager->mm_shrink.seeks = 1;
 417         register_shrinker(&manager->mm_shrink);
 418 }
 419
 420 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
 421 {
 422         unregister_shrinker(&manager->mm_shrink);
 423 }
 424
 425 static int ttm_set_pages_caching(struct page **pages,
 426                 enum ttm_caching_state cstate, unsigned cpages)
 427 {
 428         int r = 0;
 429         /* Set page caching */
 430         switch (cstate) {
 431         case tt_uncached:
 432                 r = set_pages_array_uc(pages, cpages);
 433                 if (r)
 434                         printk(KERN_ERR "[ttm] Failed to set %d pages to uc!\n",
 435                                         cpages);
 436                 break;
 437         case tt_wc:
 438                 r = set_pages_array_wc(pages, cpages);
 439                 if (r)
 440                         printk(KERN_ERR "[ttm] Failed to set %d pages to wc!\n",
 441                                         cpages);
 442                 break;
 443         default:
 444                 break;
 445         }
 446         return r;
 447 }
 448
 449 /**
 450  * Free pages the pages that failed to change the caching state. If there is
 451  * any pages that have changed their caching state already put them to the
 452  * pool.
 453  */
 454 static void ttm_handle_caching_state_failure(struct list_head *pages,
 455                 int ttm_flags, enum ttm_caching_state cstate,
 456                 struct page **failed_pages, unsigned cpages)
 457 {
 458         unsigned i;
 459         /* Failed pages has to be reed */
 460         for (i = 0; i < cpages; ++i) {
 461                 list_del(&failed_pages[i]->lru);
 462                 __free_page(failed_pages[i]);
 463         }
 464 }
 465
 466 /**
 467  * Allocate new pages with correct caching.
 468  *
 469  * This function is reentrant if caller updates count depending on number of
 470  * pages returned in pages array.
 471  */
 472 static int ttm_alloc_new_pages(struct list_head *pages, int gfp_flags,
 473                 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
 474 {
 475         struct page **caching_array;
 476         struct page *p;
 477         int r = 0;
 478         unsigned i, cpages;
 479         unsigned max_cpages = min(count,
 480                         (unsigned)(PAGE_SIZE/sizeof(struct page *)));
 481
 482         /* allocate array for page caching change */
 483         caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
 484
 485         if (!caching_array) {
 486                 printk(KERN_ERR "[ttm] unable to allocate table for new pages.");
 487                 return -ENOMEM;
 488         }
 489
 490         for (i = 0, cpages = 0; i < count; ++i) {
 491                 p = alloc_page(gfp_flags);
 492
 493                 if (!p) {
 494                         printk(KERN_ERR "[ttm] unable to get page %u\n", i);
 495
 496                         /* store already allocated pages in the pool after
 497                          * setting the caching state */
 498                         if (cpages) {
 499                                 r = ttm_set_pages_caching(caching_array, cstate, cpages);
 500                                 if (r)
 501                                         ttm_handle_caching_state_failure(pages,
 502                                                 ttm_flags, cstate,
 503                                                 caching_array, cpages);
 504                         }
 505                         r = -ENOMEM;
 506                         goto out;
 507                 }
 508
 509 #ifdef CONFIG_HIGHMEM
 510                 /* gfp flags of highmem page should never be dma32 so we
 511                  * we should be fine in such case
 512                  */
 513                 if (!PageHighMem(p))
 514 #endif
 515                 {
 516                         caching_array[cpages++] = p;
 517                         if (cpages == max_cpages) {
 518
 519                                 r = ttm_set_pages_caching(caching_array,
 520                                                 cstate, cpages);
 521                                 if (r) {
 522                                         ttm_handle_caching_state_failure(pages,
 523                                                 ttm_flags, cstate,
 524                                                 caching_array, cpages);
 525                                         goto out;
 526                                 }
 527                                 cpages = 0;
 528                         }
 529                 }
 530
 531                 list_add(&p->lru, pages);
 532         }
 533
 534         if (cpages) {
 535                 r = ttm_set_pages_caching(caching_array, cstate, cpages);
 536                 if (r)
 537                         ttm_handle_caching_state_failure(pages,
 538                                         ttm_flags, cstate,
 539                                         caching_array, cpages);
 540         }
 541 out:
 542         kfree(caching_array);
 543
 544         return r;
 545 }
 546
 547 /**
 548  * Fill the given pool if there isn't enough pages and requested number of
 549  * pages is small.
 550  */
 551 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 552                 int ttm_flags, enum ttm_caching_state cstate, unsigned count,
 553                 unsigned long *irq_flags)
 554 {
 555         struct page *p;
 556         int r;
 557         unsigned cpages = 0;
 558         /**
 559          * Only allow one pool fill operation at a time.
 560          * If pool doesn't have enough pages for the allocation new pages are
 561          * allocated from outside of pool.
 562          */
 563         if (pool->fill_lock)
 564                 return;
 565
 566         pool->fill_lock = true;
 567
 568         /* If allocation request is small and there is not enough
 569          * pages in pool we fill the pool first */
 570         if (count < _manager.options.small
 571                 && count > pool->npages) {
 572                 struct list_head new_pages;
 573                 unsigned alloc_size = _manager.options.alloc_size;
 574
 575                 /**
 576                  * Can't change page caching if in irqsave context. We have to
 577                  * drop the pool->lock.
 578                  */
 579                 spin_unlock_irqrestore(&pool->lock, *irq_flags);
 580
 581                 INIT_LIST_HEAD(&new_pages);
 582                 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
 583                                 cstate, alloc_size);
 584                 spin_lock_irqsave(&pool->lock, *irq_flags);
 585
 586                 if (!r) {
 587                         list_splice(&new_pages, &pool->list);
 588                         ++pool->nrefills;
 589                         pool->npages += alloc_size;
 590                 } else {
 591                         printk(KERN_ERR "[ttm] Failed to fill pool (%p).", pool);
 592                         /* If we have any pages left put them to the pool. */
 593                         list_for_each_entry(p, &pool->list, lru) {
 594                                 ++cpages;
 595                         }
 596                         list_splice(&new_pages, &pool->list);
 597                         pool->npages += cpages;
 598                 }
 599
 600         }
 601         pool->fill_lock = false;
 602 }
 603
 604 /**
 605  * Cut count nubmer of pages from the pool and put them to return list
 606  *
 607  * @return count of pages still to allocate to fill the request.
 608  */
 609 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
 610                 struct list_head *pages, int ttm_flags,
 611                 enum ttm_caching_state cstate, unsigned count)
 612 {
 613         unsigned long irq_flags;
 614         struct list_head *p;
 615         unsigned i;
 616
 617         spin_lock_irqsave(&pool->lock, irq_flags);
 618         ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
 619
 620         if (count >= pool->npages) {
 621                 /* take all pages from the pool */
 622                 list_splice_init(&pool->list, pages);
 623                 count -= pool->npages;
 624                 pool->npages = 0;
 625                 goto out;
 626         }
 627         /* find the last pages to include for requested number of pages. Split
 628          * pool to begin and halves to reduce search space. */
 629         if (count <= pool->npages/2) {
 630                 i = 0;
 631                 list_for_each(p, &pool->list) {
 632                         if (++i == count)
 633                                 break;
 634                 }
 635         } else {
 636                 i = pool->npages + 1;
 637                 list_for_each_prev(p, &pool->list) {
 638                         if (--i == count)
 639                                 break;
 640                 }
 641         }
 642         /* Cut count number of pages from pool */
 643         list_cut_position(pages, &pool->list, p);
 644         pool->npages -= count;
 645         count = 0;
 646 out:
 647         spin_unlock_irqrestore(&pool->lock, irq_flags);
 648         return count;
 649 }
 650
 651 /*
 652  * On success pages list will hold count number of correctly
 653  * cached pages.
 654  */
 655 int ttm_get_pages(struct list_head *pages, int flags,
 656                 enum ttm_caching_state cstate, unsigned count)
 657 {
 658         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 659         struct page *p = NULL;
 660         int gfp_flags = 0;
 661         int r;
 662
 663         /* set zero flag for page allocation if required */
 664         if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
 665                 gfp_flags |= __GFP_ZERO;
 666
 667         /* No pool for cached pages */
 668         if (pool == NULL) {
 669                 if (flags & TTM_PAGE_FLAG_DMA32)
 670                         gfp_flags |= GFP_DMA32;
 671                 else
 672                         gfp_flags |= __GFP_HIGHMEM;
 673
 674                 for (r = 0; r < count; ++r) {
 675                         p = alloc_page(gfp_flags);
 676                         if (!p) {
 677
 678                                 printk(KERN_ERR "[ttm] unable to allocate page.");
 679                                 return -ENOMEM;
 680                         }
 681
 682                         list_add(&p->lru, pages);
 683                 }
 684                 return 0;
 685         }
 686
 687
 688         /* combine zero flag to pool flags */
 689         gfp_flags |= pool->gfp_flags;
 690
 691         /* First we take pages from the pool */
 692         count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count);
 693
 694         /* clear the pages coming from the pool if requested */
 695         if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
 696                 list_for_each_entry(p, pages, lru) {
 697                         clear_page(page_address(p));
 698                 }
 699         }
 700
 701         /* If pool didn't have enough pages allocate new one. */
 702         if (count > 0) {
 703                 /* ttm_alloc_new_pages doesn't reference pool so we can run
 704                  * multiple requests in parallel.
 705                  **/
 706                 r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count);
 707                 if (r) {
 708                         /* If there is any pages in the list put them back to
 709                          * the pool. */
 710                         printk(KERN_ERR "[ttm] Failed to allocate extra pages "
 711                                         "for large request.");
 712                         ttm_put_pages(pages, 0, flags, cstate);
 713                         return r;
 714                 }
 715         }
 716
 717
 718         return 0;
 719 }
 720
 721 /* Put all pages in pages list to correct pool to wait for reuse */
 722 void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
 723                 enum ttm_caching_state cstate)
 724 {
 725         unsigned long irq_flags;
 726         struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 727         struct page *p, *tmp;
 728
 729         if (pool == NULL) {
 730                 /* No pool for this memory type so free the pages */
 731
 732                 list_for_each_entry_safe(p, tmp, pages, lru) {
 733                         __free_page(p);
 734                 }
 735                 /* Make the pages list empty */
 736                 INIT_LIST_HEAD(pages);
 737                 return;
 738         }
 739         if (page_count == 0) {
 740                 list_for_each_entry_safe(p, tmp, pages, lru) {
 741                         ++page_count;
 742                 }
 743         }
 744
 745         spin_lock_irqsave(&pool->lock, irq_flags);
 746         list_splice_init(pages, &pool->list);
 747         pool->npages += page_count;
 748         /* Check that we don't go over the pool limit */
 749         page_count = 0;
 750         if (pool->npages > _manager.options.max_size) {
 751                 page_count = pool->npages - _manager.options.max_size;
 752                 /* free at least NUM_PAGES_TO_ALLOC number of pages
 753                  * to reduce calls to set_memory_wb */
 754                 if (page_count < NUM_PAGES_TO_ALLOC)
 755                         page_count = NUM_PAGES_TO_ALLOC;
 756         }
 757         spin_unlock_irqrestore(&pool->lock, irq_flags);
 758         if (page_count)
 759                 ttm_page_pool_free(pool, page_count);
 760 }
 761
 762 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
 763                 char *name)
 764 {
 765         spin_lock_init(&pool->lock);
 766         pool->fill_lock = false;
 767         INIT_LIST_HEAD(&pool->list);
 768         pool->npages = pool->nfrees = 0;
 769         pool->gfp_flags = flags;
 770         pool->name = name;
 771 }
 772
 773 int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 774 {
 775         int ret;
 776         if (atomic_add_return(1, &_manager.page_alloc_inited) > 1)
 777                 return 0;
 778
 779         printk(KERN_INFO "[ttm] Initializing pool allocator.\n");
 780
 781         ttm_page_pool_init_locked(&_manager.wc_pool, GFP_HIGHUSER, "wc");
 782
 783         ttm_page_pool_init_locked(&_manager.uc_pool, GFP_HIGHUSER, "uc");
 784
 785         ttm_page_pool_init_locked(&_manager.wc_pool_dma32, GFP_USER | GFP_DMA32,
 786                         "wc dma");
 787
 788         ttm_page_pool_init_locked(&_manager.uc_pool_dma32, GFP_USER | GFP_DMA32,
 789                         "uc dma");
 790
 791         _manager.options.max_size = max_pages;
 792         _manager.options.small = SMALL_ALLOCATION;
 793         _manager.options.alloc_size = NUM_PAGES_TO_ALLOC;
 794
 795         kobject_init(&_manager.kobj, &ttm_pool_kobj_type);
 796         ret = kobject_add(&_manager.kobj, &glob->kobj, "pool");
 797         if (unlikely(ret != 0)) {
 798                 kobject_put(&_manager.kobj);
 799                 return ret;
 800         }
 801
 802         ttm_pool_mm_shrink_init(&_manager);
 803
 804         return 0;
 805 }
 806
 807 void ttm_page_alloc_fini()
 808 {
 809         int i;
 810
 811         if (atomic_sub_return(1, &_manager.page_alloc_inited) > 0)
 812                 return;
 813
 814         printk(KERN_INFO "[ttm] Finilizing pool allocator.\n");
 815         ttm_pool_mm_shrink_fini(&_manager);
 816
 817         for (i = 0; i < NUM_POOLS; ++i)
 818                 ttm_page_pool_free(&_manager.pools[i], FREE_ALL_PAGES);
 819
 820         kobject_put(&_manager.kobj);
 821 }
 822
 823 int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
 824 {
 825         struct ttm_page_pool *p;
 826         unsigned i;
 827         char *h[] = {"pool", "refills", "pages freed", "size"};
 828         if (atomic_read(&_manager.page_alloc_inited) == 0) {
 829                 seq_printf(m, "No pool allocator running.\n");
 830                 return 0;
 831         }
 832         seq_printf(m, "%6s %12s %13s %8s\n",
 833                         h[0], h[1], h[2], h[3]);
 834         for (i = 0; i < NUM_POOLS; ++i) {
 835                 p = &_manager.pools[i];
 836
 837                 seq_printf(m, "%6s %12ld %13ld %8d\n",
 838                                 p->name, p->nrefills,
 839                                 p->nfrees, p->npages);
 840         }
 841         return 0;
 842 }
 843 EXPORT_SYMBOL(ttm_page_alloc_debugfs);