mm/hmm.c

   1 /*
   2  * Copyright 2013 Red Hat Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * Authors: JÃ©rÃ´me Glisse <jglisse@redhat.com>
  15  */
  16 /*
  17  * Refer to include/linux/hmm.h for information about heterogeneous memory
  18  * management or HMM for short.
  19  */
  20 #include <linux/mm.h>
  21 #include <linux/hmm.h>
  22 #include <linux/init.h>
  23 #include <linux/rmap.h>
  24 #include <linux/swap.h>
  25 #include <linux/slab.h>
  26 #include <linux/sched.h>
  27 #include <linux/mmzone.h>
  28 #include <linux/pagemap.h>
  29 #include <linux/swapops.h>
  30 #include <linux/hugetlb.h>
  31 #include <linux/memremap.h>
  32 #include <linux/jump_label.h>
  33 #include <linux/mmu_notifier.h>
  34 #include <linux/memory_hotplug.h>
  35
  36 #define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
  37
  38 #if defined(CONFIG_DEVICE_PRIVATE) || defined(CONFIG_DEVICE_PUBLIC)
  39 /*
  40  * Device private memory see HMM (Documentation/vm/hmm.txt) or hmm.h
  41  */
  42 DEFINE_STATIC_KEY_FALSE(device_private_key);
  43 EXPORT_SYMBOL(device_private_key);
  44 #endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */
  45
  46
  47 #if IS_ENABLED(CONFIG_HMM_MIRROR)
  48 static const struct mmu_notifier_ops hmm_mmu_notifier_ops;
  49
  50 /*
  51  * struct hmm - HMM per mm struct
  52  *
  53  * @mm: mm struct this HMM struct is bound to
  54  * @lock: lock protecting ranges list
  55  * @sequence: we track updates to the CPU page table with a sequence number
  56  * @ranges: list of range being snapshotted
  57  * @mirrors: list of mirrors for this mm
  58  * @mmu_notifier: mmu notifier to track updates to CPU page table
  59  * @mirrors_sem: read/write semaphore protecting the mirrors list
  60  */
  61 struct hmm {
  62         struct mm_struct        *mm;
  63         spinlock_t              lock;
  64         atomic_t                sequence;
  65         struct list_head        ranges;
  66         struct list_head        mirrors;
  67         struct mmu_notifier     mmu_notifier;
  68         struct rw_semaphore     mirrors_sem;
  69 };
  70
  71 /*
  72  * hmm_register - register HMM against an mm (HMM internal)
  73  *
  74  * @mm: mm struct to attach to
  75  *
  76  * This is not intended to be used directly by device drivers. It allocates an
  77  * HMM struct if mm does not have one, and initializes it.
  78  */
  79 static struct hmm *hmm_register(struct mm_struct *mm)
  80 {
  81         struct hmm *hmm = READ_ONCE(mm->hmm);
  82         bool cleanup = false;
  83
  84         /*
  85          * The hmm struct can only be freed once the mm_struct goes away,
  86          * hence we should always have pre-allocated an new hmm struct
  87          * above.
  88          */
  89         if (hmm)
  90                 return hmm;
  91
  92         hmm = kmalloc(sizeof(*hmm), GFP_KERNEL);
  93         if (!hmm)
  94                 return NULL;
  95         INIT_LIST_HEAD(&hmm->mirrors);
  96         init_rwsem(&hmm->mirrors_sem);
  97         atomic_set(&hmm->sequence, 0);
  98         hmm->mmu_notifier.ops = NULL;
  99         INIT_LIST_HEAD(&hmm->ranges);
 100         spin_lock_init(&hmm->lock);
 101         hmm->mm = mm;
 102
 103         /*
 104          * We should only get here if hold the mmap_sem in write mode ie on
 105          * registration of first mirror through hmm_mirror_register()
 106          */
 107         hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops;
 108         if (__mmu_notifier_register(&hmm->mmu_notifier, mm)) {
 109                 kfree(hmm);
 110                 return NULL;
 111         }
 112
 113         spin_lock(&mm->page_table_lock);
 114         if (!mm->hmm)
 115                 mm->hmm = hmm;
 116         else
 117                 cleanup = true;
 118         spin_unlock(&mm->page_table_lock);
 119
 120         if (cleanup) {
 121                 mmu_notifier_unregister(&hmm->mmu_notifier, mm);
 122                 kfree(hmm);
 123         }
 124
 125         return mm->hmm;
 126 }
 127
 128 void hmm_mm_destroy(struct mm_struct *mm)
 129 {
 130         kfree(mm->hmm);
 131 }
 132
 133 static void hmm_invalidate_range(struct hmm *hmm,
 134                                  enum hmm_update_type action,
 135                                  unsigned long start,
 136                                  unsigned long end)
 137 {
 138         struct hmm_mirror *mirror;
 139         struct hmm_range *range;
 140
 141         spin_lock(&hmm->lock);
 142         list_for_each_entry(range, &hmm->ranges, list) {
 143                 unsigned long addr, idx, npages;
 144
 145                 if (end < range->start || start >= range->end)
 146                         continue;
 147
 148                 range->valid = false;
 149                 addr = max(start, range->start);
 150                 idx = (addr - range->start) >> PAGE_SHIFT;
 151                 npages = (min(range->end, end) - addr) >> PAGE_SHIFT;
 152                 memset(&range->pfns[idx], 0, sizeof(*range->pfns) * npages);
 153         }
 154         spin_unlock(&hmm->lock);
 155
 156         down_read(&hmm->mirrors_sem);
 157         list_for_each_entry(mirror, &hmm->mirrors, list)
 158                 mirror->ops->sync_cpu_device_pagetables(mirror, action,
 159                                                         start, end);
 160         up_read(&hmm->mirrors_sem);
 161 }
 162
 163 static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 164 {
 165         struct hmm_mirror *mirror;
 166         struct hmm *hmm = mm->hmm;
 167
 168         down_write(&hmm->mirrors_sem);
 169         mirror = list_first_entry_or_null(&hmm->mirrors, struct hmm_mirror,
 170                                           list);
 171         while (mirror) {
 172                 list_del_init(&mirror->list);
 173                 if (mirror->ops->release) {
 174                         /*
 175                          * Drop mirrors_sem so callback can wait on any pending
 176                          * work that might itself trigger mmu_notifier callback
 177                          * and thus would deadlock with us.
 178                          */
 179                         up_write(&hmm->mirrors_sem);
 180                         mirror->ops->release(mirror);
 181                         down_write(&hmm->mirrors_sem);
 182                 }
 183                 mirror = list_first_entry_or_null(&hmm->mirrors,
 184                                                   struct hmm_mirror, list);
 185         }
 186         up_write(&hmm->mirrors_sem);
 187 }
 188
 189 static void hmm_invalidate_range_start(struct mmu_notifier *mn,
 190                                        struct mm_struct *mm,
 191                                        unsigned long start,
 192                                        unsigned long end)
 193 {
 194         struct hmm *hmm = mm->hmm;
 195
 196         VM_BUG_ON(!hmm);
 197
 198         atomic_inc(&hmm->sequence);
 199 }
 200
 201 static void hmm_invalidate_range_end(struct mmu_notifier *mn,
 202                                      struct mm_struct *mm,
 203                                      unsigned long start,
 204                                      unsigned long end)
 205 {
 206         struct hmm *hmm = mm->hmm;
 207
 208         VM_BUG_ON(!hmm);
 209
 210         hmm_invalidate_range(mm->hmm, HMM_UPDATE_INVALIDATE, start, end);
 211 }
 212
 213 static const struct mmu_notifier_ops hmm_mmu_notifier_ops = {
 214         .release                = hmm_release,
 215         .invalidate_range_start = hmm_invalidate_range_start,
 216         .invalidate_range_end   = hmm_invalidate_range_end,
 217 };
 218
 219 /*
 220  * hmm_mirror_register() - register a mirror against an mm
 221  *
 222  * @mirror: new mirror struct to register
 223  * @mm: mm to register against
 224  *
 225  * To start mirroring a process address space, the device driver must register
 226  * an HMM mirror struct.
 227  *
 228  * THE mm->mmap_sem MUST BE HELD IN WRITE MODE !
 229  */
 230 int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm)
 231 {
 232         /* Sanity check */
 233         if (!mm || !mirror || !mirror->ops)
 234                 return -EINVAL;
 235
 236         mirror->hmm = hmm_register(mm);
 237         if (!mirror->hmm)
 238                 return -ENOMEM;
 239
 240         down_write(&mirror->hmm->mirrors_sem);
 241         list_add(&mirror->list, &mirror->hmm->mirrors);
 242         up_write(&mirror->hmm->mirrors_sem);
 243
 244         return 0;
 245 }
 246 EXPORT_SYMBOL(hmm_mirror_register);
 247
 248 /*
 249  * hmm_mirror_unregister() - unregister a mirror
 250  *
 251  * @mirror: new mirror struct to register
 252  *
 253  * Stop mirroring a process address space, and cleanup.
 254  */
 255 void hmm_mirror_unregister(struct hmm_mirror *mirror)
 256 {
 257         struct hmm *hmm = mirror->hmm;
 258
 259         down_write(&hmm->mirrors_sem);
 260         list_del_init(&mirror->list);
 261         up_write(&hmm->mirrors_sem);
 262 }
 263 EXPORT_SYMBOL(hmm_mirror_unregister);
 264
 265 struct hmm_vma_walk {
 266         struct hmm_range        *range;
 267         unsigned long           last;
 268         bool                    fault;
 269         bool                    block;
 270         bool                    write;
 271 };
 272
 273 static int hmm_vma_do_fault(struct mm_walk *walk,
 274                             unsigned long addr,
 275                             hmm_pfn_t *pfn)
 276 {
 277         unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE;
 278         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 279         struct vm_area_struct *vma = walk->vma;
 280         int r;
 281
 282         flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY;
 283         flags |= hmm_vma_walk->write ? FAULT_FLAG_WRITE : 0;
 284         r = handle_mm_fault(vma, addr, flags);
 285         if (r & VM_FAULT_RETRY)
 286                 return -EBUSY;
 287         if (r & VM_FAULT_ERROR) {
 288                 *pfn = HMM_PFN_ERROR;
 289                 return -EFAULT;
 290         }
 291
 292         return -EAGAIN;
 293 }
 294
 295 static void hmm_pfns_special(hmm_pfn_t *pfns,
 296                              unsigned long addr,
 297                              unsigned long end)
 298 {
 299         for (; addr < end; addr += PAGE_SIZE, pfns++)
 300                 *pfns = HMM_PFN_SPECIAL;
 301 }
 302
 303 static int hmm_pfns_bad(unsigned long addr,
 304                         unsigned long end,
 305                         struct mm_walk *walk)
 306 {
 307         struct hmm_range *range = walk->private;
 308         hmm_pfn_t *pfns = range->pfns;
 309         unsigned long i;
 310
 311         i = (addr - range->start) >> PAGE_SHIFT;
 312         for (; addr < end; addr += PAGE_SIZE, i++)
 313                 pfns[i] = HMM_PFN_ERROR;
 314
 315         return 0;
 316 }
 317
 318 static void hmm_pfns_clear(hmm_pfn_t *pfns,
 319                            unsigned long addr,
 320                            unsigned long end)
 321 {
 322         for (; addr < end; addr += PAGE_SIZE, pfns++)
 323                 *pfns = 0;
 324 }
 325
 326 static int hmm_vma_walk_hole(unsigned long addr,
 327                              unsigned long end,
 328                              struct mm_walk *walk)
 329 {
 330         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 331         struct hmm_range *range = hmm_vma_walk->range;
 332         hmm_pfn_t *pfns = range->pfns;
 333         unsigned long i;
 334
 335         hmm_vma_walk->last = addr;
 336         i = (addr - range->start) >> PAGE_SHIFT;
 337         for (; addr < end; addr += PAGE_SIZE, i++) {
 338                 pfns[i] = HMM_PFN_EMPTY;
 339                 if (hmm_vma_walk->fault) {
 340                         int ret;
 341
 342                         ret = hmm_vma_do_fault(walk, addr, &pfns[i]);
 343                         if (ret != -EAGAIN)
 344                                 return ret;
 345                 }
 346         }
 347
 348         return hmm_vma_walk->fault ? -EAGAIN : 0;
 349 }
 350
 351 static int hmm_vma_walk_clear(unsigned long addr,
 352                               unsigned long end,
 353                               struct mm_walk *walk)
 354 {
 355         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 356         struct hmm_range *range = hmm_vma_walk->range;
 357         hmm_pfn_t *pfns = range->pfns;
 358         unsigned long i;
 359
 360         hmm_vma_walk->last = addr;
 361         i = (addr - range->start) >> PAGE_SHIFT;
 362         for (; addr < end; addr += PAGE_SIZE, i++) {
 363                 pfns[i] = 0;
 364                 if (hmm_vma_walk->fault) {
 365                         int ret;
 366
 367                         ret = hmm_vma_do_fault(walk, addr, &pfns[i]);
 368                         if (ret != -EAGAIN)
 369                                 return ret;
 370                 }
 371         }
 372
 373         return hmm_vma_walk->fault ? -EAGAIN : 0;
 374 }
 375
 376 static int hmm_vma_walk_pmd(pmd_t *pmdp,
 377                             unsigned long start,
 378                             unsigned long end,
 379                             struct mm_walk *walk)
 380 {
 381         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 382         struct hmm_range *range = hmm_vma_walk->range;
 383         struct vm_area_struct *vma = walk->vma;
 384         hmm_pfn_t *pfns = range->pfns;
 385         unsigned long addr = start, i;
 386         bool write_fault;
 387         hmm_pfn_t flag;
 388         pte_t *ptep;
 389
 390         i = (addr - range->start) >> PAGE_SHIFT;
 391         flag = vma->vm_flags & VM_READ ? HMM_PFN_READ : 0;
 392         write_fault = hmm_vma_walk->fault & hmm_vma_walk->write;
 393
 394 again:
 395         if (pmd_none(*pmdp))
 396                 return hmm_vma_walk_hole(start, end, walk);
 397
 398         if (pmd_huge(*pmdp) && vma->vm_flags & VM_HUGETLB)
 399                 return hmm_pfns_bad(start, end, walk);
 400
 401         if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
 402                 unsigned long pfn;
 403                 pmd_t pmd;
 404
 405                 /*
 406                  * No need to take pmd_lock here, even if some other threads
 407                  * is splitting the huge pmd we will get that event through
 408                  * mmu_notifier callback.
 409                  *
 410                  * So just read pmd value and check again its a transparent
 411                  * huge or device mapping one and compute corresponding pfn
 412                  * values.
 413                  */
 414                 pmd = pmd_read_atomic(pmdp);
 415                 barrier();
 416                 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
 417                         goto again;
 418                 if (pmd_protnone(pmd))
 419                         return hmm_vma_walk_clear(start, end, walk);
 420
 421                 if (write_fault && !pmd_write(pmd))
 422                         return hmm_vma_walk_clear(start, end, walk);
 423
 424                 pfn = pmd_pfn(pmd) + pte_index(addr);
 425                 flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;
 426                 for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
 427                         pfns[i] = hmm_pfn_t_from_pfn(pfn) | flag;
 428                 return 0;
 429         }
 430
 431         if (pmd_bad(*pmdp))
 432                 return hmm_pfns_bad(start, end, walk);
 433
 434         ptep = pte_offset_map(pmdp, addr);
 435         for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
 436                 pte_t pte = *ptep;
 437
 438                 pfns[i] = 0;
 439
 440                 if (pte_none(pte)) {
 441                         pfns[i] = HMM_PFN_EMPTY;
 442                         if (hmm_vma_walk->fault)
 443                                 goto fault;
 444                         continue;
 445                 }
 446
 447                 if (!pte_present(pte)) {
 448                         swp_entry_t entry = pte_to_swp_entry(pte);
 449
 450                         if (!non_swap_entry(entry)) {
 451                                 if (hmm_vma_walk->fault)
 452                                         goto fault;
 453                                 continue;
 454                         }
 455
 456                         /*
 457                          * This is a special swap entry, ignore migration, use
 458                          * device and report anything else as error.
 459                          */
 460                         if (is_device_private_entry(entry)) {
 461                                 pfns[i] = hmm_pfn_t_from_pfn(swp_offset(entry));
 462                                 if (is_write_device_private_entry(entry)) {
 463                                         pfns[i] |= HMM_PFN_WRITE;
 464                                 } else if (write_fault)
 465                                         goto fault;
 466                                 pfns[i] |= HMM_PFN_DEVICE_UNADDRESSABLE;
 467                                 pfns[i] |= flag;
 468                         } else if (is_migration_entry(entry)) {
 469                                 if (hmm_vma_walk->fault) {
 470                                         pte_unmap(ptep);
 471                                         hmm_vma_walk->last = addr;
 472                                         migration_entry_wait(vma->vm_mm,
 473                                                              pmdp, addr);
 474                                         return -EAGAIN;
 475                                 }
 476                                 continue;
 477                         } else {
 478                                 /* Report error for everything else */
 479                                 pfns[i] = HMM_PFN_ERROR;
 480                         }
 481                         continue;
 482                 }
 483
 484                 if (write_fault && !pte_write(pte))
 485                         goto fault;
 486
 487                 pfns[i] = hmm_pfn_t_from_pfn(pte_pfn(pte)) | flag;
 488                 pfns[i] |= pte_write(pte) ? HMM_PFN_WRITE : 0;
 489                 continue;
 490
 491 fault:
 492                 pte_unmap(ptep);
 493                 /* Fault all pages in range */
 494                 return hmm_vma_walk_clear(start, end, walk);
 495         }
 496         pte_unmap(ptep - 1);
 497
 498         return 0;
 499 }
 500
 501 /*
 502  * hmm_vma_get_pfns() - snapshot CPU page table for a range of virtual addresses
 503  * @vma: virtual memory area containing the virtual address range
 504  * @range: used to track snapshot validity
 505  * @start: range virtual start address (inclusive)
 506  * @end: range virtual end address (exclusive)
 507  * @entries: array of hmm_pfn_t: provided by the caller, filled in by function
 508  * Returns: -EINVAL if invalid argument, -ENOMEM out of memory, 0 success
 509  *
 510  * This snapshots the CPU page table for a range of virtual addresses. Snapshot
 511  * validity is tracked by range struct. See hmm_vma_range_done() for further
 512  * information.
 513  *
 514  * The range struct is initialized here. It tracks the CPU page table, but only
 515  * if the function returns success (0), in which case the caller must then call
 516  * hmm_vma_range_done() to stop CPU page table update tracking on this range.
 517  *
 518  * NOT CALLING hmm_vma_range_done() IF FUNCTION RETURNS 0 WILL LEAD TO SERIOUS
 519  * MEMORY CORRUPTION ! YOU HAVE BEEN WARNED !
 520  */
 521 int hmm_vma_get_pfns(struct vm_area_struct *vma,
 522                      struct hmm_range *range,
 523                      unsigned long start,
 524                      unsigned long end,
 525                      hmm_pfn_t *pfns)
 526 {
 527         struct hmm_vma_walk hmm_vma_walk;
 528         struct mm_walk mm_walk;
 529         struct hmm *hmm;
 530
 531         /* FIXME support hugetlb fs */
 532         if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) {
 533                 hmm_pfns_special(pfns, start, end);
 534                 return -EINVAL;
 535         }
 536
 537         /* Sanity check, this really should not happen ! */
 538         if (start < vma->vm_start || start >= vma->vm_end)
 539                 return -EINVAL;
 540         if (end < vma->vm_start || end > vma->vm_end)
 541                 return -EINVAL;
 542
 543         hmm = hmm_register(vma->vm_mm);
 544         if (!hmm)
 545                 return -ENOMEM;
 546         /* Caller must have registered a mirror, via hmm_mirror_register() ! */
 547         if (!hmm->mmu_notifier.ops)
 548                 return -EINVAL;
 549
 550         /* Initialize range to track CPU page table update */
 551         range->start = start;
 552         range->pfns = pfns;
 553         range->end = end;
 554         spin_lock(&hmm->lock);
 555         range->valid = true;
 556         list_add_rcu(&range->list, &hmm->ranges);
 557         spin_unlock(&hmm->lock);
 558
 559         hmm_vma_walk.fault = false;
 560         hmm_vma_walk.range = range;
 561         mm_walk.private = &hmm_vma_walk;
 562
 563         mm_walk.vma = vma;
 564         mm_walk.mm = vma->vm_mm;
 565         mm_walk.pte_entry = NULL;
 566         mm_walk.test_walk = NULL;
 567         mm_walk.hugetlb_entry = NULL;
 568         mm_walk.pmd_entry = hmm_vma_walk_pmd;
 569         mm_walk.pte_hole = hmm_vma_walk_hole;
 570
 571         walk_page_range(start, end, &mm_walk);
 572         return 0;
 573 }
 574 EXPORT_SYMBOL(hmm_vma_get_pfns);
 575
 576 /*
 577  * hmm_vma_range_done() - stop tracking change to CPU page table over a range
 578  * @vma: virtual memory area containing the virtual address range
 579  * @range: range being tracked
 580  * Returns: false if range data has been invalidated, true otherwise
 581  *
 582  * Range struct is used to track updates to the CPU page table after a call to
 583  * either hmm_vma_get_pfns() or hmm_vma_fault(). Once the device driver is done
 584  * using the data,  or wants to lock updates to the data it got from those
 585  * functions, it must call the hmm_vma_range_done() function, which will then
 586  * stop tracking CPU page table updates.
 587  *
 588  * Note that device driver must still implement general CPU page table update
 589  * tracking either by using hmm_mirror (see hmm_mirror_register()) or by using
 590  * the mmu_notifier API directly.
 591  *
 592  * CPU page table update tracking done through hmm_range is only temporary and
 593  * to be used while trying to duplicate CPU page table contents for a range of
 594  * virtual addresses.
 595  *
 596  * There are two ways to use this :
 597  * again:
 598  *   hmm_vma_get_pfns(vma, range, start, end, pfns); or hmm_vma_fault(...);
 599  *   trans = device_build_page_table_update_transaction(pfns);
 600  *   device_page_table_lock();
 601  *   if (!hmm_vma_range_done(vma, range)) {
 602  *     device_page_table_unlock();
 603  *     goto again;
 604  *   }
 605  *   device_commit_transaction(trans);
 606  *   device_page_table_unlock();
 607  *
 608  * Or:
 609  *   hmm_vma_get_pfns(vma, range, start, end, pfns); or hmm_vma_fault(...);
 610  *   device_page_table_lock();
 611  *   hmm_vma_range_done(vma, range);
 612  *   device_update_page_table(pfns);
 613  *   device_page_table_unlock();
 614  */
 615 bool hmm_vma_range_done(struct vm_area_struct *vma, struct hmm_range *range)
 616 {
 617         unsigned long npages = (range->end - range->start) >> PAGE_SHIFT;
 618         struct hmm *hmm;
 619
 620         if (range->end <= range->start) {
 621                 BUG();
 622                 return false;
 623         }
 624
 625         hmm = hmm_register(vma->vm_mm);
 626         if (!hmm) {
 627                 memset(range->pfns, 0, sizeof(*range->pfns) * npages);
 628                 return false;
 629         }
 630
 631         spin_lock(&hmm->lock);
 632         list_del_rcu(&range->list);
 633         spin_unlock(&hmm->lock);
 634
 635         return range->valid;
 636 }
 637 EXPORT_SYMBOL(hmm_vma_range_done);
 638
 639 /*
 640  * hmm_vma_fault() - try to fault some address in a virtual address range
 641  * @vma: virtual memory area containing the virtual address range
 642  * @range: use to track pfns array content validity
 643  * @start: fault range virtual start address (inclusive)
 644  * @end: fault range virtual end address (exclusive)
 645  * @pfns: array of hmm_pfn_t, only entry with fault flag set will be faulted
 646  * @write: is it a write fault
 647  * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem)
 648  * Returns: 0 success, error otherwise (-EAGAIN means mmap_sem have been drop)
 649  *
 650  * This is similar to a regular CPU page fault except that it will not trigger
 651  * any memory migration if the memory being faulted is not accessible by CPUs.
 652  *
 653  * On error, for one virtual address in the range, the function will set the
 654  * hmm_pfn_t error flag for the corresponding pfn entry.
 655  *
 656  * Expected use pattern:
 657  * retry:
 658  *   down_read(&mm->mmap_sem);
 659  *   // Find vma and address device wants to fault, initialize hmm_pfn_t
 660  *   // array accordingly
 661  *   ret = hmm_vma_fault(vma, start, end, pfns, allow_retry);
 662  *   switch (ret) {
 663  *   case -EAGAIN:
 664  *     hmm_vma_range_done(vma, range);
 665  *     // You might want to rate limit or yield to play nicely, you may
 666  *     // also commit any valid pfn in the array assuming that you are
 667  *     // getting true from hmm_vma_range_monitor_end()
 668  *     goto retry;
 669  *   case 0:
 670  *     break;
 671  *   default:
 672  *     // Handle error !
 673  *     up_read(&mm->mmap_sem)
 674  *     return;
 675  *   }
 676  *   // Take device driver lock that serialize device page table update
 677  *   driver_lock_device_page_table_update();
 678  *   hmm_vma_range_done(vma, range);
 679  *   // Commit pfns we got from hmm_vma_fault()
 680  *   driver_unlock_device_page_table_update();
 681  *   up_read(&mm->mmap_sem)
 682  *
 683  * YOU MUST CALL hmm_vma_range_done() AFTER THIS FUNCTION RETURN SUCCESS (0)
 684  * BEFORE FREEING THE range struct OR YOU WILL HAVE SERIOUS MEMORY CORRUPTION !
 685  *
 686  * YOU HAVE BEEN WARNED !
 687  */
 688 int hmm_vma_fault(struct vm_area_struct *vma,
 689                   struct hmm_range *range,
 690                   unsigned long start,
 691                   unsigned long end,
 692                   hmm_pfn_t *pfns,
 693                   bool write,
 694                   bool block)
 695 {
 696         struct hmm_vma_walk hmm_vma_walk;
 697         struct mm_walk mm_walk;
 698         struct hmm *hmm;
 699         int ret;
 700
 701         /* Sanity check, this really should not happen ! */
 702         if (start < vma->vm_start || start >= vma->vm_end)
 703                 return -EINVAL;
 704         if (end < vma->vm_start || end > vma->vm_end)
 705                 return -EINVAL;
 706
 707         hmm = hmm_register(vma->vm_mm);
 708         if (!hmm) {
 709                 hmm_pfns_clear(pfns, start, end);
 710                 return -ENOMEM;
 711         }
 712         /* Caller must have registered a mirror using hmm_mirror_register() */
 713         if (!hmm->mmu_notifier.ops)
 714                 return -EINVAL;
 715
 716         /* Initialize range to track CPU page table update */
 717         range->start = start;
 718         range->pfns = pfns;
 719         range->end = end;
 720         spin_lock(&hmm->lock);
 721         range->valid = true;
 722         list_add_rcu(&range->list, &hmm->ranges);
 723         spin_unlock(&hmm->lock);
 724
 725         /* FIXME support hugetlb fs */
 726         if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) {
 727                 hmm_pfns_special(pfns, start, end);
 728                 return 0;
 729         }
 730
 731         hmm_vma_walk.fault = true;
 732         hmm_vma_walk.write = write;
 733         hmm_vma_walk.block = block;
 734         hmm_vma_walk.range = range;
 735         mm_walk.private = &hmm_vma_walk;
 736         hmm_vma_walk.last = range->start;
 737
 738         mm_walk.vma = vma;
 739         mm_walk.mm = vma->vm_mm;
 740         mm_walk.pte_entry = NULL;
 741         mm_walk.test_walk = NULL;
 742         mm_walk.hugetlb_entry = NULL;
 743         mm_walk.pmd_entry = hmm_vma_walk_pmd;
 744         mm_walk.pte_hole = hmm_vma_walk_hole;
 745
 746         do {
 747                 ret = walk_page_range(start, end, &mm_walk);
 748                 start = hmm_vma_walk.last;
 749         } while (ret == -EAGAIN);
 750
 751         if (ret) {
 752                 unsigned long i;
 753
 754                 i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
 755                 hmm_pfns_clear(&pfns[i], hmm_vma_walk.last, end);
 756                 hmm_vma_range_done(vma, range);
 757         }
 758         return ret;
 759 }
 760 EXPORT_SYMBOL(hmm_vma_fault);
 761 #endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
 762
 763
 764 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) ||  IS_ENABLED(CONFIG_DEVICE_PUBLIC)
 765 struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma,
 766                                        unsigned long addr)
 767 {
 768         struct page *page;
 769
 770         page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
 771         if (!page)
 772                 return NULL;
 773         lock_page(page);
 774         return page;
 775 }
 776 EXPORT_SYMBOL(hmm_vma_alloc_locked_page);
 777
 778
 779 static void hmm_devmem_ref_release(struct percpu_ref *ref)
 780 {
 781         struct hmm_devmem *devmem;
 782
 783         devmem = container_of(ref, struct hmm_devmem, ref);
 784         complete(&devmem->completion);
 785 }
 786
 787 static void hmm_devmem_ref_exit(void *data)
 788 {
 789         struct percpu_ref *ref = data;
 790         struct hmm_devmem *devmem;
 791
 792         devmem = container_of(ref, struct hmm_devmem, ref);
 793         percpu_ref_exit(ref);
 794         devm_remove_action(devmem->device, &hmm_devmem_ref_exit, data);
 795 }
 796
 797 static void hmm_devmem_ref_kill(void *data)
 798 {
 799         struct percpu_ref *ref = data;
 800         struct hmm_devmem *devmem;
 801
 802         devmem = container_of(ref, struct hmm_devmem, ref);
 803         percpu_ref_kill(ref);
 804         wait_for_completion(&devmem->completion);
 805         devm_remove_action(devmem->device, &hmm_devmem_ref_kill, data);
 806 }
 807
 808 static int hmm_devmem_fault(struct vm_area_struct *vma,
 809                             unsigned long addr,
 810                             const struct page *page,
 811                             unsigned int flags,
 812                             pmd_t *pmdp)
 813 {
 814         struct hmm_devmem *devmem = page->pgmap->data;
 815
 816         return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
 817 }
 818
 819 static void hmm_devmem_free(struct page *page, void *data)
 820 {
 821         struct hmm_devmem *devmem = data;
 822
 823         devmem->ops->free(devmem, page);
 824 }
 825
 826 static DEFINE_MUTEX(hmm_devmem_lock);
 827 static RADIX_TREE(hmm_devmem_radix, GFP_KERNEL);
 828
 829 static void hmm_devmem_radix_release(struct resource *resource)
 830 {
 831         resource_size_t key, align_start, align_size;
 832
 833         align_start = resource->start & ~(PA_SECTION_SIZE - 1);
 834         align_size = ALIGN(resource_size(resource), PA_SECTION_SIZE);
 835
 836         mutex_lock(&hmm_devmem_lock);
 837         for (key = resource->start;
 838              key <= resource->end;
 839              key += PA_SECTION_SIZE)
 840                 radix_tree_delete(&hmm_devmem_radix, key >> PA_SECTION_SHIFT);
 841         mutex_unlock(&hmm_devmem_lock);
 842 }
 843
 844 static void hmm_devmem_release(struct device *dev, void *data)
 845 {
 846         struct hmm_devmem *devmem = data;
 847         struct resource *resource = devmem->resource;
 848         unsigned long start_pfn, npages;
 849         struct zone *zone;
 850         struct page *page;
 851
 852         if (percpu_ref_tryget_live(&devmem->ref)) {
 853                 dev_WARN(dev, "%s: page mapping is still live!\n", __func__);
 854                 percpu_ref_put(&devmem->ref);
 855         }
 856
 857         /* pages are dead and unused, undo the arch mapping */
 858         start_pfn = (resource->start & ~(PA_SECTION_SIZE - 1)) >> PAGE_SHIFT;
 859         npages = ALIGN(resource_size(resource), PA_SECTION_SIZE) >> PAGE_SHIFT;
 860
 861         page = pfn_to_page(start_pfn);
 862         zone = page_zone(page);
 863
 864         mem_hotplug_begin();
 865         if (resource->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY)
 866                 __remove_pages(zone, start_pfn, npages, NULL);
 867         else
 868                 arch_remove_memory(start_pfn << PAGE_SHIFT,
 869                                    npages << PAGE_SHIFT, NULL);
 870         mem_hotplug_done();
 871
 872         hmm_devmem_radix_release(resource);
 873 }
 874
 875 static struct hmm_devmem *hmm_devmem_find(resource_size_t phys)
 876 {
 877         WARN_ON_ONCE(!rcu_read_lock_held());
 878
 879         return radix_tree_lookup(&hmm_devmem_radix, phys >> PA_SECTION_SHIFT);
 880 }
 881
 882 static int hmm_devmem_pages_create(struct hmm_devmem *devmem)
 883 {
 884         resource_size_t key, align_start, align_size, align_end;
 885         struct device *device = devmem->device;
 886         int ret, nid, is_ram;
 887         unsigned long pfn;
 888
 889         align_start = devmem->resource->start & ~(PA_SECTION_SIZE - 1);
 890         align_size = ALIGN(devmem->resource->start +
 891                            resource_size(devmem->resource),
 892                            PA_SECTION_SIZE) - align_start;
 893
 894         is_ram = region_intersects(align_start, align_size,
 895                                    IORESOURCE_SYSTEM_RAM,
 896                                    IORES_DESC_NONE);
 897         if (is_ram == REGION_MIXED) {
 898                 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
 899                                 __func__, devmem->resource);
 900                 return -ENXIO;
 901         }
 902         if (is_ram == REGION_INTERSECTS)
 903                 return -ENXIO;
 904
 905         if (devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY)
 906                 devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;
 907         else
 908                 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 909
 910         devmem->pagemap.res = *devmem->resource;
 911         devmem->pagemap.page_fault = hmm_devmem_fault;
 912         devmem->pagemap.page_free = hmm_devmem_free;
 913         devmem->pagemap.dev = devmem->device;
 914         devmem->pagemap.ref = &devmem->ref;
 915         devmem->pagemap.data = devmem;
 916
 917         mutex_lock(&hmm_devmem_lock);
 918         align_end = align_start + align_size - 1;
 919         for (key = align_start; key <= align_end; key += PA_SECTION_SIZE) {
 920                 struct hmm_devmem *dup;
 921
 922                 rcu_read_lock();
 923                 dup = hmm_devmem_find(key);
 924                 rcu_read_unlock();
 925                 if (dup) {
 926                         dev_err(device, "%s: collides with mapping for %s\n",
 927                                 __func__, dev_name(dup->device));
 928                         mutex_unlock(&hmm_devmem_lock);
 929                         ret = -EBUSY;
 930                         goto error;
 931                 }
 932                 ret = radix_tree_insert(&hmm_devmem_radix,
 933                                         key >> PA_SECTION_SHIFT,
 934                                         devmem);
 935                 if (ret) {
 936                         dev_err(device, "%s: failed: %d\n", __func__, ret);
 937                         mutex_unlock(&hmm_devmem_lock);
 938                         goto error_radix;
 939                 }
 940         }
 941         mutex_unlock(&hmm_devmem_lock);
 942
 943         nid = dev_to_node(device);
 944         if (nid < 0)
 945                 nid = numa_mem_id();
 946
 947         mem_hotplug_begin();
 948         /*
 949          * For device private memory we call add_pages() as we only need to
 950          * allocate and initialize struct page for the device memory. More-
 951          * over the device memory is un-accessible thus we do not want to
 952          * create a linear mapping for the memory like arch_add_memory()
 953          * would do.
 954          *
 955          * For device public memory, which is accesible by the CPU, we do
 956          * want the linear mapping and thus use arch_add_memory().
 957          */
 958         if (devmem->pagemap.type == MEMORY_DEVICE_PUBLIC)
 959                 ret = arch_add_memory(nid, align_start, align_size, NULL,
 960                                 false);
 961         else
 962                 ret = add_pages(nid, align_start >> PAGE_SHIFT,
 963                                 align_size >> PAGE_SHIFT, NULL, false);
 964         if (ret) {
 965                 mem_hotplug_done();
 966                 goto error_add_memory;
 967         }
 968         move_pfn_range_to_zone(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
 969                                 align_start >> PAGE_SHIFT,
 970                                 align_size >> PAGE_SHIFT, NULL);
 971         mem_hotplug_done();
 972
 973         for (pfn = devmem->pfn_first; pfn < devmem->pfn_last; pfn++) {
 974                 struct page *page = pfn_to_page(pfn);
 975
 976                 page->pgmap = &devmem->pagemap;
 977         }
 978         return 0;
 979
 980 error_add_memory:
 981         untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
 982 error_radix:
 983         hmm_devmem_radix_release(devmem->resource);
 984 error:
 985         return ret;
 986 }
 987
 988 static int hmm_devmem_match(struct device *dev, void *data, void *match_data)
 989 {
 990         struct hmm_devmem *devmem = data;
 991
 992         return devmem->resource == match_data;
 993 }
 994
 995 static void hmm_devmem_pages_remove(struct hmm_devmem *devmem)
 996 {
 997         devres_release(devmem->device, &hmm_devmem_release,
 998                        &hmm_devmem_match, devmem->resource);
 999 }
1000
1001 /*
1002  * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1003  *
1004  * @ops: memory event device driver callback (see struct hmm_devmem_ops)
1005  * @device: device struct to bind the resource too
1006  * @size: size in bytes of the device memory to add
1007  * Returns: pointer to new hmm_devmem struct ERR_PTR otherwise
1008  *
1009  * This function first finds an empty range of physical address big enough to
1010  * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1011  * in turn allocates struct pages. It does not do anything beyond that; all
1012  * events affecting the memory will go through the various callbacks provided
1013  * by hmm_devmem_ops struct.
1014  *
1015  * Device driver should call this function during device initialization and
1016  * is then responsible of memory management. HMM only provides helpers.
1017  */
1018 struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops,
1019                                   struct device *device,
1020                                   unsigned long size)
1021 {
1022         struct hmm_devmem *devmem;
1023         resource_size_t addr;
1024         int ret;
1025
1026         static_branch_enable(&device_private_key);
1027
1028         devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1029                                    GFP_KERNEL, dev_to_node(device));
1030         if (!devmem)
1031                 return ERR_PTR(-ENOMEM);
1032
1033         init_completion(&devmem->completion);
1034         devmem->pfn_first = -1UL;
1035         devmem->pfn_last = -1UL;
1036         devmem->resource = NULL;
1037         devmem->device = device;
1038         devmem->ops = ops;
1039
1040         ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1041                               0, GFP_KERNEL);
1042         if (ret)
1043                 goto error_percpu_ref;
1044
1045         ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1046         if (ret)
1047                 goto error_devm_add_action;
1048
1049         size = ALIGN(size, PA_SECTION_SIZE);
1050         addr = min((unsigned long)iomem_resource.end,
1051                    (1UL << MAX_PHYSMEM_BITS) - 1);
1052         addr = addr - size + 1UL;
1053
1054         /*
1055          * FIXME add a new helper to quickly walk resource tree and find free
1056          * range
1057          *
1058          * FIXME what about ioport_resource resource ?
1059          */
1060         for (; addr > size && addr >= iomem_resource.start; addr -= size) {
1061                 ret = region_intersects(addr, size, 0, IORES_DESC_NONE);
1062                 if (ret != REGION_DISJOINT)
1063                         continue;
1064
1065                 devmem->resource = devm_request_mem_region(device, addr, size,
1066                                                            dev_name(device));
1067                 if (!devmem->resource) {
1068                         ret = -ENOMEM;
1069                         goto error_no_resource;
1070                 }
1071                 break;
1072         }
1073         if (!devmem->resource) {
1074                 ret = -ERANGE;
1075                 goto error_no_resource;
1076         }
1077
1078         devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1079         devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1080         devmem->pfn_last = devmem->pfn_first +
1081                            (resource_size(devmem->resource) >> PAGE_SHIFT);
1082
1083         ret = hmm_devmem_pages_create(devmem);
1084         if (ret)
1085                 goto error_pages;
1086
1087         devres_add(device, devmem);
1088
1089         ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1090         if (ret) {
1091                 hmm_devmem_remove(devmem);
1092                 return ERR_PTR(ret);
1093         }
1094
1095         return devmem;
1096
1097 error_pages:
1098         devm_release_mem_region(device, devmem->resource->start,
1099                                 resource_size(devmem->resource));
1100 error_no_resource:
1101 error_devm_add_action:
1102         hmm_devmem_ref_kill(&devmem->ref);
1103         hmm_devmem_ref_exit(&devmem->ref);
1104 error_percpu_ref:
1105         devres_free(devmem);
1106         return ERR_PTR(ret);
1107 }
1108 EXPORT_SYMBOL(hmm_devmem_add);
1109
1110 struct hmm_devmem *hmm_devmem_add_resource(const struct hmm_devmem_ops *ops,
1111                                            struct device *device,
1112                                            struct resource *res)
1113 {
1114         struct hmm_devmem *devmem;
1115         int ret;
1116
1117         if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY)
1118                 return ERR_PTR(-EINVAL);
1119
1120         static_branch_enable(&device_private_key);
1121
1122         devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1123                                    GFP_KERNEL, dev_to_node(device));
1124         if (!devmem)
1125                 return ERR_PTR(-ENOMEM);
1126
1127         init_completion(&devmem->completion);
1128         devmem->pfn_first = -1UL;
1129         devmem->pfn_last = -1UL;
1130         devmem->resource = res;
1131         devmem->device = device;
1132         devmem->ops = ops;
1133
1134         ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1135                               0, GFP_KERNEL);
1136         if (ret)
1137                 goto error_percpu_ref;
1138
1139         ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1140         if (ret)
1141                 goto error_devm_add_action;
1142
1143
1144         devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1145         devmem->pfn_last = devmem->pfn_first +
1146                            (resource_size(devmem->resource) >> PAGE_SHIFT);
1147
1148         ret = hmm_devmem_pages_create(devmem);
1149         if (ret)
1150                 goto error_devm_add_action;
1151
1152         devres_add(device, devmem);
1153
1154         ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1155         if (ret) {
1156                 hmm_devmem_remove(devmem);
1157                 return ERR_PTR(ret);
1158         }
1159
1160         return devmem;
1161
1162 error_devm_add_action:
1163         hmm_devmem_ref_kill(&devmem->ref);
1164         hmm_devmem_ref_exit(&devmem->ref);
1165 error_percpu_ref:
1166         devres_free(devmem);
1167         return ERR_PTR(ret);
1168 }
1169 EXPORT_SYMBOL(hmm_devmem_add_resource);
1170
1171 /*
1172  * hmm_devmem_remove() - remove device memory (kill and free ZONE_DEVICE)
1173  *
1174  * @devmem: hmm_devmem struct use to track and manage the ZONE_DEVICE memory
1175  *
1176  * This will hot-unplug memory that was hotplugged by hmm_devmem_add on behalf
1177  * of the device driver. It will free struct page and remove the resource that
1178  * reserved the physical address range for this device memory.
1179  */
1180 void hmm_devmem_remove(struct hmm_devmem *devmem)
1181 {
1182         resource_size_t start, size;
1183         struct device *device;
1184         bool cdm = false;
1185
1186         if (!devmem)
1187                 return;
1188
1189         device = devmem->device;
1190         start = devmem->resource->start;
1191         size = resource_size(devmem->resource);
1192
1193         cdm = devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY;
1194         hmm_devmem_ref_kill(&devmem->ref);
1195         hmm_devmem_ref_exit(&devmem->ref);
1196         hmm_devmem_pages_remove(devmem);
1197
1198         if (!cdm)
1199                 devm_release_mem_region(device, start, size);
1200 }
1201 EXPORT_SYMBOL(hmm_devmem_remove);
1202
1203 /*
1204  * A device driver that wants to handle multiple devices memory through a
1205  * single fake device can use hmm_device to do so. This is purely a helper
1206  * and it is not needed to make use of any HMM functionality.
1207  */
1208 #define HMM_DEVICE_MAX 256
1209
1210 static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX);
1211 static DEFINE_SPINLOCK(hmm_device_lock);
1212 static struct class *hmm_device_class;
1213 static dev_t hmm_device_devt;
1214
1215 static void hmm_device_release(struct device *device)
1216 {
1217         struct hmm_device *hmm_device;
1218
1219         hmm_device = container_of(device, struct hmm_device, device);
1220         spin_lock(&hmm_device_lock);
1221         clear_bit(hmm_device->minor, hmm_device_mask);
1222         spin_unlock(&hmm_device_lock);
1223
1224         kfree(hmm_device);
1225 }
1226
1227 struct hmm_device *hmm_device_new(void *drvdata)
1228 {
1229         struct hmm_device *hmm_device;
1230
1231         hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL);
1232         if (!hmm_device)
1233                 return ERR_PTR(-ENOMEM);
1234
1235         spin_lock(&hmm_device_lock);
1236         hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX);
1237         if (hmm_device->minor >= HMM_DEVICE_MAX) {
1238                 spin_unlock(&hmm_device_lock);
1239                 kfree(hmm_device);
1240                 return ERR_PTR(-EBUSY);
1241         }
1242         set_bit(hmm_device->minor, hmm_device_mask);
1243         spin_unlock(&hmm_device_lock);
1244
1245         dev_set_name(&hmm_device->device, "hmm_device%d", hmm_device->minor);
1246         hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt),
1247                                         hmm_device->minor);
1248         hmm_device->device.release = hmm_device_release;
1249         dev_set_drvdata(&hmm_device->device, drvdata);
1250         hmm_device->device.class = hmm_device_class;
1251         device_initialize(&hmm_device->device);
1252
1253         return hmm_device;
1254 }
1255 EXPORT_SYMBOL(hmm_device_new);
1256
1257 void hmm_device_put(struct hmm_device *hmm_device)
1258 {
1259         put_device(&hmm_device->device);
1260 }
1261 EXPORT_SYMBOL(hmm_device_put);
1262
1263 static int __init hmm_init(void)
1264 {
1265         int ret;
1266
1267         ret = alloc_chrdev_region(&hmm_device_devt, 0,
1268                                   HMM_DEVICE_MAX,
1269                                   "hmm_device");
1270         if (ret)
1271                 return ret;
1272
1273         hmm_device_class = class_create(THIS_MODULE, "hmm_device");
1274         if (IS_ERR(hmm_device_class)) {
1275                 unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX);
1276                 return PTR_ERR(hmm_device_class);
1277         }
1278         return 0;
1279 }
1280
1281 device_initcall(hmm_init);
1282 #endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */