arch/riscv/kernel/vdso.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
   4  *                    <benh@kernel.crashing.org>
   5  * Copyright (C) 2012 ARM Limited
   6  * Copyright (C) 2015 Regents of the University of California
   7  */
   8
   9 #include <linux/elf.h>
  10 #include <linux/mm.h>
  11 #include <linux/slab.h>
  12 #include <linux/binfmts.h>
  13 #include <linux/err.h>
  14 #include <asm/page.h>
  15 #include <asm/vdso.h>
  16 #include <linux/time_namespace.h>
  17
  18 #ifdef CONFIG_GENERIC_TIME_VSYSCALL
  19 #include <vdso/datapage.h>
  20 #else
  21 struct vdso_data {
  22 };
  23 #endif
  24
  25 extern char vdso_start[], vdso_end[];
  26 #ifdef CONFIG_COMPAT
  27 extern char compat_vdso_start[], compat_vdso_end[];
  28 #endif
  29
  30 enum vvar_pages {
  31         VVAR_DATA_PAGE_OFFSET,
  32         VVAR_TIMENS_PAGE_OFFSET,
  33         VVAR_NR_PAGES,
  34 };
  35
  36 enum rv_vdso_map {
  37         RV_VDSO_MAP_VVAR,
  38         RV_VDSO_MAP_VDSO,
  39 };
  40
  41 #define VVAR_SIZE  (VVAR_NR_PAGES << PAGE_SHIFT)
  42
  43 /*
  44  * The vDSO data page.
  45  */
  46 static union {
  47         struct vdso_data        data;
  48         u8                      page[PAGE_SIZE];
  49 } vdso_data_store __page_aligned_data;
  50 struct vdso_data *vdso_data = &vdso_data_store.data;
  51
  52 struct __vdso_info {
  53         const char *name;
  54         const char *vdso_code_start;
  55         const char *vdso_code_end;
  56         unsigned long vdso_pages;
  57         /* Data Mapping */
  58         struct vm_special_mapping *dm;
  59         /* Code Mapping */
  60         struct vm_special_mapping *cm;
  61 };
  62
  63 static struct __vdso_info vdso_info;
  64 #ifdef CONFIG_COMPAT
  65 static struct __vdso_info compat_vdso_info;
  66 #endif
  67
  68 static int vdso_mremap(const struct vm_special_mapping *sm,
  69                        struct vm_area_struct *new_vma)
  70 {
  71         current->mm->context.vdso = (void *)new_vma->vm_start;
  72
  73         return 0;
  74 }
  75
  76 static void __init __vdso_init(struct __vdso_info *vdso_info)
  77 {
  78         unsigned int i;
  79         struct page **vdso_pagelist;
  80         unsigned long pfn;
  81
  82         if (memcmp(vdso_info->vdso_code_start, "\177ELF", 4))
  83                 panic("vDSO is not a valid ELF object!\n");
  84
  85         vdso_info->vdso_pages = (
  86                 vdso_info->vdso_code_end -
  87                 vdso_info->vdso_code_start) >>
  88                 PAGE_SHIFT;
  89
  90         vdso_pagelist = kcalloc(vdso_info->vdso_pages,
  91                                 sizeof(struct page *),
  92                                 GFP_KERNEL);
  93         if (vdso_pagelist == NULL)
  94                 panic("vDSO kcalloc failed!\n");
  95
  96         /* Grab the vDSO code pages. */
  97         pfn = sym_to_pfn(vdso_info->vdso_code_start);
  98
  99         for (i = 0; i < vdso_info->vdso_pages; i++)
 100                 vdso_pagelist[i] = pfn_to_page(pfn + i);
 101
 102         vdso_info->cm->pages = vdso_pagelist;
 103 }
 104
 105 #ifdef CONFIG_TIME_NS
 106 struct vdso_data *arch_get_vdso_data(void *vvar_page)
 107 {
 108         return (struct vdso_data *)(vvar_page);
 109 }
 110
 111 /*
 112  * The vvar mapping contains data for a specific time namespace, so when a task
 113  * changes namespace we must unmap its vvar data for the old namespace.
 114  * Subsequent faults will map in data for the new namespace.
 115  *
 116  * For more details see timens_setup_vdso_data().
 117  */
 118 int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
 119 {
 120         struct mm_struct *mm = task->mm;
 121         struct vm_area_struct *vma;
 122         VMA_ITERATOR(vmi, mm, 0);
 123
 124         mmap_read_lock(mm);
 125
 126         for_each_vma(vmi, vma) {
 127                 unsigned long size = vma->vm_end - vma->vm_start;
 128
 129                 if (vma_is_special_mapping(vma, vdso_info.dm))
 130                         zap_page_range(vma, vma->vm_start, size);
 131 #ifdef CONFIG_COMPAT
 132                 if (vma_is_special_mapping(vma, compat_vdso_info.dm))
 133                         zap_page_range(vma, vma->vm_start, size);
 134 #endif
 135         }
 136
 137         mmap_read_unlock(mm);
 138         return 0;
 139 }
 140
 141 static struct page *find_timens_vvar_page(struct vm_area_struct *vma)
 142 {
 143         if (likely(vma->vm_mm == current->mm))
 144                 return current->nsproxy->time_ns->vvar_page;
 145
 146         /*
 147          * VM_PFNMAP | VM_IO protect .fault() handler from being called
 148          * through interfaces like /proc/$pid/mem or
 149          * process_vm_{readv,writev}() as long as there's no .access()
 150          * in special_mapping_vmops.
 151          * For more details check_vma_flags() and __access_remote_vm()
 152          */
 153         WARN(1, "vvar_page accessed remotely");
 154
 155         return NULL;
 156 }
 157 #else
 158 static struct page *find_timens_vvar_page(struct vm_area_struct *vma)
 159 {
 160         return NULL;
 161 }
 162 #endif
 163
 164 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
 165                              struct vm_area_struct *vma, struct vm_fault *vmf)
 166 {
 167         struct page *timens_page = find_timens_vvar_page(vma);
 168         unsigned long pfn;
 169
 170         switch (vmf->pgoff) {
 171         case VVAR_DATA_PAGE_OFFSET:
 172                 if (timens_page)
 173                         pfn = page_to_pfn(timens_page);
 174                 else
 175                         pfn = sym_to_pfn(vdso_data);
 176                 break;
 177 #ifdef CONFIG_TIME_NS
 178         case VVAR_TIMENS_PAGE_OFFSET:
 179                 /*
 180                  * If a task belongs to a time namespace then a namespace
 181                  * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
 182                  * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
 183                  * offset.
 184                  * See also the comment near timens_setup_vdso_data().
 185                  */
 186                 if (!timens_page)
 187                         return VM_FAULT_SIGBUS;
 188                 pfn = sym_to_pfn(vdso_data);
 189                 break;
 190 #endif /* CONFIG_TIME_NS */
 191         default:
 192                 return VM_FAULT_SIGBUS;
 193         }
 194
 195         return vmf_insert_pfn(vma, vmf->address, pfn);
 196 }
 197
 198 static struct vm_special_mapping rv_vdso_maps[] __ro_after_init = {
 199         [RV_VDSO_MAP_VVAR] = {
 200                 .name   = "[vvar]",
 201                 .fault = vvar_fault,
 202         },
 203         [RV_VDSO_MAP_VDSO] = {
 204                 .name   = "[vdso]",
 205                 .mremap = vdso_mremap,
 206         },
 207 };
 208
 209 static struct __vdso_info vdso_info __ro_after_init = {
 210         .name = "vdso",
 211         .vdso_code_start = vdso_start,
 212         .vdso_code_end = vdso_end,
 213         .dm = &rv_vdso_maps[RV_VDSO_MAP_VVAR],
 214         .cm = &rv_vdso_maps[RV_VDSO_MAP_VDSO],
 215 };
 216
 217 #ifdef CONFIG_COMPAT
 218 static struct vm_special_mapping rv_compat_vdso_maps[] __ro_after_init = {
 219         [RV_VDSO_MAP_VVAR] = {
 220                 .name   = "[vvar]",
 221                 .fault = vvar_fault,
 222         },
 223         [RV_VDSO_MAP_VDSO] = {
 224                 .name   = "[vdso]",
 225                 .mremap = vdso_mremap,
 226         },
 227 };
 228
 229 static struct __vdso_info compat_vdso_info __ro_after_init = {
 230         .name = "compat_vdso",
 231         .vdso_code_start = compat_vdso_start,
 232         .vdso_code_end = compat_vdso_end,
 233         .dm = &rv_compat_vdso_maps[RV_VDSO_MAP_VVAR],
 234         .cm = &rv_compat_vdso_maps[RV_VDSO_MAP_VDSO],
 235 };
 236 #endif
 237
 238 static int __init vdso_init(void)
 239 {
 240         __vdso_init(&vdso_info);
 241 #ifdef CONFIG_COMPAT
 242         __vdso_init(&compat_vdso_info);
 243 #endif
 244
 245         return 0;
 246 }
 247 arch_initcall(vdso_init);
 248
 249 static int __setup_additional_pages(struct mm_struct *mm,
 250                                     struct linux_binprm *bprm,
 251                                     int uses_interp,
 252                                     struct __vdso_info *vdso_info)
 253 {
 254         unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
 255         void *ret;
 256
 257         BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
 258
 259         vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT;
 260         /* Be sure to map the data page */
 261         vdso_mapping_len = vdso_text_len + VVAR_SIZE;
 262
 263         vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
 264         if (IS_ERR_VALUE(vdso_base)) {
 265                 ret = ERR_PTR(vdso_base);
 266                 goto up_fail;
 267         }
 268
 269         ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE,
 270                 (VM_READ | VM_MAYREAD | VM_PFNMAP), vdso_info->dm);
 271         if (IS_ERR(ret))
 272                 goto up_fail;
 273
 274         vdso_base += VVAR_SIZE;
 275         mm->context.vdso = (void *)vdso_base;
 276
 277         ret =
 278            _install_special_mapping(mm, vdso_base, vdso_text_len,
 279                 (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC),
 280                 vdso_info->cm);
 281
 282         if (IS_ERR(ret))
 283                 goto up_fail;
 284
 285         return 0;
 286
 287 up_fail:
 288         mm->context.vdso = NULL;
 289         return PTR_ERR(ret);
 290 }
 291
 292 #ifdef CONFIG_COMPAT
 293 int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
 294                                        int uses_interp)
 295 {
 296         struct mm_struct *mm = current->mm;
 297         int ret;
 298
 299         if (mmap_write_lock_killable(mm))
 300                 return -EINTR;
 301
 302         ret = __setup_additional_pages(mm, bprm, uses_interp,
 303                                                         &compat_vdso_info);
 304         mmap_write_unlock(mm);
 305
 306         return ret;
 307 }
 308 #endif
 309
 310 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 311 {
 312         struct mm_struct *mm = current->mm;
 313         int ret;
 314
 315         if (mmap_write_lock_killable(mm))
 316                 return -EINTR;
 317
 318         ret = __setup_additional_pages(mm, bprm, uses_interp, &vdso_info);
 319         mmap_write_unlock(mm);
 320
 321         return ret;
 322 }