mm/damon/ops-common.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common Primitives for Data Access Monitoring
   4  *
   5  * Author: SeongJae Park <sj@kernel.org>
   6  */
   7
   8 #include <linux/mmu_notifier.h>
   9 #include <linux/page_idle.h>
  10 #include <linux/pagemap.h>
  11 #include <linux/rmap.h>
  12
  13 #include "ops-common.h"
  14
  15 /*
  16  * Get an online page for a pfn if it's in the LRU list.  Otherwise, returns
  17  * NULL.
  18  *
  19  * The body of this function is stolen from the 'page_idle_get_page()'.  We
  20  * steal rather than reuse it because the code is quite simple.
  21  */
  22 struct page *damon_get_page(unsigned long pfn)
  23 {
  24         struct page *page = pfn_to_online_page(pfn);
  25
  26         if (!page || !PageLRU(page) || !get_page_unless_zero(page))
  27                 return NULL;
  28
  29         if (unlikely(!PageLRU(page))) {
  30                 put_page(page);
  31                 page = NULL;
  32         }
  33         return page;
  34 }
  35
  36 void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
  37 {
  38         bool referenced = false;
  39         struct page *page = damon_get_page(pte_pfn(*pte));
  40
  41         if (!page)
  42                 return;
  43
  44         if (pte_young(*pte)) {
  45                 referenced = true;
  46                 *pte = pte_mkold(*pte);
  47         }
  48
  49 #ifdef CONFIG_MMU_NOTIFIER
  50         if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))
  51                 referenced = true;
  52 #endif /* CONFIG_MMU_NOTIFIER */
  53
  54         if (referenced)
  55                 set_page_young(page);
  56
  57         set_page_idle(page);
  58         put_page(page);
  59 }
  60
  61 void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
  62 {
  63 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  64         bool referenced = false;
  65         struct page *page = damon_get_page(pmd_pfn(*pmd));
  66
  67         if (!page)
  68                 return;
  69
  70         if (pmd_young(*pmd)) {
  71                 referenced = true;
  72                 *pmd = pmd_mkold(*pmd);
  73         }
  74
  75 #ifdef CONFIG_MMU_NOTIFIER
  76         if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE))
  77                 referenced = true;
  78 #endif /* CONFIG_MMU_NOTIFIER */
  79
  80         if (referenced)
  81                 set_page_young(page);
  82
  83         set_page_idle(page);
  84         put_page(page);
  85 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  86 }
  87
  88 #define DAMON_MAX_SUBSCORE      (100)
  89 #define DAMON_MAX_AGE_IN_LOG    (32)
  90
  91 int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
  92                         struct damos *s)
  93 {
  94         unsigned int max_nr_accesses;
  95         int freq_subscore;
  96         unsigned int age_in_sec;
  97         int age_in_log, age_subscore;
  98         unsigned int freq_weight = s->quota.weight_nr_accesses;
  99         unsigned int age_weight = s->quota.weight_age;
 100         int hotness;
 101
 102         max_nr_accesses = c->aggr_interval / c->sample_interval;
 103         freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
 104
 105         age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
 106         for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
 107                         age_in_log++, age_in_sec >>= 1)
 108                 ;
 109
 110         /* If frequency is 0, higher age means it's colder */
 111         if (freq_subscore == 0)
 112                 age_in_log *= -1;
 113
 114         /*
 115          * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
 116          * Scale it to be in [0, 100] and set it as age subscore.
 117          */
 118         age_in_log += DAMON_MAX_AGE_IN_LOG;
 119         age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
 120                 DAMON_MAX_AGE_IN_LOG / 2;
 121
 122         hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
 123         if (freq_weight + age_weight)
 124                 hotness /= freq_weight + age_weight;
 125         /*
 126          * Transform it to fit in [0, DAMOS_MAX_SCORE]
 127          */
 128         hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
 129
 130         /* Return coldness of the region */
 131         return DAMOS_MAX_SCORE - hotness;
 132 }
 133
 134 int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
 135                         struct damos *s)
 136 {
 137         unsigned int max_nr_accesses;
 138         int freq_subscore;
 139         unsigned int age_in_sec;
 140         int age_in_log, age_subscore;
 141         unsigned int freq_weight = s->quota.weight_nr_accesses;
 142         unsigned int age_weight = s->quota.weight_age;
 143         int hotness;
 144
 145         max_nr_accesses = c->aggr_interval / c->sample_interval;
 146         freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
 147
 148         age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
 149         for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
 150                         age_in_log++, age_in_sec >>= 1)
 151                 ;
 152
 153         /* If frequency is 0, higher age means it's colder */
 154         if (freq_subscore == 0)
 155                 age_in_log *= -1;
 156
 157         /*
 158          * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
 159          * Scale it to be in [0, 100] and set it as age subscore.
 160          */
 161         age_in_log += DAMON_MAX_AGE_IN_LOG;
 162         age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
 163                 DAMON_MAX_AGE_IN_LOG / 2;
 164
 165         hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
 166         if (freq_weight + age_weight)
 167                 hotness /= freq_weight + age_weight;
 168         /*
 169          * Transform it to fit in [0, DAMOS_MAX_SCORE]
 170          */
 171         hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
 172
 173         return hotness;
 174 }