arch/sparc/vdso/vclock_gettime.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2006 Andi Kleen, SUSE Labs.
   4  *
   5  * Fast user context implementation of clock_gettime, gettimeofday, and time.
   6  *
   7  * The code should have no internal unresolved relocations.
   8  * Check with readelf after changing.
   9  * Also alternative() doesn't work.
  10  */
  11 /*
  12  * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
  13  */
  14
  15 #include <linux/kernel.h>
  16 #include <linux/time.h>
  17 #include <linux/string.h>
  18 #include <asm/io.h>
  19 #include <asm/unistd.h>
  20 #include <asm/timex.h>
  21 #include <asm/clocksource.h>
  22 #include <asm/vvar.h>
  23
  24 #ifdef  CONFIG_SPARC64
  25 #define SYSCALL_STRING                                                  \
  26         "ta     0x6d;"                                                  \
  27         "bcs,a  1f;"                                                    \
  28         " sub   %%g0, %%o0, %%o0;"                                      \
  29         "1:"
  30 #else
  31 #define SYSCALL_STRING                                                  \
  32         "ta     0x10;"                                                  \
  33         "bcs,a  1f;"                                                    \
  34         " sub   %%g0, %%o0, %%o0;"                                      \
  35         "1:"
  36 #endif
  37
  38 #define SYSCALL_CLOBBERS                                                \
  39         "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",                 \
  40         "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",           \
  41         "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",         \
  42         "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",         \
  43         "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",         \
  44         "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",         \
  45         "cc", "memory"
  46
  47 /*
  48  * Compute the vvar page's address in the process address space, and return it
  49  * as a pointer to the vvar_data.
  50  */
  51 notrace static __always_inline struct vvar_data *get_vvar_data(void)
  52 {
  53         unsigned long ret;
  54
  55         /*
  56          * vdso data page is the first vDSO page so grab the PC
  57          * and move up a page to get to the data page.
  58          */
  59         __asm__("rd %%pc, %0" : "=r" (ret));
  60         ret &= ~(8192 - 1);
  61         ret -= 8192;
  62
  63         return (struct vvar_data *) ret;
  64 }
  65
  66 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
  67 {
  68         register long num __asm__("g1") = __NR_clock_gettime;
  69         register long o0 __asm__("o0") = clock;
  70         register long o1 __asm__("o1") = (long) ts;
  71
  72         __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
  73                              "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
  74         return o0;
  75 }
  76
  77 notrace static long vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz)
  78 {
  79         register long num __asm__("g1") = __NR_gettimeofday;
  80         register long o0 __asm__("o0") = (long) tv;
  81         register long o1 __asm__("o1") = (long) tz;
  82
  83         __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
  84                              "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
  85         return o0;
  86 }
  87
  88 #ifdef  CONFIG_SPARC64
  89 notrace static __always_inline u64 vread_tick(void)
  90 {
  91         u64     ret;
  92
  93         __asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
  94         return ret;
  95 }
  96
  97 notrace static __always_inline u64 vread_tick_stick(void)
  98 {
  99         u64     ret;
 100
 101         __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
 102         return ret;
 103 }
 104 #else
 105 notrace static __always_inline u64 vread_tick(void)
 106 {
 107         register unsigned long long ret asm("o4");
 108
 109         __asm__ __volatile__("rd %%tick, %L0\n\t"
 110                              "srlx %L0, 32, %H0"
 111                              : "=r" (ret));
 112         return ret;
 113 }
 114
 115 notrace static __always_inline u64 vread_tick_stick(void)
 116 {
 117         register unsigned long long ret asm("o4");
 118
 119         __asm__ __volatile__("rd %%asr24, %L0\n\t"
 120                              "srlx %L0, 32, %H0"
 121                              : "=r" (ret));
 122         return ret;
 123 }
 124 #endif
 125
 126 notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
 127 {
 128         u64 v;
 129         u64 cycles;
 130
 131         cycles = vread_tick();
 132         v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
 133         return v * vvar->clock.mult;
 134 }
 135
 136 notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
 137 {
 138         u64 v;
 139         u64 cycles;
 140
 141         cycles = vread_tick_stick();
 142         v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
 143         return v * vvar->clock.mult;
 144 }
 145
 146 notrace static __always_inline int do_realtime(struct vvar_data *vvar,
 147                                                struct timespec *ts)
 148 {
 149         unsigned long seq;
 150         u64 ns;
 151
 152         do {
 153                 seq = vvar_read_begin(vvar);
 154                 ts->tv_sec = vvar->wall_time_sec;
 155                 ns = vvar->wall_time_snsec;
 156                 ns += vgetsns(vvar);
 157                 ns >>= vvar->clock.shift;
 158         } while (unlikely(vvar_read_retry(vvar, seq)));
 159
 160         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 161         ts->tv_nsec = ns;
 162
 163         return 0;
 164 }
 165
 166 notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
 167                                                      struct timespec *ts)
 168 {
 169         unsigned long seq;
 170         u64 ns;
 171
 172         do {
 173                 seq = vvar_read_begin(vvar);
 174                 ts->tv_sec = vvar->wall_time_sec;
 175                 ns = vvar->wall_time_snsec;
 176                 ns += vgetsns_stick(vvar);
 177                 ns >>= vvar->clock.shift;
 178         } while (unlikely(vvar_read_retry(vvar, seq)));
 179
 180         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 181         ts->tv_nsec = ns;
 182
 183         return 0;
 184 }
 185
 186 notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
 187                                                 struct timespec *ts)
 188 {
 189         unsigned long seq;
 190         u64 ns;
 191
 192         do {
 193                 seq = vvar_read_begin(vvar);
 194                 ts->tv_sec = vvar->monotonic_time_sec;
 195                 ns = vvar->monotonic_time_snsec;
 196                 ns += vgetsns(vvar);
 197                 ns >>= vvar->clock.shift;
 198         } while (unlikely(vvar_read_retry(vvar, seq)));
 199
 200         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 201         ts->tv_nsec = ns;
 202
 203         return 0;
 204 }
 205
 206 notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
 207                                                       struct timespec *ts)
 208 {
 209         unsigned long seq;
 210         u64 ns;
 211
 212         do {
 213                 seq = vvar_read_begin(vvar);
 214                 ts->tv_sec = vvar->monotonic_time_sec;
 215                 ns = vvar->monotonic_time_snsec;
 216                 ns += vgetsns_stick(vvar);
 217                 ns >>= vvar->clock.shift;
 218         } while (unlikely(vvar_read_retry(vvar, seq)));
 219
 220         ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 221         ts->tv_nsec = ns;
 222
 223         return 0;
 224 }
 225
 226 notrace static int do_realtime_coarse(struct vvar_data *vvar,
 227                                       struct timespec *ts)
 228 {
 229         unsigned long seq;
 230
 231         do {
 232                 seq = vvar_read_begin(vvar);
 233                 ts->tv_sec = vvar->wall_time_coarse_sec;
 234                 ts->tv_nsec = vvar->wall_time_coarse_nsec;
 235         } while (unlikely(vvar_read_retry(vvar, seq)));
 236         return 0;
 237 }
 238
 239 notrace static int do_monotonic_coarse(struct vvar_data *vvar,
 240                                        struct timespec *ts)
 241 {
 242         unsigned long seq;
 243
 244         do {
 245                 seq = vvar_read_begin(vvar);
 246                 ts->tv_sec = vvar->monotonic_time_coarse_sec;
 247                 ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
 248         } while (unlikely(vvar_read_retry(vvar, seq)));
 249
 250         return 0;
 251 }
 252
 253 notrace int
 254 __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 255 {
 256         struct vvar_data *vvd = get_vvar_data();
 257
 258         switch (clock) {
 259         case CLOCK_REALTIME:
 260                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 261                         break;
 262                 return do_realtime(vvd, ts);
 263         case CLOCK_MONOTONIC:
 264                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 265                         break;
 266                 return do_monotonic(vvd, ts);
 267         case CLOCK_REALTIME_COARSE:
 268                 return do_realtime_coarse(vvd, ts);
 269         case CLOCK_MONOTONIC_COARSE:
 270                 return do_monotonic_coarse(vvd, ts);
 271         }
 272         /*
 273          * Unknown clock ID ? Fall back to the syscall.
 274          */
 275         return vdso_fallback_gettime(clock, ts);
 276 }
 277 int
 278 clock_gettime(clockid_t, struct timespec *)
 279         __attribute__((weak, alias("__vdso_clock_gettime")));
 280
 281 notrace int
 282 __vdso_clock_gettime_stick(clockid_t clock, struct timespec *ts)
 283 {
 284         struct vvar_data *vvd = get_vvar_data();
 285
 286         switch (clock) {
 287         case CLOCK_REALTIME:
 288                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 289                         break;
 290                 return do_realtime_stick(vvd, ts);
 291         case CLOCK_MONOTONIC:
 292                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
 293                         break;
 294                 return do_monotonic_stick(vvd, ts);
 295         case CLOCK_REALTIME_COARSE:
 296                 return do_realtime_coarse(vvd, ts);
 297         case CLOCK_MONOTONIC_COARSE:
 298                 return do_monotonic_coarse(vvd, ts);
 299         }
 300         /*
 301          * Unknown clock ID ? Fall back to the syscall.
 302          */
 303         return vdso_fallback_gettime(clock, ts);
 304 }
 305
 306 notrace int
 307 __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 308 {
 309         struct vvar_data *vvd = get_vvar_data();
 310
 311         if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
 312                 if (likely(tv != NULL)) {
 313                         union tstv_t {
 314                                 struct timespec ts;
 315                                 struct timeval tv;
 316                         } *tstv = (union tstv_t *) tv;
 317                         do_realtime(vvd, &tstv->ts);
 318                         /*
 319                          * Assign before dividing to ensure that the division is
 320                          * done in the type of tv_usec, not tv_nsec.
 321                          *
 322                          * There cannot be > 1 billion usec in a second:
 323                          * do_realtime() has already distributed such overflow
 324                          * into tv_sec.  So we can assign it to an int safely.
 325                          */
 326                         tstv->tv.tv_usec = tstv->ts.tv_nsec;
 327                         tstv->tv.tv_usec /= 1000;
 328                 }
 329                 if (unlikely(tz != NULL)) {
 330                         /* Avoid memcpy. Some old compilers fail to inline it */
 331                         tz->tz_minuteswest = vvd->tz_minuteswest;
 332                         tz->tz_dsttime = vvd->tz_dsttime;
 333                 }
 334                 return 0;
 335         }
 336         return vdso_fallback_gettimeofday(tv, tz);
 337 }
 338 int
 339 gettimeofday(struct timeval *, struct timezone *)
 340         __attribute__((weak, alias("__vdso_gettimeofday")));
 341
 342 notrace int
 343 __vdso_gettimeofday_stick(struct timeval *tv, struct timezone *tz)
 344 {
 345         struct vvar_data *vvd = get_vvar_data();
 346
 347         if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
 348                 if (likely(tv != NULL)) {
 349                         union tstv_t {
 350                                 struct timespec ts;
 351                                 struct timeval tv;
 352                         } *tstv = (union tstv_t *) tv;
 353                         do_realtime_stick(vvd, &tstv->ts);
 354                         /*
 355                          * Assign before dividing to ensure that the division is
 356                          * done in the type of tv_usec, not tv_nsec.
 357                          *
 358                          * There cannot be > 1 billion usec in a second:
 359                          * do_realtime() has already distributed such overflow
 360                          * into tv_sec.  So we can assign it to an int safely.
 361                          */
 362                         tstv->tv.tv_usec = tstv->ts.tv_nsec;
 363                         tstv->tv.tv_usec /= 1000;
 364                 }
 365                 if (unlikely(tz != NULL)) {
 366                         /* Avoid memcpy. Some old compilers fail to inline it */
 367                         tz->tz_minuteswest = vvd->tz_minuteswest;
 368                         tz->tz_dsttime = vvd->tz_dsttime;
 369                 }
 370                 return 0;
 371         }
 372         return vdso_fallback_gettimeofday(tv, tz);
 373 }