kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/ftrace.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/slab.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31 #include <linux/rcupdate.h>
  32
  33 #include <trace/events/sched.h>
  34
  35 #include <asm/ftrace.h>
  36 #include <asm/setup.h>
  37
  38 #include "trace_output.h"
  39 #include "trace_stat.h"
  40
  41 #define FTRACE_WARN_ON(cond)                    \
  42         do {                                    \
  43                 if (WARN_ON(cond))              \
  44                         ftrace_kill();          \
  45         } while (0)
  46
  47 #define FTRACE_WARN_ON_ONCE(cond)               \
  48         do {                                    \
  49                 if (WARN_ON_ONCE(cond))         \
  50                         ftrace_kill();          \
  51         } while (0)
  52
  53 /* hash bits for specific function selection */
  54 #define FTRACE_HASH_BITS 7
  55 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  56
  57 /* ftrace_enabled is a method to turn ftrace on or off */
  58 int ftrace_enabled __read_mostly;
  59 static int last_ftrace_enabled;
  60
  61 /* Quick disabling of function tracer. */
  62 int function_trace_stop;
  63
  64 /* List for set_ftrace_pid's pids. */
  65 LIST_HEAD(ftrace_pids);
  66 struct ftrace_pid {
  67         struct list_head list;
  68         struct pid *pid;
  69 };
  70
  71 /*
  72  * ftrace_disabled is set when an anomaly is discovered.
  73  * ftrace_disabled is much stronger than ftrace_enabled.
  74  */
  75 static int ftrace_disabled __read_mostly;
  76
  77 static DEFINE_MUTEX(ftrace_lock);
  78
  79 static struct ftrace_ops ftrace_list_end __read_mostly =
  80 {
  81         .func           = ftrace_stub,
  82 };
  83
  84 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  85 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  86 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  87 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  88
  89 /*
  90  * Traverse the ftrace_list, invoking all entries.  The reason that we
  91  * can use rcu_dereference_raw() is that elements removed from this list
  92  * are simply leaked, so there is no need to interact with a grace-period
  93  * mechanism.  The rcu_dereference_raw() calls are needed to handle
  94  * concurrent insertions into the ftrace_list.
  95  *
  96  * Silly Alpha and silly pointer-speculation compiler optimizations!
  97  */
  98 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  99 {
 100         struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/
 101
 102         while (op != &ftrace_list_end) {
 103                 op->func(ip, parent_ip);
 104                 op = rcu_dereference_raw(op->next); /*see above*/
 105         };
 106 }
 107
 108 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 109 {
 110         if (!test_tsk_trace_trace(current))
 111                 return;
 112
 113         ftrace_pid_function(ip, parent_ip);
 114 }
 115
 116 static void set_ftrace_pid_function(ftrace_func_t func)
 117 {
 118         /* do not set ftrace_pid_function to itself! */
 119         if (func != ftrace_pid_func)
 120                 ftrace_pid_function = func;
 121 }
 122
 123 /**
 124  * clear_ftrace_function - reset the ftrace function
 125  *
 126  * This NULLs the ftrace function and in essence stops
 127  * tracing.  There may be lag
 128  */
 129 void clear_ftrace_function(void)
 130 {
 131         ftrace_trace_function = ftrace_stub;
 132         __ftrace_trace_function = ftrace_stub;
 133         ftrace_pid_function = ftrace_stub;
 134 }
 135
 136 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 137 /*
 138  * For those archs that do not test ftrace_trace_stop in their
 139  * mcount call site, we need to do it from C.
 140  */
 141 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 142 {
 143         if (function_trace_stop)
 144                 return;
 145
 146         __ftrace_trace_function(ip, parent_ip);
 147 }
 148 #endif
 149
 150 static int __register_ftrace_function(struct ftrace_ops *ops)
 151 {
 152         ops->next = ftrace_list;
 153         /*
 154          * We are entering ops into the ftrace_list but another
 155          * CPU might be walking that list. We need to make sure
 156          * the ops->next pointer is valid before another CPU sees
 157          * the ops pointer included into the ftrace_list.
 158          */
 159         rcu_assign_pointer(ftrace_list, ops);
 160
 161         if (ftrace_enabled) {
 162                 ftrace_func_t func;
 163
 164                 if (ops->next == &ftrace_list_end)
 165                         func = ops->func;
 166                 else
 167                         func = ftrace_list_func;
 168
 169                 if (!list_empty(&ftrace_pids)) {
 170                         set_ftrace_pid_function(func);
 171                         func = ftrace_pid_func;
 172                 }
 173
 174                 /*
 175                  * For one func, simply call it directly.
 176                  * For more than one func, call the chain.
 177                  */
 178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 179                 ftrace_trace_function = func;
 180 #else
 181                 __ftrace_trace_function = func;
 182                 ftrace_trace_function = ftrace_test_stop_func;
 183 #endif
 184         }
 185
 186         return 0;
 187 }
 188
 189 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 190 {
 191         struct ftrace_ops **p;
 192
 193         /*
 194          * If we are removing the last function, then simply point
 195          * to the ftrace_stub.
 196          */
 197         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 198                 ftrace_trace_function = ftrace_stub;
 199                 ftrace_list = &ftrace_list_end;
 200                 return 0;
 201         }
 202
 203         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 204                 if (*p == ops)
 205                         break;
 206
 207         if (*p != ops)
 208                 return -1;
 209
 210         *p = (*p)->next;
 211
 212         if (ftrace_enabled) {
 213                 /* If we only have one func left, then call that directly */
 214                 if (ftrace_list->next == &ftrace_list_end) {
 215                         ftrace_func_t func = ftrace_list->func;
 216
 217                         if (!list_empty(&ftrace_pids)) {
 218                                 set_ftrace_pid_function(func);
 219                                 func = ftrace_pid_func;
 220                         }
 221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 222                         ftrace_trace_function = func;
 223 #else
 224                         __ftrace_trace_function = func;
 225 #endif
 226                 }
 227         }
 228
 229         return 0;
 230 }
 231
 232 static void ftrace_update_pid_func(void)
 233 {
 234         ftrace_func_t func;
 235
 236         if (ftrace_trace_function == ftrace_stub)
 237                 return;
 238
 239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 240         func = ftrace_trace_function;
 241 #else
 242         func = __ftrace_trace_function;
 243 #endif
 244
 245         if (!list_empty(&ftrace_pids)) {
 246                 set_ftrace_pid_function(func);
 247                 func = ftrace_pid_func;
 248         } else {
 249                 if (func == ftrace_pid_func)
 250                         func = ftrace_pid_function;
 251         }
 252
 253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 254         ftrace_trace_function = func;
 255 #else
 256         __ftrace_trace_function = func;
 257 #endif
 258 }
 259
 260 #ifdef CONFIG_FUNCTION_PROFILER
 261 struct ftrace_profile {
 262         struct hlist_node               node;
 263         unsigned long                   ip;
 264         unsigned long                   counter;
 265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 266         unsigned long long              time;
 267 #endif
 268 };
 269
 270 struct ftrace_profile_page {
 271         struct ftrace_profile_page      *next;
 272         unsigned long                   index;
 273         struct ftrace_profile           records[];
 274 };
 275
 276 struct ftrace_profile_stat {
 277         atomic_t                        disabled;
 278         struct hlist_head               *hash;
 279         struct ftrace_profile_page      *pages;
 280         struct ftrace_profile_page      *start;
 281         struct tracer_stat              stat;
 282 };
 283
 284 #define PROFILE_RECORDS_SIZE                                            \
 285         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 286
 287 #define PROFILES_PER_PAGE                                       \
 288         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 289
 290 static int ftrace_profile_bits __read_mostly;
 291 static int ftrace_profile_enabled __read_mostly;
 292
 293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 294 static DEFINE_MUTEX(ftrace_profile_lock);
 295
 296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 297
 298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 299
 300 static void *
 301 function_stat_next(void *v, int idx)
 302 {
 303         struct ftrace_profile *rec = v;
 304         struct ftrace_profile_page *pg;
 305
 306         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 307
 308  again:
 309         if (idx != 0)
 310                 rec++;
 311
 312         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 313                 pg = pg->next;
 314                 if (!pg)
 315                         return NULL;
 316                 rec = &pg->records[0];
 317                 if (!rec->counter)
 318                         goto again;
 319         }
 320
 321         return rec;
 322 }
 323
 324 static void *function_stat_start(struct tracer_stat *trace)
 325 {
 326         struct ftrace_profile_stat *stat =
 327                 container_of(trace, struct ftrace_profile_stat, stat);
 328
 329         if (!stat || !stat->start)
 330                 return NULL;
 331
 332         return function_stat_next(&stat->start->records[0], 0);
 333 }
 334
 335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 336 /* function graph compares on total time */
 337 static int function_stat_cmp(void *p1, void *p2)
 338 {
 339         struct ftrace_profile *a = p1;
 340         struct ftrace_profile *b = p2;
 341
 342         if (a->time < b->time)
 343                 return -1;
 344         if (a->time > b->time)
 345                 return 1;
 346         else
 347                 return 0;
 348 }
 349 #else
 350 /* not function graph compares against hits */
 351 static int function_stat_cmp(void *p1, void *p2)
 352 {
 353         struct ftrace_profile *a = p1;
 354         struct ftrace_profile *b = p2;
 355
 356         if (a->counter < b->counter)
 357                 return -1;
 358         if (a->counter > b->counter)
 359                 return 1;
 360         else
 361                 return 0;
 362 }
 363 #endif
 364
 365 static int function_stat_headers(struct seq_file *m)
 366 {
 367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 368         seq_printf(m, "  Function                               "
 369                    "Hit    Time            Avg\n"
 370                       "  --------                               "
 371                    "---    ----            ---\n");
 372 #else
 373         seq_printf(m, "  Function                               Hit\n"
 374                       "  --------                               ---\n");
 375 #endif
 376         return 0;
 377 }
 378
 379 static int function_stat_show(struct seq_file *m, void *v)
 380 {
 381         struct ftrace_profile *rec = v;
 382         char str[KSYM_SYMBOL_LEN];
 383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 384         static DEFINE_MUTEX(mutex);
 385         static struct trace_seq s;
 386         unsigned long long avg;
 387 #endif
 388
 389         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 390         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 391
 392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 393         seq_printf(m, "    ");
 394         avg = rec->time;
 395         do_div(avg, rec->counter);
 396
 397         mutex_lock(&mutex);
 398         trace_seq_init(&s);
 399         trace_print_graph_duration(rec->time, &s);
 400         trace_seq_puts(&s, "    ");
 401         trace_print_graph_duration(avg, &s);
 402         trace_print_seq(m, &s);
 403         mutex_unlock(&mutex);
 404 #endif
 405         seq_putc(m, '\n');
 406
 407         return 0;
 408 }
 409
 410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 411 {
 412         struct ftrace_profile_page *pg;
 413
 414         pg = stat->pages = stat->start;
 415
 416         while (pg) {
 417                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 418                 pg->index = 0;
 419                 pg = pg->next;
 420         }
 421
 422         memset(stat->hash, 0,
 423                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 424 }
 425
 426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 427 {
 428         struct ftrace_profile_page *pg;
 429         int functions;
 430         int pages;
 431         int i;
 432
 433         /* If we already allocated, do nothing */
 434         if (stat->pages)
 435                 return 0;
 436
 437         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 438         if (!stat->pages)
 439                 return -ENOMEM;
 440
 441 #ifdef CONFIG_DYNAMIC_FTRACE
 442         functions = ftrace_update_tot_cnt;
 443 #else
 444         /*
 445          * We do not know the number of functions that exist because
 446          * dynamic tracing is what counts them. With past experience
 447          * we have around 20K functions. That should be more than enough.
 448          * It is highly unlikely we will execute every function in
 449          * the kernel.
 450          */
 451         functions = 20000;
 452 #endif
 453
 454         pg = stat->start = stat->pages;
 455
 456         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 457
 458         for (i = 0; i < pages; i++) {
 459                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 460                 if (!pg->next)
 461                         goto out_free;
 462                 pg = pg->next;
 463         }
 464
 465         return 0;
 466
 467  out_free:
 468         pg = stat->start;
 469         while (pg) {
 470                 unsigned long tmp = (unsigned long)pg;
 471
 472                 pg = pg->next;
 473                 free_page(tmp);
 474         }
 475
 476         free_page((unsigned long)stat->pages);
 477         stat->pages = NULL;
 478         stat->start = NULL;
 479
 480         return -ENOMEM;
 481 }
 482
 483 static int ftrace_profile_init_cpu(int cpu)
 484 {
 485         struct ftrace_profile_stat *stat;
 486         int size;
 487
 488         stat = &per_cpu(ftrace_profile_stats, cpu);
 489
 490         if (stat->hash) {
 491                 /* If the profile is already created, simply reset it */
 492                 ftrace_profile_reset(stat);
 493                 return 0;
 494         }
 495
 496         /*
 497          * We are profiling all functions, but usually only a few thousand
 498          * functions are hit. We'll make a hash of 1024 items.
 499          */
 500         size = FTRACE_PROFILE_HASH_SIZE;
 501
 502         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 503
 504         if (!stat->hash)
 505                 return -ENOMEM;
 506
 507         if (!ftrace_profile_bits) {
 508                 size--;
 509
 510                 for (; size; size >>= 1)
 511                         ftrace_profile_bits++;
 512         }
 513
 514         /* Preallocate the function profiling pages */
 515         if (ftrace_profile_pages_init(stat) < 0) {
 516                 kfree(stat->hash);
 517                 stat->hash = NULL;
 518                 return -ENOMEM;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int ftrace_profile_init(void)
 525 {
 526         int cpu;
 527         int ret = 0;
 528
 529         for_each_online_cpu(cpu) {
 530                 ret = ftrace_profile_init_cpu(cpu);
 531                 if (ret)
 532                         break;
 533         }
 534
 535         return ret;
 536 }
 537
 538 /* interrupts must be disabled */
 539 static struct ftrace_profile *
 540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 541 {
 542         struct ftrace_profile *rec;
 543         struct hlist_head *hhd;
 544         struct hlist_node *n;
 545         unsigned long key;
 546
 547         key = hash_long(ip, ftrace_profile_bits);
 548         hhd = &stat->hash[key];
 549
 550         if (hlist_empty(hhd))
 551                 return NULL;
 552
 553         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 554                 if (rec->ip == ip)
 555                         return rec;
 556         }
 557
 558         return NULL;
 559 }
 560
 561 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 562                                struct ftrace_profile *rec)
 563 {
 564         unsigned long key;
 565
 566         key = hash_long(rec->ip, ftrace_profile_bits);
 567         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 568 }
 569
 570 /*
 571  * The memory is already allocated, this simply finds a new record to use.
 572  */
 573 static struct ftrace_profile *
 574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 575 {
 576         struct ftrace_profile *rec = NULL;
 577
 578         /* prevent recursion (from NMIs) */
 579         if (atomic_inc_return(&stat->disabled) != 1)
 580                 goto out;
 581
 582         /*
 583          * Try to find the function again since an NMI
 584          * could have added it
 585          */
 586         rec = ftrace_find_profiled_func(stat, ip);
 587         if (rec)
 588                 goto out;
 589
 590         if (stat->pages->index == PROFILES_PER_PAGE) {
 591                 if (!stat->pages->next)
 592                         goto out;
 593                 stat->pages = stat->pages->next;
 594         }
 595
 596         rec = &stat->pages->records[stat->pages->index++];
 597         rec->ip = ip;
 598         ftrace_add_profile(stat, rec);
 599
 600  out:
 601         atomic_dec(&stat->disabled);
 602
 603         return rec;
 604 }
 605
 606 static void
 607 function_profile_call(unsigned long ip, unsigned long parent_ip)
 608 {
 609         struct ftrace_profile_stat *stat;
 610         struct ftrace_profile *rec;
 611         unsigned long flags;
 612
 613         if (!ftrace_profile_enabled)
 614                 return;
 615
 616         local_irq_save(flags);
 617
 618         stat = &__get_cpu_var(ftrace_profile_stats);
 619         if (!stat->hash || !ftrace_profile_enabled)
 620                 goto out;
 621
 622         rec = ftrace_find_profiled_func(stat, ip);
 623         if (!rec) {
 624                 rec = ftrace_profile_alloc(stat, ip);
 625                 if (!rec)
 626                         goto out;
 627         }
 628
 629         rec->counter++;
 630  out:
 631         local_irq_restore(flags);
 632 }
 633
 634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 635 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 636 {
 637         function_profile_call(trace->func, 0);
 638         return 1;
 639 }
 640
 641 static void profile_graph_return(struct ftrace_graph_ret *trace)
 642 {
 643         struct ftrace_profile_stat *stat;
 644         unsigned long long calltime;
 645         struct ftrace_profile *rec;
 646         unsigned long flags;
 647
 648         local_irq_save(flags);
 649         stat = &__get_cpu_var(ftrace_profile_stats);
 650         if (!stat->hash || !ftrace_profile_enabled)
 651                 goto out;
 652
 653         calltime = trace->rettime - trace->calltime;
 654
 655         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 656                 int index;
 657
 658                 index = trace->depth;
 659
 660                 /* Append this call time to the parent time to subtract */
 661                 if (index)
 662                         current->ret_stack[index - 1].subtime += calltime;
 663
 664                 if (current->ret_stack[index].subtime < calltime)
 665                         calltime -= current->ret_stack[index].subtime;
 666                 else
 667                         calltime = 0;
 668         }
 669
 670         rec = ftrace_find_profiled_func(stat, trace->func);
 671         if (rec)
 672                 rec->time += calltime;
 673
 674  out:
 675         local_irq_restore(flags);
 676 }
 677
 678 static int register_ftrace_profiler(void)
 679 {
 680         return register_ftrace_graph(&profile_graph_return,
 681                                      &profile_graph_entry);
 682 }
 683
 684 static void unregister_ftrace_profiler(void)
 685 {
 686         unregister_ftrace_graph();
 687 }
 688 #else
 689 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 690 {
 691         .func           = function_profile_call,
 692 };
 693
 694 static int register_ftrace_profiler(void)
 695 {
 696         return register_ftrace_function(&ftrace_profile_ops);
 697 }
 698
 699 static void unregister_ftrace_profiler(void)
 700 {
 701         unregister_ftrace_function(&ftrace_profile_ops);
 702 }
 703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 704
 705 static ssize_t
 706 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 707                      size_t cnt, loff_t *ppos)
 708 {
 709         unsigned long val;
 710         char buf[64];           /* big enough to hold a number */
 711         int ret;
 712
 713         if (cnt >= sizeof(buf))
 714                 return -EINVAL;
 715
 716         if (copy_from_user(&buf, ubuf, cnt))
 717                 return -EFAULT;
 718
 719         buf[cnt] = 0;
 720
 721         ret = strict_strtoul(buf, 10, &val);
 722         if (ret < 0)
 723                 return ret;
 724
 725         val = !!val;
 726
 727         mutex_lock(&ftrace_profile_lock);
 728         if (ftrace_profile_enabled ^ val) {
 729                 if (val) {
 730                         ret = ftrace_profile_init();
 731                         if (ret < 0) {
 732                                 cnt = ret;
 733                                 goto out;
 734                         }
 735
 736                         ret = register_ftrace_profiler();
 737                         if (ret < 0) {
 738                                 cnt = ret;
 739                                 goto out;
 740                         }
 741                         ftrace_profile_enabled = 1;
 742                 } else {
 743                         ftrace_profile_enabled = 0;
 744                         /*
 745                          * unregister_ftrace_profiler calls stop_machine
 746                          * so this acts like an synchronize_sched.
 747                          */
 748                         unregister_ftrace_profiler();
 749                 }
 750         }
 751  out:
 752         mutex_unlock(&ftrace_profile_lock);
 753
 754         *ppos += cnt;
 755
 756         return cnt;
 757 }
 758
 759 static ssize_t
 760 ftrace_profile_read(struct file *filp, char __user *ubuf,
 761                      size_t cnt, loff_t *ppos)
 762 {
 763         char buf[64];           /* big enough to hold a number */
 764         int r;
 765
 766         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 767         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 768 }
 769
 770 static const struct file_operations ftrace_profile_fops = {
 771         .open           = tracing_open_generic,
 772         .read           = ftrace_profile_read,
 773         .write          = ftrace_profile_write,
 774 };
 775
 776 /* used to initialize the real stat files */
 777 static struct tracer_stat function_stats __initdata = {
 778         .name           = "functions",
 779         .stat_start     = function_stat_start,
 780         .stat_next      = function_stat_next,
 781         .stat_cmp       = function_stat_cmp,
 782         .stat_headers   = function_stat_headers,
 783         .stat_show      = function_stat_show
 784 };
 785
 786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 787 {
 788         struct ftrace_profile_stat *stat;
 789         struct dentry *entry;
 790         char *name;
 791         int ret;
 792         int cpu;
 793
 794         for_each_possible_cpu(cpu) {
 795                 stat = &per_cpu(ftrace_profile_stats, cpu);
 796
 797                 /* allocate enough for function name + cpu number */
 798                 name = kmalloc(32, GFP_KERNEL);
 799                 if (!name) {
 800                         /*
 801                          * The files created are permanent, if something happens
 802                          * we still do not free memory.
 803                          */
 804                         WARN(1,
 805                              "Could not allocate stat file for cpu %d\n",
 806                              cpu);
 807                         return;
 808                 }
 809                 stat->stat = function_stats;
 810                 snprintf(name, 32, "function%d", cpu);
 811                 stat->stat.name = name;
 812                 ret = register_stat_tracer(&stat->stat);
 813                 if (ret) {
 814                         WARN(1,
 815                              "Could not register function stat for cpu %d\n",
 816                              cpu);
 817                         kfree(name);
 818                         return;
 819                 }
 820         }
 821
 822         entry = debugfs_create_file("function_profile_enabled", 0644,
 823                                     d_tracer, NULL, &ftrace_profile_fops);
 824         if (!entry)
 825                 pr_warning("Could not create debugfs "
 826                            "'function_profile_enabled' entry\n");
 827 }
 828
 829 #else /* CONFIG_FUNCTION_PROFILER */
 830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 831 {
 832 }
 833 #endif /* CONFIG_FUNCTION_PROFILER */
 834
 835 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 836
 837 #ifdef CONFIG_DYNAMIC_FTRACE
 838
 839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 840 # error Dynamic ftrace depends on MCOUNT_RECORD
 841 #endif
 842
 843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 844
 845 struct ftrace_func_probe {
 846         struct hlist_node       node;
 847         struct ftrace_probe_ops *ops;
 848         unsigned long           flags;
 849         unsigned long           ip;
 850         void                    *data;
 851         struct rcu_head         rcu;
 852 };
 853
 854 enum {
 855         FTRACE_ENABLE_CALLS             = (1 << 0),
 856         FTRACE_DISABLE_CALLS            = (1 << 1),
 857         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 858         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 859         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 860         FTRACE_START_FUNC_RET           = (1 << 5),
 861         FTRACE_STOP_FUNC_RET            = (1 << 6),
 862 };
 863
 864 static int ftrace_filtered;
 865
 866 static struct dyn_ftrace *ftrace_new_addrs;
 867
 868 static DEFINE_MUTEX(ftrace_regex_lock);
 869
 870 struct ftrace_page {
 871         struct ftrace_page      *next;
 872         int                     index;
 873         struct dyn_ftrace       records[];
 874 };
 875
 876 #define ENTRIES_PER_PAGE \
 877   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 878
 879 /* estimate from running different kernels */
 880 #define NR_TO_INIT              10000
 881
 882 static struct ftrace_page       *ftrace_pages_start;
 883 static struct ftrace_page       *ftrace_pages;
 884
 885 static struct dyn_ftrace *ftrace_free_records;
 886
 887 /*
 888  * This is a double for. Do not use 'break' to break out of the loop,
 889  * you must use a goto.
 890  */
 891 #define do_for_each_ftrace_rec(pg, rec)                                 \
 892         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 893                 int _____i;                                             \
 894                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 895                         rec = &pg->records[_____i];
 896
 897 #define while_for_each_ftrace_rec()             \
 898                 }                               \
 899         }
 900
 901 static void ftrace_free_rec(struct dyn_ftrace *rec)
 902 {
 903         rec->freelist = ftrace_free_records;
 904         ftrace_free_records = rec;
 905         rec->flags |= FTRACE_FL_FREE;
 906 }
 907
 908 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 909 {
 910         struct dyn_ftrace *rec;
 911
 912         /* First check for freed records */
 913         if (ftrace_free_records) {
 914                 rec = ftrace_free_records;
 915
 916                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 917                         FTRACE_WARN_ON_ONCE(1);
 918                         ftrace_free_records = NULL;
 919                         return NULL;
 920                 }
 921
 922                 ftrace_free_records = rec->freelist;
 923                 memset(rec, 0, sizeof(*rec));
 924                 return rec;
 925         }
 926
 927         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 928                 if (!ftrace_pages->next) {
 929                         /* allocate another page */
 930                         ftrace_pages->next =
 931                                 (void *)get_zeroed_page(GFP_KERNEL);
 932                         if (!ftrace_pages->next)
 933                                 return NULL;
 934                 }
 935                 ftrace_pages = ftrace_pages->next;
 936         }
 937
 938         return &ftrace_pages->records[ftrace_pages->index++];
 939 }
 940
 941 static struct dyn_ftrace *
 942 ftrace_record_ip(unsigned long ip)
 943 {
 944         struct dyn_ftrace *rec;
 945
 946         if (ftrace_disabled)
 947                 return NULL;
 948
 949         rec = ftrace_alloc_dyn_node(ip);
 950         if (!rec)
 951                 return NULL;
 952
 953         rec->ip = ip;
 954         rec->newlist = ftrace_new_addrs;
 955         ftrace_new_addrs = rec;
 956
 957         return rec;
 958 }
 959
 960 static void print_ip_ins(const char *fmt, unsigned char *p)
 961 {
 962         int i;
 963
 964         printk(KERN_CONT "%s", fmt);
 965
 966         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 967                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 968 }
 969
 970 static void ftrace_bug(int failed, unsigned long ip)
 971 {
 972         switch (failed) {
 973         case -EFAULT:
 974                 FTRACE_WARN_ON_ONCE(1);
 975                 pr_info("ftrace faulted on modifying ");
 976                 print_ip_sym(ip);
 977                 break;
 978         case -EINVAL:
 979                 FTRACE_WARN_ON_ONCE(1);
 980                 pr_info("ftrace failed to modify ");
 981                 print_ip_sym(ip);
 982                 print_ip_ins(" actual: ", (unsigned char *)ip);
 983                 printk(KERN_CONT "\n");
 984                 break;
 985         case -EPERM:
 986                 FTRACE_WARN_ON_ONCE(1);
 987                 pr_info("ftrace faulted on writing ");
 988                 print_ip_sym(ip);
 989                 break;
 990         default:
 991                 FTRACE_WARN_ON_ONCE(1);
 992                 pr_info("ftrace faulted on unknown error ");
 993                 print_ip_sym(ip);
 994         }
 995 }
 996
 997
 998 /* Return 1 if the address range is reserved for ftrace */
 999 int ftrace_text_reserved(void *start, void *end)
1000 {
1001         struct dyn_ftrace *rec;
1002         struct ftrace_page *pg;
1003
1004         do_for_each_ftrace_rec(pg, rec) {
1005                 if (rec->ip <= (unsigned long)end &&
1006                     rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1007                         return 1;
1008         } while_for_each_ftrace_rec();
1009         return 0;
1010 }
1011
1012
1013 static int
1014 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1015 {
1016         unsigned long ftrace_addr;
1017         unsigned long flag = 0UL;
1018
1019         ftrace_addr = (unsigned long)FTRACE_ADDR;
1020
1021         /*
1022          * If this record is not to be traced or we want to disable it,
1023          * then disable it.
1024          *
1025          * If we want to enable it and filtering is off, then enable it.
1026          *
1027          * If we want to enable it and filtering is on, enable it only if
1028          * it's filtered
1029          */
1030         if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1031                 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1032                         flag = FTRACE_FL_ENABLED;
1033         }
1034
1035         /* If the state of this record hasn't changed, then do nothing */
1036         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1037                 return 0;
1038
1039         if (flag) {
1040                 rec->flags |= FTRACE_FL_ENABLED;
1041                 return ftrace_make_call(rec, ftrace_addr);
1042         }
1043
1044         rec->flags &= ~FTRACE_FL_ENABLED;
1045         return ftrace_make_nop(NULL, rec, ftrace_addr);
1046 }
1047
1048 static void ftrace_replace_code(int enable)
1049 {
1050         struct dyn_ftrace *rec;
1051         struct ftrace_page *pg;
1052         int failed;
1053
1054         do_for_each_ftrace_rec(pg, rec) {
1055                 /*
1056                  * Skip over free records, records that have
1057                  * failed and not converted.
1058                  */
1059                 if (rec->flags & FTRACE_FL_FREE ||
1060                     rec->flags & FTRACE_FL_FAILED ||
1061                     !(rec->flags & FTRACE_FL_CONVERTED))
1062                         continue;
1063
1064                 failed = __ftrace_replace_code(rec, enable);
1065                 if (failed) {
1066                         rec->flags |= FTRACE_FL_FAILED;
1067                         ftrace_bug(failed, rec->ip);
1068                         /* Stop processing */
1069                         return;
1070                 }
1071         } while_for_each_ftrace_rec();
1072 }
1073
1074 static int
1075 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1076 {
1077         unsigned long ip;
1078         int ret;
1079
1080         ip = rec->ip;
1081
1082         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1083         if (ret) {
1084                 ftrace_bug(ret, ip);
1085                 rec->flags |= FTRACE_FL_FAILED;
1086                 return 0;
1087         }
1088         return 1;
1089 }
1090
1091 /*
1092  * archs can override this function if they must do something
1093  * before the modifying code is performed.
1094  */
1095 int __weak ftrace_arch_code_modify_prepare(void)
1096 {
1097         return 0;
1098 }
1099
1100 /*
1101  * archs can override this function if they must do something
1102  * after the modifying code is performed.
1103  */
1104 int __weak ftrace_arch_code_modify_post_process(void)
1105 {
1106         return 0;
1107 }
1108
1109 static int __ftrace_modify_code(void *data)
1110 {
1111         int *command = data;
1112
1113         if (*command & FTRACE_ENABLE_CALLS)
1114                 ftrace_replace_code(1);
1115         else if (*command & FTRACE_DISABLE_CALLS)
1116                 ftrace_replace_code(0);
1117
1118         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1119                 ftrace_update_ftrace_func(ftrace_trace_function);
1120
1121         if (*command & FTRACE_START_FUNC_RET)
1122                 ftrace_enable_ftrace_graph_caller();
1123         else if (*command & FTRACE_STOP_FUNC_RET)
1124                 ftrace_disable_ftrace_graph_caller();
1125
1126         return 0;
1127 }
1128
1129 static void ftrace_run_update_code(int command)
1130 {
1131         int ret;
1132
1133         ret = ftrace_arch_code_modify_prepare();
1134         FTRACE_WARN_ON(ret);
1135         if (ret)
1136                 return;
1137
1138         stop_machine(__ftrace_modify_code, &command, NULL);
1139
1140         ret = ftrace_arch_code_modify_post_process();
1141         FTRACE_WARN_ON(ret);
1142 }
1143
1144 static ftrace_func_t saved_ftrace_func;
1145 static int ftrace_start_up;
1146
1147 static void ftrace_startup_enable(int command)
1148 {
1149         if (saved_ftrace_func != ftrace_trace_function) {
1150                 saved_ftrace_func = ftrace_trace_function;
1151                 command |= FTRACE_UPDATE_TRACE_FUNC;
1152         }
1153
1154         if (!command || !ftrace_enabled)
1155                 return;
1156
1157         ftrace_run_update_code(command);
1158 }
1159
1160 static void ftrace_startup(int command)
1161 {
1162         if (unlikely(ftrace_disabled))
1163                 return;
1164
1165         ftrace_start_up++;
1166         command |= FTRACE_ENABLE_CALLS;
1167
1168         ftrace_startup_enable(command);
1169 }
1170
1171 static void ftrace_shutdown(int command)
1172 {
1173         if (unlikely(ftrace_disabled))
1174                 return;
1175
1176         ftrace_start_up--;
1177         /*
1178          * Just warn in case of unbalance, no need to kill ftrace, it's not
1179          * critical but the ftrace_call callers may be never nopped again after
1180          * further ftrace uses.
1181          */
1182         WARN_ON_ONCE(ftrace_start_up < 0);
1183
1184         if (!ftrace_start_up)
1185                 command |= FTRACE_DISABLE_CALLS;
1186
1187         if (saved_ftrace_func != ftrace_trace_function) {
1188                 saved_ftrace_func = ftrace_trace_function;
1189                 command |= FTRACE_UPDATE_TRACE_FUNC;
1190         }
1191
1192         if (!command || !ftrace_enabled)
1193                 return;
1194
1195         ftrace_run_update_code(command);
1196 }
1197
1198 static void ftrace_startup_sysctl(void)
1199 {
1200         int command = FTRACE_ENABLE_MCOUNT;
1201
1202         if (unlikely(ftrace_disabled))
1203                 return;
1204
1205         /* Force update next time */
1206         saved_ftrace_func = NULL;
1207         /* ftrace_start_up is true if we want ftrace running */
1208         if (ftrace_start_up)
1209                 command |= FTRACE_ENABLE_CALLS;
1210
1211         ftrace_run_update_code(command);
1212 }
1213
1214 static void ftrace_shutdown_sysctl(void)
1215 {
1216         int command = FTRACE_DISABLE_MCOUNT;
1217
1218         if (unlikely(ftrace_disabled))
1219                 return;
1220
1221         /* ftrace_start_up is true if ftrace is running */
1222         if (ftrace_start_up)
1223                 command |= FTRACE_DISABLE_CALLS;
1224
1225         ftrace_run_update_code(command);
1226 }
1227
1228 static cycle_t          ftrace_update_time;
1229 static unsigned long    ftrace_update_cnt;
1230 unsigned long           ftrace_update_tot_cnt;
1231
1232 static int ftrace_update_code(struct module *mod)
1233 {
1234         struct dyn_ftrace *p;
1235         cycle_t start, stop;
1236
1237         start = ftrace_now(raw_smp_processor_id());
1238         ftrace_update_cnt = 0;
1239
1240         while (ftrace_new_addrs) {
1241
1242                 /* If something went wrong, bail without enabling anything */
1243                 if (unlikely(ftrace_disabled))
1244                         return -1;
1245
1246                 p = ftrace_new_addrs;
1247                 ftrace_new_addrs = p->newlist;
1248                 p->flags = 0L;
1249
1250                 /*
1251                  * Do the initial record convertion from mcount jump
1252                  * to the NOP instructions.
1253                  */
1254                 if (!ftrace_code_disable(mod, p)) {
1255                         ftrace_free_rec(p);
1256                         continue;
1257                 }
1258
1259                 p->flags |= FTRACE_FL_CONVERTED;
1260                 ftrace_update_cnt++;
1261
1262                 /*
1263                  * If the tracing is enabled, go ahead and enable the record.
1264                  *
1265                  * The reason not to enable the record immediatelly is the
1266                  * inherent check of ftrace_make_nop/ftrace_make_call for
1267                  * correct previous instructions.  Making first the NOP
1268                  * conversion puts the module to the correct state, thus
1269                  * passing the ftrace_make_call check.
1270                  */
1271                 if (ftrace_start_up) {
1272                         int failed = __ftrace_replace_code(p, 1);
1273                         if (failed) {
1274                                 ftrace_bug(failed, p->ip);
1275                                 ftrace_free_rec(p);
1276                         }
1277                 }
1278         }
1279
1280         stop = ftrace_now(raw_smp_processor_id());
1281         ftrace_update_time = stop - start;
1282         ftrace_update_tot_cnt += ftrace_update_cnt;
1283
1284         return 0;
1285 }
1286
1287 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1288 {
1289         struct ftrace_page *pg;
1290         int cnt;
1291         int i;
1292
1293         /* allocate a few pages */
1294         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1295         if (!ftrace_pages_start)
1296                 return -1;
1297
1298         /*
1299          * Allocate a few more pages.
1300          *
1301          * TODO: have some parser search vmlinux before
1302          *   final linking to find all calls to ftrace.
1303          *   Then we can:
1304          *    a) know how many pages to allocate.
1305          *     and/or
1306          *    b) set up the table then.
1307          *
1308          *  The dynamic code is still necessary for
1309          *  modules.
1310          */
1311
1312         pg = ftrace_pages = ftrace_pages_start;
1313
1314         cnt = num_to_init / ENTRIES_PER_PAGE;
1315         pr_info("ftrace: allocating %ld entries in %d pages\n",
1316                 num_to_init, cnt + 1);
1317
1318         for (i = 0; i < cnt; i++) {
1319                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1320
1321                 /* If we fail, we'll try later anyway */
1322                 if (!pg->next)
1323                         break;
1324
1325                 pg = pg->next;
1326         }
1327
1328         return 0;
1329 }
1330
1331 enum {
1332         FTRACE_ITER_FILTER      = (1 << 0),
1333         FTRACE_ITER_NOTRACE     = (1 << 1),
1334         FTRACE_ITER_FAILURES    = (1 << 2),
1335         FTRACE_ITER_PRINTALL    = (1 << 3),
1336         FTRACE_ITER_HASH        = (1 << 4),
1337 };
1338
1339 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1340
1341 struct ftrace_iterator {
1342         struct ftrace_page      *pg;
1343         int                     hidx;
1344         int                     idx;
1345         unsigned                flags;
1346         struct trace_parser     parser;
1347 };
1348
1349 static void *
1350 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1351 {
1352         struct ftrace_iterator *iter = m->private;
1353         struct hlist_node *hnd = v;
1354         struct hlist_head *hhd;
1355
1356         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1357
1358         (*pos)++;
1359
1360  retry:
1361         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1362                 return NULL;
1363
1364         hhd = &ftrace_func_hash[iter->hidx];
1365
1366         if (hlist_empty(hhd)) {
1367                 iter->hidx++;
1368                 hnd = NULL;
1369                 goto retry;
1370         }
1371
1372         if (!hnd)
1373                 hnd = hhd->first;
1374         else {
1375                 hnd = hnd->next;
1376                 if (!hnd) {
1377                         iter->hidx++;
1378                         goto retry;
1379                 }
1380         }
1381
1382         return hnd;
1383 }
1384
1385 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1386 {
1387         struct ftrace_iterator *iter = m->private;
1388         void *p = NULL;
1389         loff_t l;
1390
1391         if (!(iter->flags & FTRACE_ITER_HASH))
1392                 *pos = 0;
1393
1394         iter->flags |= FTRACE_ITER_HASH;
1395
1396         iter->hidx = 0;
1397         for (l = 0; l <= *pos; ) {
1398                 p = t_hash_next(m, p, &l);
1399                 if (!p)
1400                         break;
1401         }
1402         return p;
1403 }
1404
1405 static int t_hash_show(struct seq_file *m, void *v)
1406 {
1407         struct ftrace_func_probe *rec;
1408         struct hlist_node *hnd = v;
1409
1410         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1411
1412         if (rec->ops->print)
1413                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1414
1415         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1416
1417         if (rec->data)
1418                 seq_printf(m, ":%p", rec->data);
1419         seq_putc(m, '\n');
1420
1421         return 0;
1422 }
1423
1424 static void *
1425 t_next(struct seq_file *m, void *v, loff_t *pos)
1426 {
1427         struct ftrace_iterator *iter = m->private;
1428         struct dyn_ftrace *rec = NULL;
1429
1430         if (iter->flags & FTRACE_ITER_HASH)
1431                 return t_hash_next(m, v, pos);
1432
1433         (*pos)++;
1434
1435         if (iter->flags & FTRACE_ITER_PRINTALL)
1436                 return NULL;
1437
1438  retry:
1439         if (iter->idx >= iter->pg->index) {
1440                 if (iter->pg->next) {
1441                         iter->pg = iter->pg->next;
1442                         iter->idx = 0;
1443                         goto retry;
1444                 }
1445         } else {
1446                 rec = &iter->pg->records[iter->idx++];
1447                 if ((rec->flags & FTRACE_FL_FREE) ||
1448
1449                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1450                      (rec->flags & FTRACE_FL_FAILED)) ||
1451
1452                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1453                      !(rec->flags & FTRACE_FL_FAILED)) ||
1454
1455                     ((iter->flags & FTRACE_ITER_FILTER) &&
1456                      !(rec->flags & FTRACE_FL_FILTER)) ||
1457
1458                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1459                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1460                         rec = NULL;
1461                         goto retry;
1462                 }
1463         }
1464
1465         return rec;
1466 }
1467
1468 static void *t_start(struct seq_file *m, loff_t *pos)
1469 {
1470         struct ftrace_iterator *iter = m->private;
1471         void *p = NULL;
1472         loff_t l;
1473
1474         mutex_lock(&ftrace_lock);
1475         /*
1476          * For set_ftrace_filter reading, if we have the filter
1477          * off, we can short cut and just print out that all
1478          * functions are enabled.
1479          */
1480         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1481                 if (*pos > 0)
1482                         return t_hash_start(m, pos);
1483                 iter->flags |= FTRACE_ITER_PRINTALL;
1484                 return iter;
1485         }
1486
1487         if (iter->flags & FTRACE_ITER_HASH)
1488                 return t_hash_start(m, pos);
1489
1490         iter->pg = ftrace_pages_start;
1491         iter->idx = 0;
1492         for (l = 0; l <= *pos; ) {
1493                 p = t_next(m, p, &l);
1494                 if (!p)
1495                         break;
1496         }
1497
1498         if (!p && iter->flags & FTRACE_ITER_FILTER)
1499                 return t_hash_start(m, pos);
1500
1501         return p;
1502 }
1503
1504 static void t_stop(struct seq_file *m, void *p)
1505 {
1506         mutex_unlock(&ftrace_lock);
1507 }
1508
1509 static int t_show(struct seq_file *m, void *v)
1510 {
1511         struct ftrace_iterator *iter = m->private;
1512         struct dyn_ftrace *rec = v;
1513
1514         if (iter->flags & FTRACE_ITER_HASH)
1515                 return t_hash_show(m, v);
1516
1517         if (iter->flags & FTRACE_ITER_PRINTALL) {
1518                 seq_printf(m, "#### all functions enabled ####\n");
1519                 return 0;
1520         }
1521
1522         if (!rec)
1523                 return 0;
1524
1525         seq_printf(m, "%ps\n", (void *)rec->ip);
1526
1527         return 0;
1528 }
1529
1530 static const struct seq_operations show_ftrace_seq_ops = {
1531         .start = t_start,
1532         .next = t_next,
1533         .stop = t_stop,
1534         .show = t_show,
1535 };
1536
1537 static int
1538 ftrace_avail_open(struct inode *inode, struct file *file)
1539 {
1540         struct ftrace_iterator *iter;
1541         int ret;
1542
1543         if (unlikely(ftrace_disabled))
1544                 return -ENODEV;
1545
1546         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1547         if (!iter)
1548                 return -ENOMEM;
1549
1550         iter->pg = ftrace_pages_start;
1551
1552         ret = seq_open(file, &show_ftrace_seq_ops);
1553         if (!ret) {
1554                 struct seq_file *m = file->private_data;
1555
1556                 m->private = iter;
1557         } else {
1558                 kfree(iter);
1559         }
1560
1561         return ret;
1562 }
1563
1564 static int
1565 ftrace_failures_open(struct inode *inode, struct file *file)
1566 {
1567         int ret;
1568         struct seq_file *m;
1569         struct ftrace_iterator *iter;
1570
1571         ret = ftrace_avail_open(inode, file);
1572         if (!ret) {
1573                 m = (struct seq_file *)file->private_data;
1574                 iter = (struct ftrace_iterator *)m->private;
1575                 iter->flags = FTRACE_ITER_FAILURES;
1576         }
1577
1578         return ret;
1579 }
1580
1581
1582 static void ftrace_filter_reset(int enable)
1583 {
1584         struct ftrace_page *pg;
1585         struct dyn_ftrace *rec;
1586         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1587
1588         mutex_lock(&ftrace_lock);
1589         if (enable)
1590                 ftrace_filtered = 0;
1591         do_for_each_ftrace_rec(pg, rec) {
1592                 if (rec->flags & FTRACE_FL_FAILED)
1593                         continue;
1594                 rec->flags &= ~type;
1595         } while_for_each_ftrace_rec();
1596         mutex_unlock(&ftrace_lock);
1597 }
1598
1599 static int
1600 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1601 {
1602         struct ftrace_iterator *iter;
1603         int ret = 0;
1604
1605         if (unlikely(ftrace_disabled))
1606                 return -ENODEV;
1607
1608         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1609         if (!iter)
1610                 return -ENOMEM;
1611
1612         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1613                 kfree(iter);
1614                 return -ENOMEM;
1615         }
1616
1617         mutex_lock(&ftrace_regex_lock);
1618         if ((file->f_mode & FMODE_WRITE) &&
1619             (file->f_flags & O_TRUNC))
1620                 ftrace_filter_reset(enable);
1621
1622         if (file->f_mode & FMODE_READ) {
1623                 iter->pg = ftrace_pages_start;
1624                 iter->flags = enable ? FTRACE_ITER_FILTER :
1625                         FTRACE_ITER_NOTRACE;
1626
1627                 ret = seq_open(file, &show_ftrace_seq_ops);
1628                 if (!ret) {
1629                         struct seq_file *m = file->private_data;
1630                         m->private = iter;
1631                 } else {
1632                         trace_parser_put(&iter->parser);
1633                         kfree(iter);
1634                 }
1635         } else
1636                 file->private_data = iter;
1637         mutex_unlock(&ftrace_regex_lock);
1638
1639         return ret;
1640 }
1641
1642 static int
1643 ftrace_filter_open(struct inode *inode, struct file *file)
1644 {
1645         return ftrace_regex_open(inode, file, 1);
1646 }
1647
1648 static int
1649 ftrace_notrace_open(struct inode *inode, struct file *file)
1650 {
1651         return ftrace_regex_open(inode, file, 0);
1652 }
1653
1654 static loff_t
1655 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1656 {
1657         loff_t ret;
1658
1659         if (file->f_mode & FMODE_READ)
1660                 ret = seq_lseek(file, offset, origin);
1661         else
1662                 file->f_pos = ret = 1;
1663
1664         return ret;
1665 }
1666
1667 static int ftrace_match(char *str, char *regex, int len, int type)
1668 {
1669         int matched = 0;
1670         int slen;
1671
1672         switch (type) {
1673         case MATCH_FULL:
1674                 if (strcmp(str, regex) == 0)
1675                         matched = 1;
1676                 break;
1677         case MATCH_FRONT_ONLY:
1678                 if (strncmp(str, regex, len) == 0)
1679                         matched = 1;
1680                 break;
1681         case MATCH_MIDDLE_ONLY:
1682                 if (strstr(str, regex))
1683                         matched = 1;
1684                 break;
1685         case MATCH_END_ONLY:
1686                 slen = strlen(str);
1687                 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
1688                         matched = 1;
1689                 break;
1690         }
1691
1692         return matched;
1693 }
1694
1695 static int
1696 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1697 {
1698         char str[KSYM_SYMBOL_LEN];
1699
1700         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1701         return ftrace_match(str, regex, len, type);
1702 }
1703
1704 static int ftrace_match_records(char *buff, int len, int enable)
1705 {
1706         unsigned int search_len;
1707         struct ftrace_page *pg;
1708         struct dyn_ftrace *rec;
1709         unsigned long flag;
1710         char *search;
1711         int type;
1712         int not;
1713         int found = 0;
1714
1715         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1716         type = filter_parse_regex(buff, len, &search, &not);
1717
1718         search_len = strlen(search);
1719
1720         mutex_lock(&ftrace_lock);
1721         do_for_each_ftrace_rec(pg, rec) {
1722
1723                 if (rec->flags & FTRACE_FL_FAILED)
1724                         continue;
1725
1726                 if (ftrace_match_record(rec, search, search_len, type)) {
1727                         if (not)
1728                                 rec->flags &= ~flag;
1729                         else
1730                                 rec->flags |= flag;
1731                         found = 1;
1732                 }
1733                 /*
1734                  * Only enable filtering if we have a function that
1735                  * is filtered on.
1736                  */
1737                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1738                         ftrace_filtered = 1;
1739         } while_for_each_ftrace_rec();
1740         mutex_unlock(&ftrace_lock);
1741
1742         return found;
1743 }
1744
1745 static int
1746 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1747                            char *regex, int len, int type)
1748 {
1749         char str[KSYM_SYMBOL_LEN];
1750         char *modname;
1751
1752         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1753
1754         if (!modname || strcmp(modname, mod))
1755                 return 0;
1756
1757         /* blank search means to match all funcs in the mod */
1758         if (len)
1759                 return ftrace_match(str, regex, len, type);
1760         else
1761                 return 1;
1762 }
1763
1764 static int ftrace_match_module_records(char *buff, char *mod, int enable)
1765 {
1766         unsigned search_len = 0;
1767         struct ftrace_page *pg;
1768         struct dyn_ftrace *rec;
1769         int type = MATCH_FULL;
1770         char *search = buff;
1771         unsigned long flag;
1772         int not = 0;
1773         int found = 0;
1774
1775         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1776
1777         /* blank or '*' mean the same */
1778         if (strcmp(buff, "*") == 0)
1779                 buff[0] = 0;
1780
1781         /* handle the case of 'dont filter this module' */
1782         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1783                 buff[0] = 0;
1784                 not = 1;
1785         }
1786
1787         if (strlen(buff)) {
1788                 type = filter_parse_regex(buff, strlen(buff), &search, &not);
1789                 search_len = strlen(search);
1790         }
1791
1792         mutex_lock(&ftrace_lock);
1793         do_for_each_ftrace_rec(pg, rec) {
1794
1795                 if (rec->flags & FTRACE_FL_FAILED)
1796                         continue;
1797
1798                 if (ftrace_match_module_record(rec, mod,
1799                                                search, search_len, type)) {
1800                         if (not)
1801                                 rec->flags &= ~flag;
1802                         else
1803                                 rec->flags |= flag;
1804                         found = 1;
1805                 }
1806                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1807                         ftrace_filtered = 1;
1808
1809         } while_for_each_ftrace_rec();
1810         mutex_unlock(&ftrace_lock);
1811
1812         return found;
1813 }
1814
1815 /*
1816  * We register the module command as a template to show others how
1817  * to register the a command as well.
1818  */
1819
1820 static int
1821 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1822 {
1823         char *mod;
1824
1825         /*
1826          * cmd == 'mod' because we only registered this func
1827          * for the 'mod' ftrace_func_command.
1828          * But if you register one func with multiple commands,
1829          * you can tell which command was used by the cmd
1830          * parameter.
1831          */
1832
1833         /* we must have a module name */
1834         if (!param)
1835                 return -EINVAL;
1836
1837         mod = strsep(&param, ":");
1838         if (!strlen(mod))
1839                 return -EINVAL;
1840
1841         if (ftrace_match_module_records(func, mod, enable))
1842                 return 0;
1843         return -EINVAL;
1844 }
1845
1846 static struct ftrace_func_command ftrace_mod_cmd = {
1847         .name                   = "mod",
1848         .func                   = ftrace_mod_callback,
1849 };
1850
1851 static int __init ftrace_mod_cmd_init(void)
1852 {
1853         return register_ftrace_command(&ftrace_mod_cmd);
1854 }
1855 device_initcall(ftrace_mod_cmd_init);
1856
1857 static void
1858 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1859 {
1860         struct ftrace_func_probe *entry;
1861         struct hlist_head *hhd;
1862         struct hlist_node *n;
1863         unsigned long key;
1864         int resched;
1865
1866         key = hash_long(ip, FTRACE_HASH_BITS);
1867
1868         hhd = &ftrace_func_hash[key];
1869
1870         if (hlist_empty(hhd))
1871                 return;
1872
1873         /*
1874          * Disable preemption for these calls to prevent a RCU grace
1875          * period. This syncs the hash iteration and freeing of items
1876          * on the hash. rcu_read_lock is too dangerous here.
1877          */
1878         resched = ftrace_preempt_disable();
1879         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1880                 if (entry->ip == ip)
1881                         entry->ops->func(ip, parent_ip, &entry->data);
1882         }
1883         ftrace_preempt_enable(resched);
1884 }
1885
1886 static struct ftrace_ops trace_probe_ops __read_mostly =
1887 {
1888         .func           = function_trace_probe_call,
1889 };
1890
1891 static int ftrace_probe_registered;
1892
1893 static void __enable_ftrace_function_probe(void)
1894 {
1895         int i;
1896
1897         if (ftrace_probe_registered)
1898                 return;
1899
1900         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1901                 struct hlist_head *hhd = &ftrace_func_hash[i];
1902                 if (hhd->first)
1903                         break;
1904         }
1905         /* Nothing registered? */
1906         if (i == FTRACE_FUNC_HASHSIZE)
1907                 return;
1908
1909         __register_ftrace_function(&trace_probe_ops);
1910         ftrace_startup(0);
1911         ftrace_probe_registered = 1;
1912 }
1913
1914 static void __disable_ftrace_function_probe(void)
1915 {
1916         int i;
1917
1918         if (!ftrace_probe_registered)
1919                 return;
1920
1921         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1922                 struct hlist_head *hhd = &ftrace_func_hash[i];
1923                 if (hhd->first)
1924                         return;
1925         }
1926
1927         /* no more funcs left */
1928         __unregister_ftrace_function(&trace_probe_ops);
1929         ftrace_shutdown(0);
1930         ftrace_probe_registered = 0;
1931 }
1932
1933
1934 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1935 {
1936         struct ftrace_func_probe *entry =
1937                 container_of(rhp, struct ftrace_func_probe, rcu);
1938
1939         if (entry->ops->free)
1940                 entry->ops->free(&entry->data);
1941         kfree(entry);
1942 }
1943
1944
1945 int
1946 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1947                               void *data)
1948 {
1949         struct ftrace_func_probe *entry;
1950         struct ftrace_page *pg;
1951         struct dyn_ftrace *rec;
1952         int type, len, not;
1953         unsigned long key;
1954         int count = 0;
1955         char *search;
1956
1957         type = filter_parse_regex(glob, strlen(glob), &search, &not);
1958         len = strlen(search);
1959
1960         /* we do not support '!' for function probes */
1961         if (WARN_ON(not))
1962                 return -EINVAL;
1963
1964         mutex_lock(&ftrace_lock);
1965         do_for_each_ftrace_rec(pg, rec) {
1966
1967                 if (rec->flags & FTRACE_FL_FAILED)
1968                         continue;
1969
1970                 if (!ftrace_match_record(rec, search, len, type))
1971                         continue;
1972
1973                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1974                 if (!entry) {
1975                         /* If we did not process any, then return error */
1976                         if (!count)
1977                                 count = -ENOMEM;
1978                         goto out_unlock;
1979                 }
1980
1981                 count++;
1982
1983                 entry->data = data;
1984
1985                 /*
1986                  * The caller might want to do something special
1987                  * for each function we find. We call the callback
1988                  * to give the caller an opportunity to do so.
1989                  */
1990                 if (ops->callback) {
1991                         if (ops->callback(rec->ip, &entry->data) < 0) {
1992                                 /* caller does not like this func */
1993                                 kfree(entry);
1994                                 continue;
1995                         }
1996                 }
1997
1998                 entry->ops = ops;
1999                 entry->ip = rec->ip;
2000
2001                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2002                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2003
2004         } while_for_each_ftrace_rec();
2005         __enable_ftrace_function_probe();
2006
2007  out_unlock:
2008         mutex_unlock(&ftrace_lock);
2009
2010         return count;
2011 }
2012
2013 enum {
2014         PROBE_TEST_FUNC         = 1,
2015         PROBE_TEST_DATA         = 2
2016 };
2017
2018 static void
2019 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2020                                   void *data, int flags)
2021 {
2022         struct ftrace_func_probe *entry;
2023         struct hlist_node *n, *tmp;
2024         char str[KSYM_SYMBOL_LEN];
2025         int type = MATCH_FULL;
2026         int i, len = 0;
2027         char *search;
2028
2029         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2030                 glob = NULL;
2031         else if (glob) {
2032                 int not;
2033
2034                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2035                 len = strlen(search);
2036
2037                 /* we do not support '!' for function probes */
2038                 if (WARN_ON(not))
2039                         return;
2040         }
2041
2042         mutex_lock(&ftrace_lock);
2043         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2044                 struct hlist_head *hhd = &ftrace_func_hash[i];
2045
2046                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2047
2048                         /* break up if statements for readability */
2049                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2050                                 continue;
2051
2052                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2053                                 continue;
2054
2055                         /* do this last, since it is the most expensive */
2056                         if (glob) {
2057                                 kallsyms_lookup(entry->ip, NULL, NULL,
2058                                                 NULL, str);
2059                                 if (!ftrace_match(str, glob, len, type))
2060                                         continue;
2061                         }
2062
2063                         hlist_del(&entry->node);
2064                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2065                 }
2066         }
2067         __disable_ftrace_function_probe();
2068         mutex_unlock(&ftrace_lock);
2069 }
2070
2071 void
2072 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2073                                 void *data)
2074 {
2075         __unregister_ftrace_function_probe(glob, ops, data,
2076                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2077 }
2078
2079 void
2080 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2081 {
2082         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2083 }
2084
2085 void unregister_ftrace_function_probe_all(char *glob)
2086 {
2087         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2088 }
2089
2090 static LIST_HEAD(ftrace_commands);
2091 static DEFINE_MUTEX(ftrace_cmd_mutex);
2092
2093 int register_ftrace_command(struct ftrace_func_command *cmd)
2094 {
2095         struct ftrace_func_command *p;
2096         int ret = 0;
2097
2098         mutex_lock(&ftrace_cmd_mutex);
2099         list_for_each_entry(p, &ftrace_commands, list) {
2100                 if (strcmp(cmd->name, p->name) == 0) {
2101                         ret = -EBUSY;
2102                         goto out_unlock;
2103                 }
2104         }
2105         list_add(&cmd->list, &ftrace_commands);
2106  out_unlock:
2107         mutex_unlock(&ftrace_cmd_mutex);
2108
2109         return ret;
2110 }
2111
2112 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2113 {
2114         struct ftrace_func_command *p, *n;
2115         int ret = -ENODEV;
2116
2117         mutex_lock(&ftrace_cmd_mutex);
2118         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2119                 if (strcmp(cmd->name, p->name) == 0) {
2120                         ret = 0;
2121                         list_del_init(&p->list);
2122                         goto out_unlock;
2123                 }
2124         }
2125  out_unlock:
2126         mutex_unlock(&ftrace_cmd_mutex);
2127
2128         return ret;
2129 }
2130
2131 static int ftrace_process_regex(char *buff, int len, int enable)
2132 {
2133         char *func, *command, *next = buff;
2134         struct ftrace_func_command *p;
2135         int ret = -EINVAL;
2136
2137         func = strsep(&next, ":");
2138
2139         if (!next) {
2140                 if (ftrace_match_records(func, len, enable))
2141                         return 0;
2142                 return ret;
2143         }
2144
2145         /* command found */
2146
2147         command = strsep(&next, ":");
2148
2149         mutex_lock(&ftrace_cmd_mutex);
2150         list_for_each_entry(p, &ftrace_commands, list) {
2151                 if (strcmp(p->name, command) == 0) {
2152                         ret = p->func(func, command, next, enable);
2153                         goto out_unlock;
2154                 }
2155         }
2156  out_unlock:
2157         mutex_unlock(&ftrace_cmd_mutex);
2158
2159         return ret;
2160 }
2161
2162 static ssize_t
2163 ftrace_regex_write(struct file *file, const char __user *ubuf,
2164                    size_t cnt, loff_t *ppos, int enable)
2165 {
2166         struct ftrace_iterator *iter;
2167         struct trace_parser *parser;
2168         ssize_t ret, read;
2169
2170         if (!cnt)
2171                 return 0;
2172
2173         mutex_lock(&ftrace_regex_lock);
2174
2175         if (file->f_mode & FMODE_READ) {
2176                 struct seq_file *m = file->private_data;
2177                 iter = m->private;
2178         } else
2179                 iter = file->private_data;
2180
2181         parser = &iter->parser;
2182         read = trace_get_user(parser, ubuf, cnt, ppos);
2183
2184         if (read >= 0 && trace_parser_loaded(parser) &&
2185             !trace_parser_cont(parser)) {
2186                 ret = ftrace_process_regex(parser->buffer,
2187                                            parser->idx, enable);
2188                 trace_parser_clear(parser);
2189                 if (ret)
2190                         goto out_unlock;
2191         }
2192
2193         ret = read;
2194 out_unlock:
2195         mutex_unlock(&ftrace_regex_lock);
2196
2197         return ret;
2198 }
2199
2200 static ssize_t
2201 ftrace_filter_write(struct file *file, const char __user *ubuf,
2202                     size_t cnt, loff_t *ppos)
2203 {
2204         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2205 }
2206
2207 static ssize_t
2208 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2209                      size_t cnt, loff_t *ppos)
2210 {
2211         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2212 }
2213
2214 static void
2215 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2216 {
2217         if (unlikely(ftrace_disabled))
2218                 return;
2219
2220         mutex_lock(&ftrace_regex_lock);
2221         if (reset)
2222                 ftrace_filter_reset(enable);
2223         if (buf)
2224                 ftrace_match_records(buf, len, enable);
2225         mutex_unlock(&ftrace_regex_lock);
2226 }
2227
2228 /**
2229  * ftrace_set_filter - set a function to filter on in ftrace
2230  * @buf - the string that holds the function filter text.
2231  * @len - the length of the string.
2232  * @reset - non zero to reset all filters before applying this filter.
2233  *
2234  * Filters denote which functions should be enabled when tracing is enabled.
2235  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2236  */
2237 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2238 {
2239         ftrace_set_regex(buf, len, reset, 1);
2240 }
2241
2242 /**
2243  * ftrace_set_notrace - set a function to not trace in ftrace
2244  * @buf - the string that holds the function notrace text.
2245  * @len - the length of the string.
2246  * @reset - non zero to reset all filters before applying this filter.
2247  *
2248  * Notrace Filters denote which functions should not be enabled when tracing
2249  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2250  * for tracing.
2251  */
2252 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2253 {
2254         ftrace_set_regex(buf, len, reset, 0);
2255 }
2256
2257 /*
2258  * command line interface to allow users to set filters on boot up.
2259  */
2260 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
2261 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2262 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2263
2264 static int __init set_ftrace_notrace(char *str)
2265 {
2266         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2267         return 1;
2268 }
2269 __setup("ftrace_notrace=", set_ftrace_notrace);
2270
2271 static int __init set_ftrace_filter(char *str)
2272 {
2273         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2274         return 1;
2275 }
2276 __setup("ftrace_filter=", set_ftrace_filter);
2277
2278 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2279 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2280 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
2281
2282 static int __init set_graph_function(char *str)
2283 {
2284         strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2285         return 1;
2286 }
2287 __setup("ftrace_graph_filter=", set_graph_function);
2288
2289 static void __init set_ftrace_early_graph(char *buf)
2290 {
2291         int ret;
2292         char *func;
2293
2294         while (buf) {
2295                 func = strsep(&buf, ",");
2296                 /* we allow only one expression at a time */
2297                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2298                                       func);
2299                 if (ret)
2300                         printk(KERN_DEBUG "ftrace: function %s not "
2301                                           "traceable\n", func);
2302         }
2303 }
2304 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2305
2306 static void __init set_ftrace_early_filter(char *buf, int enable)
2307 {
2308         char *func;
2309
2310         while (buf) {
2311                 func = strsep(&buf, ",");
2312                 ftrace_set_regex(func, strlen(func), 0, enable);
2313         }
2314 }
2315
2316 static void __init set_ftrace_early_filters(void)
2317 {
2318         if (ftrace_filter_buf[0])
2319                 set_ftrace_early_filter(ftrace_filter_buf, 1);
2320         if (ftrace_notrace_buf[0])
2321                 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2322 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2323         if (ftrace_graph_buf[0])
2324                 set_ftrace_early_graph(ftrace_graph_buf);
2325 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2326 }
2327
2328 static int
2329 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2330 {
2331         struct seq_file *m = (struct seq_file *)file->private_data;
2332         struct ftrace_iterator *iter;
2333         struct trace_parser *parser;
2334
2335         mutex_lock(&ftrace_regex_lock);
2336         if (file->f_mode & FMODE_READ) {
2337                 iter = m->private;
2338
2339                 seq_release(inode, file);
2340         } else
2341                 iter = file->private_data;
2342
2343         parser = &iter->parser;
2344         if (trace_parser_loaded(parser)) {
2345                 parser->buffer[parser->idx] = 0;
2346                 ftrace_match_records(parser->buffer, parser->idx, enable);
2347         }
2348
2349         mutex_lock(&ftrace_lock);
2350         if (ftrace_start_up && ftrace_enabled)
2351                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2352         mutex_unlock(&ftrace_lock);
2353
2354         trace_parser_put(parser);
2355         kfree(iter);
2356
2357         mutex_unlock(&ftrace_regex_lock);
2358         return 0;
2359 }
2360
2361 static int
2362 ftrace_filter_release(struct inode *inode, struct file *file)
2363 {
2364         return ftrace_regex_release(inode, file, 1);
2365 }
2366
2367 static int
2368 ftrace_notrace_release(struct inode *inode, struct file *file)
2369 {
2370         return ftrace_regex_release(inode, file, 0);
2371 }
2372
2373 static const struct file_operations ftrace_avail_fops = {
2374         .open = ftrace_avail_open,
2375         .read = seq_read,
2376         .llseek = seq_lseek,
2377         .release = seq_release_private,
2378 };
2379
2380 static const struct file_operations ftrace_failures_fops = {
2381         .open = ftrace_failures_open,
2382         .read = seq_read,
2383         .llseek = seq_lseek,
2384         .release = seq_release_private,
2385 };
2386
2387 static const struct file_operations ftrace_filter_fops = {
2388         .open = ftrace_filter_open,
2389         .read = seq_read,
2390         .write = ftrace_filter_write,
2391         .llseek = ftrace_regex_lseek,
2392         .release = ftrace_filter_release,
2393 };
2394
2395 static const struct file_operations ftrace_notrace_fops = {
2396         .open = ftrace_notrace_open,
2397         .read = seq_read,
2398         .write = ftrace_notrace_write,
2399         .llseek = ftrace_regex_lseek,
2400         .release = ftrace_notrace_release,
2401 };
2402
2403 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2404
2405 static DEFINE_MUTEX(graph_lock);
2406
2407 int ftrace_graph_count;
2408 int ftrace_graph_filter_enabled;
2409 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2410
2411 static void *
2412 __g_next(struct seq_file *m, loff_t *pos)
2413 {
2414         if (*pos >= ftrace_graph_count)
2415                 return NULL;
2416         return &ftrace_graph_funcs[*pos];
2417 }
2418
2419 static void *
2420 g_next(struct seq_file *m, void *v, loff_t *pos)
2421 {
2422         (*pos)++;
2423         return __g_next(m, pos);
2424 }
2425
2426 static void *g_start(struct seq_file *m, loff_t *pos)
2427 {
2428         mutex_lock(&graph_lock);
2429
2430         /* Nothing, tell g_show to print all functions are enabled */
2431         if (!ftrace_graph_filter_enabled && !*pos)
2432                 return (void *)1;
2433
2434         return __g_next(m, pos);
2435 }
2436
2437 static void g_stop(struct seq_file *m, void *p)
2438 {
2439         mutex_unlock(&graph_lock);
2440 }
2441
2442 static int g_show(struct seq_file *m, void *v)
2443 {
2444         unsigned long *ptr = v;
2445
2446         if (!ptr)
2447                 return 0;
2448
2449         if (ptr == (unsigned long *)1) {
2450                 seq_printf(m, "#### all functions enabled ####\n");
2451                 return 0;
2452         }
2453
2454         seq_printf(m, "%ps\n", (void *)*ptr);
2455
2456         return 0;
2457 }
2458
2459 static const struct seq_operations ftrace_graph_seq_ops = {
2460         .start = g_start,
2461         .next = g_next,
2462         .stop = g_stop,
2463         .show = g_show,
2464 };
2465
2466 static int
2467 ftrace_graph_open(struct inode *inode, struct file *file)
2468 {
2469         int ret = 0;
2470
2471         if (unlikely(ftrace_disabled))
2472                 return -ENODEV;
2473
2474         mutex_lock(&graph_lock);
2475         if ((file->f_mode & FMODE_WRITE) &&
2476             (file->f_flags & O_TRUNC)) {
2477                 ftrace_graph_filter_enabled = 0;
2478                 ftrace_graph_count = 0;
2479                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2480         }
2481         mutex_unlock(&graph_lock);
2482
2483         if (file->f_mode & FMODE_READ)
2484                 ret = seq_open(file, &ftrace_graph_seq_ops);
2485
2486         return ret;
2487 }
2488
2489 static int
2490 ftrace_graph_release(struct inode *inode, struct file *file)
2491 {
2492         if (file->f_mode & FMODE_READ)
2493                 seq_release(inode, file);
2494         return 0;
2495 }
2496
2497 static int
2498 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2499 {
2500         struct dyn_ftrace *rec;
2501         struct ftrace_page *pg;
2502         int search_len;
2503         int fail = 1;
2504         int type, not;
2505         char *search;
2506         bool exists;
2507         int i;
2508
2509         if (ftrace_disabled)
2510                 return -ENODEV;
2511
2512         /* decode regex */
2513         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2514         if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
2515                 return -EBUSY;
2516
2517         search_len = strlen(search);
2518
2519         mutex_lock(&ftrace_lock);
2520         do_for_each_ftrace_rec(pg, rec) {
2521
2522                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2523                         continue;
2524
2525                 if (ftrace_match_record(rec, search, search_len, type)) {
2526                         /* if it is in the array */
2527                         exists = false;
2528                         for (i = 0; i < *idx; i++) {
2529                                 if (array[i] == rec->ip) {
2530                                         exists = true;
2531                                         break;
2532                                 }
2533                         }
2534
2535                         if (!not) {
2536                                 fail = 0;
2537                                 if (!exists) {
2538                                         array[(*idx)++] = rec->ip;
2539                                         if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2540                                                 goto out;
2541                                 }
2542                         } else {
2543                                 if (exists) {
2544                                         array[i] = array[--(*idx)];
2545                                         array[*idx] = 0;
2546                                         fail = 0;
2547                                 }
2548                         }
2549                 }
2550         } while_for_each_ftrace_rec();
2551 out:
2552         mutex_unlock(&ftrace_lock);
2553
2554         if (fail)
2555                 return -EINVAL;
2556
2557         ftrace_graph_filter_enabled = 1;
2558         return 0;
2559 }
2560
2561 static ssize_t
2562 ftrace_graph_write(struct file *file, const char __user *ubuf,
2563                    size_t cnt, loff_t *ppos)
2564 {
2565         struct trace_parser parser;
2566         ssize_t read, ret;
2567
2568         if (!cnt)
2569                 return 0;
2570
2571         mutex_lock(&graph_lock);
2572
2573         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2574                 ret = -ENOMEM;
2575                 goto out_unlock;
2576         }
2577
2578         read = trace_get_user(&parser, ubuf, cnt, ppos);
2579
2580         if (read >= 0 && trace_parser_loaded((&parser))) {
2581                 parser.buffer[parser.idx] = 0;
2582
2583                 /* we allow only one expression at a time */
2584                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2585                                         parser.buffer);
2586                 if (ret)
2587                         goto out_free;
2588         }
2589
2590         ret = read;
2591
2592 out_free:
2593         trace_parser_put(&parser);
2594 out_unlock:
2595         mutex_unlock(&graph_lock);
2596
2597         return ret;
2598 }
2599
2600 static const struct file_operations ftrace_graph_fops = {
2601         .open           = ftrace_graph_open,
2602         .read           = seq_read,
2603         .write          = ftrace_graph_write,
2604         .release        = ftrace_graph_release,
2605 };
2606 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2607
2608 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2609 {
2610
2611         trace_create_file("available_filter_functions", 0444,
2612                         d_tracer, NULL, &ftrace_avail_fops);
2613
2614         trace_create_file("failures", 0444,
2615                         d_tracer, NULL, &ftrace_failures_fops);
2616
2617         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2618                         NULL, &ftrace_filter_fops);
2619
2620         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2621                                     NULL, &ftrace_notrace_fops);
2622
2623 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2624         trace_create_file("set_graph_function", 0444, d_tracer,
2625                                     NULL,
2626                                     &ftrace_graph_fops);
2627 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2628
2629         return 0;
2630 }
2631
2632 static int ftrace_process_locs(struct module *mod,
2633                                unsigned long *start,
2634                                unsigned long *end)
2635 {
2636         unsigned long *p;
2637         unsigned long addr;
2638         unsigned long flags;
2639
2640         mutex_lock(&ftrace_lock);
2641         p = start;
2642         while (p < end) {
2643                 addr = ftrace_call_adjust(*p++);
2644                 /*
2645                  * Some architecture linkers will pad between
2646                  * the different mcount_loc sections of different
2647                  * object files to satisfy alignments.
2648                  * Skip any NULL pointers.
2649                  */
2650                 if (!addr)
2651                         continue;
2652                 ftrace_record_ip(addr);
2653         }
2654
2655         /* disable interrupts to prevent kstop machine */
2656         local_irq_save(flags);
2657         ftrace_update_code(mod);
2658         local_irq_restore(flags);
2659         mutex_unlock(&ftrace_lock);
2660
2661         return 0;
2662 }
2663
2664 #ifdef CONFIG_MODULES
2665 void ftrace_release_mod(struct module *mod)
2666 {
2667         struct dyn_ftrace *rec;
2668         struct ftrace_page *pg;
2669
2670         if (ftrace_disabled)
2671                 return;
2672
2673         mutex_lock(&ftrace_lock);
2674         do_for_each_ftrace_rec(pg, rec) {
2675                 if (within_module_core(rec->ip, mod)) {
2676                         /*
2677                          * rec->ip is changed in ftrace_free_rec()
2678                          * It should not between s and e if record was freed.
2679                          */
2680                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2681                         ftrace_free_rec(rec);
2682                 }
2683         } while_for_each_ftrace_rec();
2684         mutex_unlock(&ftrace_lock);
2685 }
2686
2687 static void ftrace_init_module(struct module *mod,
2688                                unsigned long *start, unsigned long *end)
2689 {
2690         if (ftrace_disabled || start == end)
2691                 return;
2692         ftrace_process_locs(mod, start, end);
2693 }
2694
2695 static int ftrace_module_notify(struct notifier_block *self,
2696                                 unsigned long val, void *data)
2697 {
2698         struct module *mod = data;
2699
2700         switch (val) {
2701         case MODULE_STATE_COMING:
2702                 ftrace_init_module(mod, mod->ftrace_callsites,
2703                                    mod->ftrace_callsites +
2704                                    mod->num_ftrace_callsites);
2705                 break;
2706         case MODULE_STATE_GOING:
2707                 ftrace_release_mod(mod);
2708                 break;
2709         }
2710
2711         return 0;
2712 }
2713 #else
2714 static int ftrace_module_notify(struct notifier_block *self,
2715                                 unsigned long val, void *data)
2716 {
2717         return 0;
2718 }
2719 #endif /* CONFIG_MODULES */
2720
2721 struct notifier_block ftrace_module_nb = {
2722         .notifier_call = ftrace_module_notify,
2723         .priority = 0,
2724 };
2725
2726 extern unsigned long __start_mcount_loc[];
2727 extern unsigned long __stop_mcount_loc[];
2728
2729 void __init ftrace_init(void)
2730 {
2731         unsigned long count, addr, flags;
2732         int ret;
2733
2734         /* Keep the ftrace pointer to the stub */
2735         addr = (unsigned long)ftrace_stub;
2736
2737         local_irq_save(flags);
2738         ftrace_dyn_arch_init(&addr);
2739         local_irq_restore(flags);
2740
2741         /* ftrace_dyn_arch_init places the return code in addr */
2742         if (addr)
2743                 goto failed;
2744
2745         count = __stop_mcount_loc - __start_mcount_loc;
2746
2747         ret = ftrace_dyn_table_alloc(count);
2748         if (ret)
2749                 goto failed;
2750
2751         last_ftrace_enabled = ftrace_enabled = 1;
2752
2753         ret = ftrace_process_locs(NULL,
2754                                   __start_mcount_loc,
2755                                   __stop_mcount_loc);
2756
2757         ret = register_module_notifier(&ftrace_module_nb);
2758         if (ret)
2759                 pr_warning("Failed to register trace ftrace module notifier\n");
2760
2761         set_ftrace_early_filters();
2762
2763         return;
2764  failed:
2765         ftrace_disabled = 1;
2766 }
2767
2768 #else
2769
2770 static int __init ftrace_nodyn_init(void)
2771 {
2772         ftrace_enabled = 1;
2773         return 0;
2774 }
2775 device_initcall(ftrace_nodyn_init);
2776
2777 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2778 static inline void ftrace_startup_enable(int command) { }
2779 /* Keep as macros so we do not need to define the commands */
2780 # define ftrace_startup(command)        do { } while (0)
2781 # define ftrace_shutdown(command)       do { } while (0)
2782 # define ftrace_startup_sysctl()        do { } while (0)
2783 # define ftrace_shutdown_sysctl()       do { } while (0)
2784 #endif /* CONFIG_DYNAMIC_FTRACE */
2785
2786 static void clear_ftrace_swapper(void)
2787 {
2788         struct task_struct *p;
2789         int cpu;
2790
2791         get_online_cpus();
2792         for_each_online_cpu(cpu) {
2793                 p = idle_task(cpu);
2794                 clear_tsk_trace_trace(p);
2795         }
2796         put_online_cpus();
2797 }
2798
2799 static void set_ftrace_swapper(void)
2800 {
2801         struct task_struct *p;
2802         int cpu;
2803
2804         get_online_cpus();
2805         for_each_online_cpu(cpu) {
2806                 p = idle_task(cpu);
2807                 set_tsk_trace_trace(p);
2808         }
2809         put_online_cpus();
2810 }
2811
2812 static void clear_ftrace_pid(struct pid *pid)
2813 {
2814         struct task_struct *p;
2815
2816         rcu_read_lock();
2817         do_each_pid_task(pid, PIDTYPE_PID, p) {
2818                 clear_tsk_trace_trace(p);
2819         } while_each_pid_task(pid, PIDTYPE_PID, p);
2820         rcu_read_unlock();
2821
2822         put_pid(pid);
2823 }
2824
2825 static void set_ftrace_pid(struct pid *pid)
2826 {
2827         struct task_struct *p;
2828
2829         rcu_read_lock();
2830         do_each_pid_task(pid, PIDTYPE_PID, p) {
2831                 set_tsk_trace_trace(p);
2832         } while_each_pid_task(pid, PIDTYPE_PID, p);
2833         rcu_read_unlock();
2834 }
2835
2836 static void clear_ftrace_pid_task(struct pid *pid)
2837 {
2838         if (pid == ftrace_swapper_pid)
2839                 clear_ftrace_swapper();
2840         else
2841                 clear_ftrace_pid(pid);
2842 }
2843
2844 static void set_ftrace_pid_task(struct pid *pid)
2845 {
2846         if (pid == ftrace_swapper_pid)
2847                 set_ftrace_swapper();
2848         else
2849                 set_ftrace_pid(pid);
2850 }
2851
2852 static int ftrace_pid_add(int p)
2853 {
2854         struct pid *pid;
2855         struct ftrace_pid *fpid;
2856         int ret = -EINVAL;
2857
2858         mutex_lock(&ftrace_lock);
2859
2860         if (!p)
2861                 pid = ftrace_swapper_pid;
2862         else
2863                 pid = find_get_pid(p);
2864
2865         if (!pid)
2866                 goto out;
2867
2868         ret = 0;
2869
2870         list_for_each_entry(fpid, &ftrace_pids, list)
2871                 if (fpid->pid == pid)
2872                         goto out_put;
2873
2874         ret = -ENOMEM;
2875
2876         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2877         if (!fpid)
2878                 goto out_put;
2879
2880         list_add(&fpid->list, &ftrace_pids);
2881         fpid->pid = pid;
2882
2883         set_ftrace_pid_task(pid);
2884
2885         ftrace_update_pid_func();
2886         ftrace_startup_enable(0);
2887
2888         mutex_unlock(&ftrace_lock);
2889         return 0;
2890
2891 out_put:
2892         if (pid != ftrace_swapper_pid)
2893                 put_pid(pid);
2894
2895 out:
2896         mutex_unlock(&ftrace_lock);
2897         return ret;
2898 }
2899
2900 static void ftrace_pid_reset(void)
2901 {
2902         struct ftrace_pid *fpid, *safe;
2903
2904         mutex_lock(&ftrace_lock);
2905         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
2906                 struct pid *pid = fpid->pid;
2907
2908                 clear_ftrace_pid_task(pid);
2909
2910                 list_del(&fpid->list);
2911                 kfree(fpid);
2912         }
2913
2914         ftrace_update_pid_func();
2915         ftrace_startup_enable(0);
2916
2917         mutex_unlock(&ftrace_lock);
2918 }
2919
2920 static void *fpid_start(struct seq_file *m, loff_t *pos)
2921 {
2922         mutex_lock(&ftrace_lock);
2923
2924         if (list_empty(&ftrace_pids) && (!*pos))
2925                 return (void *) 1;
2926
2927         return seq_list_start(&ftrace_pids, *pos);
2928 }
2929
2930 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
2931 {
2932         if (v == (void *)1)
2933                 return NULL;
2934
2935         return seq_list_next(v, &ftrace_pids, pos);
2936 }
2937
2938 static void fpid_stop(struct seq_file *m, void *p)
2939 {
2940         mutex_unlock(&ftrace_lock);
2941 }
2942
2943 static int fpid_show(struct seq_file *m, void *v)
2944 {
2945         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
2946
2947         if (v == (void *)1) {
2948                 seq_printf(m, "no pid\n");
2949                 return 0;
2950         }
2951
2952         if (fpid->pid == ftrace_swapper_pid)
2953                 seq_printf(m, "swapper tasks\n");
2954         else
2955                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
2956
2957         return 0;
2958 }
2959
2960 static const struct seq_operations ftrace_pid_sops = {
2961         .start = fpid_start,
2962         .next = fpid_next,
2963         .stop = fpid_stop,
2964         .show = fpid_show,
2965 };
2966
2967 static int
2968 ftrace_pid_open(struct inode *inode, struct file *file)
2969 {
2970         int ret = 0;
2971
2972         if ((file->f_mode & FMODE_WRITE) &&
2973             (file->f_flags & O_TRUNC))
2974                 ftrace_pid_reset();
2975
2976         if (file->f_mode & FMODE_READ)
2977                 ret = seq_open(file, &ftrace_pid_sops);
2978
2979         return ret;
2980 }
2981
2982 static ssize_t
2983 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2984                    size_t cnt, loff_t *ppos)
2985 {
2986         char buf[64], *tmp;
2987         long val;
2988         int ret;
2989
2990         if (cnt >= sizeof(buf))
2991                 return -EINVAL;
2992
2993         if (copy_from_user(&buf, ubuf, cnt))
2994                 return -EFAULT;
2995
2996         buf[cnt] = 0;
2997
2998         /*
2999          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3000          * to clean the filter quietly.
3001          */
3002         tmp = strstrip(buf);
3003         if (strlen(tmp) == 0)
3004                 return 1;
3005
3006         ret = strict_strtol(tmp, 10, &val);
3007         if (ret < 0)
3008                 return ret;
3009
3010         ret = ftrace_pid_add(val);
3011
3012         return ret ? ret : cnt;
3013 }
3014
3015 static int
3016 ftrace_pid_release(struct inode *inode, struct file *file)
3017 {
3018         if (file->f_mode & FMODE_READ)
3019                 seq_release(inode, file);
3020
3021         return 0;
3022 }
3023
3024 static const struct file_operations ftrace_pid_fops = {
3025         .open           = ftrace_pid_open,
3026         .write          = ftrace_pid_write,
3027         .read           = seq_read,
3028         .llseek         = seq_lseek,
3029         .release        = ftrace_pid_release,
3030 };
3031
3032 static __init int ftrace_init_debugfs(void)
3033 {
3034         struct dentry *d_tracer;
3035
3036         d_tracer = tracing_init_dentry();
3037         if (!d_tracer)
3038                 return 0;
3039
3040         ftrace_init_dyn_debugfs(d_tracer);
3041
3042         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3043                             NULL, &ftrace_pid_fops);
3044
3045         ftrace_profile_debugfs(d_tracer);
3046
3047         return 0;
3048 }
3049 fs_initcall(ftrace_init_debugfs);
3050
3051 /**
3052  * ftrace_kill - kill ftrace
3053  *
3054  * This function should be used by panic code. It stops ftrace
3055  * but in a not so nice way. If you need to simply kill ftrace
3056  * from a non-atomic section, use ftrace_kill.
3057  */
3058 void ftrace_kill(void)
3059 {
3060         ftrace_disabled = 1;
3061         ftrace_enabled = 0;
3062         clear_ftrace_function();
3063 }
3064
3065 /**
3066  * register_ftrace_function - register a function for profiling
3067  * @ops - ops structure that holds the function for profiling.
3068  *
3069  * Register a function to be called by all functions in the
3070  * kernel.
3071  *
3072  * Note: @ops->func and all the functions it calls must be labeled
3073  *       with "notrace", otherwise it will go into a
3074  *       recursive loop.
3075  */
3076 int register_ftrace_function(struct ftrace_ops *ops)
3077 {
3078         int ret;
3079
3080         if (unlikely(ftrace_disabled))
3081                 return -1;
3082
3083         mutex_lock(&ftrace_lock);
3084
3085         ret = __register_ftrace_function(ops);
3086         ftrace_startup(0);
3087
3088         mutex_unlock(&ftrace_lock);
3089         return ret;
3090 }
3091
3092 /**
3093  * unregister_ftrace_function - unregister a function for profiling.
3094  * @ops - ops structure that holds the function to unregister
3095  *
3096  * Unregister a function that was added to be called by ftrace profiling.
3097  */
3098 int unregister_ftrace_function(struct ftrace_ops *ops)
3099 {
3100         int ret;
3101
3102         mutex_lock(&ftrace_lock);
3103         ret = __unregister_ftrace_function(ops);
3104         ftrace_shutdown(0);
3105         mutex_unlock(&ftrace_lock);
3106
3107         return ret;
3108 }
3109
3110 int
3111 ftrace_enable_sysctl(struct ctl_table *table, int write,
3112                      void __user *buffer, size_t *lenp,
3113                      loff_t *ppos)
3114 {
3115         int ret;
3116
3117         if (unlikely(ftrace_disabled))
3118                 return -ENODEV;
3119
3120         mutex_lock(&ftrace_lock);
3121
3122         ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3123
3124         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3125                 goto out;
3126
3127         last_ftrace_enabled = !!ftrace_enabled;
3128
3129         if (ftrace_enabled) {
3130
3131                 ftrace_startup_sysctl();
3132
3133                 /* we are starting ftrace again */
3134                 if (ftrace_list != &ftrace_list_end) {
3135                         if (ftrace_list->next == &ftrace_list_end)
3136                                 ftrace_trace_function = ftrace_list->func;
3137                         else
3138                                 ftrace_trace_function = ftrace_list_func;
3139                 }
3140
3141         } else {
3142                 /* stopping ftrace calls (just send to ftrace_stub) */
3143                 ftrace_trace_function = ftrace_stub;
3144
3145                 ftrace_shutdown_sysctl();
3146         }
3147
3148  out:
3149         mutex_unlock(&ftrace_lock);
3150         return ret;
3151 }
3152
3153 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3154
3155 static int ftrace_graph_active;
3156 static struct notifier_block ftrace_suspend_notifier;
3157
3158 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3159 {
3160         return 0;
3161 }
3162
3163 /* The callbacks that hook a function */
3164 trace_func_graph_ret_t ftrace_graph_return =
3165                         (trace_func_graph_ret_t)ftrace_stub;
3166 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3167
3168 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3169 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3170 {
3171         int i;
3172         int ret = 0;
3173         unsigned long flags;
3174         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3175         struct task_struct *g, *t;
3176
3177         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3178                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3179                                         * sizeof(struct ftrace_ret_stack),
3180                                         GFP_KERNEL);
3181                 if (!ret_stack_list[i]) {
3182                         start = 0;
3183                         end = i;
3184                         ret = -ENOMEM;
3185                         goto free;
3186                 }
3187         }
3188
3189         read_lock_irqsave(&tasklist_lock, flags);
3190         do_each_thread(g, t) {
3191                 if (start == end) {
3192                         ret = -EAGAIN;
3193                         goto unlock;
3194                 }
3195
3196                 if (t->ret_stack == NULL) {
3197                         atomic_set(&t->tracing_graph_pause, 0);
3198                         atomic_set(&t->trace_overrun, 0);
3199                         t->curr_ret_stack = -1;
3200                         /* Make sure the tasks see the -1 first: */
3201                         smp_wmb();
3202                         t->ret_stack = ret_stack_list[start++];
3203                 }
3204         } while_each_thread(g, t);
3205
3206 unlock:
3207         read_unlock_irqrestore(&tasklist_lock, flags);
3208 free:
3209         for (i = start; i < end; i++)
3210                 kfree(ret_stack_list[i]);
3211         return ret;
3212 }
3213
3214 static void
3215 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3216                                 struct task_struct *next)
3217 {
3218         unsigned long long timestamp;
3219         int index;
3220
3221         /*
3222          * Does the user want to count the time a function was asleep.
3223          * If so, do not update the time stamps.
3224          */
3225         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3226                 return;
3227
3228         timestamp = trace_clock_local();
3229
3230         prev->ftrace_timestamp = timestamp;
3231
3232         /* only process tasks that we timestamped */
3233         if (!next->ftrace_timestamp)
3234                 return;
3235
3236         /*
3237          * Update all the counters in next to make up for the
3238          * time next was sleeping.
3239          */
3240         timestamp -= next->ftrace_timestamp;
3241
3242         for (index = next->curr_ret_stack; index >= 0; index--)
3243                 next->ret_stack[index].calltime += timestamp;
3244 }
3245
3246 /* Allocate a return stack for each task */
3247 static int start_graph_tracing(void)
3248 {
3249         struct ftrace_ret_stack **ret_stack_list;
3250         int ret, cpu;
3251
3252         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3253                                 sizeof(struct ftrace_ret_stack *),
3254                                 GFP_KERNEL);
3255
3256         if (!ret_stack_list)
3257                 return -ENOMEM;
3258
3259         /* The cpu_boot init_task->ret_stack will never be freed */
3260         for_each_online_cpu(cpu) {
3261                 if (!idle_task(cpu)->ret_stack)
3262                         ftrace_graph_init_task(idle_task(cpu));
3263         }
3264
3265         do {
3266                 ret = alloc_retstack_tasklist(ret_stack_list);
3267         } while (ret == -EAGAIN);
3268
3269         if (!ret) {
3270                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3271                 if (ret)
3272                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3273                                 " probe to kernel_sched_switch\n");
3274         }
3275
3276         kfree(ret_stack_list);
3277         return ret;
3278 }
3279
3280 /*
3281  * Hibernation protection.
3282  * The state of the current task is too much unstable during
3283  * suspend/restore to disk. We want to protect against that.
3284  */
3285 static int
3286 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3287                                                         void *unused)
3288 {
3289         switch (state) {
3290         case PM_HIBERNATION_PREPARE:
3291                 pause_graph_tracing();
3292                 break;
3293
3294         case PM_POST_HIBERNATION:
3295                 unpause_graph_tracing();
3296                 break;
3297         }
3298         return NOTIFY_DONE;
3299 }
3300
3301 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3302                         trace_func_graph_ent_t entryfunc)
3303 {
3304         int ret = 0;
3305
3306         mutex_lock(&ftrace_lock);
3307
3308         /* we currently allow only one tracer registered at a time */
3309         if (ftrace_graph_active) {
3310                 ret = -EBUSY;
3311                 goto out;
3312         }
3313
3314         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3315         register_pm_notifier(&ftrace_suspend_notifier);
3316
3317         ftrace_graph_active++;
3318         ret = start_graph_tracing();
3319         if (ret) {
3320                 ftrace_graph_active--;
3321                 goto out;
3322         }
3323
3324         ftrace_graph_return = retfunc;
3325         ftrace_graph_entry = entryfunc;
3326
3327         ftrace_startup(FTRACE_START_FUNC_RET);
3328
3329 out:
3330         mutex_unlock(&ftrace_lock);
3331         return ret;
3332 }
3333
3334 void unregister_ftrace_graph(void)
3335 {
3336         mutex_lock(&ftrace_lock);
3337
3338         if (unlikely(!ftrace_graph_active))
3339                 goto out;
3340
3341         ftrace_graph_active--;
3342         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3343         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3344         ftrace_graph_entry = ftrace_graph_entry_stub;
3345         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3346         unregister_pm_notifier(&ftrace_suspend_notifier);
3347
3348  out:
3349         mutex_unlock(&ftrace_lock);
3350 }
3351
3352 /* Allocate a return stack for newly created task */
3353 void ftrace_graph_init_task(struct task_struct *t)
3354 {
3355         /* Make sure we do not use the parent ret_stack */
3356         t->ret_stack = NULL;
3357         t->curr_ret_stack = -1;
3358
3359         if (ftrace_graph_active) {
3360                 struct ftrace_ret_stack *ret_stack;
3361
3362                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3363                                 * sizeof(struct ftrace_ret_stack),
3364                                 GFP_KERNEL);
3365                 if (!ret_stack)
3366                         return;
3367                 atomic_set(&t->tracing_graph_pause, 0);
3368                 atomic_set(&t->trace_overrun, 0);
3369                 t->ftrace_timestamp = 0;
3370                 /* make curr_ret_stack visable before we add the ret_stack */
3371                 smp_wmb();
3372                 t->ret_stack = ret_stack;
3373         }
3374 }
3375
3376 void ftrace_graph_exit_task(struct task_struct *t)
3377 {
3378         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3379
3380         t->ret_stack = NULL;
3381         /* NULL must become visible to IRQs before we free it: */
3382         barrier();
3383
3384         kfree(ret_stack);
3385 }
3386
3387 void ftrace_graph_stop(void)
3388 {
3389         ftrace_stop();
3390 }
3391 #endif