LCOV - code coverage report
Current view: top level - kernel/trace - trace.h (source / functions) Hit Total Coverage
Test: combined.info Lines: 14 76 18.4 %
Date: 2022-04-01 14:58:12 Functions: 0 3 0.0 %
Branches: 6 98 6.1 %

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : 
       3                 :            : #ifndef _LINUX_KERNEL_TRACE_H
       4                 :            : #define _LINUX_KERNEL_TRACE_H
       5                 :            : 
       6                 :            : #include <linux/fs.h>
       7                 :            : #include <linux/atomic.h>
       8                 :            : #include <linux/sched.h>
       9                 :            : #include <linux/clocksource.h>
      10                 :            : #include <linux/ring_buffer.h>
      11                 :            : #include <linux/mmiotrace.h>
      12                 :            : #include <linux/tracepoint.h>
      13                 :            : #include <linux/ftrace.h>
      14                 :            : #include <linux/trace.h>
      15                 :            : #include <linux/hw_breakpoint.h>
      16                 :            : #include <linux/trace_seq.h>
      17                 :            : #include <linux/trace_events.h>
      18                 :            : #include <linux/compiler.h>
      19                 :            : #include <linux/glob.h>
      20                 :            : #include <linux/irq_work.h>
      21                 :            : #include <linux/workqueue.h>
      22                 :            : 
      23                 :            : #ifdef CONFIG_FTRACE_SYSCALLS
      24                 :            : #include <asm/unistd.h>           /* For NR_SYSCALLS           */
      25                 :            : #include <asm/syscall.h>  /* some archs define it here */
      26                 :            : #endif
      27                 :            : 
      28                 :            : enum trace_type {
      29                 :            :         __TRACE_FIRST_TYPE = 0,
      30                 :            : 
      31                 :            :         TRACE_FN,
      32                 :            :         TRACE_CTX,
      33                 :            :         TRACE_WAKE,
      34                 :            :         TRACE_STACK,
      35                 :            :         TRACE_PRINT,
      36                 :            :         TRACE_BPRINT,
      37                 :            :         TRACE_MMIO_RW,
      38                 :            :         TRACE_MMIO_MAP,
      39                 :            :         TRACE_BRANCH,
      40                 :            :         TRACE_GRAPH_RET,
      41                 :            :         TRACE_GRAPH_ENT,
      42                 :            :         TRACE_USER_STACK,
      43                 :            :         TRACE_BLK,
      44                 :            :         TRACE_BPUTS,
      45                 :            :         TRACE_HWLAT,
      46                 :            :         TRACE_RAW_DATA,
      47                 :            : 
      48                 :            :         __TRACE_LAST_TYPE,
      49                 :            : };
      50                 :            : 
      51                 :            : 
      52                 :            : #undef __field
      53                 :            : #define __field(type, item)             type    item;
      54                 :            : 
      55                 :            : #undef __field_fn
      56                 :            : #define __field_fn(type, item)          type    item;
      57                 :            : 
      58                 :            : #undef __field_struct
      59                 :            : #define __field_struct(type, item)      __field(type, item)
      60                 :            : 
      61                 :            : #undef __field_desc
      62                 :            : #define __field_desc(type, container, item)
      63                 :            : 
      64                 :            : #undef __array
      65                 :            : #define __array(type, item, size)       type    item[size];
      66                 :            : 
      67                 :            : #undef __array_desc
      68                 :            : #define __array_desc(type, container, item, size)
      69                 :            : 
      70                 :            : #undef __dynamic_array
      71                 :            : #define __dynamic_array(type, item)     type    item[];
      72                 :            : 
      73                 :            : #undef F_STRUCT
      74                 :            : #define F_STRUCT(args...)               args
      75                 :            : 
      76                 :            : #undef FTRACE_ENTRY
      77                 :            : #define FTRACE_ENTRY(name, struct_name, id, tstruct, print)             \
      78                 :            :         struct struct_name {                                            \
      79                 :            :                 struct trace_entry      ent;                            \
      80                 :            :                 tstruct                                                 \
      81                 :            :         }
      82                 :            : 
      83                 :            : #undef FTRACE_ENTRY_DUP
      84                 :            : #define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk)
      85                 :            : 
      86                 :            : #undef FTRACE_ENTRY_REG
      87                 :            : #define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print, regfn)  \
      88                 :            :         FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print))
      89                 :            : 
      90                 :            : #undef FTRACE_ENTRY_PACKED
      91                 :            : #define FTRACE_ENTRY_PACKED(name, struct_name, id, tstruct, print)      \
      92                 :            :         FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print)) __packed
      93                 :            : 
      94                 :            : #include "trace_entries.h"
      95                 :            : 
      96                 :            : /* Use this for memory failure errors */
      97                 :            : #define MEM_FAIL(condition, fmt, ...) ({                        \
      98                 :            :         static bool __section(.data.once) __warned;             \
      99                 :            :         int __ret_warn_once = !!(condition);                    \
     100                 :            :                                                                 \
     101                 :            :         if (unlikely(__ret_warn_once && !__warned)) {           \
     102                 :            :                 __warned = true;                                \
     103                 :            :                 pr_err("ERROR: " fmt, ##__VA_ARGS__);         \
     104                 :            :         }                                                       \
     105                 :            :         unlikely(__ret_warn_once);                              \
     106                 :            : })
     107                 :            : 
     108                 :            : /*
     109                 :            :  * syscalls are special, and need special handling, this is why
     110                 :            :  * they are not included in trace_entries.h
     111                 :            :  */
     112                 :            : struct syscall_trace_enter {
     113                 :            :         struct trace_entry      ent;
     114                 :            :         int                     nr;
     115                 :            :         unsigned long           args[];
     116                 :            : };
     117                 :            : 
     118                 :            : struct syscall_trace_exit {
     119                 :            :         struct trace_entry      ent;
     120                 :            :         int                     nr;
     121                 :            :         long                    ret;
     122                 :            : };
     123                 :            : 
     124                 :            : struct kprobe_trace_entry_head {
     125                 :            :         struct trace_entry      ent;
     126                 :            :         unsigned long           ip;
     127                 :            : };
     128                 :            : 
     129                 :            : struct kretprobe_trace_entry_head {
     130                 :            :         struct trace_entry      ent;
     131                 :            :         unsigned long           func;
     132                 :            :         unsigned long           ret_ip;
     133                 :            : };
     134                 :            : 
     135                 :            : /*
     136                 :            :  * trace_flag_type is an enumeration that holds different
     137                 :            :  * states when a trace occurs. These are:
     138                 :            :  *  IRQS_OFF            - interrupts were disabled
     139                 :            :  *  IRQS_NOSUPPORT      - arch does not support irqs_disabled_flags
     140                 :            :  *  NEED_RESCHED        - reschedule is requested
     141                 :            :  *  HARDIRQ             - inside an interrupt handler
     142                 :            :  *  SOFTIRQ             - inside a softirq handler
     143                 :            :  */
     144                 :            : enum trace_flag_type {
     145                 :            :         TRACE_FLAG_IRQS_OFF             = 0x01,
     146                 :            :         TRACE_FLAG_IRQS_NOSUPPORT       = 0x02,
     147                 :            :         TRACE_FLAG_NEED_RESCHED         = 0x04,
     148                 :            :         TRACE_FLAG_HARDIRQ              = 0x08,
     149                 :            :         TRACE_FLAG_SOFTIRQ              = 0x10,
     150                 :            :         TRACE_FLAG_PREEMPT_RESCHED      = 0x20,
     151                 :            :         TRACE_FLAG_NMI                  = 0x40,
     152                 :            : };
     153                 :            : 
     154                 :            : #define TRACE_BUF_SIZE          1024
     155                 :            : 
     156                 :            : struct trace_array;
     157                 :            : 
     158                 :            : /*
     159                 :            :  * The CPU trace array - it consists of thousands of trace entries
     160                 :            :  * plus some other descriptor data: (for example which task started
     161                 :            :  * the trace, etc.)
     162                 :            :  */
     163                 :            : struct trace_array_cpu {
     164                 :            :         atomic_t                disabled;
     165                 :            :         void                    *buffer_page;   /* ring buffer spare */
     166                 :            : 
     167                 :            :         unsigned long           entries;
     168                 :            :         unsigned long           saved_latency;
     169                 :            :         unsigned long           critical_start;
     170                 :            :         unsigned long           critical_end;
     171                 :            :         unsigned long           critical_sequence;
     172                 :            :         unsigned long           nice;
     173                 :            :         unsigned long           policy;
     174                 :            :         unsigned long           rt_priority;
     175                 :            :         unsigned long           skipped_entries;
     176                 :            :         u64                     preempt_timestamp;
     177                 :            :         pid_t                   pid;
     178                 :            :         kuid_t                  uid;
     179                 :            :         char                    comm[TASK_COMM_LEN];
     180                 :            : 
     181                 :            :         bool                    ignore_pid;
     182                 :            : #ifdef CONFIG_FUNCTION_TRACER
     183                 :            :         bool                    ftrace_ignore_pid;
     184                 :            : #endif
     185                 :            : };
     186                 :            : 
     187                 :            : struct tracer;
     188                 :            : struct trace_option_dentry;
     189                 :            : 
     190                 :            : struct array_buffer {
     191                 :            :         struct trace_array              *tr;
     192                 :            :         struct trace_buffer             *buffer;
     193                 :            :         struct trace_array_cpu __percpu *data;
     194                 :            :         u64                             time_start;
     195                 :            :         int                             cpu;
     196                 :            : };
     197                 :            : 
     198                 :            : #define TRACE_FLAGS_MAX_SIZE            32
     199                 :            : 
     200                 :            : struct trace_options {
     201                 :            :         struct tracer                   *tracer;
     202                 :            :         struct trace_option_dentry      *topts;
     203                 :            : };
     204                 :            : 
     205                 :            : struct trace_pid_list {
     206                 :            :         int                             pid_max;
     207                 :            :         unsigned long                   *pids;
     208                 :            : };
     209                 :            : 
     210                 :            : typedef bool (*cond_update_fn_t)(struct trace_array *tr, void *cond_data);
     211                 :            : 
     212                 :            : /**
     213                 :            :  * struct cond_snapshot - conditional snapshot data and callback
     214                 :            :  *
     215                 :            :  * The cond_snapshot structure encapsulates a callback function and
     216                 :            :  * data associated with the snapshot for a given tracing instance.
     217                 :            :  *
     218                 :            :  * When a snapshot is taken conditionally, by invoking
     219                 :            :  * tracing_snapshot_cond(tr, cond_data), the cond_data passed in is
     220                 :            :  * passed in turn to the cond_snapshot.update() function.  That data
     221                 :            :  * can be compared by the update() implementation with the cond_data
     222                 :            :  * contained wihin the struct cond_snapshot instance associated with
     223                 :            :  * the trace_array.  Because the tr->max_lock is held throughout the
     224                 :            :  * update() call, the update() function can directly retrieve the
     225                 :            :  * cond_snapshot and cond_data associated with the per-instance
     226                 :            :  * snapshot associated with the trace_array.
     227                 :            :  *
     228                 :            :  * The cond_snapshot.update() implementation can save data to be
     229                 :            :  * associated with the snapshot if it decides to, and returns 'true'
     230                 :            :  * in that case, or it returns 'false' if the conditional snapshot
     231                 :            :  * shouldn't be taken.
     232                 :            :  *
     233                 :            :  * The cond_snapshot instance is created and associated with the
     234                 :            :  * user-defined cond_data by tracing_cond_snapshot_enable().
     235                 :            :  * Likewise, the cond_snapshot instance is destroyed and is no longer
     236                 :            :  * associated with the trace instance by
     237                 :            :  * tracing_cond_snapshot_disable().
     238                 :            :  *
     239                 :            :  * The method below is required.
     240                 :            :  *
     241                 :            :  * @update: When a conditional snapshot is invoked, the update()
     242                 :            :  *      callback function is invoked with the tr->max_lock held.  The
     243                 :            :  *      update() implementation signals whether or not to actually
     244                 :            :  *      take the snapshot, by returning 'true' if so, 'false' if no
     245                 :            :  *      snapshot should be taken.  Because the max_lock is held for
     246                 :            :  *      the duration of update(), the implementation is safe to
     247                 :            :  *      directly retrieven and save any implementation data it needs
     248                 :            :  *      to in association with the snapshot.
     249                 :            :  */
     250                 :            : struct cond_snapshot {
     251                 :            :         void                            *cond_data;
     252                 :            :         cond_update_fn_t                update;
     253                 :            : };
     254                 :            : 
     255                 :            : /*
     256                 :            :  * The trace array - an array of per-CPU trace arrays. This is the
     257                 :            :  * highest level data structure that individual tracers deal with.
     258                 :            :  * They have on/off state as well:
     259                 :            :  */
     260                 :            : struct trace_array {
     261                 :            :         struct list_head        list;
     262                 :            :         char                    *name;
     263                 :            :         struct array_buffer     array_buffer;
     264                 :            : #ifdef CONFIG_TRACER_MAX_TRACE
     265                 :            :         /*
     266                 :            :          * The max_buffer is used to snapshot the trace when a maximum
     267                 :            :          * latency is reached, or when the user initiates a snapshot.
     268                 :            :          * Some tracers will use this to store a maximum trace while
     269                 :            :          * it continues examining live traces.
     270                 :            :          *
     271                 :            :          * The buffers for the max_buffer are set up the same as the array_buffer
     272                 :            :          * When a snapshot is taken, the buffer of the max_buffer is swapped
     273                 :            :          * with the buffer of the array_buffer and the buffers are reset for
     274                 :            :          * the array_buffer so the tracing can continue.
     275                 :            :          */
     276                 :            :         struct array_buffer     max_buffer;
     277                 :            :         bool                    allocated_snapshot;
     278                 :            : #endif
     279                 :            : #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
     280                 :            :         unsigned long           max_latency;
     281                 :            : #ifdef CONFIG_FSNOTIFY
     282                 :            :         struct dentry           *d_max_latency;
     283                 :            :         struct work_struct      fsnotify_work;
     284                 :            :         struct irq_work         fsnotify_irqwork;
     285                 :            : #endif
     286                 :            : #endif
     287                 :            :         struct trace_pid_list   __rcu *filtered_pids;
     288                 :            :         /*
     289                 :            :          * max_lock is used to protect the swapping of buffers
     290                 :            :          * when taking a max snapshot. The buffers themselves are
     291                 :            :          * protected by per_cpu spinlocks. But the action of the swap
     292                 :            :          * needs its own lock.
     293                 :            :          *
     294                 :            :          * This is defined as a arch_spinlock_t in order to help
     295                 :            :          * with performance when lockdep debugging is enabled.
     296                 :            :          *
     297                 :            :          * It is also used in other places outside the update_max_tr
     298                 :            :          * so it needs to be defined outside of the
     299                 :            :          * CONFIG_TRACER_MAX_TRACE.
     300                 :            :          */
     301                 :            :         arch_spinlock_t         max_lock;
     302                 :            :         int                     buffer_disabled;
     303                 :            : #ifdef CONFIG_FTRACE_SYSCALLS
     304                 :            :         int                     sys_refcount_enter;
     305                 :            :         int                     sys_refcount_exit;
     306                 :            :         struct trace_event_file __rcu *enter_syscall_files[NR_syscalls];
     307                 :            :         struct trace_event_file __rcu *exit_syscall_files[NR_syscalls];
     308                 :            : #endif
     309                 :            :         int                     stop_count;
     310                 :            :         int                     clock_id;
     311                 :            :         int                     nr_topts;
     312                 :            :         bool                    clear_trace;
     313                 :            :         int                     buffer_percent;
     314                 :            :         unsigned int            n_err_log_entries;
     315                 :            :         struct tracer           *current_trace;
     316                 :            :         unsigned int            trace_flags;
     317                 :            :         unsigned char           trace_flags_index[TRACE_FLAGS_MAX_SIZE];
     318                 :            :         unsigned int            flags;
     319                 :            :         raw_spinlock_t          start_lock;
     320                 :            :         struct list_head        err_log;
     321                 :            :         struct dentry           *dir;
     322                 :            :         struct dentry           *options;
     323                 :            :         struct dentry           *percpu_dir;
     324                 :            :         struct dentry           *event_dir;
     325                 :            :         struct trace_options    *topts;
     326                 :            :         struct list_head        systems;
     327                 :            :         struct list_head        events;
     328                 :            :         struct trace_event_file *trace_marker_file;
     329                 :            :         cpumask_var_t           tracing_cpumask; /* only trace on set CPUs */
     330                 :            :         int                     ref;
     331                 :            : #ifdef CONFIG_FUNCTION_TRACER
     332                 :            :         struct ftrace_ops       *ops;
     333                 :            :         struct trace_pid_list   __rcu *function_pids;
     334                 :            : #ifdef CONFIG_DYNAMIC_FTRACE
     335                 :            :         /* All of these are protected by the ftrace_lock */
     336                 :            :         struct list_head        func_probes;
     337                 :            :         struct list_head        mod_trace;
     338                 :            :         struct list_head        mod_notrace;
     339                 :            : #endif
     340                 :            :         /* function tracing enabled */
     341                 :            :         int                     function_enabled;
     342                 :            : #endif
     343                 :            :         int                     time_stamp_abs_ref;
     344                 :            :         struct list_head        hist_vars;
     345                 :            : #ifdef CONFIG_TRACER_SNAPSHOT
     346                 :            :         struct cond_snapshot    *cond_snapshot;
     347                 :            : #endif
     348                 :            : };
     349                 :            : 
     350                 :            : enum {
     351                 :            :         TRACE_ARRAY_FL_GLOBAL   = (1 << 0)
     352                 :            : };
     353                 :            : 
     354                 :            : extern struct list_head ftrace_trace_arrays;
     355                 :            : 
     356                 :            : extern struct mutex trace_types_lock;
     357                 :            : 
     358                 :            : extern int trace_array_get(struct trace_array *tr);
     359                 :            : extern int tracing_check_open_get_tr(struct trace_array *tr);
     360                 :            : extern struct trace_array *trace_array_find(const char *instance);
     361                 :            : extern struct trace_array *trace_array_find_get(const char *instance);
     362                 :            : 
     363                 :            : extern int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs);
     364                 :            : extern int tracing_set_clock(struct trace_array *tr, const char *clockstr);
     365                 :            : 
     366                 :            : extern bool trace_clock_in_ns(struct trace_array *tr);
     367                 :            : 
     368                 :            : /*
     369                 :            :  * The global tracer (top) should be the first trace array added,
     370                 :            :  * but we check the flag anyway.
     371                 :            :  */
     372                 :         12 : static inline struct trace_array *top_trace_array(void)
     373                 :            : {
     374                 :         12 :         struct trace_array *tr;
     375                 :            : 
     376   [ +  -  +  -  :         12 :         if (list_empty(&ftrace_trace_arrays))
          +  -  -  -  -  
                      - ]
     377                 :            :                 return NULL;
     378                 :            : 
     379                 :         12 :         tr = list_entry(ftrace_trace_arrays.prev,
     380                 :            :                         typeof(*tr), list);
     381   [ -  +  -  +  :         12 :         WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
          -  +  -  -  -  
                      - ]
     382                 :            :         return tr;
     383                 :            : }
     384                 :            : 
     385                 :            : #define FTRACE_CMP_TYPE(var, type) \
     386                 :            :         __builtin_types_compatible_p(typeof(var), type *)
     387                 :            : 
     388                 :            : #undef IF_ASSIGN
     389                 :            : #define IF_ASSIGN(var, entry, etype, id)                        \
     390                 :            :         if (FTRACE_CMP_TYPE(var, etype)) {                      \
     391                 :            :                 var = (typeof(var))(entry);                     \
     392                 :            :                 WARN_ON(id != 0 && (entry)->type != id);     \
     393                 :            :                 break;                                          \
     394                 :            :         }
     395                 :            : 
     396                 :            : /* Will cause compile errors if type is not found. */
     397                 :            : extern void __ftrace_bad_type(void);
     398                 :            : 
     399                 :            : /*
     400                 :            :  * The trace_assign_type is a verifier that the entry type is
     401                 :            :  * the same as the type being assigned. To add new types simply
     402                 :            :  * add a line with the following format:
     403                 :            :  *
     404                 :            :  * IF_ASSIGN(var, ent, type, id);
     405                 :            :  *
     406                 :            :  *  Where "type" is the trace type that includes the trace_entry
     407                 :            :  *  as the "ent" item. And "id" is the trace identifier that is
     408                 :            :  *  used in the trace_type enum.
     409                 :            :  *
     410                 :            :  *  If the type can have more than one id, then use zero.
     411                 :            :  */
     412                 :            : #define trace_assign_type(var, ent)                                     \
     413                 :            :         do {                                                            \
     414                 :            :                 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN);     \
     415                 :            :                 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0);        \
     416                 :            :                 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK);   \
     417                 :            :                 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
     418                 :            :                 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT);   \
     419                 :            :                 IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT); \
     420                 :            :                 IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS);   \
     421                 :            :                 IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT);   \
     422                 :            :                 IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\
     423                 :            :                 IF_ASSIGN(var, ent, struct trace_mmiotrace_rw,          \
     424                 :            :                           TRACE_MMIO_RW);                               \
     425                 :            :                 IF_ASSIGN(var, ent, struct trace_mmiotrace_map,         \
     426                 :            :                           TRACE_MMIO_MAP);                              \
     427                 :            :                 IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
     428                 :            :                 IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry,      \
     429                 :            :                           TRACE_GRAPH_ENT);             \
     430                 :            :                 IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry,      \
     431                 :            :                           TRACE_GRAPH_RET);             \
     432                 :            :                 __ftrace_bad_type();                                    \
     433                 :            :         } while (0)
     434                 :            : 
     435                 :            : /*
     436                 :            :  * An option specific to a tracer. This is a boolean value.
     437                 :            :  * The bit is the bit index that sets its value on the
     438                 :            :  * flags value in struct tracer_flags.
     439                 :            :  */
     440                 :            : struct tracer_opt {
     441                 :            :         const char      *name; /* Will appear on the trace_options file */
     442                 :            :         u32             bit; /* Mask assigned in val field in tracer_flags */
     443                 :            : };
     444                 :            : 
     445                 :            : /*
     446                 :            :  * The set of specific options for a tracer. Your tracer
     447                 :            :  * have to set the initial value of the flags val.
     448                 :            :  */
     449                 :            : struct tracer_flags {
     450                 :            :         u32                     val;
     451                 :            :         struct tracer_opt       *opts;
     452                 :            :         struct tracer           *trace;
     453                 :            : };
     454                 :            : 
     455                 :            : /* Makes more easy to define a tracer opt */
     456                 :            : #define TRACER_OPT(s, b)        .name = #s, .bit = b
     457                 :            : 
     458                 :            : 
     459                 :            : struct trace_option_dentry {
     460                 :            :         struct tracer_opt               *opt;
     461                 :            :         struct tracer_flags             *flags;
     462                 :            :         struct trace_array              *tr;
     463                 :            :         struct dentry                   *entry;
     464                 :            : };
     465                 :            : 
     466                 :            : /**
     467                 :            :  * struct tracer - a specific tracer and its callbacks to interact with tracefs
     468                 :            :  * @name: the name chosen to select it on the available_tracers file
     469                 :            :  * @init: called when one switches to this tracer (echo name > current_tracer)
     470                 :            :  * @reset: called when one switches to another tracer
     471                 :            :  * @start: called when tracing is unpaused (echo 1 > tracing_on)
     472                 :            :  * @stop: called when tracing is paused (echo 0 > tracing_on)
     473                 :            :  * @update_thresh: called when tracing_thresh is updated
     474                 :            :  * @open: called when the trace file is opened
     475                 :            :  * @pipe_open: called when the trace_pipe file is opened
     476                 :            :  * @close: called when the trace file is released
     477                 :            :  * @pipe_close: called when the trace_pipe file is released
     478                 :            :  * @read: override the default read callback on trace_pipe
     479                 :            :  * @splice_read: override the default splice_read callback on trace_pipe
     480                 :            :  * @selftest: selftest to run on boot (see trace_selftest.c)
     481                 :            :  * @print_headers: override the first lines that describe your columns
     482                 :            :  * @print_line: callback that prints a trace
     483                 :            :  * @set_flag: signals one of your private flags changed (trace_options file)
     484                 :            :  * @flags: your private flags
     485                 :            :  */
     486                 :            : struct tracer {
     487                 :            :         const char              *name;
     488                 :            :         int                     (*init)(struct trace_array *tr);
     489                 :            :         void                    (*reset)(struct trace_array *tr);
     490                 :            :         void                    (*start)(struct trace_array *tr);
     491                 :            :         void                    (*stop)(struct trace_array *tr);
     492                 :            :         int                     (*update_thresh)(struct trace_array *tr);
     493                 :            :         void                    (*open)(struct trace_iterator *iter);
     494                 :            :         void                    (*pipe_open)(struct trace_iterator *iter);
     495                 :            :         void                    (*close)(struct trace_iterator *iter);
     496                 :            :         void                    (*pipe_close)(struct trace_iterator *iter);
     497                 :            :         ssize_t                 (*read)(struct trace_iterator *iter,
     498                 :            :                                         struct file *filp, char __user *ubuf,
     499                 :            :                                         size_t cnt, loff_t *ppos);
     500                 :            :         ssize_t                 (*splice_read)(struct trace_iterator *iter,
     501                 :            :                                                struct file *filp,
     502                 :            :                                                loff_t *ppos,
     503                 :            :                                                struct pipe_inode_info *pipe,
     504                 :            :                                                size_t len,
     505                 :            :                                                unsigned int flags);
     506                 :            : #ifdef CONFIG_FTRACE_STARTUP_TEST
     507                 :            :         int                     (*selftest)(struct tracer *trace,
     508                 :            :                                             struct trace_array *tr);
     509                 :            : #endif
     510                 :            :         void                    (*print_header)(struct seq_file *m);
     511                 :            :         enum print_line_t       (*print_line)(struct trace_iterator *iter);
     512                 :            :         /* If you handled the flag setting, return 0 */
     513                 :            :         int                     (*set_flag)(struct trace_array *tr,
     514                 :            :                                             u32 old_flags, u32 bit, int set);
     515                 :            :         /* Return 0 if OK with change, else return non-zero */
     516                 :            :         int                     (*flag_changed)(struct trace_array *tr,
     517                 :            :                                                 u32 mask, int set);
     518                 :            :         struct tracer           *next;
     519                 :            :         struct tracer_flags     *flags;
     520                 :            :         int                     enabled;
     521                 :            :         int                     ref;
     522                 :            :         bool                    print_max;
     523                 :            :         bool                    allow_instances;
     524                 :            : #ifdef CONFIG_TRACER_MAX_TRACE
     525                 :            :         bool                    use_max_tr;
     526                 :            : #endif
     527                 :            :         /* True if tracer cannot be enabled in kernel param */
     528                 :            :         bool                    noboot;
     529                 :            : };
     530                 :            : 
     531                 :            : 
     532                 :            : /* Only current can touch trace_recursion */
     533                 :            : 
     534                 :            : /*
     535                 :            :  * For function tracing recursion:
     536                 :            :  *  The order of these bits are important.
     537                 :            :  *
     538                 :            :  *  When function tracing occurs, the following steps are made:
     539                 :            :  *   If arch does not support a ftrace feature:
     540                 :            :  *    call internal function (uses INTERNAL bits) which calls...
     541                 :            :  *   If callback is registered to the "global" list, the list
     542                 :            :  *    function is called and recursion checks the GLOBAL bits.
     543                 :            :  *    then this function calls...
     544                 :            :  *   The function callback, which can use the FTRACE bits to
     545                 :            :  *    check for recursion.
     546                 :            :  *
     547                 :            :  * Now if the arch does not suppport a feature, and it calls
     548                 :            :  * the global list function which calls the ftrace callback
     549                 :            :  * all three of these steps will do a recursion protection.
     550                 :            :  * There's no reason to do one if the previous caller already
     551                 :            :  * did. The recursion that we are protecting against will
     552                 :            :  * go through the same steps again.
     553                 :            :  *
     554                 :            :  * To prevent the multiple recursion checks, if a recursion
     555                 :            :  * bit is set that is higher than the MAX bit of the current
     556                 :            :  * check, then we know that the check was made by the previous
     557                 :            :  * caller, and we can skip the current check.
     558                 :            :  */
     559                 :            : enum {
     560                 :            :         TRACE_BUFFER_BIT,
     561                 :            :         TRACE_BUFFER_NMI_BIT,
     562                 :            :         TRACE_BUFFER_IRQ_BIT,
     563                 :            :         TRACE_BUFFER_SIRQ_BIT,
     564                 :            : 
     565                 :            :         /* Start of function recursion bits */
     566                 :            :         TRACE_FTRACE_BIT,
     567                 :            :         TRACE_FTRACE_NMI_BIT,
     568                 :            :         TRACE_FTRACE_IRQ_BIT,
     569                 :            :         TRACE_FTRACE_SIRQ_BIT,
     570                 :            : 
     571                 :            :         /* INTERNAL_BITs must be greater than FTRACE_BITs */
     572                 :            :         TRACE_INTERNAL_BIT,
     573                 :            :         TRACE_INTERNAL_NMI_BIT,
     574                 :            :         TRACE_INTERNAL_IRQ_BIT,
     575                 :            :         TRACE_INTERNAL_SIRQ_BIT,
     576                 :            : 
     577                 :            :         TRACE_BRANCH_BIT,
     578                 :            : /*
     579                 :            :  * Abuse of the trace_recursion.
     580                 :            :  * As we need a way to maintain state if we are tracing the function
     581                 :            :  * graph in irq because we want to trace a particular function that
     582                 :            :  * was called in irq context but we have irq tracing off. Since this
     583                 :            :  * can only be modified by current, we can reuse trace_recursion.
     584                 :            :  */
     585                 :            :         TRACE_IRQ_BIT,
     586                 :            : 
     587                 :            :         /* Set if the function is in the set_graph_function file */
     588                 :            :         TRACE_GRAPH_BIT,
     589                 :            : 
     590                 :            :         /*
     591                 :            :          * In the very unlikely case that an interrupt came in
     592                 :            :          * at a start of graph tracing, and we want to trace
     593                 :            :          * the function in that interrupt, the depth can be greater
     594                 :            :          * than zero, because of the preempted start of a previous
     595                 :            :          * trace. In an even more unlikely case, depth could be 2
     596                 :            :          * if a softirq interrupted the start of graph tracing,
     597                 :            :          * followed by an interrupt preempting a start of graph
     598                 :            :          * tracing in the softirq, and depth can even be 3
     599                 :            :          * if an NMI came in at the start of an interrupt function
     600                 :            :          * that preempted a softirq start of a function that
     601                 :            :          * preempted normal context!!!! Luckily, it can't be
     602                 :            :          * greater than 3, so the next two bits are a mask
     603                 :            :          * of what the depth is when we set TRACE_GRAPH_BIT
     604                 :            :          */
     605                 :            : 
     606                 :            :         TRACE_GRAPH_DEPTH_START_BIT,
     607                 :            :         TRACE_GRAPH_DEPTH_END_BIT,
     608                 :            : 
     609                 :            :         /*
     610                 :            :          * To implement set_graph_notrace, if this bit is set, we ignore
     611                 :            :          * function graph tracing of called functions, until the return
     612                 :            :          * function is called to clear it.
     613                 :            :          */
     614                 :            :         TRACE_GRAPH_NOTRACE_BIT,
     615                 :            : };
     616                 :            : 
     617                 :            : #define trace_recursion_set(bit)        do { (current)->trace_recursion |= (1<<(bit)); } while (0)
     618                 :            : #define trace_recursion_clear(bit)      do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
     619                 :            : #define trace_recursion_test(bit)       ((current)->trace_recursion & (1<<(bit)))
     620                 :            : 
     621                 :            : #define trace_recursion_depth() \
     622                 :            :         (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
     623                 :            : #define trace_recursion_set_depth(depth) \
     624                 :            :         do {                                                            \
     625                 :            :                 current->trace_recursion &=                              \
     626                 :            :                         ~(3 << TRACE_GRAPH_DEPTH_START_BIT);              \
     627                 :            :                 current->trace_recursion |=                          \
     628                 :            :                         ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
     629                 :            :         } while (0)
     630                 :            : 
     631                 :            : #define TRACE_CONTEXT_BITS      4
     632                 :            : 
     633                 :            : #define TRACE_FTRACE_START      TRACE_FTRACE_BIT
     634                 :            : #define TRACE_FTRACE_MAX        ((1 << (TRACE_FTRACE_START + TRACE_CONTEXT_BITS)) - 1)
     635                 :            : 
     636                 :            : #define TRACE_LIST_START        TRACE_INTERNAL_BIT
     637                 :            : #define TRACE_LIST_MAX          ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
     638                 :            : 
     639                 :            : #define TRACE_CONTEXT_MASK      TRACE_LIST_MAX
     640                 :            : 
     641                 :            : static __always_inline int trace_get_context_bit(void)
     642                 :            : {
     643                 :            :         int bit;
     644                 :            : 
     645                 :            :         if (in_interrupt()) {
     646                 :            :                 if (in_nmi())
     647                 :            :                         bit = 0;
     648                 :            : 
     649                 :            :                 else if (in_irq())
     650                 :            :                         bit = 1;
     651                 :            :                 else
     652                 :            :                         bit = 2;
     653                 :            :         } else
     654                 :            :                 bit = 3;
     655                 :            : 
     656                 :            :         return bit;
     657                 :            : }
     658                 :            : 
     659                 :            : static __always_inline int trace_test_and_set_recursion(int start, int max)
     660                 :            : {
     661                 :            :         unsigned int val = current->trace_recursion;
     662                 :            :         int bit;
     663                 :            : 
     664                 :            :         /* A previous recursion check was made */
     665                 :            :         if ((val & TRACE_CONTEXT_MASK) > max)
     666                 :            :                 return 0;
     667                 :            : 
     668                 :            :         bit = trace_get_context_bit() + start;
     669                 :            :         if (unlikely(val & (1 << bit)))
     670                 :            :                 return -1;
     671                 :            : 
     672                 :            :         val |= 1 << bit;
     673                 :            :         current->trace_recursion = val;
     674                 :            :         barrier();
     675                 :            : 
     676                 :            :         return bit;
     677                 :            : }
     678                 :            : 
     679                 :            : static __always_inline void trace_clear_recursion(int bit)
     680                 :            : {
     681                 :            :         unsigned int val = current->trace_recursion;
     682                 :            : 
     683                 :            :         if (!bit)
     684                 :            :                 return;
     685                 :            : 
     686                 :            :         bit = 1 << bit;
     687                 :            :         val &= ~bit;
     688                 :            : 
     689                 :            :         barrier();
     690                 :            :         current->trace_recursion = val;
     691                 :            : }
     692                 :            : 
     693                 :            : static inline struct ring_buffer_iter *
     694                 :          0 : trace_buffer_iter(struct trace_iterator *iter, int cpu)
     695                 :            : {
     696   [ #  #  #  #  :          0 :         return iter->buffer_iter ? iter->buffer_iter[cpu] : NULL;
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     697                 :            : }
     698                 :            : 
     699                 :            : int tracer_init(struct tracer *t, struct trace_array *tr);
     700                 :            : int tracing_is_enabled(void);
     701                 :            : void tracing_reset_online_cpus(struct array_buffer *buf);
     702                 :            : void tracing_reset_current(int cpu);
     703                 :            : void tracing_reset_all_online_cpus(void);
     704                 :            : int tracing_open_generic(struct inode *inode, struct file *filp);
     705                 :            : int tracing_open_generic_tr(struct inode *inode, struct file *filp);
     706                 :            : bool tracing_is_disabled(void);
     707                 :            : bool tracer_tracing_is_on(struct trace_array *tr);
     708                 :            : void tracer_tracing_on(struct trace_array *tr);
     709                 :            : void tracer_tracing_off(struct trace_array *tr);
     710                 :            : struct dentry *trace_create_file(const char *name,
     711                 :            :                                  umode_t mode,
     712                 :            :                                  struct dentry *parent,
     713                 :            :                                  void *data,
     714                 :            :                                  const struct file_operations *fops);
     715                 :            : 
     716                 :            : struct dentry *tracing_init_dentry(void);
     717                 :            : 
     718                 :            : struct ring_buffer_event;
     719                 :            : 
     720                 :            : struct ring_buffer_event *
     721                 :            : trace_buffer_lock_reserve(struct trace_buffer *buffer,
     722                 :            :                           int type,
     723                 :            :                           unsigned long len,
     724                 :            :                           unsigned long flags,
     725                 :            :                           int pc);
     726                 :            : 
     727                 :            : struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
     728                 :            :                                                 struct trace_array_cpu *data);
     729                 :            : 
     730                 :            : struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
     731                 :            :                                           int *ent_cpu, u64 *ent_ts);
     732                 :            : 
     733                 :            : void trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
     734                 :            :                                         struct ring_buffer_event *event);
     735                 :            : 
     736                 :            : int trace_empty(struct trace_iterator *iter);
     737                 :            : 
     738                 :            : void *trace_find_next_entry_inc(struct trace_iterator *iter);
     739                 :            : 
     740                 :            : void trace_init_global_iter(struct trace_iterator *iter);
     741                 :            : 
     742                 :            : void tracing_iter_reset(struct trace_iterator *iter, int cpu);
     743                 :            : 
     744                 :            : unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu);
     745                 :            : unsigned long trace_total_entries(struct trace_array *tr);
     746                 :            : 
     747                 :            : void trace_function(struct trace_array *tr,
     748                 :            :                     unsigned long ip,
     749                 :            :                     unsigned long parent_ip,
     750                 :            :                     unsigned long flags, int pc);
     751                 :            : void trace_graph_function(struct trace_array *tr,
     752                 :            :                     unsigned long ip,
     753                 :            :                     unsigned long parent_ip,
     754                 :            :                     unsigned long flags, int pc);
     755                 :            : void trace_latency_header(struct seq_file *m);
     756                 :            : void trace_default_header(struct seq_file *m);
     757                 :            : void print_trace_header(struct seq_file *m, struct trace_iterator *iter);
     758                 :            : int trace_empty(struct trace_iterator *iter);
     759                 :            : 
     760                 :            : void trace_graph_return(struct ftrace_graph_ret *trace);
     761                 :            : int trace_graph_entry(struct ftrace_graph_ent *trace);
     762                 :            : void set_graph_array(struct trace_array *tr);
     763                 :            : 
     764                 :            : void tracing_start_cmdline_record(void);
     765                 :            : void tracing_stop_cmdline_record(void);
     766                 :            : void tracing_start_tgid_record(void);
     767                 :            : void tracing_stop_tgid_record(void);
     768                 :            : 
     769                 :            : int register_tracer(struct tracer *type);
     770                 :            : int is_tracing_stopped(void);
     771                 :            : 
     772                 :            : loff_t tracing_lseek(struct file *file, loff_t offset, int whence);
     773                 :            : 
     774                 :            : extern cpumask_var_t __read_mostly tracing_buffer_mask;
     775                 :            : 
     776                 :            : #define for_each_tracing_cpu(cpu)       \
     777                 :            :         for_each_cpu(cpu, tracing_buffer_mask)
     778                 :            : 
     779                 :            : extern unsigned long nsecs_to_usecs(unsigned long nsecs);
     780                 :            : 
     781                 :            : extern unsigned long tracing_thresh;
     782                 :            : 
     783                 :            : /* PID filtering */
     784                 :            : 
     785                 :            : extern int pid_max;
     786                 :            : 
     787                 :            : bool trace_find_filtered_pid(struct trace_pid_list *filtered_pids,
     788                 :            :                              pid_t search_pid);
     789                 :            : bool trace_ignore_this_task(struct trace_pid_list *filtered_pids,
     790                 :            :                             struct task_struct *task);
     791                 :            : void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
     792                 :            :                                   struct task_struct *self,
     793                 :            :                                   struct task_struct *task);
     794                 :            : void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos);
     795                 :            : void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos);
     796                 :            : int trace_pid_show(struct seq_file *m, void *v);
     797                 :            : void trace_free_pid_list(struct trace_pid_list *pid_list);
     798                 :            : int trace_pid_write(struct trace_pid_list *filtered_pids,
     799                 :            :                     struct trace_pid_list **new_pid_list,
     800                 :            :                     const char __user *ubuf, size_t cnt);
     801                 :            : 
     802                 :            : #ifdef CONFIG_TRACER_MAX_TRACE
     803                 :            : void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
     804                 :            :                    void *cond_data);
     805                 :            : void update_max_tr_single(struct trace_array *tr,
     806                 :            :                           struct task_struct *tsk, int cpu);
     807                 :            : #endif /* CONFIG_TRACER_MAX_TRACE */
     808                 :            : 
     809                 :            : #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
     810                 :            :         defined(CONFIG_FSNOTIFY)
     811                 :            : 
     812                 :            : void latency_fsnotify(struct trace_array *tr);
     813                 :            : 
     814                 :            : #else
     815                 :            : 
     816                 :            : static inline void latency_fsnotify(struct trace_array *tr) { }
     817                 :            : 
     818                 :            : #endif
     819                 :            : 
     820                 :            : #ifdef CONFIG_STACKTRACE
     821                 :            : void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
     822                 :            :                    int pc);
     823                 :            : #else
     824                 :            : static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
     825                 :            :                                  int skip, int pc)
     826                 :            : {
     827                 :            : }
     828                 :            : #endif /* CONFIG_STACKTRACE */
     829                 :            : 
     830                 :            : extern u64 ftrace_now(int cpu);
     831                 :            : 
     832                 :            : extern void trace_find_cmdline(int pid, char comm[]);
     833                 :            : extern int trace_find_tgid(int pid);
     834                 :            : extern void trace_event_follow_fork(struct trace_array *tr, bool enable);
     835                 :            : 
     836                 :            : #ifdef CONFIG_DYNAMIC_FTRACE
     837                 :            : extern unsigned long ftrace_update_tot_cnt;
     838                 :            : extern unsigned long ftrace_number_of_pages;
     839                 :            : extern unsigned long ftrace_number_of_groups;
     840                 :            : void ftrace_init_trace_array(struct trace_array *tr);
     841                 :            : #else
     842                 :          0 : static inline void ftrace_init_trace_array(struct trace_array *tr) { }
     843                 :            : #endif
     844                 :            : #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
     845                 :            : extern int DYN_FTRACE_TEST_NAME(void);
     846                 :            : #define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
     847                 :            : extern int DYN_FTRACE_TEST_NAME2(void);
     848                 :            : 
     849                 :            : extern bool ring_buffer_expanded;
     850                 :            : extern bool tracing_selftest_disabled;
     851                 :            : 
     852                 :            : #ifdef CONFIG_FTRACE_STARTUP_TEST
     853                 :            : extern int trace_selftest_startup_function(struct tracer *trace,
     854                 :            :                                            struct trace_array *tr);
     855                 :            : extern int trace_selftest_startup_function_graph(struct tracer *trace,
     856                 :            :                                                  struct trace_array *tr);
     857                 :            : extern int trace_selftest_startup_irqsoff(struct tracer *trace,
     858                 :            :                                           struct trace_array *tr);
     859                 :            : extern int trace_selftest_startup_preemptoff(struct tracer *trace,
     860                 :            :                                              struct trace_array *tr);
     861                 :            : extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
     862                 :            :                                                  struct trace_array *tr);
     863                 :            : extern int trace_selftest_startup_wakeup(struct tracer *trace,
     864                 :            :                                          struct trace_array *tr);
     865                 :            : extern int trace_selftest_startup_nop(struct tracer *trace,
     866                 :            :                                          struct trace_array *tr);
     867                 :            : extern int trace_selftest_startup_branch(struct tracer *trace,
     868                 :            :                                          struct trace_array *tr);
     869                 :            : /*
     870                 :            :  * Tracer data references selftest functions that only occur
     871                 :            :  * on boot up. These can be __init functions. Thus, when selftests
     872                 :            :  * are enabled, then the tracers need to reference __init functions.
     873                 :            :  */
     874                 :            : #define __tracer_data           __refdata
     875                 :            : #else
     876                 :            : /* Tracers are seldom changed. Optimize when selftests are disabled. */
     877                 :            : #define __tracer_data           __read_mostly
     878                 :            : #endif /* CONFIG_FTRACE_STARTUP_TEST */
     879                 :            : 
     880                 :            : extern void *head_page(struct trace_array_cpu *data);
     881                 :            : extern unsigned long long ns2usecs(u64 nsec);
     882                 :            : extern int
     883                 :            : trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
     884                 :            : extern int
     885                 :            : trace_vprintk(unsigned long ip, const char *fmt, va_list args);
     886                 :            : extern int
     887                 :            : trace_array_vprintk(struct trace_array *tr,
     888                 :            :                     unsigned long ip, const char *fmt, va_list args);
     889                 :            : int trace_array_printk_buf(struct trace_buffer *buffer,
     890                 :            :                            unsigned long ip, const char *fmt, ...);
     891                 :            : void trace_printk_seq(struct trace_seq *s);
     892                 :            : enum print_line_t print_trace_line(struct trace_iterator *iter);
     893                 :            : 
     894                 :            : extern char trace_find_mark(unsigned long long duration);
     895                 :            : 
     896                 :            : struct ftrace_hash;
     897                 :            : 
     898                 :            : struct ftrace_mod_load {
     899                 :            :         struct list_head        list;
     900                 :            :         char                    *func;
     901                 :            :         char                    *module;
     902                 :            :         int                      enable;
     903                 :            : };
     904                 :            : 
     905                 :            : enum {
     906                 :            :         FTRACE_HASH_FL_MOD      = (1 << 0),
     907                 :            : };
     908                 :            : 
     909                 :            : struct ftrace_hash {
     910                 :            :         unsigned long           size_bits;
     911                 :            :         struct hlist_head       *buckets;
     912                 :            :         unsigned long           count;
     913                 :            :         unsigned long           flags;
     914                 :            :         struct rcu_head         rcu;
     915                 :            : };
     916                 :            : 
     917                 :            : struct ftrace_func_entry *
     918                 :            : ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip);
     919                 :            : 
     920                 :            : static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)
     921                 :            : {
     922                 :            :         return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));
     923                 :            : }
     924                 :            : 
     925                 :            : /* Standard output formatting function used for function return traces */
     926                 :            : #ifdef CONFIG_FUNCTION_GRAPH_TRACER
     927                 :            : 
     928                 :            : /* Flag options */
     929                 :            : #define TRACE_GRAPH_PRINT_OVERRUN       0x1
     930                 :            : #define TRACE_GRAPH_PRINT_CPU           0x2
     931                 :            : #define TRACE_GRAPH_PRINT_OVERHEAD      0x4
     932                 :            : #define TRACE_GRAPH_PRINT_PROC          0x8
     933                 :            : #define TRACE_GRAPH_PRINT_DURATION      0x10
     934                 :            : #define TRACE_GRAPH_PRINT_ABS_TIME      0x20
     935                 :            : #define TRACE_GRAPH_PRINT_REL_TIME      0x40
     936                 :            : #define TRACE_GRAPH_PRINT_IRQS          0x80
     937                 :            : #define TRACE_GRAPH_PRINT_TAIL          0x100
     938                 :            : #define TRACE_GRAPH_SLEEP_TIME          0x200
     939                 :            : #define TRACE_GRAPH_GRAPH_TIME          0x400
     940                 :            : #define TRACE_GRAPH_PRINT_FILL_SHIFT    28
     941                 :            : #define TRACE_GRAPH_PRINT_FILL_MASK     (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT)
     942                 :            : 
     943                 :            : extern void ftrace_graph_sleep_time_control(bool enable);
     944                 :            : 
     945                 :            : #ifdef CONFIG_FUNCTION_PROFILER
     946                 :            : extern void ftrace_graph_graph_time_control(bool enable);
     947                 :            : #else
     948                 :            : static inline void ftrace_graph_graph_time_control(bool enable) { }
     949                 :            : #endif
     950                 :            : 
     951                 :            : extern enum print_line_t
     952                 :            : print_graph_function_flags(struct trace_iterator *iter, u32 flags);
     953                 :            : extern void print_graph_headers_flags(struct seq_file *s, u32 flags);
     954                 :            : extern void
     955                 :            : trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
     956                 :            : extern void graph_trace_open(struct trace_iterator *iter);
     957                 :            : extern void graph_trace_close(struct trace_iterator *iter);
     958                 :            : extern int __trace_graph_entry(struct trace_array *tr,
     959                 :            :                                struct ftrace_graph_ent *trace,
     960                 :            :                                unsigned long flags, int pc);
     961                 :            : extern void __trace_graph_return(struct trace_array *tr,
     962                 :            :                                  struct ftrace_graph_ret *trace,
     963                 :            :                                  unsigned long flags, int pc);
     964                 :            : 
     965                 :            : #ifdef CONFIG_DYNAMIC_FTRACE
     966                 :            : extern struct ftrace_hash __rcu *ftrace_graph_hash;
     967                 :            : extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash;
     968                 :            : 
     969                 :            : static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
     970                 :            : {
     971                 :            :         unsigned long addr = trace->func;
     972                 :            :         int ret = 0;
     973                 :            :         struct ftrace_hash *hash;
     974                 :            : 
     975                 :            :         preempt_disable_notrace();
     976                 :            : 
     977                 :            :         /*
     978                 :            :          * Have to open code "rcu_dereference_sched()" because the
     979                 :            :          * function graph tracer can be called when RCU is not
     980                 :            :          * "watching".
     981                 :            :          * Protected with schedule_on_each_cpu(ftrace_sync)
     982                 :            :          */
     983                 :            :         hash = rcu_dereference_protected(ftrace_graph_hash, !preemptible());
     984                 :            : 
     985                 :            :         if (ftrace_hash_empty(hash)) {
     986                 :            :                 ret = 1;
     987                 :            :                 goto out;
     988                 :            :         }
     989                 :            : 
     990                 :            :         if (ftrace_lookup_ip(hash, addr)) {
     991                 :            : 
     992                 :            :                 /*
     993                 :            :                  * This needs to be cleared on the return functions
     994                 :            :                  * when the depth is zero.
     995                 :            :                  */
     996                 :            :                 trace_recursion_set(TRACE_GRAPH_BIT);
     997                 :            :                 trace_recursion_set_depth(trace->depth);
     998                 :            : 
     999                 :            :                 /*
    1000                 :            :                  * If no irqs are to be traced, but a set_graph_function
    1001                 :            :                  * is set, and called by an interrupt handler, we still
    1002                 :            :                  * want to trace it.
    1003                 :            :                  */
    1004                 :            :                 if (in_irq())
    1005                 :            :                         trace_recursion_set(TRACE_IRQ_BIT);
    1006                 :            :                 else
    1007                 :            :                         trace_recursion_clear(TRACE_IRQ_BIT);
    1008                 :            :                 ret = 1;
    1009                 :            :         }
    1010                 :            : 
    1011                 :            : out:
    1012                 :            :         preempt_enable_notrace();
    1013                 :            :         return ret;
    1014                 :            : }
    1015                 :            : 
    1016                 :            : static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
    1017                 :            : {
    1018                 :            :         if (trace_recursion_test(TRACE_GRAPH_BIT) &&
    1019                 :            :             trace->depth == trace_recursion_depth())
    1020                 :            :                 trace_recursion_clear(TRACE_GRAPH_BIT);
    1021                 :            : }
    1022                 :            : 
    1023                 :            : static inline int ftrace_graph_notrace_addr(unsigned long addr)
    1024                 :            : {
    1025                 :            :         int ret = 0;
    1026                 :            :         struct ftrace_hash *notrace_hash;
    1027                 :            : 
    1028                 :            :         preempt_disable_notrace();
    1029                 :            : 
    1030                 :            :         /*
    1031                 :            :          * Have to open code "rcu_dereference_sched()" because the
    1032                 :            :          * function graph tracer can be called when RCU is not
    1033                 :            :          * "watching".
    1034                 :            :          * Protected with schedule_on_each_cpu(ftrace_sync)
    1035                 :            :          */
    1036                 :            :         notrace_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
    1037                 :            :                                                  !preemptible());
    1038                 :            : 
    1039                 :            :         if (ftrace_lookup_ip(notrace_hash, addr))
    1040                 :            :                 ret = 1;
    1041                 :            : 
    1042                 :            :         preempt_enable_notrace();
    1043                 :            :         return ret;
    1044                 :            : }
    1045                 :            : #else
    1046                 :            : static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
    1047                 :            : {
    1048                 :            :         return 1;
    1049                 :            : }
    1050                 :            : 
    1051                 :            : static inline int ftrace_graph_notrace_addr(unsigned long addr)
    1052                 :            : {
    1053                 :            :         return 0;
    1054                 :            : }
    1055                 :            : static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
    1056                 :            : { }
    1057                 :            : #endif /* CONFIG_DYNAMIC_FTRACE */
    1058                 :            : 
    1059                 :            : extern unsigned int fgraph_max_depth;
    1060                 :            : 
    1061                 :            : static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace)
    1062                 :            : {
    1063                 :            :         /* trace it when it is-nested-in or is a function enabled. */
    1064                 :            :         return !(trace_recursion_test(TRACE_GRAPH_BIT) ||
    1065                 :            :                  ftrace_graph_addr(trace)) ||
    1066                 :            :                 (trace->depth < 0) ||
    1067                 :            :                 (fgraph_max_depth && trace->depth >= fgraph_max_depth);
    1068                 :            : }
    1069                 :            : 
    1070                 :            : #else /* CONFIG_FUNCTION_GRAPH_TRACER */
    1071                 :            : static inline enum print_line_t
    1072                 :            : print_graph_function_flags(struct trace_iterator *iter, u32 flags)
    1073                 :            : {
    1074                 :            :         return TRACE_TYPE_UNHANDLED;
    1075                 :            : }
    1076                 :            : #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
    1077                 :            : 
    1078                 :            : extern struct list_head ftrace_pids;
    1079                 :            : 
    1080                 :            : #ifdef CONFIG_FUNCTION_TRACER
    1081                 :            : struct ftrace_func_command {
    1082                 :            :         struct list_head        list;
    1083                 :            :         char                    *name;
    1084                 :            :         int                     (*func)(struct trace_array *tr,
    1085                 :            :                                         struct ftrace_hash *hash,
    1086                 :            :                                         char *func, char *cmd,
    1087                 :            :                                         char *params, int enable);
    1088                 :            : };
    1089                 :            : extern bool ftrace_filter_param __initdata;
    1090                 :            : static inline int ftrace_trace_task(struct trace_array *tr)
    1091                 :            : {
    1092                 :            :         return !this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
    1093                 :            : }
    1094                 :            : extern int ftrace_is_dead(void);
    1095                 :            : int ftrace_create_function_files(struct trace_array *tr,
    1096                 :            :                                  struct dentry *parent);
    1097                 :            : void ftrace_destroy_function_files(struct trace_array *tr);
    1098                 :            : void ftrace_init_global_array_ops(struct trace_array *tr);
    1099                 :            : void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func);
    1100                 :            : void ftrace_reset_array_ops(struct trace_array *tr);
    1101                 :            : void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer);
    1102                 :            : void ftrace_init_tracefs_toplevel(struct trace_array *tr,
    1103                 :            :                                   struct dentry *d_tracer);
    1104                 :            : void ftrace_clear_pids(struct trace_array *tr);
    1105                 :            : int init_function_trace(void);
    1106                 :            : void ftrace_pid_follow_fork(struct trace_array *tr, bool enable);
    1107                 :            : #else
    1108                 :            : static inline int ftrace_trace_task(struct trace_array *tr)
    1109                 :            : {
    1110                 :            :         return 1;
    1111                 :            : }
    1112                 :          0 : static inline int ftrace_is_dead(void) { return 0; }
    1113                 :            : static inline int
    1114                 :          3 : ftrace_create_function_files(struct trace_array *tr,
    1115                 :            :                              struct dentry *parent)
    1116                 :            : {
    1117                 :          3 :         return 0;
    1118                 :            : }
    1119                 :          0 : static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
    1120                 :            : static inline __init void
    1121                 :          3 : ftrace_init_global_array_ops(struct trace_array *tr) { }
    1122                 :            : static inline void ftrace_reset_array_ops(struct trace_array *tr) { }
    1123                 :          3 : static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }
    1124                 :          3 : static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { }
    1125                 :          0 : static inline void ftrace_clear_pids(struct trace_array *tr) { }
    1126                 :          3 : static inline int init_function_trace(void) { return 0; }
    1127                 :            : static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { }
    1128                 :            : /* ftace_func_t type is not defined, use macro instead of static inline */
    1129                 :            : #define ftrace_init_array_ops(tr, func) do { } while (0)
    1130                 :            : #endif /* CONFIG_FUNCTION_TRACER */
    1131                 :            : 
    1132                 :            : #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
    1133                 :            : 
    1134                 :            : struct ftrace_probe_ops {
    1135                 :            :         void                    (*func)(unsigned long ip,
    1136                 :            :                                         unsigned long parent_ip,
    1137                 :            :                                         struct trace_array *tr,
    1138                 :            :                                         struct ftrace_probe_ops *ops,
    1139                 :            :                                         void *data);
    1140                 :            :         int                     (*init)(struct ftrace_probe_ops *ops,
    1141                 :            :                                         struct trace_array *tr,
    1142                 :            :                                         unsigned long ip, void *init_data,
    1143                 :            :                                         void **data);
    1144                 :            :         void                    (*free)(struct ftrace_probe_ops *ops,
    1145                 :            :                                         struct trace_array *tr,
    1146                 :            :                                         unsigned long ip, void *data);
    1147                 :            :         int                     (*print)(struct seq_file *m,
    1148                 :            :                                          unsigned long ip,
    1149                 :            :                                          struct ftrace_probe_ops *ops,
    1150                 :            :                                          void *data);
    1151                 :            : };
    1152                 :            : 
    1153                 :            : struct ftrace_func_mapper;
    1154                 :            : typedef int (*ftrace_mapper_func)(void *data);
    1155                 :            : 
    1156                 :            : struct ftrace_func_mapper *allocate_ftrace_func_mapper(void);
    1157                 :            : void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
    1158                 :            :                                            unsigned long ip);
    1159                 :            : int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
    1160                 :            :                                unsigned long ip, void *data);
    1161                 :            : void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
    1162                 :            :                                    unsigned long ip);
    1163                 :            : void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
    1164                 :            :                              ftrace_mapper_func free_func);
    1165                 :            : 
    1166                 :            : extern int
    1167                 :            : register_ftrace_function_probe(char *glob, struct trace_array *tr,
    1168                 :            :                                struct ftrace_probe_ops *ops, void *data);
    1169                 :            : extern int
    1170                 :            : unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
    1171                 :            :                                       struct ftrace_probe_ops *ops);
    1172                 :            : extern void clear_ftrace_function_probes(struct trace_array *tr);
    1173                 :            : 
    1174                 :            : int register_ftrace_command(struct ftrace_func_command *cmd);
    1175                 :            : int unregister_ftrace_command(struct ftrace_func_command *cmd);
    1176                 :            : 
    1177                 :            : void ftrace_create_filter_files(struct ftrace_ops *ops,
    1178                 :            :                                 struct dentry *parent);
    1179                 :            : void ftrace_destroy_filter_files(struct ftrace_ops *ops);
    1180                 :            : 
    1181                 :            : extern int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
    1182                 :            :                              int len, int reset);
    1183                 :            : extern int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
    1184                 :            :                               int len, int reset);
    1185                 :            : #else
    1186                 :            : struct ftrace_func_command;
    1187                 :            : 
    1188                 :            : static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)
    1189                 :            : {
    1190                 :            :         return -EINVAL;
    1191                 :            : }
    1192                 :            : static inline __init int unregister_ftrace_command(char *cmd_name)
    1193                 :            : {
    1194                 :            :         return -EINVAL;
    1195                 :            : }
    1196                 :          0 : static inline void clear_ftrace_function_probes(struct trace_array *tr)
    1197                 :            : {
    1198                 :          0 : }
    1199                 :            : 
    1200                 :            : /*
    1201                 :            :  * The ops parameter passed in is usually undefined.
    1202                 :            :  * This must be a macro.
    1203                 :            :  */
    1204                 :            : #define ftrace_create_filter_files(ops, parent) do { } while (0)
    1205                 :            : #define ftrace_destroy_filter_files(ops) do { } while (0)
    1206                 :            : #endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */
    1207                 :            : 
    1208                 :            : bool ftrace_event_is_function(struct trace_event_call *call);
    1209                 :            : 
    1210                 :            : /*
    1211                 :            :  * struct trace_parser - servers for reading the user input separated by spaces
    1212                 :            :  * @cont: set if the input is not complete - no final space char was found
    1213                 :            :  * @buffer: holds the parsed user input
    1214                 :            :  * @idx: user input length
    1215                 :            :  * @size: buffer size
    1216                 :            :  */
    1217                 :            : struct trace_parser {
    1218                 :            :         bool            cont;
    1219                 :            :         char            *buffer;
    1220                 :            :         unsigned        idx;
    1221                 :            :         unsigned        size;
    1222                 :            : };
    1223                 :            : 
    1224                 :          0 : static inline bool trace_parser_loaded(struct trace_parser *parser)
    1225                 :            : {
    1226         [ #  # ]:          0 :         return (parser->idx != 0);
    1227                 :            : }
    1228                 :            : 
    1229                 :            : static inline bool trace_parser_cont(struct trace_parser *parser)
    1230                 :            : {
    1231                 :            :         return parser->cont;
    1232                 :            : }
    1233                 :            : 
    1234                 :          0 : static inline void trace_parser_clear(struct trace_parser *parser)
    1235                 :            : {
    1236                 :          0 :         parser->cont = false;
    1237                 :          0 :         parser->idx = 0;
    1238                 :          0 : }
    1239                 :            : 
    1240                 :            : extern int trace_parser_get_init(struct trace_parser *parser, int size);
    1241                 :            : extern void trace_parser_put(struct trace_parser *parser);
    1242                 :            : extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
    1243                 :            :         size_t cnt, loff_t *ppos);
    1244                 :            : 
    1245                 :            : /*
    1246                 :            :  * Only create function graph options if function graph is configured.
    1247                 :            :  */
    1248                 :            : #ifdef CONFIG_FUNCTION_GRAPH_TRACER
    1249                 :            : # define FGRAPH_FLAGS                                           \
    1250                 :            :                 C(DISPLAY_GRAPH,        "display-graph"),
    1251                 :            : #else
    1252                 :            : # define FGRAPH_FLAGS
    1253                 :            : #endif
    1254                 :            : 
    1255                 :            : #ifdef CONFIG_BRANCH_TRACER
    1256                 :            : # define BRANCH_FLAGS                                   \
    1257                 :            :                 C(BRANCH,               "branch"),
    1258                 :            : #else
    1259                 :            : # define BRANCH_FLAGS
    1260                 :            : #endif
    1261                 :            : 
    1262                 :            : #ifdef CONFIG_FUNCTION_TRACER
    1263                 :            : # define FUNCTION_FLAGS                                         \
    1264                 :            :                 C(FUNCTION,             "function-trace"),    \
    1265                 :            :                 C(FUNC_FORK,            "function-fork"),
    1266                 :            : # define FUNCTION_DEFAULT_FLAGS         TRACE_ITER_FUNCTION
    1267                 :            : #else
    1268                 :            : # define FUNCTION_FLAGS
    1269                 :            : # define FUNCTION_DEFAULT_FLAGS         0UL
    1270                 :            : # define TRACE_ITER_FUNC_FORK           0UL
    1271                 :            : #endif
    1272                 :            : 
    1273                 :            : #ifdef CONFIG_STACKTRACE
    1274                 :            : # define STACK_FLAGS                            \
    1275                 :            :                 C(STACKTRACE,           "stacktrace"),
    1276                 :            : #else
    1277                 :            : # define STACK_FLAGS
    1278                 :            : #endif
    1279                 :            : 
    1280                 :            : /*
    1281                 :            :  * trace_iterator_flags is an enumeration that defines bit
    1282                 :            :  * positions into trace_flags that controls the output.
    1283                 :            :  *
    1284                 :            :  * NOTE: These bits must match the trace_options array in
    1285                 :            :  *       trace.c (this macro guarantees it).
    1286                 :            :  */
    1287                 :            : #define TRACE_FLAGS                                             \
    1288                 :            :                 C(PRINT_PARENT,         "print-parent"),      \
    1289                 :            :                 C(SYM_OFFSET,           "sym-offset"),                \
    1290                 :            :                 C(SYM_ADDR,             "sym-addr"),          \
    1291                 :            :                 C(VERBOSE,              "verbose"),           \
    1292                 :            :                 C(RAW,                  "raw"),                       \
    1293                 :            :                 C(HEX,                  "hex"),                       \
    1294                 :            :                 C(BIN,                  "bin"),                       \
    1295                 :            :                 C(BLOCK,                "block"),             \
    1296                 :            :                 C(PRINTK,               "trace_printk"),      \
    1297                 :            :                 C(ANNOTATE,             "annotate"),          \
    1298                 :            :                 C(USERSTACKTRACE,       "userstacktrace"),    \
    1299                 :            :                 C(SYM_USEROBJ,          "sym-userobj"),               \
    1300                 :            :                 C(PRINTK_MSGONLY,       "printk-msg-only"),   \
    1301                 :            :                 C(CONTEXT_INFO,         "context-info"),   /* Print pid/cpu/time */ \
    1302                 :            :                 C(LATENCY_FMT,          "latency-format"),    \
    1303                 :            :                 C(RECORD_CMD,           "record-cmd"),                \
    1304                 :            :                 C(RECORD_TGID,          "record-tgid"),               \
    1305                 :            :                 C(OVERWRITE,            "overwrite"),         \
    1306                 :            :                 C(STOP_ON_FREE,         "disable_on_free"),   \
    1307                 :            :                 C(IRQ_INFO,             "irq-info"),          \
    1308                 :            :                 C(MARKERS,              "markers"),           \
    1309                 :            :                 C(EVENT_FORK,           "event-fork"),                \
    1310                 :            :                 FUNCTION_FLAGS                                  \
    1311                 :            :                 FGRAPH_FLAGS                                    \
    1312                 :            :                 STACK_FLAGS                                     \
    1313                 :            :                 BRANCH_FLAGS
    1314                 :            : 
    1315                 :            : /*
    1316                 :            :  * By defining C, we can make TRACE_FLAGS a list of bit names
    1317                 :            :  * that will define the bits for the flag masks.
    1318                 :            :  */
    1319                 :            : #undef C
    1320                 :            : #define C(a, b) TRACE_ITER_##a##_BIT
    1321                 :            : 
    1322                 :            : enum trace_iterator_bits {
    1323                 :            :         TRACE_FLAGS
    1324                 :            :         /* Make sure we don't go more than we have bits for */
    1325                 :            :         TRACE_ITER_LAST_BIT
    1326                 :            : };
    1327                 :            : 
    1328                 :            : /*
    1329                 :            :  * By redefining C, we can make TRACE_FLAGS a list of masks that
    1330                 :            :  * use the bits as defined above.
    1331                 :            :  */
    1332                 :            : #undef C
    1333                 :            : #define C(a, b) TRACE_ITER_##a = (1 << TRACE_ITER_##a##_BIT)
    1334                 :            : 
    1335                 :            : enum trace_iterator_flags { TRACE_FLAGS };
    1336                 :            : 
    1337                 :            : /*
    1338                 :            :  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
    1339                 :            :  * control the output of kernel symbols.
    1340                 :            :  */
    1341                 :            : #define TRACE_ITER_SYM_MASK \
    1342                 :            :         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
    1343                 :            : 
    1344                 :            : extern struct tracer nop_trace;
    1345                 :            : 
    1346                 :            : #ifdef CONFIG_BRANCH_TRACER
    1347                 :            : extern int enable_branch_tracing(struct trace_array *tr);
    1348                 :            : extern void disable_branch_tracing(void);
    1349                 :            : static inline int trace_branch_enable(struct trace_array *tr)
    1350                 :            : {
    1351                 :            :         if (tr->trace_flags & TRACE_ITER_BRANCH)
    1352                 :            :                 return enable_branch_tracing(tr);
    1353                 :            :         return 0;
    1354                 :            : }
    1355                 :            : static inline void trace_branch_disable(void)
    1356                 :            : {
    1357                 :            :         /* due to races, always disable */
    1358                 :            :         disable_branch_tracing();
    1359                 :            : }
    1360                 :            : #else
    1361                 :          0 : static inline int trace_branch_enable(struct trace_array *tr)
    1362                 :            : {
    1363                 :          0 :         return 0;
    1364                 :            : }
    1365                 :          0 : static inline void trace_branch_disable(void)
    1366                 :            : {
    1367         [ #  # ]:          0 : }
    1368                 :            : #endif /* CONFIG_BRANCH_TRACER */
    1369                 :            : 
    1370                 :            : /* set ring buffers to default size if not already done so */
    1371                 :            : int tracing_update_buffers(void);
    1372                 :            : 
    1373                 :            : struct ftrace_event_field {
    1374                 :            :         struct list_head        link;
    1375                 :            :         const char              *name;
    1376                 :            :         const char              *type;
    1377                 :            :         int                     filter_type;
    1378                 :            :         int                     offset;
    1379                 :            :         int                     size;
    1380                 :            :         int                     is_signed;
    1381                 :            : };
    1382                 :            : 
    1383                 :            : struct prog_entry;
    1384                 :            : 
    1385                 :            : struct event_filter {
    1386                 :            :         struct prog_entry __rcu *prog;
    1387                 :            :         char                    *filter_string;
    1388                 :            : };
    1389                 :            : 
    1390                 :            : struct event_subsystem {
    1391                 :            :         struct list_head        list;
    1392                 :            :         const char              *name;
    1393                 :            :         struct event_filter     *filter;
    1394                 :            :         int                     ref_count;
    1395                 :            : };
    1396                 :            : 
    1397                 :            : struct trace_subsystem_dir {
    1398                 :            :         struct list_head                list;
    1399                 :            :         struct event_subsystem          *subsystem;
    1400                 :            :         struct trace_array              *tr;
    1401                 :            :         struct dentry                   *entry;
    1402                 :            :         int                             ref_count;
    1403                 :            :         int                             nr_events;
    1404                 :            : };
    1405                 :            : 
    1406                 :            : extern int call_filter_check_discard(struct trace_event_call *call, void *rec,
    1407                 :            :                                      struct trace_buffer *buffer,
    1408                 :            :                                      struct ring_buffer_event *event);
    1409                 :            : 
    1410                 :            : void trace_buffer_unlock_commit_regs(struct trace_array *tr,
    1411                 :            :                                      struct trace_buffer *buffer,
    1412                 :            :                                      struct ring_buffer_event *event,
    1413                 :            :                                      unsigned long flags, int pc,
    1414                 :            :                                      struct pt_regs *regs);
    1415                 :            : 
    1416                 :          0 : static inline void trace_buffer_unlock_commit(struct trace_array *tr,
    1417                 :            :                                               struct trace_buffer *buffer,
    1418                 :            :                                               struct ring_buffer_event *event,
    1419                 :            :                                               unsigned long flags, int pc)
    1420                 :            : {
    1421                 :          0 :         trace_buffer_unlock_commit_regs(tr, buffer, event, flags, pc, NULL);
    1422                 :          0 : }
    1423                 :            : 
    1424                 :            : DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
    1425                 :            : DECLARE_PER_CPU(int, trace_buffered_event_cnt);
    1426                 :            : void trace_buffered_event_disable(void);
    1427                 :            : void trace_buffered_event_enable(void);
    1428                 :            : 
    1429                 :            : static inline void
    1430                 :          0 : __trace_event_discard_commit(struct trace_buffer *buffer,
    1431                 :            :                              struct ring_buffer_event *event)
    1432                 :            : {
    1433   [ #  #  #  # ]:          0 :         if (this_cpu_read(trace_buffered_event) == event) {
    1434                 :            :                 /* Simply release the temp buffer */
    1435                 :          0 :                 this_cpu_dec(trace_buffered_event_cnt);
    1436                 :          0 :                 return;
    1437                 :            :         }
    1438                 :          0 :         ring_buffer_discard_commit(buffer, event);
    1439                 :            : }
    1440                 :            : 
    1441                 :            : /*
    1442                 :            :  * Helper function for event_trigger_unlock_commit{_regs}().
    1443                 :            :  * If there are event triggers attached to this event that requires
    1444                 :            :  * filtering against its fields, then they wil be called as the
    1445                 :            :  * entry already holds the field information of the current event.
    1446                 :            :  *
    1447                 :            :  * It also checks if the event should be discarded or not.
    1448                 :            :  * It is to be discarded if the event is soft disabled and the
    1449                 :            :  * event was only recorded to process triggers, or if the event
    1450                 :            :  * filter is active and this event did not match the filters.
    1451                 :            :  *
    1452                 :            :  * Returns true if the event is discarded, false otherwise.
    1453                 :            :  */
    1454                 :            : static inline bool
    1455                 :          0 : __event_trigger_test_discard(struct trace_event_file *file,
    1456                 :            :                              struct trace_buffer *buffer,
    1457                 :            :                              struct ring_buffer_event *event,
    1458                 :            :                              void *entry,
    1459                 :            :                              enum event_trigger_type *tt)
    1460                 :            : {
    1461                 :          0 :         unsigned long eflags = file->flags;
    1462                 :            : 
    1463         [ #  # ]:          0 :         if (eflags & EVENT_FILE_FL_TRIGGER_COND)
    1464                 :          0 :                 *tt = event_triggers_call(file, entry, event);
    1465                 :            : 
    1466         [ #  # ]:          0 :         if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
    1467   [ #  #  #  # ]:          0 :             (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
    1468                 :          0 :              !filter_match_preds(file->filter, entry))) {
    1469                 :          0 :                 __trace_event_discard_commit(buffer, event);
    1470                 :          0 :                 return true;
    1471                 :            :         }
    1472                 :            : 
    1473                 :            :         return false;
    1474                 :            : }
    1475                 :            : 
    1476                 :            : /**
    1477                 :            :  * event_trigger_unlock_commit - handle triggers and finish event commit
    1478                 :            :  * @file: The file pointer assoctiated to the event
    1479                 :            :  * @buffer: The ring buffer that the event is being written to
    1480                 :            :  * @event: The event meta data in the ring buffer
    1481                 :            :  * @entry: The event itself
    1482                 :            :  * @irq_flags: The state of the interrupts at the start of the event
    1483                 :            :  * @pc: The state of the preempt count at the start of the event.
    1484                 :            :  *
    1485                 :            :  * This is a helper function to handle triggers that require data
    1486                 :            :  * from the event itself. It also tests the event against filters and
    1487                 :            :  * if the event is soft disabled and should be discarded.
    1488                 :            :  */
    1489                 :            : static inline void
    1490                 :          0 : event_trigger_unlock_commit(struct trace_event_file *file,
    1491                 :            :                             struct trace_buffer *buffer,
    1492                 :            :                             struct ring_buffer_event *event,
    1493                 :            :                             void *entry, unsigned long irq_flags, int pc)
    1494                 :            : {
    1495                 :          0 :         enum event_trigger_type tt = ETT_NONE;
    1496                 :            : 
    1497         [ #  # ]:          0 :         if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
    1498                 :          0 :                 trace_buffer_unlock_commit(file->tr, buffer, event, irq_flags, pc);
    1499                 :            : 
    1500         [ #  # ]:          0 :         if (tt)
    1501                 :          0 :                 event_triggers_post_call(file, tt);
    1502                 :          0 : }
    1503                 :            : 
    1504                 :            : /**
    1505                 :            :  * event_trigger_unlock_commit_regs - handle triggers and finish event commit
    1506                 :            :  * @file: The file pointer assoctiated to the event
    1507                 :            :  * @buffer: The ring buffer that the event is being written to
    1508                 :            :  * @event: The event meta data in the ring buffer
    1509                 :            :  * @entry: The event itself
    1510                 :            :  * @irq_flags: The state of the interrupts at the start of the event
    1511                 :            :  * @pc: The state of the preempt count at the start of the event.
    1512                 :            :  *
    1513                 :            :  * This is a helper function to handle triggers that require data
    1514                 :            :  * from the event itself. It also tests the event against filters and
    1515                 :            :  * if the event is soft disabled and should be discarded.
    1516                 :            :  *
    1517                 :            :  * Same as event_trigger_unlock_commit() but calls
    1518                 :            :  * trace_buffer_unlock_commit_regs() instead of trace_buffer_unlock_commit().
    1519                 :            :  */
    1520                 :            : static inline void
    1521                 :          0 : event_trigger_unlock_commit_regs(struct trace_event_file *file,
    1522                 :            :                                  struct trace_buffer *buffer,
    1523                 :            :                                  struct ring_buffer_event *event,
    1524                 :            :                                  void *entry, unsigned long irq_flags, int pc,
    1525                 :            :                                  struct pt_regs *regs)
    1526                 :            : {
    1527                 :          0 :         enum event_trigger_type tt = ETT_NONE;
    1528                 :            : 
    1529         [ #  # ]:          0 :         if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
    1530                 :          0 :                 trace_buffer_unlock_commit_regs(file->tr, buffer, event,
    1531                 :            :                                                 irq_flags, pc, regs);
    1532                 :            : 
    1533         [ #  # ]:          0 :         if (tt)
    1534                 :          0 :                 event_triggers_post_call(file, tt);
    1535                 :          0 : }
    1536                 :            : 
    1537                 :            : #define FILTER_PRED_INVALID     ((unsigned short)-1)
    1538                 :            : #define FILTER_PRED_IS_RIGHT    (1 << 15)
    1539                 :            : #define FILTER_PRED_FOLD        (1 << 15)
    1540                 :            : 
    1541                 :            : /*
    1542                 :            :  * The max preds is the size of unsigned short with
    1543                 :            :  * two flags at the MSBs. One bit is used for both the IS_RIGHT
    1544                 :            :  * and FOLD flags. The other is reserved.
    1545                 :            :  *
    1546                 :            :  * 2^14 preds is way more than enough.
    1547                 :            :  */
    1548                 :            : #define MAX_FILTER_PRED         16384
    1549                 :            : 
    1550                 :            : struct filter_pred;
    1551                 :            : struct regex;
    1552                 :            : 
    1553                 :            : typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
    1554                 :            : 
    1555                 :            : typedef int (*regex_match_func)(char *str, struct regex *r, int len);
    1556                 :            : 
    1557                 :            : enum regex_type {
    1558                 :            :         MATCH_FULL = 0,
    1559                 :            :         MATCH_FRONT_ONLY,
    1560                 :            :         MATCH_MIDDLE_ONLY,
    1561                 :            :         MATCH_END_ONLY,
    1562                 :            :         MATCH_GLOB,
    1563                 :            :         MATCH_INDEX,
    1564                 :            : };
    1565                 :            : 
    1566                 :            : struct regex {
    1567                 :            :         char                    pattern[MAX_FILTER_STR_VAL];
    1568                 :            :         int                     len;
    1569                 :            :         int                     field_len;
    1570                 :            :         regex_match_func        match;
    1571                 :            : };
    1572                 :            : 
    1573                 :            : struct filter_pred {
    1574                 :            :         filter_pred_fn_t        fn;
    1575                 :            :         u64                     val;
    1576                 :            :         struct regex            regex;
    1577                 :            :         unsigned short          *ops;
    1578                 :            :         struct ftrace_event_field *field;
    1579                 :            :         int                     offset;
    1580                 :            :         int                     not;
    1581                 :            :         int                     op;
    1582                 :            : };
    1583                 :            : 
    1584                 :          0 : static inline bool is_string_field(struct ftrace_event_field *field)
    1585                 :            : {
    1586                 :          0 :         return field->filter_type == FILTER_DYN_STRING ||
    1587                 :            :                field->filter_type == FILTER_STATIC_STRING ||
    1588   [ #  #  #  # ]:          0 :                field->filter_type == FILTER_PTR_STRING ||
    1589                 :            :                field->filter_type == FILTER_COMM;
    1590                 :            : }
    1591                 :            : 
    1592                 :            : static inline bool is_function_field(struct ftrace_event_field *field)
    1593                 :            : {
    1594                 :            :         return field->filter_type == FILTER_TRACE_FN;
    1595                 :            : }
    1596                 :            : 
    1597                 :            : extern enum regex_type
    1598                 :            : filter_parse_regex(char *buff, int len, char **search, int *not);
    1599                 :            : extern void print_event_filter(struct trace_event_file *file,
    1600                 :            :                                struct trace_seq *s);
    1601                 :            : extern int apply_event_filter(struct trace_event_file *file,
    1602                 :            :                               char *filter_string);
    1603                 :            : extern int apply_subsystem_event_filter(struct trace_subsystem_dir *dir,
    1604                 :            :                                         char *filter_string);
    1605                 :            : extern void print_subsystem_event_filter(struct event_subsystem *system,
    1606                 :            :                                          struct trace_seq *s);
    1607                 :            : extern int filter_assign_type(const char *type);
    1608                 :            : extern int create_event_filter(struct trace_array *tr,
    1609                 :            :                                struct trace_event_call *call,
    1610                 :            :                                char *filter_str, bool set_str,
    1611                 :            :                                struct event_filter **filterp);
    1612                 :            : extern void free_event_filter(struct event_filter *filter);
    1613                 :            : 
    1614                 :            : struct ftrace_event_field *
    1615                 :            : trace_find_event_field(struct trace_event_call *call, char *name);
    1616                 :            : 
    1617                 :            : extern void trace_event_enable_cmd_record(bool enable);
    1618                 :            : extern void trace_event_enable_tgid_record(bool enable);
    1619                 :            : 
    1620                 :            : extern int event_trace_init(void);
    1621                 :            : extern int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr);
    1622                 :            : extern int event_trace_del_tracer(struct trace_array *tr);
    1623                 :            : 
    1624                 :            : extern struct trace_event_file *__find_event_file(struct trace_array *tr,
    1625                 :            :                                                   const char *system,
    1626                 :            :                                                   const char *event);
    1627                 :            : extern struct trace_event_file *find_event_file(struct trace_array *tr,
    1628                 :            :                                                 const char *system,
    1629                 :            :                                                 const char *event);
    1630                 :            : 
    1631                 :          0 : static inline void *event_file_data(struct file *filp)
    1632                 :            : {
    1633   [ #  #  #  #  :          0 :         return READ_ONCE(file_inode(filp)->i_private);
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    1634                 :            : }
    1635                 :            : 
    1636                 :            : extern struct mutex event_mutex;
    1637                 :            : extern struct list_head ftrace_events;
    1638                 :            : 
    1639                 :            : extern const struct file_operations event_trigger_fops;
    1640                 :            : extern const struct file_operations event_hist_fops;
    1641                 :            : extern const struct file_operations event_inject_fops;
    1642                 :            : 
    1643                 :            : #ifdef CONFIG_HIST_TRIGGERS
    1644                 :            : extern int register_trigger_hist_cmd(void);
    1645                 :            : extern int register_trigger_hist_enable_disable_cmds(void);
    1646                 :            : #else
    1647                 :          3 : static inline int register_trigger_hist_cmd(void) { return 0; }
    1648                 :          3 : static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
    1649                 :            : #endif
    1650                 :            : 
    1651                 :            : extern int register_trigger_cmds(void);
    1652                 :            : extern void clear_event_triggers(struct trace_array *tr);
    1653                 :            : 
    1654                 :            : struct event_trigger_data {
    1655                 :            :         unsigned long                   count;
    1656                 :            :         int                             ref;
    1657                 :            :         struct event_trigger_ops        *ops;
    1658                 :            :         struct event_command            *cmd_ops;
    1659                 :            :         struct event_filter __rcu       *filter;
    1660                 :            :         char                            *filter_str;
    1661                 :            :         void                            *private_data;
    1662                 :            :         bool                            paused;
    1663                 :            :         bool                            paused_tmp;
    1664                 :            :         struct list_head                list;
    1665                 :            :         char                            *name;
    1666                 :            :         struct list_head                named_list;
    1667                 :            :         struct event_trigger_data       *named_data;
    1668                 :            : };
    1669                 :            : 
    1670                 :            : /* Avoid typos */
    1671                 :            : #define ENABLE_EVENT_STR        "enable_event"
    1672                 :            : #define DISABLE_EVENT_STR       "disable_event"
    1673                 :            : #define ENABLE_HIST_STR         "enable_hist"
    1674                 :            : #define DISABLE_HIST_STR        "disable_hist"
    1675                 :            : 
    1676                 :            : struct enable_trigger_data {
    1677                 :            :         struct trace_event_file         *file;
    1678                 :            :         bool                            enable;
    1679                 :            :         bool                            hist;
    1680                 :            : };
    1681                 :            : 
    1682                 :            : extern int event_enable_trigger_print(struct seq_file *m,
    1683                 :            :                                       struct event_trigger_ops *ops,
    1684                 :            :                                       struct event_trigger_data *data);
    1685                 :            : extern void event_enable_trigger_free(struct event_trigger_ops *ops,
    1686                 :            :                                       struct event_trigger_data *data);
    1687                 :            : extern int event_enable_trigger_func(struct event_command *cmd_ops,
    1688                 :            :                                      struct trace_event_file *file,
    1689                 :            :                                      char *glob, char *cmd, char *param);
    1690                 :            : extern int event_enable_register_trigger(char *glob,
    1691                 :            :                                          struct event_trigger_ops *ops,
    1692                 :            :                                          struct event_trigger_data *data,
    1693                 :            :                                          struct trace_event_file *file);
    1694                 :            : extern void event_enable_unregister_trigger(char *glob,
    1695                 :            :                                             struct event_trigger_ops *ops,
    1696                 :            :                                             struct event_trigger_data *test,
    1697                 :            :                                             struct trace_event_file *file);
    1698                 :            : extern void trigger_data_free(struct event_trigger_data *data);
    1699                 :            : extern int event_trigger_init(struct event_trigger_ops *ops,
    1700                 :            :                               struct event_trigger_data *data);
    1701                 :            : extern int trace_event_trigger_enable_disable(struct trace_event_file *file,
    1702                 :            :                                               int trigger_enable);
    1703                 :            : extern void update_cond_flag(struct trace_event_file *file);
    1704                 :            : extern int set_trigger_filter(char *filter_str,
    1705                 :            :                               struct event_trigger_data *trigger_data,
    1706                 :            :                               struct trace_event_file *file);
    1707                 :            : extern struct event_trigger_data *find_named_trigger(const char *name);
    1708                 :            : extern bool is_named_trigger(struct event_trigger_data *test);
    1709                 :            : extern int save_named_trigger(const char *name,
    1710                 :            :                               struct event_trigger_data *data);
    1711                 :            : extern void del_named_trigger(struct event_trigger_data *data);
    1712                 :            : extern void pause_named_trigger(struct event_trigger_data *data);
    1713                 :            : extern void unpause_named_trigger(struct event_trigger_data *data);
    1714                 :            : extern void set_named_trigger_data(struct event_trigger_data *data,
    1715                 :            :                                    struct event_trigger_data *named_data);
    1716                 :            : extern struct event_trigger_data *
    1717                 :            : get_named_trigger_data(struct event_trigger_data *data);
    1718                 :            : extern int register_event_command(struct event_command *cmd);
    1719                 :            : extern int unregister_event_command(struct event_command *cmd);
    1720                 :            : extern int register_trigger_hist_enable_disable_cmds(void);
    1721                 :            : 
    1722                 :            : /**
    1723                 :            :  * struct event_trigger_ops - callbacks for trace event triggers
    1724                 :            :  *
    1725                 :            :  * The methods in this structure provide per-event trigger hooks for
    1726                 :            :  * various trigger operations.
    1727                 :            :  *
    1728                 :            :  * All the methods below, except for @init() and @free(), must be
    1729                 :            :  * implemented.
    1730                 :            :  *
    1731                 :            :  * @func: The trigger 'probe' function called when the triggering
    1732                 :            :  *      event occurs.  The data passed into this callback is the data
    1733                 :            :  *      that was supplied to the event_command @reg() function that
    1734                 :            :  *      registered the trigger (see struct event_command) along with
    1735                 :            :  *      the trace record, rec.
    1736                 :            :  *
    1737                 :            :  * @init: An optional initialization function called for the trigger
    1738                 :            :  *      when the trigger is registered (via the event_command reg()
    1739                 :            :  *      function).  This can be used to perform per-trigger
    1740                 :            :  *      initialization such as incrementing a per-trigger reference
    1741                 :            :  *      count, for instance.  This is usually implemented by the
    1742                 :            :  *      generic utility function @event_trigger_init() (see
    1743                 :            :  *      trace_event_triggers.c).
    1744                 :            :  *
    1745                 :            :  * @free: An optional de-initialization function called for the
    1746                 :            :  *      trigger when the trigger is unregistered (via the
    1747                 :            :  *      event_command @reg() function).  This can be used to perform
    1748                 :            :  *      per-trigger de-initialization such as decrementing a
    1749                 :            :  *      per-trigger reference count and freeing corresponding trigger
    1750                 :            :  *      data, for instance.  This is usually implemented by the
    1751                 :            :  *      generic utility function @event_trigger_free() (see
    1752                 :            :  *      trace_event_triggers.c).
    1753                 :            :  *
    1754                 :            :  * @print: The callback function invoked to have the trigger print
    1755                 :            :  *      itself.  This is usually implemented by a wrapper function
    1756                 :            :  *      that calls the generic utility function @event_trigger_print()
    1757                 :            :  *      (see trace_event_triggers.c).
    1758                 :            :  */
    1759                 :            : struct event_trigger_ops {
    1760                 :            :         void                    (*func)(struct event_trigger_data *data,
    1761                 :            :                                         void *rec,
    1762                 :            :                                         struct ring_buffer_event *rbe);
    1763                 :            :         int                     (*init)(struct event_trigger_ops *ops,
    1764                 :            :                                         struct event_trigger_data *data);
    1765                 :            :         void                    (*free)(struct event_trigger_ops *ops,
    1766                 :            :                                         struct event_trigger_data *data);
    1767                 :            :         int                     (*print)(struct seq_file *m,
    1768                 :            :                                          struct event_trigger_ops *ops,
    1769                 :            :                                          struct event_trigger_data *data);
    1770                 :            : };
    1771                 :            : 
    1772                 :            : /**
    1773                 :            :  * struct event_command - callbacks and data members for event commands
    1774                 :            :  *
    1775                 :            :  * Event commands are invoked by users by writing the command name
    1776                 :            :  * into the 'trigger' file associated with a trace event.  The
    1777                 :            :  * parameters associated with a specific invocation of an event
    1778                 :            :  * command are used to create an event trigger instance, which is
    1779                 :            :  * added to the list of trigger instances associated with that trace
    1780                 :            :  * event.  When the event is hit, the set of triggers associated with
    1781                 :            :  * that event is invoked.
    1782                 :            :  *
    1783                 :            :  * The data members in this structure provide per-event command data
    1784                 :            :  * for various event commands.
    1785                 :            :  *
    1786                 :            :  * All the data members below, except for @post_trigger, must be set
    1787                 :            :  * for each event command.
    1788                 :            :  *
    1789                 :            :  * @name: The unique name that identifies the event command.  This is
    1790                 :            :  *      the name used when setting triggers via trigger files.
    1791                 :            :  *
    1792                 :            :  * @trigger_type: A unique id that identifies the event command
    1793                 :            :  *      'type'.  This value has two purposes, the first to ensure that
    1794                 :            :  *      only one trigger of the same type can be set at a given time
    1795                 :            :  *      for a particular event e.g. it doesn't make sense to have both
    1796                 :            :  *      a traceon and traceoff trigger attached to a single event at
    1797                 :            :  *      the same time, so traceon and traceoff have the same type
    1798                 :            :  *      though they have different names.  The @trigger_type value is
    1799                 :            :  *      also used as a bit value for deferring the actual trigger
    1800                 :            :  *      action until after the current event is finished.  Some
    1801                 :            :  *      commands need to do this if they themselves log to the trace
    1802                 :            :  *      buffer (see the @post_trigger() member below).  @trigger_type
    1803                 :            :  *      values are defined by adding new values to the trigger_type
    1804                 :            :  *      enum in include/linux/trace_events.h.
    1805                 :            :  *
    1806                 :            :  * @flags: See the enum event_command_flags below.
    1807                 :            :  *
    1808                 :            :  * All the methods below, except for @set_filter() and @unreg_all(),
    1809                 :            :  * must be implemented.
    1810                 :            :  *
    1811                 :            :  * @func: The callback function responsible for parsing and
    1812                 :            :  *      registering the trigger written to the 'trigger' file by the
    1813                 :            :  *      user.  It allocates the trigger instance and registers it with
    1814                 :            :  *      the appropriate trace event.  It makes use of the other
    1815                 :            :  *      event_command callback functions to orchestrate this, and is
    1816                 :            :  *      usually implemented by the generic utility function
    1817                 :            :  *      @event_trigger_callback() (see trace_event_triggers.c).
    1818                 :            :  *
    1819                 :            :  * @reg: Adds the trigger to the list of triggers associated with the
    1820                 :            :  *      event, and enables the event trigger itself, after
    1821                 :            :  *      initializing it (via the event_trigger_ops @init() function).
    1822                 :            :  *      This is also where commands can use the @trigger_type value to
    1823                 :            :  *      make the decision as to whether or not multiple instances of
    1824                 :            :  *      the trigger should be allowed.  This is usually implemented by
    1825                 :            :  *      the generic utility function @register_trigger() (see
    1826                 :            :  *      trace_event_triggers.c).
    1827                 :            :  *
    1828                 :            :  * @unreg: Removes the trigger from the list of triggers associated
    1829                 :            :  *      with the event, and disables the event trigger itself, after
    1830                 :            :  *      initializing it (via the event_trigger_ops @free() function).
    1831                 :            :  *      This is usually implemented by the generic utility function
    1832                 :            :  *      @unregister_trigger() (see trace_event_triggers.c).
    1833                 :            :  *
    1834                 :            :  * @unreg_all: An optional function called to remove all the triggers
    1835                 :            :  *      from the list of triggers associated with the event.  Called
    1836                 :            :  *      when a trigger file is opened in truncate mode.
    1837                 :            :  *
    1838                 :            :  * @set_filter: An optional function called to parse and set a filter
    1839                 :            :  *      for the trigger.  If no @set_filter() method is set for the
    1840                 :            :  *      event command, filters set by the user for the command will be
    1841                 :            :  *      ignored.  This is usually implemented by the generic utility
    1842                 :            :  *      function @set_trigger_filter() (see trace_event_triggers.c).
    1843                 :            :  *
    1844                 :            :  * @get_trigger_ops: The callback function invoked to retrieve the
    1845                 :            :  *      event_trigger_ops implementation associated with the command.
    1846                 :            :  */
    1847                 :            : struct event_command {
    1848                 :            :         struct list_head        list;
    1849                 :            :         char                    *name;
    1850                 :            :         enum event_trigger_type trigger_type;
    1851                 :            :         int                     flags;
    1852                 :            :         int                     (*func)(struct event_command *cmd_ops,
    1853                 :            :                                         struct trace_event_file *file,
    1854                 :            :                                         char *glob, char *cmd, char *params);
    1855                 :            :         int                     (*reg)(char *glob,
    1856                 :            :                                        struct event_trigger_ops *ops,
    1857                 :            :                                        struct event_trigger_data *data,
    1858                 :            :                                        struct trace_event_file *file);
    1859                 :            :         void                    (*unreg)(char *glob,
    1860                 :            :                                          struct event_trigger_ops *ops,
    1861                 :            :                                          struct event_trigger_data *data,
    1862                 :            :                                          struct trace_event_file *file);
    1863                 :            :         void                    (*unreg_all)(struct trace_event_file *file);
    1864                 :            :         int                     (*set_filter)(char *filter_str,
    1865                 :            :                                               struct event_trigger_data *data,
    1866                 :            :                                               struct trace_event_file *file);
    1867                 :            :         struct event_trigger_ops *(*get_trigger_ops)(char *cmd, char *param);
    1868                 :            : };
    1869                 :            : 
    1870                 :            : /**
    1871                 :            :  * enum event_command_flags - flags for struct event_command
    1872                 :            :  *
    1873                 :            :  * @POST_TRIGGER: A flag that says whether or not this command needs
    1874                 :            :  *      to have its action delayed until after the current event has
    1875                 :            :  *      been closed.  Some triggers need to avoid being invoked while
    1876                 :            :  *      an event is currently in the process of being logged, since
    1877                 :            :  *      the trigger may itself log data into the trace buffer.  Thus
    1878                 :            :  *      we make sure the current event is committed before invoking
    1879                 :            :  *      those triggers.  To do that, the trigger invocation is split
    1880                 :            :  *      in two - the first part checks the filter using the current
    1881                 :            :  *      trace record; if a command has the @post_trigger flag set, it
    1882                 :            :  *      sets a bit for itself in the return value, otherwise it
    1883                 :            :  *      directly invokes the trigger.  Once all commands have been
    1884                 :            :  *      either invoked or set their return flag, the current record is
    1885                 :            :  *      either committed or discarded.  At that point, if any commands
    1886                 :            :  *      have deferred their triggers, those commands are finally
    1887                 :            :  *      invoked following the close of the current event.  In other
    1888                 :            :  *      words, if the event_trigger_ops @func() probe implementation
    1889                 :            :  *      itself logs to the trace buffer, this flag should be set,
    1890                 :            :  *      otherwise it can be left unspecified.
    1891                 :            :  *
    1892                 :            :  * @NEEDS_REC: A flag that says whether or not this command needs
    1893                 :            :  *      access to the trace record in order to perform its function,
    1894                 :            :  *      regardless of whether or not it has a filter associated with
    1895                 :            :  *      it (filters make a trigger require access to the trace record
    1896                 :            :  *      but are not always present).
    1897                 :            :  */
    1898                 :            : enum event_command_flags {
    1899                 :            :         EVENT_CMD_FL_POST_TRIGGER       = 1,
    1900                 :            :         EVENT_CMD_FL_NEEDS_REC          = 2,
    1901                 :            : };
    1902                 :            : 
    1903                 :          0 : static inline bool event_command_post_trigger(struct event_command *cmd_ops)
    1904                 :            : {
    1905   [ #  #  #  # ]:          0 :         return cmd_ops->flags & EVENT_CMD_FL_POST_TRIGGER;
    1906                 :            : }
    1907                 :            : 
    1908                 :          0 : static inline bool event_command_needs_rec(struct event_command *cmd_ops)
    1909                 :            : {
    1910         [ #  # ]:          0 :         return cmd_ops->flags & EVENT_CMD_FL_NEEDS_REC;
    1911                 :            : }
    1912                 :            : 
    1913                 :            : extern int trace_event_enable_disable(struct trace_event_file *file,
    1914                 :            :                                       int enable, int soft_disable);
    1915                 :            : extern int tracing_alloc_snapshot(void);
    1916                 :            : extern void tracing_snapshot_cond(struct trace_array *tr, void *cond_data);
    1917                 :            : extern int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update);
    1918                 :            : 
    1919                 :            : extern int tracing_snapshot_cond_disable(struct trace_array *tr);
    1920                 :            : extern void *tracing_cond_snapshot_data(struct trace_array *tr);
    1921                 :            : 
    1922                 :            : extern const char *__start___trace_bprintk_fmt[];
    1923                 :            : extern const char *__stop___trace_bprintk_fmt[];
    1924                 :            : 
    1925                 :            : extern const char *__start___tracepoint_str[];
    1926                 :            : extern const char *__stop___tracepoint_str[];
    1927                 :            : 
    1928                 :            : void trace_printk_control(bool enabled);
    1929                 :            : void trace_printk_start_comm(void);
    1930                 :            : int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
    1931                 :            : int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
    1932                 :            : 
    1933                 :            : /* Used from boot time tracer */
    1934                 :            : extern int trace_set_options(struct trace_array *tr, char *option);
    1935                 :            : extern int tracing_set_tracer(struct trace_array *tr, const char *buf);
    1936                 :            : extern ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
    1937                 :            :                                           unsigned long size, int cpu_id);
    1938                 :            : extern int tracing_set_cpumask(struct trace_array *tr,
    1939                 :            :                                 cpumask_var_t tracing_cpumask_new);
    1940                 :            : 
    1941                 :            : 
    1942                 :            : #define MAX_EVENT_NAME_LEN      64
    1943                 :            : 
    1944                 :            : extern int trace_run_command(const char *buf, int (*createfn)(int, char**));
    1945                 :            : extern ssize_t trace_parse_run_command(struct file *file,
    1946                 :            :                 const char __user *buffer, size_t count, loff_t *ppos,
    1947                 :            :                 int (*createfn)(int, char**));
    1948                 :            : 
    1949                 :            : extern unsigned int err_pos(char *cmd, const char *str);
    1950                 :            : extern void tracing_log_err(struct trace_array *tr,
    1951                 :            :                             const char *loc, const char *cmd,
    1952                 :            :                             const char **errs, u8 type, u8 pos);
    1953                 :            : 
    1954                 :            : /*
    1955                 :            :  * Normal trace_printk() and friends allocates special buffers
    1956                 :            :  * to do the manipulation, as well as saves the print formats
    1957                 :            :  * into sections to display. But the trace infrastructure wants
    1958                 :            :  * to use these without the added overhead at the price of being
    1959                 :            :  * a bit slower (used mainly for warnings, where we don't care
    1960                 :            :  * about performance). The internal_trace_puts() is for such
    1961                 :            :  * a purpose.
    1962                 :            :  */
    1963                 :            : #define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
    1964                 :            : 
    1965                 :            : #undef FTRACE_ENTRY
    1966                 :            : #define FTRACE_ENTRY(call, struct_name, id, tstruct, print)     \
    1967                 :            :         extern struct trace_event_call                                  \
    1968                 :            :         __aligned(4) event_##call;
    1969                 :            : #undef FTRACE_ENTRY_DUP
    1970                 :            : #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \
    1971                 :            :         FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
    1972                 :            : #undef FTRACE_ENTRY_PACKED
    1973                 :            : #define FTRACE_ENTRY_PACKED(call, struct_name, id, tstruct, print) \
    1974                 :            :         FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
    1975                 :            : 
    1976                 :            : #include "trace_entries.h"
    1977                 :            : 
    1978                 :            : #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_FUNCTION_TRACER)
    1979                 :            : int perf_ftrace_event_register(struct trace_event_call *call,
    1980                 :            :                                enum trace_reg type, void *data);
    1981                 :            : #else
    1982                 :            : #define perf_ftrace_event_register NULL
    1983                 :            : #endif
    1984                 :            : 
    1985                 :            : #ifdef CONFIG_FTRACE_SYSCALLS
    1986                 :            : void init_ftrace_syscalls(void);
    1987                 :            : const char *get_syscall_name(int syscall);
    1988                 :            : #else
    1989                 :          3 : static inline void init_ftrace_syscalls(void) { }
    1990                 :            : static inline const char *get_syscall_name(int syscall)
    1991                 :            : {
    1992                 :            :         return NULL;
    1993                 :            : }
    1994                 :            : #endif
    1995                 :            : 
    1996                 :            : #ifdef CONFIG_EVENT_TRACING
    1997                 :            : void trace_event_init(void);
    1998                 :            : void trace_event_eval_update(struct trace_eval_map **map, int len);
    1999                 :            : /* Used from boot time tracer */
    2000                 :            : extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set);
    2001                 :            : extern int trigger_process_regex(struct trace_event_file *file, char *buff);
    2002                 :            : #else
    2003                 :            : static inline void __init trace_event_init(void) { }
    2004                 :            : static inline void trace_event_eval_update(struct trace_eval_map **map, int len) { }
    2005                 :            : #endif
    2006                 :            : 
    2007                 :            : #ifdef CONFIG_TRACER_SNAPSHOT
    2008                 :            : void tracing_snapshot_instance(struct trace_array *tr);
    2009                 :            : int tracing_alloc_snapshot_instance(struct trace_array *tr);
    2010                 :            : #else
    2011                 :            : static inline void tracing_snapshot_instance(struct trace_array *tr) { }
    2012                 :            : static inline int tracing_alloc_snapshot_instance(struct trace_array *tr)
    2013                 :            : {
    2014                 :            :         return 0;
    2015                 :            : }
    2016                 :            : #endif
    2017                 :            : 
    2018                 :            : #ifdef CONFIG_PREEMPT_TRACER
    2019                 :            : void tracer_preempt_on(unsigned long a0, unsigned long a1);
    2020                 :            : void tracer_preempt_off(unsigned long a0, unsigned long a1);
    2021                 :            : #else
    2022                 :            : static inline void tracer_preempt_on(unsigned long a0, unsigned long a1) { }
    2023                 :            : static inline void tracer_preempt_off(unsigned long a0, unsigned long a1) { }
    2024                 :            : #endif
    2025                 :            : #ifdef CONFIG_IRQSOFF_TRACER
    2026                 :            : void tracer_hardirqs_on(unsigned long a0, unsigned long a1);
    2027                 :            : void tracer_hardirqs_off(unsigned long a0, unsigned long a1);
    2028                 :            : #else
    2029                 :            : static inline void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { }
    2030                 :            : static inline void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { }
    2031                 :            : #endif
    2032                 :            : 
    2033                 :            : extern struct trace_iterator *tracepoint_print_iter;
    2034                 :            : 
    2035                 :            : /*
    2036                 :            :  * Reset the state of the trace_iterator so that it can read consumed data.
    2037                 :            :  * Normally, the trace_iterator is used for reading the data when it is not
    2038                 :            :  * consumed, and must retain state.
    2039                 :            :  */
    2040                 :          0 : static __always_inline void trace_iterator_reset(struct trace_iterator *iter)
    2041                 :            : {
    2042                 :          0 :         const size_t offset = offsetof(struct trace_iterator, seq);
    2043                 :            : 
    2044                 :            :         /*
    2045                 :            :          * Keep gcc from complaining about overwriting more than just one
    2046                 :            :          * member in the structure.
    2047                 :            :          */
    2048                 :          0 :         memset((char *)iter + offset, 0, sizeof(struct trace_iterator) - offset);
    2049                 :            : 
    2050                 :          0 :         iter->pos = -1;
    2051                 :            : }
    2052                 :            : 
    2053                 :            : #endif /* _LINUX_KERNEL_TRACE_H */

Generated by: LCOV version 1.14