1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/notifier.h>
23 #include <linux/irqflags.h>
24 #include <linux/debugfs.h>
25 #include <linux/tracefs.h>
26 #include <linux/pagemap.h>
27 #include <linux/hardirq.h>
28 #include <linux/linkage.h>
29 #include <linux/uaccess.h>
30 #include <linux/vmalloc.h>
31 #include <linux/ftrace.h>
32 #include <linux/module.h>
33 #include <linux/percpu.h>
34 #include <linux/splice.h>
35 #include <linux/kdebug.h>
36 #include <linux/string.h>
37 #include <linux/mount.h>
38 #include <linux/rwsem.h>
39 #include <linux/slab.h>
40 #include <linux/ctype.h>
41 #include <linux/init.h>
42 #include <linux/poll.h>
43 #include <linux/nmi.h>
45 #include <linux/trace.h>
46 #include <linux/sched/clock.h>
47 #include <linux/sched/rt.h>
48 #include <linux/fsnotify.h>
49 #include <linux/irq_work.h>
50 #include <linux/workqueue.h>
53 #include "trace_output.h"
56 * On boot up, the ring buffer is set to the minimum size, so that
57 * we do not waste memory on systems that are not using tracing.
59 bool ring_buffer_expanded;
62 * We need to change this state when a selftest is running.
63 * A selftest will lurk into the ring-buffer to count the
64 * entries inserted during the selftest although some concurrent
65 * insertions into the ring-buffer such as trace_printk could occurred
66 * at the same time, giving false positive or negative results.
68 static bool __read_mostly tracing_selftest_running;
71 * If a tracer is running, we do not want to run SELFTEST.
73 bool __read_mostly tracing_selftest_disabled;
75 /* Pipe tracepoints to printk */
76 struct trace_iterator *tracepoint_print_iter;
77 int tracepoint_printk;
78 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
80 /* For tracers that don't implement custom flags */
81 static struct tracer_opt dummy_tracer_opt[] = {
86 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
92 * To prevent the comm cache from being overwritten when no
93 * tracing is active, only save the comm when a trace event
96 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
99 * Kill all tracing for good (never come back).
100 * It is initialized to 1 but will turn to zero if the initialization
101 * of the tracer is successful. But that is the only place that sets
104 static int tracing_disabled = 1;
106 cpumask_var_t __read_mostly tracing_buffer_mask;
109 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
112 * is set, then ftrace_dump is called. This will output the contents
113 * of the ftrace buffers to the console. This is very useful for
114 * capturing traces that lead to crashes and outputing it to a
117 * It is default off, but you can enable it with either specifying
118 * "ftrace_dump_on_oops" in the kernel command line, or setting
119 * /proc/sys/kernel/ftrace_dump_on_oops
120 * Set 1 if you want to dump buffers of all CPUs
121 * Set 2 if you want to dump the buffer of the CPU that triggered oops
124 enum ftrace_dump_mode ftrace_dump_on_oops;
126 /* When set, tracing will stop when a WARN*() is hit */
127 int __disable_trace_on_warning;
129 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
130 /* Map of enums to their values, for "eval_map" file */
131 struct trace_eval_map_head {
133 unsigned long length;
136 union trace_eval_map_item;
138 struct trace_eval_map_tail {
140 * "end" is first and points to NULL as it must be different
141 * than "mod" or "eval_string"
143 union trace_eval_map_item *next;
144 const char *end; /* points to NULL */
147 static DEFINE_MUTEX(trace_eval_mutex);
150 * The trace_eval_maps are saved in an array with two extra elements,
151 * one at the beginning, and one at the end. The beginning item contains
152 * the count of the saved maps (head.length), and the module they
153 * belong to if not built in (head.mod). The ending item contains a
154 * pointer to the next array of saved eval_map items.
156 union trace_eval_map_item {
157 struct trace_eval_map map;
158 struct trace_eval_map_head head;
159 struct trace_eval_map_tail tail;
162 static union trace_eval_map_item *trace_eval_maps;
163 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
165 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
166 static void ftrace_trace_userstack(struct ring_buffer *buffer,
167 unsigned long flags, int pc);
169 #define MAX_TRACER_SIZE 100
170 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
171 static char *default_bootup_tracer;
173 static bool allocate_snapshot;
175 static int __init set_cmdline_ftrace(char *str)
177 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
178 default_bootup_tracer = bootup_tracer_buf;
179 /* We are using ftrace early, expand it */
180 ring_buffer_expanded = true;
183 __setup("ftrace=", set_cmdline_ftrace);
185 static int __init set_ftrace_dump_on_oops(char *str)
187 if (*str++ != '=' || !*str) {
188 ftrace_dump_on_oops = DUMP_ALL;
192 if (!strcmp("orig_cpu", str)) {
193 ftrace_dump_on_oops = DUMP_ORIG;
199 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
201 static int __init stop_trace_on_warning(char *str)
203 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
204 __disable_trace_on_warning = 1;
207 __setup("traceoff_on_warning", stop_trace_on_warning);
209 static int __init boot_alloc_snapshot(char *str)
211 allocate_snapshot = true;
212 /* We also need the main ring buffer expanded */
213 ring_buffer_expanded = true;
216 __setup("alloc_snapshot", boot_alloc_snapshot);
219 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
221 static int __init set_trace_boot_options(char *str)
223 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
226 __setup("trace_options=", set_trace_boot_options);
228 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
229 static char *trace_boot_clock __initdata;
231 static int __init set_trace_boot_clock(char *str)
233 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
234 trace_boot_clock = trace_boot_clock_buf;
237 __setup("trace_clock=", set_trace_boot_clock);
239 static int __init set_tracepoint_printk(char *str)
241 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
242 tracepoint_printk = 1;
245 __setup("tp_printk", set_tracepoint_printk);
247 unsigned long long ns2usecs(u64 nsec)
254 /* trace_flags holds trace_options default values */
255 #define TRACE_DEFAULT_FLAGS \
256 (FUNCTION_DEFAULT_FLAGS | \
257 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
258 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
259 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
260 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
262 /* trace_options that are only supported by global_trace */
263 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
264 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
266 /* trace_flags that are default zero for instances */
267 #define ZEROED_TRACE_FLAGS \
268 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
271 * The global_trace is the descriptor that holds the top-level tracing
272 * buffers for the live tracing.
274 static struct trace_array global_trace = {
275 .trace_flags = TRACE_DEFAULT_FLAGS,
278 LIST_HEAD(ftrace_trace_arrays);
280 int trace_array_get(struct trace_array *this_tr)
282 struct trace_array *tr;
285 mutex_lock(&trace_types_lock);
286 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
293 mutex_unlock(&trace_types_lock);
298 static void __trace_array_put(struct trace_array *this_tr)
300 WARN_ON(!this_tr->ref);
305 * trace_array_put - Decrement the reference counter for this trace array.
307 * NOTE: Use this when we no longer need the trace array returned by
308 * trace_array_get_by_name(). This ensures the trace array can be later
312 void trace_array_put(struct trace_array *this_tr)
317 mutex_lock(&trace_types_lock);
318 __trace_array_put(this_tr);
319 mutex_unlock(&trace_types_lock);
321 EXPORT_SYMBOL_GPL(trace_array_put);
323 int tracing_check_open_get_tr(struct trace_array *tr)
327 ret = security_locked_down(LOCKDOWN_TRACEFS);
331 if (tracing_disabled)
334 if (tr && trace_array_get(tr) < 0)
340 int call_filter_check_discard(struct trace_event_call *call, void *rec,
341 struct ring_buffer *buffer,
342 struct ring_buffer_event *event)
344 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
345 !filter_match_preds(call->filter, rec)) {
346 __trace_event_discard_commit(buffer, event);
353 void trace_free_pid_list(struct trace_pid_list *pid_list)
355 vfree(pid_list->pids);
360 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
361 * @filtered_pids: The list of pids to check
362 * @search_pid: The PID to find in @filtered_pids
364 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
367 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
370 * If pid_max changed after filtered_pids was created, we
371 * by default ignore all pids greater than the previous pid_max.
373 if (search_pid >= filtered_pids->pid_max)
376 return test_bit(search_pid, filtered_pids->pids);
380 * trace_ignore_this_task - should a task be ignored for tracing
381 * @filtered_pids: The list of pids to check
382 * @task: The task that should be ignored if not filtered
384 * Checks if @task should be traced or not from @filtered_pids.
385 * Returns true if @task should *NOT* be traced.
386 * Returns false if @task should be traced.
389 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
392 * Return false, because if filtered_pids does not exist,
393 * all pids are good to trace.
398 return !trace_find_filtered_pid(filtered_pids, task->pid);
402 * trace_filter_add_remove_task - Add or remove a task from a pid_list
403 * @pid_list: The list to modify
404 * @self: The current task for fork or NULL for exit
405 * @task: The task to add or remove
407 * If adding a task, if @self is defined, the task is only added if @self
408 * is also included in @pid_list. This happens on fork and tasks should
409 * only be added when the parent is listed. If @self is NULL, then the
410 * @task pid will be removed from the list, which would happen on exit
413 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
414 struct task_struct *self,
415 struct task_struct *task)
420 /* For forks, we only add if the forking task is listed */
422 if (!trace_find_filtered_pid(pid_list, self->pid))
426 /* Sorry, but we don't support pid_max changing after setting */
427 if (task->pid >= pid_list->pid_max)
430 /* "self" is set for forks, and NULL for exits */
432 set_bit(task->pid, pid_list->pids);
434 clear_bit(task->pid, pid_list->pids);
438 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
439 * @pid_list: The pid list to show
440 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
441 * @pos: The position of the file
443 * This is used by the seq_file "next" operation to iterate the pids
444 * listed in a trace_pid_list structure.
446 * Returns the pid+1 as we want to display pid of zero, but NULL would
447 * stop the iteration.
449 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
451 unsigned long pid = (unsigned long)v;
455 /* pid already is +1 of the actual prevous bit */
456 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
458 /* Return pid + 1 to allow zero to be represented */
459 if (pid < pid_list->pid_max)
460 return (void *)(pid + 1);
466 * trace_pid_start - Used for seq_file to start reading pid lists
467 * @pid_list: The pid list to show
468 * @pos: The position of the file
470 * This is used by seq_file "start" operation to start the iteration
473 * Returns the pid+1 as we want to display pid of zero, but NULL would
474 * stop the iteration.
476 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
481 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
482 if (pid >= pid_list->pid_max)
485 /* Return pid + 1 so that zero can be the exit value */
486 for (pid++; pid && l < *pos;
487 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
493 * trace_pid_show - show the current pid in seq_file processing
494 * @m: The seq_file structure to write into
495 * @v: A void pointer of the pid (+1) value to display
497 * Can be directly used by seq_file operations to display the current
500 int trace_pid_show(struct seq_file *m, void *v)
502 unsigned long pid = (unsigned long)v - 1;
504 seq_printf(m, "%lu\n", pid);
508 /* 128 should be much more than enough */
509 #define PID_BUF_SIZE 127
511 int trace_pid_write(struct trace_pid_list *filtered_pids,
512 struct trace_pid_list **new_pid_list,
513 const char __user *ubuf, size_t cnt)
515 struct trace_pid_list *pid_list;
516 struct trace_parser parser;
524 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
528 * Always recreate a new array. The write is an all or nothing
529 * operation. Always create a new array when adding new pids by
530 * the user. If the operation fails, then the current list is
533 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
535 trace_parser_put(&parser);
539 pid_list->pid_max = READ_ONCE(pid_max);
541 /* Only truncating will shrink pid_max */
542 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
543 pid_list->pid_max = filtered_pids->pid_max;
545 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
546 if (!pid_list->pids) {
547 trace_parser_put(&parser);
553 /* copy the current bits to the new max */
554 for_each_set_bit(pid, filtered_pids->pids,
555 filtered_pids->pid_max) {
556 set_bit(pid, pid_list->pids);
565 ret = trace_get_user(&parser, ubuf, cnt, &pos);
566 if (ret < 0 || !trace_parser_loaded(&parser))
574 if (kstrtoul(parser.buffer, 0, &val))
576 if (val >= pid_list->pid_max)
581 set_bit(pid, pid_list->pids);
584 trace_parser_clear(&parser);
587 trace_parser_put(&parser);
590 trace_free_pid_list(pid_list);
595 /* Cleared the list of pids */
596 trace_free_pid_list(pid_list);
601 *new_pid_list = pid_list;
606 static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
610 /* Early boot up does not have a buffer yet */
612 return trace_clock_local();
614 ts = ring_buffer_time_stamp(buf->buffer, cpu);
615 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
620 u64 ftrace_now(int cpu)
622 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
626 * tracing_is_enabled - Show if global_trace has been disabled
628 * Shows if the global trace has been enabled or not. It uses the
629 * mirror flag "buffer_disabled" to be used in fast paths such as for
630 * the irqsoff tracer. But it may be inaccurate due to races. If you
631 * need to know the accurate state, use tracing_is_on() which is a little
632 * slower, but accurate.
634 int tracing_is_enabled(void)
637 * For quick access (irqsoff uses this in fast path), just
638 * return the mirror variable of the state of the ring buffer.
639 * It's a little racy, but we don't really care.
642 return !global_trace.buffer_disabled;
646 * trace_buf_size is the size in bytes that is allocated
647 * for a buffer. Note, the number of bytes is always rounded
650 * This number is purposely set to a low number of 16384.
651 * If the dump on oops happens, it will be much appreciated
652 * to not have to wait for all that output. Anyway this can be
653 * boot time and run time configurable.
655 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
657 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
659 /* trace_types holds a link list of available tracers. */
660 static struct tracer *trace_types __read_mostly;
663 * trace_types_lock is used to protect the trace_types list.
665 DEFINE_MUTEX(trace_types_lock);
668 * serialize the access of the ring buffer
670 * ring buffer serializes readers, but it is low level protection.
671 * The validity of the events (which returns by ring_buffer_peek() ..etc)
672 * are not protected by ring buffer.
674 * The content of events may become garbage if we allow other process consumes
675 * these events concurrently:
676 * A) the page of the consumed events may become a normal page
677 * (not reader page) in ring buffer, and this page will be rewrited
678 * by events producer.
679 * B) The page of the consumed events may become a page for splice_read,
680 * and this page will be returned to system.
682 * These primitives allow multi process access to different cpu ring buffer
685 * These primitives don't distinguish read-only and read-consume access.
686 * Multi read-only access are also serialized.
690 static DECLARE_RWSEM(all_cpu_access_lock);
691 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
693 static inline void trace_access_lock(int cpu)
695 if (cpu == RING_BUFFER_ALL_CPUS) {
696 /* gain it for accessing the whole ring buffer. */
697 down_write(&all_cpu_access_lock);
699 /* gain it for accessing a cpu ring buffer. */
701 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
702 down_read(&all_cpu_access_lock);
704 /* Secondly block other access to this @cpu ring buffer. */
705 mutex_lock(&per_cpu(cpu_access_lock, cpu));
709 static inline void trace_access_unlock(int cpu)
711 if (cpu == RING_BUFFER_ALL_CPUS) {
712 up_write(&all_cpu_access_lock);
714 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
715 up_read(&all_cpu_access_lock);
719 static inline void trace_access_lock_init(void)
723 for_each_possible_cpu(cpu)
724 mutex_init(&per_cpu(cpu_access_lock, cpu));
729 static DEFINE_MUTEX(access_lock);
731 static inline void trace_access_lock(int cpu)
734 mutex_lock(&access_lock);
737 static inline void trace_access_unlock(int cpu)
740 mutex_unlock(&access_lock);
743 static inline void trace_access_lock_init(void)
749 #ifdef CONFIG_STACKTRACE
750 static void __ftrace_trace_stack(struct ring_buffer *buffer,
752 int skip, int pc, struct pt_regs *regs);
753 static inline void ftrace_trace_stack(struct trace_array *tr,
754 struct ring_buffer *buffer,
756 int skip, int pc, struct pt_regs *regs);
759 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
761 int skip, int pc, struct pt_regs *regs)
764 static inline void ftrace_trace_stack(struct trace_array *tr,
765 struct ring_buffer *buffer,
767 int skip, int pc, struct pt_regs *regs)
773 static __always_inline void
774 trace_event_setup(struct ring_buffer_event *event,
775 int type, unsigned long flags, int pc)
777 struct trace_entry *ent = ring_buffer_event_data(event);
779 tracing_generic_entry_update(ent, type, flags, pc);
782 static __always_inline struct ring_buffer_event *
783 __trace_buffer_lock_reserve(struct ring_buffer *buffer,
786 unsigned long flags, int pc)
788 struct ring_buffer_event *event;
790 event = ring_buffer_lock_reserve(buffer, len);
792 trace_event_setup(event, type, flags, pc);
797 void tracer_tracing_on(struct trace_array *tr)
799 if (tr->trace_buffer.buffer)
800 ring_buffer_record_on(tr->trace_buffer.buffer);
802 * This flag is looked at when buffers haven't been allocated
803 * yet, or by some tracers (like irqsoff), that just want to
804 * know if the ring buffer has been disabled, but it can handle
805 * races of where it gets disabled but we still do a record.
806 * As the check is in the fast path of the tracers, it is more
807 * important to be fast than accurate.
809 tr->buffer_disabled = 0;
810 /* Make the flag seen by readers */
815 * tracing_on - enable tracing buffers
817 * This function enables tracing buffers that may have been
818 * disabled with tracing_off.
820 void tracing_on(void)
822 tracer_tracing_on(&global_trace);
824 EXPORT_SYMBOL_GPL(tracing_on);
827 static __always_inline void
828 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
830 __this_cpu_write(trace_taskinfo_save, true);
832 /* If this is the temp buffer, we need to commit fully */
833 if (this_cpu_read(trace_buffered_event) == event) {
834 /* Length is in event->array[0] */
835 ring_buffer_write(buffer, event->array[0], &event->array[1]);
836 /* Release the temp buffer */
837 this_cpu_dec(trace_buffered_event_cnt);
839 ring_buffer_unlock_commit(buffer, event);
843 * __trace_puts - write a constant string into the trace buffer.
844 * @ip: The address of the caller
845 * @str: The constant string to write
846 * @size: The size of the string.
848 int __trace_puts(unsigned long ip, const char *str, int size)
850 struct ring_buffer_event *event;
851 struct ring_buffer *buffer;
852 struct print_entry *entry;
853 unsigned long irq_flags;
857 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
860 pc = preempt_count();
862 if (unlikely(tracing_selftest_running || tracing_disabled))
865 alloc = sizeof(*entry) + size + 2; /* possible \n added */
867 local_save_flags(irq_flags);
868 buffer = global_trace.trace_buffer.buffer;
869 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
874 entry = ring_buffer_event_data(event);
877 memcpy(&entry->buf, str, size);
879 /* Add a newline if necessary */
880 if (entry->buf[size - 1] != '\n') {
881 entry->buf[size] = '\n';
882 entry->buf[size + 1] = '\0';
884 entry->buf[size] = '\0';
886 __buffer_unlock_commit(buffer, event);
887 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
891 EXPORT_SYMBOL_GPL(__trace_puts);
894 * __trace_bputs - write the pointer to a constant string into trace buffer
895 * @ip: The address of the caller
896 * @str: The constant string to write to the buffer to
898 int __trace_bputs(unsigned long ip, const char *str)
900 struct ring_buffer_event *event;
901 struct ring_buffer *buffer;
902 struct bputs_entry *entry;
903 unsigned long irq_flags;
904 int size = sizeof(struct bputs_entry);
907 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
910 pc = preempt_count();
912 if (unlikely(tracing_selftest_running || tracing_disabled))
915 local_save_flags(irq_flags);
916 buffer = global_trace.trace_buffer.buffer;
917 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
922 entry = ring_buffer_event_data(event);
926 __buffer_unlock_commit(buffer, event);
927 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
931 EXPORT_SYMBOL_GPL(__trace_bputs);
933 #ifdef CONFIG_TRACER_SNAPSHOT
934 void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data)
936 struct tracer *tracer = tr->current_trace;
940 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
941 internal_trace_puts("*** snapshot is being ignored ***\n");
945 if (!tr->allocated_snapshot) {
946 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
947 internal_trace_puts("*** stopping trace here! ***\n");
952 /* Note, snapshot can not be used when the tracer uses it */
953 if (tracer->use_max_tr) {
954 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
955 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
959 local_irq_save(flags);
960 update_max_tr(tr, current, smp_processor_id(), cond_data);
961 local_irq_restore(flags);
964 void tracing_snapshot_instance(struct trace_array *tr)
966 tracing_snapshot_instance_cond(tr, NULL);
970 * tracing_snapshot - take a snapshot of the current buffer.
972 * This causes a swap between the snapshot buffer and the current live
973 * tracing buffer. You can use this to take snapshots of the live
974 * trace when some condition is triggered, but continue to trace.
976 * Note, make sure to allocate the snapshot with either
977 * a tracing_snapshot_alloc(), or by doing it manually
978 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
980 * If the snapshot buffer is not allocated, it will stop tracing.
981 * Basically making a permanent snapshot.
983 void tracing_snapshot(void)
985 struct trace_array *tr = &global_trace;
987 tracing_snapshot_instance(tr);
989 EXPORT_SYMBOL_GPL(tracing_snapshot);
992 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
993 * @tr: The tracing instance to snapshot
994 * @cond_data: The data to be tested conditionally, and possibly saved
996 * This is the same as tracing_snapshot() except that the snapshot is
997 * conditional - the snapshot will only happen if the
998 * cond_snapshot.update() implementation receiving the cond_data
999 * returns true, which means that the trace array's cond_snapshot
1000 * update() operation used the cond_data to determine whether the
1001 * snapshot should be taken, and if it was, presumably saved it along
1002 * with the snapshot.
1004 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1006 tracing_snapshot_instance_cond(tr, cond_data);
1008 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1011 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1012 * @tr: The tracing instance
1014 * When the user enables a conditional snapshot using
1015 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1016 * with the snapshot. This accessor is used to retrieve it.
1018 * Should not be called from cond_snapshot.update(), since it takes
1019 * the tr->max_lock lock, which the code calling
1020 * cond_snapshot.update() has already done.
1022 * Returns the cond_data associated with the trace array's snapshot.
1024 void *tracing_cond_snapshot_data(struct trace_array *tr)
1026 void *cond_data = NULL;
1028 arch_spin_lock(&tr->max_lock);
1030 if (tr->cond_snapshot)
1031 cond_data = tr->cond_snapshot->cond_data;
1033 arch_spin_unlock(&tr->max_lock);
1037 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1039 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
1040 struct trace_buffer *size_buf, int cpu_id);
1041 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
1043 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1047 if (!tr->allocated_snapshot) {
1049 /* allocate spare buffer */
1050 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1051 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
1055 tr->allocated_snapshot = true;
1061 static void free_snapshot(struct trace_array *tr)
1064 * We don't free the ring buffer. instead, resize it because
1065 * The max_tr ring buffer has some state (e.g. ring->clock) and
1066 * we want preserve it.
1068 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1069 set_buffer_entries(&tr->max_buffer, 1);
1070 tracing_reset_online_cpus(&tr->max_buffer);
1071 tr->allocated_snapshot = false;
1075 * tracing_alloc_snapshot - allocate snapshot buffer.
1077 * This only allocates the snapshot buffer if it isn't already
1078 * allocated - it doesn't also take a snapshot.
1080 * This is meant to be used in cases where the snapshot buffer needs
1081 * to be set up for events that can't sleep but need to be able to
1082 * trigger a snapshot.
1084 int tracing_alloc_snapshot(void)
1086 struct trace_array *tr = &global_trace;
1089 ret = tracing_alloc_snapshot_instance(tr);
1094 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1097 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1099 * This is similar to tracing_snapshot(), but it will allocate the
1100 * snapshot buffer if it isn't already allocated. Use this only
1101 * where it is safe to sleep, as the allocation may sleep.
1103 * This causes a swap between the snapshot buffer and the current live
1104 * tracing buffer. You can use this to take snapshots of the live
1105 * trace when some condition is triggered, but continue to trace.
1107 void tracing_snapshot_alloc(void)
1111 ret = tracing_alloc_snapshot();
1117 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1120 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1121 * @tr: The tracing instance
1122 * @cond_data: User data to associate with the snapshot
1123 * @update: Implementation of the cond_snapshot update function
1125 * Check whether the conditional snapshot for the given instance has
1126 * already been enabled, or if the current tracer is already using a
1127 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1128 * save the cond_data and update function inside.
1130 * Returns 0 if successful, error otherwise.
1132 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1133 cond_update_fn_t update)
1135 struct cond_snapshot *cond_snapshot;
1138 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1142 cond_snapshot->cond_data = cond_data;
1143 cond_snapshot->update = update;
1145 mutex_lock(&trace_types_lock);
1147 ret = tracing_alloc_snapshot_instance(tr);
1151 if (tr->current_trace->use_max_tr) {
1157 * The cond_snapshot can only change to NULL without the
1158 * trace_types_lock. We don't care if we race with it going
1159 * to NULL, but we want to make sure that it's not set to
1160 * something other than NULL when we get here, which we can
1161 * do safely with only holding the trace_types_lock and not
1162 * having to take the max_lock.
1164 if (tr->cond_snapshot) {
1169 arch_spin_lock(&tr->max_lock);
1170 tr->cond_snapshot = cond_snapshot;
1171 arch_spin_unlock(&tr->max_lock);
1173 mutex_unlock(&trace_types_lock);
1178 mutex_unlock(&trace_types_lock);
1179 kfree(cond_snapshot);
1182 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1185 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1186 * @tr: The tracing instance
1188 * Check whether the conditional snapshot for the given instance is
1189 * enabled; if so, free the cond_snapshot associated with it,
1190 * otherwise return -EINVAL.
1192 * Returns 0 if successful, error otherwise.
1194 int tracing_snapshot_cond_disable(struct trace_array *tr)
1198 arch_spin_lock(&tr->max_lock);
1200 if (!tr->cond_snapshot)
1203 kfree(tr->cond_snapshot);
1204 tr->cond_snapshot = NULL;
1207 arch_spin_unlock(&tr->max_lock);
1211 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1213 void tracing_snapshot(void)
1215 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1217 EXPORT_SYMBOL_GPL(tracing_snapshot);
1218 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1220 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1222 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1223 int tracing_alloc_snapshot(void)
1225 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1228 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1229 void tracing_snapshot_alloc(void)
1234 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1235 void *tracing_cond_snapshot_data(struct trace_array *tr)
1239 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1240 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1244 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1245 int tracing_snapshot_cond_disable(struct trace_array *tr)
1249 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1250 #endif /* CONFIG_TRACER_SNAPSHOT */
1252 void tracer_tracing_off(struct trace_array *tr)
1254 if (tr->trace_buffer.buffer)
1255 ring_buffer_record_off(tr->trace_buffer.buffer);
1257 * This flag is looked at when buffers haven't been allocated
1258 * yet, or by some tracers (like irqsoff), that just want to
1259 * know if the ring buffer has been disabled, but it can handle
1260 * races of where it gets disabled but we still do a record.
1261 * As the check is in the fast path of the tracers, it is more
1262 * important to be fast than accurate.
1264 tr->buffer_disabled = 1;
1265 /* Make the flag seen by readers */
1270 * tracing_off - turn off tracing buffers
1272 * This function stops the tracing buffers from recording data.
1273 * It does not disable any overhead the tracers themselves may
1274 * be causing. This function simply causes all recording to
1275 * the ring buffers to fail.
1277 void tracing_off(void)
1279 tracer_tracing_off(&global_trace);
1281 EXPORT_SYMBOL_GPL(tracing_off);
1283 void disable_trace_on_warning(void)
1285 if (__disable_trace_on_warning)
1290 * tracer_tracing_is_on - show real state of ring buffer enabled
1291 * @tr : the trace array to know if ring buffer is enabled
1293 * Shows real state of the ring buffer if it is enabled or not.
1295 bool tracer_tracing_is_on(struct trace_array *tr)
1297 if (tr->trace_buffer.buffer)
1298 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1299 return !tr->buffer_disabled;
1303 * tracing_is_on - show state of ring buffers enabled
1305 int tracing_is_on(void)
1307 return tracer_tracing_is_on(&global_trace);
1309 EXPORT_SYMBOL_GPL(tracing_is_on);
1311 static int __init set_buf_size(char *str)
1313 unsigned long buf_size;
1317 buf_size = memparse(str, &str);
1318 /* nr_entries can not be zero */
1321 trace_buf_size = buf_size;
1324 __setup("trace_buf_size=", set_buf_size);
1326 static int __init set_tracing_thresh(char *str)
1328 unsigned long threshold;
1333 ret = kstrtoul(str, 0, &threshold);
1336 tracing_thresh = threshold * 1000;
1339 __setup("tracing_thresh=", set_tracing_thresh);
1341 unsigned long nsecs_to_usecs(unsigned long nsecs)
1343 return nsecs / 1000;
1347 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1348 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1349 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1350 * of strings in the order that the evals (enum) were defined.
1355 /* These must match the bit postions in trace_iterator_flags */
1356 static const char *trace_options[] = {
1364 int in_ns; /* is this clock in nanoseconds? */
1365 } trace_clocks[] = {
1366 { trace_clock_local, "local", 1 },
1367 { trace_clock_global, "global", 1 },
1368 { trace_clock_counter, "counter", 0 },
1369 { trace_clock_jiffies, "uptime", 0 },
1370 { trace_clock, "perf", 1 },
1371 { ktime_get_mono_fast_ns, "mono", 1 },
1372 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1373 { ktime_get_boot_fast_ns, "boot", 1 },
1377 bool trace_clock_in_ns(struct trace_array *tr)
1379 if (trace_clocks[tr->clock_id].in_ns)
1386 * trace_parser_get_init - gets the buffer for trace parser
1388 int trace_parser_get_init(struct trace_parser *parser, int size)
1390 memset(parser, 0, sizeof(*parser));
1392 parser->buffer = kmalloc(size, GFP_KERNEL);
1393 if (!parser->buffer)
1396 parser->size = size;
1401 * trace_parser_put - frees the buffer for trace parser
1403 void trace_parser_put(struct trace_parser *parser)
1405 kfree(parser->buffer);
1406 parser->buffer = NULL;
1410 * trace_get_user - reads the user input string separated by space
1411 * (matched by isspace(ch))
1413 * For each string found the 'struct trace_parser' is updated,
1414 * and the function returns.
1416 * Returns number of bytes read.
1418 * See kernel/trace/trace.h for 'struct trace_parser' details.
1420 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1421 size_t cnt, loff_t *ppos)
1428 trace_parser_clear(parser);
1430 ret = get_user(ch, ubuf++);
1438 * The parser is not finished with the last write,
1439 * continue reading the user input without skipping spaces.
1441 if (!parser->cont) {
1442 /* skip white space */
1443 while (cnt && isspace(ch)) {
1444 ret = get_user(ch, ubuf++);
1453 /* only spaces were written */
1454 if (isspace(ch) || !ch) {
1461 /* read the non-space input */
1462 while (cnt && !isspace(ch) && ch) {
1463 if (parser->idx < parser->size - 1)
1464 parser->buffer[parser->idx++] = ch;
1469 ret = get_user(ch, ubuf++);
1476 /* We either got finished input or we have to wait for another call. */
1477 if (isspace(ch) || !ch) {
1478 parser->buffer[parser->idx] = 0;
1479 parser->cont = false;
1480 } else if (parser->idx < parser->size - 1) {
1481 parser->cont = true;
1482 parser->buffer[parser->idx++] = ch;
1483 /* Make sure the parsed string always terminates with '\0'. */
1484 parser->buffer[parser->idx] = 0;
1497 /* TODO add a seq_buf_to_buffer() */
1498 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1502 if (trace_seq_used(s) <= s->seq.readpos)
1505 len = trace_seq_used(s) - s->seq.readpos;
1508 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1510 s->seq.readpos += cnt;
1514 unsigned long __read_mostly tracing_thresh;
1515 static const struct file_operations tracing_max_lat_fops;
1517 #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1518 defined(CONFIG_FSNOTIFY)
1520 static struct workqueue_struct *fsnotify_wq;
1522 static void latency_fsnotify_workfn(struct work_struct *work)
1524 struct trace_array *tr = container_of(work, struct trace_array,
1526 fsnotify(tr->d_max_latency->d_inode, FS_MODIFY,
1527 tr->d_max_latency->d_inode, FSNOTIFY_EVENT_INODE, NULL, 0);
1530 static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1532 struct trace_array *tr = container_of(iwork, struct trace_array,
1534 queue_work(fsnotify_wq, &tr->fsnotify_work);
1537 static void trace_create_maxlat_file(struct trace_array *tr,
1538 struct dentry *d_tracer)
1540 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1541 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1542 tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
1543 d_tracer, &tr->max_latency,
1544 &tracing_max_lat_fops);
1547 __init static int latency_fsnotify_init(void)
1549 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1550 WQ_UNBOUND | WQ_HIGHPRI, 0);
1552 pr_err("Unable to allocate tr_max_lat_wq\n");
1558 late_initcall_sync(latency_fsnotify_init);
1560 void latency_fsnotify(struct trace_array *tr)
1565 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1566 * possible that we are called from __schedule() or do_idle(), which
1567 * could cause a deadlock.
1569 irq_work_queue(&tr->fsnotify_irqwork);
1573 * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1574 * defined(CONFIG_FSNOTIFY)
1578 #define trace_create_maxlat_file(tr, d_tracer) \
1579 trace_create_file("tracing_max_latency", 0644, d_tracer, \
1580 &tr->max_latency, &tracing_max_lat_fops)
1584 #ifdef CONFIG_TRACER_MAX_TRACE
1586 * Copy the new maximum trace into the separate maximum-trace
1587 * structure. (this way the maximum trace is permanently saved,
1588 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1591 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1593 struct trace_buffer *trace_buf = &tr->trace_buffer;
1594 struct trace_buffer *max_buf = &tr->max_buffer;
1595 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1596 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1599 max_buf->time_start = data->preempt_timestamp;
1601 max_data->saved_latency = tr->max_latency;
1602 max_data->critical_start = data->critical_start;
1603 max_data->critical_end = data->critical_end;
1605 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1606 max_data->pid = tsk->pid;
1608 * If tsk == current, then use current_uid(), as that does not use
1609 * RCU. The irq tracer can be called out of RCU scope.
1612 max_data->uid = current_uid();
1614 max_data->uid = task_uid(tsk);
1616 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1617 max_data->policy = tsk->policy;
1618 max_data->rt_priority = tsk->rt_priority;
1620 /* record this tasks comm */
1621 tracing_record_cmdline(tsk);
1622 latency_fsnotify(tr);
1626 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1628 * @tsk: the task with the latency
1629 * @cpu: The cpu that initiated the trace.
1630 * @cond_data: User data associated with a conditional snapshot
1632 * Flip the buffers between the @tr and the max_tr and record information
1633 * about which task was the cause of this latency.
1636 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1642 WARN_ON_ONCE(!irqs_disabled());
1644 if (!tr->allocated_snapshot) {
1645 /* Only the nop tracer should hit this when disabling */
1646 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1650 arch_spin_lock(&tr->max_lock);
1652 /* Inherit the recordable setting from trace_buffer */
1653 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1654 ring_buffer_record_on(tr->max_buffer.buffer);
1656 ring_buffer_record_off(tr->max_buffer.buffer);
1658 #ifdef CONFIG_TRACER_SNAPSHOT
1659 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1662 swap(tr->trace_buffer.buffer, tr->max_buffer.buffer);
1664 __update_max_tr(tr, tsk, cpu);
1667 arch_spin_unlock(&tr->max_lock);
1671 * update_max_tr_single - only copy one trace over, and reset the rest
1673 * @tsk: task with the latency
1674 * @cpu: the cpu of the buffer to copy.
1676 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1679 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1686 WARN_ON_ONCE(!irqs_disabled());
1687 if (!tr->allocated_snapshot) {
1688 /* Only the nop tracer should hit this when disabling */
1689 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1693 arch_spin_lock(&tr->max_lock);
1695 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1697 if (ret == -EBUSY) {
1699 * We failed to swap the buffer due to a commit taking
1700 * place on this CPU. We fail to record, but we reset
1701 * the max trace buffer (no one writes directly to it)
1702 * and flag that it failed.
1704 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1705 "Failed to swap buffers due to commit in progress\n");
1708 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1710 __update_max_tr(tr, tsk, cpu);
1711 arch_spin_unlock(&tr->max_lock);
1713 #endif /* CONFIG_TRACER_MAX_TRACE */
1715 static int wait_on_pipe(struct trace_iterator *iter, int full)
1717 /* Iterators are static, they should be filled or empty */
1718 if (trace_buffer_iter(iter, iter->cpu_file))
1721 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1725 #ifdef CONFIG_FTRACE_STARTUP_TEST
1726 static bool selftests_can_run;
1728 struct trace_selftests {
1729 struct list_head list;
1730 struct tracer *type;
1733 static LIST_HEAD(postponed_selftests);
1735 static int save_selftest(struct tracer *type)
1737 struct trace_selftests *selftest;
1739 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1743 selftest->type = type;
1744 list_add(&selftest->list, &postponed_selftests);
1748 static int run_tracer_selftest(struct tracer *type)
1750 struct trace_array *tr = &global_trace;
1751 struct tracer *saved_tracer = tr->current_trace;
1754 if (!type->selftest || tracing_selftest_disabled)
1758 * If a tracer registers early in boot up (before scheduling is
1759 * initialized and such), then do not run its selftests yet.
1760 * Instead, run it a little later in the boot process.
1762 if (!selftests_can_run)
1763 return save_selftest(type);
1766 * Run a selftest on this tracer.
1767 * Here we reset the trace buffer, and set the current
1768 * tracer to be this tracer. The tracer can then run some
1769 * internal tracing to verify that everything is in order.
1770 * If we fail, we do not register this tracer.
1772 tracing_reset_online_cpus(&tr->trace_buffer);
1774 tr->current_trace = type;
1776 #ifdef CONFIG_TRACER_MAX_TRACE
1777 if (type->use_max_tr) {
1778 /* If we expanded the buffers, make sure the max is expanded too */
1779 if (ring_buffer_expanded)
1780 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1781 RING_BUFFER_ALL_CPUS);
1782 tr->allocated_snapshot = true;
1786 /* the test is responsible for initializing and enabling */
1787 pr_info("Testing tracer %s: ", type->name);
1788 ret = type->selftest(type, tr);
1789 /* the test is responsible for resetting too */
1790 tr->current_trace = saved_tracer;
1792 printk(KERN_CONT "FAILED!\n");
1793 /* Add the warning after printing 'FAILED' */
1797 /* Only reset on passing, to avoid touching corrupted buffers */
1798 tracing_reset_online_cpus(&tr->trace_buffer);
1800 #ifdef CONFIG_TRACER_MAX_TRACE
1801 if (type->use_max_tr) {
1802 tr->allocated_snapshot = false;
1804 /* Shrink the max buffer again */
1805 if (ring_buffer_expanded)
1806 ring_buffer_resize(tr->max_buffer.buffer, 1,
1807 RING_BUFFER_ALL_CPUS);
1811 printk(KERN_CONT "PASSED\n");
1815 static __init int init_trace_selftests(void)
1817 struct trace_selftests *p, *n;
1818 struct tracer *t, **last;
1821 selftests_can_run = true;
1823 mutex_lock(&trace_types_lock);
1825 if (list_empty(&postponed_selftests))
1828 pr_info("Running postponed tracer tests:\n");
1830 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1831 /* This loop can take minutes when sanitizers are enabled, so
1832 * lets make sure we allow RCU processing.
1835 ret = run_tracer_selftest(p->type);
1836 /* If the test fails, then warn and remove from available_tracers */
1838 WARN(1, "tracer: %s failed selftest, disabling\n",
1840 last = &trace_types;
1841 for (t = trace_types; t; t = t->next) {
1854 mutex_unlock(&trace_types_lock);
1858 core_initcall(init_trace_selftests);
1860 static inline int run_tracer_selftest(struct tracer *type)
1864 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1866 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1868 static void __init apply_trace_boot_options(void);
1871 * register_tracer - register a tracer with the ftrace system.
1872 * @type: the plugin for the tracer
1874 * Register a new plugin tracer.
1876 int __init register_tracer(struct tracer *type)
1882 pr_info("Tracer must have a name\n");
1886 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1887 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1891 mutex_lock(&trace_types_lock);
1893 tracing_selftest_running = true;
1895 for (t = trace_types; t; t = t->next) {
1896 if (strcmp(type->name, t->name) == 0) {
1898 pr_info("Tracer %s already registered\n",
1905 if (!type->set_flag)
1906 type->set_flag = &dummy_set_flag;
1908 /*allocate a dummy tracer_flags*/
1909 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1914 type->flags->val = 0;
1915 type->flags->opts = dummy_tracer_opt;
1917 if (!type->flags->opts)
1918 type->flags->opts = dummy_tracer_opt;
1920 /* store the tracer for __set_tracer_option */
1921 type->flags->trace = type;
1923 ret = run_tracer_selftest(type);
1927 type->next = trace_types;
1929 add_tracer_options(&global_trace, type);
1932 tracing_selftest_running = false;
1933 mutex_unlock(&trace_types_lock);
1935 if (ret || !default_bootup_tracer)
1938 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1941 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1942 /* Do we want this tracer to start on bootup? */
1943 tracing_set_tracer(&global_trace, type->name);
1944 default_bootup_tracer = NULL;
1946 apply_trace_boot_options();
1948 /* disable other selftests, since this will break it. */
1949 tracing_selftest_disabled = true;
1950 #ifdef CONFIG_FTRACE_STARTUP_TEST
1951 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1959 static void tracing_reset_cpu(struct trace_buffer *buf, int cpu)
1961 struct ring_buffer *buffer = buf->buffer;
1966 ring_buffer_record_disable(buffer);
1968 /* Make sure all commits have finished */
1970 ring_buffer_reset_cpu(buffer, cpu);
1972 ring_buffer_record_enable(buffer);
1975 void tracing_reset_online_cpus(struct trace_buffer *buf)
1977 struct ring_buffer *buffer = buf->buffer;
1983 ring_buffer_record_disable(buffer);
1985 /* Make sure all commits have finished */
1988 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1990 for_each_online_cpu(cpu)
1991 ring_buffer_reset_cpu(buffer, cpu);
1993 ring_buffer_record_enable(buffer);
1996 /* Must have trace_types_lock held */
1997 void tracing_reset_all_online_cpus(void)
1999 struct trace_array *tr;
2001 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2002 if (!tr->clear_trace)
2004 tr->clear_trace = false;
2005 tracing_reset_online_cpus(&tr->trace_buffer);
2006 #ifdef CONFIG_TRACER_MAX_TRACE
2007 tracing_reset_online_cpus(&tr->max_buffer);
2012 static int *tgid_map;
2014 #define SAVED_CMDLINES_DEFAULT 128
2015 #define NO_CMDLINE_MAP UINT_MAX
2016 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2017 struct saved_cmdlines_buffer {
2018 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2019 unsigned *map_cmdline_to_pid;
2020 unsigned cmdline_num;
2022 char *saved_cmdlines;
2024 static struct saved_cmdlines_buffer *savedcmd;
2026 /* temporary disable recording */
2027 static atomic_t trace_record_taskinfo_disabled __read_mostly;
2029 static inline char *get_saved_cmdlines(int idx)
2031 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2034 static inline void set_cmdline(int idx, const char *cmdline)
2036 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2039 static int allocate_cmdlines_buffer(unsigned int val,
2040 struct saved_cmdlines_buffer *s)
2042 s->map_cmdline_to_pid = kmalloc_array(val,
2043 sizeof(*s->map_cmdline_to_pid),
2045 if (!s->map_cmdline_to_pid)
2048 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2049 if (!s->saved_cmdlines) {
2050 kfree(s->map_cmdline_to_pid);
2055 s->cmdline_num = val;
2056 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2057 sizeof(s->map_pid_to_cmdline));
2058 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2059 val * sizeof(*s->map_cmdline_to_pid));
2064 static int trace_create_savedcmd(void)
2068 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2072 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2082 int is_tracing_stopped(void)
2084 return global_trace.stop_count;
2088 * tracing_start - quick start of the tracer
2090 * If tracing is enabled but was stopped by tracing_stop,
2091 * this will start the tracer back up.
2093 void tracing_start(void)
2095 struct ring_buffer *buffer;
2096 unsigned long flags;
2098 if (tracing_disabled)
2101 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2102 if (--global_trace.stop_count) {
2103 if (global_trace.stop_count < 0) {
2104 /* Someone screwed up their debugging */
2106 global_trace.stop_count = 0;
2111 /* Prevent the buffers from switching */
2112 arch_spin_lock(&global_trace.max_lock);
2114 buffer = global_trace.trace_buffer.buffer;
2116 ring_buffer_record_enable(buffer);
2118 #ifdef CONFIG_TRACER_MAX_TRACE
2119 buffer = global_trace.max_buffer.buffer;
2121 ring_buffer_record_enable(buffer);
2124 arch_spin_unlock(&global_trace.max_lock);
2127 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2130 static void tracing_start_tr(struct trace_array *tr)
2132 struct ring_buffer *buffer;
2133 unsigned long flags;
2135 if (tracing_disabled)
2138 /* If global, we need to also start the max tracer */
2139 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2140 return tracing_start();
2142 raw_spin_lock_irqsave(&tr->start_lock, flags);
2144 if (--tr->stop_count) {
2145 if (tr->stop_count < 0) {
2146 /* Someone screwed up their debugging */
2153 buffer = tr->trace_buffer.buffer;
2155 ring_buffer_record_enable(buffer);
2158 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2162 * tracing_stop - quick stop of the tracer
2164 * Light weight way to stop tracing. Use in conjunction with
2167 void tracing_stop(void)
2169 struct ring_buffer *buffer;
2170 unsigned long flags;
2172 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2173 if (global_trace.stop_count++)
2176 /* Prevent the buffers from switching */
2177 arch_spin_lock(&global_trace.max_lock);
2179 buffer = global_trace.trace_buffer.buffer;
2181 ring_buffer_record_disable(buffer);
2183 #ifdef CONFIG_TRACER_MAX_TRACE
2184 buffer = global_trace.max_buffer.buffer;
2186 ring_buffer_record_disable(buffer);
2189 arch_spin_unlock(&global_trace.max_lock);
2192 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2195 static void tracing_stop_tr(struct trace_array *tr)
2197 struct ring_buffer *buffer;
2198 unsigned long flags;
2200 /* If global, we need to also stop the max tracer */
2201 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2202 return tracing_stop();
2204 raw_spin_lock_irqsave(&tr->start_lock, flags);
2205 if (tr->stop_count++)
2208 buffer = tr->trace_buffer.buffer;
2210 ring_buffer_record_disable(buffer);
2213 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2216 static int trace_save_cmdline(struct task_struct *tsk)
2220 /* treat recording of idle task as a success */
2224 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2228 * It's not the end of the world if we don't get
2229 * the lock, but we also don't want to spin
2230 * nor do we want to disable interrupts,
2231 * so if we miss here, then better luck next time.
2233 if (!arch_spin_trylock(&trace_cmdline_lock))
2236 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2237 if (idx == NO_CMDLINE_MAP) {
2238 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2241 * Check whether the cmdline buffer at idx has a pid
2242 * mapped. We are going to overwrite that entry so we
2243 * need to clear the map_pid_to_cmdline. Otherwise we
2244 * would read the new comm for the old pid.
2246 pid = savedcmd->map_cmdline_to_pid[idx];
2247 if (pid != NO_CMDLINE_MAP)
2248 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2250 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2251 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2253 savedcmd->cmdline_idx = idx;
2256 set_cmdline(idx, tsk->comm);
2258 arch_spin_unlock(&trace_cmdline_lock);
2263 static void __trace_find_cmdline(int pid, char comm[])
2268 strcpy(comm, "<idle>");
2272 if (WARN_ON_ONCE(pid < 0)) {
2273 strcpy(comm, "<XXX>");
2277 if (pid > PID_MAX_DEFAULT) {
2278 strcpy(comm, "<...>");
2282 map = savedcmd->map_pid_to_cmdline[pid];
2283 if (map != NO_CMDLINE_MAP)
2284 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2286 strcpy(comm, "<...>");
2289 void trace_find_cmdline(int pid, char comm[])
2292 arch_spin_lock(&trace_cmdline_lock);
2294 __trace_find_cmdline(pid, comm);
2296 arch_spin_unlock(&trace_cmdline_lock);
2300 int trace_find_tgid(int pid)
2302 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2305 return tgid_map[pid];
2308 static int trace_save_tgid(struct task_struct *tsk)
2310 /* treat recording of idle task as a success */
2314 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2317 tgid_map[tsk->pid] = tsk->tgid;
2321 static bool tracing_record_taskinfo_skip(int flags)
2323 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2325 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2327 if (!__this_cpu_read(trace_taskinfo_save))
2333 * tracing_record_taskinfo - record the task info of a task
2335 * @task: task to record
2336 * @flags: TRACE_RECORD_CMDLINE for recording comm
2337 * TRACE_RECORD_TGID for recording tgid
2339 void tracing_record_taskinfo(struct task_struct *task, int flags)
2343 if (tracing_record_taskinfo_skip(flags))
2347 * Record as much task information as possible. If some fail, continue
2348 * to try to record the others.
2350 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2351 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2353 /* If recording any information failed, retry again soon. */
2357 __this_cpu_write(trace_taskinfo_save, false);
2361 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2363 * @prev: previous task during sched_switch
2364 * @next: next task during sched_switch
2365 * @flags: TRACE_RECORD_CMDLINE for recording comm
2366 * TRACE_RECORD_TGID for recording tgid
2368 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2369 struct task_struct *next, int flags)
2373 if (tracing_record_taskinfo_skip(flags))
2377 * Record as much task information as possible. If some fail, continue
2378 * to try to record the others.
2380 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2381 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2382 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2383 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2385 /* If recording any information failed, retry again soon. */
2389 __this_cpu_write(trace_taskinfo_save, false);
2392 /* Helpers to record a specific task information */
2393 void tracing_record_cmdline(struct task_struct *task)
2395 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2398 void tracing_record_tgid(struct task_struct *task)
2400 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2404 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2405 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2406 * simplifies those functions and keeps them in sync.
2408 enum print_line_t trace_handle_return(struct trace_seq *s)
2410 return trace_seq_has_overflowed(s) ?
2411 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2413 EXPORT_SYMBOL_GPL(trace_handle_return);
2416 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
2417 unsigned long flags, int pc)
2419 struct task_struct *tsk = current;
2421 entry->preempt_count = pc & 0xff;
2422 entry->pid = (tsk) ? tsk->pid : 0;
2425 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2426 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2428 TRACE_FLAG_IRQS_NOSUPPORT |
2430 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
2431 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2432 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2433 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2434 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2436 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2438 struct ring_buffer_event *
2439 trace_buffer_lock_reserve(struct ring_buffer *buffer,
2442 unsigned long flags, int pc)
2444 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2447 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2448 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2449 static int trace_buffered_event_ref;
2452 * trace_buffered_event_enable - enable buffering events
2454 * When events are being filtered, it is quicker to use a temporary
2455 * buffer to write the event data into if there's a likely chance
2456 * that it will not be committed. The discard of the ring buffer
2457 * is not as fast as committing, and is much slower than copying
2460 * When an event is to be filtered, allocate per cpu buffers to
2461 * write the event data into, and if the event is filtered and discarded
2462 * it is simply dropped, otherwise, the entire data is to be committed
2465 void trace_buffered_event_enable(void)
2467 struct ring_buffer_event *event;
2471 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2473 if (trace_buffered_event_ref++)
2476 for_each_tracing_cpu(cpu) {
2477 page = alloc_pages_node(cpu_to_node(cpu),
2478 GFP_KERNEL | __GFP_NORETRY, 0);
2482 event = page_address(page);
2483 memset(event, 0, sizeof(*event));
2485 per_cpu(trace_buffered_event, cpu) = event;
2488 if (cpu == smp_processor_id() &&
2489 this_cpu_read(trace_buffered_event) !=
2490 per_cpu(trace_buffered_event, cpu))
2497 trace_buffered_event_disable();
2500 static void enable_trace_buffered_event(void *data)
2502 /* Probably not needed, but do it anyway */
2504 this_cpu_dec(trace_buffered_event_cnt);
2507 static void disable_trace_buffered_event(void *data)
2509 this_cpu_inc(trace_buffered_event_cnt);
2513 * trace_buffered_event_disable - disable buffering events
2515 * When a filter is removed, it is faster to not use the buffered
2516 * events, and to commit directly into the ring buffer. Free up
2517 * the temp buffers when there are no more users. This requires
2518 * special synchronization with current events.
2520 void trace_buffered_event_disable(void)
2524 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2526 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2529 if (--trace_buffered_event_ref)
2533 /* For each CPU, set the buffer as used. */
2534 smp_call_function_many(tracing_buffer_mask,
2535 disable_trace_buffered_event, NULL, 1);
2538 /* Wait for all current users to finish */
2541 for_each_tracing_cpu(cpu) {
2542 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2543 per_cpu(trace_buffered_event, cpu) = NULL;
2546 * Make sure trace_buffered_event is NULL before clearing
2547 * trace_buffered_event_cnt.
2552 /* Do the work on each cpu */
2553 smp_call_function_many(tracing_buffer_mask,
2554 enable_trace_buffered_event, NULL, 1);
2558 static struct ring_buffer *temp_buffer;
2560 struct ring_buffer_event *
2561 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2562 struct trace_event_file *trace_file,
2563 int type, unsigned long len,
2564 unsigned long flags, int pc)
2566 struct ring_buffer_event *entry;
2569 *current_rb = trace_file->tr->trace_buffer.buffer;
2571 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2572 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2573 (entry = this_cpu_read(trace_buffered_event))) {
2574 /* Try to use the per cpu buffer first */
2575 val = this_cpu_inc_return(trace_buffered_event_cnt);
2577 trace_event_setup(entry, type, flags, pc);
2578 entry->array[0] = len;
2581 this_cpu_dec(trace_buffered_event_cnt);
2584 entry = __trace_buffer_lock_reserve(*current_rb,
2585 type, len, flags, pc);
2587 * If tracing is off, but we have triggers enabled
2588 * we still need to look at the event data. Use the temp_buffer
2589 * to store the trace event for the tigger to use. It's recusive
2590 * safe and will not be recorded anywhere.
2592 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2593 *current_rb = temp_buffer;
2594 entry = __trace_buffer_lock_reserve(*current_rb,
2595 type, len, flags, pc);
2599 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2601 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2602 static DEFINE_MUTEX(tracepoint_printk_mutex);
2604 static void output_printk(struct trace_event_buffer *fbuffer)
2606 struct trace_event_call *event_call;
2607 struct trace_event *event;
2608 unsigned long flags;
2609 struct trace_iterator *iter = tracepoint_print_iter;
2611 /* We should never get here if iter is NULL */
2612 if (WARN_ON_ONCE(!iter))
2615 event_call = fbuffer->trace_file->event_call;
2616 if (!event_call || !event_call->event.funcs ||
2617 !event_call->event.funcs->trace)
2620 event = &fbuffer->trace_file->event_call->event;
2622 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2623 trace_seq_init(&iter->seq);
2624 iter->ent = fbuffer->entry;
2625 event_call->event.funcs->trace(iter, 0, event);
2626 trace_seq_putc(&iter->seq, 0);
2627 printk("%s", iter->seq.buffer);
2629 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2632 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2633 void __user *buffer, size_t *lenp,
2636 int save_tracepoint_printk;
2639 mutex_lock(&tracepoint_printk_mutex);
2640 save_tracepoint_printk = tracepoint_printk;
2642 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2645 * This will force exiting early, as tracepoint_printk
2646 * is always zero when tracepoint_printk_iter is not allocated
2648 if (!tracepoint_print_iter)
2649 tracepoint_printk = 0;
2651 if (save_tracepoint_printk == tracepoint_printk)
2654 if (tracepoint_printk)
2655 static_key_enable(&tracepoint_printk_key.key);
2657 static_key_disable(&tracepoint_printk_key.key);
2660 mutex_unlock(&tracepoint_printk_mutex);
2665 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2667 if (static_key_false(&tracepoint_printk_key.key))
2668 output_printk(fbuffer);
2670 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2671 fbuffer->event, fbuffer->entry,
2672 fbuffer->flags, fbuffer->pc);
2674 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2679 * trace_buffer_unlock_commit_regs()
2680 * trace_event_buffer_commit()
2681 * trace_event_raw_event_xxx()
2683 # define STACK_SKIP 3
2685 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2686 struct ring_buffer *buffer,
2687 struct ring_buffer_event *event,
2688 unsigned long flags, int pc,
2689 struct pt_regs *regs)
2691 __buffer_unlock_commit(buffer, event);
2694 * If regs is not set, then skip the necessary functions.
2695 * Note, we can still get here via blktrace, wakeup tracer
2696 * and mmiotrace, but that's ok if they lose a function or
2697 * two. They are not that meaningful.
2699 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2700 ftrace_trace_userstack(buffer, flags, pc);
2704 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2707 trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2708 struct ring_buffer_event *event)
2710 __buffer_unlock_commit(buffer, event);
2714 trace_process_export(struct trace_export *export,
2715 struct ring_buffer_event *event)
2717 struct trace_entry *entry;
2718 unsigned int size = 0;
2720 entry = ring_buffer_event_data(event);
2721 size = ring_buffer_event_length(event);
2722 export->write(export, entry, size);
2725 static DEFINE_MUTEX(ftrace_export_lock);
2727 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2729 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2731 static inline void ftrace_exports_enable(void)
2733 static_branch_enable(&ftrace_exports_enabled);
2736 static inline void ftrace_exports_disable(void)
2738 static_branch_disable(&ftrace_exports_enabled);
2741 static void ftrace_exports(struct ring_buffer_event *event)
2743 struct trace_export *export;
2745 preempt_disable_notrace();
2747 export = rcu_dereference_raw_check(ftrace_exports_list);
2749 trace_process_export(export, event);
2750 export = rcu_dereference_raw_check(export->next);
2753 preempt_enable_notrace();
2757 add_trace_export(struct trace_export **list, struct trace_export *export)
2759 rcu_assign_pointer(export->next, *list);
2761 * We are entering export into the list but another
2762 * CPU might be walking that list. We need to make sure
2763 * the export->next pointer is valid before another CPU sees
2764 * the export pointer included into the list.
2766 rcu_assign_pointer(*list, export);
2770 rm_trace_export(struct trace_export **list, struct trace_export *export)
2772 struct trace_export **p;
2774 for (p = list; *p != NULL; p = &(*p)->next)
2781 rcu_assign_pointer(*p, (*p)->next);
2787 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2790 ftrace_exports_enable();
2792 add_trace_export(list, export);
2796 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2800 ret = rm_trace_export(list, export);
2802 ftrace_exports_disable();
2807 int register_ftrace_export(struct trace_export *export)
2809 if (WARN_ON_ONCE(!export->write))
2812 mutex_lock(&ftrace_export_lock);
2814 add_ftrace_export(&ftrace_exports_list, export);
2816 mutex_unlock(&ftrace_export_lock);
2820 EXPORT_SYMBOL_GPL(register_ftrace_export);
2822 int unregister_ftrace_export(struct trace_export *export)
2826 mutex_lock(&ftrace_export_lock);
2828 ret = rm_ftrace_export(&ftrace_exports_list, export);
2830 mutex_unlock(&ftrace_export_lock);
2834 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2837 trace_function(struct trace_array *tr,
2838 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2841 struct trace_event_call *call = &event_function;
2842 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2843 struct ring_buffer_event *event;
2844 struct ftrace_entry *entry;
2846 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2850 entry = ring_buffer_event_data(event);
2852 entry->parent_ip = parent_ip;
2854 if (!call_filter_check_discard(call, entry, buffer, event)) {
2855 if (static_branch_unlikely(&ftrace_exports_enabled))
2856 ftrace_exports(event);
2857 __buffer_unlock_commit(buffer, event);
2861 #ifdef CONFIG_STACKTRACE
2863 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2864 #define FTRACE_KSTACK_NESTING 4
2866 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2868 struct ftrace_stack {
2869 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2873 struct ftrace_stacks {
2874 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2877 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2878 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2880 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2881 unsigned long flags,
2882 int skip, int pc, struct pt_regs *regs)
2884 struct trace_event_call *call = &event_kernel_stack;
2885 struct ring_buffer_event *event;
2886 unsigned int size, nr_entries;
2887 struct ftrace_stack *fstack;
2888 struct stack_entry *entry;
2892 * Add one, for this function and the call to save_stack_trace()
2893 * If regs is set, then these functions will not be in the way.
2895 #ifndef CONFIG_UNWINDER_ORC
2901 * Since events can happen in NMIs there's no safe way to
2902 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2903 * or NMI comes in, it will just have to use the default
2904 * FTRACE_STACK_SIZE.
2906 preempt_disable_notrace();
2908 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2910 /* This should never happen. If it does, yell once and skip */
2911 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
2915 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2916 * interrupt will either see the value pre increment or post
2917 * increment. If the interrupt happens pre increment it will have
2918 * restored the counter when it returns. We just need a barrier to
2919 * keep gcc from moving things around.
2923 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2924 size = ARRAY_SIZE(fstack->calls);
2927 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2930 nr_entries = stack_trace_save(fstack->calls, size, skip);
2933 size = nr_entries * sizeof(unsigned long);
2934 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2935 sizeof(*entry) + size, flags, pc);
2938 entry = ring_buffer_event_data(event);
2940 memcpy(&entry->caller, fstack->calls, size);
2941 entry->size = nr_entries;
2943 if (!call_filter_check_discard(call, entry, buffer, event))
2944 __buffer_unlock_commit(buffer, event);
2947 /* Again, don't let gcc optimize things here */
2949 __this_cpu_dec(ftrace_stack_reserve);
2950 preempt_enable_notrace();
2954 static inline void ftrace_trace_stack(struct trace_array *tr,
2955 struct ring_buffer *buffer,
2956 unsigned long flags,
2957 int skip, int pc, struct pt_regs *regs)
2959 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2962 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2965 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2968 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2970 if (rcu_is_watching()) {
2971 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2976 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2977 * but if the above rcu_is_watching() failed, then the NMI
2978 * triggered someplace critical, and rcu_irq_enter() should
2979 * not be called from NMI.
2981 if (unlikely(in_nmi()))
2984 rcu_irq_enter_irqson();
2985 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2986 rcu_irq_exit_irqson();
2990 * trace_dump_stack - record a stack back trace in the trace buffer
2991 * @skip: Number of functions to skip (helper handlers)
2993 void trace_dump_stack(int skip)
2995 unsigned long flags;
2997 if (tracing_disabled || tracing_selftest_running)
3000 local_save_flags(flags);
3002 #ifndef CONFIG_UNWINDER_ORC
3003 /* Skip 1 to skip this function. */
3006 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
3007 flags, skip, preempt_count(), NULL);
3009 EXPORT_SYMBOL_GPL(trace_dump_stack);
3011 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
3012 static DEFINE_PER_CPU(int, user_stack_count);
3015 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
3017 struct trace_event_call *call = &event_user_stack;
3018 struct ring_buffer_event *event;
3019 struct userstack_entry *entry;
3021 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
3025 * NMIs can not handle page faults, even with fix ups.
3026 * The save user stack can (and often does) fault.
3028 if (unlikely(in_nmi()))
3032 * prevent recursion, since the user stack tracing may
3033 * trigger other kernel events.
3036 if (__this_cpu_read(user_stack_count))
3039 __this_cpu_inc(user_stack_count);
3041 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3042 sizeof(*entry), flags, pc);
3044 goto out_drop_count;
3045 entry = ring_buffer_event_data(event);
3047 entry->tgid = current->tgid;
3048 memset(&entry->caller, 0, sizeof(entry->caller));
3050 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3051 if (!call_filter_check_discard(call, entry, buffer, event))
3052 __buffer_unlock_commit(buffer, event);
3055 __this_cpu_dec(user_stack_count);
3059 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
3060 static void ftrace_trace_userstack(struct ring_buffer *buffer,
3061 unsigned long flags, int pc)
3064 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3066 #endif /* CONFIG_STACKTRACE */
3068 /* created for use with alloc_percpu */
3069 struct trace_buffer_struct {
3071 char buffer[4][TRACE_BUF_SIZE];
3074 static struct trace_buffer_struct *trace_percpu_buffer;
3077 * Thise allows for lockless recording. If we're nested too deeply, then
3078 * this returns NULL.
3080 static char *get_trace_buf(void)
3082 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3084 if (!buffer || buffer->nesting >= 4)
3089 /* Interrupts must see nesting incremented before we use the buffer */
3091 return &buffer->buffer[buffer->nesting][0];
3094 static void put_trace_buf(void)
3096 /* Don't let the decrement of nesting leak before this */
3098 this_cpu_dec(trace_percpu_buffer->nesting);
3101 static int alloc_percpu_trace_buffer(void)
3103 struct trace_buffer_struct *buffers;
3105 buffers = alloc_percpu(struct trace_buffer_struct);
3106 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
3109 trace_percpu_buffer = buffers;
3113 static int buffers_allocated;
3115 void trace_printk_init_buffers(void)
3117 if (buffers_allocated)
3120 if (alloc_percpu_trace_buffer())
3123 /* trace_printk() is for debug use only. Don't use it in production. */
3126 pr_warn("**********************************************************\n");
3127 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3129 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3131 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3132 pr_warn("** unsafe for production use. **\n");
3134 pr_warn("** If you see this message and you are not debugging **\n");
3135 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3137 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3138 pr_warn("**********************************************************\n");
3140 /* Expand the buffers to set size */
3141 tracing_update_buffers();
3143 buffers_allocated = 1;
3146 * trace_printk_init_buffers() can be called by modules.
3147 * If that happens, then we need to start cmdline recording
3148 * directly here. If the global_trace.buffer is already
3149 * allocated here, then this was called by module code.
3151 if (global_trace.trace_buffer.buffer)
3152 tracing_start_cmdline_record();
3154 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3156 void trace_printk_start_comm(void)
3158 /* Start tracing comms if trace printk is set */
3159 if (!buffers_allocated)
3161 tracing_start_cmdline_record();
3164 static void trace_printk_start_stop_comm(int enabled)
3166 if (!buffers_allocated)
3170 tracing_start_cmdline_record();
3172 tracing_stop_cmdline_record();
3176 * trace_vbprintk - write binary msg to tracing buffer
3177 * @ip: The address of the caller
3178 * @fmt: The string format to write to the buffer
3179 * @args: Arguments for @fmt
3181 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3183 struct trace_event_call *call = &event_bprint;
3184 struct ring_buffer_event *event;
3185 struct ring_buffer *buffer;
3186 struct trace_array *tr = &global_trace;
3187 struct bprint_entry *entry;
3188 unsigned long flags;
3190 int len = 0, size, pc;
3192 if (unlikely(tracing_selftest_running || tracing_disabled))
3195 /* Don't pollute graph traces with trace_vprintk internals */
3196 pause_graph_tracing();
3198 pc = preempt_count();
3199 preempt_disable_notrace();
3201 tbuffer = get_trace_buf();
3207 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3209 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3212 local_save_flags(flags);
3213 size = sizeof(*entry) + sizeof(u32) * len;
3214 buffer = tr->trace_buffer.buffer;
3215 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3219 entry = ring_buffer_event_data(event);
3223 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3224 if (!call_filter_check_discard(call, entry, buffer, event)) {
3225 __buffer_unlock_commit(buffer, event);
3226 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
3233 preempt_enable_notrace();
3234 unpause_graph_tracing();
3238 EXPORT_SYMBOL_GPL(trace_vbprintk);
3242 __trace_array_vprintk(struct ring_buffer *buffer,
3243 unsigned long ip, const char *fmt, va_list args)
3245 struct trace_event_call *call = &event_print;
3246 struct ring_buffer_event *event;
3247 int len = 0, size, pc;
3248 struct print_entry *entry;
3249 unsigned long flags;
3252 if (tracing_disabled || tracing_selftest_running)
3255 /* Don't pollute graph traces with trace_vprintk internals */
3256 pause_graph_tracing();
3258 pc = preempt_count();
3259 preempt_disable_notrace();
3262 tbuffer = get_trace_buf();
3268 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3270 local_save_flags(flags);
3271 size = sizeof(*entry) + len + 1;
3272 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3276 entry = ring_buffer_event_data(event);
3279 memcpy(&entry->buf, tbuffer, len + 1);
3280 if (!call_filter_check_discard(call, entry, buffer, event)) {
3281 __buffer_unlock_commit(buffer, event);
3282 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3289 preempt_enable_notrace();
3290 unpause_graph_tracing();
3296 int trace_array_vprintk(struct trace_array *tr,
3297 unsigned long ip, const char *fmt, va_list args)
3299 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3303 int trace_array_printk(struct trace_array *tr,
3304 unsigned long ip, const char *fmt, ...)
3309 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3316 ret = trace_array_vprintk(tr, ip, fmt, ap);
3320 EXPORT_SYMBOL_GPL(trace_array_printk);
3323 int trace_array_printk_buf(struct ring_buffer *buffer,
3324 unsigned long ip, const char *fmt, ...)
3329 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3333 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3339 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3341 return trace_array_vprintk(&global_trace, ip, fmt, args);
3343 EXPORT_SYMBOL_GPL(trace_vprintk);
3345 static void trace_iterator_increment(struct trace_iterator *iter)
3347 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3351 ring_buffer_read(buf_iter, NULL);
3354 static struct trace_entry *
3355 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3356 unsigned long *lost_events)
3358 struct ring_buffer_event *event;
3359 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3362 event = ring_buffer_iter_peek(buf_iter, ts);
3364 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
3368 iter->ent_size = ring_buffer_event_length(event);
3369 return ring_buffer_event_data(event);
3375 static struct trace_entry *
3376 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3377 unsigned long *missing_events, u64 *ent_ts)
3379 struct ring_buffer *buffer = iter->trace_buffer->buffer;
3380 struct trace_entry *ent, *next = NULL;
3381 unsigned long lost_events = 0, next_lost = 0;
3382 int cpu_file = iter->cpu_file;
3383 u64 next_ts = 0, ts;
3389 * If we are in a per_cpu trace file, don't bother by iterating over
3390 * all cpu and peek directly.
3392 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3393 if (ring_buffer_empty_cpu(buffer, cpu_file))
3395 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3397 *ent_cpu = cpu_file;
3402 for_each_tracing_cpu(cpu) {
3404 if (ring_buffer_empty_cpu(buffer, cpu))
3407 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3410 * Pick the entry with the smallest timestamp:
3412 if (ent && (!next || ts < next_ts)) {
3416 next_lost = lost_events;
3417 next_size = iter->ent_size;
3421 iter->ent_size = next_size;
3424 *ent_cpu = next_cpu;
3430 *missing_events = next_lost;
3435 /* Find the next real entry, without updating the iterator itself */
3436 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3437 int *ent_cpu, u64 *ent_ts)
3439 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3442 /* Find the next real entry, and increment the iterator to the next entry */
3443 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3445 iter->ent = __find_next_entry(iter, &iter->cpu,
3446 &iter->lost_events, &iter->ts);
3449 trace_iterator_increment(iter);
3451 return iter->ent ? iter : NULL;
3454 static void trace_consume(struct trace_iterator *iter)
3456 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
3457 &iter->lost_events);
3460 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3462 struct trace_iterator *iter = m->private;
3466 WARN_ON_ONCE(iter->leftover);
3470 /* can't go backwards */
3475 ent = trace_find_next_entry_inc(iter);
3479 while (ent && iter->idx < i)
3480 ent = trace_find_next_entry_inc(iter);
3487 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3489 struct ring_buffer_event *event;
3490 struct ring_buffer_iter *buf_iter;
3491 unsigned long entries = 0;
3494 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
3496 buf_iter = trace_buffer_iter(iter, cpu);
3500 ring_buffer_iter_reset(buf_iter);
3503 * We could have the case with the max latency tracers
3504 * that a reset never took place on a cpu. This is evident
3505 * by the timestamp being before the start of the buffer.
3507 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3508 if (ts >= iter->trace_buffer->time_start)
3511 ring_buffer_read(buf_iter, NULL);
3514 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
3518 * The current tracer is copied to avoid a global locking
3521 static void *s_start(struct seq_file *m, loff_t *pos)
3523 struct trace_iterator *iter = m->private;
3524 struct trace_array *tr = iter->tr;
3525 int cpu_file = iter->cpu_file;
3531 * copy the tracer to avoid using a global lock all around.
3532 * iter->trace is a copy of current_trace, the pointer to the
3533 * name may be used instead of a strcmp(), as iter->trace->name
3534 * will point to the same string as current_trace->name.
3536 mutex_lock(&trace_types_lock);
3537 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3538 *iter->trace = *tr->current_trace;
3539 mutex_unlock(&trace_types_lock);
3541 #ifdef CONFIG_TRACER_MAX_TRACE
3542 if (iter->snapshot && iter->trace->use_max_tr)
3543 return ERR_PTR(-EBUSY);
3546 if (!iter->snapshot)
3547 atomic_inc(&trace_record_taskinfo_disabled);
3549 if (*pos != iter->pos) {
3554 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3555 for_each_tracing_cpu(cpu)
3556 tracing_iter_reset(iter, cpu);
3558 tracing_iter_reset(iter, cpu_file);
3561 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3566 * If we overflowed the seq_file before, then we want
3567 * to just reuse the trace_seq buffer again.
3573 p = s_next(m, p, &l);
3577 trace_event_read_lock();
3578 trace_access_lock(cpu_file);
3582 static void s_stop(struct seq_file *m, void *p)
3584 struct trace_iterator *iter = m->private;
3586 #ifdef CONFIG_TRACER_MAX_TRACE
3587 if (iter->snapshot && iter->trace->use_max_tr)
3591 if (!iter->snapshot)
3592 atomic_dec(&trace_record_taskinfo_disabled);
3594 trace_access_unlock(iter->cpu_file);
3595 trace_event_read_unlock();
3599 get_total_entries_cpu(struct trace_buffer *buf, unsigned long *total,
3600 unsigned long *entries, int cpu)
3602 unsigned long count;
3604 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3606 * If this buffer has skipped entries, then we hold all
3607 * entries for the trace and we need to ignore the
3608 * ones before the time stamp.
3610 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3611 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3612 /* total is the same as the entries */
3616 ring_buffer_overrun_cpu(buf->buffer, cpu);
3621 get_total_entries(struct trace_buffer *buf,
3622 unsigned long *total, unsigned long *entries)
3630 for_each_tracing_cpu(cpu) {
3631 get_total_entries_cpu(buf, &t, &e, cpu);
3637 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3639 unsigned long total, entries;
3644 get_total_entries_cpu(&tr->trace_buffer, &total, &entries, cpu);
3649 unsigned long trace_total_entries(struct trace_array *tr)
3651 unsigned long total, entries;
3656 get_total_entries(&tr->trace_buffer, &total, &entries);
3661 static void print_lat_help_header(struct seq_file *m)
3663 seq_puts(m, "# _------=> CPU# \n"
3664 "# / _-----=> irqs-off \n"
3665 "# | / _----=> need-resched \n"
3666 "# || / _---=> hardirq/softirq \n"
3667 "# ||| / _--=> preempt-depth \n"
3669 "# cmd pid ||||| time | caller \n"
3670 "# \\ / ||||| \\ | / \n");
3673 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
3675 unsigned long total;
3676 unsigned long entries;
3678 get_total_entries(buf, &total, &entries);
3679 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3680 entries, total, num_online_cpus());
3684 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3687 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3689 print_event_info(buf, m);
3691 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3692 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3695 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3698 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3699 const char *space = " ";
3700 int prec = tgid ? 10 : 2;
3702 print_event_info(buf, m);
3704 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
3705 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
3706 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
3707 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
3708 seq_printf(m, "# %.*s||| / delay\n", prec, space);
3709 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
3710 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
3714 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3716 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3717 struct trace_buffer *buf = iter->trace_buffer;
3718 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3719 struct tracer *type = iter->trace;
3720 unsigned long entries;
3721 unsigned long total;
3722 const char *name = "preemption";
3726 get_total_entries(buf, &total, &entries);
3728 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3730 seq_puts(m, "# -----------------------------------"
3731 "---------------------------------\n");
3732 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3733 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3734 nsecs_to_usecs(data->saved_latency),
3738 #if defined(CONFIG_PREEMPT_NONE)
3740 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3742 #elif defined(CONFIG_PREEMPT)
3744 #elif defined(CONFIG_PREEMPT_RT)
3749 /* These are reserved for later use */
3752 seq_printf(m, " #P:%d)\n", num_online_cpus());
3756 seq_puts(m, "# -----------------\n");
3757 seq_printf(m, "# | task: %.16s-%d "
3758 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3759 data->comm, data->pid,
3760 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3761 data->policy, data->rt_priority);
3762 seq_puts(m, "# -----------------\n");
3764 if (data->critical_start) {
3765 seq_puts(m, "# => started at: ");
3766 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3767 trace_print_seq(m, &iter->seq);
3768 seq_puts(m, "\n# => ended at: ");
3769 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3770 trace_print_seq(m, &iter->seq);
3771 seq_puts(m, "\n#\n");
3777 static void test_cpu_buff_start(struct trace_iterator *iter)
3779 struct trace_seq *s = &iter->seq;
3780 struct trace_array *tr = iter->tr;
3782 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3785 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3788 if (cpumask_available(iter->started) &&
3789 cpumask_test_cpu(iter->cpu, iter->started))
3792 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3795 if (cpumask_available(iter->started))
3796 cpumask_set_cpu(iter->cpu, iter->started);
3798 /* Don't print started cpu buffer for the first entry of the trace */
3800 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3804 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3806 struct trace_array *tr = iter->tr;
3807 struct trace_seq *s = &iter->seq;
3808 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3809 struct trace_entry *entry;
3810 struct trace_event *event;
3814 test_cpu_buff_start(iter);
3816 event = ftrace_find_event(entry->type);
3818 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3819 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3820 trace_print_lat_context(iter);
3822 trace_print_context(iter);
3825 if (trace_seq_has_overflowed(s))
3826 return TRACE_TYPE_PARTIAL_LINE;
3829 return event->funcs->trace(iter, sym_flags, event);
3831 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3833 return trace_handle_return(s);
3836 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3838 struct trace_array *tr = iter->tr;
3839 struct trace_seq *s = &iter->seq;
3840 struct trace_entry *entry;
3841 struct trace_event *event;
3845 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3846 trace_seq_printf(s, "%d %d %llu ",
3847 entry->pid, iter->cpu, iter->ts);
3849 if (trace_seq_has_overflowed(s))
3850 return TRACE_TYPE_PARTIAL_LINE;
3852 event = ftrace_find_event(entry->type);
3854 return event->funcs->raw(iter, 0, event);
3856 trace_seq_printf(s, "%d ?\n", entry->type);
3858 return trace_handle_return(s);
3861 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3863 struct trace_array *tr = iter->tr;
3864 struct trace_seq *s = &iter->seq;
3865 unsigned char newline = '\n';
3866 struct trace_entry *entry;
3867 struct trace_event *event;
3871 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3872 SEQ_PUT_HEX_FIELD(s, entry->pid);
3873 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3874 SEQ_PUT_HEX_FIELD(s, iter->ts);
3875 if (trace_seq_has_overflowed(s))
3876 return TRACE_TYPE_PARTIAL_LINE;
3879 event = ftrace_find_event(entry->type);
3881 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3882 if (ret != TRACE_TYPE_HANDLED)
3886 SEQ_PUT_FIELD(s, newline);
3888 return trace_handle_return(s);
3891 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3893 struct trace_array *tr = iter->tr;
3894 struct trace_seq *s = &iter->seq;
3895 struct trace_entry *entry;
3896 struct trace_event *event;
3900 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3901 SEQ_PUT_FIELD(s, entry->pid);
3902 SEQ_PUT_FIELD(s, iter->cpu);
3903 SEQ_PUT_FIELD(s, iter->ts);
3904 if (trace_seq_has_overflowed(s))
3905 return TRACE_TYPE_PARTIAL_LINE;
3908 event = ftrace_find_event(entry->type);
3909 return event ? event->funcs->binary(iter, 0, event) :
3913 int trace_empty(struct trace_iterator *iter)
3915 struct ring_buffer_iter *buf_iter;
3918 /* If we are looking at one CPU buffer, only check that one */
3919 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3920 cpu = iter->cpu_file;
3921 buf_iter = trace_buffer_iter(iter, cpu);
3923 if (!ring_buffer_iter_empty(buf_iter))
3926 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3932 for_each_tracing_cpu(cpu) {
3933 buf_iter = trace_buffer_iter(iter, cpu);
3935 if (!ring_buffer_iter_empty(buf_iter))
3938 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3946 /* Called with trace_event_read_lock() held. */
3947 enum print_line_t print_trace_line(struct trace_iterator *iter)
3949 struct trace_array *tr = iter->tr;
3950 unsigned long trace_flags = tr->trace_flags;
3951 enum print_line_t ret;
3953 if (iter->lost_events) {
3954 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3955 iter->cpu, iter->lost_events);
3956 if (trace_seq_has_overflowed(&iter->seq))
3957 return TRACE_TYPE_PARTIAL_LINE;
3960 if (iter->trace && iter->trace->print_line) {
3961 ret = iter->trace->print_line(iter);
3962 if (ret != TRACE_TYPE_UNHANDLED)
3966 if (iter->ent->type == TRACE_BPUTS &&
3967 trace_flags & TRACE_ITER_PRINTK &&
3968 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3969 return trace_print_bputs_msg_only(iter);
3971 if (iter->ent->type == TRACE_BPRINT &&
3972 trace_flags & TRACE_ITER_PRINTK &&
3973 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3974 return trace_print_bprintk_msg_only(iter);
3976 if (iter->ent->type == TRACE_PRINT &&
3977 trace_flags & TRACE_ITER_PRINTK &&
3978 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3979 return trace_print_printk_msg_only(iter);
3981 if (trace_flags & TRACE_ITER_BIN)
3982 return print_bin_fmt(iter);
3984 if (trace_flags & TRACE_ITER_HEX)
3985 return print_hex_fmt(iter);
3987 if (trace_flags & TRACE_ITER_RAW)
3988 return print_raw_fmt(iter);
3990 return print_trace_fmt(iter);
3993 void trace_latency_header(struct seq_file *m)
3995 struct trace_iterator *iter = m->private;
3996 struct trace_array *tr = iter->tr;
3998 /* print nothing if the buffers are empty */
3999 if (trace_empty(iter))
4002 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4003 print_trace_header(m, iter);
4005 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4006 print_lat_help_header(m);
4009 void trace_default_header(struct seq_file *m)
4011 struct trace_iterator *iter = m->private;
4012 struct trace_array *tr = iter->tr;
4013 unsigned long trace_flags = tr->trace_flags;
4015 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4018 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4019 /* print nothing if the buffers are empty */
4020 if (trace_empty(iter))
4022 print_trace_header(m, iter);
4023 if (!(trace_flags & TRACE_ITER_VERBOSE))
4024 print_lat_help_header(m);
4026 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4027 if (trace_flags & TRACE_ITER_IRQ_INFO)
4028 print_func_help_header_irq(iter->trace_buffer,
4031 print_func_help_header(iter->trace_buffer, m,
4037 static void test_ftrace_alive(struct seq_file *m)
4039 if (!ftrace_is_dead())
4041 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4042 "# MAY BE MISSING FUNCTION EVENTS\n");
4045 #ifdef CONFIG_TRACER_MAX_TRACE
4046 static void show_snapshot_main_help(struct seq_file *m)
4048 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4049 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4050 "# Takes a snapshot of the main buffer.\n"
4051 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4052 "# (Doesn't have to be '2' works with any number that\n"
4053 "# is not a '0' or '1')\n");
4056 static void show_snapshot_percpu_help(struct seq_file *m)
4058 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4059 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4060 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4061 "# Takes a snapshot of the main buffer for this cpu.\n");
4063 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4064 "# Must use main snapshot file to allocate.\n");
4066 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4067 "# (Doesn't have to be '2' works with any number that\n"
4068 "# is not a '0' or '1')\n");
4071 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4073 if (iter->tr->allocated_snapshot)
4074 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4076 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4078 seq_puts(m, "# Snapshot commands:\n");
4079 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4080 show_snapshot_main_help(m);
4082 show_snapshot_percpu_help(m);
4085 /* Should never be called */
4086 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4089 static int s_show(struct seq_file *m, void *v)
4091 struct trace_iterator *iter = v;
4094 if (iter->ent == NULL) {
4096 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4098 test_ftrace_alive(m);
4100 if (iter->snapshot && trace_empty(iter))
4101 print_snapshot_help(m, iter);
4102 else if (iter->trace && iter->trace->print_header)
4103 iter->trace->print_header(m);
4105 trace_default_header(m);
4107 } else if (iter->leftover) {
4109 * If we filled the seq_file buffer earlier, we
4110 * want to just show it now.
4112 ret = trace_print_seq(m, &iter->seq);
4114 /* ret should this time be zero, but you never know */
4115 iter->leftover = ret;
4118 print_trace_line(iter);
4119 ret = trace_print_seq(m, &iter->seq);
4121 * If we overflow the seq_file buffer, then it will
4122 * ask us for this data again at start up.
4124 * ret is 0 if seq_file write succeeded.
4127 iter->leftover = ret;
4134 * Should be used after trace_array_get(), trace_types_lock
4135 * ensures that i_cdev was already initialized.
4137 static inline int tracing_get_cpu(struct inode *inode)
4139 if (inode->i_cdev) /* See trace_create_cpu_file() */
4140 return (long)inode->i_cdev - 1;
4141 return RING_BUFFER_ALL_CPUS;
4144 static const struct seq_operations tracer_seq_ops = {
4151 static struct trace_iterator *
4152 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4154 struct trace_array *tr = inode->i_private;
4155 struct trace_iterator *iter;
4158 if (tracing_disabled)
4159 return ERR_PTR(-ENODEV);
4161 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4163 return ERR_PTR(-ENOMEM);
4165 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4167 if (!iter->buffer_iter)
4171 * We make a copy of the current tracer to avoid concurrent
4172 * changes on it while we are reading.
4174 mutex_lock(&trace_types_lock);
4175 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4179 *iter->trace = *tr->current_trace;
4181 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4186 #ifdef CONFIG_TRACER_MAX_TRACE
4187 /* Currently only the top directory has a snapshot */
4188 if (tr->current_trace->print_max || snapshot)
4189 iter->trace_buffer = &tr->max_buffer;
4192 iter->trace_buffer = &tr->trace_buffer;
4193 iter->snapshot = snapshot;
4195 iter->cpu_file = tracing_get_cpu(inode);
4196 mutex_init(&iter->mutex);
4198 /* Notify the tracer early; before we stop tracing. */
4199 if (iter->trace && iter->trace->open)
4200 iter->trace->open(iter);
4202 /* Annotate start of buffers if we had overruns */
4203 if (ring_buffer_overruns(iter->trace_buffer->buffer))
4204 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4206 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4207 if (trace_clocks[tr->clock_id].in_ns)
4208 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4210 /* stop the trace while dumping if we are not opening "snapshot" */
4211 if (!iter->snapshot)
4212 tracing_stop_tr(tr);
4214 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4215 for_each_tracing_cpu(cpu) {
4216 iter->buffer_iter[cpu] =
4217 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4220 ring_buffer_read_prepare_sync();
4221 for_each_tracing_cpu(cpu) {
4222 ring_buffer_read_start(iter->buffer_iter[cpu]);
4223 tracing_iter_reset(iter, cpu);
4226 cpu = iter->cpu_file;
4227 iter->buffer_iter[cpu] =
4228 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4230 ring_buffer_read_prepare_sync();
4231 ring_buffer_read_start(iter->buffer_iter[cpu]);
4232 tracing_iter_reset(iter, cpu);
4235 mutex_unlock(&trace_types_lock);
4240 mutex_unlock(&trace_types_lock);
4242 kfree(iter->buffer_iter);
4244 seq_release_private(inode, file);
4245 return ERR_PTR(-ENOMEM);
4248 int tracing_open_generic(struct inode *inode, struct file *filp)
4252 ret = tracing_check_open_get_tr(NULL);
4256 filp->private_data = inode->i_private;
4260 bool tracing_is_disabled(void)
4262 return (tracing_disabled) ? true: false;
4266 * Open and update trace_array ref count.
4267 * Must have the current trace_array passed to it.
4269 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4271 struct trace_array *tr = inode->i_private;
4274 ret = tracing_check_open_get_tr(tr);
4278 filp->private_data = inode->i_private;
4283 static int tracing_release(struct inode *inode, struct file *file)
4285 struct trace_array *tr = inode->i_private;
4286 struct seq_file *m = file->private_data;
4287 struct trace_iterator *iter;
4290 if (!(file->f_mode & FMODE_READ)) {
4291 trace_array_put(tr);
4295 /* Writes do not use seq_file */
4297 mutex_lock(&trace_types_lock);
4299 for_each_tracing_cpu(cpu) {
4300 if (iter->buffer_iter[cpu])
4301 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4304 if (iter->trace && iter->trace->close)
4305 iter->trace->close(iter);
4307 if (!iter->snapshot)
4308 /* reenable tracing if it was previously enabled */
4309 tracing_start_tr(tr);
4311 __trace_array_put(tr);
4313 mutex_unlock(&trace_types_lock);
4315 mutex_destroy(&iter->mutex);
4316 free_cpumask_var(iter->started);
4318 kfree(iter->buffer_iter);
4319 seq_release_private(inode, file);
4324 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4326 struct trace_array *tr = inode->i_private;
4328 trace_array_put(tr);
4332 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4334 struct trace_array *tr = inode->i_private;
4336 trace_array_put(tr);
4338 return single_release(inode, file);
4341 static int tracing_open(struct inode *inode, struct file *file)
4343 struct trace_array *tr = inode->i_private;
4344 struct trace_iterator *iter;
4347 ret = tracing_check_open_get_tr(tr);
4351 /* If this file was open for write, then erase contents */
4352 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4353 int cpu = tracing_get_cpu(inode);
4354 struct trace_buffer *trace_buf = &tr->trace_buffer;
4356 #ifdef CONFIG_TRACER_MAX_TRACE
4357 if (tr->current_trace->print_max)
4358 trace_buf = &tr->max_buffer;
4361 if (cpu == RING_BUFFER_ALL_CPUS)
4362 tracing_reset_online_cpus(trace_buf);
4364 tracing_reset_cpu(trace_buf, cpu);
4367 if (file->f_mode & FMODE_READ) {
4368 iter = __tracing_open(inode, file, false);
4370 ret = PTR_ERR(iter);
4371 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4372 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4376 trace_array_put(tr);
4382 * Some tracers are not suitable for instance buffers.
4383 * A tracer is always available for the global array (toplevel)
4384 * or if it explicitly states that it is.
4387 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4389 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4392 /* Find the next tracer that this trace array may use */
4393 static struct tracer *
4394 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4396 while (t && !trace_ok_for_array(t, tr))
4403 t_next(struct seq_file *m, void *v, loff_t *pos)
4405 struct trace_array *tr = m->private;
4406 struct tracer *t = v;
4411 t = get_tracer_for_array(tr, t->next);
4416 static void *t_start(struct seq_file *m, loff_t *pos)
4418 struct trace_array *tr = m->private;
4422 mutex_lock(&trace_types_lock);
4424 t = get_tracer_for_array(tr, trace_types);
4425 for (; t && l < *pos; t = t_next(m, t, &l))
4431 static void t_stop(struct seq_file *m, void *p)
4433 mutex_unlock(&trace_types_lock);
4436 static int t_show(struct seq_file *m, void *v)
4438 struct tracer *t = v;
4443 seq_puts(m, t->name);
4452 static const struct seq_operations show_traces_seq_ops = {
4459 static int show_traces_open(struct inode *inode, struct file *file)
4461 struct trace_array *tr = inode->i_private;
4465 ret = tracing_check_open_get_tr(tr);
4469 ret = seq_open(file, &show_traces_seq_ops);
4471 trace_array_put(tr);
4475 m = file->private_data;
4481 static int show_traces_release(struct inode *inode, struct file *file)
4483 struct trace_array *tr = inode->i_private;
4485 trace_array_put(tr);
4486 return seq_release(inode, file);
4490 tracing_write_stub(struct file *filp, const char __user *ubuf,
4491 size_t count, loff_t *ppos)
4496 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4500 if (file->f_mode & FMODE_READ)
4501 ret = seq_lseek(file, offset, whence);
4503 file->f_pos = ret = 0;
4508 static const struct file_operations tracing_fops = {
4509 .open = tracing_open,
4511 .write = tracing_write_stub,
4512 .llseek = tracing_lseek,
4513 .release = tracing_release,
4516 static const struct file_operations show_traces_fops = {
4517 .open = show_traces_open,
4519 .llseek = seq_lseek,
4520 .release = show_traces_release,
4524 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4525 size_t count, loff_t *ppos)
4527 struct trace_array *tr = file_inode(filp)->i_private;
4531 len = snprintf(NULL, 0, "%*pb\n",
4532 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4533 mask_str = kmalloc(len, GFP_KERNEL);
4537 len = snprintf(mask_str, len, "%*pb\n",
4538 cpumask_pr_args(tr->tracing_cpumask));
4543 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4552 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4553 size_t count, loff_t *ppos)
4555 struct trace_array *tr = file_inode(filp)->i_private;
4556 cpumask_var_t tracing_cpumask_new;
4559 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4562 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4566 local_irq_disable();
4567 arch_spin_lock(&tr->max_lock);
4568 for_each_tracing_cpu(cpu) {
4570 * Increase/decrease the disabled counter if we are
4571 * about to flip a bit in the cpumask:
4573 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4574 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4575 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4576 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
4578 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4579 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4580 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4581 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
4584 arch_spin_unlock(&tr->max_lock);
4587 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4588 free_cpumask_var(tracing_cpumask_new);
4593 free_cpumask_var(tracing_cpumask_new);
4598 static const struct file_operations tracing_cpumask_fops = {
4599 .open = tracing_open_generic_tr,
4600 .read = tracing_cpumask_read,
4601 .write = tracing_cpumask_write,
4602 .release = tracing_release_generic_tr,
4603 .llseek = generic_file_llseek,
4606 static int tracing_trace_options_show(struct seq_file *m, void *v)
4608 struct tracer_opt *trace_opts;
4609 struct trace_array *tr = m->private;
4613 mutex_lock(&trace_types_lock);
4614 tracer_flags = tr->current_trace->flags->val;
4615 trace_opts = tr->current_trace->flags->opts;
4617 for (i = 0; trace_options[i]; i++) {
4618 if (tr->trace_flags & (1 << i))
4619 seq_printf(m, "%s\n", trace_options[i]);
4621 seq_printf(m, "no%s\n", trace_options[i]);
4624 for (i = 0; trace_opts[i].name; i++) {
4625 if (tracer_flags & trace_opts[i].bit)
4626 seq_printf(m, "%s\n", trace_opts[i].name);
4628 seq_printf(m, "no%s\n", trace_opts[i].name);
4630 mutex_unlock(&trace_types_lock);
4635 static int __set_tracer_option(struct trace_array *tr,
4636 struct tracer_flags *tracer_flags,
4637 struct tracer_opt *opts, int neg)
4639 struct tracer *trace = tracer_flags->trace;
4642 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4647 tracer_flags->val &= ~opts->bit;
4649 tracer_flags->val |= opts->bit;
4653 /* Try to assign a tracer specific option */
4654 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4656 struct tracer *trace = tr->current_trace;
4657 struct tracer_flags *tracer_flags = trace->flags;
4658 struct tracer_opt *opts = NULL;
4661 for (i = 0; tracer_flags->opts[i].name; i++) {
4662 opts = &tracer_flags->opts[i];
4664 if (strcmp(cmp, opts->name) == 0)
4665 return __set_tracer_option(tr, trace->flags, opts, neg);
4671 /* Some tracers require overwrite to stay enabled */
4672 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4674 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4680 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4682 /* do nothing if flag is already set */
4683 if (!!(tr->trace_flags & mask) == !!enabled)
4686 /* Give the tracer a chance to approve the change */
4687 if (tr->current_trace->flag_changed)
4688 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4692 tr->trace_flags |= mask;
4694 tr->trace_flags &= ~mask;
4696 if (mask == TRACE_ITER_RECORD_CMD)
4697 trace_event_enable_cmd_record(enabled);
4699 if (mask == TRACE_ITER_RECORD_TGID) {
4701 tgid_map = kvcalloc(PID_MAX_DEFAULT + 1,
4705 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4709 trace_event_enable_tgid_record(enabled);
4712 if (mask == TRACE_ITER_EVENT_FORK)
4713 trace_event_follow_fork(tr, enabled);
4715 if (mask == TRACE_ITER_FUNC_FORK)
4716 ftrace_pid_follow_fork(tr, enabled);
4718 if (mask == TRACE_ITER_OVERWRITE) {
4719 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
4720 #ifdef CONFIG_TRACER_MAX_TRACE
4721 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4725 if (mask == TRACE_ITER_PRINTK) {
4726 trace_printk_start_stop_comm(enabled);
4727 trace_printk_control(enabled);
4733 static int trace_set_options(struct trace_array *tr, char *option)
4738 size_t orig_len = strlen(option);
4741 cmp = strstrip(option);
4743 len = str_has_prefix(cmp, "no");
4749 mutex_lock(&trace_types_lock);
4751 ret = match_string(trace_options, -1, cmp);
4752 /* If no option could be set, test the specific tracer options */
4754 ret = set_tracer_option(tr, cmp, neg);
4756 ret = set_tracer_flag(tr, 1 << ret, !neg);
4758 mutex_unlock(&trace_types_lock);
4761 * If the first trailing whitespace is replaced with '\0' by strstrip,
4762 * turn it back into a space.
4764 if (orig_len > strlen(option))
4765 option[strlen(option)] = ' ';
4770 static void __init apply_trace_boot_options(void)
4772 char *buf = trace_boot_options_buf;
4776 option = strsep(&buf, ",");
4782 trace_set_options(&global_trace, option);
4784 /* Put back the comma to allow this to be called again */
4791 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4792 size_t cnt, loff_t *ppos)
4794 struct seq_file *m = filp->private_data;
4795 struct trace_array *tr = m->private;
4799 if (cnt >= sizeof(buf))
4802 if (copy_from_user(buf, ubuf, cnt))
4807 ret = trace_set_options(tr, buf);
4816 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4818 struct trace_array *tr = inode->i_private;
4821 ret = tracing_check_open_get_tr(tr);
4825 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4827 trace_array_put(tr);
4832 static const struct file_operations tracing_iter_fops = {
4833 .open = tracing_trace_options_open,
4835 .llseek = seq_lseek,
4836 .release = tracing_single_release_tr,
4837 .write = tracing_trace_options_write,
4840 static const char readme_msg[] =
4841 "tracing mini-HOWTO:\n\n"
4842 "# echo 0 > tracing_on : quick way to disable tracing\n"
4843 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4844 " Important files:\n"
4845 " trace\t\t\t- The static contents of the buffer\n"
4846 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4847 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4848 " current_tracer\t- function and latency tracers\n"
4849 " available_tracers\t- list of configured tracers for current_tracer\n"
4850 " error_log\t- error log for failed commands (that support it)\n"
4851 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4852 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4853 " trace_clock\t\t-change the clock used to order events\n"
4854 " local: Per cpu clock but may not be synced across CPUs\n"
4855 " global: Synced across CPUs but slows tracing down.\n"
4856 " counter: Not a clock, but just an increment\n"
4857 " uptime: Jiffy counter from time of boot\n"
4858 " perf: Same clock that perf events use\n"
4859 #ifdef CONFIG_X86_64
4860 " x86-tsc: TSC cycle counter\n"
4862 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4863 " delta: Delta difference against a buffer-wide timestamp\n"
4864 " absolute: Absolute (standalone) timestamp\n"
4865 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4866 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4867 " tracing_cpumask\t- Limit which CPUs to trace\n"
4868 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4869 "\t\t\t Remove sub-buffer with rmdir\n"
4870 " trace_options\t\t- Set format or modify how tracing happens\n"
4871 "\t\t\t Disable an option by prefixing 'no' to the\n"
4872 "\t\t\t option name\n"
4873 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4874 #ifdef CONFIG_DYNAMIC_FTRACE
4875 "\n available_filter_functions - list of functions that can be filtered on\n"
4876 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4877 "\t\t\t functions\n"
4878 "\t accepts: func_full_name or glob-matching-pattern\n"
4879 "\t modules: Can select a group via module\n"
4880 "\t Format: :mod:<module-name>\n"
4881 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4882 "\t triggers: a command to perform when function is hit\n"
4883 "\t Format: <function>:<trigger>[:count]\n"
4884 "\t trigger: traceon, traceoff\n"
4885 "\t\t enable_event:<system>:<event>\n"
4886 "\t\t disable_event:<system>:<event>\n"
4887 #ifdef CONFIG_STACKTRACE
4890 #ifdef CONFIG_TRACER_SNAPSHOT
4895 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4896 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4897 "\t The first one will disable tracing every time do_fault is hit\n"
4898 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4899 "\t The first time do trap is hit and it disables tracing, the\n"
4900 "\t counter will decrement to 2. If tracing is already disabled,\n"
4901 "\t the counter will not decrement. It only decrements when the\n"
4902 "\t trigger did work\n"
4903 "\t To remove trigger without count:\n"
4904 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4905 "\t To remove trigger with a count:\n"
4906 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4907 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4908 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4909 "\t modules: Can select a group via module command :mod:\n"
4910 "\t Does not accept triggers\n"
4911 #endif /* CONFIG_DYNAMIC_FTRACE */
4912 #ifdef CONFIG_FUNCTION_TRACER
4913 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4916 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4917 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4918 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4919 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4921 #ifdef CONFIG_TRACER_SNAPSHOT
4922 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4923 "\t\t\t snapshot buffer. Read the contents for more\n"
4924 "\t\t\t information\n"
4926 #ifdef CONFIG_STACK_TRACER
4927 " stack_trace\t\t- Shows the max stack trace when active\n"
4928 " stack_max_size\t- Shows current max stack size that was traced\n"
4929 "\t\t\t Write into this file to reset the max size (trigger a\n"
4930 "\t\t\t new trace)\n"
4931 #ifdef CONFIG_DYNAMIC_FTRACE
4932 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4935 #endif /* CONFIG_STACK_TRACER */
4936 #ifdef CONFIG_DYNAMIC_EVENTS
4937 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
4938 "\t\t\t Write into this file to define/undefine new trace events.\n"
4940 #ifdef CONFIG_KPROBE_EVENTS
4941 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
4942 "\t\t\t Write into this file to define/undefine new trace events.\n"
4944 #ifdef CONFIG_UPROBE_EVENTS
4945 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
4946 "\t\t\t Write into this file to define/undefine new trace events.\n"
4948 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4949 "\t accepts: event-definitions (one definition per line)\n"
4950 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4951 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4952 #ifdef CONFIG_HIST_TRIGGERS
4953 "\t s:[synthetic/]<event> <field> [<field>]\n"
4955 "\t -:[<group>/]<event>\n"
4956 #ifdef CONFIG_KPROBE_EVENTS
4957 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4958 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4960 #ifdef CONFIG_UPROBE_EVENTS
4961 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
4963 "\t args: <name>=fetcharg[:type]\n"
4964 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4965 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4966 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
4968 "\t $stack<index>, $stack, $retval, $comm,\n"
4970 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
4971 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4972 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
4973 "\t <type>\\[<array-size>\\]\n"
4974 #ifdef CONFIG_HIST_TRIGGERS
4975 "\t field: <stype> <name>;\n"
4976 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4977 "\t [unsigned] char/int/long\n"
4980 " events/\t\t- Directory containing all trace event subsystems:\n"
4981 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4982 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4983 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4985 " filter\t\t- If set, only events passing filter are traced\n"
4986 " events/<system>/<event>/\t- Directory containing control files for\n"
4988 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4989 " filter\t\t- If set, only events passing filter are traced\n"
4990 " trigger\t\t- If set, a command to perform when event is hit\n"
4991 "\t Format: <trigger>[:count][if <filter>]\n"
4992 "\t trigger: traceon, traceoff\n"
4993 "\t enable_event:<system>:<event>\n"
4994 "\t disable_event:<system>:<event>\n"
4995 #ifdef CONFIG_HIST_TRIGGERS
4996 "\t enable_hist:<system>:<event>\n"
4997 "\t disable_hist:<system>:<event>\n"
4999 #ifdef CONFIG_STACKTRACE
5002 #ifdef CONFIG_TRACER_SNAPSHOT
5005 #ifdef CONFIG_HIST_TRIGGERS
5006 "\t\t hist (see below)\n"
5008 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5009 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5010 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5011 "\t events/block/block_unplug/trigger\n"
5012 "\t The first disables tracing every time block_unplug is hit.\n"
5013 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5014 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5015 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5016 "\t Like function triggers, the counter is only decremented if it\n"
5017 "\t enabled or disabled tracing.\n"
5018 "\t To remove a trigger without a count:\n"
5019 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5020 "\t To remove a trigger with a count:\n"
5021 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5022 "\t Filters can be ignored when removing a trigger.\n"
5023 #ifdef CONFIG_HIST_TRIGGERS
5024 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5025 "\t Format: hist:keys=<field1[,field2,...]>\n"
5026 "\t [:values=<field1[,field2,...]>]\n"
5027 "\t [:sort=<field1[,field2,...]>]\n"
5028 "\t [:size=#entries]\n"
5029 "\t [:pause][:continue][:clear]\n"
5030 "\t [:name=histname1]\n"
5031 "\t [:<handler>.<action>]\n"
5032 "\t [if <filter>]\n\n"
5033 "\t When a matching event is hit, an entry is added to a hash\n"
5034 "\t table using the key(s) and value(s) named, and the value of a\n"
5035 "\t sum called 'hitcount' is incremented. Keys and values\n"
5036 "\t correspond to fields in the event's format description. Keys\n"
5037 "\t can be any field, or the special string 'stacktrace'.\n"
5038 "\t Compound keys consisting of up to two fields can be specified\n"
5039 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5040 "\t fields. Sort keys consisting of up to two fields can be\n"
5041 "\t specified using the 'sort' keyword. The sort direction can\n"
5042 "\t be modified by appending '.descending' or '.ascending' to a\n"
5043 "\t sort field. The 'size' parameter can be used to specify more\n"
5044 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5045 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5046 "\t its histogram data will be shared with other triggers of the\n"
5047 "\t same name, and trigger hits will update this common data.\n\n"
5048 "\t Reading the 'hist' file for the event will dump the hash\n"
5049 "\t table in its entirety to stdout. If there are multiple hist\n"
5050 "\t triggers attached to an event, there will be a table for each\n"
5051 "\t trigger in the output. The table displayed for a named\n"
5052 "\t trigger will be the same as any other instance having the\n"
5053 "\t same name. The default format used to display a given field\n"
5054 "\t can be modified by appending any of the following modifiers\n"
5055 "\t to the field name, as applicable:\n\n"
5056 "\t .hex display a number as a hex value\n"
5057 "\t .sym display an address as a symbol\n"
5058 "\t .sym-offset display an address as a symbol and offset\n"
5059 "\t .execname display a common_pid as a program name\n"
5060 "\t .syscall display a syscall id as a syscall name\n"
5061 "\t .log2 display log2 value rather than raw number\n"
5062 "\t .usecs display a common_timestamp in microseconds\n\n"
5063 "\t The 'pause' parameter can be used to pause an existing hist\n"
5064 "\t trigger or to start a hist trigger but not log any events\n"
5065 "\t until told to do so. 'continue' can be used to start or\n"
5066 "\t restart a paused hist trigger.\n\n"
5067 "\t The 'clear' parameter will clear the contents of a running\n"
5068 "\t hist trigger and leave its current paused/active state\n"
5070 "\t The enable_hist and disable_hist triggers can be used to\n"
5071 "\t have one event conditionally start and stop another event's\n"
5072 "\t already-attached hist trigger. The syntax is analogous to\n"
5073 "\t the enable_event and disable_event triggers.\n\n"
5074 "\t Hist trigger handlers and actions are executed whenever a\n"
5075 "\t a histogram entry is added or updated. They take the form:\n\n"
5076 "\t <handler>.<action>\n\n"
5077 "\t The available handlers are:\n\n"
5078 "\t onmatch(matching.event) - invoke on addition or update\n"
5079 "\t onmax(var) - invoke if var exceeds current max\n"
5080 "\t onchange(var) - invoke action if var changes\n\n"
5081 "\t The available actions are:\n\n"
5082 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5083 "\t save(field,...) - save current event fields\n"
5084 #ifdef CONFIG_TRACER_SNAPSHOT
5085 "\t snapshot() - snapshot the trace buffer\n"
5091 tracing_readme_read(struct file *filp, char __user *ubuf,
5092 size_t cnt, loff_t *ppos)
5094 return simple_read_from_buffer(ubuf, cnt, ppos,
5095 readme_msg, strlen(readme_msg));
5098 static const struct file_operations tracing_readme_fops = {
5099 .open = tracing_open_generic,
5100 .read = tracing_readme_read,
5101 .llseek = generic_file_llseek,
5104 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5108 if (*pos || m->count)
5113 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
5114 if (trace_find_tgid(*ptr))
5121 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5131 v = saved_tgids_next(m, v, &l);
5139 static void saved_tgids_stop(struct seq_file *m, void *v)
5143 static int saved_tgids_show(struct seq_file *m, void *v)
5145 int pid = (int *)v - tgid_map;
5147 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5151 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5152 .start = saved_tgids_start,
5153 .stop = saved_tgids_stop,
5154 .next = saved_tgids_next,
5155 .show = saved_tgids_show,
5158 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5162 ret = tracing_check_open_get_tr(NULL);
5166 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5170 static const struct file_operations tracing_saved_tgids_fops = {
5171 .open = tracing_saved_tgids_open,
5173 .llseek = seq_lseek,
5174 .release = seq_release,
5177 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5179 unsigned int *ptr = v;
5181 if (*pos || m->count)
5186 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5188 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5197 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5203 arch_spin_lock(&trace_cmdline_lock);
5205 v = &savedcmd->map_cmdline_to_pid[0];
5207 v = saved_cmdlines_next(m, v, &l);
5215 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5217 arch_spin_unlock(&trace_cmdline_lock);
5221 static int saved_cmdlines_show(struct seq_file *m, void *v)
5223 char buf[TASK_COMM_LEN];
5224 unsigned int *pid = v;
5226 __trace_find_cmdline(*pid, buf);
5227 seq_printf(m, "%d %s\n", *pid, buf);
5231 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5232 .start = saved_cmdlines_start,
5233 .next = saved_cmdlines_next,
5234 .stop = saved_cmdlines_stop,
5235 .show = saved_cmdlines_show,
5238 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5242 ret = tracing_check_open_get_tr(NULL);
5246 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5249 static const struct file_operations tracing_saved_cmdlines_fops = {
5250 .open = tracing_saved_cmdlines_open,
5252 .llseek = seq_lseek,
5253 .release = seq_release,
5257 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5258 size_t cnt, loff_t *ppos)
5263 arch_spin_lock(&trace_cmdline_lock);
5264 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5265 arch_spin_unlock(&trace_cmdline_lock);
5267 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5270 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5272 kfree(s->saved_cmdlines);
5273 kfree(s->map_cmdline_to_pid);
5277 static int tracing_resize_saved_cmdlines(unsigned int val)
5279 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5281 s = kmalloc(sizeof(*s), GFP_KERNEL);
5285 if (allocate_cmdlines_buffer(val, s) < 0) {
5290 arch_spin_lock(&trace_cmdline_lock);
5291 savedcmd_temp = savedcmd;
5293 arch_spin_unlock(&trace_cmdline_lock);
5294 free_saved_cmdlines_buffer(savedcmd_temp);
5300 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5301 size_t cnt, loff_t *ppos)
5306 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5310 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5311 if (!val || val > PID_MAX_DEFAULT)
5314 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5323 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5324 .open = tracing_open_generic,
5325 .read = tracing_saved_cmdlines_size_read,
5326 .write = tracing_saved_cmdlines_size_write,
5329 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5330 static union trace_eval_map_item *
5331 update_eval_map(union trace_eval_map_item *ptr)
5333 if (!ptr->map.eval_string) {
5334 if (ptr->tail.next) {
5335 ptr = ptr->tail.next;
5336 /* Set ptr to the next real item (skip head) */
5344 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5346 union trace_eval_map_item *ptr = v;
5349 * Paranoid! If ptr points to end, we don't want to increment past it.
5350 * This really should never happen.
5352 ptr = update_eval_map(ptr);
5353 if (WARN_ON_ONCE(!ptr))
5360 ptr = update_eval_map(ptr);
5365 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5367 union trace_eval_map_item *v;
5370 mutex_lock(&trace_eval_mutex);
5372 v = trace_eval_maps;
5376 while (v && l < *pos) {
5377 v = eval_map_next(m, v, &l);
5383 static void eval_map_stop(struct seq_file *m, void *v)
5385 mutex_unlock(&trace_eval_mutex);
5388 static int eval_map_show(struct seq_file *m, void *v)
5390 union trace_eval_map_item *ptr = v;
5392 seq_printf(m, "%s %ld (%s)\n",
5393 ptr->map.eval_string, ptr->map.eval_value,
5399 static const struct seq_operations tracing_eval_map_seq_ops = {
5400 .start = eval_map_start,
5401 .next = eval_map_next,
5402 .stop = eval_map_stop,
5403 .show = eval_map_show,
5406 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5410 ret = tracing_check_open_get_tr(NULL);
5414 return seq_open(filp, &tracing_eval_map_seq_ops);
5417 static const struct file_operations tracing_eval_map_fops = {
5418 .open = tracing_eval_map_open,
5420 .llseek = seq_lseek,
5421 .release = seq_release,
5424 static inline union trace_eval_map_item *
5425 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5427 /* Return tail of array given the head */
5428 return ptr + ptr->head.length + 1;
5432 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5435 struct trace_eval_map **stop;
5436 struct trace_eval_map **map;
5437 union trace_eval_map_item *map_array;
5438 union trace_eval_map_item *ptr;
5443 * The trace_eval_maps contains the map plus a head and tail item,
5444 * where the head holds the module and length of array, and the
5445 * tail holds a pointer to the next list.
5447 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5449 pr_warn("Unable to allocate trace eval mapping\n");
5453 mutex_lock(&trace_eval_mutex);
5455 if (!trace_eval_maps)
5456 trace_eval_maps = map_array;
5458 ptr = trace_eval_maps;
5460 ptr = trace_eval_jmp_to_tail(ptr);
5461 if (!ptr->tail.next)
5463 ptr = ptr->tail.next;
5466 ptr->tail.next = map_array;
5468 map_array->head.mod = mod;
5469 map_array->head.length = len;
5472 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5473 map_array->map = **map;
5476 memset(map_array, 0, sizeof(*map_array));
5478 mutex_unlock(&trace_eval_mutex);
5481 static void trace_create_eval_file(struct dentry *d_tracer)
5483 trace_create_file("eval_map", 0444, d_tracer,
5484 NULL, &tracing_eval_map_fops);
5487 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5488 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5489 static inline void trace_insert_eval_map_file(struct module *mod,
5490 struct trace_eval_map **start, int len) { }
5491 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5493 static void trace_insert_eval_map(struct module *mod,
5494 struct trace_eval_map **start, int len)
5496 struct trace_eval_map **map;
5503 trace_event_eval_update(map, len);
5505 trace_insert_eval_map_file(mod, start, len);
5509 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5510 size_t cnt, loff_t *ppos)
5512 struct trace_array *tr = filp->private_data;
5513 char buf[MAX_TRACER_SIZE+2];
5516 mutex_lock(&trace_types_lock);
5517 r = sprintf(buf, "%s\n", tr->current_trace->name);
5518 mutex_unlock(&trace_types_lock);
5520 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5523 int tracer_init(struct tracer *t, struct trace_array *tr)
5525 tracing_reset_online_cpus(&tr->trace_buffer);
5529 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
5533 for_each_tracing_cpu(cpu)
5534 per_cpu_ptr(buf->data, cpu)->entries = val;
5537 #ifdef CONFIG_TRACER_MAX_TRACE
5538 /* resize @tr's buffer to the size of @size_tr's entries */
5539 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5540 struct trace_buffer *size_buf, int cpu_id)
5544 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5545 for_each_tracing_cpu(cpu) {
5546 ret = ring_buffer_resize(trace_buf->buffer,
5547 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5550 per_cpu_ptr(trace_buf->data, cpu)->entries =
5551 per_cpu_ptr(size_buf->data, cpu)->entries;
5554 ret = ring_buffer_resize(trace_buf->buffer,
5555 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5557 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5558 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5563 #endif /* CONFIG_TRACER_MAX_TRACE */
5565 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5566 unsigned long size, int cpu)
5571 * If kernel or user changes the size of the ring buffer
5572 * we use the size that was given, and we can forget about
5573 * expanding it later.
5575 ring_buffer_expanded = true;
5577 /* May be called before buffers are initialized */
5578 if (!tr->trace_buffer.buffer)
5581 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
5585 #ifdef CONFIG_TRACER_MAX_TRACE
5586 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5587 !tr->current_trace->use_max_tr)
5590 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5592 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5593 &tr->trace_buffer, cpu);
5596 * AARGH! We are left with different
5597 * size max buffer!!!!
5598 * The max buffer is our "snapshot" buffer.
5599 * When a tracer needs a snapshot (one of the
5600 * latency tracers), it swaps the max buffer
5601 * with the saved snap shot. We succeeded to
5602 * update the size of the main buffer, but failed to
5603 * update the size of the max buffer. But when we tried
5604 * to reset the main buffer to the original size, we
5605 * failed there too. This is very unlikely to
5606 * happen, but if it does, warn and kill all
5610 tracing_disabled = 1;
5615 if (cpu == RING_BUFFER_ALL_CPUS)
5616 set_buffer_entries(&tr->max_buffer, size);
5618 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5621 #endif /* CONFIG_TRACER_MAX_TRACE */
5623 if (cpu == RING_BUFFER_ALL_CPUS)
5624 set_buffer_entries(&tr->trace_buffer, size);
5626 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
5631 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5632 unsigned long size, int cpu_id)
5636 mutex_lock(&trace_types_lock);
5638 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5639 /* make sure, this cpu is enabled in the mask */
5640 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5646 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5651 mutex_unlock(&trace_types_lock);
5658 * tracing_update_buffers - used by tracing facility to expand ring buffers
5660 * To save on memory when the tracing is never used on a system with it
5661 * configured in. The ring buffers are set to a minimum size. But once
5662 * a user starts to use the tracing facility, then they need to grow
5663 * to their default size.
5665 * This function is to be called when a tracer is about to be used.
5667 int tracing_update_buffers(void)
5671 mutex_lock(&trace_types_lock);
5672 if (!ring_buffer_expanded)
5673 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5674 RING_BUFFER_ALL_CPUS);
5675 mutex_unlock(&trace_types_lock);
5680 struct trace_option_dentry;
5683 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5686 * Used to clear out the tracer before deletion of an instance.
5687 * Must have trace_types_lock held.
5689 static void tracing_set_nop(struct trace_array *tr)
5691 if (tr->current_trace == &nop_trace)
5694 tr->current_trace->enabled--;
5696 if (tr->current_trace->reset)
5697 tr->current_trace->reset(tr);
5699 tr->current_trace = &nop_trace;
5702 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5704 /* Only enable if the directory has been created already. */
5708 create_trace_option_files(tr, t);
5711 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5714 #ifdef CONFIG_TRACER_MAX_TRACE
5719 mutex_lock(&trace_types_lock);
5721 if (!ring_buffer_expanded) {
5722 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5723 RING_BUFFER_ALL_CPUS);
5729 for (t = trace_types; t; t = t->next) {
5730 if (strcmp(t->name, buf) == 0)
5737 if (t == tr->current_trace)
5740 #ifdef CONFIG_TRACER_SNAPSHOT
5741 if (t->use_max_tr) {
5742 arch_spin_lock(&tr->max_lock);
5743 if (tr->cond_snapshot)
5745 arch_spin_unlock(&tr->max_lock);
5750 /* Some tracers won't work on kernel command line */
5751 if (system_state < SYSTEM_RUNNING && t->noboot) {
5752 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5757 /* Some tracers are only allowed for the top level buffer */
5758 if (!trace_ok_for_array(t, tr)) {
5763 /* If trace pipe files are being read, we can't change the tracer */
5764 if (tr->current_trace->ref) {
5769 trace_branch_disable();
5771 tr->current_trace->enabled--;
5773 if (tr->current_trace->reset)
5774 tr->current_trace->reset(tr);
5776 /* Current trace needs to be nop_trace before synchronize_rcu */
5777 tr->current_trace = &nop_trace;
5779 #ifdef CONFIG_TRACER_MAX_TRACE
5780 had_max_tr = tr->allocated_snapshot;
5782 if (had_max_tr && !t->use_max_tr) {
5784 * We need to make sure that the update_max_tr sees that
5785 * current_trace changed to nop_trace to keep it from
5786 * swapping the buffers after we resize it.
5787 * The update_max_tr is called from interrupts disabled
5788 * so a synchronized_sched() is sufficient.
5795 #ifdef CONFIG_TRACER_MAX_TRACE
5796 if (t->use_max_tr && !had_max_tr) {
5797 ret = tracing_alloc_snapshot_instance(tr);
5804 ret = tracer_init(t, tr);
5809 tr->current_trace = t;
5810 tr->current_trace->enabled++;
5811 trace_branch_enable(tr);
5813 mutex_unlock(&trace_types_lock);
5819 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5820 size_t cnt, loff_t *ppos)
5822 struct trace_array *tr = filp->private_data;
5823 char buf[MAX_TRACER_SIZE+1];
5830 if (cnt > MAX_TRACER_SIZE)
5831 cnt = MAX_TRACER_SIZE;
5833 if (copy_from_user(buf, ubuf, cnt))
5838 /* strip ending whitespace. */
5839 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5842 err = tracing_set_tracer(tr, buf);
5852 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5853 size_t cnt, loff_t *ppos)
5858 r = snprintf(buf, sizeof(buf), "%ld\n",
5859 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5860 if (r > sizeof(buf))
5862 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5866 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5867 size_t cnt, loff_t *ppos)
5872 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5882 tracing_thresh_read(struct file *filp, char __user *ubuf,
5883 size_t cnt, loff_t *ppos)
5885 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5889 tracing_thresh_write(struct file *filp, const char __user *ubuf,
5890 size_t cnt, loff_t *ppos)
5892 struct trace_array *tr = filp->private_data;
5895 mutex_lock(&trace_types_lock);
5896 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5900 if (tr->current_trace->update_thresh) {
5901 ret = tr->current_trace->update_thresh(tr);
5908 mutex_unlock(&trace_types_lock);
5913 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5916 tracing_max_lat_read(struct file *filp, char __user *ubuf,
5917 size_t cnt, loff_t *ppos)
5919 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5923 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5924 size_t cnt, loff_t *ppos)
5926 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5931 static int tracing_open_pipe(struct inode *inode, struct file *filp)
5933 struct trace_array *tr = inode->i_private;
5934 struct trace_iterator *iter;
5937 ret = tracing_check_open_get_tr(tr);
5941 mutex_lock(&trace_types_lock);
5943 /* create a buffer to store the information to pass to userspace */
5944 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5947 __trace_array_put(tr);
5951 trace_seq_init(&iter->seq);
5952 iter->trace = tr->current_trace;
5954 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5959 /* trace pipe does not show start of buffer */
5960 cpumask_setall(iter->started);
5962 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5963 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5965 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5966 if (trace_clocks[tr->clock_id].in_ns)
5967 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5970 iter->trace_buffer = &tr->trace_buffer;
5971 iter->cpu_file = tracing_get_cpu(inode);
5972 mutex_init(&iter->mutex);
5973 filp->private_data = iter;
5975 if (iter->trace->pipe_open)
5976 iter->trace->pipe_open(iter);
5978 nonseekable_open(inode, filp);
5980 tr->current_trace->ref++;
5982 mutex_unlock(&trace_types_lock);
5987 __trace_array_put(tr);
5988 mutex_unlock(&trace_types_lock);
5992 static int tracing_release_pipe(struct inode *inode, struct file *file)
5994 struct trace_iterator *iter = file->private_data;
5995 struct trace_array *tr = inode->i_private;
5997 mutex_lock(&trace_types_lock);
5999 tr->current_trace->ref--;
6001 if (iter->trace->pipe_close)
6002 iter->trace->pipe_close(iter);
6004 mutex_unlock(&trace_types_lock);
6006 free_cpumask_var(iter->started);
6007 mutex_destroy(&iter->mutex);
6010 trace_array_put(tr);
6016 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6018 struct trace_array *tr = iter->tr;
6020 /* Iterators are static, they should be filled or empty */
6021 if (trace_buffer_iter(iter, iter->cpu_file))
6022 return EPOLLIN | EPOLLRDNORM;
6024 if (tr->trace_flags & TRACE_ITER_BLOCK)
6026 * Always select as readable when in blocking mode
6028 return EPOLLIN | EPOLLRDNORM;
6030 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
6035 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6037 struct trace_iterator *iter = filp->private_data;
6039 return trace_poll(iter, filp, poll_table);
6042 /* Must be called with iter->mutex held. */
6043 static int tracing_wait_pipe(struct file *filp)
6045 struct trace_iterator *iter = filp->private_data;
6048 while (trace_empty(iter)) {
6050 if ((filp->f_flags & O_NONBLOCK)) {
6055 * We block until we read something and tracing is disabled.
6056 * We still block if tracing is disabled, but we have never
6057 * read anything. This allows a user to cat this file, and
6058 * then enable tracing. But after we have read something,
6059 * we give an EOF when tracing is again disabled.
6061 * iter->pos will be 0 if we haven't read anything.
6063 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6066 mutex_unlock(&iter->mutex);
6068 ret = wait_on_pipe(iter, 0);
6070 mutex_lock(&iter->mutex);
6083 tracing_read_pipe(struct file *filp, char __user *ubuf,
6084 size_t cnt, loff_t *ppos)
6086 struct trace_iterator *iter = filp->private_data;
6090 * Avoid more than one consumer on a single file descriptor
6091 * This is just a matter of traces coherency, the ring buffer itself
6094 mutex_lock(&iter->mutex);
6096 /* return any leftover data */
6097 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6101 trace_seq_init(&iter->seq);
6103 if (iter->trace->read) {
6104 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6110 sret = tracing_wait_pipe(filp);
6114 /* stop when tracing is finished */
6115 if (trace_empty(iter)) {
6120 if (cnt >= PAGE_SIZE)
6121 cnt = PAGE_SIZE - 1;
6123 /* reset all but tr, trace, and overruns */
6124 memset(&iter->seq, 0,
6125 sizeof(struct trace_iterator) -
6126 offsetof(struct trace_iterator, seq));
6127 cpumask_clear(iter->started);
6128 trace_seq_init(&iter->seq);
6131 trace_event_read_lock();
6132 trace_access_lock(iter->cpu_file);
6133 while (trace_find_next_entry_inc(iter) != NULL) {
6134 enum print_line_t ret;
6135 int save_len = iter->seq.seq.len;
6137 ret = print_trace_line(iter);
6138 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6139 /* don't print partial lines */
6140 iter->seq.seq.len = save_len;
6143 if (ret != TRACE_TYPE_NO_CONSUME)
6144 trace_consume(iter);
6146 if (trace_seq_used(&iter->seq) >= cnt)
6150 * Setting the full flag means we reached the trace_seq buffer
6151 * size and we should leave by partial output condition above.
6152 * One of the trace_seq_* functions is not used properly.
6154 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6157 trace_access_unlock(iter->cpu_file);
6158 trace_event_read_unlock();
6160 /* Now copy what we have to the user */
6161 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6162 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6163 trace_seq_init(&iter->seq);
6166 * If there was nothing to send to user, in spite of consuming trace
6167 * entries, go back to wait for more entries.
6173 mutex_unlock(&iter->mutex);
6178 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6181 __free_page(spd->pages[idx]);
6184 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
6185 .confirm = generic_pipe_buf_confirm,
6186 .release = generic_pipe_buf_release,
6187 .steal = generic_pipe_buf_steal,
6188 .get = generic_pipe_buf_get,
6192 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6198 /* Seq buffer is page-sized, exactly what we need. */
6200 save_len = iter->seq.seq.len;
6201 ret = print_trace_line(iter);
6203 if (trace_seq_has_overflowed(&iter->seq)) {
6204 iter->seq.seq.len = save_len;
6209 * This should not be hit, because it should only
6210 * be set if the iter->seq overflowed. But check it
6211 * anyway to be safe.
6213 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6214 iter->seq.seq.len = save_len;
6218 count = trace_seq_used(&iter->seq) - save_len;
6221 iter->seq.seq.len = save_len;
6225 if (ret != TRACE_TYPE_NO_CONSUME)
6226 trace_consume(iter);
6228 if (!trace_find_next_entry_inc(iter)) {
6238 static ssize_t tracing_splice_read_pipe(struct file *filp,
6240 struct pipe_inode_info *pipe,
6244 struct page *pages_def[PIPE_DEF_BUFFERS];
6245 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6246 struct trace_iterator *iter = filp->private_data;
6247 struct splice_pipe_desc spd = {
6249 .partial = partial_def,
6250 .nr_pages = 0, /* This gets updated below. */
6251 .nr_pages_max = PIPE_DEF_BUFFERS,
6252 .ops = &tracing_pipe_buf_ops,
6253 .spd_release = tracing_spd_release_pipe,
6259 if (splice_grow_spd(pipe, &spd))
6262 mutex_lock(&iter->mutex);
6264 if (iter->trace->splice_read) {
6265 ret = iter->trace->splice_read(iter, filp,
6266 ppos, pipe, len, flags);
6271 ret = tracing_wait_pipe(filp);
6275 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6280 trace_event_read_lock();
6281 trace_access_lock(iter->cpu_file);
6283 /* Fill as many pages as possible. */
6284 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6285 spd.pages[i] = alloc_page(GFP_KERNEL);
6289 rem = tracing_fill_pipe_page(rem, iter);
6291 /* Copy the data into the page, so we can start over. */
6292 ret = trace_seq_to_buffer(&iter->seq,
6293 page_address(spd.pages[i]),
6294 trace_seq_used(&iter->seq));
6296 __free_page(spd.pages[i]);
6299 spd.partial[i].offset = 0;
6300 spd.partial[i].len = trace_seq_used(&iter->seq);
6302 trace_seq_init(&iter->seq);
6305 trace_access_unlock(iter->cpu_file);
6306 trace_event_read_unlock();
6307 mutex_unlock(&iter->mutex);
6312 ret = splice_to_pipe(pipe, &spd);
6316 splice_shrink_spd(&spd);
6320 mutex_unlock(&iter->mutex);
6325 tracing_entries_read(struct file *filp, char __user *ubuf,
6326 size_t cnt, loff_t *ppos)
6328 struct inode *inode = file_inode(filp);
6329 struct trace_array *tr = inode->i_private;
6330 int cpu = tracing_get_cpu(inode);
6335 mutex_lock(&trace_types_lock);
6337 if (cpu == RING_BUFFER_ALL_CPUS) {
6338 int cpu, buf_size_same;
6343 /* check if all cpu sizes are same */
6344 for_each_tracing_cpu(cpu) {
6345 /* fill in the size from first enabled cpu */
6347 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
6348 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
6354 if (buf_size_same) {
6355 if (!ring_buffer_expanded)
6356 r = sprintf(buf, "%lu (expanded: %lu)\n",
6358 trace_buf_size >> 10);
6360 r = sprintf(buf, "%lu\n", size >> 10);
6362 r = sprintf(buf, "X\n");
6364 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
6366 mutex_unlock(&trace_types_lock);
6368 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6373 tracing_entries_write(struct file *filp, const char __user *ubuf,
6374 size_t cnt, loff_t *ppos)
6376 struct inode *inode = file_inode(filp);
6377 struct trace_array *tr = inode->i_private;
6381 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6385 /* must have at least 1 entry */
6389 /* value is in KB */
6391 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6401 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6402 size_t cnt, loff_t *ppos)
6404 struct trace_array *tr = filp->private_data;
6407 unsigned long size = 0, expanded_size = 0;
6409 mutex_lock(&trace_types_lock);
6410 for_each_tracing_cpu(cpu) {
6411 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
6412 if (!ring_buffer_expanded)
6413 expanded_size += trace_buf_size >> 10;
6415 if (ring_buffer_expanded)
6416 r = sprintf(buf, "%lu\n", size);
6418 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6419 mutex_unlock(&trace_types_lock);
6421 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6425 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6426 size_t cnt, loff_t *ppos)
6429 * There is no need to read what the user has written, this function
6430 * is just to make sure that there is no error when "echo" is used
6439 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6441 struct trace_array *tr = inode->i_private;
6443 /* disable tracing ? */
6444 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6445 tracer_tracing_off(tr);
6446 /* resize the ring buffer to 0 */
6447 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6449 trace_array_put(tr);
6455 tracing_mark_write(struct file *filp, const char __user *ubuf,
6456 size_t cnt, loff_t *fpos)
6458 struct trace_array *tr = filp->private_data;
6459 struct ring_buffer_event *event;
6460 enum event_trigger_type tt = ETT_NONE;
6461 struct ring_buffer *buffer;
6462 struct print_entry *entry;
6463 unsigned long irq_flags;
6468 /* Used in tracing_mark_raw_write() as well */
6469 #define FAULTED_STR "<faulted>"
6470 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6472 if (tracing_disabled)
6475 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6478 if (cnt > TRACE_BUF_SIZE)
6479 cnt = TRACE_BUF_SIZE;
6481 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6483 local_save_flags(irq_flags);
6484 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6486 /* If less than "<faulted>", then make sure we can still add that */
6487 if (cnt < FAULTED_SIZE)
6488 size += FAULTED_SIZE - cnt;
6490 buffer = tr->trace_buffer.buffer;
6491 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6492 irq_flags, preempt_count());
6493 if (unlikely(!event))
6494 /* Ring buffer disabled, return as if not open for write */
6497 entry = ring_buffer_event_data(event);
6498 entry->ip = _THIS_IP_;
6500 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6502 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6509 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6510 /* do not add \n before testing triggers, but add \0 */
6511 entry->buf[cnt] = '\0';
6512 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6515 if (entry->buf[cnt - 1] != '\n') {
6516 entry->buf[cnt] = '\n';
6517 entry->buf[cnt + 1] = '\0';
6519 entry->buf[cnt] = '\0';
6521 __buffer_unlock_commit(buffer, event);
6524 event_triggers_post_call(tr->trace_marker_file, tt);
6532 /* Limit it for now to 3K (including tag) */
6533 #define RAW_DATA_MAX_SIZE (1024*3)
6536 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6537 size_t cnt, loff_t *fpos)
6539 struct trace_array *tr = filp->private_data;
6540 struct ring_buffer_event *event;
6541 struct ring_buffer *buffer;
6542 struct raw_data_entry *entry;
6543 unsigned long irq_flags;
6548 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6550 if (tracing_disabled)
6553 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6556 /* The marker must at least have a tag id */
6557 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6560 if (cnt > TRACE_BUF_SIZE)
6561 cnt = TRACE_BUF_SIZE;
6563 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6565 local_save_flags(irq_flags);
6566 size = sizeof(*entry) + cnt;
6567 if (cnt < FAULT_SIZE_ID)
6568 size += FAULT_SIZE_ID - cnt;
6570 buffer = tr->trace_buffer.buffer;
6571 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6572 irq_flags, preempt_count());
6574 /* Ring buffer disabled, return as if not open for write */
6577 entry = ring_buffer_event_data(event);
6579 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6582 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6587 __buffer_unlock_commit(buffer, event);
6595 static int tracing_clock_show(struct seq_file *m, void *v)
6597 struct trace_array *tr = m->private;
6600 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6602 "%s%s%s%s", i ? " " : "",
6603 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6604 i == tr->clock_id ? "]" : "");
6610 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6614 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6615 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6618 if (i == ARRAY_SIZE(trace_clocks))
6621 mutex_lock(&trace_types_lock);
6625 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
6628 * New clock may not be consistent with the previous clock.
6629 * Reset the buffer so that it doesn't have incomparable timestamps.
6631 tracing_reset_online_cpus(&tr->trace_buffer);
6633 #ifdef CONFIG_TRACER_MAX_TRACE
6634 if (tr->max_buffer.buffer)
6635 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6636 tracing_reset_online_cpus(&tr->max_buffer);
6639 mutex_unlock(&trace_types_lock);
6644 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6645 size_t cnt, loff_t *fpos)
6647 struct seq_file *m = filp->private_data;
6648 struct trace_array *tr = m->private;
6650 const char *clockstr;
6653 if (cnt >= sizeof(buf))
6656 if (copy_from_user(buf, ubuf, cnt))
6661 clockstr = strstrip(buf);
6663 ret = tracing_set_clock(tr, clockstr);
6672 static int tracing_clock_open(struct inode *inode, struct file *file)
6674 struct trace_array *tr = inode->i_private;
6677 ret = tracing_check_open_get_tr(tr);
6681 ret = single_open(file, tracing_clock_show, inode->i_private);
6683 trace_array_put(tr);
6688 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6690 struct trace_array *tr = m->private;
6692 mutex_lock(&trace_types_lock);
6694 if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer))
6695 seq_puts(m, "delta [absolute]\n");
6697 seq_puts(m, "[delta] absolute\n");
6699 mutex_unlock(&trace_types_lock);
6704 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6706 struct trace_array *tr = inode->i_private;
6709 ret = tracing_check_open_get_tr(tr);
6713 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6715 trace_array_put(tr);
6720 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6724 mutex_lock(&trace_types_lock);
6726 if (abs && tr->time_stamp_abs_ref++)
6730 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6735 if (--tr->time_stamp_abs_ref)
6739 ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs);
6741 #ifdef CONFIG_TRACER_MAX_TRACE
6742 if (tr->max_buffer.buffer)
6743 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6746 mutex_unlock(&trace_types_lock);
6751 struct ftrace_buffer_info {
6752 struct trace_iterator iter;
6754 unsigned int spare_cpu;
6758 #ifdef CONFIG_TRACER_SNAPSHOT
6759 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6761 struct trace_array *tr = inode->i_private;
6762 struct trace_iterator *iter;
6766 ret = tracing_check_open_get_tr(tr);
6770 if (file->f_mode & FMODE_READ) {
6771 iter = __tracing_open(inode, file, true);
6773 ret = PTR_ERR(iter);
6775 /* Writes still need the seq_file to hold the private data */
6777 m = kzalloc(sizeof(*m), GFP_KERNEL);
6780 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6788 iter->trace_buffer = &tr->max_buffer;
6789 iter->cpu_file = tracing_get_cpu(inode);
6791 file->private_data = m;
6795 trace_array_put(tr);
6801 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6804 struct seq_file *m = filp->private_data;
6805 struct trace_iterator *iter = m->private;
6806 struct trace_array *tr = iter->tr;
6810 ret = tracing_update_buffers();
6814 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6818 mutex_lock(&trace_types_lock);
6820 if (tr->current_trace->use_max_tr) {
6825 arch_spin_lock(&tr->max_lock);
6826 if (tr->cond_snapshot)
6828 arch_spin_unlock(&tr->max_lock);
6834 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6838 if (tr->allocated_snapshot)
6842 /* Only allow per-cpu swap if the ring buffer supports it */
6843 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6844 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6849 if (tr->allocated_snapshot)
6850 ret = resize_buffer_duplicate_size(&tr->max_buffer,
6851 &tr->trace_buffer, iter->cpu_file);
6853 ret = tracing_alloc_snapshot_instance(tr);
6856 local_irq_disable();
6857 /* Now, we're going to swap */
6858 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6859 update_max_tr(tr, current, smp_processor_id(), NULL);
6861 update_max_tr_single(tr, current, iter->cpu_file);
6865 if (tr->allocated_snapshot) {
6866 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6867 tracing_reset_online_cpus(&tr->max_buffer);
6869 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
6879 mutex_unlock(&trace_types_lock);
6883 static int tracing_snapshot_release(struct inode *inode, struct file *file)
6885 struct seq_file *m = file->private_data;
6888 ret = tracing_release(inode, file);
6890 if (file->f_mode & FMODE_READ)
6893 /* If write only, the seq_file is just a stub */
6901 static int tracing_buffers_open(struct inode *inode, struct file *filp);
6902 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6903 size_t count, loff_t *ppos);
6904 static int tracing_buffers_release(struct inode *inode, struct file *file);
6905 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6906 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6908 static int snapshot_raw_open(struct inode *inode, struct file *filp)
6910 struct ftrace_buffer_info *info;
6913 /* The following checks for tracefs lockdown */
6914 ret = tracing_buffers_open(inode, filp);
6918 info = filp->private_data;
6920 if (info->iter.trace->use_max_tr) {
6921 tracing_buffers_release(inode, filp);
6925 info->iter.snapshot = true;
6926 info->iter.trace_buffer = &info->iter.tr->max_buffer;
6931 #endif /* CONFIG_TRACER_SNAPSHOT */
6934 static const struct file_operations tracing_thresh_fops = {
6935 .open = tracing_open_generic,
6936 .read = tracing_thresh_read,
6937 .write = tracing_thresh_write,
6938 .llseek = generic_file_llseek,
6941 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6942 static const struct file_operations tracing_max_lat_fops = {
6943 .open = tracing_open_generic,
6944 .read = tracing_max_lat_read,
6945 .write = tracing_max_lat_write,
6946 .llseek = generic_file_llseek,
6950 static const struct file_operations set_tracer_fops = {
6951 .open = tracing_open_generic,
6952 .read = tracing_set_trace_read,
6953 .write = tracing_set_trace_write,
6954 .llseek = generic_file_llseek,
6957 static const struct file_operations tracing_pipe_fops = {
6958 .open = tracing_open_pipe,
6959 .poll = tracing_poll_pipe,
6960 .read = tracing_read_pipe,
6961 .splice_read = tracing_splice_read_pipe,
6962 .release = tracing_release_pipe,
6963 .llseek = no_llseek,
6966 static const struct file_operations tracing_entries_fops = {
6967 .open = tracing_open_generic_tr,
6968 .read = tracing_entries_read,
6969 .write = tracing_entries_write,
6970 .llseek = generic_file_llseek,
6971 .release = tracing_release_generic_tr,
6974 static const struct file_operations tracing_total_entries_fops = {
6975 .open = tracing_open_generic_tr,
6976 .read = tracing_total_entries_read,
6977 .llseek = generic_file_llseek,
6978 .release = tracing_release_generic_tr,
6981 static const struct file_operations tracing_free_buffer_fops = {
6982 .open = tracing_open_generic_tr,
6983 .write = tracing_free_buffer_write,
6984 .release = tracing_free_buffer_release,
6987 static const struct file_operations tracing_mark_fops = {
6988 .open = tracing_open_generic_tr,
6989 .write = tracing_mark_write,
6990 .llseek = generic_file_llseek,
6991 .release = tracing_release_generic_tr,
6994 static const struct file_operations tracing_mark_raw_fops = {
6995 .open = tracing_open_generic_tr,
6996 .write = tracing_mark_raw_write,
6997 .llseek = generic_file_llseek,
6998 .release = tracing_release_generic_tr,
7001 static const struct file_operations trace_clock_fops = {
7002 .open = tracing_clock_open,
7004 .llseek = seq_lseek,
7005 .release = tracing_single_release_tr,
7006 .write = tracing_clock_write,
7009 static const struct file_operations trace_time_stamp_mode_fops = {
7010 .open = tracing_time_stamp_mode_open,
7012 .llseek = seq_lseek,
7013 .release = tracing_single_release_tr,
7016 #ifdef CONFIG_TRACER_SNAPSHOT
7017 static const struct file_operations snapshot_fops = {
7018 .open = tracing_snapshot_open,
7020 .write = tracing_snapshot_write,
7021 .llseek = tracing_lseek,
7022 .release = tracing_snapshot_release,
7025 static const struct file_operations snapshot_raw_fops = {
7026 .open = snapshot_raw_open,
7027 .read = tracing_buffers_read,
7028 .release = tracing_buffers_release,
7029 .splice_read = tracing_buffers_splice_read,
7030 .llseek = no_llseek,
7033 #endif /* CONFIG_TRACER_SNAPSHOT */
7035 #define TRACING_LOG_ERRS_MAX 8
7036 #define TRACING_LOG_LOC_MAX 128
7038 #define CMD_PREFIX " Command: "
7041 const char **errs; /* ptr to loc-specific array of err strings */
7042 u8 type; /* index into errs -> specific err string */
7043 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7047 struct tracing_log_err {
7048 struct list_head list;
7049 struct err_info info;
7050 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7051 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7054 static DEFINE_MUTEX(tracing_err_log_lock);
7056 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7058 struct tracing_log_err *err;
7060 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7061 err = kzalloc(sizeof(*err), GFP_KERNEL);
7063 err = ERR_PTR(-ENOMEM);
7064 tr->n_err_log_entries++;
7069 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7070 list_del(&err->list);
7076 * err_pos - find the position of a string within a command for error careting
7077 * @cmd: The tracing command that caused the error
7078 * @str: The string to position the caret at within @cmd
7080 * Finds the position of the first occurence of @str within @cmd. The
7081 * return value can be passed to tracing_log_err() for caret placement
7084 * Returns the index within @cmd of the first occurence of @str or 0
7085 * if @str was not found.
7087 unsigned int err_pos(char *cmd, const char *str)
7091 if (WARN_ON(!strlen(cmd)))
7094 found = strstr(cmd, str);
7102 * tracing_log_err - write an error to the tracing error log
7103 * @tr: The associated trace array for the error (NULL for top level array)
7104 * @loc: A string describing where the error occurred
7105 * @cmd: The tracing command that caused the error
7106 * @errs: The array of loc-specific static error strings
7107 * @type: The index into errs[], which produces the specific static err string
7108 * @pos: The position the caret should be placed in the cmd
7110 * Writes an error into tracing/error_log of the form:
7112 * <loc>: error: <text>
7116 * tracing/error_log is a small log file containing the last
7117 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7118 * unless there has been a tracing error, and the error log can be
7119 * cleared and have its memory freed by writing the empty string in
7120 * truncation mode to it i.e. echo > tracing/error_log.
7122 * NOTE: the @errs array along with the @type param are used to
7123 * produce a static error string - this string is not copied and saved
7124 * when the error is logged - only a pointer to it is saved. See
7125 * existing callers for examples of how static strings are typically
7126 * defined for use with tracing_log_err().
7128 void tracing_log_err(struct trace_array *tr,
7129 const char *loc, const char *cmd,
7130 const char **errs, u8 type, u8 pos)
7132 struct tracing_log_err *err;
7137 mutex_lock(&tracing_err_log_lock);
7138 err = get_tracing_log_err(tr);
7139 if (PTR_ERR(err) == -ENOMEM) {
7140 mutex_unlock(&tracing_err_log_lock);
7144 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7145 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7147 err->info.errs = errs;
7148 err->info.type = type;
7149 err->info.pos = pos;
7150 err->info.ts = local_clock();
7152 list_add_tail(&err->list, &tr->err_log);
7153 mutex_unlock(&tracing_err_log_lock);
7156 static void clear_tracing_err_log(struct trace_array *tr)
7158 struct tracing_log_err *err, *next;
7160 mutex_lock(&tracing_err_log_lock);
7161 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7162 list_del(&err->list);
7166 tr->n_err_log_entries = 0;
7167 mutex_unlock(&tracing_err_log_lock);
7170 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7172 struct trace_array *tr = m->private;
7174 mutex_lock(&tracing_err_log_lock);
7176 return seq_list_start(&tr->err_log, *pos);
7179 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7181 struct trace_array *tr = m->private;
7183 return seq_list_next(v, &tr->err_log, pos);
7186 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7188 mutex_unlock(&tracing_err_log_lock);
7191 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7195 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7197 for (i = 0; i < pos; i++)
7202 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7204 struct tracing_log_err *err = v;
7207 const char *err_text = err->info.errs[err->info.type];
7208 u64 sec = err->info.ts;
7211 nsec = do_div(sec, NSEC_PER_SEC);
7212 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7213 err->loc, err_text);
7214 seq_printf(m, "%s", err->cmd);
7215 tracing_err_log_show_pos(m, err->info.pos);
7221 static const struct seq_operations tracing_err_log_seq_ops = {
7222 .start = tracing_err_log_seq_start,
7223 .next = tracing_err_log_seq_next,
7224 .stop = tracing_err_log_seq_stop,
7225 .show = tracing_err_log_seq_show
7228 static int tracing_err_log_open(struct inode *inode, struct file *file)
7230 struct trace_array *tr = inode->i_private;
7233 ret = tracing_check_open_get_tr(tr);
7237 /* If this file was opened for write, then erase contents */
7238 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7239 clear_tracing_err_log(tr);
7241 if (file->f_mode & FMODE_READ) {
7242 ret = seq_open(file, &tracing_err_log_seq_ops);
7244 struct seq_file *m = file->private_data;
7247 trace_array_put(tr);
7253 static ssize_t tracing_err_log_write(struct file *file,
7254 const char __user *buffer,
7255 size_t count, loff_t *ppos)
7260 static int tracing_err_log_release(struct inode *inode, struct file *file)
7262 struct trace_array *tr = inode->i_private;
7264 trace_array_put(tr);
7266 if (file->f_mode & FMODE_READ)
7267 seq_release(inode, file);
7272 static const struct file_operations tracing_err_log_fops = {
7273 .open = tracing_err_log_open,
7274 .write = tracing_err_log_write,
7276 .llseek = seq_lseek,
7277 .release = tracing_err_log_release,
7280 static int tracing_buffers_open(struct inode *inode, struct file *filp)
7282 struct trace_array *tr = inode->i_private;
7283 struct ftrace_buffer_info *info;
7286 ret = tracing_check_open_get_tr(tr);
7290 info = kzalloc(sizeof(*info), GFP_KERNEL);
7292 trace_array_put(tr);
7296 mutex_lock(&trace_types_lock);
7299 info->iter.cpu_file = tracing_get_cpu(inode);
7300 info->iter.trace = tr->current_trace;
7301 info->iter.trace_buffer = &tr->trace_buffer;
7303 /* Force reading ring buffer for first read */
7304 info->read = (unsigned int)-1;
7306 filp->private_data = info;
7308 tr->current_trace->ref++;
7310 mutex_unlock(&trace_types_lock);
7312 ret = nonseekable_open(inode, filp);
7314 trace_array_put(tr);
7320 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7322 struct ftrace_buffer_info *info = filp->private_data;
7323 struct trace_iterator *iter = &info->iter;
7325 return trace_poll(iter, filp, poll_table);
7329 tracing_buffers_read(struct file *filp, char __user *ubuf,
7330 size_t count, loff_t *ppos)
7332 struct ftrace_buffer_info *info = filp->private_data;
7333 struct trace_iterator *iter = &info->iter;
7340 #ifdef CONFIG_TRACER_MAX_TRACE
7341 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7346 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
7348 if (IS_ERR(info->spare)) {
7349 ret = PTR_ERR(info->spare);
7352 info->spare_cpu = iter->cpu_file;
7358 /* Do we have previous read data to read? */
7359 if (info->read < PAGE_SIZE)
7363 trace_access_lock(iter->cpu_file);
7364 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
7368 trace_access_unlock(iter->cpu_file);
7371 if (trace_empty(iter)) {
7372 if ((filp->f_flags & O_NONBLOCK))
7375 ret = wait_on_pipe(iter, 0);
7386 size = PAGE_SIZE - info->read;
7390 ret = copy_to_user(ubuf, info->spare + info->read, size);
7402 static int tracing_buffers_release(struct inode *inode, struct file *file)
7404 struct ftrace_buffer_info *info = file->private_data;
7405 struct trace_iterator *iter = &info->iter;
7407 mutex_lock(&trace_types_lock);
7409 iter->tr->current_trace->ref--;
7411 __trace_array_put(iter->tr);
7414 ring_buffer_free_read_page(iter->trace_buffer->buffer,
7415 info->spare_cpu, info->spare);
7418 mutex_unlock(&trace_types_lock);
7424 struct ring_buffer *buffer;
7427 refcount_t refcount;
7430 static void buffer_ref_release(struct buffer_ref *ref)
7432 if (!refcount_dec_and_test(&ref->refcount))
7434 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7438 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7439 struct pipe_buffer *buf)
7441 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7443 buffer_ref_release(ref);
7447 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7448 struct pipe_buffer *buf)
7450 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7452 if (refcount_read(&ref->refcount) > INT_MAX/2)
7455 refcount_inc(&ref->refcount);
7459 /* Pipe buffer operations for a buffer. */
7460 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7461 .confirm = generic_pipe_buf_confirm,
7462 .release = buffer_pipe_buf_release,
7463 .steal = generic_pipe_buf_nosteal,
7464 .get = buffer_pipe_buf_get,
7468 * Callback from splice_to_pipe(), if we need to release some pages
7469 * at the end of the spd in case we error'ed out in filling the pipe.
7471 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7473 struct buffer_ref *ref =
7474 (struct buffer_ref *)spd->partial[i].private;
7476 buffer_ref_release(ref);
7477 spd->partial[i].private = 0;
7481 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7482 struct pipe_inode_info *pipe, size_t len,
7485 struct ftrace_buffer_info *info = file->private_data;
7486 struct trace_iterator *iter = &info->iter;
7487 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7488 struct page *pages_def[PIPE_DEF_BUFFERS];
7489 struct splice_pipe_desc spd = {
7491 .partial = partial_def,
7492 .nr_pages_max = PIPE_DEF_BUFFERS,
7493 .ops = &buffer_pipe_buf_ops,
7494 .spd_release = buffer_spd_release,
7496 struct buffer_ref *ref;
7500 #ifdef CONFIG_TRACER_MAX_TRACE
7501 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7505 if (*ppos & (PAGE_SIZE - 1))
7508 if (len & (PAGE_SIZE - 1)) {
7509 if (len < PAGE_SIZE)
7514 if (splice_grow_spd(pipe, &spd))
7518 trace_access_lock(iter->cpu_file);
7519 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
7521 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7525 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7531 refcount_set(&ref->refcount, 1);
7532 ref->buffer = iter->trace_buffer->buffer;
7533 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7534 if (IS_ERR(ref->page)) {
7535 ret = PTR_ERR(ref->page);
7540 ref->cpu = iter->cpu_file;
7542 r = ring_buffer_read_page(ref->buffer, &ref->page,
7543 len, iter->cpu_file, 1);
7545 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7551 page = virt_to_page(ref->page);
7553 spd.pages[i] = page;
7554 spd.partial[i].len = PAGE_SIZE;
7555 spd.partial[i].offset = 0;
7556 spd.partial[i].private = (unsigned long)ref;
7560 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
7563 trace_access_unlock(iter->cpu_file);
7566 /* did we read anything? */
7567 if (!spd.nr_pages) {
7572 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7575 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7582 ret = splice_to_pipe(pipe, &spd);
7584 splice_shrink_spd(&spd);
7589 static const struct file_operations tracing_buffers_fops = {
7590 .open = tracing_buffers_open,
7591 .read = tracing_buffers_read,
7592 .poll = tracing_buffers_poll,
7593 .release = tracing_buffers_release,
7594 .splice_read = tracing_buffers_splice_read,
7595 .llseek = no_llseek,
7599 tracing_stats_read(struct file *filp, char __user *ubuf,
7600 size_t count, loff_t *ppos)
7602 struct inode *inode = file_inode(filp);
7603 struct trace_array *tr = inode->i_private;
7604 struct trace_buffer *trace_buf = &tr->trace_buffer;
7605 int cpu = tracing_get_cpu(inode);
7606 struct trace_seq *s;
7608 unsigned long long t;
7609 unsigned long usec_rem;
7611 s = kmalloc(sizeof(*s), GFP_KERNEL);
7617 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7618 trace_seq_printf(s, "entries: %ld\n", cnt);
7620 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7621 trace_seq_printf(s, "overrun: %ld\n", cnt);
7623 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7624 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7626 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7627 trace_seq_printf(s, "bytes: %ld\n", cnt);
7629 if (trace_clocks[tr->clock_id].in_ns) {
7630 /* local or global for trace_clock */
7631 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7632 usec_rem = do_div(t, USEC_PER_SEC);
7633 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7636 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7637 usec_rem = do_div(t, USEC_PER_SEC);
7638 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7640 /* counter or tsc mode for trace_clock */
7641 trace_seq_printf(s, "oldest event ts: %llu\n",
7642 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7644 trace_seq_printf(s, "now ts: %llu\n",
7645 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7648 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7649 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7651 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7652 trace_seq_printf(s, "read events: %ld\n", cnt);
7654 count = simple_read_from_buffer(ubuf, count, ppos,
7655 s->buffer, trace_seq_used(s));
7662 static const struct file_operations tracing_stats_fops = {
7663 .open = tracing_open_generic_tr,
7664 .read = tracing_stats_read,
7665 .llseek = generic_file_llseek,
7666 .release = tracing_release_generic_tr,
7669 #ifdef CONFIG_DYNAMIC_FTRACE
7672 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7673 size_t cnt, loff_t *ppos)
7679 /* 256 should be plenty to hold the amount needed */
7680 buf = kmalloc(256, GFP_KERNEL);
7684 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
7685 ftrace_update_tot_cnt,
7686 ftrace_number_of_pages,
7687 ftrace_number_of_groups);
7689 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7694 static const struct file_operations tracing_dyn_info_fops = {
7695 .open = tracing_open_generic,
7696 .read = tracing_read_dyn_info,
7697 .llseek = generic_file_llseek,
7699 #endif /* CONFIG_DYNAMIC_FTRACE */
7701 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7703 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7704 struct trace_array *tr, struct ftrace_probe_ops *ops,
7707 tracing_snapshot_instance(tr);
7711 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7712 struct trace_array *tr, struct ftrace_probe_ops *ops,
7715 struct ftrace_func_mapper *mapper = data;
7719 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7729 tracing_snapshot_instance(tr);
7733 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7734 struct ftrace_probe_ops *ops, void *data)
7736 struct ftrace_func_mapper *mapper = data;
7739 seq_printf(m, "%ps:", (void *)ip);
7741 seq_puts(m, "snapshot");
7744 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7747 seq_printf(m, ":count=%ld\n", *count);
7749 seq_puts(m, ":unlimited\n");
7755 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7756 unsigned long ip, void *init_data, void **data)
7758 struct ftrace_func_mapper *mapper = *data;
7761 mapper = allocate_ftrace_func_mapper();
7767 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7771 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7772 unsigned long ip, void *data)
7774 struct ftrace_func_mapper *mapper = data;
7779 free_ftrace_func_mapper(mapper, NULL);
7783 ftrace_func_mapper_remove_ip(mapper, ip);
7786 static struct ftrace_probe_ops snapshot_probe_ops = {
7787 .func = ftrace_snapshot,
7788 .print = ftrace_snapshot_print,
7791 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7792 .func = ftrace_count_snapshot,
7793 .print = ftrace_snapshot_print,
7794 .init = ftrace_snapshot_init,
7795 .free = ftrace_snapshot_free,
7799 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7800 char *glob, char *cmd, char *param, int enable)
7802 struct ftrace_probe_ops *ops;
7803 void *count = (void *)-1;
7810 /* hash funcs only work with set_ftrace_filter */
7814 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7817 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7822 number = strsep(¶m, ":");
7824 if (!strlen(number))
7828 * We use the callback data field (which is a pointer)
7831 ret = kstrtoul(number, 0, (unsigned long *)&count);
7836 ret = tracing_alloc_snapshot_instance(tr);
7840 ret = register_ftrace_function_probe(glob, tr, ops, count);
7843 return ret < 0 ? ret : 0;
7846 static struct ftrace_func_command ftrace_snapshot_cmd = {
7848 .func = ftrace_trace_snapshot_callback,
7851 static __init int register_snapshot_cmd(void)
7853 return register_ftrace_command(&ftrace_snapshot_cmd);
7856 static inline __init int register_snapshot_cmd(void) { return 0; }
7857 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7859 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7861 if (WARN_ON(!tr->dir))
7862 return ERR_PTR(-ENODEV);
7864 /* Top directory uses NULL as the parent */
7865 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7868 /* All sub buffers have a descriptor */
7872 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7874 struct dentry *d_tracer;
7877 return tr->percpu_dir;
7879 d_tracer = tracing_get_dentry(tr);
7880 if (IS_ERR(d_tracer))
7883 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7885 WARN_ONCE(!tr->percpu_dir,
7886 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7888 return tr->percpu_dir;
7891 static struct dentry *
7892 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7893 void *data, long cpu, const struct file_operations *fops)
7895 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7897 if (ret) /* See tracing_get_cpu() */
7898 d_inode(ret)->i_cdev = (void *)(cpu + 1);
7903 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7905 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7906 struct dentry *d_cpu;
7907 char cpu_dir[30]; /* 30 characters should be more than enough */
7912 snprintf(cpu_dir, 30, "cpu%ld", cpu);
7913 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7915 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7919 /* per cpu trace_pipe */
7920 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
7921 tr, cpu, &tracing_pipe_fops);
7924 trace_create_cpu_file("trace", 0644, d_cpu,
7925 tr, cpu, &tracing_fops);
7927 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
7928 tr, cpu, &tracing_buffers_fops);
7930 trace_create_cpu_file("stats", 0444, d_cpu,
7931 tr, cpu, &tracing_stats_fops);
7933 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
7934 tr, cpu, &tracing_entries_fops);
7936 #ifdef CONFIG_TRACER_SNAPSHOT
7937 trace_create_cpu_file("snapshot", 0644, d_cpu,
7938 tr, cpu, &snapshot_fops);
7940 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
7941 tr, cpu, &snapshot_raw_fops);
7945 #ifdef CONFIG_FTRACE_SELFTEST
7946 /* Let selftest have access to static functions in this file */
7947 #include "trace_selftest.c"
7951 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7954 struct trace_option_dentry *topt = filp->private_data;
7957 if (topt->flags->val & topt->opt->bit)
7962 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7966 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7969 struct trace_option_dentry *topt = filp->private_data;
7973 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7977 if (val != 0 && val != 1)
7980 if (!!(topt->flags->val & topt->opt->bit) != val) {
7981 mutex_lock(&trace_types_lock);
7982 ret = __set_tracer_option(topt->tr, topt->flags,
7984 mutex_unlock(&trace_types_lock);
7995 static const struct file_operations trace_options_fops = {
7996 .open = tracing_open_generic,
7997 .read = trace_options_read,
7998 .write = trace_options_write,
7999 .llseek = generic_file_llseek,
8003 * In order to pass in both the trace_array descriptor as well as the index
8004 * to the flag that the trace option file represents, the trace_array
8005 * has a character array of trace_flags_index[], which holds the index
8006 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8007 * The address of this character array is passed to the flag option file
8008 * read/write callbacks.
8010 * In order to extract both the index and the trace_array descriptor,
8011 * get_tr_index() uses the following algorithm.
8015 * As the pointer itself contains the address of the index (remember
8018 * Then to get the trace_array descriptor, by subtracting that index
8019 * from the ptr, we get to the start of the index itself.
8021 * ptr - idx == &index[0]
8023 * Then a simple container_of() from that pointer gets us to the
8024 * trace_array descriptor.
8026 static void get_tr_index(void *data, struct trace_array **ptr,
8027 unsigned int *pindex)
8029 *pindex = *(unsigned char *)data;
8031 *ptr = container_of(data - *pindex, struct trace_array,
8036 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8039 void *tr_index = filp->private_data;
8040 struct trace_array *tr;
8044 get_tr_index(tr_index, &tr, &index);
8046 if (tr->trace_flags & (1 << index))
8051 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8055 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8058 void *tr_index = filp->private_data;
8059 struct trace_array *tr;
8064 get_tr_index(tr_index, &tr, &index);
8066 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8070 if (val != 0 && val != 1)
8073 mutex_lock(&trace_types_lock);
8074 ret = set_tracer_flag(tr, 1 << index, val);
8075 mutex_unlock(&trace_types_lock);
8085 static const struct file_operations trace_options_core_fops = {
8086 .open = tracing_open_generic,
8087 .read = trace_options_core_read,
8088 .write = trace_options_core_write,
8089 .llseek = generic_file_llseek,
8092 struct dentry *trace_create_file(const char *name,
8094 struct dentry *parent,
8096 const struct file_operations *fops)
8100 ret = tracefs_create_file(name, mode, parent, data, fops);
8102 pr_warn("Could not create tracefs '%s' entry\n", name);
8108 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8110 struct dentry *d_tracer;
8115 d_tracer = tracing_get_dentry(tr);
8116 if (IS_ERR(d_tracer))
8119 tr->options = tracefs_create_dir("options", d_tracer);
8121 pr_warn("Could not create tracefs directory 'options'\n");
8129 create_trace_option_file(struct trace_array *tr,
8130 struct trace_option_dentry *topt,
8131 struct tracer_flags *flags,
8132 struct tracer_opt *opt)
8134 struct dentry *t_options;
8136 t_options = trace_options_init_dentry(tr);
8140 topt->flags = flags;
8144 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
8145 &trace_options_fops);
8150 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8152 struct trace_option_dentry *topts;
8153 struct trace_options *tr_topts;
8154 struct tracer_flags *flags;
8155 struct tracer_opt *opts;
8162 flags = tracer->flags;
8164 if (!flags || !flags->opts)
8168 * If this is an instance, only create flags for tracers
8169 * the instance may have.
8171 if (!trace_ok_for_array(tracer, tr))
8174 for (i = 0; i < tr->nr_topts; i++) {
8175 /* Make sure there's no duplicate flags. */
8176 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8182 for (cnt = 0; opts[cnt].name; cnt++)
8185 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8189 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8196 tr->topts = tr_topts;
8197 tr->topts[tr->nr_topts].tracer = tracer;
8198 tr->topts[tr->nr_topts].topts = topts;
8201 for (cnt = 0; opts[cnt].name; cnt++) {
8202 create_trace_option_file(tr, &topts[cnt], flags,
8204 WARN_ONCE(topts[cnt].entry == NULL,
8205 "Failed to create trace option: %s",
8210 static struct dentry *
8211 create_trace_option_core_file(struct trace_array *tr,
8212 const char *option, long index)
8214 struct dentry *t_options;
8216 t_options = trace_options_init_dentry(tr);
8220 return trace_create_file(option, 0644, t_options,
8221 (void *)&tr->trace_flags_index[index],
8222 &trace_options_core_fops);
8225 static void create_trace_options_dir(struct trace_array *tr)
8227 struct dentry *t_options;
8228 bool top_level = tr == &global_trace;
8231 t_options = trace_options_init_dentry(tr);
8235 for (i = 0; trace_options[i]; i++) {
8237 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8238 create_trace_option_core_file(tr, trace_options[i], i);
8243 rb_simple_read(struct file *filp, char __user *ubuf,
8244 size_t cnt, loff_t *ppos)
8246 struct trace_array *tr = filp->private_data;
8250 r = tracer_tracing_is_on(tr);
8251 r = sprintf(buf, "%d\n", r);
8253 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8257 rb_simple_write(struct file *filp, const char __user *ubuf,
8258 size_t cnt, loff_t *ppos)
8260 struct trace_array *tr = filp->private_data;
8261 struct ring_buffer *buffer = tr->trace_buffer.buffer;
8265 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8270 mutex_lock(&trace_types_lock);
8271 if (!!val == tracer_tracing_is_on(tr)) {
8272 val = 0; /* do nothing */
8274 tracer_tracing_on(tr);
8275 if (tr->current_trace->start)
8276 tr->current_trace->start(tr);
8278 tracer_tracing_off(tr);
8279 if (tr->current_trace->stop)
8280 tr->current_trace->stop(tr);
8282 mutex_unlock(&trace_types_lock);
8290 static const struct file_operations rb_simple_fops = {
8291 .open = tracing_open_generic_tr,
8292 .read = rb_simple_read,
8293 .write = rb_simple_write,
8294 .release = tracing_release_generic_tr,
8295 .llseek = default_llseek,
8299 buffer_percent_read(struct file *filp, char __user *ubuf,
8300 size_t cnt, loff_t *ppos)
8302 struct trace_array *tr = filp->private_data;
8306 r = tr->buffer_percent;
8307 r = sprintf(buf, "%d\n", r);
8309 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8313 buffer_percent_write(struct file *filp, const char __user *ubuf,
8314 size_t cnt, loff_t *ppos)
8316 struct trace_array *tr = filp->private_data;
8320 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8330 tr->buffer_percent = val;
8337 static const struct file_operations buffer_percent_fops = {
8338 .open = tracing_open_generic_tr,
8339 .read = buffer_percent_read,
8340 .write = buffer_percent_write,
8341 .release = tracing_release_generic_tr,
8342 .llseek = default_llseek,
8345 static struct dentry *trace_instance_dir;
8348 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8351 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
8353 enum ring_buffer_flags rb_flags;
8355 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
8359 buf->buffer = ring_buffer_alloc(size, rb_flags);
8363 buf->data = alloc_percpu(struct trace_array_cpu);
8365 ring_buffer_free(buf->buffer);
8370 /* Allocate the first page for all buffers */
8371 set_buffer_entries(&tr->trace_buffer,
8372 ring_buffer_size(tr->trace_buffer.buffer, 0));
8377 static int allocate_trace_buffers(struct trace_array *tr, int size)
8381 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
8385 #ifdef CONFIG_TRACER_MAX_TRACE
8386 ret = allocate_trace_buffer(tr, &tr->max_buffer,
8387 allocate_snapshot ? size : 1);
8389 ring_buffer_free(tr->trace_buffer.buffer);
8390 tr->trace_buffer.buffer = NULL;
8391 free_percpu(tr->trace_buffer.data);
8392 tr->trace_buffer.data = NULL;
8395 tr->allocated_snapshot = allocate_snapshot;
8398 * Only the top level trace array gets its snapshot allocated
8399 * from the kernel command line.
8401 allocate_snapshot = false;
8406 static void free_trace_buffer(struct trace_buffer *buf)
8409 ring_buffer_free(buf->buffer);
8411 free_percpu(buf->data);
8416 static void free_trace_buffers(struct trace_array *tr)
8421 free_trace_buffer(&tr->trace_buffer);
8423 #ifdef CONFIG_TRACER_MAX_TRACE
8424 free_trace_buffer(&tr->max_buffer);
8428 static void init_trace_flags_index(struct trace_array *tr)
8432 /* Used by the trace options files */
8433 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8434 tr->trace_flags_index[i] = i;
8437 static void __update_tracer_options(struct trace_array *tr)
8441 for (t = trace_types; t; t = t->next)
8442 add_tracer_options(tr, t);
8445 static void update_tracer_options(struct trace_array *tr)
8447 mutex_lock(&trace_types_lock);
8448 __update_tracer_options(tr);
8449 mutex_unlock(&trace_types_lock);
8452 static struct trace_array *trace_array_create(const char *name)
8454 struct trace_array *tr;
8458 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8460 return ERR_PTR(ret);
8462 tr->name = kstrdup(name, GFP_KERNEL);
8466 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8469 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8471 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8473 raw_spin_lock_init(&tr->start_lock);
8475 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8477 tr->current_trace = &nop_trace;
8479 INIT_LIST_HEAD(&tr->systems);
8480 INIT_LIST_HEAD(&tr->events);
8481 INIT_LIST_HEAD(&tr->hist_vars);
8482 INIT_LIST_HEAD(&tr->err_log);
8484 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8487 tr->dir = tracefs_create_dir(name, trace_instance_dir);
8491 ret = event_trace_add_tracer(tr->dir, tr);
8493 tracefs_remove_recursive(tr->dir);
8497 ftrace_init_trace_array(tr);
8499 init_tracer_tracefs(tr, tr->dir);
8500 init_trace_flags_index(tr);
8501 __update_tracer_options(tr);
8503 list_add(&tr->list, &ftrace_trace_arrays);
8511 free_trace_buffers(tr);
8512 free_cpumask_var(tr->tracing_cpumask);
8516 return ERR_PTR(ret);
8519 static int instance_mkdir(const char *name)
8521 struct trace_array *tr;
8524 mutex_lock(&event_mutex);
8525 mutex_lock(&trace_types_lock);
8528 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8529 if (tr->name && strcmp(tr->name, name) == 0)
8533 tr = trace_array_create(name);
8535 ret = PTR_ERR_OR_ZERO(tr);
8538 mutex_unlock(&trace_types_lock);
8539 mutex_unlock(&event_mutex);
8544 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8545 * @name: The name of the trace array to be looked up/created.
8547 * Returns pointer to trace array with given name.
8548 * NULL, if it cannot be created.
8550 * NOTE: This function increments the reference counter associated with the
8551 * trace array returned. This makes sure it cannot be freed while in use.
8552 * Use trace_array_put() once the trace array is no longer needed.
8555 struct trace_array *trace_array_get_by_name(const char *name)
8557 struct trace_array *tr;
8559 mutex_lock(&event_mutex);
8560 mutex_lock(&trace_types_lock);
8562 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8563 if (tr->name && strcmp(tr->name, name) == 0)
8567 tr = trace_array_create(name);
8575 mutex_unlock(&trace_types_lock);
8576 mutex_unlock(&event_mutex);
8579 EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8581 static int __remove_instance(struct trace_array *tr)
8585 /* Reference counter for a newly created trace array = 1. */
8586 if (tr->ref > 1 || (tr->current_trace && tr->current_trace->ref))
8589 list_del(&tr->list);
8591 /* Disable all the flags that were enabled coming in */
8592 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8593 if ((1 << i) & ZEROED_TRACE_FLAGS)
8594 set_tracer_flag(tr, 1 << i, 0);
8597 tracing_set_nop(tr);
8598 clear_ftrace_function_probes(tr);
8599 event_trace_del_tracer(tr);
8600 ftrace_clear_pids(tr);
8601 ftrace_destroy_function_files(tr);
8602 tracefs_remove_recursive(tr->dir);
8603 free_trace_buffers(tr);
8605 for (i = 0; i < tr->nr_topts; i++) {
8606 kfree(tr->topts[i].topts);
8610 free_cpumask_var(tr->tracing_cpumask);
8618 int trace_array_destroy(struct trace_array *this_tr)
8620 struct trace_array *tr;
8626 mutex_lock(&event_mutex);
8627 mutex_lock(&trace_types_lock);
8631 /* Making sure trace array exists before destroying it. */
8632 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8633 if (tr == this_tr) {
8634 ret = __remove_instance(tr);
8639 mutex_unlock(&trace_types_lock);
8640 mutex_unlock(&event_mutex);
8644 EXPORT_SYMBOL_GPL(trace_array_destroy);
8646 static int instance_rmdir(const char *name)
8648 struct trace_array *tr;
8651 mutex_lock(&event_mutex);
8652 mutex_lock(&trace_types_lock);
8655 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8656 if (tr->name && strcmp(tr->name, name) == 0) {
8657 ret = __remove_instance(tr);
8662 mutex_unlock(&trace_types_lock);
8663 mutex_unlock(&event_mutex);
8668 static __init void create_trace_instances(struct dentry *d_tracer)
8670 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8673 if (WARN_ON(!trace_instance_dir))
8678 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8680 struct trace_event_file *file;
8683 trace_create_file("available_tracers", 0444, d_tracer,
8684 tr, &show_traces_fops);
8686 trace_create_file("current_tracer", 0644, d_tracer,
8687 tr, &set_tracer_fops);
8689 trace_create_file("tracing_cpumask", 0644, d_tracer,
8690 tr, &tracing_cpumask_fops);
8692 trace_create_file("trace_options", 0644, d_tracer,
8693 tr, &tracing_iter_fops);
8695 trace_create_file("trace", 0644, d_tracer,
8698 trace_create_file("trace_pipe", 0444, d_tracer,
8699 tr, &tracing_pipe_fops);
8701 trace_create_file("buffer_size_kb", 0644, d_tracer,
8702 tr, &tracing_entries_fops);
8704 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8705 tr, &tracing_total_entries_fops);
8707 trace_create_file("free_buffer", 0200, d_tracer,
8708 tr, &tracing_free_buffer_fops);
8710 trace_create_file("trace_marker", 0220, d_tracer,
8711 tr, &tracing_mark_fops);
8713 file = __find_event_file(tr, "ftrace", "print");
8714 if (file && file->dir)
8715 trace_create_file("trigger", 0644, file->dir, file,
8716 &event_trigger_fops);
8717 tr->trace_marker_file = file;
8719 trace_create_file("trace_marker_raw", 0220, d_tracer,
8720 tr, &tracing_mark_raw_fops);
8722 trace_create_file("trace_clock", 0644, d_tracer, tr,
8725 trace_create_file("tracing_on", 0644, d_tracer,
8726 tr, &rb_simple_fops);
8728 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8729 &trace_time_stamp_mode_fops);
8731 tr->buffer_percent = 50;
8733 trace_create_file("buffer_percent", 0444, d_tracer,
8734 tr, &buffer_percent_fops);
8736 create_trace_options_dir(tr);
8738 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8739 trace_create_maxlat_file(tr, d_tracer);
8742 if (ftrace_create_function_files(tr, d_tracer))
8743 WARN(1, "Could not allocate function filter files");
8745 #ifdef CONFIG_TRACER_SNAPSHOT
8746 trace_create_file("snapshot", 0644, d_tracer,
8747 tr, &snapshot_fops);
8750 trace_create_file("error_log", 0644, d_tracer,
8751 tr, &tracing_err_log_fops);
8753 for_each_tracing_cpu(cpu)
8754 tracing_init_tracefs_percpu(tr, cpu);
8756 ftrace_init_tracefs(tr, d_tracer);
8759 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
8761 struct vfsmount *mnt;
8762 struct file_system_type *type;
8765 * To maintain backward compatibility for tools that mount
8766 * debugfs to get to the tracing facility, tracefs is automatically
8767 * mounted to the debugfs/tracing directory.
8769 type = get_fs_type("tracefs");
8772 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
8773 put_filesystem(type);
8782 * tracing_init_dentry - initialize top level trace array
8784 * This is called when creating files or directories in the tracing
8785 * directory. It is called via fs_initcall() by any of the boot up code
8786 * and expects to return the dentry of the top level tracing directory.
8788 struct dentry *tracing_init_dentry(void)
8790 struct trace_array *tr = &global_trace;
8792 /* The top level trace array uses NULL as parent */
8796 if (WARN_ON(!tracefs_initialized()) ||
8797 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8798 WARN_ON(!debugfs_initialized())))
8799 return ERR_PTR(-ENODEV);
8802 * As there may still be users that expect the tracing
8803 * files to exist in debugfs/tracing, we must automount
8804 * the tracefs file system there, so older tools still
8805 * work with the newer kerenl.
8807 tr->dir = debugfs_create_automount("tracing", NULL,
8808 trace_automount, NULL);
8813 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8814 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8816 static void __init trace_eval_init(void)
8820 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8821 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8824 #ifdef CONFIG_MODULES
8825 static void trace_module_add_evals(struct module *mod)
8827 if (!mod->num_trace_evals)
8831 * Modules with bad taint do not have events created, do
8832 * not bother with enums either.
8834 if (trace_module_has_bad_taint(mod))
8837 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8840 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8841 static void trace_module_remove_evals(struct module *mod)
8843 union trace_eval_map_item *map;
8844 union trace_eval_map_item **last = &trace_eval_maps;
8846 if (!mod->num_trace_evals)
8849 mutex_lock(&trace_eval_mutex);
8851 map = trace_eval_maps;
8854 if (map->head.mod == mod)
8856 map = trace_eval_jmp_to_tail(map);
8857 last = &map->tail.next;
8858 map = map->tail.next;
8863 *last = trace_eval_jmp_to_tail(map)->tail.next;
8866 mutex_unlock(&trace_eval_mutex);
8869 static inline void trace_module_remove_evals(struct module *mod) { }
8870 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
8872 static int trace_module_notify(struct notifier_block *self,
8873 unsigned long val, void *data)
8875 struct module *mod = data;
8878 case MODULE_STATE_COMING:
8879 trace_module_add_evals(mod);
8881 case MODULE_STATE_GOING:
8882 trace_module_remove_evals(mod);
8889 static struct notifier_block trace_module_nb = {
8890 .notifier_call = trace_module_notify,
8893 #endif /* CONFIG_MODULES */
8895 static __init int tracer_init_tracefs(void)
8897 struct dentry *d_tracer;
8899 trace_access_lock_init();
8901 d_tracer = tracing_init_dentry();
8902 if (IS_ERR(d_tracer))
8907 init_tracer_tracefs(&global_trace, d_tracer);
8908 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
8910 trace_create_file("tracing_thresh", 0644, d_tracer,
8911 &global_trace, &tracing_thresh_fops);
8913 trace_create_file("README", 0444, d_tracer,
8914 NULL, &tracing_readme_fops);
8916 trace_create_file("saved_cmdlines", 0444, d_tracer,
8917 NULL, &tracing_saved_cmdlines_fops);
8919 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8920 NULL, &tracing_saved_cmdlines_size_fops);
8922 trace_create_file("saved_tgids", 0444, d_tracer,
8923 NULL, &tracing_saved_tgids_fops);
8927 trace_create_eval_file(d_tracer);
8929 #ifdef CONFIG_MODULES
8930 register_module_notifier(&trace_module_nb);
8933 #ifdef CONFIG_DYNAMIC_FTRACE
8934 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8935 NULL, &tracing_dyn_info_fops);
8938 create_trace_instances(d_tracer);
8940 update_tracer_options(&global_trace);
8945 static int trace_panic_handler(struct notifier_block *this,
8946 unsigned long event, void *unused)
8948 if (ftrace_dump_on_oops)
8949 ftrace_dump(ftrace_dump_on_oops);
8953 static struct notifier_block trace_panic_notifier = {
8954 .notifier_call = trace_panic_handler,
8956 .priority = 150 /* priority: INT_MAX >= x >= 0 */
8959 static int trace_die_handler(struct notifier_block *self,
8965 if (ftrace_dump_on_oops)
8966 ftrace_dump(ftrace_dump_on_oops);
8974 static struct notifier_block trace_die_notifier = {
8975 .notifier_call = trace_die_handler,
8980 * printk is set to max of 1024, we really don't need it that big.
8981 * Nothing should be printing 1000 characters anyway.
8983 #define TRACE_MAX_PRINT 1000
8986 * Define here KERN_TRACE so that we have one place to modify
8987 * it if we decide to change what log level the ftrace dump
8990 #define KERN_TRACE KERN_EMERG
8993 trace_printk_seq(struct trace_seq *s)
8995 /* Probably should print a warning here. */
8996 if (s->seq.len >= TRACE_MAX_PRINT)
8997 s->seq.len = TRACE_MAX_PRINT;
9000 * More paranoid code. Although the buffer size is set to
9001 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9002 * an extra layer of protection.
9004 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9005 s->seq.len = s->seq.size - 1;
9007 /* should be zero ended, but we are paranoid. */
9008 s->buffer[s->seq.len] = 0;
9010 printk(KERN_TRACE "%s", s->buffer);
9015 void trace_init_global_iter(struct trace_iterator *iter)
9017 iter->tr = &global_trace;
9018 iter->trace = iter->tr->current_trace;
9019 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9020 iter->trace_buffer = &global_trace.trace_buffer;
9022 if (iter->trace && iter->trace->open)
9023 iter->trace->open(iter);
9025 /* Annotate start of buffers if we had overruns */
9026 if (ring_buffer_overruns(iter->trace_buffer->buffer))
9027 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9029 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9030 if (trace_clocks[iter->tr->clock_id].in_ns)
9031 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9034 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9036 /* use static because iter can be a bit big for the stack */
9037 static struct trace_iterator iter;
9038 static atomic_t dump_running;
9039 struct trace_array *tr = &global_trace;
9040 unsigned int old_userobj;
9041 unsigned long flags;
9044 /* Only allow one dump user at a time. */
9045 if (atomic_inc_return(&dump_running) != 1) {
9046 atomic_dec(&dump_running);
9051 * Always turn off tracing when we dump.
9052 * We don't need to show trace output of what happens
9053 * between multiple crashes.
9055 * If the user does a sysrq-z, then they can re-enable
9056 * tracing with echo 1 > tracing_on.
9060 local_irq_save(flags);
9061 printk_nmi_direct_enter();
9063 /* Simulate the iterator */
9064 trace_init_global_iter(&iter);
9066 for_each_tracing_cpu(cpu) {
9067 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
9070 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9072 /* don't look at user memory in panic mode */
9073 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9075 switch (oops_dump_mode) {
9077 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9080 iter.cpu_file = raw_smp_processor_id();
9085 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9086 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9089 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9091 /* Did function tracer already get disabled? */
9092 if (ftrace_is_dead()) {
9093 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9094 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9098 * We need to stop all tracing on all CPUS to read the
9099 * the next buffer. This is a bit expensive, but is
9100 * not done often. We fill all what we can read,
9101 * and then release the locks again.
9104 while (!trace_empty(&iter)) {
9107 printk(KERN_TRACE "---------------------------------\n");
9111 trace_iterator_reset(&iter);
9112 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9114 if (trace_find_next_entry_inc(&iter) != NULL) {
9117 ret = print_trace_line(&iter);
9118 if (ret != TRACE_TYPE_NO_CONSUME)
9119 trace_consume(&iter);
9121 touch_nmi_watchdog();
9123 trace_printk_seq(&iter.seq);
9127 printk(KERN_TRACE " (ftrace buffer empty)\n");
9129 printk(KERN_TRACE "---------------------------------\n");
9132 tr->trace_flags |= old_userobj;
9134 for_each_tracing_cpu(cpu) {
9135 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
9137 atomic_dec(&dump_running);
9138 printk_nmi_direct_exit();
9139 local_irq_restore(flags);
9141 EXPORT_SYMBOL_GPL(ftrace_dump);
9143 int trace_run_command(const char *buf, int (*createfn)(int, char **))
9150 argv = argv_split(GFP_KERNEL, buf, &argc);
9155 ret = createfn(argc, argv);
9162 #define WRITE_BUFSIZE 4096
9164 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9165 size_t count, loff_t *ppos,
9166 int (*createfn)(int, char **))
9168 char *kbuf, *buf, *tmp;
9173 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9177 while (done < count) {
9178 size = count - done;
9180 if (size >= WRITE_BUFSIZE)
9181 size = WRITE_BUFSIZE - 1;
9183 if (copy_from_user(kbuf, buffer + done, size)) {
9190 tmp = strchr(buf, '\n');
9193 size = tmp - buf + 1;
9196 if (done + size < count) {
9199 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9200 pr_warn("Line length is too long: Should be less than %d\n",
9208 /* Remove comments */
9209 tmp = strchr(buf, '#');
9214 ret = trace_run_command(buf, createfn);
9219 } while (done < count);
9229 __init static int tracer_alloc_buffers(void)
9235 * Make sure we don't accidently add more trace options
9236 * than we have bits for.
9238 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9240 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9243 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9244 goto out_free_buffer_mask;
9246 /* Only allocate trace_printk buffers if a trace_printk exists */
9247 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
9248 /* Must be called before global_trace.buffer is allocated */
9249 trace_printk_init_buffers();
9251 /* To save memory, keep the ring buffer size to its minimum */
9252 if (ring_buffer_expanded)
9253 ring_buf_size = trace_buf_size;
9257 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9258 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9260 raw_spin_lock_init(&global_trace.start_lock);
9263 * The prepare callbacks allocates some memory for the ring buffer. We
9264 * don't free the buffer if the if the CPU goes down. If we were to free
9265 * the buffer, then the user would lose any trace that was in the
9266 * buffer. The memory will be removed once the "instance" is removed.
9268 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9269 "trace/RB:preapre", trace_rb_cpu_prepare,
9272 goto out_free_cpumask;
9273 /* Used for event triggers */
9275 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9277 goto out_rm_hp_state;
9279 if (trace_create_savedcmd() < 0)
9280 goto out_free_temp_buffer;
9282 /* TODO: make the number of buffers hot pluggable with CPUS */
9283 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9284 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
9286 goto out_free_savedcmd;
9289 if (global_trace.buffer_disabled)
9292 if (trace_boot_clock) {
9293 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9295 pr_warn("Trace clock %s not defined, going back to default\n",
9300 * register_tracer() might reference current_trace, so it
9301 * needs to be set before we register anything. This is
9302 * just a bootstrap of current_trace anyway.
9304 global_trace.current_trace = &nop_trace;
9306 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9308 ftrace_init_global_array_ops(&global_trace);
9310 init_trace_flags_index(&global_trace);
9312 register_tracer(&nop_trace);
9314 /* Function tracing may start here (via kernel command line) */
9315 init_function_trace();
9317 /* All seems OK, enable tracing */
9318 tracing_disabled = 0;
9320 atomic_notifier_chain_register(&panic_notifier_list,
9321 &trace_panic_notifier);
9323 register_die_notifier(&trace_die_notifier);
9325 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9327 INIT_LIST_HEAD(&global_trace.systems);
9328 INIT_LIST_HEAD(&global_trace.events);
9329 INIT_LIST_HEAD(&global_trace.hist_vars);
9330 INIT_LIST_HEAD(&global_trace.err_log);
9331 list_add(&global_trace.list, &ftrace_trace_arrays);
9333 apply_trace_boot_options();
9335 register_snapshot_cmd();
9340 free_saved_cmdlines_buffer(savedcmd);
9341 out_free_temp_buffer:
9342 ring_buffer_free(temp_buffer);
9344 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9346 free_cpumask_var(global_trace.tracing_cpumask);
9347 out_free_buffer_mask:
9348 free_cpumask_var(tracing_buffer_mask);
9353 void __init early_trace_init(void)
9355 if (tracepoint_printk) {
9356 tracepoint_print_iter =
9357 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9358 if (WARN_ON(!tracepoint_print_iter))
9359 tracepoint_printk = 0;
9361 static_key_enable(&tracepoint_printk_key.key);
9363 tracer_alloc_buffers();
9366 void __init trace_init(void)
9371 __init static int clear_boot_tracer(void)
9374 * The default tracer at boot buffer is an init section.
9375 * This function is called in lateinit. If we did not
9376 * find the boot tracer, then clear it out, to prevent
9377 * later registration from accessing the buffer that is
9378 * about to be freed.
9380 if (!default_bootup_tracer)
9383 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9384 default_bootup_tracer);
9385 default_bootup_tracer = NULL;
9390 fs_initcall(tracer_init_tracefs);
9391 late_initcall_sync(clear_boot_tracer);
9393 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9394 __init static int tracing_set_default_clock(void)
9396 /* sched_clock_stable() is determined in late_initcall */
9397 if (!trace_boot_clock && !sched_clock_stable()) {
9399 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9400 "If you want to keep using the local clock, then add:\n"
9401 " \"trace_clock=local\"\n"
9402 "on the kernel command line\n");
9403 tracing_set_clock(&global_trace, "global");
9408 late_initcall_sync(tracing_set_default_clock);