diff options
Diffstat (limited to 'include/trace/events/sched.h')
| -rw-r--r-- | include/trace/events/sched.h | 869 |
1 files changed, 863 insertions, 6 deletions
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 9f7fd0c6662b..73cd7e502d4c 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -8,6 +8,8 @@ #include <linux/tracepoint.h> #include <linux/binfmts.h> +struct rq; + /* * Tracepoint for calling kthread_stop, performed to end a kthread: */ @@ -51,6 +53,653 @@ TRACE_EVENT(sched_kthread_stop_ret, ); /* + * Tracepoint for task enqueue/dequeue: + */ +TRACE_EVENT(sched_enq_deq_task, + + TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed), + + TP_ARGS(p, enqueue, cpus_allowed), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, prio ) + __field( int, cpu ) + __field( bool, enqueue ) + __field(unsigned int, nr_running ) + __field(unsigned long, cpu_load ) + __field(unsigned int, rt_nr_running ) + __field(unsigned int, cpus_allowed ) +#ifdef CONFIG_SCHED_HMP + __field(unsigned int, demand ) + __field(unsigned int, pred_demand ) +#endif + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->cpu = task_cpu(p); + __entry->enqueue = enqueue; + __entry->nr_running = task_rq(p)->nr_running; + __entry->cpu_load = task_rq(p)->cpu_load[0]; + __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; + __entry->cpus_allowed = cpus_allowed; +#ifdef CONFIG_SCHED_HMP + __entry->demand = p->ravg.demand; + __entry->pred_demand = p->ravg.pred_demand; +#endif + ), + + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u cpu_load=%lu rt_nr_running=%u affine=%x" +#ifdef CONFIG_SCHED_HMP + " demand=%u pred_demand=%u" +#endif + , __entry->cpu, + __entry->enqueue ? "enqueue" : "dequeue", + __entry->comm, __entry->pid, + __entry->prio, __entry->nr_running, + __entry->cpu_load, __entry->rt_nr_running, __entry->cpus_allowed +#ifdef CONFIG_SCHED_HMP + , __entry->demand, __entry->pred_demand +#endif + ) +); + +#ifdef CONFIG_SCHED_HMP + +struct group_cpu_time; +struct migration_sum_data; +extern const char *task_event_names[]; +extern const char *migrate_type_names[]; + +TRACE_EVENT(sched_task_load, + + TP_PROTO(struct task_struct *p, bool boost, int reason, + bool sync, bool need_idle, u32 flags, int best_cpu), + + TP_ARGS(p, boost, reason, sync, need_idle, flags, best_cpu), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, demand ) + __field( bool, boost ) + __field( int, reason ) + __field( bool, sync ) + __field( bool, need_idle ) + __field( u32, flags ) + __field( int, best_cpu ) + __field( u64, latency ) + __field( int, grp_id ) + __field( u64, avg_burst ) + __field( u64, avg_sleep ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->demand = p->ravg.demand; + __entry->boost = boost; + __entry->reason = reason; + __entry->sync = sync; + __entry->need_idle = need_idle; + __entry->flags = flags; + __entry->best_cpu = best_cpu; + __entry->latency = p->state == TASK_WAKING ? + sched_ktime_clock() - + p->ravg.mark_start : 0; + __entry->grp_id = p->grp ? p->grp->id : 0; + __entry->avg_burst = p->ravg.avg_burst; + __entry->avg_sleep = p->ravg.avg_sleep_time; + ), + + TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d flags=%x grp=%d best_cpu=%d latency=%llu avg_burst=%llu avg_sleep=%llu", + __entry->pid, __entry->comm, __entry->demand, + __entry->boost, __entry->reason, __entry->sync, + __entry->need_idle, __entry->flags, __entry->grp_id, + __entry->best_cpu, __entry->latency, __entry->avg_burst, + __entry->avg_sleep) +); + +TRACE_EVENT(sched_set_preferred_cluster, + + TP_PROTO(struct related_thread_group *grp, u64 total_demand), + + TP_ARGS(grp, total_demand), + + TP_STRUCT__entry( + __field( int, id ) + __field( u64, demand ) + __field( int, cluster_first_cpu ) + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, task_demand ) + ), + + TP_fast_assign( + __entry->id = grp->id; + __entry->demand = total_demand; + __entry->cluster_first_cpu = grp->preferred_cluster ? + cluster_first_cpu(grp->preferred_cluster) + : -1; + ), + + TP_printk("group_id %d total_demand %llu preferred_cluster_first_cpu %d", + __entry->id, __entry->demand, + __entry->cluster_first_cpu) +); + +DECLARE_EVENT_CLASS(sched_cpu_load, + + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + + TP_ARGS(rq, idle, irqload, power_cost, temp), + + TP_STRUCT__entry( + __field(unsigned int, cpu ) + __field(unsigned int, idle ) + __field(unsigned int, nr_running ) + __field(unsigned int, nr_big_tasks ) + __field(unsigned int, load_scale_factor ) + __field(unsigned int, capacity ) + __field( u64, cumulative_runnable_avg ) + __field( u64, irqload ) + __field(unsigned int, max_freq ) + __field(unsigned int, power_cost ) + __field( int, cstate ) + __field( int, dstate ) + __field( int, temp ) + ), + + TP_fast_assign( + __entry->cpu = rq->cpu; + __entry->idle = idle; + __entry->nr_running = rq->nr_running; + __entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks; + __entry->load_scale_factor = cpu_load_scale_factor(rq->cpu); + __entry->capacity = cpu_capacity(rq->cpu); + __entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg; + __entry->irqload = irqload; + __entry->max_freq = cpu_max_freq(rq->cpu); + __entry->power_cost = power_cost; + __entry->cstate = rq->cstate; + __entry->dstate = rq->cluster->dstate; + __entry->temp = temp; + ), + + TP_printk("cpu %u idle %d nr_run %u nr_big %u lsf %u capacity %u cr_avg %llu irqload %llu fmax %u power_cost %u cstate %d dstate %d temp %d", + __entry->cpu, __entry->idle, __entry->nr_running, __entry->nr_big_tasks, + __entry->load_scale_factor, __entry->capacity, + __entry->cumulative_runnable_avg, __entry->irqload, + __entry->max_freq, __entry->power_cost, __entry->cstate, + __entry->dstate, __entry->temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_wakeup, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_lb, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_cgroup, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +TRACE_EVENT(sched_set_boost, + + TP_PROTO(int type), + + TP_ARGS(type), + + TP_STRUCT__entry( + __field(int, type ) + ), + + TP_fast_assign( + __entry->type = type; + ), + + TP_printk("type %d", __entry->type) +); + +#if defined(CREATE_TRACE_POINTS) && defined(CONFIG_SCHED_HMP) +static inline void __window_data(u32 *dst, u32 *src) +{ + if (src) + memcpy(dst, src, nr_cpu_ids * sizeof(u32)); + else + memset(dst, 0, nr_cpu_ids * sizeof(u32)); +} + +struct trace_seq; +const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len) +{ + int i; + const char *ret = p->buffer + seq_buf_used(&p->seq); + + for (i = 0; i < buf_len; i++) + trace_seq_printf(p, "%u ", buf[i]); + + trace_seq_putc(p, 0); + + return ret; +} + +static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new) +{ + if (curr) + if (new) + return rq->nt_curr_runnable_sum; + else + return rq->curr_runnable_sum; + else + if (new) + return rq->nt_prev_runnable_sum; + else + return rq->prev_runnable_sum; +} + +static inline s64 __grp_update_sum(struct rq *rq, bool curr, bool new) +{ + if (curr) + if (new) + return rq->grp_time.nt_curr_runnable_sum; + else + return rq->grp_time.curr_runnable_sum; + else + if (new) + return rq->grp_time.nt_prev_runnable_sum; + else + return rq->grp_time.prev_runnable_sum; +} + +static inline s64 +__get_update_sum(struct rq *rq, enum migrate_types migrate_type, + bool src, bool new, bool curr) +{ + switch (migrate_type) { + case RQ_TO_GROUP: + if (src) + return __rq_update_sum(rq, curr, new); + else + return __grp_update_sum(rq, curr, new); + case GROUP_TO_RQ: + if (src) + return __grp_update_sum(rq, curr, new); + else + return __rq_update_sum(rq, curr, new); + default: + WARN_ON_ONCE(1); + return -1; + } +} +#endif + +TRACE_EVENT(sched_update_task_ravg, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, u64 cycles, u64 exec_time, + struct group_cpu_time *cpu_time), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cycles, exec_time, cpu_time), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( pid_t, cur_pid ) + __field(unsigned int, cur_freq ) + __field( u64, wallclock ) + __field( u64, mark_start ) + __field( u64, delta_m ) + __field( u64, win_start ) + __field( u64, delta ) + __field( u64, irqtime ) + __field(enum task_event, evt ) + __field(unsigned int, demand ) + __field(unsigned int, sum ) + __field( int, cpu ) + __field(unsigned int, pred_demand ) + __field( u64, rq_cs ) + __field( u64, rq_ps ) + __field( u64, grp_cs ) + __field( u64, grp_ps ) + __field( u64, grp_nt_cs ) + __field( u64, grp_nt_ps ) + __field( u32, curr_window ) + __field( u32, prev_window ) + __dynamic_array(u32, curr_sum, nr_cpu_ids ) + __dynamic_array(u32, prev_sum, nr_cpu_ids ) + __field( u64, nt_cs ) + __field( u64, nt_ps ) + __field( u32, active_windows ) + __field( u8, curr_top ) + __field( u8, prev_top ) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = rq->window_start; + __entry->delta = (wallclock - rq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + __entry->cur_pid = rq->curr->pid; + __entry->cur_freq = cpu_cycles_to_freq(cycles, exec_time); + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = p->ravg.mark_start; + __entry->delta_m = (wallclock - p->ravg.mark_start); + __entry->demand = p->ravg.demand; + __entry->sum = p->ravg.sum; + __entry->irqtime = irqtime; + __entry->pred_demand = p->ravg.pred_demand; + __entry->rq_cs = rq->curr_runnable_sum; + __entry->rq_ps = rq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->grp_nt_cs = cpu_time ? cpu_time->nt_curr_runnable_sum : 0; + __entry->grp_nt_ps = cpu_time ? cpu_time->nt_prev_runnable_sum : 0; + __entry->curr_window = p->ravg.curr_window; + __entry->prev_window = p->ravg.prev_window; + __window_data(__get_dynamic_array(curr_sum), p->ravg.curr_window_cpu); + __window_data(__get_dynamic_array(prev_sum), p->ravg.prev_window_cpu); + __entry->nt_cs = rq->nt_curr_runnable_sum; + __entry->nt_ps = rq->nt_prev_runnable_sum; + __entry->active_windows = p->ravg.active_windows; + __entry->curr_top = rq->curr_top; + __entry->prev_top = rq->prev_top; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_wins %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", + __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->cur_freq, __entry->cur_pid, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, + __entry->sum, __entry->irqtime, __entry->pred_demand, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids), + __entry->prev_window, + __window_print(p, __get_dynamic_array(prev_sum), nr_cpu_ids), + __entry->nt_cs, __entry->nt_ps, + __entry->active_windows, __entry->grp_cs, + __entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps, + __entry->curr_top, __entry->prev_top) +); + +TRACE_EVENT(sched_get_task_cpu_cycles, + + TP_PROTO(int cpu, int event, u64 cycles, u64 exec_time, struct task_struct *p), + + TP_ARGS(cpu, event, cycles, exec_time, p), + + TP_STRUCT__entry( + __field(int, cpu ) + __field(int, event ) + __field(u64, cycles ) + __field(u64, exec_time ) + __field(u32, freq ) + __field(u32, legacy_freq ) + __field(u32, max_freq) + __field(pid_t, pid ) + __array(char, comm, TASK_COMM_LEN ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->event = event; + __entry->cycles = cycles; + __entry->exec_time = exec_time; + __entry->freq = cpu_cycles_to_freq(cycles, exec_time); + __entry->legacy_freq = cpu_cur_freq(cpu); + __entry->max_freq = cpu_max_freq(cpu); + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + ), + + TP_printk("cpu=%d event=%d cycles=%llu exec_time=%llu freq=%u legacy_freq=%u max_freq=%u task=%d (%s)", + __entry->cpu, __entry->event, __entry->cycles, + __entry->exec_time, __entry->freq, __entry->legacy_freq, + __entry->max_freq, __entry->pid, __entry->comm) +); + +TRACE_EVENT(sched_update_history, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, + enum task_event evt), + + TP_ARGS(rq, p, runtime, samples, evt), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, samples ) + __field(enum task_event, evt ) + __field(unsigned int, demand ) + __field(unsigned int, pred_demand ) + __array( u32, hist, RAVG_HIST_SIZE_MAX) + __field(unsigned int, nr_big_tasks ) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->samples = samples; + __entry->evt = evt; + __entry->demand = p->ravg.demand; + __entry->pred_demand = p->ravg.pred_demand; + memcpy(__entry->hist, p->ravg.sum_history, + RAVG_HIST_SIZE_MAX * sizeof(u32)); + __entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks; + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u samples %d event %s demand %u pred_demand %u" + " (hist: %u %u %u %u %u) cpu %d nr_big %u", + __entry->pid, __entry->comm, + __entry->runtime, __entry->samples, + task_event_names[__entry->evt], + __entry->demand, __entry->pred_demand, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3], + __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) +); + +TRACE_EVENT(sched_reset_all_window_stats, + + TP_PROTO(u64 window_start, u64 window_size, u64 time_taken, + int reason, unsigned int old_val, unsigned int new_val), + + TP_ARGS(window_start, window_size, time_taken, + reason, old_val, new_val), + + TP_STRUCT__entry( + __field( u64, window_start ) + __field( u64, window_size ) + __field( u64, time_taken ) + __field( int, reason ) + __field(unsigned int, old_val ) + __field(unsigned int, new_val ) + ), + + TP_fast_assign( + __entry->window_start = window_start; + __entry->window_size = window_size; + __entry->time_taken = time_taken; + __entry->reason = reason; + __entry->old_val = old_val; + __entry->new_val = new_val; + ), + + TP_printk("time_taken %llu window_start %llu window_size %llu reason %s old_val %u new_val %u", + __entry->time_taken, __entry->window_start, + __entry->window_size, + sched_window_reset_reasons[__entry->reason], + __entry->old_val, __entry->new_val) +); + +TRACE_EVENT(sched_update_pred_demand, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int pct, + unsigned int pred_demand), + + TP_ARGS(rq, p, runtime, pct, pred_demand), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, pct ) + __field(unsigned int, pred_demand ) + __array( u8, bucket, NUM_BUSY_BUCKETS) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->pct = pct; + __entry->pred_demand = pred_demand; + memcpy(__entry->bucket, p->ravg.busy_buckets, + NUM_BUSY_BUCKETS * sizeof(u8)); + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)", + __entry->pid, __entry->comm, + __entry->runtime, __entry->pct, __entry->cpu, + __entry->pred_demand, __entry->bucket[0], __entry->bucket[1], + __entry->bucket[2], __entry->bucket[3],__entry->bucket[4], + __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], + __entry->bucket[8], __entry->bucket[9]) +); + +TRACE_EVENT(sched_migration_update_sum, + + TP_PROTO(struct task_struct *p, enum migrate_types migrate_type, struct rq *rq), + + TP_ARGS(p, migrate_type, rq), + + TP_STRUCT__entry( + __field(int, tcpu ) + __field(int, pid ) + __field(enum migrate_types, migrate_type ) + __field( s64, src_cs ) + __field( s64, src_ps ) + __field( s64, dst_cs ) + __field( s64, dst_ps ) + __field( s64, src_nt_cs ) + __field( s64, src_nt_ps ) + __field( s64, dst_nt_cs ) + __field( s64, dst_nt_ps ) + ), + + TP_fast_assign( + __entry->tcpu = task_cpu(p); + __entry->pid = p->pid; + __entry->migrate_type = migrate_type; + __entry->src_cs = __get_update_sum(rq, migrate_type, + true, false, true); + __entry->src_ps = __get_update_sum(rq, migrate_type, + true, false, false); + __entry->dst_cs = __get_update_sum(rq, migrate_type, + false, false, true); + __entry->dst_ps = __get_update_sum(rq, migrate_type, + false, false, false); + __entry->src_nt_cs = __get_update_sum(rq, migrate_type, + true, true, true); + __entry->src_nt_ps = __get_update_sum(rq, migrate_type, + true, true, false); + __entry->dst_nt_cs = __get_update_sum(rq, migrate_type, + false, true, true); + __entry->dst_nt_ps = __get_update_sum(rq, migrate_type, + false, true, false); + ), + + TP_printk("pid %d task_cpu %d migrate_type %s src_cs %llu src_ps %llu dst_cs %lld dst_ps %lld src_nt_cs %llu src_nt_ps %llu dst_nt_cs %lld dst_nt_ps %lld", + __entry->pid, __entry->tcpu, migrate_type_names[__entry->migrate_type], + __entry->src_cs, __entry->src_ps, __entry->dst_cs, __entry->dst_ps, + __entry->src_nt_cs, __entry->src_nt_ps, __entry->dst_nt_cs, __entry->dst_nt_ps) +); + +TRACE_EVENT(sched_get_busy, + + TP_PROTO(int cpu, u64 load, u64 nload, u64 pload, int early, bool aggregated), + + TP_ARGS(cpu, load, nload, pload, early, aggregated), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( u64, load ) + __field( u64, nload ) + __field( u64, pload ) + __field( int, early ) + __field( bool, aggregated ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->load = load; + __entry->nload = nload; + __entry->pload = pload; + __entry->early = early; + __entry->aggregated = aggregated; + ), + + TP_printk("cpu %d load %lld new_task_load %lld predicted_load %lld early %d aggregated %d", + __entry->cpu, __entry->load, __entry->nload, + __entry->pload, __entry->early, __entry->aggregated) +); + +TRACE_EVENT(sched_freq_alert, + + TP_PROTO(int cpu, int pd_notif, int check_groups, struct rq *rq, + u64 new_load), + + TP_ARGS(cpu, pd_notif, check_groups, rq, new_load), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( int, pd_notif ) + __field( int, check_groups ) + __field( u64, old_busy_time ) + __field( u64, ps ) + __field( u64, new_load ) + __field( u64, old_pred ) + __field( u64, new_pred ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->pd_notif = pd_notif; + __entry->check_groups = check_groups; + __entry->old_busy_time = rq->old_busy_time; + __entry->ps = rq->prev_runnable_sum; + __entry->new_load = new_load; + __entry->old_pred = rq->old_estimated_time; + __entry->new_pred = rq->hmp_stats.pred_demands_sum; + ), + + TP_printk("cpu %d pd_notif=%d check_groups %d old_busy_time=%llu prev_sum=%lld new_load=%llu old_pred=%llu new_pred=%llu", + __entry->cpu, __entry->pd_notif, __entry->check_groups, + __entry->old_busy_time, __entry->ps, __entry->new_load, + __entry->old_pred, __entry->new_pred) +); + +#endif /* CONFIG_SCHED_HMP */ + +/* * Tracepoint for waking up a task: */ DECLARE_EVENT_CLASS(sched_wakeup_template, @@ -166,14 +815,16 @@ TRACE_EVENT(sched_switch, */ TRACE_EVENT(sched_migrate_task, - TP_PROTO(struct task_struct *p, int dest_cpu), + TP_PROTO(struct task_struct *p, int dest_cpu, + unsigned int load), - TP_ARGS(p, dest_cpu), + TP_ARGS(p, dest_cpu, load), TP_STRUCT__entry( __array( char, comm, TASK_COMM_LEN ) __field( pid_t, pid ) __field( int, prio ) + __field(unsigned int, load ) __field( int, orig_cpu ) __field( int, dest_cpu ) ), @@ -182,12 +833,13 @@ TRACE_EVENT(sched_migrate_task, memcpy(__entry->comm, p->comm, TASK_COMM_LEN); __entry->pid = p->pid; __entry->prio = p->prio; + __entry->load = load; __entry->orig_cpu = task_cpu(p); __entry->dest_cpu = dest_cpu; ), - TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d", - __entry->comm, __entry->pid, __entry->prio, + TP_printk("comm=%s pid=%d prio=%d load=%d orig_cpu=%d dest_cpu=%d", + __entry->comm, __entry->pid, __entry->prio, __entry->load, __entry->orig_cpu, __entry->dest_cpu) ); @@ -216,6 +868,56 @@ TRACE_EVENT(sched_cpu_hotplug, __entry->status ? "online" : "offline", __entry->error) ); +/* + * Tracepoint for load balancing: + */ +#if NR_CPUS > 32 +#error "Unsupported NR_CPUS for lb tracepoint." +#endif +TRACE_EVENT(sched_load_balance, + + TP_PROTO(int cpu, enum cpu_idle_type idle, int balance, + unsigned long group_mask, int busiest_nr_running, + unsigned long imbalance, unsigned int env_flags, int ld_moved, + unsigned int balance_interval), + + TP_ARGS(cpu, idle, balance, group_mask, busiest_nr_running, + imbalance, env_flags, ld_moved, balance_interval), + + TP_STRUCT__entry( + __field( int, cpu) + __field( enum cpu_idle_type, idle) + __field( int, balance) + __field( unsigned long, group_mask) + __field( int, busiest_nr_running) + __field( unsigned long, imbalance) + __field( unsigned int, env_flags) + __field( int, ld_moved) + __field( unsigned int, balance_interval) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->idle = idle; + __entry->balance = balance; + __entry->group_mask = group_mask; + __entry->busiest_nr_running = busiest_nr_running; + __entry->imbalance = imbalance; + __entry->env_flags = env_flags; + __entry->ld_moved = ld_moved; + __entry->balance_interval = balance_interval; + ), + + TP_printk("cpu=%d state=%s balance=%d group=%#lx busy_nr=%d imbalance=%ld flags=%#x ld_moved=%d bal_int=%d", + __entry->cpu, + __entry->idle == CPU_IDLE ? "idle" : + (__entry->idle == CPU_NEWLY_IDLE ? "newly_idle" : "busy"), + __entry->balance, + __entry->group_mask, __entry->busiest_nr_running, + __entry->imbalance, __entry->env_flags, __entry->ld_moved, + __entry->balance_interval) +); + DECLARE_EVENT_CLASS(sched_process_template, TP_PROTO(struct task_struct *p), @@ -612,6 +1314,162 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TP_printk("cpu=%d", __entry->cpu) ); +TRACE_EVENT(sched_get_nr_running_avg, + + TP_PROTO(int avg, int big_avg, int iowait_avg, + unsigned int max_nr, unsigned int big_max_nr), + + TP_ARGS(avg, big_avg, iowait_avg, max_nr, big_max_nr), + + TP_STRUCT__entry( + __field( int, avg ) + __field( int, big_avg ) + __field( int, iowait_avg ) + __field( unsigned int, max_nr ) + __field( unsigned int, big_max_nr ) + ), + + TP_fast_assign( + __entry->avg = avg; + __entry->big_avg = big_avg; + __entry->iowait_avg = iowait_avg; + __entry->max_nr = max_nr; + __entry->big_max_nr = big_max_nr; + ), + + TP_printk("avg=%d big_avg=%d iowait_avg=%d max_nr=%u big_max_nr=%u", + __entry->avg, __entry->big_avg, __entry->iowait_avg, + __entry->max_nr, __entry->big_max_nr) +); + +TRACE_EVENT(core_ctl_eval_need, + + TP_PROTO(unsigned int cpu, unsigned int old_need, + unsigned int new_need, unsigned int updated), + TP_ARGS(cpu, old_need, new_need, updated), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, old_need) + __field(u32, new_need) + __field(u32, updated) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->old_need = old_need; + __entry->new_need = new_need; + __entry->updated = updated; + ), + TP_printk("cpu=%u, old_need=%u, new_need=%u, updated=%u", __entry->cpu, + __entry->old_need, __entry->new_need, __entry->updated) +); + +TRACE_EVENT(core_ctl_set_busy, + + TP_PROTO(unsigned int cpu, unsigned int busy, + unsigned int old_is_busy, unsigned int is_busy), + TP_ARGS(cpu, busy, old_is_busy, is_busy), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, busy) + __field(u32, old_is_busy) + __field(u32, is_busy) + __field(bool, high_irqload) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->busy = busy; + __entry->old_is_busy = old_is_busy; + __entry->is_busy = is_busy; + __entry->high_irqload = sched_cpu_high_irqload(cpu); + ), + TP_printk("cpu=%u, busy=%u, old_is_busy=%u, new_is_busy=%u high_irqload=%d", + __entry->cpu, __entry->busy, __entry->old_is_busy, + __entry->is_busy, __entry->high_irqload) +); + +TRACE_EVENT(core_ctl_set_boost, + + TP_PROTO(u32 refcount, s32 ret), + TP_ARGS(refcount, ret), + TP_STRUCT__entry( + __field(u32, refcount) + __field(s32, ret) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->ret = ret; + ), + TP_printk("refcount=%u, ret=%d", __entry->refcount, __entry->ret) +); + +/** + * sched_isolate - called when cores are isolated/unisolated + * + * @acutal_mask: mask of cores actually isolated/unisolated + * @req_mask: mask of cores requested isolated/unisolated + * @online_mask: cpu online mask + * @time: amount of time in us it took to isolate/unisolate + * @isolate: 1 if isolating, 0 if unisolating + * + */ +TRACE_EVENT(sched_isolate, + + TP_PROTO(unsigned int requested_cpu, unsigned int isolated_cpus, + u64 start_time, unsigned char isolate), + + TP_ARGS(requested_cpu, isolated_cpus, start_time, isolate), + + TP_STRUCT__entry( + __field(u32, requested_cpu) + __field(u32, isolated_cpus) + __field(u32, time) + __field(unsigned char, isolate) + ), + + TP_fast_assign( + __entry->requested_cpu = requested_cpu; + __entry->isolated_cpus = isolated_cpus; + __entry->time = div64_u64(sched_clock() - start_time, 1000); + __entry->isolate = isolate; + ), + + TP_printk("iso cpu=%u cpus=0x%x time=%u us isolated=%d", + __entry->requested_cpu, __entry->isolated_cpus, + __entry->time, __entry->isolate) +); + +TRACE_EVENT(sched_preempt_disable, + + TP_PROTO(u64 delta, bool irqs_disabled, + unsigned long caddr0, unsigned long caddr1, + unsigned long caddr2, unsigned long caddr3), + + TP_ARGS(delta, irqs_disabled, caddr0, caddr1, caddr2, caddr3), + + TP_STRUCT__entry( + __field(u64, delta) + __field(bool, irqs_disabled) + __field(void*, caddr0) + __field(void*, caddr1) + __field(void*, caddr2) + __field(void*, caddr3) + ), + + TP_fast_assign( + __entry->delta = delta; + __entry->irqs_disabled = irqs_disabled; + __entry->caddr0 = (void *)caddr0; + __entry->caddr1 = (void *)caddr1; + __entry->caddr2 = (void *)caddr2; + __entry->caddr3 = (void *)caddr3; + ), + + TP_printk("delta=%llu(ns) irqs_d=%d Callers:(%pf<-%pf<-%pf<-%pf)", + __entry->delta, __entry->irqs_disabled, + __entry->caddr0, __entry->caddr1, + __entry->caddr2, __entry->caddr3) +); + TRACE_EVENT(sched_contrib_scale_f, TP_PROTO(int cpu, unsigned long freq_scale_factor, @@ -1130,8 +1988,7 @@ TRACE_EVENT(walt_update_history, __entry->samples = samples; __entry->evt = evt; __entry->demand = p->ravg.demand; - __entry->walt_avg = (__entry->demand << 10); - do_div(__entry->walt_avg, walt_ravg_window); + __entry->walt_avg = (__entry->demand << 10) / walt_ravg_window, __entry->pelt_avg = p->se.avg.util_avg; memcpy(__entry->hist, p->ravg.sum_history, RAVG_HIST_SIZE_MAX * sizeof(u32)); |