diff options
Diffstat (limited to 'include/trace/events/sched.h')
| -rw-r--r-- | include/trace/events/sched.h | 924 |
1 files changed, 771 insertions, 153 deletions
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index dffaffab4bc8..a56443be1c70 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -8,6 +8,8 @@ #include <linux/tracepoint.h> #include <linux/binfmts.h> +struct rq; + /* * Tracepoint for calling kthread_stop, performed to end a kthread: */ @@ -51,6 +53,602 @@ TRACE_EVENT(sched_kthread_stop_ret, ); /* + * Tracepoint for task enqueue/dequeue: + */ +TRACE_EVENT(sched_enq_deq_task, + + TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed), + + TP_ARGS(p, enqueue, cpus_allowed), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, prio ) + __field( int, cpu ) + __field( bool, enqueue ) + __field(unsigned int, nr_running ) + __field(unsigned long, cpu_load ) + __field(unsigned int, rt_nr_running ) + __field(unsigned int, cpus_allowed ) +#ifdef CONFIG_SCHED_HMP + __field(unsigned int, demand ) + __field(unsigned int, pred_demand ) +#endif + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->cpu = task_cpu(p); + __entry->enqueue = enqueue; + __entry->nr_running = task_rq(p)->nr_running; + __entry->cpu_load = task_rq(p)->cpu_load[0]; + __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; + __entry->cpus_allowed = cpus_allowed; +#ifdef CONFIG_SCHED_HMP + __entry->demand = p->ravg.demand; + __entry->pred_demand = p->ravg.pred_demand; +#endif + ), + + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u cpu_load=%lu rt_nr_running=%u affine=%x" +#ifdef CONFIG_SCHED_HMP + " demand=%u pred_demand=%u" +#endif + , __entry->cpu, + __entry->enqueue ? "enqueue" : "dequeue", + __entry->comm, __entry->pid, + __entry->prio, __entry->nr_running, + __entry->cpu_load, __entry->rt_nr_running, __entry->cpus_allowed +#ifdef CONFIG_SCHED_HMP + , __entry->demand, __entry->pred_demand +#endif + ) +); + +#ifdef CONFIG_SCHED_HMP + +struct group_cpu_time; +struct migration_sum_data; +extern const char *task_event_names[]; +extern const char *migrate_type_names[]; + +TRACE_EVENT(sched_task_load, + + TP_PROTO(struct task_struct *p, bool boost, int reason, + bool sync, bool need_idle, u32 flags, int best_cpu), + + TP_ARGS(p, boost, reason, sync, need_idle, flags, best_cpu), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, demand ) + __field( bool, boost ) + __field( int, reason ) + __field( bool, sync ) + __field( bool, need_idle ) + __field( u32, flags ) + __field( int, best_cpu ) + __field( u64, latency ) + __field( int, grp_id ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->demand = p->ravg.demand; + __entry->boost = boost; + __entry->reason = reason; + __entry->sync = sync; + __entry->need_idle = need_idle; + __entry->flags = flags; + __entry->best_cpu = best_cpu; + __entry->latency = p->state == TASK_WAKING ? + sched_ktime_clock() - + p->ravg.mark_start : 0; + __entry->grp_id = p->grp ? p->grp->id : 0; + ), + + TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d flags=%x grp=%d best_cpu=%d latency=%llu", + __entry->pid, __entry->comm, __entry->demand, + __entry->boost, __entry->reason, __entry->sync, + __entry->need_idle, __entry->flags, __entry->grp_id, + __entry->best_cpu, __entry->latency) +); + +TRACE_EVENT(sched_set_preferred_cluster, + + TP_PROTO(struct related_thread_group *grp, u64 total_demand), + + TP_ARGS(grp, total_demand), + + TP_STRUCT__entry( + __field( int, id ) + __field( u64, demand ) + __field( int, cluster_first_cpu ) + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, task_demand ) + ), + + TP_fast_assign( + __entry->id = grp->id; + __entry->demand = total_demand; + __entry->cluster_first_cpu = grp->preferred_cluster ? + cluster_first_cpu(grp->preferred_cluster) + : -1; + ), + + TP_printk("group_id %d total_demand %llu preferred_cluster_first_cpu %d", + __entry->id, __entry->demand, + __entry->cluster_first_cpu) +); + +DECLARE_EVENT_CLASS(sched_cpu_load, + + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + + TP_ARGS(rq, idle, irqload, power_cost, temp), + + TP_STRUCT__entry( + __field(unsigned int, cpu ) + __field(unsigned int, idle ) + __field(unsigned int, nr_running ) + __field(unsigned int, nr_big_tasks ) + __field(unsigned int, load_scale_factor ) + __field(unsigned int, capacity ) + __field( u64, cumulative_runnable_avg ) + __field( u64, irqload ) + __field(unsigned int, max_freq ) + __field(unsigned int, power_cost ) + __field( int, cstate ) + __field( int, dstate ) + __field( int, temp ) + ), + + TP_fast_assign( + __entry->cpu = rq->cpu; + __entry->idle = idle; + __entry->nr_running = rq->nr_running; + __entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks; + __entry->load_scale_factor = cpu_load_scale_factor(rq->cpu); + __entry->capacity = cpu_capacity(rq->cpu); + __entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg; + __entry->irqload = irqload; + __entry->max_freq = cpu_max_freq(rq->cpu); + __entry->power_cost = power_cost; + __entry->cstate = rq->cstate; + __entry->dstate = rq->cluster->dstate; + __entry->temp = temp; + ), + + TP_printk("cpu %u idle %d nr_run %u nr_big %u lsf %u capacity %u cr_avg %llu irqload %llu fmax %u power_cost %u cstate %d dstate %d temp %d", + __entry->cpu, __entry->idle, __entry->nr_running, __entry->nr_big_tasks, + __entry->load_scale_factor, __entry->capacity, + __entry->cumulative_runnable_avg, __entry->irqload, + __entry->max_freq, __entry->power_cost, __entry->cstate, + __entry->dstate, __entry->temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_wakeup, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_lb, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_cgroup, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +TRACE_EVENT(sched_set_boost, + + TP_PROTO(int type), + + TP_ARGS(type), + + TP_STRUCT__entry( + __field(int, type ) + ), + + TP_fast_assign( + __entry->type = type; + ), + + TP_printk("type %d", __entry->type) +); + +#if defined(CREATE_TRACE_POINTS) && defined(CONFIG_SCHED_HMP) +static inline void __window_data(u32 *dst, u32 *src) +{ + if (src) + memcpy(dst, src, nr_cpu_ids * sizeof(u32)); + else + memset(dst, 0, nr_cpu_ids * sizeof(u32)); +} + +struct trace_seq; +const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len) +{ + int i; + const char *ret = p->buffer + seq_buf_used(&p->seq); + + for (i = 0; i < buf_len; i++) + trace_seq_printf(p, "%u ", buf[i]); + + trace_seq_putc(p, 0); + + return ret; +} +#endif + +TRACE_EVENT(sched_update_task_ravg, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, u64 cycles, u64 exec_time, + struct group_cpu_time *cpu_time), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cycles, exec_time, cpu_time), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( pid_t, cur_pid ) + __field(unsigned int, cur_freq ) + __field( u64, wallclock ) + __field( u64, mark_start ) + __field( u64, delta_m ) + __field( u64, win_start ) + __field( u64, delta ) + __field( u64, irqtime ) + __field(enum task_event, evt ) + __field(unsigned int, demand ) + __field(unsigned int, sum ) + __field( int, cpu ) + __field(unsigned int, pred_demand ) + __field( u64, rq_cs ) + __field( u64, rq_ps ) + __field( u64, grp_cs ) + __field( u64, grp_ps ) + __field( u64, grp_nt_cs ) + __field( u64, grp_nt_ps ) + __field( u32, curr_window ) + __field( u32, prev_window ) + __dynamic_array(u32, curr_sum, nr_cpu_ids ) + __dynamic_array(u32, prev_sum, nr_cpu_ids ) + __field( u64, nt_cs ) + __field( u64, nt_ps ) + __field( u32, active_windows ) + __field( u8, curr_top ) + __field( u8, prev_top ) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = rq->window_start; + __entry->delta = (wallclock - rq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + __entry->cur_pid = rq->curr->pid; + __entry->cur_freq = cpu_cycles_to_freq(cycles, exec_time); + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = p->ravg.mark_start; + __entry->delta_m = (wallclock - p->ravg.mark_start); + __entry->demand = p->ravg.demand; + __entry->sum = p->ravg.sum; + __entry->irqtime = irqtime; + __entry->pred_demand = p->ravg.pred_demand; + __entry->rq_cs = rq->curr_runnable_sum; + __entry->rq_ps = rq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->grp_nt_cs = cpu_time ? cpu_time->nt_curr_runnable_sum : 0; + __entry->grp_nt_ps = cpu_time ? cpu_time->nt_prev_runnable_sum : 0; + __entry->curr_window = p->ravg.curr_window; + __entry->prev_window = p->ravg.prev_window; + __window_data(__get_dynamic_array(curr_sum), p->ravg.curr_window_cpu); + __window_data(__get_dynamic_array(prev_sum), p->ravg.prev_window_cpu); + __entry->nt_cs = rq->nt_curr_runnable_sum; + __entry->nt_ps = rq->nt_prev_runnable_sum; + __entry->active_windows = p->ravg.active_windows; + __entry->curr_top = rq->curr_top; + __entry->prev_top = rq->prev_top; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_wins %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", + __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->cur_freq, __entry->cur_pid, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, + __entry->sum, __entry->irqtime, __entry->pred_demand, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids), + __entry->prev_window, + __window_print(p, __get_dynamic_array(prev_sum), nr_cpu_ids), + __entry->nt_cs, __entry->nt_ps, + __entry->active_windows, __entry->grp_cs, + __entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps, + __entry->curr_top, __entry->prev_top) +); + +TRACE_EVENT(sched_get_task_cpu_cycles, + + TP_PROTO(int cpu, int event, u64 cycles, u64 exec_time), + + TP_ARGS(cpu, event, cycles, exec_time), + + TP_STRUCT__entry( + __field(int, cpu ) + __field(int, event ) + __field(u64, cycles ) + __field(u64, exec_time ) + __field(u32, freq ) + __field(u32, legacy_freq ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->event = event; + __entry->cycles = cycles; + __entry->exec_time = exec_time; + __entry->freq = cpu_cycles_to_freq(cycles, exec_time); + __entry->legacy_freq = cpu_cur_freq(cpu); + ), + + TP_printk("cpu=%d event=%d cycles=%llu exec_time=%llu freq=%u legacy_freq=%u", + __entry->cpu, __entry->event, __entry->cycles, + __entry->exec_time, __entry->freq, __entry->legacy_freq) +); + +TRACE_EVENT(sched_update_history, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, + enum task_event evt), + + TP_ARGS(rq, p, runtime, samples, evt), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, samples ) + __field(enum task_event, evt ) + __field(unsigned int, demand ) + __field(unsigned int, pred_demand ) + __array( u32, hist, RAVG_HIST_SIZE_MAX) + __field(unsigned int, nr_big_tasks ) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->samples = samples; + __entry->evt = evt; + __entry->demand = p->ravg.demand; + __entry->pred_demand = p->ravg.pred_demand; + memcpy(__entry->hist, p->ravg.sum_history, + RAVG_HIST_SIZE_MAX * sizeof(u32)); + __entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks; + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u samples %d event %s demand %u pred_demand %u" + " (hist: %u %u %u %u %u) cpu %d nr_big %u", + __entry->pid, __entry->comm, + __entry->runtime, __entry->samples, + task_event_names[__entry->evt], + __entry->demand, __entry->pred_demand, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3], + __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) +); + +TRACE_EVENT(sched_reset_all_window_stats, + + TP_PROTO(u64 window_start, u64 window_size, u64 time_taken, + int reason, unsigned int old_val, unsigned int new_val), + + TP_ARGS(window_start, window_size, time_taken, + reason, old_val, new_val), + + TP_STRUCT__entry( + __field( u64, window_start ) + __field( u64, window_size ) + __field( u64, time_taken ) + __field( int, reason ) + __field(unsigned int, old_val ) + __field(unsigned int, new_val ) + ), + + TP_fast_assign( + __entry->window_start = window_start; + __entry->window_size = window_size; + __entry->time_taken = time_taken; + __entry->reason = reason; + __entry->old_val = old_val; + __entry->new_val = new_val; + ), + + TP_printk("time_taken %llu window_start %llu window_size %llu reason %s old_val %u new_val %u", + __entry->time_taken, __entry->window_start, + __entry->window_size, + sched_window_reset_reasons[__entry->reason], + __entry->old_val, __entry->new_val) +); + +TRACE_EVENT(sched_update_pred_demand, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int pct, + unsigned int pred_demand), + + TP_ARGS(rq, p, runtime, pct, pred_demand), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, pct ) + __field(unsigned int, pred_demand ) + __array( u8, bucket, NUM_BUSY_BUCKETS) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->pct = pct; + __entry->pred_demand = pred_demand; + memcpy(__entry->bucket, p->ravg.busy_buckets, + NUM_BUSY_BUCKETS * sizeof(u8)); + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)", + __entry->pid, __entry->comm, + __entry->runtime, __entry->pct, __entry->cpu, + __entry->pred_demand, __entry->bucket[0], __entry->bucket[1], + __entry->bucket[2], __entry->bucket[3],__entry->bucket[4], + __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], + __entry->bucket[8], __entry->bucket[9]) +); + +TRACE_EVENT(sched_migration_update_sum, + + TP_PROTO(struct task_struct *p, enum migrate_types migrate_type, struct migration_sum_data *d), + + TP_ARGS(p, migrate_type, d), + + TP_STRUCT__entry( + __field(int, tcpu ) + __field(int, pid ) + __field( u64, cs ) + __field( u64, ps ) + __field( s64, nt_cs ) + __field( s64, nt_ps ) + __field(enum migrate_types, migrate_type ) + __field( s64, src_cs ) + __field( s64, src_ps ) + __field( s64, dst_cs ) + __field( s64, dst_ps ) + __field( s64, src_nt_cs ) + __field( s64, src_nt_ps ) + __field( s64, dst_nt_cs ) + __field( s64, dst_nt_ps ) + ), + + TP_fast_assign( + __entry->tcpu = task_cpu(p); + __entry->pid = p->pid; + __entry->migrate_type = migrate_type; + __entry->src_cs = d->src_rq ? + d->src_rq->curr_runnable_sum : + d->src_cpu_time->curr_runnable_sum; + __entry->src_ps = d->src_rq ? + d->src_rq->prev_runnable_sum : + d->src_cpu_time->prev_runnable_sum; + __entry->dst_cs = d->dst_rq ? + d->dst_rq->curr_runnable_sum : + d->dst_cpu_time->curr_runnable_sum; + __entry->dst_ps = d->dst_rq ? + d->dst_rq->prev_runnable_sum : + d->dst_cpu_time->prev_runnable_sum; + __entry->src_nt_cs = d->src_rq ? + d->src_rq->nt_curr_runnable_sum : + d->src_cpu_time->nt_curr_runnable_sum; + __entry->src_nt_ps = d->src_rq ? + d->src_rq->nt_prev_runnable_sum : + d->src_cpu_time->nt_prev_runnable_sum; + __entry->dst_nt_cs = d->dst_rq ? + d->dst_rq->nt_curr_runnable_sum : + d->dst_cpu_time->nt_curr_runnable_sum; + __entry->dst_nt_ps = d->dst_rq ? + d->dst_rq->nt_prev_runnable_sum : + d->dst_cpu_time->nt_prev_runnable_sum; + ), + + TP_printk("pid %d task_cpu %d migrate_type %s src_cs %llu src_ps %llu dst_cs %lld dst_ps %lld src_nt_cs %llu src_nt_ps %llu dst_nt_cs %lld dst_nt_ps %lld", + __entry->pid, __entry->tcpu, migrate_type_names[__entry->migrate_type], + __entry->src_cs, __entry->src_ps, __entry->dst_cs, __entry->dst_ps, + __entry->src_nt_cs, __entry->src_nt_ps, __entry->dst_nt_cs, __entry->dst_nt_ps) +); + +TRACE_EVENT(sched_get_busy, + + TP_PROTO(int cpu, u64 load, u64 nload, u64 pload, int early), + + TP_ARGS(cpu, load, nload, pload, early), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( u64, load ) + __field( u64, nload ) + __field( u64, pload ) + __field( int, early ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->load = load; + __entry->nload = nload; + __entry->pload = pload; + __entry->early = early; + ), + + TP_printk("cpu %d load %lld new_task_load %lld predicted_load %lld early %d", + __entry->cpu, __entry->load, __entry->nload, + __entry->pload, __entry->early) +); + +TRACE_EVENT(sched_freq_alert, + + TP_PROTO(int cpu, int pd_notif, int check_groups, struct rq *rq, + u64 new_load), + + TP_ARGS(cpu, pd_notif, check_groups, rq, new_load), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( int, pd_notif ) + __field( int, check_groups ) + __field( u64, old_busy_time ) + __field( u64, ps ) + __field( u64, new_load ) + __field( u64, old_pred ) + __field( u64, new_pred ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->pd_notif = pd_notif; + __entry->check_groups = check_groups; + __entry->old_busy_time = rq->old_busy_time; + __entry->ps = rq->prev_runnable_sum; + __entry->new_load = new_load; + __entry->old_pred = rq->old_estimated_time; + __entry->new_pred = rq->hmp_stats.pred_demands_sum; + ), + + TP_printk("cpu %d pd_notif=%d check_groups %d old_busy_time=%llu prev_sum=%lld new_load=%llu old_pred=%llu new_pred=%llu", + __entry->cpu, __entry->pd_notif, __entry->check_groups, + __entry->old_busy_time, __entry->ps, __entry->new_load, + __entry->old_pred, __entry->new_pred) +); + +#endif /* CONFIG_SCHED_HMP */ + +/* * Tracepoint for waking up a task: */ DECLARE_EVENT_CLASS(sched_wakeup_template, @@ -166,14 +764,16 @@ TRACE_EVENT(sched_switch, */ TRACE_EVENT(sched_migrate_task, - TP_PROTO(struct task_struct *p, int dest_cpu), + TP_PROTO(struct task_struct *p, int dest_cpu, + unsigned int load), - TP_ARGS(p, dest_cpu), + TP_ARGS(p, dest_cpu, load), TP_STRUCT__entry( __array( char, comm, TASK_COMM_LEN ) __field( pid_t, pid ) __field( int, prio ) + __field(unsigned int, load ) __field( int, orig_cpu ) __field( int, dest_cpu ) ), @@ -182,12 +782,13 @@ TRACE_EVENT(sched_migrate_task, memcpy(__entry->comm, p->comm, TASK_COMM_LEN); __entry->pid = p->pid; __entry->prio = p->prio; + __entry->load = load; __entry->orig_cpu = task_cpu(p); __entry->dest_cpu = dest_cpu; ), - TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d", - __entry->comm, __entry->pid, __entry->prio, + TP_printk("comm=%s pid=%d prio=%d load=%d orig_cpu=%d dest_cpu=%d", + __entry->comm, __entry->pid, __entry->prio, __entry->load, __entry->orig_cpu, __entry->dest_cpu) ); @@ -216,6 +817,56 @@ TRACE_EVENT(sched_cpu_hotplug, __entry->status ? "online" : "offline", __entry->error) ); +/* + * Tracepoint for load balancing: + */ +#if NR_CPUS > 32 +#error "Unsupported NR_CPUS for lb tracepoint." +#endif +TRACE_EVENT(sched_load_balance, + + TP_PROTO(int cpu, enum cpu_idle_type idle, int balance, + unsigned long group_mask, int busiest_nr_running, + unsigned long imbalance, unsigned int env_flags, int ld_moved, + unsigned int balance_interval), + + TP_ARGS(cpu, idle, balance, group_mask, busiest_nr_running, + imbalance, env_flags, ld_moved, balance_interval), + + TP_STRUCT__entry( + __field( int, cpu) + __field( enum cpu_idle_type, idle) + __field( int, balance) + __field( unsigned long, group_mask) + __field( int, busiest_nr_running) + __field( unsigned long, imbalance) + __field( unsigned int, env_flags) + __field( int, ld_moved) + __field( unsigned int, balance_interval) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->idle = idle; + __entry->balance = balance; + __entry->group_mask = group_mask; + __entry->busiest_nr_running = busiest_nr_running; + __entry->imbalance = imbalance; + __entry->env_flags = env_flags; + __entry->ld_moved = ld_moved; + __entry->balance_interval = balance_interval; + ), + + TP_printk("cpu=%d state=%s balance=%d group=%#lx busy_nr=%d imbalance=%ld flags=%#x ld_moved=%d bal_int=%d", + __entry->cpu, + __entry->idle == CPU_IDLE ? "idle" : + (__entry->idle == CPU_NEWLY_IDLE ? "newly_idle" : "busy"), + __entry->balance, + __entry->group_mask, __entry->busiest_nr_running, + __entry->imbalance, __entry->env_flags, __entry->ld_moved, + __entry->balance_interval) +); + DECLARE_EVENT_CLASS(sched_process_template, TP_PROTO(struct task_struct *p), @@ -612,6 +1263,122 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TP_printk("cpu=%d", __entry->cpu) ); +TRACE_EVENT(sched_get_nr_running_avg, + + TP_PROTO(int avg, int big_avg, int iowait_avg), + + TP_ARGS(avg, big_avg, iowait_avg), + + TP_STRUCT__entry( + __field( int, avg ) + __field( int, big_avg ) + __field( int, iowait_avg ) + ), + + TP_fast_assign( + __entry->avg = avg; + __entry->big_avg = big_avg; + __entry->iowait_avg = iowait_avg; + ), + + TP_printk("avg=%d big_avg=%d iowait_avg=%d", + __entry->avg, __entry->big_avg, __entry->iowait_avg) +); + +TRACE_EVENT(core_ctl_eval_need, + + TP_PROTO(unsigned int cpu, unsigned int old_need, + unsigned int new_need, unsigned int updated), + TP_ARGS(cpu, old_need, new_need, updated), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, old_need) + __field(u32, new_need) + __field(u32, updated) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->old_need = old_need; + __entry->new_need = new_need; + __entry->updated = updated; + ), + TP_printk("cpu=%u, old_need=%u, new_need=%u, updated=%u", __entry->cpu, + __entry->old_need, __entry->new_need, __entry->updated) +); + +TRACE_EVENT(core_ctl_set_busy, + + TP_PROTO(unsigned int cpu, unsigned int busy, + unsigned int old_is_busy, unsigned int is_busy), + TP_ARGS(cpu, busy, old_is_busy, is_busy), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, busy) + __field(u32, old_is_busy) + __field(u32, is_busy) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->busy = busy; + __entry->old_is_busy = old_is_busy; + __entry->is_busy = is_busy; + ), + TP_printk("cpu=%u, busy=%u, old_is_busy=%u, new_is_busy=%u", + __entry->cpu, __entry->busy, __entry->old_is_busy, + __entry->is_busy) +); + +TRACE_EVENT(core_ctl_set_boost, + + TP_PROTO(u32 refcount, s32 ret), + TP_ARGS(refcount, ret), + TP_STRUCT__entry( + __field(u32, refcount) + __field(s32, ret) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->ret = ret; + ), + TP_printk("refcount=%u, ret=%d", __entry->refcount, __entry->ret) +); + +/** + * sched_isolate - called when cores are isolated/unisolated + * + * @acutal_mask: mask of cores actually isolated/unisolated + * @req_mask: mask of cores requested isolated/unisolated + * @online_mask: cpu online mask + * @time: amount of time in us it took to isolate/unisolate + * @isolate: 1 if isolating, 0 if unisolating + * + */ +TRACE_EVENT(sched_isolate, + + TP_PROTO(unsigned int requested_cpu, unsigned int isolated_cpus, + u64 start_time, unsigned char isolate), + + TP_ARGS(requested_cpu, isolated_cpus, start_time, isolate), + + TP_STRUCT__entry( + __field(u32, requested_cpu) + __field(u32, isolated_cpus) + __field(u32, time) + __field(unsigned char, isolate) + ), + + TP_fast_assign( + __entry->requested_cpu = requested_cpu; + __entry->isolated_cpus = isolated_cpus; + __entry->time = div64_u64(sched_clock() - start_time, 1000); + __entry->isolate = isolate; + ), + + TP_printk("iso cpu=%u cpus=0x%x time=%u us isolated=%d", + __entry->requested_cpu, __entry->isolated_cpus, + __entry->time, __entry->isolate) +); + TRACE_EVENT(sched_contrib_scale_f, TP_PROTO(int cpu, unsigned long freq_scale_factor, @@ -957,155 +1724,6 @@ TRACE_EVENT(sched_overutilized, TP_printk("overutilized=%d", __entry->overutilized ? 1 : 0) ); -#ifdef CONFIG_SCHED_WALT -struct rq; - -TRACE_EVENT(walt_update_task_ravg, - - TP_PROTO(struct task_struct *p, struct rq *rq, int evt, - u64 wallclock, u64 irqtime), - - TP_ARGS(p, rq, evt, wallclock, irqtime), - - TP_STRUCT__entry( - __array( char, comm, TASK_COMM_LEN ) - __field( pid_t, pid ) - __field( pid_t, cur_pid ) - __field(unsigned int, cur_freq ) - __field( u64, wallclock ) - __field( u64, mark_start ) - __field( u64, delta_m ) - __field( u64, win_start ) - __field( u64, delta ) - __field( u64, irqtime ) - __field( int, evt ) - __field(unsigned int, demand ) - __field(unsigned int, sum ) - __field( int, cpu ) - __field( u64, cs ) - __field( u64, ps ) - __field( u32, curr_window ) - __field( u32, prev_window ) - __field( u64, nt_cs ) - __field( u64, nt_ps ) - __field( u32, active_windows ) - ), - - TP_fast_assign( - __entry->wallclock = wallclock; - __entry->win_start = rq->window_start; - __entry->delta = (wallclock - rq->window_start); - __entry->evt = evt; - __entry->cpu = rq->cpu; - __entry->cur_pid = rq->curr->pid; - __entry->cur_freq = rq->cur_freq; - memcpy(__entry->comm, p->comm, TASK_COMM_LEN); - __entry->pid = p->pid; - __entry->mark_start = p->ravg.mark_start; - __entry->delta_m = (wallclock - p->ravg.mark_start); - __entry->demand = p->ravg.demand; - __entry->sum = p->ravg.sum; - __entry->irqtime = irqtime; - __entry->cs = rq->curr_runnable_sum; - __entry->ps = rq->prev_runnable_sum; - __entry->curr_window = p->ravg.curr_window; - __entry->prev_window = p->ravg.prev_window; - __entry->nt_cs = rq->nt_curr_runnable_sum; - __entry->nt_ps = rq->nt_prev_runnable_sum; - __entry->active_windows = p->ravg.active_windows; - ), - - TP_printk("wc %llu ws %llu delta %llu event %d cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u sum %u irqtime %llu" - " cs %llu ps %llu cur_window %u prev_window %u nt_cs %llu nt_ps %llu active_wins %u" - , __entry->wallclock, __entry->win_start, __entry->delta, - __entry->evt, __entry->cpu, - __entry->cur_freq, __entry->cur_pid, - __entry->pid, __entry->comm, __entry->mark_start, - __entry->delta_m, __entry->demand, - __entry->sum, __entry->irqtime, - __entry->cs, __entry->ps, - __entry->curr_window, __entry->prev_window, - __entry->nt_cs, __entry->nt_ps, - __entry->active_windows - ) -); - -TRACE_EVENT(walt_update_history, - - TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, - int evt), - - TP_ARGS(rq, p, runtime, samples, evt), - - TP_STRUCT__entry( - __array( char, comm, TASK_COMM_LEN ) - __field( pid_t, pid ) - __field(unsigned int, runtime ) - __field( int, samples ) - __field( int, evt ) - __field( u64, demand ) - __field(unsigned int, walt_avg ) - __field(unsigned int, pelt_avg ) - __array( u32, hist, RAVG_HIST_SIZE_MAX) - __field( int, cpu ) - ), - - TP_fast_assign( - memcpy(__entry->comm, p->comm, TASK_COMM_LEN); - __entry->pid = p->pid; - __entry->runtime = runtime; - __entry->samples = samples; - __entry->evt = evt; - __entry->demand = p->ravg.demand; - __entry->walt_avg = (__entry->demand << 10); - do_div(__entry->walt_avg, walt_ravg_window); - __entry->pelt_avg = p->se.avg.util_avg; - memcpy(__entry->hist, p->ravg.sum_history, - RAVG_HIST_SIZE_MAX * sizeof(u32)); - __entry->cpu = rq->cpu; - ), - - TP_printk("%d (%s): runtime %u samples %d event %d demand %llu" - " walt %u pelt %u (hist: %u %u %u %u %u) cpu %d", - __entry->pid, __entry->comm, - __entry->runtime, __entry->samples, __entry->evt, - __entry->demand, - __entry->walt_avg, - __entry->pelt_avg, - __entry->hist[0], __entry->hist[1], - __entry->hist[2], __entry->hist[3], - __entry->hist[4], __entry->cpu) -); - -TRACE_EVENT(walt_migration_update_sum, - - TP_PROTO(struct rq *rq, struct task_struct *p), - - TP_ARGS(rq, p), - - TP_STRUCT__entry( - __field(int, cpu ) - __field(int, pid ) - __field( u64, cs ) - __field( u64, ps ) - __field( s64, nt_cs ) - __field( s64, nt_ps ) - ), - - TP_fast_assign( - __entry->cpu = cpu_of(rq); - __entry->cs = rq->curr_runnable_sum; - __entry->ps = rq->prev_runnable_sum; - __entry->nt_cs = (s64)rq->nt_curr_runnable_sum; - __entry->nt_ps = (s64)rq->nt_prev_runnable_sum; - __entry->pid = p->pid; - ), - - TP_printk("cpu %d: cs %llu ps %llu nt_cs %lld nt_ps %lld pid %d", - __entry->cpu, __entry->cs, __entry->ps, - __entry->nt_cs, __entry->nt_ps, __entry->pid) -); -#endif /* CONFIG_SCHED_WALT */ #endif /* CONFIG_SMP */ |