Merge pull request #75 from sched-ext/htejun

scx: Build fix after kernel update

Author: Byte-Lab

scheds/c/scx_flatcg.bpf.c

@@ -302,16 +302,18 @@ static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc)
 	bpf_spin_unlock(&cgv_tree_lock);
 }
 
-void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
+s32 BPF_STRUCT_OPS(fcg_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
 {
 	struct fcg_task_ctx *taskc;
-	struct cgroup *cgrp;
-	struct fcg_cgrp_ctx *cgc;
+	bool is_idle = false;
+	s32 cpu;
+
+	cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
 
 	taskc = bpf_task_storage_get(&task_ctx, p, 0, 0);
 	if (!taskc) {
 		scx_bpf_error("task_ctx lookup failed");
-		return;
+		return cpu;
 	}
 
 	/*
@@ -321,7 +323,7 @@ void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
 	 * affinities so that we don't have to worry about per-cgroup dq's
 	 * containing tasks that can't be executed from some CPUs.
 	 */
-	if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) {
+	if (is_idle || p->nr_cpus_allowed != nr_cpus) {
 		/*
 		 * Tell fcg_stopping() that this bypassed the regular scheduling
 		 * path and should be force charged to the cgroup. 0 is used to
@@ -338,14 +340,28 @@ void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
 		 * implement per-cgroup fallback dq's instead so that we have
 		 * more control over when tasks with custom cpumask get issued.
 		 */
-		if ((enq_flags & SCX_ENQ_LOCAL) ||
+		if (is_idle ||
 		    (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) {
 			stat_inc(FCG_STAT_LOCAL);
-			scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
+			scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
 		} else {
 			stat_inc(FCG_STAT_GLOBAL);
-			scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
+			scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
 		}
+	}
+
+	return cpu;
+}
+
+void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
+{
+	struct fcg_task_ctx *taskc;
+	struct cgroup *cgrp;
+	struct fcg_cgrp_ctx *cgc;
+
+	taskc = bpf_task_storage_get(&task_ctx, p, 0, 0);
+	if (!taskc) {
+		scx_bpf_error("task_ctx lookup failed");
 		return;
 	}
 
@@ -756,8 +772,8 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev)
 	}
 }
 
-s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
+s32 BPF_STRUCT_OPS(fcg_init_task, struct task_struct *p,
+		   struct scx_init_task_args *args)
 {
 	struct fcg_task_ctx *taskc;
 	struct fcg_cgrp_ctx *cgc;
@@ -893,13 +909,14 @@ void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei)
 
 SEC(".struct_ops.link")
 struct sched_ext_ops flatcg_ops = {
+	.select_cpu		= (void *)fcg_select_cpu,
 	.enqueue		= (void *)fcg_enqueue,
 	.dispatch		= (void *)fcg_dispatch,
 	.runnable		= (void *)fcg_runnable,
 	.running		= (void *)fcg_running,
 	.stopping		= (void *)fcg_stopping,
 	.quiescent		= (void *)fcg_quiescent,
-	.prep_enable		= (void *)fcg_prep_enable,
+	.init_task		= (void *)fcg_init_task,
 	.cgroup_set_weight	= (void *)fcg_cgroup_set_weight,
 	.cgroup_init		= (void *)fcg_cgroup_init,
 	.cgroup_exit		= (void *)fcg_cgroup_exit,

scheds/c/scx_nest.bpf.c

@@ -92,9 +92,6 @@ struct task_ctx {
 	 * if the task should attach to the core that it will execute on next.
 	 */
 	s32 prev_cpu;
-
-	/* Dispatch directly to local_dsq */
-	bool force_local;
 };
 
 struct {
@@ -231,8 +228,6 @@ s32 BPF_STRUCT_OPS(nest_select_cpu, struct task_struct *p, s32 prev_cpu,
 		return -ENOENT;
 	}
 
-	// Unset below if we can't find a core to migrate to.
-	tctx->force_local = true;
 	tctx->prev_cpu = prev_cpu;
 
 	bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(primary));
@@ -337,7 +332,6 @@ s32 BPF_STRUCT_OPS(nest_select_cpu, struct task_struct *p, s32 prev_cpu,
 	}
 
 	bpf_rcu_read_unlock();
-	tctx->force_local = false;
 	return prev_cpu;
 
 promote_to_primary:
@@ -374,30 +368,21 @@ s32 BPF_STRUCT_OPS(nest_select_cpu, struct task_struct *p, s32 prev_cpu,
 	}
 	bpf_rcu_read_unlock();
 	update_attached(tctx, prev_cpu, cpu);
+	scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, 0);
 	return cpu;
 }
 
 void BPF_STRUCT_OPS(nest_enqueue, struct task_struct *p, u64 enq_flags)
 {
 	struct task_ctx *tctx;
 	u64 vtime = p->scx.dsq_vtime;
-	s32 cpu = bpf_get_smp_processor_id();
 
 	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
 	if (!tctx) {
 		scx_bpf_error("Unable to find task ctx");
 		return;
 	}
 
-	if (tctx->force_local || (enq_flags & SCX_ENQ_LOCAL)) {
-		tctx->force_local = false;
-		if (enq_flags & SCX_ENQ_LOCAL)
-			update_attached(tctx, tctx->prev_cpu, cpu);
-
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags);
-		return;
-	}
-
 	/*
 	 * Limit the amount of budget that an idling task can accumulate
 	 * to one slice.
@@ -495,8 +480,8 @@ void BPF_STRUCT_OPS(nest_stopping, struct task_struct *p, bool runnable)
 	p->scx.dsq_vtime += (slice_ns - p->scx.slice) * 100 / p->scx.weight;
 }
 
-s32 BPF_STRUCT_OPS(nest_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
+s32 BPF_STRUCT_OPS(nest_init_task, struct task_struct *p,
+		   struct scx_init_task_args *args)
 {
 	struct task_ctx *tctx;
 	struct bpf_cpumask *cpumask;
@@ -524,8 +509,7 @@ s32 BPF_STRUCT_OPS(nest_prep_enable, struct task_struct *p,
 	return 0;
 }
 
-void BPF_STRUCT_OPS(nest_enable, struct task_struct *p,
-		    struct scx_enable_args *args)
+void BPF_STRUCT_OPS(nest_enable, struct task_struct *p)
 {
 	p->scx.dsq_vtime = vtime_now;
 }
@@ -682,7 +666,7 @@ struct sched_ext_ops nest_ops = {
 	.dispatch		= (void *)nest_dispatch,
 	.running		= (void *)nest_running,
 	.stopping		= (void *)nest_stopping,
-	.prep_enable		= (void *)nest_prep_enable,
+	.init_task		= (void *)nest_init_task,
 	.enable			= (void *)nest_enable,
 	.init			= (void *)nest_init,
 	.exit			= (void *)nest_exit,

scheds/c/scx_qmap.bpf.c

@@ -354,8 +354,8 @@ void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args
 		__sync_fetch_and_add(&nr_reenqueued, cnt);
 }
 
-s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
+s32 BPF_STRUCT_OPS(qmap_init_task, struct task_struct *p,
+		   struct scx_init_task_args *args)
 {
 	if (p->tgid == disallow_tgid)
 		p->scx.disallow = true;
@@ -391,7 +391,7 @@ struct sched_ext_ops qmap_ops = {
 	.dispatch		= (void *)qmap_dispatch,
 	.core_sched_before	= (void *)qmap_core_sched_before,
 	.cpu_release		= (void *)qmap_cpu_release,
-	.prep_enable		= (void *)qmap_prep_enable,
+	.init_task		= (void *)qmap_init_task,
 	.init			= (void *)qmap_init,
 	.exit			= (void *)qmap_exit,
 	.flags			= SCX_OPS_ENQ_LAST,

scheds/c/scx_simple.bpf.c

@@ -51,19 +51,22 @@ static inline bool vtime_before(u64 a, u64 b)
 	return (s64)(a - b) < 0;
 }
 
-void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
+s32 BPF_STRUCT_OPS(simple_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
 {
-	/*
-	 * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that
-	 * running @p on its CPU directly shouldn't affect fairness. Just queue
-	 * it on the local FIFO.
-	 */
-	if (enq_flags & SCX_ENQ_LOCAL) {
+	bool is_idle = false;
+	s32 cpu;
+
+	cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
+	if (is_idle) {
 		stat_inc(0);	/* count local queueing */
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
-		return;
+		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
 	}
 
+	return cpu;
+}
+
+void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
+{
 	stat_inc(1);	/* count global queueing */
 
 	if (fifo_sched) {
@@ -120,8 +123,7 @@ void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable)
 	p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight;
 }
 
-void BPF_STRUCT_OPS(simple_enable, struct task_struct *p,
-		    struct scx_enable_args *args)
+void BPF_STRUCT_OPS(simple_enable, struct task_struct *p)
 {
 	p->scx.dsq_vtime = vtime_now;
 }
@@ -141,6 +143,7 @@ void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei)
 
 SEC(".struct_ops.link")
 struct sched_ext_ops simple_ops = {
+	.select_cpu		= (void *)simple_select_cpu,
 	.enqueue		= (void *)simple_enqueue,
 	.dispatch		= (void *)simple_dispatch,
 	.running		= (void *)simple_running,

scheds/c/scx_userland.bpf.c

@@ -300,8 +300,8 @@ void BPF_STRUCT_OPS(userland_update_idle, s32 cpu, bool idle)
 	}
 }
 
-s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
+s32 BPF_STRUCT_OPS(userland_init_task, struct task_struct *p,
+		   struct scx_init_task_args *args)
 {
 	if (bpf_task_storage_get(&task_ctx_stor, p, 0,
 				 BPF_LOCAL_STORAGE_GET_F_CREATE))
@@ -340,7 +340,7 @@ struct sched_ext_ops userland_ops = {
 	.enqueue		= (void *)userland_enqueue,
 	.dispatch		= (void *)userland_dispatch,
 	.update_idle		= (void *)userland_update_idle,
-	.prep_enable		= (void *)userland_prep_enable,
+	.init_task		= (void *)userland_init_task,
 	.init			= (void *)userland_init,
 	.exit			= (void *)userland_exit,
 	.flags			= SCX_OPS_ENQ_LAST | SCX_OPS_KEEP_BUILTIN_IDLE,

scheds/include/scx/common.bpf.h

@@ -68,6 +68,7 @@ const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym;
 const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym;
 void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym;
 void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
+s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
 bool scx_bpf_task_running(const struct task_struct *p) __ksym;
 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
 struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym;