diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -292,6 +292,7 @@ */ u_char tdq_ts_ticks; int tdq_id; /* (c) cpuid. */ + int tdq_do_idle; struct runq tdq_runq; /* (t) Run queue. */ char tdq_name[TDQ_NAME_LEN]; #ifdef KTR @@ -1499,9 +1500,10 @@ cpu = sched_lowest(ccg, mask, pri, INT_MAX, ts->ts_cpu, r); if (cpu >= 0) SCHED_STAT_INC(pickcpu_intrbind); - } else - /* Search the LLC for the least loaded idle CPU we can run now. */ - if (ccg != NULL) { + } else if (ccg != NULL) { + /* + * Search the LLC for the least loaded idle CPU we can run now. + */ cpu = sched_lowest(ccg, mask, max(pri, PRI_MAX_TIMESHARE), INT_MAX, ts->ts_cpu, r); if (cpu >= 0) @@ -1575,6 +1577,8 @@ struct thread *td; TDQ_LOCK_ASSERT(tdq, MA_OWNED); + if (__predict_false(tdq->tdq_do_idle)) + return (NULL); /* Return NULL for idle thread */ td = runq_choose_realtime(&tdq->tdq_runq); if (td != NULL) return (td); @@ -2660,16 +2664,11 @@ thread_unlock(td); } -void -sched_preempt(struct thread *td) +static void +sched_preempt_locked(struct tdq *tdq, struct thread *td) { - struct tdq *tdq; int flags; - SDT_PROBE2(sched, , , surrender, td, td->td_proc); - - thread_lock(td); - tdq = TDQ_SELF(); TDQ_LOCK_ASSERT(tdq, MA_OWNED); if (td->td_priority > tdq->tdq_lowpri) { if (td->td_critnest == 1) { @@ -2681,12 +2680,35 @@ return; } td->td_owepreempt = 1; - } else { + } else tdq->tdq_owepreempt = 0; - } thread_unlock(td); } +void +sched_preempt(struct thread *td) +{ + struct tdq *tdq; + + SDT_PROBE2(sched, , , surrender, td, td->td_proc); + + thread_lock(td); + tdq = TDQ_SELF(); + sched_preempt_locked(tdq, td); +} + +void +sched_do_idle(struct thread *td, bool do_idle) +{ + struct tdq *tdq; + + thread_lock(td); + tdq = TDQ_SELF(); + TDQ_LOCK_ASSERT(tdq, MA_OWNED); + tdq->tdq_do_idle = do_idle; + sched_preempt_locked(tdq, td); +} + /* * Fix priorities on return to user-space. Priorities may be elevated due * to static priorities in msleep() or similar. @@ -2841,6 +2863,8 @@ tdq = TDQ_SELF(); TDQ_LOCK_ASSERT(tdq, MA_OWNED); td = tdq_choose(tdq); + KASSERT(td != PCPU_GET(idlethread), ("sched_choose: idle thread " + "explicitly returned; tdq_choose should return NULL instead")); if (td != NULL) { tdq_runq_rem(tdq, td); tdq->tdq_lowpri = td->td_priority; diff --git a/sys/sys/sched.h b/sys/sys/sched.h --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -147,6 +147,7 @@ void sched_clock(struct thread *td, int cnt); void sched_idletd(void *); void sched_preempt(struct thread *td); +void sched_do_idle(struct thread *td, bool do_idle); void sched_relinquish(struct thread *td); void sched_rem(struct thread *td); void sched_wakeup(struct thread *td, int srqflags);