diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -184,6 +184,9 @@ int flags, bool fp); static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp); +static bool __always_inline lockmgr_sleepgen_valid(struct lock *lk, + u_int sleepgen); + static void lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper) { @@ -581,7 +584,7 @@ static __noinline int lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, - const char *file, int line, struct lockmgr_wait *lwa) + u_int sleepgen, const char *file, int line, struct lockmgr_wait *lwa) { uintptr_t tid, x; int error = 0; @@ -653,6 +656,14 @@ * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); + + /* Check sleepgen. */ + if (!lockmgr_sleepgen_valid(lk, sleepgen)) { + sleepq_release(&lk->lock_object); + error = ENOLCK; + break; + } + x = lockmgr_read_value(lk); retry_sleepq: @@ -766,7 +777,7 @@ static __noinline int lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, - const char *file, int line, struct lockmgr_wait *lwa) + u_int sleepgen, const char *file, int line, struct lockmgr_wait *lwa) { struct lock_class *class; uintptr_t tid, x, v; @@ -868,6 +879,14 @@ * probabilly will need to manipulate waiters flags. */ sleepq_lock(&lk->lock_object); + + /* Check sleepgen. */ + if (!lockmgr_sleepgen_valid(lk, sleepgen)) { + sleepq_release(&lk->lock_object); + error = ENOLCK; + break; + } + x = lockmgr_read_value(lk); retry_sleepq: @@ -1024,7 +1043,8 @@ } out_xlock: - error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa); + error = lockmgr_xlock_hard(lk, flags, ilk, LK_SLEEPGEN_INVALID, file, + line, lwa); flags &= ~LK_INTERLOCK; out: lockmgr_exit(flags, ilk, 0); @@ -1043,6 +1063,8 @@ if (SCHEDULER_STOPPED()) return (0); + /* XXX SLEEPGEN ASSERTS? */ + op = flags & LK_TYPE_MASK; locked = false; switch (op) { @@ -1058,8 +1080,8 @@ file, line, flags); locked = true; } else { - return (lockmgr_slock_hard(lk, flags, ilk, file, line, - NULL)); + return (lockmgr_slock_hard(lk, flags, ilk, + LK_SLEEPGEN_INVALID, file, line, NULL)); } break; case LK_EXCLUSIVE: @@ -1074,8 +1096,8 @@ flags); locked = true; } else { - return (lockmgr_xlock_hard(lk, flags, ilk, file, line, - NULL)); + return (lockmgr_xlock_hard(lk, flags, ilk, + LK_SLEEPGEN_INVALID, file, line, NULL)); } break; case LK_UPGRADE: @@ -1243,7 +1265,8 @@ return (0); } - return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL)); + return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL, + LK_SLEEPGEN_INVALID, file, line, NULL)); } int @@ -1264,7 +1287,8 @@ return (0); } - return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL)); + return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL, + LK_SLEEPGEN_INVALID, file, line, NULL)); } int @@ -1299,15 +1323,16 @@ } int -__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, +__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk_, const char *wmesg, int pri, int timo, const char *file, int line) { GIANT_DECLARE; struct lockmgr_wait lwa; struct lock_class *class; + struct lock_object *ilk; const char *iwmesg; uintptr_t tid, v, x; - u_int op, realexslp; + u_int op, realexslp, sleepgen; int error, ipri, itimo, queue, wakeup_swapper; #ifdef LOCK_PROFILING uint64_t waittime = 0; @@ -1324,6 +1349,14 @@ ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri; itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo; + /* XXX may want to push some of this down into slock and xlock hard */ + ilk = NULL; + sleepgen = LK_SLEEPGEN_INVALID; + if ((flags & LK_INTERLOCK) != 0) + ilk = ilk_; + else if ((flags & LK_SLEEPGEN) != 0) + sleepgen = (uintptr_t)ilk_; + lwa.iwmesg = iwmesg; lwa.ipri = ipri; lwa.itimo = itimo; @@ -1338,6 +1371,15 @@ KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL, ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d", __func__, file, line)); + KASSERT((flags & (LK_INTERLOCK | LK_SLEEPGEN)) != + (LK_INTERLOCK | LK_SLEEPGEN), + ("%s: Invalid flags (LK_INTERLOCK | LK_SLEEPGEN) @ %s:%d ", + __func__, file, line)); + KASSERT((flags & LK_SLEEPGEN) == 0 || (sleepgen & 1) == 1, + ("%s: Invalid sleepgen @ %s:%d", __func__, file, line)); + KASSERT((flags & LK_SLEEPGEN) == 0 || op == LK_SHARED || + op == LK_EXCLUSIVE, + ("%s: Invalid op with LK_SLEEPGEN @ %s:%d", __func__, file, line)); KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), ("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread, lk->lock_object.lo_name, file, line)); @@ -1363,14 +1405,16 @@ wakeup_swapper = 0; switch (op) { case LK_SHARED: - return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa)); + return (lockmgr_slock_hard(lk, flags, ilk, sleepgen, file, + line, &lwa)); break; case LK_UPGRADE: case LK_TRYUPGRADE: return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa)); break; case LK_EXCLUSIVE: - return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa)); + return (lockmgr_xlock_hard(lk, flags, ilk, sleepgen, file, + line, &lwa)); break; case LK_DOWNGRADE: _lockmgr_assert(lk, KA_XLOCKED, file, line); @@ -1653,6 +1697,71 @@ } } +static __always_inline bool +lockmgr_sleepgen_valid(struct lock *lk, u_int sleepgen) +{ + return (sleepgen == LK_SLEEPGEN_INVALID || + atomic_load_int(&((struct lock_sleepgen *)lk)->lksg_sleepgen) == + sleepgen); +} + +static void +lockmgr_wakeall(struct lock *lk) +{ + uintptr_t x, v; + int wakeup_swapper; + + wakeup_swapper = 0; + + sleepq_lock(&lk->lock_object); + lk->lk_exslpfail = 0; + for (;;) { + x = lockmgr_read_value(lk); + v = x & ~LK_ALL_WAITERS; + if (x == v || atomic_fcmpset_ptr(&lk->lk_lock, &x, v)) + break; + cpu_spinwait(); + } + if ((x & LK_EXCLUSIVE_WAITERS) != 0) { + wakeup_swapper |= sleepq_broadcast(&lk->lock_object, + SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); + } + if ((x & LK_SHARED_WAITERS) != 0) { + wakeup_swapper |= sleepq_broadcast(&lk->lock_object, + SLEEPQ_LK, 0, SQ_SHARED_QUEUE); + } + sleepq_release(&lk->lock_object); + + if (wakeup_swapper) + kick_proc0(); +} + +void +_lockmgr_sleepgen_invalidate_hard(struct lock_sleepgen *lksg) +{ + atomic_add_int(&lksg->lksg_sleepgen, LK_SLEEPGEN_INCR); + lockmgr_wakeall(&lksg->lksg_lock); +} + +u_int ( +lockmgr_sleepgen_acquire)(struct lock_sleepgen *lksg) +{ + return _lockmgr_sleepgen_acquire(lksg); +} + +void ( +lockmgr_sleepgen_release)(struct lock_sleepgen *lksg) +{ + _lockmgr_sleepgen_release(lksg); +} + + +void ( +lockmgr_sleepgen_invalidate)(struct lock_sleepgen *lksg) +{ + _lockmgr_sleepgen_invalidate(lksg); +} + void lockmgr_printinfo(const struct lock *lk) { diff --git a/sys/sys/_lockmgr.h b/sys/sys/_lockmgr.h --- a/sys/sys/_lockmgr.h +++ b/sys/sys/_lockmgr.h @@ -46,4 +46,10 @@ #endif }; +struct lock_sleepgen { + struct lock lksg_lock; + u_int lksg_holders; + u_int lksg_sleepgen; +}; + #endif diff --git a/sys/sys/lockmgr.h b/sys/sys/lockmgr.h --- a/sys/sys/lockmgr.h +++ b/sys/sys/lockmgr.h @@ -115,6 +115,15 @@ NULL, wmesg, prio, timo, file, line)); } +static __inline int +_lockmgr_args_sleepgen(struct lock_sleepgen *lksg, u_int flags, u_int sleepgen, + const char *wmesg, int prio, int timo, const char *file, int line) +{ + + return (__lockmgr_args(&lksg->lksg_lock, flags, + (void *)(uintptr_t)sleepgen, wmesg, prio, timo, file, line)); +} + /* * Define aliases in order to complete lockmgr KPI. */ @@ -173,6 +182,7 @@ #define LK_TIMELOCK 0x001000 #define LK_NODDLKTREAT 0x002000 #define LK_ADAPTIVE 0x004000 +#define LK_SLEEPGEN 0x008000 /* * Operations for lockmgr(). @@ -208,6 +218,61 @@ #define KA_NOTRECURSED LA_NOTRECURSED #endif +/* XXX organize me */ +#define LK_SLEEPGEN_INVALID 0 +#define LK_SLEEPGEN_INIT 1 +#define LK_SLEEPGEN_INCR 2 + +#define _lockmgr_sleepgen_acquire(lksg) __extension__ ({ \ + atomic_add_acq_int(&(lksg)->lksg_holders, 1); \ + atomic_load_acq_int(&(lksg)->lksg_sleepgen); \ +}) + +#define _lockmgr_sleepgen_release(lksg) \ + atomic_add_int(&(lksg)->lksg_holders, -1) + +#define _lockmgr_sleepgen_invalidate(lksg) do { \ + lockmgr_assert(&(lksg)->lksg_lock, KA_LOCKED); \ + KASSERT(((lksg)->lksg_sleepgen & 1) == 1, \ + ("lockmgr_sleepgen_invalidate: lock %p bad sleepgen %u", \ + (lksg), (lksg)->lksg_sleepgen)); \ + atomic_thread_fence_rel(); \ + if (atomic_load_int(&(lksg)->lksg_holders) != 0) \ + _lockmgr_sleepgen_invalidate_hard((lksg)); \ +} while (0) + +u_int lockmgr_sleepgen_acquire(struct lock_sleepgen *); +void lockmgr_sleepgen_release(struct lock_sleepgen *); +void lockmgr_sleepgen_invalidate(struct lock_sleepgen *); +void _lockmgr_sleepgen_invalidate_hard(struct lock_sleepgen *); + +/* Macros for inline expansion. */ +#define lockmgr_sleepgen_acquire(lksg) _lockmgr_sleepgen_acquire((lksg)) +#define lockmgr_sleepgen_release(lksg) _lockmgr_sleepgen_release((lksg)) +#define lockmgr_sleepgen_invalidate(lksg) \ + _lockmgr_sleepgen_invalidate((lksg)) + +#define lockmgr_args_sleepgen_cond(lksg, flags, wmesg, prio, timo, \ + cond) __extension__ ({ \ + int _error; \ + u_int _sleepgen; \ + _sleepgen = lockmgr_sleepgen_acquire((lksg)); \ + /* Check cond after acquiring the sleepgen. */ \ + if (cond) { \ + _error = _lockmgr_args_sleepgen((lksg), (flags), \ + _sleepgen, (wmesg), (prio), (timo), LOCK_FILE, \ + LOCK_LINE); \ + } else { \ + _error = ENOLCK; \ + } \ + lockmgr_sleepgen_release((lksg)); \ + _error; \ +}) + +#define lockmgr_sleepgen_cond(lksg, flags, wmesg, cond) \ + lockmgr_args_sleepgen_cond((lksg), (flags), (wmesg), \ + LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, (cond)) + #endif /* _KERNEL */ #endif /* !_SYS_LOCKMGR_H_ */