diff --git a/sys/net/if_llatbl.h b/sys/net/if_llatbl.h --- a/sys/net/if_llatbl.h +++ b/sys/net/if_llatbl.h @@ -174,6 +174,7 @@ int llt_maxentries; struct llentries *lle_head; struct ifnet *llt_ifp; + struct mtx llt_lock; llt_lookup_t *llt_lookup; llt_alloc_t *llt_alloc_entry; @@ -192,6 +193,12 @@ llt_post_resolved_t *llt_post_resolved; }; +#define LLTABLE_LOCK(llt) mtx_lock(&(llt)->llt_lock) +#define LLTABLE_UNLOCK(llt) mtx_unlock(&(llt)->llt_lock) +#define LLTABLE_LOCK_ASSERT(llt) mtx_assert(&(llt)->llt_lock, MA_OWNED) +#define LLTABLE_RLOCK_ASSERT(llt) MPASS(in_epoch(net_epoch_preempt) || \ + mtx_owned(&(llt)->llt_lock)) + MALLOC_DECLARE(M_LLTABLE); /* @@ -261,7 +268,7 @@ struct ifnet *lltable_get_ifp(const struct lltable *llt); int lltable_get_af(const struct lltable *llt); -bool lltable_acquire_wlock(struct ifnet *ifp, struct llentry *lle); +bool lltable_trylock(struct llentry *lle); int lltable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg); diff --git a/sys/net/if_llatbl.c b/sys/net/if_llatbl.c --- a/sys/net/if_llatbl.c +++ b/sys/net/if_llatbl.c @@ -206,7 +206,7 @@ if ((lle->la_flags & LLE_LINKED) != 0) return (0); - IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp); + LLTABLE_LOCK_ASSERT(llt); if (llt->llt_maxentries > 0 && llt->llt_entries >= llt->llt_maxentries) @@ -233,7 +233,7 @@ return (0); llt = lle->lle_tbl; - IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp); + LLTABLE_LOCK_ASSERT(llt); KASSERT(llt->llt_entries > 0, ("%s: lltable %p (%s) entries %d <= 0", __func__, llt, if_name(llt->llt_ifp), llt->llt_entries)); @@ -283,12 +283,12 @@ pmd.flags = flags; CK_LIST_INIT(&pmd.dchain); - IF_AFDATA_WLOCK(llt->llt_ifp); + LLTABLE_LOCK(llt); /* Push matching lles to chain */ lltable_foreach_lle(llt, htable_prefix_free_cb, &pmd); llentries_unlink(llt, &pmd.dchain); - IF_AFDATA_WUNLOCK(llt->llt_ifp); + LLTABLE_UNLOCK(llt); CK_LIST_FOREACH_SAFE(lle, &pmd.dchain, lle_chain, next) lltable_free_entry(llt, lle); @@ -298,6 +298,7 @@ htable_free_tbl(struct lltable *llt) { + mtx_destroy(&llt->llt_lock); free(llt->lle_head, M_LLTABLE); free(llt, M_LLTABLE); } @@ -357,14 +358,12 @@ * On failure, false is returned and lle wlock is still held. */ bool -lltable_acquire_wlock(struct ifnet *ifp, struct llentry *lle) +lltable_trylock(struct llentry *lle) { NET_EPOCH_ASSERT(); - /* Perform real LLE update */ - /* use afdata WLOCK to update fields */ LLE_WUNLOCK(lle); - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(lle->lle_tbl); LLE_WLOCK(lle); /* @@ -372,7 +371,7 @@ * this lle. Check and return */ if ((lle->la_flags & LLE_DELETED) != 0) { - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(lle->lle_tbl); return (false); } @@ -392,13 +391,13 @@ const char *linkhdr, size_t linkhdrsize, int lladdr_off) { - if (!lltable_acquire_wlock(ifp, lle)) + if (!lltable_trylock(lle)) return (0); /* Update data */ lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, lladdr_off); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(lle->lle_tbl); return (1); } @@ -630,9 +629,9 @@ if (llt->llt_ifp->if_flags & IFF_LOOPBACK) return; - IF_AFDATA_WLOCK(llt->llt_ifp); + LLTABLE_LOCK(llt); lltable_foreach_lle(llt, llentry_update_ifaddr, llt->llt_ifp); - IF_AFDATA_WUNLOCK(llt->llt_ifp); + LLTABLE_UNLOCK(llt); } /* @@ -696,11 +695,11 @@ lltable_unlink(llt); CK_LIST_INIT(&dchain); - IF_AFDATA_WLOCK(llt->llt_ifp); + LLTABLE_LOCK(llt); /* Push all lles to @dchain */ lltable_foreach_lle(llt, lltable_free_cb, &dchain); llentries_unlink(llt, &dchain); - IF_AFDATA_WUNLOCK(llt->llt_ifp); + LLTABLE_UNLOCK(llt); CK_LIST_FOREACH_SAFE(lle, &dchain, lle_chain, next) { llentry_free(lle); @@ -722,24 +721,22 @@ const struct sockaddr *l3addr) { struct llentry *lle; - struct ifnet *ifp; - ifp = llt->llt_ifp; - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(llt); lle = lla_lookup(llt, LLE_SF(l3addr->sa_family, LLE_EXCLUSIVE), l3addr); if (lle == NULL) { - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(llt); return (ENOENT); } if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) { - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(llt); LLE_WUNLOCK(lle); return (EPERM); } lltable_unlink_entry(llt, lle); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(llt); llt->llt_delete_entry(llt, lle); @@ -798,10 +795,10 @@ lmd.func = func; lmd.farg = farg; - IF_AFDATA_WLOCK(llt->llt_ifp); + LLTABLE_LOCK(llt); lltable_foreach_lle(llt, lltable_delete_conditional_cb, &lmd); llentries_unlink(llt, &lmd.dchain); - IF_AFDATA_WUNLOCK(llt->llt_ifp); + LLTABLE_UNLOCK(llt); CK_LIST_FOREACH_SAFE(lle, &lmd.dchain, lle_chain, next) llt->llt_delete_entry(llt, lle); @@ -817,6 +814,7 @@ llt->llt_hsize = hsize; llt->lle_head = malloc(sizeof(struct llentries) * hsize, M_LLTABLE, M_WAITOK | M_ZERO); + mtx_init(&llt->llt_lock, "lltable", NULL, MTX_DEF); for (i = 0; i < llt->llt_hsize; i++) CK_LIST_INIT(&llt->lle_head[i]); @@ -1021,13 +1019,13 @@ /* Try to link new entry */ lle_tmp = NULL; - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(llt); LLE_WLOCK(lle); lle_tmp = lla_lookup(llt, LLE_EXCLUSIVE, dst); if (lle_tmp != NULL) { /* Check if we are trying to replace immutable entry */ if ((lle_tmp->la_flags & LLE_IFADDR) != 0) { - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(llt); LLE_WUNLOCK(lle_tmp); lltable_free_entry(llt, lle); return (EPERM); @@ -1036,7 +1034,7 @@ lltable_unlink_entry(llt, lle_tmp); } lltable_link_entry(llt, lle); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(llt); if (lle_tmp != NULL) { EVENTHANDLER_INVOKE(lle_event, lle_tmp,LLENTRY_EXPIRED); diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c --- a/sys/netinet/if_ether.c +++ b/sys/netinet/if_ether.c @@ -288,7 +288,7 @@ /* XXX: LOR avoidance. We still have ref on lle. */ LLE_WUNLOCK(lle); - IF_AFDATA_LOCK(ifp); + LLTABLE_LOCK(LLTABLE(ifp)); LLE_WLOCK(lle); /* Guard against race with other llentry_free(). */ @@ -296,7 +296,7 @@ LLE_REMREF(lle); lltable_unlink_entry(lle->lle_tbl, lle); } - IF_AFDATA_UNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE(ifp)); size_t pkts_dropped = llentry_free(lle); @@ -488,13 +488,13 @@ return (EINVAL); } - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(LLTABLE(ifp)); LLE_WLOCK(la); la_tmp = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); /* Prefer ANY existing lle over newly-created one */ if (la_tmp == NULL) lltable_link_entry(LLTABLE(ifp), la); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE(ifp)); if (la_tmp != NULL) { lltable_free_entry(LLTABLE(ifp), la); la = la_tmp; @@ -961,7 +961,7 @@ lltable_set_entry_addr(ifp, la, linkhdr, linkhdrsize, lladdr_off); - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(LLTABLE(ifp)); LLE_WLOCK(la); la_tmp = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); @@ -983,7 +983,7 @@ */ if (la_tmp == NULL) lltable_link_entry(LLTABLE(ifp), la); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE(ifp)); if (la_tmp == NULL) { arp_mark_lle_reachable(la, ifp); @@ -1301,7 +1301,7 @@ return; } - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(LLTABLE(ifp)); LLE_WLOCK(lle); /* Unlink any entry if exists */ lle_tmp = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); @@ -1309,7 +1309,7 @@ lltable_unlink_entry(LLTABLE(ifp), lle_tmp); lltable_link_entry(LLTABLE(ifp), lle); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE(ifp)); if (lle_tmp != NULL) EVENTHANDLER_INVOKE(lle_event, lle_tmp, LLENTRY_EXPIRED); diff --git a/sys/netinet/in.c b/sys/netinet/in.c --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -1546,7 +1546,7 @@ /* Unlink entry from table if not already */ if ((lle->la_flags & LLE_LINKED) != 0) { - IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp); + LLTABLE_LOCK_ASSERT(llt); lltable_unlink_entry(llt, lle); } @@ -1728,7 +1728,7 @@ const struct sockaddr_in *sin = (const struct sockaddr_in *)l3addr; struct llentry *lle; - IF_AFDATA_LOCK_ASSERT(llt->llt_ifp); + LLTABLE_RLOCK_ASSERT(llt); KASSERT(l3addr->sa_family == AF_INET, ("sin_family %d", l3addr->sa_family)); KASSERT((flags & (LLE_UNLOCKED | LLE_EXCLUSIVE)) != diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -2253,15 +2253,13 @@ static void in6_lltable_free_entry(struct lltable *llt, struct llentry *lle) { - struct ifnet *ifp __diagused; LLE_WLOCK_ASSERT(lle); KASSERT(llt != NULL, ("lltable is NULL")); /* Unlink entry from table */ if ((lle->la_flags & LLE_LINKED) != 0) { - ifp = llt->llt_ifp; - IF_AFDATA_WLOCK_ASSERT(ifp); + LLTABLE_LOCK_ASSERT(llt); lltable_unlink_entry(llt, lle); } @@ -2421,7 +2419,7 @@ int family = flags >> 16; struct llentry *lle; - IF_AFDATA_LOCK_ASSERT(llt->llt_ifp); + LLTABLE_RLOCK_ASSERT(llt); KASSERT(l3addr->sa_family == AF_INET6, ("sin_family %d", l3addr->sa_family)); KASSERT((flags & (LLE_UNLOCKED | LLE_EXCLUSIVE)) != @@ -2445,7 +2443,7 @@ LLE_RLOCK(lle); /* - * If the afdata lock is not held, the LLE may have been unlinked while + * If the lltable lock is not held, the LLE may have been unlinked while * we were blocked on the LLE lock. Check for this case. */ if (__predict_false((lle->la_flags & LLE_LINKED) == 0)) { @@ -2743,9 +2741,9 @@ return; llt = LLTABLE6(ifp); - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(llt); need_purge = ((llt->llt_flags & LLT_ADDEDPROXY) != 0); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(llt); /* * Ever added proxy ndp entries, leave solicited node multicast diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c --- a/sys/netinet6/mld6.c +++ b/sys/netinet6/mld6.c @@ -165,8 +165,8 @@ * scope ID is only used by MLD to select the outgoing interface. * * During interface attach and detach, MLD will take MLD_LOCK *after* - * the IF_AFDATA_LOCK. - * As in6_setscope() takes IF_AFDATA_LOCK then SCOPE_LOCK, we can't call + * the LLTABLE_LOCK. + * As in6_setscope() takes LLTABLE_LOCK then SCOPE_LOCK, we can't call * it with MLD_LOCK held without triggering an LOR. A netisr with indirect * dispatch could work around this, but we'd rather not do that, as it * can introduce other races. @@ -182,7 +182,7 @@ * calls in6_setscope() internally whilst MLD_LOCK is held. This will * trigger a LOR warning in WITNESS when the ifnet is detached. * - * The right answer is probably to make IF_AFDATA_LOCK an rwlock, given + * The right answer is probably to make LLTABLE_LOCK an rwlock, given * how it's used across the network stack. Here we're simply exploiting * the fact that MLD runs at a similar layer in the stack to scope6.c. * @@ -553,7 +553,7 @@ * Hook for domifdetach. * Runs after link-layer cleanup; free MLD state. * - * SMPng: Normally called with IF_AFDATA_LOCK held. + * SMPng: Normally called with LLTABLE_LOCK held. */ void mld_domifdetach(struct ifnet *ifp) diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -1217,7 +1217,7 @@ sin6.sin6_family = AF_INET6; sin6.sin6_addr = *addr6; - IF_AFDATA_LOCK_ASSERT(ifp); + LLTABLE_RLOCK_ASSERT(LLTABLE6(ifp)); ln = lla_lookup(LLTABLE6(ifp), flags, (struct sockaddr *)&sin6); @@ -1342,7 +1342,7 @@ int rc = 0; NET_EPOCH_ASSERT(); - IF_AFDATA_UNLOCK_ASSERT(ifp); + if (nd6_is_new_addr_neighbor(addr, ifp)) return (1); @@ -1414,10 +1414,10 @@ NET_EPOCH_ASSERT(); LLE_WLOCK_ASSERT(lle); - if (!lltable_acquire_wlock(ifp, lle)) + if (!lltable_trylock(lle)) return (false); bool ret = nd6_try_set_entry_addr_locked(ifp, lle, lladdr); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(lle->lle_tbl); return (ret); } @@ -1556,7 +1556,7 @@ * free(9) in llentry_free() if someone else holds one as well. */ LLE_WUNLOCK(ln); - IF_AFDATA_LOCK(ifp); + LLTABLE_LOCK(ln->lle_tbl); LLE_WLOCK(ln); /* Guard against race with other llentry_free(). */ if (ln->la_flags & LLE_LINKED) { @@ -1564,7 +1564,7 @@ LLE_REMREF(ln); lltable_unlink_entry(ln->lle_tbl, ln); } - IF_AFDATA_UNLOCK(ifp); + LLTABLE_UNLOCK(ln->lle_tbl); nd6_free_children(ln); @@ -1969,7 +1969,6 @@ int lladdr_off; NET_EPOCH_ASSERT(); - IF_AFDATA_UNLOCK_ASSERT(ifp); KASSERT(ifp != NULL, ("%s: ifp == NULL", __func__)); KASSERT(from != NULL, ("%s: from == NULL", __func__)); @@ -2011,13 +2010,13 @@ lladdr_off); } - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(LLTABLE6(ifp)); LLE_WLOCK(ln); /* Prefer any existing lle over newly-created one */ ln_tmp = nd6_lookup(from, LLE_SF(AF_INET6, LLE_EXCLUSIVE), ifp); if (ln_tmp == NULL) lltable_link_entry(LLTABLE6(ifp), ln); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE6(ifp)); if (ln_tmp == NULL) { /* No existing lle, mark as new entry (6,7) */ is_newentry = 1; @@ -2337,7 +2336,7 @@ return (NULL); } - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(LLTABLE6(ifp)); LLE_WLOCK(lle); /* Prefer any existing entry over newly-created one */ lle_tmp = nd6_lookup(addr, LLE_SF(AF_INET6, LLE_EXCLUSIVE), ifp); @@ -2363,7 +2362,7 @@ LLE_WUNLOCK(lle); lle = child_lle; } - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE6(ifp)); return (lle); } @@ -2610,14 +2609,14 @@ if (ln == NULL) return (ENOBUFS); - IF_AFDATA_WLOCK(ifp); + LLTABLE_LOCK(LLTABLE6(ifp)); LLE_WLOCK(ln); /* Unlink any entry if exists */ ln_tmp = lla_lookup(LLTABLE6(ifp), LLE_SF(AF_INET6, LLE_EXCLUSIVE), dst); if (ln_tmp != NULL) lltable_unlink_entry(LLTABLE6(ifp), ln_tmp); lltable_link_entry(LLTABLE6(ifp), ln); - IF_AFDATA_WUNLOCK(ifp); + LLTABLE_UNLOCK(LLTABLE6(ifp)); if (ln_tmp != NULL) EVENTHANDLER_INVOKE(lle_event, ln_tmp, LLENTRY_EXPIRED); diff --git a/sys/netlink/route/neigh.c b/sys/netlink/route/neigh.c --- a/sys/netlink/route/neigh.c +++ b/sys/netlink/route/neigh.c @@ -37,7 +37,6 @@ #include #include -#include #include #include #include @@ -431,7 +430,7 @@ lle->la_expire = attrs.ndaf_next_ts - time_second + time_uptime; /* llentry created, try to insert or update */ - IF_AFDATA_WLOCK(attrs.nda_ifp); + LLTABLE_LOCK(llt); LLE_WLOCK(lle); struct llentry *lle_tmp = lla_lookup(llt, LLE_EXCLUSIVE, attrs.nda_dst); if (lle_tmp != NULL) { @@ -454,7 +453,7 @@ else error = ENOENT; } - IF_AFDATA_WUNLOCK(attrs.nda_ifp); + LLTABLE_UNLOCK(llt); if (error != 0) { /* throw away the newly allocated llentry */