diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 9ce8f440e38f..da15f4d7c1ae 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -40,6 +40,23 @@ void br_recalculate_neigh_suppress_enabled(struct net_bridge *br) } #if IS_ENABLED(CONFIG_INET) +static bool +br_is_neigh_suppress_enabled_vid(const struct net_bridge_port *p, u16 vid) +{ + const struct net_bridge_vlan *v = NULL; + + if (p && vid && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) { + struct net_bridge_vlan_group *vg; + + vg = nbp_vlan_group_rcu(p); + v = br_vlan_find(vg, vid); + if (!v) + return false; + } + + return br_is_neigh_suppress_enabled(p, v); +} + static void br_arp_send(struct net_bridge *br, struct net_bridge_port *p, struct net_device *dev, __be32 dest_ip, __be32 src_ip, const unsigned char *dest_hw, @@ -159,7 +176,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, return; if (br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) { - if (br_is_neigh_suppress_enabled(p, vid)) + if (br_is_neigh_suppress_enabled_vid(p, vid)) return; if (is_unicast_ether_addr(eth_hdr(skb)->h_dest) && parp->ar_op == htons(ARPOP_REQUEST)) @@ -211,7 +228,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, if ((p && test_bit(BR_PROXYARP_BIT, &p->flags)) || (dst && test_bit(BR_PROXYARP_WIFI_BIT, &dst->flags)) || - br_is_neigh_suppress_enabled(dst, vid)) { + br_is_neigh_suppress_enabled_vid(dst, vid)) { if (!vid) br_arp_send(br, p, skb->dev, sip, tip, sha, ha, sha, 0, 0); @@ -424,7 +441,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, BR_INPUT_SKB_CB(skb)->proxyarp_replied = 0; BR_INPUT_SKB_CB(skb)->grat_arp = 0; - if (br_is_neigh_suppress_enabled(p, vid)) + if (br_is_neigh_suppress_enabled_vid(p, vid)) return; if (is_unicast_ether_addr(eth_hdr(skb)->h_dest) && @@ -486,7 +503,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; - if (br_is_neigh_suppress_enabled(dst, vid)) { + if (br_is_neigh_suppress_enabled_vid(dst, vid)) { if (vid != 0) br_nd_send(br, p, skb, n, ha, skb->vlan_proto, @@ -509,35 +526,25 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, } #endif -bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, u16 vid) +bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, + const struct net_bridge_vlan *v) { if (!p) return false; - if (vid && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) { - struct net_bridge_vlan_group *vg = nbp_vlan_group_rcu(p); - struct net_bridge_vlan *v; - - v = br_vlan_find(vg, vid); - if (!v) - return false; + if (v && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) return !!(READ_ONCE(v->priv_flags) & BR_VLFLAG_NEIGH_SUPPRESS_ENABLED); - } + return test_bit(BR_NEIGH_SUPPRESS_BIT, &p->flags); } -bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, u16 vid) +bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, + const struct net_bridge_vlan *v) { - if (vid && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) { - struct net_bridge_vlan_group *vg = nbp_vlan_group_rcu(p); - struct net_bridge_vlan *v; - - v = br_vlan_find(vg, vid); - if (!v) - return false; + if (v && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) return !!(READ_ONCE(v->priv_flags) & BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED); - } + return test_bit(BR_NEIGH_FORWARD_GRAT_BIT, &p->flags); } diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index e01c44a90d84..ce9ea9ac3d0a 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -89,10 +89,10 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) dest = eth_hdr(skb)->h_dest; if (is_broadcast_ether_addr(dest)) { - br_flood(br, skb, BR_PKT_BROADCAST, false, true, vid); + br_flood(br, vlan, skb, BR_PKT_BROADCAST, false, true); } else if (is_multicast_ether_addr(dest)) { if (unlikely(netpoll_tx_running(dev))) { - br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid); + br_flood(br, vlan, skb, BR_PKT_MULTICAST, false, true); goto out; } if (br_multicast_rcv(&brmctx, &pmctx_null, vlan, skb, vid)) { @@ -105,11 +105,11 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) br_multicast_querier_exists(brmctx, eth_hdr(skb), mdst)) br_multicast_flood(mdst, skb, brmctx, false, true); else - br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid); + br_flood(br, vlan, skb, BR_PKT_MULTICAST, false, true); } else if ((dst = br_fdb_find_rcu(br, dest, vid)) != NULL) { br_forward(READ_ONCE(dst->dst), skb, false, true); } else { - br_flood(br, skb, BR_PKT_UNICAST, false, true, vid); + br_flood(br, vlan, skb, BR_PKT_UNICAST, false, true); } out: rcu_read_unlock(); diff --git a/net/bridge/br_forward.c b/net/bridge/br_forward.c index 46c762ca5177..b5eece1ff9e8 100644 --- a/net/bridge/br_forward.c +++ b/net/bridge/br_forward.c @@ -7,7 +7,6 @@ * Lennert Buytenhek */ -#include #include #include #include @@ -17,16 +16,31 @@ #include #include "br_private.h" -/* Don't forward packets to originating port or forwarding disabled */ -static inline int should_deliver(const struct net_bridge_port *p, - const struct sk_buff *skb) -{ - struct net_bridge_vlan_group *vg; +struct br_fwd_dst { + const struct net_bridge_port *port; + struct net_bridge_vlan *vlan; +}; + +static bool should_deliver_vlan(const struct br_fwd_dst *fwd, + const struct sk_buff *skb) +{ + if (fwd->vlan) + return br_vlan_state_allowed(br_vlan_get_state(fwd->vlan), + false); + + return br_allowed_egress(nbp_vlan_group_rcu(fwd->port), skb); +} + +/* Don't forward packets to originating port or forwarding disabled */ +static __always_inline bool should_deliver(const struct br_fwd_dst *fwd, + const struct sk_buff *skb) +{ + const struct net_bridge_port *p = fwd->port; - vg = nbp_vlan_group_rcu(p); return (test_bit(BR_HAIRPIN_MODE_BIT, &p->flags) || skb->dev != p->dev) && (br_mst_is_enabled(p) || p->state == BR_STATE_FORWARDING) && - br_allowed_egress(vg, skb) && nbp_switchdev_allowed_egress(p, skb) && + should_deliver_vlan(fwd, skb) && + nbp_switchdev_allowed_egress(p, skb) && !br_skb_isolated(p, skb); } @@ -70,9 +84,10 @@ int br_forward_finish(struct net *net, struct sock *sk, struct sk_buff *skb) } EXPORT_SYMBOL_GPL(br_forward_finish); -static void __br_forward(const struct net_bridge_port *to, +static void __br_forward(const struct br_fwd_dst *fwd, struct sk_buff *skb, bool local_orig) { + const struct net_bridge_port *to = fwd->port; struct net_bridge_vlan_group *vg; struct net_device *indev; struct net *net; @@ -83,8 +98,8 @@ static void __br_forward(const struct net_bridge_port *to, */ nbp_switchdev_frame_mark_tx_fwd_offload(to, skb); - vg = nbp_vlan_group_rcu(to); - skb = br_handle_vlan(to->br, to, vg, skb); + vg = fwd->vlan ? NULL : nbp_vlan_group_rcu(to); + skb = br_handle_vlan(to->br, to, vg, fwd->vlan, skb); if (!skb) return; @@ -117,7 +132,7 @@ static void __br_forward(const struct net_bridge_port *to, br_forward_finish); } -static int deliver_clone(const struct net_bridge_port *prev, +static int deliver_clone(const struct br_fwd_dst *fwd, struct sk_buff *skb, bool local_orig) { struct net_device *dev = BR_INPUT_SKB_CB(skb)->brdev; @@ -128,7 +143,7 @@ static int deliver_clone(const struct net_bridge_port *prev, return -ENOMEM; } - __br_forward(prev, skb, local_orig); + __br_forward(fwd, skb, local_orig); return 0; } @@ -144,6 +159,8 @@ static int deliver_clone(const struct net_bridge_port *prev, void br_forward(const struct net_bridge_port *to, struct sk_buff *skb, bool local_rcv, bool local_orig) { + struct br_fwd_dst fwd; + if (unlikely(!to)) goto out; @@ -159,11 +176,13 @@ void br_forward(const struct net_bridge_port *to, to = backup_port; } - if (should_deliver(to, skb)) { + fwd.port = to; + fwd.vlan = NULL; + if (should_deliver(&fwd, skb)) { if (local_rcv) - deliver_clone(to, skb, local_orig); + deliver_clone(&fwd, skb, local_orig); else - __br_forward(to, skb, local_orig); + __br_forward(&fwd, skb, local_orig); return; } @@ -173,95 +192,162 @@ void br_forward(const struct net_bridge_port *to, } EXPORT_SYMBOL_GPL(br_forward); -static struct net_bridge_port *maybe_deliver( - struct net_bridge_port *prev, struct net_bridge_port *p, - struct sk_buff *skb, bool local_orig) +static int maybe_deliver(struct br_fwd_dst *prev, const struct br_fwd_dst *fwd, + struct sk_buff *skb, bool local_orig) { + const struct net_bridge_port *p = fwd->port; u8 igmp_type = br_multicast_igmp_type(skb); int err; - if (!should_deliver(p, skb)) - return prev; + if (!should_deliver(fwd, skb)) + return 0; nbp_switchdev_frame_mark_tx_fwd_to_hwdom(p, skb); - if (!prev) + if (!prev->port) goto out; err = deliver_clone(prev, skb, local_orig); if (err) - return ERR_PTR(err); + return err; out: br_multicast_count(p->br, p, skb, igmp_type, BR_MCAST_DIR_TX); + *prev = *fwd; - return p; + return 0; } -/* called under rcu_read_lock */ -void br_flood(struct net_bridge *br, struct sk_buff *skb, - enum br_pkt_type pkt_type, bool local_rcv, bool local_orig, - u16 vid) +static void br_flood_finish(const struct br_fwd_dst *fwd, int err, + struct sk_buff *skb, + bool local_rcv, bool local_orig) { enum skb_drop_reason reason = SKB_DROP_REASON_NO_TX_TARGET; - struct net_bridge_port *prev = NULL; - struct net_bridge_port *p; - br_tc_skb_miss_set(skb, pkt_type != BR_PKT_BROADCAST); + if (err || !fwd->port) { + if (err) + reason = err == -ENOMEM ? SKB_DROP_REASON_NOMEM : + SKB_DROP_REASON_NOT_SPECIFIED; - list_for_each_entry_rcu(p, &br->port_list, list) { - /* Do not flood unicast traffic to ports that turn it off, nor - * other traffic if flood off, except for traffic we originate + if (!local_rcv) + kfree_skb_reason(skb, reason); + return; + } + + if (local_rcv) + deliver_clone(fwd, skb, local_orig); + else + __br_forward(fwd, skb, local_orig); +} + +static int br_flood_port(struct br_fwd_dst *prev, + const struct br_fwd_dst *fwd, struct sk_buff *skb, + enum br_pkt_type pkt_type, bool local_orig) +{ + const struct net_bridge_port *p = fwd->port; + + /* Do not flood unicast traffic to ports that turn it off, nor + * other traffic if flood off, except for traffic we originate + */ + switch (pkt_type) { + case BR_PKT_UNICAST: + if (!test_bit(BR_FLOOD_BIT, &p->flags)) + return 0; + break; + case BR_PKT_MULTICAST: + if (!test_bit(BR_MCAST_FLOOD_BIT, &p->flags) && + skb->dev != p->br->dev) + return 0; + break; + case BR_PKT_BROADCAST: + if (!test_bit(BR_BCAST_FLOOD_BIT, &p->flags) && + skb->dev != p->br->dev) + return 0; + break; + } + + /* Do not flood to ports that enable proxy ARP */ + if (test_bit(BR_PROXYARP_BIT, &p->flags)) + return 0; + if (BR_INPUT_SKB_CB(skb)->proxyarp_replied) { + if (test_bit(BR_PROXYARP_WIFI_BIT, &p->flags)) + return 0; + /* For gratuitous ARPs/NAs, check neigh_forward_grat. + * For regular ARPs/NDs, check only neigh_suppress. */ - switch (pkt_type) { - case BR_PKT_UNICAST: - if (!test_bit(BR_FLOOD_BIT, &p->flags)) - continue; - break; - case BR_PKT_MULTICAST: - if (!test_bit(BR_MCAST_FLOOD_BIT, &p->flags) && skb->dev != br->dev) - continue; - break; - case BR_PKT_BROADCAST: - if (!test_bit(BR_BCAST_FLOOD_BIT, &p->flags) && skb->dev != br->dev) - continue; - break; - } + if (br_is_neigh_suppress_enabled(p, fwd->vlan) && + (!BR_INPUT_SKB_CB(skb)->grat_arp || + !br_is_neigh_forward_grat_enabled(p, fwd->vlan))) + return 0; + } - /* Do not flood to ports that enable proxy ARP */ - if (test_bit(BR_PROXYARP_BIT, &p->flags)) - continue; - if (BR_INPUT_SKB_CB(skb)->proxyarp_replied) { - if (test_bit(BR_PROXYARP_WIFI_BIT, &p->flags)) - continue; - /* For gratuitous ARPs/NAs, check neigh_forward_grat. - * For regular ARPs/NDs, check only neigh_suppress. - */ - if (br_is_neigh_suppress_enabled(p, vid) && - (!BR_INPUT_SKB_CB(skb)->grat_arp || - !br_is_neigh_forward_grat_enabled(p, vid))) - continue; - } + return maybe_deliver(prev, fwd, skb, local_orig); +} - prev = maybe_deliver(prev, p, skb, local_orig); - if (IS_ERR(prev)) { - reason = PTR_ERR(prev) == -ENOMEM ? SKB_DROP_REASON_NOMEM : - SKB_DROP_REASON_NOT_SPECIFIED; - goto out; +static int br_flood_vlan(struct br_fwd_dst *prev, + struct net_bridge_vlan *v, struct sk_buff *skb, + enum br_pkt_type pkt_type, bool local_orig) +{ + struct net_bridge_vlan_port_array *array; + struct net_bridge_vlan *masterv, *pv; + struct br_fwd_dst dst; + int err; + + masterv = br_vlan_is_master(v) ? v : v->brvlan; + array = rcu_dereference(masterv->port_array); + if (array) { + unsigned int i; + + for (i = 0; i < array->count; i++) { + pv = array->vlans[i]; + dst.port = pv->port; + dst.vlan = pv; + err = br_flood_port(prev, &dst, skb, pkt_type, + local_orig); + if (err) + return err; + } + } else { + list_for_each_entry_rcu(pv, &masterv->port_vlist, port_vlist) { + dst.port = pv->port; + dst.vlan = pv; + err = br_flood_port(prev, &dst, skb, pkt_type, + local_orig); + if (err) + return err; } } - if (!prev) - goto out; + return 0; +} - if (local_rcv) - deliver_clone(prev, skb, local_orig); - else - __br_forward(prev, skb, local_orig); - return; +/* called under rcu_read_lock */ +void br_flood(struct net_bridge *br, struct net_bridge_vlan *v, + struct sk_buff *skb, enum br_pkt_type pkt_type, + bool local_rcv, bool local_orig) +{ + struct br_fwd_dst prev = {}; + int err = 0; -out: - if (!local_rcv) - kfree_skb_reason(skb, reason); + br_tc_skb_miss_set(skb, pkt_type != BR_PKT_BROADCAST); + + if (v) { + err = br_flood_vlan(&prev, v, skb, pkt_type, local_orig); + } else { + struct net_bridge_port *p; + + list_for_each_entry_rcu(p, &br->port_list, list) { + struct br_fwd_dst fwd = { + .port = p, + }; + + err = br_flood_port(&prev, &fwd, skb, pkt_type, + local_orig); + if (err) + break; + } + } + + br_flood_finish(&prev, err, skb, local_rcv, local_orig); } #ifdef CONFIG_BRIDGE_IGMP_SNOOPING @@ -270,9 +356,12 @@ static void maybe_deliver_addr(struct net_bridge_port *p, struct sk_buff *skb, { struct net_device *dev = BR_INPUT_SKB_CB(skb)->brdev; const unsigned char *src = eth_hdr(skb)->h_source; + struct br_fwd_dst fwd = { + .port = p, + }; struct sk_buff *nskb; - if (!should_deliver(p, skb)) + if (!should_deliver(&fwd, skb)) return; /* Even with hairpin, no soliloquies - prevent breaking IPv6 DAD */ @@ -292,7 +381,7 @@ static void maybe_deliver_addr(struct net_bridge_port *p, struct sk_buff *skb, if (!is_broadcast_ether_addr(addr)) memcpy(eth_hdr(skb)->h_dest, addr, ETH_ALEN); - __br_forward(p, skb, local_orig); + __br_forward(&fwd, skb, local_orig); } /* called with rcu_read_lock */ @@ -301,11 +390,11 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst, struct net_bridge_mcast *brmctx, bool local_rcv, bool local_orig) { - enum skb_drop_reason reason = SKB_DROP_REASON_NO_TX_TARGET; - struct net_bridge_port *prev = NULL; struct net_bridge_port_group *p; bool allow_mode_include = true; + struct br_fwd_dst prev = {}; struct hlist_node *rp; + int err = 0; rp = br_multicast_get_first_rport_node(brmctx, skb); @@ -321,6 +410,7 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst, while (p || rp) { struct net_bridge_port *port, *lport, *rport; + struct br_fwd_dst fwd = {}; lport = p ? p->key.port : NULL; rport = br_multicast_rport_from_node_skb(rp, skb); @@ -342,12 +432,11 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst, port = rport; } - prev = maybe_deliver(prev, port, skb, local_orig); - if (IS_ERR(prev)) { - reason = PTR_ERR(prev) == -ENOMEM ? SKB_DROP_REASON_NOMEM : - SKB_DROP_REASON_NOT_SPECIFIED; - goto out; - } + fwd.port = port; + err = maybe_deliver(&prev, &fwd, skb, local_orig); + if (err) + break; + delivered: if ((unsigned long)lport >= (unsigned long)port) p = rcu_dereference(p->next); @@ -355,17 +444,6 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst, rp = rcu_dereference(hlist_next_rcu(rp)); } - if (!prev) - goto out; - - if (local_rcv) - deliver_clone(prev, skb, local_orig); - else - __br_forward(prev, skb, local_orig); - return; - -out: - if (!local_rcv) - kfree_skb_reason(skb, reason); + br_flood_finish(&prev, err, skb, local_rcv, local_orig); } #endif diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index 8bed72baf161..4357d78524a6 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -58,7 +58,7 @@ static int br_pass_frame_up(struct sk_buff *skb, bool promisc) indev = skb->dev; skb->dev = brdev; - skb = br_handle_vlan(br, NULL, vg, skb); + skb = br_handle_vlan(br, NULL, vg, NULL, skb); if (!skb) return NET_RX_DROP; /* update the multicast stats if the packet is IGMP/MLD */ @@ -226,7 +226,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb br_forward(READ_ONCE(dst->dst), skb, local_rcv, false); } else { if (!mcast_hit) - br_flood(br, skb, pkt_type, local_rcv, false, vid); + br_flood(br, vlan, skb, pkt_type, local_rcv, false); else br_multicast_flood(mdst, skb, brmctx, local_rcv, false); } diff --git a/net/bridge/br_mst.c b/net/bridge/br_mst.c index 63fc1dd0e9bc..252d5c927f71 100644 --- a/net/bridge/br_mst.c +++ b/net/bridge/br_mst.c @@ -79,16 +79,11 @@ int br_mst_get_state(const struct net_device *dev, u16 msti, u8 *state) } EXPORT_SYMBOL_GPL(br_mst_get_state); -static void br_mst_vlan_set_state(struct net_bridge_vlan_group *vg, - struct net_bridge_vlan *v, - u8 state) +static void br_mst_vlan_set_state(struct net_bridge_vlan *v, u8 state) { if (br_vlan_get_state(v) == state) return; - if (v->vid == br_get_pvid(vg)) - br_vlan_set_pvid_state(vg, state); - br_vlan_set_state(v, state); } @@ -129,7 +124,7 @@ int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, if (READ_ONCE(v->brvlan->msti) != msti) continue; - br_mst_vlan_set_state(vg, v, state); + br_mst_vlan_set_state(v, state); } out_rcu_unlock: @@ -149,13 +144,13 @@ static void br_mst_vlan_sync_state(struct net_bridge_vlan *pv, u16 msti) * it. */ if (v != pv && v->brvlan->msti == msti) { - br_mst_vlan_set_state(vg, pv, br_vlan_get_state(v)); + br_mst_vlan_set_state(pv, br_vlan_get_state(v)); return; } } /* Otherwise, start out in a new MSTI with all ports disabled. */ - return br_mst_vlan_set_state(vg, pv, BR_STATE_DISABLED); + return br_mst_vlan_set_state(pv, BR_STATE_DISABLED); } int br_mst_vlan_set_msti(struct net_bridge_vlan *mv, u16 msti) diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index 09c397e30330..67117fb3dc88 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -190,6 +190,17 @@ enum { BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED = BIT(6), }; +/* start publishing arrays when there're > BR_VLAN_PORT_ARRAY_THRESHOLD + * port-VLANs + */ +#define BR_VLAN_PORT_ARRAY_THRESHOLD 8 + +struct net_bridge_vlan_port_array { + struct rcu_head rcu; + unsigned int count; + struct net_bridge_vlan *vlans[]; +}; + /** * struct net_bridge_vlan - per-vlan entry * @@ -210,6 +221,8 @@ enum { * @port_mcast_ctx: if MASTER flag unset, this is the per-port/vlan multicast * context * @msti: if MASTER flag set, this holds the VLANs MST instance + * @port_array: if MASTER flag set, this is the port-VLAN array + * @port_vlist: if MASTER flag set, this is the port-VLAN list * @vlist: sorted list of VLAN entries * @rcu: used for entry destruction * @@ -244,6 +257,8 @@ struct net_bridge_vlan { u16 msti; + struct net_bridge_vlan_port_array __rcu *port_array; + struct list_head port_vlist; struct list_head vlist; struct rcu_head rcu; @@ -256,8 +271,7 @@ struct net_bridge_vlan { * @tunnel_hash: Hash table to map from tunnel key ID (e.g. VXLAN VNI) to VLAN * @vlan_list: sorted VLAN entry list * @num_vlans: number of total VLAN entries - * @pvid: PVID VLAN id - * @pvid_state: PVID's STP state (e.g. forwarding, learning, blocking) + * @pvid: RCU-protected PVID VLAN entry * * IMPORTANT: Be careful when checking if there're VLAN entries using list * primitives because the bridge can have entries in its list which @@ -269,9 +283,8 @@ struct net_bridge_vlan_group { struct rhashtable vlan_hash; struct rhashtable tunnel_hash; struct list_head vlan_list; + struct net_bridge_vlan __rcu *pvid; u16 num_vlans; - u16 pvid; - u8 pvid_state; }; /* bridge fdb flags */ @@ -912,9 +925,9 @@ int br_dev_queue_push_xmit(struct net *net, struct sock *sk, struct sk_buff *skb void br_forward(const struct net_bridge_port *to, struct sk_buff *skb, bool local_rcv, bool local_orig); int br_forward_finish(struct net *net, struct sock *sk, struct sk_buff *skb); -void br_flood(struct net_bridge *br, struct sk_buff *skb, - enum br_pkt_type pkt_type, bool local_rcv, bool local_orig, - u16 vid); +void br_flood(struct net_bridge *br, struct net_bridge_vlan *v, + struct sk_buff *skb, enum br_pkt_type pkt_type, + bool local_rcv, bool local_orig); /* return true if both source port and dest port are isolated */ static inline bool br_skb_isolated(const struct net_bridge_port *to, @@ -1593,6 +1606,7 @@ bool br_should_learn(struct net_bridge_port *p, struct sk_buff *skb, u16 *vid); struct sk_buff *br_handle_vlan(struct net_bridge *br, const struct net_bridge_port *port, struct net_bridge_vlan_group *vg, + struct net_bridge_vlan *vlan, struct sk_buff *skb); int br_vlan_add(struct net_bridge *br, u16 vid, u16 flags, bool *changed, struct netlink_ext_ack *extack); @@ -1687,10 +1701,16 @@ static inline int br_vlan_get_tag(const struct sk_buff *skb, u16 *vid) static inline u16 br_get_pvid(const struct net_bridge_vlan_group *vg) { + struct net_bridge_vlan *pvid; + if (!vg) return 0; - return READ_ONCE(vg->pvid); + pvid = rcu_dereference_rtnl(vg->pvid); + if (!pvid) + return 0; + + return pvid->vid; } static inline u16 br_vlan_flags(const struct net_bridge_vlan *v, u16 pvid) @@ -1724,6 +1744,7 @@ static inline bool br_should_learn(struct net_bridge_port *p, static inline struct sk_buff *br_handle_vlan(struct net_bridge *br, const struct net_bridge_port *port, struct net_bridge_vlan_group *vg, + struct net_bridge_vlan *vlan, struct sk_buff *skb) { return skb; @@ -1922,17 +1943,6 @@ static inline void br_vlan_set_state(struct net_bridge_vlan *v, u8 state) br_multicast_update_vlan_mcast_ctx(v, state); } -static inline u8 br_vlan_get_pvid_state(const struct net_bridge_vlan_group *vg) -{ - return READ_ONCE(vg->pvid_state); -} - -static inline void br_vlan_set_pvid_state(struct net_bridge_vlan_group *vg, - u8 state) -{ - WRITE_ONCE(vg->pvid_state, state); -} - /* learn_allow is true at ingress and false at egress */ static inline bool br_vlan_state_allowed(u8 state, bool learn_allow) { @@ -1945,6 +1955,11 @@ static inline bool br_vlan_state_allowed(u8 state, bool learn_allow) return false; } } +#else +static inline bool br_vlan_state_allowed(u8 state, bool learn_allow) +{ + return false; +} #endif /* br_mst.c */ @@ -2375,6 +2390,8 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, u16 vid, struct net_bridge_port *p, struct nd_msg *msg); struct nd_msg *br_is_nd_neigh_msg(struct sk_buff *skb); -bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, u16 vid); -bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, u16 vid); +bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, + const struct net_bridge_vlan *v); +bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, + const struct net_bridge_vlan *v); #endif diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index 34ede0f3e452..ce5aa15c4540 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -35,21 +35,21 @@ static struct net_bridge_vlan *br_vlan_lookup(struct rhashtable *tbl, u16 vid) } static void __vlan_add_pvid(struct net_bridge_vlan_group *vg, - const struct net_bridge_vlan *v) + struct net_bridge_vlan *v) { - if (vg->pvid == v->vid) + if (rcu_access_pointer(vg->pvid) == v) return; - br_vlan_set_pvid_state(vg, br_vlan_get_state(v)); - WRITE_ONCE(vg->pvid, v->vid); + RCU_INIT_POINTER(vg->pvid, v); } -static void __vlan_delete_pvid(struct net_bridge_vlan_group *vg, u16 vid) +static void __vlan_delete_pvid(struct net_bridge_vlan_group *vg, + struct net_bridge_vlan *v) { - if (vg->pvid != vid) + if (rcu_access_pointer(vg->pvid) != v) return; - WRITE_ONCE(vg->pvid, 0); + RCU_INIT_POINTER(vg->pvid, NULL); } /* Update the BRIDGE_VLAN_INFO_PVID and BRIDGE_VLAN_INFO_UNTAGGED flags of @v. @@ -60,6 +60,7 @@ static bool __vlan_flags_update(struct net_bridge_vlan *v, u16 flags, bool commit) { struct net_bridge_vlan_group *vg; + struct net_bridge_vlan *pvid; u16 vlan_flags; bool change; @@ -70,7 +71,8 @@ static bool __vlan_flags_update(struct net_bridge_vlan *v, u16 flags, /* check if anything would be changed on commit */ vlan_flags = v->flags; - change = !!(flags & BRIDGE_VLAN_INFO_PVID) == !!(vg->pvid != v->vid) || + pvid = rtnl_dereference(vg->pvid); + change = !!(flags & BRIDGE_VLAN_INFO_PVID) == !!(pvid != v) || ((flags ^ vlan_flags) & BRIDGE_VLAN_INFO_UNTAGGED); if (!commit) @@ -79,7 +81,7 @@ static bool __vlan_flags_update(struct net_bridge_vlan *v, u16 flags, if (flags & BRIDGE_VLAN_INFO_PVID) __vlan_add_pvid(vg, v); else - __vlan_delete_pvid(vg, v->vid); + __vlan_delete_pvid(vg, v); if (flags & BRIDGE_VLAN_INFO_UNTAGGED) vlan_flags |= BRIDGE_VLAN_INFO_UNTAGGED; @@ -128,10 +130,12 @@ static void __vlan_add_list(struct net_bridge_vlan *v) struct list_head *headp, *hpos; struct net_bridge_vlan *vent; - if (br_vlan_is_master(v)) + if (br_vlan_is_master(v)) { vg = br_vlan_group(v->br); - else + } else { vg = nbp_vlan_group(v->port); + list_add_rcu(&v->port_vlist, &v->brvlan->port_vlist); + } headp = &vg->vlan_list; list_for_each_prev(hpos, headp) { @@ -145,6 +149,10 @@ static void __vlan_add_list(struct net_bridge_vlan *v) static void __vlan_del_list(struct net_bridge_vlan *v) { list_del_rcu(&v->vlist); + if (br_vlan_is_master(v)) + WARN_ON(!list_empty(&v->port_vlist)); + else + list_del_rcu(&v->port_vlist); } static int __vlan_vid_del(struct net_device *dev, struct net_bridge *br, @@ -250,6 +258,36 @@ static void br_vlan_init_state(struct net_bridge_vlan *v) v->msti = 0; } +static unsigned int br_vlan_num_ports(const struct net_bridge_vlan *masterv) +{ + return refcount_read(&masterv->refcnt) - br_vlan_is_brentry(masterv); +} + +static void br_vlan_rebuild_port_array(struct net_bridge_vlan *masterv, + unsigned int count) +{ + struct net_bridge_vlan_port_array *array = NULL, *old; + unsigned int i = 0; + + WARN_ON(!br_vlan_is_master(masterv)); + + if (count > BR_VLAN_PORT_ARRAY_THRESHOLD) + array = kvmalloc(struct_size(array, vlans, count), GFP_KERNEL); + + if (array) { + struct net_bridge_vlan *pv; + + array->count = count; + list_for_each_entry(pv, &masterv->port_vlist, port_vlist) + array->vlans[i++] = pv; + } + + old = rtnl_dereference(masterv->port_array); + rcu_assign_pointer(masterv->port_array, array); + if (old) + kvfree_rcu(old, rcu); +} + /* This is the shared VLAN add function which works for both ports and bridge * devices. There are four possible calls to this function in terms of the * vlan entry type: @@ -333,6 +371,7 @@ static int __vlan_add(struct net_bridge_vlan *v, u16 flags, } br_multicast_ctx_init(br, v, &v->br_mcast_ctx); v->priv_flags |= BR_VLFLAG_GLOBAL_MCAST_ENABLED; + INIT_LIST_HEAD(&v->port_vlist); } /* Add the dev mac and count the vlan only if it's usable */ @@ -359,8 +398,10 @@ static int __vlan_add(struct net_bridge_vlan *v, u16 flags, __vlan_flags_commit(v, flags); br_multicast_toggle_one_vlan(v, true); - if (p) + if (p) { + br_vlan_rebuild_port_array(masterv, br_vlan_num_ports(masterv)); nbp_vlan_set_vlan_dev_state(p, v->vid); + } out: return err; @@ -403,7 +444,7 @@ static void __vlan_del(struct net_bridge_vlan *v) masterv = v->brvlan; } - __vlan_delete_pvid(vg, v->vid); + __vlan_delete_pvid(vg, v); if (p) { err = __vlan_vid_del(p->dev, p->br, v); if (err) @@ -429,6 +470,9 @@ static void __vlan_del(struct net_bridge_vlan *v) rhashtable_remove_fast(&vg->vlan_hash, &v->vnode, br_vlan_rht_params); __vlan_del_list(v); + /* -1 because br_vlan_put_master() is called later */ + br_vlan_rebuild_port_array(masterv, + br_vlan_num_ports(masterv) - 1); nbp_vlan_set_vlan_dev_state(p, v->vid); br_multicast_toggle_one_vlan(v, false); br_multicast_port_ctx_deinit(&v->port_mcast_ctx); @@ -453,7 +497,7 @@ static void __vlan_flush(const struct net_bridge *br, struct net_bridge_vlan *vlan, *tmp; u16 v_start = 0, v_end = 0; - __vlan_delete_pvid(vg, vg->pvid); + __vlan_delete_pvid(vg, rtnl_dereference(vg->pvid)); list_for_each_entry_safe(vlan, tmp, &vg->vlan_list, vlist) { /* take care of disjoint ranges */ if (!v_start) { @@ -476,10 +520,10 @@ static void __vlan_flush(const struct net_bridge *br, struct sk_buff *br_handle_vlan(struct net_bridge *br, const struct net_bridge_port *p, struct net_bridge_vlan_group *vg, + struct net_bridge_vlan *v, struct sk_buff *skb) { struct pcpu_sw_netstats *stats; - struct net_bridge_vlan *v; u16 vid; /* If this packet was not filtered at input, let it pass */ @@ -490,19 +534,22 @@ struct sk_buff *br_handle_vlan(struct net_bridge *br, * a valid vlan id. If the vlan id has untagged flag set, * send untagged; otherwise, send tagged. */ - br_vlan_get_tag(skb, &vid); - v = br_vlan_find(vg, vid); - /* Vlan entry must be configured at this point. The - * only exception is the bridge is set in promisc mode and the - * packet is destined for the bridge device. In this case - * pass the packet as is. - */ - if (!v || !br_vlan_should_use(v)) { - if ((br->dev->flags & IFF_PROMISC) && skb->dev == br->dev) { - goto out; - } else { - kfree_skb(skb); - return NULL; + if (!v) { + br_vlan_get_tag(skb, &vid); + v = br_vlan_find(vg, vid); + /* Vlan entry must be configured at this point. The + * only exception is the bridge is set in promisc mode and the + * packet is destined for the bridge device. In this case + * pass the packet as is. + */ + if (!v || !br_vlan_should_use(v)) { + if ((br->dev->flags & IFF_PROMISC) && + skb->dev == br->dev) { + goto out; + } else { + kfree_skb(skb); + return NULL; + } } } if (br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) { @@ -579,42 +626,32 @@ static bool __allowed_ingress(const struct net_bridge *br, } if (!*vid) { - u16 pvid = br_get_pvid(vg); - + v = vg ? rcu_dereference(vg->pvid) : NULL; /* Frame had a tag with VID 0 or did not have a tag. * See if pvid is set on this port. That tells us which * vlan untagged or priority-tagged traffic belongs to. */ - if (!pvid) + if (!v) goto drop; /* PVID is set on this port. Any untagged or priority-tagged * ingress frame is considered to belong to this vlan. */ - *vid = pvid; + *vid = v->vid; if (likely(!tagged)) /* Untagged Frame. */ - __vlan_hwaccel_put_tag(skb, br->vlan_proto, pvid); + __vlan_hwaccel_put_tag(skb, br->vlan_proto, v->vid); else /* Priority-tagged Frame. * At this point, we know that skb->vlan_tci VID * field was 0. * We update only VID field and preserve PCP field. */ - skb->vlan_tci |= pvid; - - /* if snooping and stats are disabled we can avoid the lookup */ - if (!br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) && - !br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) { - if (*state == BR_STATE_FORWARDING) { - *state = br_vlan_get_pvid_state(vg); - if (!br_vlan_state_allowed(*state, true)) - goto drop; - } - return true; - } + skb->vlan_tci |= v->vid; + } else { + v = br_vlan_find(vg, *vid); } - v = br_vlan_find(vg, *vid); + if (!v || !br_vlan_should_use(v)) goto drop; @@ -697,11 +734,10 @@ bool br_should_learn(struct net_bridge_port *p, struct sk_buff *skb, u16 *vid) *vid = 0; if (!*vid) { - *vid = br_get_pvid(vg); - if (!*vid || - !br_vlan_state_allowed(br_vlan_get_pvid_state(vg), true)) + v = rcu_dereference(vg->pvid); + if (!v || !br_vlan_state_allowed(br_vlan_get_state(v), true)) return false; - + *vid = v->vid; return true; } @@ -1061,17 +1097,10 @@ int br_vlan_set_stats_per_port(struct net_bridge *br, unsigned long val) static bool vlan_default_pvid(struct net_bridge_vlan_group *vg, u16 vid) { - struct net_bridge_vlan *v; + struct net_bridge_vlan *pvid = rtnl_dereference(vg->pvid); - if (vid != vg->pvid) - return false; - - v = br_vlan_lookup(&vg->vlan_hash, vid); - if (v && br_vlan_should_use(v) && - (v->flags & BRIDGE_VLAN_INFO_UNTAGGED)) - return true; - - return false; + return pvid && pvid->vid == vid && br_vlan_should_use(pvid) && + (pvid->flags & BRIDGE_VLAN_INFO_UNTAGGED); } static void br_vlan_disable_default_pvid(struct net_bridge *br) @@ -1524,7 +1553,7 @@ int br_vlan_get_info(const struct net_device *dev, u16 vid, p_vinfo->vid = vid; p_vinfo->flags = v->flags; - if (vid == br_get_pvid(vg)) + if (v == rcu_access_pointer(vg->pvid)) p_vinfo->flags |= BRIDGE_VLAN_INFO_PVID; return 0; } @@ -1551,7 +1580,7 @@ int br_vlan_get_info_rcu(const struct net_device *dev, u16 vid, p_vinfo->vid = vid; p_vinfo->flags = READ_ONCE(v->flags); - if (vid == br_get_pvid(vg)) + if (v == rcu_access_pointer(vg->pvid)) p_vinfo->flags |= BRIDGE_VLAN_INFO_PVID; return 0; } @@ -1956,7 +1985,7 @@ void br_vlan_notify(const struct net_bridge *br, goto out_kfree; flags = v->flags; - if (br_get_pvid(vg) == v->vid) + if (v == rcu_access_pointer(vg->pvid)) flags |= BRIDGE_VLAN_INFO_PVID; break; case RTM_DELVLAN: diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c index 506668fc31d9..e4b7a147e42b 100644 --- a/net/bridge/br_vlan_options.c +++ b/net/bridge/br_vlan_options.c @@ -110,8 +110,7 @@ size_t br_vlan_opts_nl_size(void) + 0; } -static int br_vlan_modify_state(struct net_bridge_vlan_group *vg, - struct net_bridge_vlan *v, +static int br_vlan_modify_state(struct net_bridge_vlan *v, u8 state, bool *changed, struct netlink_ext_ack *extack) @@ -143,9 +142,6 @@ static int br_vlan_modify_state(struct net_bridge_vlan_group *vg, if (v->state == state) return 0; - if (v->vid == br_get_pvid(vg)) - br_vlan_set_pvid_state(vg, state); - br_vlan_set_state(v, state); *changed = true; @@ -216,7 +212,6 @@ static int br_vlan_modify_tunnel(const struct net_bridge_port *p, static int br_vlan_process_one_opts(const struct net_bridge *br, const struct net_bridge_port *p, - struct net_bridge_vlan_group *vg, struct net_bridge_vlan *v, struct nlattr **tb, bool *changed, @@ -228,7 +223,7 @@ static int br_vlan_process_one_opts(const struct net_bridge *br, if (tb[BRIDGE_VLANDB_ENTRY_STATE]) { u8 state = nla_get_u8(tb[BRIDGE_VLANDB_ENTRY_STATE]); - err = br_vlan_modify_state(vg, v, state, changed, extack); + err = br_vlan_modify_state(v, state, changed, extack); if (err) return err; } @@ -339,8 +334,7 @@ int br_vlan_process_options(const struct net_bridge *br, break; } - err = br_vlan_process_one_opts(br, p, vg, v, tb, &changed, - extack); + err = br_vlan_process_one_opts(br, p, v, tb, &changed, extack); if (err) break;