diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 832439b8a898..2f19fb07bb2d 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -278,9 +278,10 @@ rx_handler_result_t tap_handle_frame(struct sk_buff **pskb) if (q->flags & IFF_VNET_HDR) features |= tap->tap_features; if (netif_needs_gso(skb, features)) { - struct sk_buff *segs = __skb_gso_segment(skb, features, false); + struct sk_buff *segs; struct sk_buff *next; + segs = __skb_gso_segment(skb, features, false, 0); if (IS_ERR(segs)) { drop_reason = SKB_DROP_REASON_SKB_GSO_SEG; goto drop; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5b4b92e76b67..8fbe3a5d8169 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -5500,7 +5500,14 @@ void netif_stacked_transfer_operstate(const struct net_device *rootdev, netdev_features_t passthru_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features); -netdev_features_t netif_skb_features(struct sk_buff *skb); +netdev_features_t __netif_skb_features(struct sk_buff *skb, + bool check_gso_limits); + +static inline netdev_features_t netif_skb_features(struct sk_buff *skb) +{ + return __netif_skb_features(skb, true); +} + void skb_warn_bad_offload(const struct sk_buff *skb); static inline bool net_gso_ok(netdev_features_t features, int gso_type) @@ -5566,15 +5573,6 @@ netif_get_gro_max_size(const struct net_device *dev, const struct sk_buff *skb) READ_ONCE(dev->gro_ipv4_max_size); } -static inline unsigned int -netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb) -{ - /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ - return skb->protocol == htons(ETH_P_IPV6) ? - READ_ONCE(dev->gso_max_size) : - READ_ONCE(dev->gso_ipv4_max_size); -} - static inline bool netif_is_macsec(const struct net_device *dev) { return dev->priv_flags & IFF_MACSEC; diff --git a/include/net/gso.h b/include/net/gso.h index 0749230d414e..fe3260b381d4 100644 --- a/include/net/gso.h +++ b/include/net/gso.h @@ -21,6 +21,8 @@ struct skb_gso_cb { __u16 csum_start; /* Number of IPv4/IPv6 GSO handler entries for this packet. */ u8 recursion_counter; + /* Max MSS segs per output skb, 0 = no limit */ + u8 max_segs; }; #define SKB_GSO_CB_OFFSET 32 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET)) @@ -83,12 +85,13 @@ static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res) } struct sk_buff *__skb_gso_segment(struct sk_buff *skb, - netdev_features_t features, bool tx_path); + netdev_features_t features, bool tx_path, + unsigned int max_segs); static inline struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features) { - return __skb_gso_segment(skb, features, true); + return __skb_gso_segment(skb, features, true, 0); } struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb, diff --git a/include/net/udp.h b/include/net/udp.h index 1fee17274745..5bc25dcf25fb 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -613,7 +613,7 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk, /* the GSO CB lays after the UDP one, no need to save and restore any * CB fragment */ - segs = __skb_gso_segment(skb, features, false); + segs = __skb_gso_segment(skb, features, false, 0); if (IS_ERR_OR_NULL(segs)) { drop_count = skb_shinfo(skb)->gso_segs; goto drop; diff --git a/net/core/dev.c b/net/core/dev.c index a6951ea08961..8eb9bf1ff9eb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3888,16 +3888,33 @@ static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb) return false; } -static netdev_features_t gso_features_check(const struct sk_buff *skb, - struct net_device *dev, - netdev_features_t features) +static unsigned int +netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb) { - u16 gso_segs = skb_shinfo(skb)->gso_segs; + /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ + return vlan_get_protocol(skb) == htons(ETH_P_IPV6) ? + READ_ONCE(dev->gso_max_size) : + READ_ONCE(dev->gso_ipv4_max_size); +} - if (gso_segs > READ_ONCE(dev->gso_max_segs)) - return features & ~NETIF_F_GSO_MASK; +static bool +gso_within_dev_limits(const struct sk_buff *skb, const struct net_device *dev) +{ + if (skb_shinfo(skb)->gso_segs > READ_ONCE(dev->gso_max_segs)) + return false; if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) + return false; + + return true; +} + +static netdev_features_t gso_features_check(const struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features, + bool check_limits) +{ + if (check_limits && !gso_within_dev_limits(skb, dev)) return features & ~NETIF_F_GSO_MASK; if (!skb_shinfo(skb)->gso_type) { @@ -3946,13 +3963,15 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, return features; } -netdev_features_t netif_skb_features(struct sk_buff *skb) +netdev_features_t +__netif_skb_features(struct sk_buff *skb, bool check_gso_limits) { struct net_device *dev = skb->dev; netdev_features_t features = dev->features; if (skb_is_gso(skb)) - features = gso_features_check(skb, dev, features); + features = gso_features_check(skb, dev, features, + check_gso_limits); /* If encapsulation offload request, verify we are testing * hardware encapsulation features instead of standard @@ -3975,7 +3994,54 @@ netdev_features_t netif_skb_features(struct sk_buff *skb) return harmonize_features(skb, features); } -EXPORT_SYMBOL(netif_skb_features); +EXPORT_SYMBOL(__netif_skb_features); + +static unsigned int +skb_gso_output_max_segs(struct sk_buff *skb, struct net_device *dev) +{ + unsigned int mss = skb_shinfo(skb)->gso_size; + unsigned int gso_max_size, hdr_len, max_segs; + netdev_features_t features; + struct tcphdr _tcph, *th; + + if (!skb_is_gso(skb) || !skb_is_gso_tcp(skb) || + skb->encapsulation || skb_has_frag_list(skb)) + return 0; + + /* + * The transport offset can be unset or stale, so hdr_len is only taken + * from a header at the checksum start. + */ + if (!skb_transport_header_was_set(skb) || + skb_checksum_start(skb) != skb_transport_header(skb)) + return 0; + + th = skb_header_pointer(skb, skb_transport_offset(skb), sizeof(_tcph), + &_tcph); + if (!th || __tcp_hdrlen(th) < sizeof(*th)) + return 0; + + hdr_len = skb_transport_header(skb) - skb_mac_header(skb) + + __tcp_hdrlen(th); + + /* The output is still a GSO skb: recheck without the limit checks */ + features = __netif_skb_features(skb, false) | NETIF_F_GSO_ROBUST; + if (!net_gso_ok(features, skb_shinfo(skb)->gso_type)) + return 0; + + gso_max_size = netif_get_gso_max_size(dev, skb); + gso_max_size = min(gso_max_size, GSO_LEGACY_MAX_SIZE); + + /* + * gso_within_dev_limits() accepts gso_segs == gso_max_segs but + * rejects skb->len >= gso_max_size, so only the size limit needs the + * - 1; the inner min() keeps that subtraction from wrapping. + */ + max_segs = (gso_max_size - min(gso_max_size, hdr_len + 1)) / mss; + max_segs = min(max_segs, READ_ONCE(dev->gso_max_segs)); + + return max_segs; +} static int xmit_one(struct sk_buff *skb, struct net_device *dev, struct netdev_queue *txq, bool more) @@ -4140,9 +4206,13 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device goto out_null; if (netif_needs_gso(skb, features)) { + unsigned int max_segs = 0; struct sk_buff *segs; - segs = skb_gso_segment(skb, features); + if (unlikely(!gso_within_dev_limits(skb, dev))) + max_segs = skb_gso_output_max_segs(skb, dev); + + segs = __skb_gso_segment(skb, features, true, max_segs); if (IS_ERR(segs)) { goto out_kfree_skb; } else if (segs) { diff --git a/net/core/gso.c b/net/core/gso.c index e96ef6350064..b5bd269e56e4 100644 --- a/net/core/gso.c +++ b/net/core/gso.c @@ -77,6 +77,8 @@ static bool skb_needs_check(const struct sk_buff *skb, bool tx_path) * @skb: buffer to segment * @features: features for the output path (see dev->features) * @tx_path: whether it is called in TX path + * @max_segs: maximum MSS segments per output GSO skb, 0 means no limit; + * set only for an unencapsulated TCP skb without a frag_list * * This function segments the given skb and returns a list of segments. * @@ -86,7 +88,8 @@ static bool skb_needs_check(const struct sk_buff *skb, bool tx_path) * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb. */ struct sk_buff *__skb_gso_segment(struct sk_buff *skb, - netdev_features_t features, bool tx_path) + netdev_features_t features, bool tx_path, + unsigned int max_segs) { struct sk_buff *segs; @@ -118,6 +121,7 @@ struct sk_buff *__skb_gso_segment(struct sk_buff *skb, SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); SKB_GSO_CB(skb)->encap_level = 0; SKB_GSO_CB(skb)->recursion_counter = 0; + SKB_GSO_CB(skb)->max_segs = min(max_segs, U8_MAX); skb_reset_mac_header(skb); skb_reset_mac_len(skb); diff --git a/net/core/net_test.c b/net/core/net_test.c index 9c3a590865d2..bc2ef7612e7d 100644 --- a/net/core/net_test.c +++ b/net/core/net_test.c @@ -5,6 +5,7 @@ /* GSO */ #include +#include static const char hdr[] = "abcdefgh"; #define GSO_TEST_SIZE 1000 @@ -34,6 +35,10 @@ enum gso_test_nr { GSO_TEST_FRAG_LIST_PURE, GSO_TEST_FRAG_LIST_NON_UNIFORM, GSO_TEST_GSO_BY_FRAGS, + GSO_TEST_RESEGMENT, + GSO_TEST_RESEGMENT_MULTI, + GSO_TEST_RESEGMENT_TAIL, + GSO_TEST_RESEGMENT_ONE_MSS, }; struct gso_test_case { @@ -46,10 +51,12 @@ struct gso_test_case { const unsigned int *frags; unsigned int nr_frag_skbs; const unsigned int *frag_skbs; + unsigned int max_segs; /* output as expected */ unsigned int nr_segs; const unsigned int *segs; + const unsigned int *gso_segs; }; static struct gso_test_case cases[] = { @@ -135,6 +142,68 @@ static struct gso_test_case cases[] = { .nr_segs = 4, .segs = (const unsigned int[]) { 100, 200, 300, 400 }, }, + { + .id = GSO_TEST_RESEGMENT, + .name = "resegment", + .linear_len = GSO_TEST_SIZE, + .nr_frags = 3, + .frags = (const unsigned int[]) { + GSO_TEST_SIZE, GSO_TEST_SIZE, 3, + }, + .max_segs = 2, + .nr_segs = 2, + .segs = (const unsigned int[]) { + 2 * GSO_TEST_SIZE, GSO_TEST_SIZE + 3, + }, + .gso_segs = (const unsigned int[]) { 2, 2 }, + }, + { + .id = GSO_TEST_RESEGMENT_MULTI, + .name = "resegment_multi", + .linear_len = 2 * GSO_TEST_SIZE, + .nr_frags = 4, + .frags = (const unsigned int[]) { + GSO_TEST_SIZE, GSO_TEST_SIZE, GSO_TEST_SIZE, 3, + }, + .max_segs = 2, + .nr_segs = 3, + .segs = (const unsigned int[]) { + 2 * GSO_TEST_SIZE, 2 * GSO_TEST_SIZE, GSO_TEST_SIZE + 3, + }, + .gso_segs = (const unsigned int[]) { 2, 2, 2 }, + }, + { + /* The last output is not a GSO skb. */ + .id = GSO_TEST_RESEGMENT_TAIL, + .name = "resegment_tail", + .linear_len = GSO_TEST_SIZE, + .nr_frags = 2, + .frags = (const unsigned int[]) { + GSO_TEST_SIZE, 400, + }, + .max_segs = 2, + .nr_segs = 2, + .segs = (const unsigned int[]) { + 2 * GSO_TEST_SIZE, 400, + }, + .gso_segs = (const unsigned int[]) { 2, 0 }, + }, + { + /* max_segs of 1 is the same as no limit. */ + .id = GSO_TEST_RESEGMENT_ONE_MSS, + .name = "resegment_one_mss", + .linear_len = GSO_TEST_SIZE, + .nr_frags = 3, + .frags = (const unsigned int[]) { + GSO_TEST_SIZE, GSO_TEST_SIZE, 3, + }, + .max_segs = 1, + .nr_segs = 4, + .segs = (const unsigned int[]) { + GSO_TEST_SIZE, GSO_TEST_SIZE, GSO_TEST_SIZE, 3, + }, + .gso_segs = (const unsigned int[]) { 0, 0, 0, 0 }, + }, }; static void gso_test_case_to_desc(struct gso_test_case *t, char *desc) @@ -226,6 +295,7 @@ static void gso_test_func(struct kunit *test) if (tcase->id == GSO_TEST_FRAG_LIST_NON_UNIFORM) features &= ~NETIF_F_SG; + SKB_GSO_CB(skb)->max_segs = tcase->max_segs; segs = skb_segment(skb, features); if (IS_ERR(segs)) { KUNIT_FAIL(test, "segs error %pe", segs); @@ -239,6 +309,9 @@ static void gso_test_func(struct kunit *test) for (cur = segs, i = 0; cur; cur = next, i++) { next = cur->next; + if (i >= tcase->nr_segs) + goto consume; + KUNIT_ASSERT_EQ(test, cur->len, sizeof(hdr) + tcase->segs[i]); /* segs have skb->data pointing to the mac header */ @@ -247,11 +320,25 @@ static void gso_test_func(struct kunit *test) /* header was copied to all segs */ KUNIT_ASSERT_EQ(test, memcmp(skb_mac_header(cur), hdr, sizeof(hdr)), 0); + if (tcase->gso_segs) { + unsigned int gso_segs = tcase->gso_segs[i]; + + KUNIT_EXPECT_EQ(test, skb_shinfo(cur)->gso_segs, + gso_segs); + if (!gso_segs) { + KUNIT_EXPECT_FALSE(test, skb_is_gso(cur)); + } else { + KUNIT_EXPECT_TRUE(test, skb_is_gso(cur)); + KUNIT_EXPECT_EQ(test, skb_shinfo(cur)->gso_size, + GSO_TEST_SIZE); + } + } /* last seg can be found through segs->prev pointer */ if (!next) KUNIT_ASSERT_PTR_EQ(test, cur, last); +consume: consume_skb(cur); } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index d56f4f0102f7..b25d8e2c9aaf 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -4797,6 +4797,7 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, struct sk_buff *segs = NULL; struct sk_buff *tail = NULL; struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; + unsigned int max_segs = SKB_GSO_CB(head_skb)->max_segs; unsigned int mss = skb_shinfo(head_skb)->gso_size; bool gso_by_frags = mss == GSO_BY_FRAGS; unsigned int doffset = head_skb->data - skb_mac_header(head_skb); @@ -4843,7 +4844,7 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, csum = !!can_checksum_protocol(features, proto); if (sg && csum && !gso_by_frags) { - if (!(features & NETIF_F_GSO_PARTIAL)) { + if (!max_segs && !(features & NETIF_F_GSO_PARTIAL)) { struct sk_buff *iter; unsigned int frag_len; @@ -4877,8 +4878,11 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, * doesn't fit into an MSS sized block, so take care of that * now. */ - DEBUG_NET_WARN_ON_ONCE(len / mss > GSO_MAX_SEGS); - partial_segs = min(len / mss, GSO_MAX_SEGS); + DEBUG_NET_WARN_ON_ONCE(!max_segs && len / mss > GSO_MAX_SEGS); + if (max_segs) + partial_segs = min(len / mss, max_segs); + else + partial_segs = min(len / mss, GSO_MAX_SEGS); if (partial_segs > 1) mss *= partial_segs; else @@ -5125,7 +5129,7 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, } if (tail->len - doffset <= gso_size) - skb_shinfo(tail)->gso_size = 0; + skb_gso_reset(tail); else if (tail != segs) skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); } diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 42835694f35c..b2a0b09fd4a5 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -375,7 +375,7 @@ static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb, int err; BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET); - segs = __skb_gso_segment(skb, NETIF_F_SG, false); + segs = __skb_gso_segment(skb, NETIF_F_SG, false, 0); if (IS_ERR(segs)) return PTR_ERR(segs); if (segs == NULL)