diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 519a0156a05c..a6d69aaa6cd2 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -162,6 +162,8 @@ struct xfrm_dev_offload { */ struct net_device *real_dev; unsigned long offload_handle; + /* Snapshot the attached device index for dump paths. */ + int ifindex; u8 dir : 2; u8 type : 2; u8 flags : 2; diff --git a/net/core/dev.c b/net/core/dev.c index 4b3d5cfdf6e0..5933c5dab09e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4018,6 +4018,9 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, return NULL; } +/* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS) + * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()). + */ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) { netdev_features_t features; @@ -4089,7 +4092,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d skb->prev = skb; skb = validate_xmit_skb(skb, dev, again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) continue; if (!head) @@ -4860,8 +4863,11 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) goto recursion_alert; skb = validate_xmit_skb(skb, dev, &again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) { + if (PTR_ERR(skb) == -EINPROGRESS) + rc = NET_XMIT_SUCCESS; goto out; + } HARD_TX_LOCK(dev, txq, cpu); diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 125ea9a5b8a0..3b749475f6ed 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -88,6 +88,7 @@ static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, xdst->u.rt6.rt6i_idev = in6_dev_get(dev); if (!xdst->u.rt6.rt6i_idev) { netdev_put(dev, &xdst->u.dst.dev_tracker); + xdst->u.dst.dev = NULL; return -ENODEV; } diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 630f3dd31cc5..f153bf695b9d 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -182,7 +182,7 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur err = x->type_offload->xmit(x, skb, esp_features); if (err) { if (err == -EINPROGRESS) - return NULL; + return ERR_PTR(-EINPROGRESS); XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); kfree_skb(skb); @@ -224,7 +224,15 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur pskb = skb2; } - return skb; + /* skb_gso_segment() set skb->prev to the last segment, but async + * crypto may have stolen it above without updating ->prev. Repoint + * it at the last retained segment so validate_xmit_skb_list() does + * not chain onto a segment now owned by the crypto engine. + */ + if (skb) + skb->prev = pskb; + + return skb ? skb : ERR_PTR(-EINPROGRESS); } EXPORT_SYMBOL_GPL(validate_xmit_xfrm); @@ -313,6 +321,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, } xso->dev = dev; + xso->ifindex = dev->ifindex; netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC); if (xuo->flags & XFRM_OFFLOAD_INBOUND) diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index ad810d1f97c0..597aedeac26e 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -480,6 +480,7 @@ static int iptfs_skb_add_frags(struct sk_buff *skb, } __skb_frag_ref(tofrag); shinfo->nr_frags++; + shinfo->flags |= SKBFL_SHARED_FRAG; /* see if we are done */ fraglen = tofrag->len; diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index 458931062a04..eb1b6f67739e 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -55,8 +55,10 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb, ka->encap_sport, sock_net_uid(net, NULL)); rt = ip_route_output_key(net, &fl4); - if (IS_ERR(rt)) + if (IS_ERR(rt)) { + kfree_skb(skb); return PTR_ERR(rt); + } skb_dst_set(skb, &rt->dst); @@ -101,6 +103,7 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb, dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); if (IS_ERR(dst)) { local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock); + kfree_skb(skb); return PTR_ERR(dst); } @@ -118,7 +121,6 @@ static void nat_keepalive_send(struct nat_keepalive *ka) sizeof(struct ipv6hdr)) + sizeof(struct udphdr); const u8 nat_ka_payload = 0xFF; - int err = -EAFNOSUPPORT; struct sk_buff *skb; struct udphdr *uh; @@ -140,16 +142,17 @@ static void nat_keepalive_send(struct nat_keepalive *ka) switch (ka->family) { case AF_INET: - err = nat_keepalive_send_ipv4(skb, ka); + nat_keepalive_send_ipv4(skb, ka); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - err = nat_keepalive_send_ipv6(skb, ka, uh); + nat_keepalive_send_ipv6(skb, ka, uh); break; #endif - } - if (err) + default: kfree_skb(skb); + break; + } } struct nat_keepalive_work_ctx { diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 7ef861a0e823..932a313b9460 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -1329,8 +1329,8 @@ static void xfrm_hash_rebuild(struct work_struct *work) } } - if (policy->selector.prefixlen_d < dbits || - policy->selector.prefixlen_s < sbits) + if (policy->selector.prefixlen_d >= dbits && + policy->selector.prefixlen_s >= sbits) continue; bin = xfrm_policy_inexact_alloc_bin(policy, dir); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index c58cd024e3c6..36a4f6793ede 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -1547,6 +1547,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, xso->type = XFRM_DEV_OFFLOAD_PACKET; xso->dir = xdo->dir; xso->dev = dev; + xso->ifindex = dev->ifindex; xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC); error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, @@ -2071,8 +2072,11 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, x->mode_cbs = orig->mode_cbs; if (x->mode_cbs && x->mode_cbs->clone_state) { - if (x->mode_cbs->clone_state(x, orig)) + if (x->mode_cbs->clone_state(x, orig)) { + if (!x->mode_data) + x->mode_cbs = NULL; goto error; + } } x->props.reqid = m->new_reqid; @@ -3010,7 +3014,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (sockptr_is_null(optval) && !optlen) { xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); - __sk_dst_reset(sk); + sk_dst_reset(sk); return 0; } @@ -3050,7 +3054,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (err >= 0) { xfrm_sk_policy_insert(sk, err, pol); xfrm_pol_put(pol); - __sk_dst_reset(sk); + sk_dst_reset(sk); err = 0; } @@ -3291,6 +3295,8 @@ int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) if (x->mode_cbs->init_state) err = x->mode_cbs->init_state(x); module_put(x->mode_cbs->owner); + if (err && !x->mode_data) + x->mode_cbs = NULL; } error: return err; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 6384795ee6b2..d6db63304ba6 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -1201,17 +1201,26 @@ static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb) return 0; } -static void xso_to_xuo(const struct xfrm_dev_offload *xso, - struct xfrm_user_offload *xuo) +static void xso_to_xuo_ifindex(const struct xfrm_dev_offload *xso, int ifindex, + struct xfrm_user_offload *xuo) { - xuo->ifindex = xso->dev->ifindex; + xuo->ifindex = ifindex; if (xso->dir == XFRM_DEV_OFFLOAD_IN) xuo->flags = XFRM_OFFLOAD_INBOUND; if (xso->type == XFRM_DEV_OFFLOAD_PACKET) xuo->flags |= XFRM_OFFLOAD_PACKET; } -static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) +#ifdef CONFIG_XFRM_MIGRATE +static void xso_to_xuo(const struct xfrm_dev_offload *xso, + struct xfrm_user_offload *xuo) +{ + xso_to_xuo_ifindex(xso, xso->dev->ifindex, xuo); +} +#endif + +static int copy_user_offload_ifindex(const struct xfrm_dev_offload *xso, + int ifindex, struct sk_buff *skb) { struct xfrm_user_offload *xuo; struct nlattr *attr; @@ -1222,11 +1231,22 @@ static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) xuo = nla_data(attr); memset(xuo, 0, sizeof(*xuo)); - xso_to_xuo(xso, xuo); + xso_to_xuo_ifindex(xso, ifindex, xuo); return 0; } +static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) +{ + return copy_user_offload_ifindex(xso, xso->dev->ifindex, skb); +} + +static int copy_user_state_offload(const struct xfrm_dev_offload *xso, + struct sk_buff *skb) +{ + return copy_user_offload_ifindex(xso, READ_ONCE(xso->ifindex), skb); +} + static bool xfrm_redact(void) { return IS_ENABLED(CONFIG_SECURITY) && @@ -1433,8 +1453,8 @@ static int copy_to_user_state_extra(struct xfrm_state *x, &x->replay); if (ret) goto out; - if(x->xso.dev) - ret = copy_user_offload(&x->xso, skb); + if (READ_ONCE(x->xso.dev)) + ret = copy_user_state_offload(&x->xso, skb); if (ret) goto out; if (x->if_id) { @@ -2104,13 +2124,12 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family, switch (ut[i].mode) { case XFRM_MODE_TUNNEL: case XFRM_MODE_BEET: + case XFRM_MODE_IPTFS: if (ut[i].optional && dir == XFRM_POLICY_OUT) { NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy"); return -EINVAL; } break; - case XFRM_MODE_IPTFS: - break; default: if (ut[i].family != prev_family) { NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change"); @@ -4046,8 +4065,8 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) l += nla_total_size(sizeof(*x->coaddr)); if (x->props.extra_flags) l += nla_total_size(sizeof(x->props.extra_flags)); - if (x->xso.dev) - l += nla_total_size(sizeof(struct xfrm_user_offload)); + if (READ_ONCE(x->xso.dev)) + l += nla_total_size(sizeof(struct xfrm_user_offload)); if (x->props.smark.v | x->props.smark.m) { l += nla_total_size(sizeof(x->props.smark.v)); l += nla_total_size(sizeof(x->props.smark.m));