diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 29a36709e7f3..ad8a16146ac5 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -33,6 +33,12 @@ #define IP_VS_HDR_INVERSE 1 #define IP_VS_HDR_ICMP 2 +/* + * Hash table: for virtual service lookups + */ +#define IP_VS_SVC_TAB_BITS 8 +#define IP_VS_SVC_TAB_SIZE BIT(IP_VS_SVC_TAB_BITS) +#define IP_VS_SVC_TAB_MASK (IP_VS_SVC_TAB_SIZE - 1) /* Generic access of ipvs struct */ static inline struct netns_ipvs *net_ipvs(struct net* net) @@ -265,6 +271,18 @@ static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, pr_err(msg, ##__VA_ARGS__); \ } while (0) +/* For arrays per family */ +enum { + IP_VS_AF_INET, + IP_VS_AF_INET6, + IP_VS_AF_MAX +}; + +static inline int ip_vs_af_index(int af) +{ + return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; +} + /* The port number of FTP service (in network order). */ #define FTPPORT cpu_to_be16(21) #define FTPDATA cpu_to_be16(20) @@ -673,8 +691,7 @@ struct ip_vs_dest_user_kern { * forwarding entries. */ struct ip_vs_service { - struct hlist_node s_list; /* for normal service table */ - struct hlist_node f_list; /* for fwmark-based service table */ + struct hlist_node s_list; /* node in service table */ atomic_t refcnt; /* reference counter */ u16 af; /* address family */ @@ -931,21 +948,22 @@ struct netns_ipvs { #endif /* ip_vs_conn */ atomic_t conn_count; /* connection counter */ + atomic_t no_cport_conns[IP_VS_AF_MAX]; /* ip_vs_ctl */ struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ - int num_services; /* no of virtual services */ - int num_services6; /* IPv6 virtual services */ - /* Trash for destinations */ struct list_head dest_trash; spinlock_t dest_trash_lock; struct timer_list dest_trash_timer; /* expiration timer */ /* Service counters */ - atomic_t ftpsvc_counter; - atomic_t nullsvc_counter; - atomic_t conn_out_counter; + atomic_t num_services[IP_VS_AF_MAX]; /* Services */ + atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ + atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ + atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ + atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ + atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ #ifdef CONFIG_SYSCTL /* delayed work for expiring no dest connections */ @@ -956,6 +974,7 @@ struct netns_ipvs { int drop_counter; int old_secure_tcp; atomic_t dropentry; + s8 dropentry_counters[8]; /* locks in ctl.c */ spinlock_t dropentry_lock; /* drop entry handling */ spinlock_t droppacket_lock; /* drop packet handling */ @@ -1041,6 +1060,10 @@ struct netns_ipvs { */ unsigned int mixed_address_family_dests; unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ + + /* the service mutex that protect svc_table and svc_fwm_table */ + struct mutex service_mutex; + struct hlist_head svc_table[IP_VS_SVC_TAB_SIZE]; /* Services */ }; #define DEFAULT_SYNC_THRESHOLD 3 diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 426534a711b0..40e8106e71f0 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -122,17 +122,6 @@ struct nft_regs { }; }; -struct nft_regs_track { - struct { - const struct nft_expr *selector; - const struct nft_expr *bitwise; - u8 num_reg; - } regs[NFT_REG32_NUM]; - - const struct nft_expr *cur; - const struct nft_expr *last; -}; - /* Store/load an u8, u16 or u64 integer to/from the u32 data register. * * Note, when using concatenations, register allocation happens at 32-bit @@ -427,8 +416,6 @@ int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp); void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr); int nft_expr_dump(struct sk_buff *skb, unsigned int attr, const struct nft_expr *expr, bool reset); -bool nft_expr_reduce_bitwise(struct nft_regs_track *track, - const struct nft_expr *expr); struct nft_set_ext; @@ -941,7 +928,6 @@ struct nft_offload_ctx; * @destroy_clone: destruction clone function * @dump: function to dump parameters * @validate: validate expression, called during loop detection - * @reduce: reduce expression * @gc: garbage collection expression * @offload: hardware offload expression * @offload_action: function to report true/false to allocate one slot or not in the flow @@ -975,8 +961,6 @@ struct nft_expr_ops { bool reset); int (*validate)(const struct nft_ctx *ctx, const struct nft_expr *expr); - bool (*reduce)(struct nft_regs_track *track, - const struct nft_expr *expr); bool (*gc)(struct net *net, const struct nft_expr *expr); int (*offload)(struct nft_offload_ctx *ctx, @@ -1954,20 +1938,4 @@ static inline u64 nft_net_tstamp(const struct net *net) return nft_pernet(net)->tstamp; } -#define __NFT_REDUCE_READONLY 1UL -#define NFT_REDUCE_READONLY (void *)__NFT_REDUCE_READONLY - -void nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg, u8 len); -void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len); -void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg); - -static inline bool nft_reg_track_cmp(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg) -{ - return track->regs[dreg].selector && - track->regs[dreg].selector->ops == expr->ops && - track->regs[dreg].num_reg == 0; -} - #endif /* _NET_NF_TABLES_H */ diff --git a/include/net/netfilter/nft_fib.h b/include/net/netfilter/nft_fib.h index 7370fba844ef..e0422456f27b 100644 --- a/include/net/netfilter/nft_fib.h +++ b/include/net/netfilter/nft_fib.h @@ -66,6 +66,4 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs, void nft_fib_store_result(void *reg, const struct nft_fib *priv, const struct net_device *dev); -bool nft_fib_reduce(struct nft_regs_track *track, - const struct nft_expr *expr); #endif diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h index d602263590fe..f74e63290603 100644 --- a/include/net/netfilter/nft_meta.h +++ b/include/net/netfilter/nft_meta.h @@ -43,9 +43,6 @@ void nft_meta_set_destroy(const struct nft_ctx *ctx, int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr); -bool nft_meta_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr); - struct nft_inner_tun_ctx; void nft_meta_inner_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt, diff --git a/net/bridge/netfilter/nft_meta_bridge.c b/net/bridge/netfilter/nft_meta_bridge.c index b7af36bbd306..7763e78abb00 100644 --- a/net/bridge/netfilter/nft_meta_bridge.c +++ b/net/bridge/netfilter/nft_meta_bridge.c @@ -112,7 +112,6 @@ static const struct nft_expr_ops nft_meta_bridge_get_ops = { .eval = nft_meta_bridge_get_eval, .init = nft_meta_bridge_get_init, .dump = nft_meta_get_dump, - .reduce = nft_meta_get_reduce, }; static void nft_meta_bridge_set_eval(const struct nft_expr *expr, @@ -159,24 +158,6 @@ static int nft_meta_bridge_set_init(const struct nft_ctx *ctx, return 0; } -static bool nft_meta_bridge_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_meta_bridge_get_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - static int nft_meta_bridge_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { @@ -202,7 +183,6 @@ static const struct nft_expr_ops nft_meta_bridge_set_ops = { .init = nft_meta_bridge_set_init, .destroy = nft_meta_set_destroy, .dump = nft_meta_set_dump, - .reduce = nft_meta_bridge_set_reduce, .validate = nft_meta_bridge_set_validate, }; diff --git a/net/bridge/netfilter/nft_reject_bridge.c b/net/bridge/netfilter/nft_reject_bridge.c index 1cb5c16e97b7..cd2b04236a99 100644 --- a/net/bridge/netfilter/nft_reject_bridge.c +++ b/net/bridge/netfilter/nft_reject_bridge.c @@ -184,7 +184,6 @@ static const struct nft_expr_ops nft_reject_bridge_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_bridge_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_bridge_type __read_mostly = { diff --git a/net/ipv4/netfilter/nft_dup_ipv4.c b/net/ipv4/netfilter/nft_dup_ipv4.c index ef5dd88107dd..d53a65ddbd7b 100644 --- a/net/ipv4/netfilter/nft_dup_ipv4.c +++ b/net/ipv4/netfilter/nft_dup_ipv4.c @@ -76,7 +76,6 @@ static const struct nft_expr_ops nft_dup_ipv4_ops = { .eval = nft_dup_ipv4_eval, .init = nft_dup_ipv4_init, .dump = nft_dup_ipv4_dump, - .reduce = NFT_REDUCE_READONLY, }; static const struct nla_policy nft_dup_ipv4_policy[NFTA_DUP_MAX + 1] = { diff --git a/net/ipv4/netfilter/nft_fib_ipv4.c b/net/ipv4/netfilter/nft_fib_ipv4.c index 82af6cd76d13..9d0c6d75109b 100644 --- a/net/ipv4/netfilter/nft_fib_ipv4.c +++ b/net/ipv4/netfilter/nft_fib_ipv4.c @@ -163,7 +163,6 @@ static const struct nft_expr_ops nft_fib4_type_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops nft_fib4_ops = { @@ -173,7 +172,6 @@ static const struct nft_expr_ops nft_fib4_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops * diff --git a/net/ipv4/netfilter/nft_reject_ipv4.c b/net/ipv4/netfilter/nft_reject_ipv4.c index 6cb213bb7256..55fc23a8f7a7 100644 --- a/net/ipv4/netfilter/nft_reject_ipv4.c +++ b/net/ipv4/netfilter/nft_reject_ipv4.c @@ -45,7 +45,6 @@ static const struct nft_expr_ops nft_reject_ipv4_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_ipv4_type __read_mostly = { diff --git a/net/ipv6/netfilter/nft_dup_ipv6.c b/net/ipv6/netfilter/nft_dup_ipv6.c index 492a811828a7..95ec27b3971c 100644 --- a/net/ipv6/netfilter/nft_dup_ipv6.c +++ b/net/ipv6/netfilter/nft_dup_ipv6.c @@ -74,7 +74,6 @@ static const struct nft_expr_ops nft_dup_ipv6_ops = { .eval = nft_dup_ipv6_eval, .init = nft_dup_ipv6_init, .dump = nft_dup_ipv6_dump, - .reduce = NFT_REDUCE_READONLY, }; static const struct nla_policy nft_dup_ipv6_policy[NFTA_DUP_MAX + 1] = { diff --git a/net/ipv6/netfilter/nft_fib_ipv6.c b/net/ipv6/netfilter/nft_fib_ipv6.c index 421036a3605b..dc375b725b28 100644 --- a/net/ipv6/netfilter/nft_fib_ipv6.c +++ b/net/ipv6/netfilter/nft_fib_ipv6.c @@ -225,7 +225,6 @@ static const struct nft_expr_ops nft_fib6_type_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops nft_fib6_ops = { @@ -235,7 +234,6 @@ static const struct nft_expr_ops nft_fib6_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops * diff --git a/net/ipv6/netfilter/nft_reject_ipv6.c b/net/ipv6/netfilter/nft_reject_ipv6.c index 5c61294f410e..ed69c768797e 100644 --- a/net/ipv6/netfilter/nft_reject_ipv6.c +++ b/net/ipv6/netfilter/nft_reject_ipv6.c @@ -46,7 +46,6 @@ static const struct nft_expr_ops nft_reject_ipv6_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_ipv6_type __read_mostly = { diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 50cc492c7553..66057db63d02 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -59,9 +59,6 @@ static struct hlist_head *ip_vs_conn_tab __read_mostly; /* SLAB cache for IPVS connections */ static struct kmem_cache *ip_vs_conn_cachep __read_mostly; -/* counter for no client port connections */ -static atomic_t ip_vs_conn_no_cport_cnt = ATOMIC_INIT(0); - /* random value for IPVS connection hash */ static unsigned int ip_vs_conn_rnd __read_mostly; @@ -294,10 +291,16 @@ struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p) struct ip_vs_conn *cp; cp = __ip_vs_conn_in_get(p); - if (!cp && atomic_read(&ip_vs_conn_no_cport_cnt)) { - struct ip_vs_conn_param cport_zero_p = *p; - cport_zero_p.cport = 0; - cp = __ip_vs_conn_in_get(&cport_zero_p); + if (!cp) { + struct netns_ipvs *ipvs = p->ipvs; + int af_id = ip_vs_af_index(p->af); + + if (atomic_read(&ipvs->no_cport_conns[af_id])) { + struct ip_vs_conn_param cport_zero_p = *p; + + cport_zero_p.cport = 0; + cp = __ip_vs_conn_in_get(&cport_zero_p); + } } IP_VS_DBG_BUF(9, "lookup/in %s %s:%d->%s:%d %s\n", @@ -490,9 +493,12 @@ void ip_vs_conn_put(struct ip_vs_conn *cp) void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport) { if (ip_vs_conn_unhash(cp)) { + struct netns_ipvs *ipvs = cp->ipvs; + int af_id = ip_vs_af_index(cp->af); + spin_lock_bh(&cp->lock); if (cp->flags & IP_VS_CONN_F_NO_CPORT) { - atomic_dec(&ip_vs_conn_no_cport_cnt); + atomic_dec(&ipvs->no_cport_conns[af_id]); cp->flags &= ~IP_VS_CONN_F_NO_CPORT; cp->cport = cport; } @@ -891,8 +897,11 @@ static void ip_vs_conn_expire(struct timer_list *t) if (unlikely(cp->app != NULL)) ip_vs_unbind_app(cp); ip_vs_unbind_dest(cp); - if (cp->flags & IP_VS_CONN_F_NO_CPORT) - atomic_dec(&ip_vs_conn_no_cport_cnt); + if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) { + int af_id = ip_vs_af_index(cp->af); + + atomic_dec(&ipvs->no_cport_conns[af_id]); + } if (cp->flags & IP_VS_CONN_F_ONE_PACKET) ip_vs_conn_rcu_free(&cp->rcu_head); else @@ -999,8 +1008,11 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, cp->out_seq.delta = 0; atomic_inc(&ipvs->conn_count); - if (flags & IP_VS_CONN_F_NO_CPORT) - atomic_inc(&ip_vs_conn_no_cport_cnt); + if (unlikely(flags & IP_VS_CONN_F_NO_CPORT)) { + int af_id = ip_vs_af_index(cp->af); + + atomic_inc(&ipvs->no_cport_conns[af_id]); + } /* Bind the connection with a destination server */ cp->dest = NULL; @@ -1257,6 +1269,7 @@ static const struct seq_operations ip_vs_conn_sync_seq_ops = { }; #endif +#ifdef CONFIG_SYSCTL /* Randomly drop connection entries before running out of memory * Can be used for DATA and CTL conns. For TPL conns there are exceptions: @@ -1266,12 +1279,7 @@ static const struct seq_operations ip_vs_conn_sync_seq_ops = { */ static inline int todrop_entry(struct ip_vs_conn *cp) { - /* - * The drop rate array needs tuning for real environments. - * Called from timer bh only => no locking - */ - static const signed char todrop_rate[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; - static signed char todrop_counter[9] = {0}; + struct netns_ipvs *ipvs = cp->ipvs; int i; /* if the conn entry hasn't lasted for 60 seconds, don't drop it. @@ -1280,15 +1288,17 @@ static inline int todrop_entry(struct ip_vs_conn *cp) if (time_before(cp->timeout + jiffies, cp->timer.expires + 60*HZ)) return 0; - /* Don't drop the entry if its number of incoming packets is not - located in [0, 8] */ + /* Drop only conns with number of incoming packets in [1..8] range */ i = atomic_read(&cp->in_pkts); - if (i > 8 || i < 0) return 0; + if (i > 8 || i < 1) + return 0; - if (!todrop_rate[i]) return 0; - if (--todrop_counter[i] > 0) return 0; + i--; + if (--ipvs->dropentry_counters[i] > 0) + return 0; - todrop_counter[i] = todrop_rate[i]; + /* Prefer to drop conns with less number of incoming packets */ + ipvs->dropentry_counters[i] = i + 1; return 1; } @@ -1368,7 +1378,7 @@ void ip_vs_random_dropentry(struct netns_ipvs *ipvs) } rcu_read_unlock(); } - +#endif /* * Flush all the connection entries in the ip_vs_conn_tab @@ -1450,7 +1460,11 @@ void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs) */ int __net_init ip_vs_conn_net_init(struct netns_ipvs *ipvs) { + int idx; + atomic_set(&ipvs->conn_count, 0); + for (idx = 0; idx < IP_VS_AF_MAX; idx++) + atomic_set(&ipvs->no_cport_conns[idx], 0); #ifdef CONFIG_PROC_FS if (!proc_create_net("ip_vs_conn", 0, ipvs->net->proc_net, diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 90d56f92c0f6..869f18e0e835 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1400,7 +1400,7 @@ ip_vs_out_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *stat return handle_response(af, skb, pd, cp, &iph, hooknum); /* Check for real-server-started requests */ - if (atomic_read(&ipvs->conn_out_counter)) { + if (atomic_read(&ipvs->conn_out_counter[ip_vs_af_index(af)])) { /* Currently only for UDP: * connection oriented protocols typically use * ephemeral ports for outgoing connections, so diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index 068702894377..564e09f0f90a 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -48,7 +48,7 @@ MODULE_ALIAS_GENL_FAMILY(IPVS_GENL_NAME); -DEFINE_MUTEX(__ip_vs_mutex); /* Serialize configuration with sockopt/netlink */ +static struct lock_class_key __ipvs_service_key; /* sysctl variables */ @@ -293,17 +293,6 @@ ip_vs_use_count_dec(void) } -/* - * Hash table: for virtual service lookups - */ -#define IP_VS_SVC_TAB_BITS 8 -#define IP_VS_SVC_TAB_SIZE (1 << IP_VS_SVC_TAB_BITS) -#define IP_VS_SVC_TAB_MASK (IP_VS_SVC_TAB_SIZE - 1) - -/* the service table hashed by */ -static struct hlist_head ip_vs_svc_table[IP_VS_SVC_TAB_SIZE]; -/* the service table hashed by fwmark */ -static struct hlist_head ip_vs_svc_fwm_table[IP_VS_SVC_TAB_SIZE]; /* @@ -338,8 +327,8 @@ static inline unsigned int ip_vs_svc_fwm_hashkey(struct netns_ipvs *ipvs, __u32 } /* - * Hashes a service in the ip_vs_svc_table by - * or in the ip_vs_svc_fwm_table by fwmark. + * Hashes a service in the svc_table by + * or by fwmark. * Should be called with locked tables. */ static int ip_vs_svc_hash(struct ip_vs_service *svc) @@ -354,18 +343,17 @@ static int ip_vs_svc_hash(struct ip_vs_service *svc) if (svc->fwmark == 0) { /* - * Hash it by in ip_vs_svc_table + * Hash it by */ hash = ip_vs_svc_hashkey(svc->ipvs, svc->af, svc->protocol, &svc->addr, svc->port); - hlist_add_head_rcu(&svc->s_list, &ip_vs_svc_table[hash]); } else { /* - * Hash it by fwmark in svc_fwm_table + * Hash it by fwmark */ hash = ip_vs_svc_fwm_hashkey(svc->ipvs, svc->fwmark); - hlist_add_head_rcu(&svc->f_list, &ip_vs_svc_fwm_table[hash]); } + hlist_add_head_rcu(&svc->s_list, &svc->ipvs->svc_table[hash]); svc->flags |= IP_VS_SVC_F_HASHED; /* increase its refcnt because it is referenced by the svc table */ @@ -375,7 +363,7 @@ static int ip_vs_svc_hash(struct ip_vs_service *svc) /* - * Unhashes a service from svc_table / svc_fwm_table. + * Unhashes a service from svc_table. * Should be called with locked tables. */ static int ip_vs_svc_unhash(struct ip_vs_service *svc) @@ -386,13 +374,8 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc) return 0; } - if (svc->fwmark == 0) { - /* Remove it from the svc_table table */ - hlist_del_rcu(&svc->s_list); - } else { - /* Remove it from the svc_fwm_table table */ - hlist_del_rcu(&svc->f_list); - } + /* Remove it from svc_table */ + hlist_del_rcu(&svc->s_list); svc->flags &= ~IP_VS_SVC_F_HASHED; atomic_dec(&svc->refcnt); @@ -413,12 +396,10 @@ __ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u16 protocol, /* Check for "full" addressed entries */ hash = ip_vs_svc_hashkey(ipvs, af, protocol, vaddr, vport); - hlist_for_each_entry_rcu(svc, &ip_vs_svc_table[hash], s_list) { - if ((svc->af == af) - && ip_vs_addr_equal(af, &svc->addr, vaddr) - && (svc->port == vport) - && (svc->protocol == protocol) - && (svc->ipvs == ipvs)) { + hlist_for_each_entry_rcu(svc, &ipvs->svc_table[hash], s_list) { + if (svc->af == af && ip_vs_addr_equal(af, &svc->addr, vaddr) && + svc->port == vport && svc->protocol == protocol && + !svc->fwmark) { /* HIT */ return svc; } @@ -440,9 +421,8 @@ __ip_vs_svc_fwm_find(struct netns_ipvs *ipvs, int af, __u32 fwmark) /* Check for fwmark addressed entries */ hash = ip_vs_svc_fwm_hashkey(ipvs, fwmark); - hlist_for_each_entry_rcu(svc, &ip_vs_svc_fwm_table[hash], f_list) { - if (svc->fwmark == fwmark && svc->af == af - && (svc->ipvs == ipvs)) { + hlist_for_each_entry_rcu(svc, &ipvs->svc_table[hash], s_list) { + if (svc->fwmark == fwmark && svc->af == af) { /* HIT */ return svc; } @@ -456,35 +436,42 @@ struct ip_vs_service * ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol, const union nf_inet_addr *vaddr, __be16 vport) { - struct ip_vs_service *svc; + struct ip_vs_service *svc = NULL; + int af_id = ip_vs_af_index(af); /* * Check the table hashed by fwmark first */ - if (fwmark) { + if (fwmark && atomic_read(&ipvs->fwm_services[af_id])) { svc = __ip_vs_svc_fwm_find(ipvs, af, fwmark); if (svc) goto out; } + if (!atomic_read(&ipvs->nonfwm_services[af_id])) + goto out; + /* * Check the table hashed by * for "full" addressed entries */ svc = __ip_vs_service_find(ipvs, af, protocol, vaddr, vport); + if (svc) + goto out; - if (!svc && protocol == IPPROTO_TCP && - atomic_read(&ipvs->ftpsvc_counter) && + if (protocol == IPPROTO_TCP && + atomic_read(&ipvs->ftpsvc_counter[af_id]) && (vport == FTPDATA || !inet_port_requires_bind_service(ipvs->net, ntohs(vport)))) { /* * Check if ftp service entry exists, the packet * might belong to FTP data connections. */ svc = __ip_vs_service_find(ipvs, af, protocol, vaddr, FTPPORT); + if (svc) + goto out; } - if (svc == NULL - && atomic_read(&ipvs->nullsvc_counter)) { + if (atomic_read(&ipvs->nullsvc_counter[af_id])) { /* * Check if the catch-all port (port zero) exists */ @@ -829,7 +816,6 @@ static void ip_vs_dest_free(struct ip_vs_dest *dest) { struct ip_vs_service *svc = rcu_dereference_protected(dest->svc, 1); - __ip_vs_dst_cache_reset(dest); __ip_vs_svc_put(svc); call_rcu(&dest->rcu_head, ip_vs_dest_rcu_free); } @@ -1032,10 +1018,6 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, dest->af = udest->af; - spin_lock_bh(&dest->dst_lock); - __ip_vs_dst_cache_reset(dest); - spin_unlock_bh(&dest->dst_lock); - if (add) { list_add_rcu(&dest->n_list, &svc->destinations); svc->num_dests++; @@ -1043,6 +1025,10 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, if (sched && sched->add_dest) sched->add_dest(svc, dest); } else { + spin_lock_bh(&dest->dst_lock); + __ip_vs_dst_cache_reset(dest); + spin_unlock_bh(&dest->dst_lock); + sched = rcu_dereference_protected(svc->scheduler, 1); if (sched && sched->upd_dest) sched->upd_dest(svc, dest); @@ -1277,6 +1263,10 @@ static void __ip_vs_unlink_dest(struct ip_vs_service *svc, { dest->flags &= ~IP_VS_DEST_F_AVAILABLE; + spin_lock_bh(&dest->dst_lock); + __ip_vs_dst_cache_reset(dest); + spin_unlock_bh(&dest->dst_lock); + /* * Remove it from the d-linked destination list. */ @@ -1369,6 +1359,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u, { int ret = 0; struct ip_vs_scheduler *sched = NULL; + int af_id = ip_vs_af_index(u->af); struct ip_vs_pe *pe = NULL; struct ip_vs_service *svc = NULL; int ret_hooks = -1; @@ -1413,8 +1404,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u, } #endif - if ((u->af == AF_INET && !ipvs->num_services) || - (u->af == AF_INET6 && !ipvs->num_services6)) { + if (!atomic_read(&ipvs->num_services[af_id])) { ret = ip_vs_register_hooks(ipvs, u->af); if (ret < 0) goto out_err; @@ -1461,21 +1451,21 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u, /* Update the virtual service counters */ if (svc->port == FTPPORT) - atomic_inc(&ipvs->ftpsvc_counter); - else if (svc->port == 0) - atomic_inc(&ipvs->nullsvc_counter); + atomic_inc(&ipvs->ftpsvc_counter[af_id]); + else if (!svc->port && !svc->fwmark) + atomic_inc(&ipvs->nullsvc_counter[af_id]); if (pe && pe->conn_out) - atomic_inc(&ipvs->conn_out_counter); + atomic_inc(&ipvs->conn_out_counter[af_id]); /* Bind the ct retriever */ RCU_INIT_POINTER(svc->pe, pe); pe = NULL; - /* Count only IPv4 services for old get/setsockopt interface */ - if (svc->af == AF_INET) - ipvs->num_services++; - else if (svc->af == AF_INET6) - ipvs->num_services6++; + if (svc->fwmark) + atomic_inc(&ipvs->fwm_services[af_id]); + else + atomic_inc(&ipvs->nonfwm_services[af_id]); + atomic_inc(&ipvs->num_services[af_id]); /* Hash the service into the service table */ ip_vs_svc_hash(svc); @@ -1520,6 +1510,8 @@ ip_vs_edit_service(struct ip_vs_service *svc, struct ip_vs_service_user_kern *u) struct ip_vs_pe *pe = NULL, *old_pe = NULL; int ret = 0; bool new_pe_conn_out, old_pe_conn_out; + struct netns_ipvs *ipvs = svc->ipvs; + int af_id = ip_vs_af_index(svc->af); /* * Lookup the scheduler, by 'u->sched_name' @@ -1588,9 +1580,9 @@ ip_vs_edit_service(struct ip_vs_service *svc, struct ip_vs_service_user_kern *u) new_pe_conn_out = (pe && pe->conn_out) ? true : false; old_pe_conn_out = (old_pe && old_pe->conn_out) ? true : false; if (new_pe_conn_out && !old_pe_conn_out) - atomic_inc(&svc->ipvs->conn_out_counter); + atomic_inc(&ipvs->conn_out_counter[af_id]); if (old_pe_conn_out && !new_pe_conn_out) - atomic_dec(&svc->ipvs->conn_out_counter); + atomic_dec(&ipvs->conn_out_counter[af_id]); } out: @@ -1610,16 +1602,15 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) struct ip_vs_scheduler *old_sched; struct ip_vs_pe *old_pe; struct netns_ipvs *ipvs = svc->ipvs; + int af_id = ip_vs_af_index(svc->af); - if (svc->af == AF_INET) { - ipvs->num_services--; - if (!ipvs->num_services) - ip_vs_unregister_hooks(ipvs, svc->af); - } else if (svc->af == AF_INET6) { - ipvs->num_services6--; - if (!ipvs->num_services6) - ip_vs_unregister_hooks(ipvs, svc->af); - } + atomic_dec(&ipvs->num_services[af_id]); + if (!atomic_read(&ipvs->num_services[af_id])) + ip_vs_unregister_hooks(ipvs, svc->af); + if (svc->fwmark) + atomic_dec(&ipvs->fwm_services[af_id]); + else + atomic_dec(&ipvs->nonfwm_services[af_id]); ip_vs_stop_estimator(svc->ipvs, &svc->stats); @@ -1631,7 +1622,7 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) /* Unbind persistence engine, keep svc->pe */ old_pe = rcu_dereference_protected(svc->pe, 1); if (old_pe && old_pe->conn_out) - atomic_dec(&ipvs->conn_out_counter); + atomic_dec(&ipvs->conn_out_counter[af_id]); ip_vs_pe_put(old_pe); /* @@ -1646,9 +1637,9 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) * Update the virtual service counters */ if (svc->port == FTPPORT) - atomic_dec(&ipvs->ftpsvc_counter); - else if (svc->port == 0) - atomic_dec(&ipvs->nullsvc_counter); + atomic_dec(&ipvs->ftpsvc_counter[af_id]); + else if (!svc->port && !svc->fwmark) + atomic_dec(&ipvs->nullsvc_counter[af_id]); /* * Free the service if nobody refers to it @@ -1697,28 +1688,11 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup) struct ip_vs_service *svc; struct hlist_node *n; - /* - * Flush the service table hashed by - */ for(idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry_safe(svc, n, &ip_vs_svc_table[idx], - s_list) { - if (svc->ipvs == ipvs) - ip_vs_unlink_service(svc, cleanup); - } + hlist_for_each_entry_safe(svc, n, &ipvs->svc_table[idx], + s_list) + ip_vs_unlink_service(svc, cleanup); } - - /* - * Flush the service table hashed by fwmark - */ - for(idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry_safe(svc, n, &ip_vs_svc_fwm_table[idx], - f_list) { - if (svc->ipvs == ipvs) - ip_vs_unlink_service(svc, cleanup); - } - } - return 0; } @@ -1732,12 +1706,12 @@ void ip_vs_service_nets_cleanup(struct list_head *net_list) struct net *net; /* Check for "full" addressed entries */ - mutex_lock(&__ip_vs_mutex); list_for_each_entry(net, net_list, exit_list) { ipvs = net_ipvs(net); + mutex_lock(&ipvs->service_mutex); ip_vs_flush(ipvs, true); + mutex_unlock(&ipvs->service_mutex); } - mutex_unlock(&__ip_vs_mutex); } /* Put all references for device (dst_cache) */ @@ -1775,34 +1749,15 @@ static int ip_vs_dst_event(struct notifier_block *this, unsigned long event, if (event != NETDEV_DOWN || !ipvs) return NOTIFY_DONE; IP_VS_DBG(3, "%s() dev=%s\n", __func__, dev->name); - mutex_lock(&__ip_vs_mutex); + rcu_read_lock(); for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry(svc, &ip_vs_svc_table[idx], s_list) { - if (svc->ipvs == ipvs) { - list_for_each_entry(dest, &svc->destinations, - n_list) { - ip_vs_forget_dev(dest, dev); - } - } - } - - hlist_for_each_entry(svc, &ip_vs_svc_fwm_table[idx], f_list) { - if (svc->ipvs == ipvs) { - list_for_each_entry(dest, &svc->destinations, - n_list) { - ip_vs_forget_dev(dest, dev); - } - } - - } + hlist_for_each_entry_rcu(svc, &ipvs->svc_table[idx], s_list) + list_for_each_entry_rcu(dest, &svc->destinations, + n_list) + ip_vs_forget_dev(dest, dev); } + rcu_read_unlock(); - spin_lock_bh(&ipvs->dest_trash_lock); - list_for_each_entry(dest, &ipvs->dest_trash, t_list) { - ip_vs_forget_dev(dest, dev); - } - spin_unlock_bh(&ipvs->dest_trash_lock); - mutex_unlock(&__ip_vs_mutex); return NOTIFY_DONE; } @@ -1826,17 +1781,8 @@ static int ip_vs_zero_all(struct netns_ipvs *ipvs) struct ip_vs_service *svc; for(idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry(svc, &ip_vs_svc_table[idx], s_list) { - if (svc->ipvs == ipvs) - ip_vs_zero_service(svc); - } - } - - for(idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry(svc, &ip_vs_svc_fwm_table[idx], f_list) { - if (svc->ipvs == ipvs) - ip_vs_zero_service(svc); - } + hlist_for_each_entry(svc, &ipvs->svc_table[idx], s_list) + ip_vs_zero_service(svc); } ip_vs_zero_stats(&ipvs->tot_stats->s); @@ -2272,7 +2218,6 @@ static struct ctl_table vs_vars[] = { struct ip_vs_iter { struct seq_net_private p; /* Do not move this, netns depends upon it*/ - struct hlist_head *table; int bucket; }; @@ -2295,7 +2240,6 @@ static inline const char *ip_vs_fwd_name(unsigned int flags) } -/* Get the Nth entry in the two lists */ static struct ip_vs_service *ip_vs_info_array(struct seq_file *seq, loff_t pos) { struct net *net = seq_file_net(seq); @@ -2304,29 +2248,14 @@ static struct ip_vs_service *ip_vs_info_array(struct seq_file *seq, loff_t pos) int idx; struct ip_vs_service *svc; - /* look in hash by protocol */ for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry_rcu(svc, &ip_vs_svc_table[idx], s_list) { - if ((svc->ipvs == ipvs) && pos-- == 0) { - iter->table = ip_vs_svc_table; + hlist_for_each_entry_rcu(svc, &ipvs->svc_table[idx], s_list) { + if (pos-- == 0) { iter->bucket = idx; return svc; } } } - - /* keep looking in fwmark */ - for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry_rcu(svc, &ip_vs_svc_fwm_table[idx], - f_list) { - if ((svc->ipvs == ipvs) && pos-- == 0) { - iter->table = ip_vs_svc_fwm_table; - iter->bucket = idx; - return svc; - } - } - } - return NULL; } @@ -2343,6 +2272,8 @@ static void *ip_vs_info_seq_next(struct seq_file *seq, void *v, loff_t *pos) struct hlist_node *e; struct ip_vs_iter *iter; struct ip_vs_service *svc; + struct net *net = seq_file_net(seq); + struct netns_ipvs *ipvs = net_ipvs(net); ++*pos; if (v == SEQ_START_TOKEN) @@ -2351,38 +2282,17 @@ static void *ip_vs_info_seq_next(struct seq_file *seq, void *v, loff_t *pos) svc = v; iter = seq->private; - if (iter->table == ip_vs_svc_table) { - /* next service in table hashed by protocol */ - e = rcu_dereference(hlist_next_rcu(&svc->s_list)); - if (e) - return hlist_entry(e, struct ip_vs_service, s_list); - - while (++iter->bucket < IP_VS_SVC_TAB_SIZE) { - hlist_for_each_entry_rcu(svc, - &ip_vs_svc_table[iter->bucket], - s_list) { - return svc; - } - } - - iter->table = ip_vs_svc_fwm_table; - iter->bucket = -1; - goto scan_fwmark; - } - - /* next service in hashed by fwmark */ - e = rcu_dereference(hlist_next_rcu(&svc->f_list)); + e = rcu_dereference(hlist_next_rcu(&svc->s_list)); if (e) - return hlist_entry(e, struct ip_vs_service, f_list); + return hlist_entry(e, struct ip_vs_service, s_list); - scan_fwmark: while (++iter->bucket < IP_VS_SVC_TAB_SIZE) { hlist_for_each_entry_rcu(svc, - &ip_vs_svc_fwm_table[iter->bucket], - f_list) + &ipvs->svc_table[iter->bucket], + s_list) { return svc; + } } - return NULL; } @@ -2404,17 +2314,12 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) seq_puts(seq, " -> RemoteAddress:Port Forward Weight ActiveConn InActConn\n"); } else { - struct net *net = seq_file_net(seq); - struct netns_ipvs *ipvs = net_ipvs(net); const struct ip_vs_service *svc = v; - const struct ip_vs_iter *iter = seq->private; const struct ip_vs_dest *dest; struct ip_vs_scheduler *sched = rcu_dereference(svc->scheduler); char *sched_name = sched ? sched->name : "none"; - if (svc->ipvs != ipvs) - return 0; - if (iter->table == ip_vs_svc_table) { + if (!svc->fwmark) { #ifdef CONFIG_IP_VS_IPV6 if (svc->af == AF_INET6) seq_printf(seq, "%s [%pI6]:%04X %s ", @@ -2736,7 +2641,7 @@ do_ip_vs_set_ctl(struct sock *sk, int cmd, sockptr_t ptr, unsigned int len) return ret; } - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); if (cmd == IP_VS_SO_SET_FLUSH) { /* Flush the virtual service */ ret = ip_vs_flush(ipvs, false); @@ -2833,7 +2738,7 @@ do_ip_vs_set_ctl(struct sock *sk, int cmd, sockptr_t ptr, unsigned int len) } out_unlock: - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); return ret; } @@ -2871,9 +2776,9 @@ __ip_vs_get_service_entries(struct netns_ipvs *ipvs, int ret = 0; for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry(svc, &ip_vs_svc_table[idx], s_list) { + hlist_for_each_entry(svc, &ipvs->svc_table[idx], s_list) { /* Only expose IPv4 entries to old interface */ - if (svc->af != AF_INET || (svc->ipvs != ipvs)) + if (svc->af != AF_INET) continue; if (count >= get->num_services) @@ -2889,24 +2794,6 @@ __ip_vs_get_service_entries(struct netns_ipvs *ipvs, } } - for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - hlist_for_each_entry(svc, &ip_vs_svc_fwm_table[idx], f_list) { - /* Only expose IPv4 entries to old interface */ - if (svc->af != AF_INET || (svc->ipvs != ipvs)) - continue; - - if (count >= get->num_services) - goto out; - memset(&entry, 0, sizeof(entry)); - ip_vs_copy_service(&entry, svc); - if (copy_to_user(&uptr->entrytable[count], - &entry, sizeof(entry))) { - ret = -EFAULT; - goto out; - } - count++; - } - } out: return ret; } @@ -3061,7 +2948,7 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len) return ret; } - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); switch (cmd) { case IP_VS_SO_GET_VERSION: { @@ -3082,7 +2969,8 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len) struct ip_vs_getinfo info; info.version = IP_VS_VERSION_CODE; info.size = ip_vs_conn_tab_size; - info.num_services = ipvs->num_services; + info.num_services = + atomic_read(&ipvs->num_services[IP_VS_AF_INET]); if (copy_to_user(user, &info, sizeof(info)) != 0) ret = -EFAULT; } @@ -3160,7 +3048,7 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len) } out: - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); return ret; } @@ -3339,9 +3227,9 @@ static int ip_vs_genl_fill_service(struct sk_buff *skb, goto nla_put_failure; } - sched = rcu_dereference_protected(svc->scheduler, 1); + sched = rcu_dereference(svc->scheduler); sched_name = sched ? sched->name : "none"; - pe = rcu_dereference_protected(svc->pe, 1); + pe = rcu_dereference(svc->pe); if (nla_put_string(skb, IPVS_SVC_ATTR_SCHED_NAME, sched_name) || (pe && nla_put_string(skb, IPVS_SVC_ATTR_PE_NAME, pe->name)) || nla_put(skb, IPVS_SVC_ATTR_FLAGS, sizeof(flags), &flags) || @@ -3395,21 +3283,10 @@ static int ip_vs_genl_dump_services(struct sk_buff *skb, struct net *net = sock_net(skb->sk); struct netns_ipvs *ipvs = net_ipvs(net); - mutex_lock(&__ip_vs_mutex); + rcu_read_lock(); for (i = 0; i < IP_VS_SVC_TAB_SIZE; i++) { - hlist_for_each_entry(svc, &ip_vs_svc_table[i], s_list) { - if (++idx <= start || (svc->ipvs != ipvs)) - continue; - if (ip_vs_genl_dump_service(skb, svc, cb) < 0) { - idx--; - goto nla_put_failure; - } - } - } - - for (i = 0; i < IP_VS_SVC_TAB_SIZE; i++) { - hlist_for_each_entry(svc, &ip_vs_svc_fwm_table[i], f_list) { - if (++idx <= start || (svc->ipvs != ipvs)) + hlist_for_each_entry_rcu(svc, &ipvs->svc_table[i], s_list) { + if (++idx <= start) continue; if (ip_vs_genl_dump_service(skb, svc, cb) < 0) { idx--; @@ -3419,7 +3296,7 @@ static int ip_vs_genl_dump_services(struct sk_buff *skb, } nla_put_failure: - mutex_unlock(&__ip_vs_mutex); + rcu_read_unlock(); cb->args[0] = idx; return skb->len; @@ -3475,13 +3352,11 @@ static int ip_vs_genl_parse_service(struct netns_ipvs *ipvs, usvc->fwmark = 0; } - rcu_read_lock(); if (usvc->fwmark) svc = __ip_vs_svc_fwm_find(ipvs, usvc->af, usvc->fwmark); else svc = __ip_vs_service_find(ipvs, usvc->af, usvc->protocol, &usvc->addr, usvc->port); - rcu_read_unlock(); *ret_svc = svc; /* If a full entry was requested, check for the additional fields */ @@ -3608,7 +3483,7 @@ static int ip_vs_genl_dump_dests(struct sk_buff *skb, struct net *net = sock_net(skb->sk); struct netns_ipvs *ipvs = net_ipvs(net); - mutex_lock(&__ip_vs_mutex); + rcu_read_lock(); /* Try to find the service for which to dump destinations */ if (nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrs, IPVS_CMD_ATTR_MAX, ip_vs_cmd_policy, cb->extack)) @@ -3620,7 +3495,7 @@ static int ip_vs_genl_dump_dests(struct sk_buff *skb, goto out_err; /* Dump the destinations */ - list_for_each_entry(dest, &svc->destinations, n_list) { + list_for_each_entry_rcu(dest, &svc->destinations, n_list) { if (++idx <= start) continue; if (ip_vs_genl_dump_dest(skb, dest, cb) < 0) { @@ -3633,7 +3508,7 @@ static int ip_vs_genl_dump_dests(struct sk_buff *skb, cb->args[0] = idx; out_err: - mutex_unlock(&__ip_vs_mutex); + rcu_read_unlock(); return skb->len; } @@ -3916,7 +3791,7 @@ static int ip_vs_genl_set_cmd(struct sk_buff *skb, struct genl_info *info) cmd = info->genlhdr->cmd; - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); if (cmd == IPVS_CMD_FLUSH) { ret = ip_vs_flush(ipvs, false); @@ -3936,9 +3811,12 @@ static int ip_vs_genl_set_cmd(struct sk_buff *skb, struct genl_info *info) if (cmd == IPVS_CMD_NEW_SERVICE || cmd == IPVS_CMD_SET_SERVICE) need_full_svc = true; + /* We use function that requires RCU lock (hlist_bl) */ + rcu_read_lock(); ret = ip_vs_genl_parse_service(ipvs, &usvc, info->attrs[IPVS_CMD_ATTR_SERVICE], need_full_svc, &svc); + rcu_read_unlock(); if (ret) goto out; @@ -4028,7 +3906,7 @@ static int ip_vs_genl_set_cmd(struct sk_buff *skb, struct genl_info *info) } out: - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); return ret; } @@ -4058,7 +3936,7 @@ static int ip_vs_genl_get_cmd(struct sk_buff *skb, struct genl_info *info) if (!msg) return -ENOMEM; - mutex_lock(&__ip_vs_mutex); + rcu_read_lock(); reply = genlmsg_put_reply(msg, info, &ip_vs_genl_family, 0, reply_cmd); if (reply == NULL) @@ -4126,7 +4004,7 @@ static int ip_vs_genl_get_cmd(struct sk_buff *skb, struct genl_info *info) out_err: nlmsg_free(msg); out: - mutex_unlock(&__ip_vs_mutex); + rcu_read_unlock(); return ret; } @@ -4243,6 +4121,7 @@ static struct genl_family ip_vs_genl_family __ro_after_init = { .small_ops = ip_vs_genl_ops, .n_small_ops = ARRAY_SIZE(ip_vs_genl_ops), .resv_start_op = IPVS_CMD_FLUSH + 1, + .parallel_ops = 1, }; static int __init ip_vs_genl_register(void) @@ -4425,6 +4304,11 @@ int __net_init ip_vs_control_net_init(struct netns_ipvs *ipvs) int ret = -ENOMEM; int idx; + /* Initialize service_mutex, svc_table per netns */ + __mutex_init(&ipvs->service_mutex, "ipvs->service_mutex", &__ipvs_service_key); + for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) + INIT_HLIST_HEAD(&ipvs->svc_table[idx]); + /* Initialize rs_table */ for (idx = 0; idx < IP_VS_RTAB_SIZE; idx++) INIT_HLIST_HEAD(&ipvs->rs_table[idx]); @@ -4432,9 +4316,14 @@ int __net_init ip_vs_control_net_init(struct netns_ipvs *ipvs) INIT_LIST_HEAD(&ipvs->dest_trash); spin_lock_init(&ipvs->dest_trash_lock); timer_setup(&ipvs->dest_trash_timer, ip_vs_dest_trash_expire, 0); - atomic_set(&ipvs->ftpsvc_counter, 0); - atomic_set(&ipvs->nullsvc_counter, 0); - atomic_set(&ipvs->conn_out_counter, 0); + for (idx = 0; idx < IP_VS_AF_MAX; idx++) { + atomic_set(&ipvs->num_services[idx], 0); + atomic_set(&ipvs->fwm_services[idx], 0); + atomic_set(&ipvs->nonfwm_services[idx], 0); + atomic_set(&ipvs->ftpsvc_counter[idx], 0); + atomic_set(&ipvs->nullsvc_counter[idx], 0); + atomic_set(&ipvs->conn_out_counter[idx], 0); + } INIT_DELAYED_WORK(&ipvs->est_reload_work, est_reload_work_handler); @@ -4529,17 +4418,8 @@ void ip_vs_unregister_nl_ioctl(void) int __init ip_vs_control_init(void) { - int idx; int ret; - /* Initialize svc_table, ip_vs_svc_fwm_table */ - for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) { - INIT_HLIST_HEAD(&ip_vs_svc_table[idx]); - INIT_HLIST_HEAD(&ip_vs_svc_fwm_table[idx]); - } - - smp_wmb(); /* Do we really need it now ? */ - ret = register_netdevice_notifier(&ip_vs_dst_notifier); if (ret < 0) return ret; diff --git a/net/netfilter/ipvs/ip_vs_est.c b/net/netfilter/ipvs/ip_vs_est.c index 77f4f637ff67..dc207172ca9f 100644 --- a/net/netfilter/ipvs/ip_vs_est.c +++ b/net/netfilter/ipvs/ip_vs_est.c @@ -602,7 +602,7 @@ static void ip_vs_est_drain_temp_list(struct netns_ipvs *ipvs) while (1) { int max = 16; - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); while (max-- > 0) { est = hlist_entry_safe(ipvs->est_temp_list.first, @@ -622,12 +622,12 @@ static void ip_vs_est_drain_temp_list(struct netns_ipvs *ipvs) } goto unlock; } - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); cond_resched(); } unlock: - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); } /* Calculate limits for all kthreads */ @@ -647,9 +647,9 @@ static int ip_vs_est_calc_limits(struct netns_ipvs *ipvs, int *chain_max) u64 val; INIT_HLIST_HEAD(&chain); - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); kd = ipvs->est_kt_arr[0]; - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); s = kd ? kd->calc_stats : NULL; if (!s) goto out; @@ -748,7 +748,7 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs) if (!ip_vs_est_calc_limits(ipvs, &chain_max)) return; - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); /* Stop all other tasks, so that we can immediately move the * estimators to est_temp_list without RCU grace period @@ -815,9 +815,9 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs) /* Give chance estimators to be added (to est_temp_list) * and deleted (releasing kthread contexts) */ - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); cond_resched(); - mutex_lock(&__ip_vs_mutex); + mutex_lock(&ipvs->service_mutex); /* Current kt released ? */ if (id >= ipvs->est_kt_count) @@ -893,7 +893,7 @@ static void ip_vs_est_calc_phase(struct netns_ipvs *ipvs) mutex_unlock(&ipvs->est_mutex); unlock: - mutex_unlock(&__ip_vs_mutex); + mutex_unlock(&ipvs->service_mutex); } void ip_vs_zero_estimator(struct ip_vs_stats *stats) diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 4389bfe3050d..394b5b5f2ccd 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -336,9 +336,11 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, goto err_unreach; } /* It is forbidden to attach dest->dest_dst if - * device is going down. + * device is going down or if server is removed and + * stored in dest_trash. */ - if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) && + dest->flags & IP_VS_DEST_F_AVAILABLE) __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); else noref = 0; @@ -513,9 +515,11 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, rt = dst_rt6_info(dst); cookie = rt6_get_cookie(rt); /* It is forbidden to attach dest->dest_dst if - * device is going down. + * device is going down or if server is removed and + * stored in dest_trash. */ - if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) && + dest->flags & IP_VS_DEST_F_AVAILABLE) __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); else noref = 0; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 0c5a4855b97d..c8d5f7e93dfd 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -932,58 +932,6 @@ static int nft_delflowtable(struct nft_ctx *ctx, return 0; } -static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg) -{ - int i; - - for (i = track->regs[dreg].num_reg; i > 0; i--) - __nft_reg_track_cancel(track, dreg - i); -} - -static void __nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, - u8 dreg, u8 num_reg) -{ - track->regs[dreg].selector = expr; - track->regs[dreg].bitwise = NULL; - track->regs[dreg].num_reg = num_reg; -} - -void nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg, u8 len) -{ - unsigned int regcount; - int i; - - __nft_reg_track_clobber(track, dreg); - - regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE); - for (i = 0; i < regcount; i++, dreg++) - __nft_reg_track_update(track, expr, dreg, i); -} -EXPORT_SYMBOL_GPL(nft_reg_track_update); - -void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len) -{ - unsigned int regcount; - int i; - - __nft_reg_track_clobber(track, dreg); - - regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE); - for (i = 0; i < regcount; i++, dreg++) - __nft_reg_track_cancel(track, dreg); -} -EXPORT_SYMBOL_GPL(nft_reg_track_cancel); - -void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg) -{ - track->regs[dreg].selector = NULL; - track->regs[dreg].bitwise = NULL; - track->regs[dreg].num_reg = 0; -} -EXPORT_SYMBOL_GPL(__nft_reg_track_cancel); - /* * Tables */ @@ -4128,7 +4076,6 @@ int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain) nft_chain_vstate_update(ctx, chain); return 0; } -EXPORT_SYMBOL_GPL(nft_chain_validate); static int nft_table_validate(struct net *net, const struct nft_table *table) { @@ -4745,7 +4692,6 @@ struct nft_set *nft_set_lookup_global(const struct net *net, } return set; } -EXPORT_SYMBOL_GPL(nft_set_lookup_global); static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set, const char *name) @@ -5818,7 +5764,6 @@ int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set, return 0; } -EXPORT_SYMBOL_GPL(nf_tables_bind_set); static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding, bool event) @@ -5898,7 +5843,6 @@ void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set) nft_use_inc_restore(&set->use); } -EXPORT_SYMBOL_GPL(nf_tables_activate_set); void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set, struct nft_set_binding *binding, @@ -5938,14 +5882,12 @@ void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set, phase == NFT_TRANS_COMMIT); } } -EXPORT_SYMBOL_GPL(nf_tables_deactivate_set); void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set) { if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) nft_set_destroy(ctx, set); } -EXPORT_SYMBOL_GPL(nf_tables_destroy_set); const struct nft_set_ext_type nft_set_ext_types[] = { [NFT_SET_EXT_KEY] = { @@ -6783,7 +6725,6 @@ void nft_set_elem_destroy(const struct nft_set *set, __nft_set_elem_destroy(&ctx, set, elem_priv, destroy_expr); } -EXPORT_SYMBOL_GPL(nft_set_elem_destroy); /* Drop references and destroy. Called from abort path. */ static void nft_trans_set_elem_destroy(const struct nft_ctx *ctx, struct nft_trans_elem *te) @@ -6910,7 +6851,6 @@ struct nft_set_ext *nft_set_catchall_lookup(const struct net *net, return NULL; } -EXPORT_SYMBOL_GPL(nft_set_catchall_lookup); static int nft_setelem_catchall_insert(const struct net *net, struct nft_set *set, @@ -8038,7 +7978,6 @@ struct nft_object *nft_obj_lookup(const struct net *net, rcu_read_unlock(); return ERR_PTR(-ENOENT); } -EXPORT_SYMBOL_GPL(nft_obj_lookup); static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table, const struct nlattr *nla, @@ -10181,16 +10120,9 @@ void nf_tables_trans_destroy_flush_work(struct net *net) } EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work); -static bool nft_expr_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - return false; -} - static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain) { const struct nft_expr *expr, *last; - struct nft_regs_track track = {}; unsigned int size, data_size; void *data, *data_boundary; struct nft_rule_dp *prule; @@ -10227,15 +10159,7 @@ static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *cha return -ENOMEM; size = 0; - track.last = nft_expr_last(rule); nft_rule_for_each_expr(expr, last, rule) { - track.cur = expr; - - if (nft_expr_reduce(&track, expr)) { - expr = track.cur; - continue; - } - if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary)) return -ENOMEM; @@ -11860,7 +11784,6 @@ void nft_data_release(const struct nft_data *data, enum nft_data_types type) WARN_ON(1); } } -EXPORT_SYMBOL_GPL(nft_data_release); int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data, enum nft_data_types type, unsigned int len) @@ -11887,7 +11810,6 @@ int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data, nla_nest_end(skb, nest); return err; } -EXPORT_SYMBOL_GPL(nft_data_dump); static void __nft_release_hook(struct net *net, struct nft_table *table) { diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c index d550910aabec..a4ff781f334d 100644 --- a/net/netfilter/nft_bitwise.c +++ b/net/netfilter/nft_bitwise.c @@ -391,61 +391,12 @@ static int nft_bitwise_offload(struct nft_offload_ctx *ctx, return 0; } -static bool nft_bitwise_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_bitwise *priv = nft_expr_priv(expr); - const struct nft_bitwise *bitwise; - unsigned int regcount; - u8 dreg; - int i; - - if (!track->regs[priv->sreg].selector) - return false; - - bitwise = nft_expr_priv(track->regs[priv->dreg].selector); - if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector && - track->regs[priv->sreg].num_reg == 0 && - track->regs[priv->dreg].bitwise && - track->regs[priv->dreg].bitwise->ops == expr->ops && - priv->sreg == bitwise->sreg && - priv->sreg2 == bitwise->sreg2 && - priv->dreg == bitwise->dreg && - priv->op == bitwise->op && - priv->len == bitwise->len && - !memcmp(&priv->mask, &bitwise->mask, sizeof(priv->mask)) && - !memcmp(&priv->xor, &bitwise->xor, sizeof(priv->xor)) && - !memcmp(&priv->data, &bitwise->data, sizeof(priv->data))) { - track->cur = expr; - return true; - } - - if (track->regs[priv->sreg].bitwise || - track->regs[priv->sreg].num_reg != 0) { - nft_reg_track_cancel(track, priv->dreg, priv->len); - return false; - } - - if (priv->sreg != priv->dreg) { - nft_reg_track_update(track, track->regs[priv->sreg].selector, - priv->dreg, priv->len); - } - - dreg = priv->dreg; - regcount = DIV_ROUND_UP(priv->len, NFT_REG32_SIZE); - for (i = 0; i < regcount; i++, dreg++) - track->regs[dreg].bitwise = expr; - - return false; -} - static const struct nft_expr_ops nft_bitwise_ops = { .type = &nft_bitwise_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_bitwise)), .eval = nft_bitwise_eval, .init = nft_bitwise_init, .dump = nft_bitwise_dump, - .reduce = nft_bitwise_reduce, .offload = nft_bitwise_offload, }; @@ -548,48 +499,12 @@ static int nft_bitwise_fast_offload(struct nft_offload_ctx *ctx, return 0; } -static bool nft_bitwise_fast_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); - const struct nft_bitwise_fast_expr *bitwise; - - if (!track->regs[priv->sreg].selector) - return false; - - bitwise = nft_expr_priv(track->regs[priv->dreg].selector); - if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector && - track->regs[priv->dreg].bitwise && - track->regs[priv->dreg].bitwise->ops == expr->ops && - priv->sreg == bitwise->sreg && - priv->dreg == bitwise->dreg && - priv->mask == bitwise->mask && - priv->xor == bitwise->xor) { - track->cur = expr; - return true; - } - - if (track->regs[priv->sreg].bitwise) { - nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE); - return false; - } - - if (priv->sreg != priv->dreg) { - track->regs[priv->dreg].selector = - track->regs[priv->sreg].selector; - } - track->regs[priv->dreg].bitwise = expr; - - return false; -} - const struct nft_expr_ops nft_bitwise_fast_ops = { .type = &nft_bitwise_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_bitwise_fast_expr)), .eval = NULL, /* inlined */ .init = nft_bitwise_fast_init, .dump = nft_bitwise_fast_dump, - .reduce = nft_bitwise_fast_reduce, .offload = nft_bitwise_fast_offload, }; @@ -626,22 +541,3 @@ struct nft_expr_type nft_bitwise_type __read_mostly = { .maxattr = NFTA_BITWISE_MAX, .owner = THIS_MODULE, }; - -bool nft_expr_reduce_bitwise(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_expr *last = track->last; - const struct nft_expr *next; - - if (expr == last) - return false; - - next = nft_expr_next(expr); - if (next->ops == &nft_bitwise_ops) - return nft_bitwise_reduce(track, next); - else if (next->ops == &nft_bitwise_fast_ops) - return nft_bitwise_fast_reduce(track, next); - - return false; -} -EXPORT_SYMBOL_GPL(nft_expr_reduce_bitwise); diff --git a/net/netfilter/nft_byteorder.c b/net/netfilter/nft_byteorder.c index af9206a3afd1..744878773dac 100644 --- a/net/netfilter/nft_byteorder.c +++ b/net/netfilter/nft_byteorder.c @@ -170,23 +170,12 @@ static int nft_byteorder_dump(struct sk_buff *skb, return -1; } -static bool nft_byteorder_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - struct nft_byteorder *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, priv->len); - - return false; -} - static const struct nft_expr_ops nft_byteorder_ops = { .type = &nft_byteorder_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_byteorder)), .eval = nft_byteorder_eval, .init = nft_byteorder_init, .dump = nft_byteorder_dump, - .reduce = nft_byteorder_reduce, }; struct nft_expr_type nft_byteorder_type __read_mostly = { diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c index 2605f43737bc..b61dc9c3383e 100644 --- a/net/netfilter/nft_cmp.c +++ b/net/netfilter/nft_cmp.c @@ -190,7 +190,6 @@ static const struct nft_expr_ops nft_cmp_ops = { .eval = nft_cmp_eval, .init = nft_cmp_init, .dump = nft_cmp_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_cmp_offload, }; @@ -282,7 +281,6 @@ const struct nft_expr_ops nft_cmp_fast_ops = { .eval = NULL, /* inlined */ .init = nft_cmp_fast_init, .dump = nft_cmp_fast_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_cmp_fast_offload, }; @@ -376,7 +374,6 @@ const struct nft_expr_ops nft_cmp16_fast_ops = { .eval = NULL, /* inlined */ .init = nft_cmp16_fast_init, .dump = nft_cmp16_fast_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_cmp16_fast_offload, }; diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 08f620311b03..5021a01ba42c 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -778,14 +778,6 @@ static const struct nfnetlink_subsystem nfnl_compat_subsys = { static struct nft_expr_type nft_match_type; -static bool nft_match_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct xt_match *match = expr->ops->data; - - return strcmp(match->name, "comment") == 0; -} - static const struct nft_expr_ops * nft_match_select_ops(const struct nft_ctx *ctx, const struct nlattr * const tb[]) @@ -828,7 +820,6 @@ nft_match_select_ops(const struct nft_ctx *ctx, ops->dump = nft_match_dump; ops->validate = nft_match_validate; ops->data = match; - ops->reduce = nft_match_reduce; matchsize = NFT_EXPR_SIZE(XT_ALIGN(match->matchsize)); if (matchsize > NFT_MATCH_LARGE_THRESH) { @@ -918,7 +909,6 @@ nft_target_select_ops(const struct nft_ctx *ctx, ops->dump = nft_target_dump; ops->validate = nft_target_validate; ops->data = target; - ops->reduce = NFT_REDUCE_READONLY; if (family == NFPROTO_BRIDGE) ops->eval = nft_target_eval_bridge; diff --git a/net/netfilter/nft_connlimit.c b/net/netfilter/nft_connlimit.c index 657764774a2d..47d817983e81 100644 --- a/net/netfilter/nft_connlimit.c +++ b/net/netfilter/nft_connlimit.c @@ -258,7 +258,6 @@ static const struct nft_expr_ops nft_connlimit_ops = { .destroy_clone = nft_connlimit_destroy_clone, .dump = nft_connlimit_dump, .gc = nft_connlimit_gc, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_connlimit_type __read_mostly = { diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index 169ae93688bc..3fa6369790f4 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -313,7 +313,6 @@ static const struct nft_expr_ops nft_counter_ops = { .destroy_clone = nft_counter_destroy, .dump = nft_counter_dump, .clone = nft_counter_clone, - .reduce = NFT_REDUCE_READONLY, .offload = nft_counter_offload, .offload_stats = nft_counter_offload_stats, }; diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 6f2ae7cad731..b6abd5f8de92 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -698,29 +698,6 @@ static int nft_ct_get_dump(struct sk_buff *skb, return -1; } -static bool nft_ct_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_ct *priv = nft_expr_priv(expr); - const struct nft_ct *ct; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - ct = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != ct->key) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static int nft_ct_set_dump(struct sk_buff *skb, const struct nft_expr *expr, bool reset) { @@ -755,27 +732,8 @@ static const struct nft_expr_ops nft_ct_get_ops = { .init = nft_ct_get_init, .destroy = nft_ct_get_destroy, .dump = nft_ct_get_dump, - .reduce = nft_ct_get_reduce, }; -static bool nft_ct_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_ct_get_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - #ifdef CONFIG_MITIGATION_RETPOLINE static const struct nft_expr_ops nft_ct_get_fast_ops = { .type = &nft_ct_type, @@ -784,7 +742,6 @@ static const struct nft_expr_ops nft_ct_get_fast_ops = { .init = nft_ct_get_init, .destroy = nft_ct_get_destroy, .dump = nft_ct_get_dump, - .reduce = nft_ct_set_reduce, }; #endif @@ -795,7 +752,6 @@ static const struct nft_expr_ops nft_ct_set_ops = { .init = nft_ct_set_init, .destroy = nft_ct_set_destroy, .dump = nft_ct_set_dump, - .reduce = nft_ct_set_reduce, }; #ifdef CONFIG_NF_CONNTRACK_ZONES @@ -806,7 +762,6 @@ static const struct nft_expr_ops nft_ct_set_zone_ops = { .init = nft_ct_set_init, .destroy = nft_ct_set_destroy, .dump = nft_ct_set_dump, - .reduce = nft_ct_set_reduce, }; #endif @@ -876,7 +831,6 @@ static const struct nft_expr_ops nft_notrack_ops = { .type = &nft_notrack_type, .size = NFT_EXPR_SIZE(0), .eval = nft_notrack_eval, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_notrack_type __read_mostly = { diff --git a/net/netfilter/nft_dup_netdev.c b/net/netfilter/nft_dup_netdev.c index 0573f96ce079..06866799e946 100644 --- a/net/netfilter/nft_dup_netdev.c +++ b/net/netfilter/nft_dup_netdev.c @@ -80,7 +80,6 @@ static const struct nft_expr_ops nft_dup_netdev_ops = { .eval = nft_dup_netdev_eval, .init = nft_dup_netdev_init, .dump = nft_dup_netdev_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_dup_netdev_offload, .offload_action = nft_dup_netdev_offload_action, }; diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index 7807d8129664..6bff6287e7d5 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -421,7 +421,6 @@ static const struct nft_expr_ops nft_dynset_ops = { .activate = nft_dynset_activate, .deactivate = nft_dynset_deactivate, .dump = nft_dynset_dump, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_dynset_type __read_mostly = { diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c index 7eedf4e3ae9c..5f01269a49bd 100644 --- a/net/netfilter/nft_exthdr.c +++ b/net/netfilter/nft_exthdr.c @@ -702,40 +702,12 @@ static int nft_exthdr_dump_strip(struct sk_buff *skb, return nft_exthdr_dump_common(skb, priv); } -static bool nft_exthdr_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_exthdr *priv = nft_expr_priv(expr); - const struct nft_exthdr *exthdr; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - exthdr = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->type != exthdr->type || - priv->op != exthdr->op || - priv->flags != exthdr->flags || - priv->offset != exthdr->offset || - priv->len != exthdr->len) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static const struct nft_expr_ops nft_exthdr_ipv6_ops = { .type = &nft_exthdr_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)), .eval = nft_exthdr_ipv6_eval, .init = nft_exthdr_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; static const struct nft_expr_ops nft_exthdr_ipv4_ops = { @@ -744,7 +716,6 @@ static const struct nft_expr_ops nft_exthdr_ipv4_ops = { .eval = nft_exthdr_ipv4_eval, .init = nft_exthdr_ipv4_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; static const struct nft_expr_ops nft_exthdr_tcp_ops = { @@ -753,7 +724,6 @@ static const struct nft_expr_ops nft_exthdr_tcp_ops = { .eval = nft_exthdr_tcp_eval, .init = nft_exthdr_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; static const struct nft_expr_ops nft_exthdr_tcp_set_ops = { @@ -762,7 +732,6 @@ static const struct nft_expr_ops nft_exthdr_tcp_set_ops = { .eval = nft_exthdr_tcp_set_eval, .init = nft_exthdr_tcp_set_init, .dump = nft_exthdr_dump_set, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_exthdr_tcp_strip_ops = { @@ -771,7 +740,6 @@ static const struct nft_expr_ops nft_exthdr_tcp_strip_ops = { .eval = nft_exthdr_tcp_strip_eval, .init = nft_exthdr_tcp_strip_init, .dump = nft_exthdr_dump_strip, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_exthdr_sctp_ops = { @@ -780,7 +748,6 @@ static const struct nft_expr_ops nft_exthdr_sctp_ops = { .eval = nft_exthdr_sctp_eval, .init = nft_exthdr_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; #ifdef CONFIG_NFT_EXTHDR_DCCP @@ -790,7 +757,6 @@ static const struct nft_expr_ops nft_exthdr_dccp_ops = { .eval = nft_exthdr_dccp_eval, .init = nft_exthdr_dccp_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; #endif diff --git a/net/netfilter/nft_fib.c b/net/netfilter/nft_fib.c index 96e02a83c045..f7dc0e54375f 100644 --- a/net/netfilter/nft_fib.c +++ b/net/netfilter/nft_fib.c @@ -162,48 +162,6 @@ void nft_fib_store_result(void *reg, const struct nft_fib *priv, } EXPORT_SYMBOL_GPL(nft_fib_store_result); -bool nft_fib_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_fib *priv = nft_expr_priv(expr); - unsigned int len = NFT_REG32_SIZE; - const struct nft_fib *fib; - - switch (priv->result) { - case NFT_FIB_RESULT_OIF: - break; - case NFT_FIB_RESULT_OIFNAME: - if (priv->flags & NFTA_FIB_F_PRESENT) - len = NFT_REG32_SIZE; - else - len = IFNAMSIZ; - break; - case NFT_FIB_RESULT_ADDRTYPE: - break; - default: - WARN_ON_ONCE(1); - break; - } - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, len); - return false; - } - - fib = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->result != fib->result || - priv->flags != fib->flags) { - nft_reg_track_update(track, expr, priv->dreg, len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} -EXPORT_SYMBOL_GPL(nft_fib_reduce); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Query routing table from nftables"); MODULE_AUTHOR("Florian Westphal "); diff --git a/net/netfilter/nft_fib_inet.c b/net/netfilter/nft_fib_inet.c index 666a3741d20b..a88d44e163d1 100644 --- a/net/netfilter/nft_fib_inet.c +++ b/net/netfilter/nft_fib_inet.c @@ -49,7 +49,6 @@ static const struct nft_expr_ops nft_fib_inet_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static struct nft_expr_type nft_fib_inet_type __read_mostly = { diff --git a/net/netfilter/nft_fib_netdev.c b/net/netfilter/nft_fib_netdev.c index 9121ec64e918..3f3478abd845 100644 --- a/net/netfilter/nft_fib_netdev.c +++ b/net/netfilter/nft_fib_netdev.c @@ -58,7 +58,6 @@ static const struct nft_expr_ops nft_fib_netdev_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static struct nft_expr_type nft_fib_netdev_type __read_mostly = { diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c index 179d0e59e2b5..32b4281038dd 100644 --- a/net/netfilter/nft_flow_offload.c +++ b/net/netfilter/nft_flow_offload.c @@ -225,7 +225,6 @@ static const struct nft_expr_ops nft_flow_offload_ops = { .destroy = nft_flow_offload_destroy, .validate = nft_flow_offload_validate, .dump = nft_flow_offload_dump, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_flow_offload_type __read_mostly = { diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c index 152a9fb4d23a..ad48dcd45abe 100644 --- a/net/netfilter/nft_fwd_netdev.c +++ b/net/netfilter/nft_fwd_netdev.c @@ -218,7 +218,6 @@ static const struct nft_expr_ops nft_fwd_neigh_netdev_ops = { .init = nft_fwd_neigh_init, .dump = nft_fwd_neigh_dump, .validate = nft_fwd_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_fwd_netdev_ops = { @@ -228,7 +227,6 @@ static const struct nft_expr_ops nft_fwd_netdev_ops = { .init = nft_fwd_netdev_init, .dump = nft_fwd_netdev_dump, .validate = nft_fwd_validate, - .reduce = NFT_REDUCE_READONLY, .offload = nft_fwd_netdev_offload, .offload_action = nft_fwd_netdev_offload_action, }; diff --git a/net/netfilter/nft_hash.c b/net/netfilter/nft_hash.c index 5d034bbb6913..1cf41e0a0e0c 100644 --- a/net/netfilter/nft_hash.c +++ b/net/netfilter/nft_hash.c @@ -166,16 +166,6 @@ static int nft_jhash_dump(struct sk_buff *skb, return -1; } -static bool nft_jhash_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_jhash *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, sizeof(u32)); - - return false; -} - static int nft_symhash_dump(struct sk_buff *skb, const struct nft_expr *expr, bool reset) { @@ -196,30 +186,6 @@ static int nft_symhash_dump(struct sk_buff *skb, return -1; } -static bool nft_symhash_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - struct nft_symhash *priv = nft_expr_priv(expr); - struct nft_symhash *symhash; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, sizeof(u32)); - return false; - } - - symhash = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->offset != symhash->offset || - priv->modulus != symhash->modulus) { - nft_reg_track_update(track, expr, priv->dreg, sizeof(u32)); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} - static struct nft_expr_type nft_hash_type; static const struct nft_expr_ops nft_jhash_ops = { .type = &nft_hash_type, @@ -227,7 +193,6 @@ static const struct nft_expr_ops nft_jhash_ops = { .eval = nft_jhash_eval, .init = nft_jhash_init, .dump = nft_jhash_dump, - .reduce = nft_jhash_reduce, }; static const struct nft_expr_ops nft_symhash_ops = { @@ -236,7 +201,6 @@ static const struct nft_expr_ops nft_symhash_ops = { .eval = nft_symhash_eval, .init = nft_symhash_init, .dump = nft_symhash_dump, - .reduce = nft_symhash_reduce, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_immediate.c b/net/netfilter/nft_immediate.c index 02ee5fb69871..37c29947b380 100644 --- a/net/netfilter/nft_immediate.c +++ b/net/netfilter/nft_immediate.c @@ -320,17 +320,6 @@ static bool nft_immediate_offload_action(const struct nft_expr *expr) return false; } -static bool nft_immediate_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_immediate_expr *priv = nft_expr_priv(expr); - - if (priv->dreg != NFT_REG_VERDICT) - nft_reg_track_cancel(track, priv->dreg, priv->dlen); - - return false; -} - static const struct nft_expr_ops nft_imm_ops = { .type = &nft_imm_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_immediate_expr)), @@ -341,7 +330,6 @@ static const struct nft_expr_ops nft_imm_ops = { .destroy = nft_immediate_destroy, .dump = nft_immediate_dump, .validate = nft_immediate_validate, - .reduce = nft_immediate_reduce, .offload = nft_immediate_offload, .offload_action = nft_immediate_offload_action, }; diff --git a/net/netfilter/nft_last.c b/net/netfilter/nft_last.c index de1b6066bfa8..e845779268d3 100644 --- a/net/netfilter/nft_last.c +++ b/net/netfilter/nft_last.c @@ -125,7 +125,6 @@ static const struct nft_expr_ops nft_last_ops = { .destroy = nft_last_destroy, .clone = nft_last_clone, .dump = nft_last_dump, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_last_type __read_mostly = { diff --git a/net/netfilter/nft_limit.c b/net/netfilter/nft_limit.c index 21d26b79b460..0daeb0b23c20 100644 --- a/net/netfilter/nft_limit.c +++ b/net/netfilter/nft_limit.c @@ -243,7 +243,6 @@ static const struct nft_expr_ops nft_limit_pkts_ops = { .destroy = nft_limit_pkts_destroy, .clone = nft_limit_pkts_clone, .dump = nft_limit_pkts_dump, - .reduce = NFT_REDUCE_READONLY, }; static void nft_limit_bytes_eval(const struct nft_expr *expr, @@ -299,7 +298,6 @@ static const struct nft_expr_ops nft_limit_bytes_ops = { .dump = nft_limit_bytes_dump, .clone = nft_limit_bytes_clone, .destroy = nft_limit_bytes_destroy, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_log.c b/net/netfilter/nft_log.c index bf01cf8a8907..da0c0d1c9cea 100644 --- a/net/netfilter/nft_log.c +++ b/net/netfilter/nft_log.c @@ -235,7 +235,6 @@ static const struct nft_expr_ops nft_log_ops = { .init = nft_log_init, .destroy = nft_log_destroy, .dump = nft_log_dump, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_log_type __read_mostly = { diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index fc2d7c5d83c8..e4e619027542 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -266,17 +266,6 @@ static int nft_lookup_validate(const struct nft_ctx *ctx, return 0; } -static bool nft_lookup_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_lookup *priv = nft_expr_priv(expr); - - if (priv->set->flags & NFT_SET_MAP) - nft_reg_track_cancel(track, priv->dreg, priv->set->dlen); - - return false; -} - static const struct nft_expr_ops nft_lookup_ops = { .type = &nft_lookup_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_lookup)), @@ -287,7 +276,6 @@ static const struct nft_expr_ops nft_lookup_ops = { .destroy = nft_lookup_destroy, .dump = nft_lookup_dump, .validate = nft_lookup_validate, - .reduce = nft_lookup_reduce, }; struct nft_expr_type nft_lookup_type __read_mostly = { diff --git a/net/netfilter/nft_masq.c b/net/netfilter/nft_masq.c index 868bd4d73555..2b01128737a3 100644 --- a/net/netfilter/nft_masq.c +++ b/net/netfilter/nft_masq.c @@ -143,7 +143,6 @@ static const struct nft_expr_ops nft_masq_ipv4_ops = { .destroy = nft_masq_ipv4_destroy, .dump = nft_masq_dump, .validate = nft_masq_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_masq_ipv4_type __read_mostly = { @@ -171,7 +170,6 @@ static const struct nft_expr_ops nft_masq_ipv6_ops = { .destroy = nft_masq_ipv6_destroy, .dump = nft_masq_dump, .validate = nft_masq_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_masq_ipv6_type __read_mostly = { @@ -213,7 +211,6 @@ static const struct nft_expr_ops nft_masq_inet_ops = { .destroy = nft_masq_inet_destroy, .dump = nft_masq_dump, .validate = nft_masq_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_masq_inet_type __read_mostly = { diff --git a/net/netfilter/nft_meta.c b/net/netfilter/nft_meta.c index 05cd1e6e6a2f..983158274c68 100644 --- a/net/netfilter/nft_meta.c +++ b/net/netfilter/nft_meta.c @@ -742,60 +742,16 @@ static int nft_meta_get_offload(struct nft_offload_ctx *ctx, return 0; } -bool nft_meta_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_meta *priv = nft_expr_priv(expr); - const struct nft_meta *meta; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - meta = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != meta->key || - priv->dreg != meta->dreg) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} -EXPORT_SYMBOL_GPL(nft_meta_get_reduce); - static const struct nft_expr_ops nft_meta_get_ops = { .type = &nft_meta_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_meta)), .eval = nft_meta_get_eval, .init = nft_meta_get_init, .dump = nft_meta_get_dump, - .reduce = nft_meta_get_reduce, .validate = nft_meta_get_validate, .offload = nft_meta_get_offload, }; -static bool nft_meta_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_meta_get_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - static const struct nft_expr_ops nft_meta_set_ops = { .type = &nft_meta_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_meta)), @@ -803,7 +759,6 @@ static const struct nft_expr_ops nft_meta_set_ops = { .init = nft_meta_set_init, .destroy = nft_meta_set_destroy, .dump = nft_meta_set_dump, - .reduce = nft_meta_set_reduce, .validate = nft_meta_set_validate, }; diff --git a/net/netfilter/nft_nat.c b/net/netfilter/nft_nat.c index 6e21f72c5b57..e32cd9fbc7c2 100644 --- a/net/netfilter/nft_nat.c +++ b/net/netfilter/nft_nat.c @@ -320,7 +320,6 @@ static const struct nft_expr_ops nft_nat_ops = { .destroy = nft_nat_destroy, .dump = nft_nat_dump, .validate = nft_nat_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_nat_type __read_mostly = { @@ -351,7 +350,6 @@ static const struct nft_expr_ops nft_nat_inet_ops = { .destroy = nft_nat_destroy, .dump = nft_nat_dump, .validate = nft_nat_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_inet_nat_type __read_mostly = { diff --git a/net/netfilter/nft_numgen.c b/net/netfilter/nft_numgen.c index bd058babfc82..06e87dfd76e7 100644 --- a/net/netfilter/nft_numgen.c +++ b/net/netfilter/nft_numgen.c @@ -84,16 +84,6 @@ static int nft_ng_inc_init(const struct nft_ctx *ctx, return err; } -static bool nft_ng_inc_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_ng_inc *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE); - - return false; -} - static int nft_ng_dump(struct sk_buff *skb, enum nft_registers dreg, u32 modulus, enum nft_ng_types type, u32 offset) { @@ -178,16 +168,6 @@ static int nft_ng_random_dump(struct sk_buff *skb, priv->offset); } -static bool nft_ng_random_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_ng_random *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE); - - return false; -} - static struct nft_expr_type nft_ng_type; static const struct nft_expr_ops nft_ng_inc_ops = { .type = &nft_ng_type, @@ -196,7 +176,6 @@ static const struct nft_expr_ops nft_ng_inc_ops = { .init = nft_ng_inc_init, .destroy = nft_ng_inc_destroy, .dump = nft_ng_inc_dump, - .reduce = nft_ng_inc_reduce, }; static const struct nft_expr_ops nft_ng_random_ops = { @@ -205,7 +184,6 @@ static const struct nft_expr_ops nft_ng_random_ops = { .eval = nft_ng_random_eval, .init = nft_ng_random_init, .dump = nft_ng_random_dump, - .reduce = nft_ng_random_reduce, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_objref.c b/net/netfilter/nft_objref.c index 1a62e384766a..633cce69568f 100644 --- a/net/netfilter/nft_objref.c +++ b/net/netfilter/nft_objref.c @@ -123,7 +123,6 @@ static const struct nft_expr_ops nft_objref_ops = { .deactivate = nft_objref_deactivate, .dump = nft_objref_dump, .validate = nft_objref_validate, - .reduce = NFT_REDUCE_READONLY, }; struct nft_objref_map { @@ -245,7 +244,6 @@ static const struct nft_expr_ops nft_objref_map_ops = { .destroy = nft_objref_map_destroy, .dump = nft_objref_map_dump, .validate = nft_objref_map_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c index 1c0b493ef0a9..39ccd67ed265 100644 --- a/net/netfilter/nft_osf.c +++ b/net/netfilter/nft_osf.c @@ -127,30 +127,6 @@ static int nft_osf_validate(const struct nft_ctx *ctx, return nft_chain_validate_hooks(ctx->chain, hooks); } -static bool nft_osf_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - struct nft_osf *priv = nft_expr_priv(expr); - struct nft_osf *osf; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, NFT_OSF_MAXGENRELEN); - return false; - } - - osf = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->flags != osf->flags || - priv->ttl != osf->ttl) { - nft_reg_track_update(track, expr, priv->dreg, NFT_OSF_MAXGENRELEN); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} - static struct nft_expr_type nft_osf_type; static const struct nft_expr_ops nft_osf_op = { .eval = nft_osf_eval, @@ -159,7 +135,6 @@ static const struct nft_expr_ops nft_osf_op = { .dump = nft_osf_dump, .type = &nft_osf_type, .validate = nft_osf_validate, - .reduce = nft_osf_reduce, }; static struct nft_expr_type nft_osf_type __read_mostly = { diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index b0214418f75a..973d56af03ff 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -250,31 +250,6 @@ static int nft_payload_dump(struct sk_buff *skb, return -1; } -static bool nft_payload_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_payload *priv = nft_expr_priv(expr); - const struct nft_payload *payload; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - payload = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->base != payload->base || - priv->offset != payload->offset || - priv->len != payload->len) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static bool nft_payload_offload_mask(struct nft_offload_reg *reg, u32 priv_len, u32 field_len) { @@ -578,7 +553,6 @@ static const struct nft_expr_ops nft_payload_ops = { .eval = nft_payload_eval, .init = nft_payload_init, .dump = nft_payload_dump, - .reduce = nft_payload_reduce, .offload = nft_payload_offload, }; @@ -588,7 +562,6 @@ const struct nft_expr_ops nft_payload_fast_ops = { .eval = nft_payload_eval, .init = nft_payload_init, .dump = nft_payload_dump, - .reduce = nft_payload_reduce, .offload = nft_payload_offload, }; @@ -1012,32 +985,12 @@ static int nft_payload_set_dump(struct sk_buff *skb, return -1; } -static bool nft_payload_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_payload_ops && - track->regs[i].selector->ops != &nft_payload_fast_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - static const struct nft_expr_ops nft_payload_set_ops = { .type = &nft_payload_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)), .eval = nft_payload_set_eval, .init = nft_payload_set_init, .dump = nft_payload_set_dump, - .reduce = nft_payload_set_reduce, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_queue.c b/net/netfilter/nft_queue.c index 344fe311878f..8eb13a02942e 100644 --- a/net/netfilter/nft_queue.c +++ b/net/netfilter/nft_queue.c @@ -191,7 +191,6 @@ static const struct nft_expr_ops nft_queue_ops = { .init = nft_queue_init, .dump = nft_queue_dump, .validate = nft_queue_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_queue_sreg_ops = { @@ -201,7 +200,6 @@ static const struct nft_expr_ops nft_queue_sreg_ops = { .init = nft_queue_sreg_init, .dump = nft_queue_sreg_dump, .validate = nft_queue_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_quota.c b/net/netfilter/nft_quota.c index cb6c0e04ff67..bb3cf3d16e79 100644 --- a/net/netfilter/nft_quota.c +++ b/net/netfilter/nft_quota.c @@ -266,7 +266,6 @@ static const struct nft_expr_ops nft_quota_ops = { .destroy = nft_quota_destroy, .clone = nft_quota_clone, .dump = nft_quota_dump, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_quota_type __read_mostly = { diff --git a/net/netfilter/nft_range.c b/net/netfilter/nft_range.c index ea382f7bbd78..cbb02644b836 100644 --- a/net/netfilter/nft_range.c +++ b/net/netfilter/nft_range.c @@ -138,7 +138,6 @@ static const struct nft_expr_ops nft_range_ops = { .eval = nft_range_eval, .init = nft_range_init, .dump = nft_range_dump, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_range_type __read_mostly = { diff --git a/net/netfilter/nft_redir.c b/net/netfilter/nft_redir.c index 95eedad85c83..58ae802db8f5 100644 --- a/net/netfilter/nft_redir.c +++ b/net/netfilter/nft_redir.c @@ -146,7 +146,6 @@ static const struct nft_expr_ops nft_redir_ipv4_ops = { .destroy = nft_redir_ipv4_destroy, .dump = nft_redir_dump, .validate = nft_redir_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_redir_ipv4_type __read_mostly = { @@ -174,7 +173,6 @@ static const struct nft_expr_ops nft_redir_ipv6_ops = { .destroy = nft_redir_ipv6_destroy, .dump = nft_redir_dump, .validate = nft_redir_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_redir_ipv6_type __read_mostly = { @@ -203,7 +201,6 @@ static const struct nft_expr_ops nft_redir_inet_ops = { .destroy = nft_redir_inet_destroy, .dump = nft_redir_dump, .validate = nft_redir_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_redir_inet_type __read_mostly = { diff --git a/net/netfilter/nft_reject_inet.c b/net/netfilter/nft_reject_inet.c index 49020e67304a..dcae83ddc32e 100644 --- a/net/netfilter/nft_reject_inet.c +++ b/net/netfilter/nft_reject_inet.c @@ -79,7 +79,6 @@ static const struct nft_expr_ops nft_reject_inet_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_inet_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_inet_type __read_mostly = { diff --git a/net/netfilter/nft_reject_netdev.c b/net/netfilter/nft_reject_netdev.c index 2558ce1505d9..b53e81e4ca75 100644 --- a/net/netfilter/nft_reject_netdev.c +++ b/net/netfilter/nft_reject_netdev.c @@ -158,7 +158,6 @@ static const struct nft_expr_ops nft_reject_netdev_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_netdev_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_netdev_type __read_mostly = { diff --git a/net/netfilter/nft_rt.c b/net/netfilter/nft_rt.c index dc50b9a5bd68..ad527f3596c0 100644 --- a/net/netfilter/nft_rt.c +++ b/net/netfilter/nft_rt.c @@ -195,7 +195,6 @@ static const struct nft_expr_ops nft_rt_get_ops = { .init = nft_rt_get_init, .dump = nft_rt_get_dump, .validate = nft_rt_validate, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_rt_type __read_mostly = { diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index 644d4b916705..ef567a948703 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -684,9 +684,9 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, cond_resched(); - write_lock_bh(&priv->lock); + write_lock(&priv->lock); err = __nft_rbtree_insert(net, set, rbe, elem_priv, tstamp, last); - write_unlock_bh(&priv->lock); + write_unlock(&priv->lock); if (nft_rbtree_interval_end(rbe)) priv->start_rbe_cookie = 0; @@ -696,13 +696,6 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, return err; } -static void nft_rbtree_erase(struct nft_rbtree *priv, struct nft_rbtree_elem *rbe) -{ - write_lock_bh(&priv->lock); - rb_erase(&rbe->node, &priv->root); - write_unlock_bh(&priv->lock); -} - static void nft_rbtree_remove(const struct net *net, const struct nft_set *set, struct nft_elem_priv *elem_priv) @@ -710,7 +703,9 @@ static void nft_rbtree_remove(const struct net *net, struct nft_rbtree_elem *rbe = nft_elem_priv_cast(elem_priv); struct nft_rbtree *priv = nft_set_priv(set); - nft_rbtree_erase(priv, rbe); + write_lock(&priv->lock); + rb_erase(&rbe->node, &priv->root); + write_unlock(&priv->lock); } static void nft_rbtree_activate(const struct net *net, @@ -871,9 +866,9 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, nft_rbtree_do_walk(ctx, set, iter); break; case NFT_ITER_READ: - read_lock_bh(&priv->lock); + read_lock(&priv->lock); nft_rbtree_do_walk(ctx, set, iter); - read_unlock_bh(&priv->lock); + read_unlock(&priv->lock); break; default: iter->err = -EINVAL; @@ -909,14 +904,14 @@ static void nft_rbtree_gc_scan(struct nft_set *set) /* end element needs to be removed first, it has * no timeout extension. */ - write_lock_bh(&priv->lock); + write_lock(&priv->lock); if (rbe_end) { nft_rbtree_gc_elem_move(net, set, priv, rbe_end); rbe_end = NULL; } nft_rbtree_gc_elem_move(net, set, priv, rbe); - write_unlock_bh(&priv->lock); + write_unlock(&priv->lock); } priv->last_gc = jiffies; diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c index 36affbb697c2..c55a1310226a 100644 --- a/net/netfilter/nft_socket.c +++ b/net/netfilter/nft_socket.c @@ -249,31 +249,6 @@ static int nft_socket_dump(struct sk_buff *skb, return 0; } -static bool nft_socket_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_socket *priv = nft_expr_priv(expr); - const struct nft_socket *socket; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - socket = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != socket->key || - priv->dreg != socket->dreg || - priv->level != socket->level) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static int nft_socket_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { @@ -296,7 +271,6 @@ static const struct nft_expr_ops nft_socket_ops = { .init = nft_socket_init, .dump = nft_socket_dump, .validate = nft_socket_validate, - .reduce = nft_socket_reduce, }; static struct nft_expr_type nft_socket_type __read_mostly = { diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index b71ef18b0e8c..8e452a874969 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -292,7 +292,6 @@ static const struct nft_expr_ops nft_synproxy_ops = { .dump = nft_synproxy_dump, .type = &nft_synproxy_type, .validate = nft_synproxy_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_synproxy_type __read_mostly = { diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c index 50481280abd2..f2101af8c867 100644 --- a/net/netfilter/nft_tproxy.c +++ b/net/netfilter/nft_tproxy.c @@ -331,7 +331,6 @@ static const struct nft_expr_ops nft_tproxy_ops = { .init = nft_tproxy_init, .destroy = nft_tproxy_destroy, .dump = nft_tproxy_dump, - .reduce = NFT_REDUCE_READONLY, .validate = nft_tproxy_validate, }; diff --git a/net/netfilter/nft_tunnel.c b/net/netfilter/nft_tunnel.c index a12486ae089d..f5cadba91417 100644 --- a/net/netfilter/nft_tunnel.c +++ b/net/netfilter/nft_tunnel.c @@ -124,31 +124,6 @@ static int nft_tunnel_get_dump(struct sk_buff *skb, return -1; } -static bool nft_tunnel_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_tunnel *priv = nft_expr_priv(expr); - const struct nft_tunnel *tunnel; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - tunnel = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != tunnel->key || - priv->dreg != tunnel->dreg || - priv->mode != tunnel->mode) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} - static struct nft_expr_type nft_tunnel_type; static const struct nft_expr_ops nft_tunnel_get_ops = { .type = &nft_tunnel_type, @@ -156,7 +131,6 @@ static const struct nft_expr_ops nft_tunnel_get_ops = { .eval = nft_tunnel_get_eval, .init = nft_tunnel_get_init, .dump = nft_tunnel_get_dump, - .reduce = nft_tunnel_get_reduce, }; static struct nft_expr_type nft_tunnel_type __read_mostly = { diff --git a/net/netfilter/nft_xfrm.c b/net/netfilter/nft_xfrm.c index 3210cfc966ab..7ffe6a2690d1 100644 --- a/net/netfilter/nft_xfrm.c +++ b/net/netfilter/nft_xfrm.c @@ -259,32 +259,6 @@ static int nft_xfrm_validate(const struct nft_ctx *ctx, const struct nft_expr *e return nft_chain_validate_hooks(ctx->chain, hooks); } -static bool nft_xfrm_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_xfrm *priv = nft_expr_priv(expr); - const struct nft_xfrm *xfrm; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - xfrm = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != xfrm->key || - priv->dreg != xfrm->dreg || - priv->dir != xfrm->dir || - priv->spnum != xfrm->spnum) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static struct nft_expr_type nft_xfrm_type; static const struct nft_expr_ops nft_xfrm_get_ops = { .type = &nft_xfrm_type, @@ -293,7 +267,6 @@ static const struct nft_expr_ops nft_xfrm_get_ops = { .init = nft_xfrm_get_init, .dump = nft_xfrm_get_dump, .validate = nft_xfrm_validate, - .reduce = nft_xfrm_reduce, }; static struct nft_expr_type nft_xfrm_type __read_mostly = {