diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 23923cc4aa47..0ce7da20aba3 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -248,6 +248,9 @@ struct nfsd_net { /* courtesy scan requests the reaper has not retired yet */ atomic_long_t nfsd_shrink_backlog; + /* delegation scan requests the reaper has not retired yet */ + atomic_long_t nfsd_deleg_backlog; + /* when deleg_reaper() last swept the client list */ time64_t nfsd_last_recall_any; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 48ffbee3419c..88f6c0a0d8b3 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -93,7 +93,7 @@ static void nfs4_free_ol_stateid(struct nfs4_stid *stid); static void nfsd4_end_grace(struct nfsd_net *nn); static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps); static void nfsd4_file_hash_remove(struct nfs4_file *fi); -static void deleg_reaper(struct nfsd_net *nn); +static void deleg_reaper(struct nfsd_net *nn, unsigned long backlog); static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) __releases(&s->sc_client->cl_lock); @@ -5603,7 +5603,7 @@ nfsd4_deleg_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) { struct nfsd_net *nn = shrink->private_data; time64_t elapsed; - long count; + long backlog, count; count = atomic_long_read(&nn->nfsd_delegations); if (!count) @@ -5620,8 +5620,14 @@ nfsd4_deleg_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) if (elapsed < NFSD_RECALL_ANY_COOLDOWN_SECS) return 0; - queue_work(laundry_wq, &nn->nfsd_deleg_work); - return count; + /* + * Unlike the courtesy shrinker, this one queues no work. + * Nothing is recalled until a scan request arrives. Subtract + * the requests already recorded, or concurrent reclaimers + * each see the whole namespace and stack a scan on top of it. + */ + backlog = atomic_long_read(&nn->nfsd_deleg_backlog); + return count > backlog ? count - backlog : 0; } static unsigned long @@ -5645,6 +5651,7 @@ nfsd4_deleg_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { struct nfsd_net *nn = shrink->private_data; + atomic_long_add(sc->nr_to_scan, &nn->nfsd_deleg_backlog); queue_work(laundry_wq, &nn->nfsd_deleg_work); /* @@ -7894,6 +7901,7 @@ nfs4_laundromat(struct nfsd_net *nn) struct nfs4_cpntf_state *cps; struct nfs4_client *clp; copy_stateid_t *cps_t; + long held, host, n; int i; if (clients_still_reclaiming(nn)) { @@ -8007,8 +8015,22 @@ nfs4_laundromat(struct nfsd_net *nn) /* service the server-to-server copy delayed unmount list */ nfsd4_ssc_expire_umount(nn); #endif - if (atomic_long_read(&num_delegations) >= max_delegations) - deleg_reaper(nn); + /* + * set_max_delegations() computes a zero max_delegations on a + * server with very little memory. @host is a divisor below. + */ + host = atomic_long_read(&num_delegations); + if (host && host >= max_delegations) { + /* + * max_delegations bounds the host, but the laundromat + * runs once per network namespace. Requesting the whole + * overage in each would multiply the request, so take + * only this namespace's share. + */ + held = atomic_long_read(&nn->nfsd_delegations); + n = host - max_delegations + 1; + deleg_reaper(nn, DIV64_U64_ROUND_UP((u64)n * held, host)); + } out: return max_t(time64_t, lt.new_timeo, NFSD_LAUNDROMAT_MINTIMEOUT); } @@ -8036,27 +8058,58 @@ courtesy_client_reaper(struct nfsd_net *nn) nfs4_process_client_reaplist(&reaplist); } +/* The two passes in deleg_reaper() must agree on which clients are asked. */ +static bool +deleg_reaper_eligible(const struct nfs4_client *clp, time64_t now) +{ + if (clp->cl_minorversion == 0) + return false; + if (clp->cl_state != NFSD4_ACTIVE) + return false; + if (atomic_read(&clp->cl_delegs_in_recall)) + return false; + if (test_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) + return false; + if (now - clp->cl_ra_time < NFSD_RECALL_ANY_COOLDOWN_SECS) + return false; + if (clp->cl_cb_state != NFSD4_CB_UP) + return false; + return true; +} + static void -deleg_reaper(struct nfsd_net *nn) +deleg_reaper(struct nfsd_net *nn, unsigned long backlog) { struct list_head *pos, *next; struct nfs4_client *clp; + unsigned long remaining, share, total; unsigned int count; + time64_t now; + + /* + * Recalling a delegation before it is needed costs the client + * an OPEN when it next touches the file. Leave + * nfsd_last_recall_any unstamped so the next sweep is not + * delayed. + */ + if (!backlog) + return; + now = ktime_get_boottime_seconds(); spin_lock(&nn->client_lock); - list_for_each_safe(pos, next, &nn->client_lru) { + + /* + * Only the clients this sweep asks contribute to the + * apportionment. Dividing the request among holders that are + * skipped under-serves it, and the shortfall goes nowhere: + * nfsd4_deleg_shrinker_worker() has already cleared + * nfsd_deleg_backlog. + */ + total = 0; + list_for_each(pos, &nn->client_lru) { clp = list_entry(pos, struct nfs4_client, cl_lru); - if (clp->cl_minorversion == 0) - continue; - if (clp->cl_state != NFSD4_ACTIVE) - continue; - if (atomic_read(&clp->cl_delegs_in_recall)) - continue; - if (ktime_get_boottime_seconds() - clp->cl_ra_time < - NFSD_RECALL_ANY_COOLDOWN_SECS) - continue; - if (clp->cl_cb_state != NFSD4_CB_UP) + if (!deleg_reaper_eligible(clp, now)) continue; /* * This read races with hash_delegation_locked() and @@ -8064,6 +8117,25 @@ deleg_reaper(struct nfsd_net *nn) * count only skews the keep value; the next * laundromat pass sees a more current one. */ + total += data_race(READ_ONCE(clp->cl_deleg_count)); + } + if (!total) + goto out; + + /* + * Reclaim asks in batches and is not bound by what the count + * callback reported, so the backlog can exceed what these + * clients hold. Cap it to keep each share within the client's + * own count. + */ + backlog = min(backlog, total); + remaining = backlog; + + list_for_each_safe(pos, next, &nn->client_lru) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + + if (!deleg_reaper_eligible(clp, now)) + continue; count = data_race(READ_ONCE(clp->cl_deleg_count)); if (!count) continue; @@ -8072,26 +8144,34 @@ deleg_reaper(struct nfsd_net *nn) /* release in nfsd4_cb_recall_any_release */ kref_get(&clp->cl_nfsdfs.cl_ref); - clp->cl_ra_time = ktime_get_boottime_seconds(); + clp->cl_ra_time = now; /* - * Ask for a single delegation. Recalling one before it - * is needed costs the client an OPEN when it next - * touches the file. + * Rounding up guarantees every holder gives up at least + * one. The round-up can overshoot @backlog, so stop + * once the request is met. client_lru is ordered by + * last renewal, so the least active clients are asked + * first. */ - clp->cl_ra->ra_keep = count - 1; + share = DIV64_U64_ROUND_UP((u64)backlog * count, total); + share = min(share, remaining); + remaining -= share; + clp->cl_ra->ra_keep = count - share; clp->cl_ra->ra_bmval[0] = BIT(RCA4_TYPE_MASK_RDATA_DLG) | BIT(RCA4_TYPE_MASK_WDATA_DLG) | BIT(RCA4_TYPE_MASK_DIR_DLG); trace_nfsd_cb_recall_any(clp->cl_ra); nfsd4_run_cb(&clp->cl_ra->ra_cb); + if (!remaining) + break; } +out: spin_unlock(&nn->client_lock); /* * Stamp the sweep even when no recall went out. A sweep that * found nothing eligible finds nothing on an immediate retry. */ - WRITE_ONCE(nn->nfsd_last_recall_any, ktime_get_boottime_seconds()); + WRITE_ONCE(nn->nfsd_last_recall_any, now); } static void @@ -8117,7 +8197,7 @@ nfsd4_deleg_shrinker_worker(struct work_struct *work) struct nfsd_net *nn = container_of(work, struct nfsd_net, nfsd_deleg_work); - deleg_reaper(nn); + deleg_reaper(nn, atomic_long_xchg(&nn->nfsd_deleg_backlog, 0)); } static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp) @@ -10082,6 +10162,7 @@ static int nfs4_state_create_net(struct net *net) INIT_WORK(&nn->nfsd_courtesy_work, nfsd4_courtesy_shrinker_worker); INIT_WORK(&nn->nfsd_deleg_work, nfsd4_deleg_shrinker_worker); atomic_long_set(&nn->nfsd_shrink_backlog, 0); + atomic_long_set(&nn->nfsd_deleg_backlog, 0); nn->nfsd_last_recall_any = 0; get_net(net);