NFSv4.1: do not use deviceids after MDS clientid invalidation

Mark all deviceids established under an expired MDS clientid as invalid.
Stop all new i/o through DS and send through the MDS.
Don't use any new LAYOUTGETs that use the invalid deviceid. Purge all layouts
established under the expired MDS clientid.
Remove the MDS clientid deviceid and data servers reference

Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 51c19093022ed342045b1b64b39b4740166a8698..af9bf9eed4ca463fce37dace1f59b846c4f63ed0 100644 (file)
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -334,6 +334,9 @@ filelayout_read_pagelist(struct nfs_read_data *data)
                __func__, data->inode->i_ino,
                data->args.pgbase, (size_t)data->args.count, offset);
 
+       if (test_bit(NFS_DEVICEID_INVALID, &FILELAYOUT_DEVID_NODE(lseg)->flags))
+               return PNFS_NOT_ATTEMPTED;
+
        /* Retrieve the correct rpc_client for the byte range */
        j = nfs4_fl_calc_j_index(lseg, offset);
        idx = nfs4_fl_calc_ds_index(lseg, j);
@@ -373,6 +376,9 @@ filelayout_write_pagelist(struct nfs_write_data *data, int sync)
        struct nfs_fh *fh;
        int status;
 
+       if (test_bit(NFS_DEVICEID_INVALID, &FILELAYOUT_DEVID_NODE(lseg)->flags))
+               return PNFS_NOT_ATTEMPTED;
+
        /* Retrieve the correct rpc_client for the byte range */
        j = nfs4_fl_calc_j_index(lseg, offset);
        idx = nfs4_fl_calc_ds_index(lseg, j);
@@ -456,6 +462,10 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo,
                        goto out;
        } else
                dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
+       /* Found deviceid is being reaped */
+       if (test_bit(NFS_DEVICEID_INVALID, &dsaddr->id_node.flags))
+                       goto out_put;
+
        fl->dsaddr = dsaddr;
 
        if (fl->first_stripe_index < 0 ||

diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index 68cce730b800e0269b3f8ee021c124b2c3d97779..2e42284253fa600ba9266afc6111711653558949 100644 (file)
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -96,6 +96,12 @@ FILELAYOUT_LSEG(struct pnfs_layout_segment *lseg)
                            generic_hdr);
 }
 
+static inline struct nfs4_deviceid_node *
+FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
+{
+       return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
+}
+
 extern struct nfs_fh *
 nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);
 

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 5fc2e5d755a5735597aad3aa48dccdcddd791dae..5b3cc3f4bb391520da6186039887bdb158915c57 100644 (file)
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -452,6 +452,9 @@ pnfs_destroy_all_layouts(struct nfs_client *clp)
        struct pnfs_layout_hdr *lo;
        LIST_HEAD(tmp_list);
 
+       nfs4_deviceid_mark_client_invalid(clp);
+       nfs4_deviceid_purge_client(clp);
+
        spin_lock(&clp->cl_lock);
        rcu_read_lock();
        list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {

diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 678c4c7b14d971523df852272d27d751fb45e872..a59736eae6ecc54f9c7b4ee5f52f34367bbb2ea7 100644 (file)
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -192,12 +192,20 @@ struct pnfs_layout_segment *pnfs_update_layout(struct inode *ino,
                                               enum pnfs_iomode iomode,
                                               gfp_t gfp_flags);
 
+void nfs4_deviceid_mark_client_invalid(struct nfs_client *clp);
+
+/* nfs4_deviceid_flags */
+enum {
+       NFS_DEVICEID_INVALID = 0,       /* set when MDS clientid recalled */
+};
+
 /* pnfs_dev.c */
 struct nfs4_deviceid_node {
        struct hlist_node               node;
        struct hlist_node               tmpnode;
        const struct pnfs_layoutdriver_type *ld;
        const struct nfs_client         *nfs_client;
+       unsigned long                   flags;
        struct nfs4_deviceid            deviceid;
        atomic_t                        ref;
 };

diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
index fb9498d91f6aef456c070a34b7032bd37e84e7de..6fda5228ef5627d45da3cdf4933e6a098b018820 100644 (file)
--- a/fs/nfs/pnfs_dev.c
+++ b/fs/nfs/pnfs_dev.c
@@ -156,6 +156,7 @@ nfs4_init_deviceid_node(struct nfs4_deviceid_node *d,
        INIT_HLIST_NODE(&d->tmpnode);
        d->ld = ld;
        d->nfs_client = nfs_client;
+       d->flags = 0;
        d->deviceid = *id;
        atomic_set(&d->ref, 1);
 }
@@ -253,3 +254,22 @@ nfs4_deviceid_purge_client(const struct nfs_client *clp)
        for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++)
                _deviceid_purge_client(clp, h);
 }
+
+/*
+ * Stop use of all deviceids associated with an nfs_client
+ */
+void
+nfs4_deviceid_mark_client_invalid(struct nfs_client *clp)
+{
+       struct nfs4_deviceid_node *d;
+       struct hlist_node *n;
+       int i;
+
+       rcu_read_lock();
+       for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE; i ++){
+               hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[i], node)
+                       if (d->nfs_client == clp)
+                               set_bit(NFS_DEVICEID_INVALID, &d->flags);
+       }
+       rcu_read_unlock();
+}