NFS/flexfiles: Send LAYOUTERROR when failing over mirrored reads

When a read to the preferred mirror returns an error, the flexfiles
driver records the error in the inode list and currently marks the
layout for return before failing over the attempted read to the next
mirror.
What we actually want to do is fire off a LAYOUTERROR to notify the
MDS that there is an issue with the preferred mirror, then we fail
over. Only once we've failed to read from all mirrors should we
return the layout.

Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index a8e9bdd978e74f5ea16a0618eb561f4cba95c67f..eb86da30d4b922d6104479f188f0c6f8d056a67a 100644 (file)
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -1252,7 +1252,7 @@ static int ff_layout_read_done_cb(struct rpc_task *task,
                if (ff_layout_choose_best_ds_for_read(hdr->lseg,
                                        hdr->pgio_mirror_idx + 1,
                                        &hdr->pgio_mirror_idx))
-                       goto out_eagain;
+                       goto out_layouterror;
                set_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
                return task->tk_status;
        case -NFS4ERR_RESET_TO_MDS:
@@ -1263,6 +1263,8 @@ static int ff_layout_read_done_cb(struct rpc_task *task,
        }
 
        return 0;
+out_layouterror:
+       ff_layout_send_layouterror(hdr->lseg);
 out_eagain:
        rpc_restart_call_prepare(task);
        return -EAGAIN;
@@ -1412,9 +1414,10 @@ static void ff_layout_read_release(void *data)
        struct nfs_pgio_header *hdr = data;
 
        ff_layout_read_record_layoutstats_done(&hdr->task, hdr);
-       if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags))
+       if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags)) {
+               ff_layout_send_layouterror(hdr->lseg);
                pnfs_read_resend_pnfs(hdr);
-       else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags))
+       } else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags))
                ff_layout_reset_read(hdr);
        pnfs_generic_rw_release(data);
 }
@@ -1586,9 +1589,10 @@ static void ff_layout_write_release(void *data)
        struct nfs_pgio_header *hdr = data;
 
        ff_layout_write_record_layoutstats_done(&hdr->task, hdr);
-       if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags))
+       if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags)) {
+               ff_layout_send_layouterror(hdr->lseg);
                ff_layout_reset_write(hdr, true);
-       else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags))
+       } else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags))
                ff_layout_reset_write(hdr, false);
        pnfs_generic_rw_release(data);
 }
@@ -2119,6 +2123,52 @@ out_nomem:
        return -ENOMEM;
 }
 
+#ifdef CONFIG_NFS_V4_2
+void
+ff_layout_send_layouterror(struct pnfs_layout_segment *lseg)
+{
+       struct pnfs_layout_hdr *lo = lseg->pls_layout;
+       struct nfs42_layout_error *errors;
+       LIST_HEAD(head);
+
+       if (!nfs_server_capable(lo->plh_inode, NFS_CAP_LAYOUTERROR))
+               return;
+       ff_layout_fetch_ds_ioerr(lo, &lseg->pls_range, &head, -1);
+       if (list_empty(&head))
+               return;
+
+       errors = kmalloc_array(NFS42_LAYOUTERROR_MAX,
+                       sizeof(*errors), GFP_NOFS);
+       if (errors != NULL) {
+               const struct nfs4_ff_layout_ds_err *pos;
+               size_t n = 0;
+
+               list_for_each_entry(pos, &head, list) {
+                       errors[n].offset = pos->offset;
+                       errors[n].length = pos->length;
+                       nfs4_stateid_copy(&errors[n].stateid, &pos->stateid);
+                       errors[n].errors[0].dev_id = pos->deviceid;
+                       errors[n].errors[0].status = pos->status;
+                       errors[n].errors[0].opnum = pos->opnum;
+                       n++;
+                       if (!list_is_last(&pos->list, &head) &&
+                           n < NFS42_LAYOUTERROR_MAX)
+                               continue;
+                       if (nfs42_proc_layouterror(lseg, errors, n) < 0)
+                               break;
+                       n = 0;
+               }
+               kfree(errors);
+       }
+       ff_layout_free_ds_ioerr(&head);
+}
+#else
+void
+ff_layout_send_layouterror(struct pnfs_layout_segment *lseg)
+{
+}
+#endif
+
 static int
 ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen)
 {

diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index 8a2d5d630af9d0a05da2ca574845c265a338ab4d..31a62820a5c66c3a983443c5dd75316b656884cb 100644 (file)
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -213,6 +213,7 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
                             struct nfs4_ff_layout_mirror *mirror, u64 offset,
                             u64 length, int status, enum nfs_opnum4 opnum,
                             gfp_t gfp_flags);
+void ff_layout_send_layouterror(struct pnfs_layout_segment *lseg);
 int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head);
 void ff_layout_free_ds_ioerr(struct list_head *head);
 unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,

diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
index ca7a6203b3cb03e8513f788bf4b86985224a89c1..c174f23afc6d440b517302ad0ca5c06ef420d029 100644 (file)
--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -326,7 +326,6 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
        spin_lock(&flo->generic_hdr.plh_inode->i_lock);
        ff_layout_add_ds_error_locked(flo, dserr);
        spin_unlock(&flo->generic_hdr.plh_inode->i_lock);
-
        return 0;
 }
 
@@ -458,6 +457,7 @@ out_fail:
                                 mirror, lseg->pls_range.offset,
                                 lseg->pls_range.length, NFS4ERR_NXIO,
                                 OP_ILLEGAL, GFP_NOIO);
+       ff_layout_send_layouterror(lseg);
        if (fail_return || !ff_layout_has_available_ds(lseg))
                pnfs_error_mark_layout_for_return(ino, lseg);
        ds = NULL;