blob: 5044142c1216289e46f28e5205bfe981e089f4dd [file] [log] [blame]
Ricardo Labiaga85e174b2010-10-20 00:17:58 -04001/*
2 * pNFS functions to call and manage layout drivers.
3 *
4 * Copyright (c) 2002 [year of first publication]
5 * The Regents of the University of Michigan
6 * All Rights Reserved
7 *
8 * Dean Hildebrand <dhildebz@umich.edu>
9 *
10 * Permission is granted to use, copy, create derivative works, and
11 * redistribute this software and such derivative works for any purpose,
12 * so long as the name of the University of Michigan is not used in
13 * any advertising or publicity pertaining to the use or distribution
14 * of this software without specific, written prior authorization. If
15 * the above copyright notice or any other identification of the
16 * University of Michigan is included in any copy of any portion of
17 * this software, then the disclaimer below must also be included.
18 *
19 * This software is provided as is, without representation or warranty
20 * of any kind either express or implied, including without limitation
21 * the implied warranties of merchantability, fitness for a particular
22 * purpose, or noninfringement. The Regents of the University of
23 * Michigan shall not be liable for any damages, including special,
24 * indirect, incidental, or consequential damages, with respect to any
25 * claim arising out of or in connection with the use of the software,
26 * even if it has been or is hereafter advised of the possibility of
27 * such damages.
28 */
29
30#include <linux/nfs_fs.h>
Trond Myklebust493292d2011-07-13 15:58:28 -040031#include <linux/nfs_page.h>
Paul Gortmaker143cb492011-07-01 14:23:34 -040032#include <linux/module.h>
Andy Adamson974cec82010-10-20 00:18:02 -040033#include "internal.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040034#include "pnfs.h"
Andy Adamson64419a92011-03-01 01:34:16 +000035#include "iostat.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040036
37#define NFSDBG_FACILITY NFSDBG_PNFS
Trond Myklebust25c75332012-09-18 17:01:12 -040038#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040039
Fred Isaman02c35fc2010-10-20 00:17:59 -040040/* Locking:
41 *
42 * pnfs_spinlock:
43 * protects pnfs_modules_tbl.
44 */
45static DEFINE_SPINLOCK(pnfs_spinlock);
46
47/*
48 * pnfs_modules_tbl holds all pnfs modules
49 */
50static LIST_HEAD(pnfs_modules_tbl);
51
52/* Return the registered pnfs layout driver module matching given id */
53static struct pnfs_layoutdriver_type *
54find_pnfs_driver_locked(u32 id)
55{
56 struct pnfs_layoutdriver_type *local;
57
58 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
59 if (local->id == id)
60 goto out;
61 local = NULL;
62out:
63 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
64 return local;
65}
66
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040067static struct pnfs_layoutdriver_type *
68find_pnfs_driver(u32 id)
69{
Fred Isaman02c35fc2010-10-20 00:17:59 -040070 struct pnfs_layoutdriver_type *local;
71
72 spin_lock(&pnfs_spinlock);
73 local = find_pnfs_driver_locked(id);
Trond Myklebust0a9c63f2012-06-15 13:02:58 -040074 if (local != NULL && !try_module_get(local->owner)) {
75 dprintk("%s: Could not grab reference on module\n", __func__);
76 local = NULL;
77 }
Fred Isaman02c35fc2010-10-20 00:17:59 -040078 spin_unlock(&pnfs_spinlock);
79 return local;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040080}
81
82void
83unset_pnfs_layoutdriver(struct nfs_server *nfss)
84{
Benny Halevy738fd0f32011-07-30 20:52:36 -040085 if (nfss->pnfs_curr_ld) {
86 if (nfss->pnfs_curr_ld->clear_layoutdriver)
87 nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
Trond Myklebust2a4c8992012-06-14 13:08:38 -040088 /* Decrement the MDS count. Purge the deviceid cache if zero */
89 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
90 nfs4_deviceid_purge_client(nfss->nfs_client);
Fred Isaman02c35fc2010-10-20 00:17:59 -040091 module_put(nfss->pnfs_curr_ld->owner);
Benny Halevy738fd0f32011-07-30 20:52:36 -040092 }
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040093 nfss->pnfs_curr_ld = NULL;
94}
95
96/*
97 * Try to set the server's pnfs module to the pnfs layout type specified by id.
98 * Currently only one pNFS layout driver per filesystem is supported.
99 *
100 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
101 */
102void
Benny Halevy738fd0f32011-07-30 20:52:36 -0400103set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
104 u32 id)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400105{
106 struct pnfs_layoutdriver_type *ld_type = NULL;
107
108 if (id == 0)
109 goto out_no_driver;
110 if (!(server->nfs_client->cl_exchange_flags &
111 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500112 printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
113 __func__, id, server->nfs_client->cl_exchange_flags);
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400114 goto out_no_driver;
115 }
116 ld_type = find_pnfs_driver(id);
117 if (!ld_type) {
118 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
119 ld_type = find_pnfs_driver(id);
120 if (!ld_type) {
121 dprintk("%s: No pNFS module found for %u.\n",
122 __func__, id);
123 goto out_no_driver;
124 }
125 }
126 server->pnfs_curr_ld = ld_type;
Benny Halevy738fd0f32011-07-30 20:52:36 -0400127 if (ld_type->set_layoutdriver
128 && ld_type->set_layoutdriver(server, mntfh)) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500129 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
130 "driver %u.\n", __func__, id);
Benny Halevy738fd0f32011-07-30 20:52:36 -0400131 module_put(ld_type->owner);
132 goto out_no_driver;
133 }
Trond Myklebust2a4c8992012-06-14 13:08:38 -0400134 /* Bump the MDS count */
135 atomic_inc(&server->nfs_client->cl_mds_count);
Christoph Hellwigea8eecd2011-03-01 01:34:21 +0000136
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400137 dprintk("%s: pNFS module for %u set\n", __func__, id);
138 return;
139
140out_no_driver:
141 dprintk("%s: Using NFSv4 I/O\n", __func__);
142 server->pnfs_curr_ld = NULL;
143}
Fred Isaman02c35fc2010-10-20 00:17:59 -0400144
145int
146pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
147{
148 int status = -EINVAL;
149 struct pnfs_layoutdriver_type *tmp;
150
151 if (ld_type->id == 0) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500152 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
Fred Isaman02c35fc2010-10-20 00:17:59 -0400153 return status;
154 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400155 if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500156 printk(KERN_ERR "NFS: %s Layout driver must provide "
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400157 "alloc_lseg and free_lseg.\n", __func__);
158 return status;
159 }
Fred Isaman02c35fc2010-10-20 00:17:59 -0400160
161 spin_lock(&pnfs_spinlock);
162 tmp = find_pnfs_driver_locked(ld_type->id);
163 if (!tmp) {
164 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
165 status = 0;
166 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
167 ld_type->name);
168 } else {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500169 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
Fred Isaman02c35fc2010-10-20 00:17:59 -0400170 __func__, ld_type->id);
171 }
172 spin_unlock(&pnfs_spinlock);
173
174 return status;
175}
176EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);
177
178void
179pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
180{
181 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
182 spin_lock(&pnfs_spinlock);
183 list_del(&ld_type->pnfs_tblid);
184 spin_unlock(&pnfs_spinlock);
185}
186EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
Benny Halevye5e94012010-10-20 00:18:01 -0400187
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400188/*
189 * pNFS client layout cache
190 */
191
Fred Isamancc6e5342011-01-06 11:36:28 +0000192/* Need to hold i_lock if caller does not already hold reference */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000193void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400194pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
Benny Halevye5e94012010-10-20 00:18:01 -0400195{
Fred Isamancc6e5342011-01-06 11:36:28 +0000196 atomic_inc(&lo->plh_refcount);
197}
198
Benny Halevy636fb9c2011-05-22 19:51:33 +0300199static struct pnfs_layout_hdr *
200pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
201{
202 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
Trond Myklebust57934272012-09-20 20:37:23 -0400203 return ld->alloc_layout_hdr(ino, gfp_flags);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300204}
205
206static void
207pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
208{
Trond Myklebust9c626382012-09-20 17:31:43 -0400209 struct nfs_server *server = NFS_SERVER(lo->plh_inode);
210 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
211
212 if (!list_empty(&lo->plh_layouts)) {
213 struct nfs_client *clp = server->nfs_client;
214
215 spin_lock(&clp->cl_lock);
216 list_del_init(&lo->plh_layouts);
217 spin_unlock(&clp->cl_lock);
218 }
Peng Tao9fa40752011-07-30 20:52:32 -0400219 put_rpccred(lo->plh_lc_cred);
Trond Myklebust57934272012-09-20 20:37:23 -0400220 return ld->free_layout_hdr(lo);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300221}
222
Fred Isamancc6e5342011-01-06 11:36:28 +0000223static void
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400224pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
Fred Isamancc6e5342011-01-06 11:36:28 +0000225{
Trond Myklebustbb346f62012-09-20 15:52:13 -0400226 struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
Fred Isamancc6e5342011-01-06 11:36:28 +0000227 dprintk("%s: freeing layout cache %p\n", __func__, lo);
Trond Myklebustbb346f62012-09-20 15:52:13 -0400228 nfsi->layout = NULL;
229 /* Reset MDS Threshold I/O counters */
230 nfsi->write_io = 0;
231 nfsi->read_io = 0;
Benny Halevye5e94012010-10-20 00:18:01 -0400232}
233
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400234void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400235pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
Andy Adamson974cec82010-10-20 00:18:02 -0400236{
Fred Isamancc6e5342011-01-06 11:36:28 +0000237 struct inode *inode = lo->plh_inode;
238
239 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400240 pnfs_detach_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000241 spin_unlock(&inode->i_lock);
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400242 pnfs_free_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000243 }
Andy Adamson974cec82010-10-20 00:18:02 -0400244}
245
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400246static int
247pnfs_iomode_to_fail_bit(u32 iomode)
248{
249 return iomode == IOMODE_RW ?
250 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
251}
252
253static void
Trond Myklebust3e621212012-09-24 13:07:16 -0400254pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400255{
Trond Myklebust25c75332012-09-18 17:01:12 -0400256 lo->plh_retry_timestamp = jiffies;
Yanchuan Nian39e88fc2013-01-04 20:19:49 +0800257 if (!test_and_set_bit(fail_bit, &lo->plh_flags))
Trond Myklebust3e621212012-09-24 13:07:16 -0400258 atomic_inc(&lo->plh_refcount);
259}
260
261static void
262pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
263{
264 if (test_and_clear_bit(fail_bit, &lo->plh_flags))
265 atomic_dec(&lo->plh_refcount);
266}
267
268static void
269pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
270{
271 struct inode *inode = lo->plh_inode;
Trond Myklebust115ce572012-09-20 21:19:43 -0400272 struct pnfs_layout_range range = {
273 .iomode = iomode,
274 .offset = 0,
275 .length = NFS4_MAX_UINT64,
276 };
277 LIST_HEAD(head);
Trond Myklebust3e621212012-09-24 13:07:16 -0400278
279 spin_lock(&inode->i_lock);
280 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
Trond Myklebust115ce572012-09-20 21:19:43 -0400281 pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
Trond Myklebust3e621212012-09-24 13:07:16 -0400282 spin_unlock(&inode->i_lock);
Trond Myklebust115ce572012-09-20 21:19:43 -0400283 pnfs_free_lseg_list(&head);
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400284 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
285 iomode == IOMODE_RW ? "RW" : "READ");
286}
287
288static bool
289pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
290{
Trond Myklebust25c75332012-09-18 17:01:12 -0400291 unsigned long start, end;
Trond Myklebust3e621212012-09-24 13:07:16 -0400292 int fail_bit = pnfs_iomode_to_fail_bit(iomode);
293
294 if (test_bit(fail_bit, &lo->plh_flags) == 0)
Trond Myklebust25c75332012-09-18 17:01:12 -0400295 return false;
296 end = jiffies;
297 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
298 if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
299 /* It is time to retry the failed layoutgets */
Trond Myklebust3e621212012-09-24 13:07:16 -0400300 pnfs_layout_clear_fail_bit(lo, fail_bit);
Trond Myklebust25c75332012-09-18 17:01:12 -0400301 return false;
302 }
303 return true;
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400304}
305
Andy Adamson974cec82010-10-20 00:18:02 -0400306static void
307init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
308{
Fred Isaman566052c2011-01-06 11:36:20 +0000309 INIT_LIST_HEAD(&lseg->pls_list);
Peng Taoa9bae562011-07-30 20:52:33 -0400310 INIT_LIST_HEAD(&lseg->pls_lc_list);
Fred Isaman4541d162011-01-06 11:36:23 +0000311 atomic_set(&lseg->pls_refcount, 1);
312 smp_mb();
313 set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
Fred Isaman566052c2011-01-06 11:36:20 +0000314 lseg->pls_layout = lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400315}
316
Trond Myklebust905ca192012-09-20 20:46:49 -0400317static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400318{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000319 struct inode *ino = lseg->pls_layout->plh_inode;
Andy Adamson974cec82010-10-20 00:18:02 -0400320
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400321 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400322}
323
Fred Isamand684d2a2011-03-01 01:34:13 +0000324static void
Trond Myklebust57036a32012-09-20 16:33:30 -0400325pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
326 struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400327{
Trond Myklebust57036a32012-09-20 16:33:30 -0400328 struct inode *inode = lo->plh_inode;
Fred Isamand684d2a2011-03-01 01:34:13 +0000329
Benny Halevyd20581a2011-05-22 19:52:03 +0300330 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Fred Isamand684d2a2011-03-01 01:34:13 +0000331 list_del_init(&lseg->pls_list);
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400332 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
333 atomic_dec(&lo->plh_refcount);
Trond Myklebust173f77e2012-09-21 15:49:42 -0400334 if (list_empty(&lo->plh_segs))
335 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Fred Isamand684d2a2011-03-01 01:34:13 +0000336 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
337}
338
Fred Isamanbae724e2011-03-01 01:34:15 +0000339void
Trond Myklebust9369a432012-09-18 20:57:08 -0400340pnfs_put_lseg(struct pnfs_layout_segment *lseg)
Fred Isamand684d2a2011-03-01 01:34:13 +0000341{
Trond Myklebust57036a32012-09-20 16:33:30 -0400342 struct pnfs_layout_hdr *lo;
Fred Isamand684d2a2011-03-01 01:34:13 +0000343 struct inode *inode;
344
345 if (!lseg)
346 return;
347
Fred Isaman4541d162011-01-06 11:36:23 +0000348 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
349 atomic_read(&lseg->pls_refcount),
350 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Trond Myklebust57036a32012-09-20 16:33:30 -0400351 lo = lseg->pls_layout;
352 inode = lo->plh_inode;
Fred Isamand684d2a2011-03-01 01:34:13 +0000353 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400354 pnfs_get_layout_hdr(lo);
Trond Myklebust57036a32012-09-20 16:33:30 -0400355 pnfs_layout_remove_lseg(lo, lseg);
Fred Isamand684d2a2011-03-01 01:34:13 +0000356 spin_unlock(&inode->i_lock);
Trond Myklebust905ca192012-09-20 20:46:49 -0400357 pnfs_free_lseg(lseg);
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400358 pnfs_put_layout_hdr(lo);
Fred Isaman4541d162011-01-06 11:36:23 +0000359 }
Andy Adamson974cec82010-10-20 00:18:02 -0400360}
Trond Myklebust9369a432012-09-18 20:57:08 -0400361EXPORT_SYMBOL_GPL(pnfs_put_lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400362
Benny Halevyfb3296e2011-05-22 19:47:26 +0300363static inline u64
364end_offset(u64 start, u64 len)
Fred Isaman4541d162011-01-06 11:36:23 +0000365{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300366 u64 end;
367
368 end = start + len;
369 return end >= start ? end : NFS4_MAX_UINT64;
370}
371
Benny Halevyfb3296e2011-05-22 19:47:26 +0300372/*
373 * is l2 fully contained in l1?
374 * start1 end1
375 * [----------------------------------)
376 * start2 end2
377 * [----------------)
378 */
379static inline int
380lo_seg_contained(struct pnfs_layout_range *l1,
381 struct pnfs_layout_range *l2)
382{
383 u64 start1 = l1->offset;
384 u64 end1 = end_offset(start1, l1->length);
385 u64 start2 = l2->offset;
386 u64 end2 = end_offset(start2, l2->length);
387
388 return (start1 <= start2) && (end1 >= end2);
389}
390
391/*
392 * is l1 and l2 intersecting?
393 * start1 end1
394 * [----------------------------------)
395 * start2 end2
396 * [----------------)
397 */
398static inline int
399lo_seg_intersecting(struct pnfs_layout_range *l1,
400 struct pnfs_layout_range *l2)
401{
402 u64 start1 = l1->offset;
403 u64 end1 = end_offset(start1, l1->length);
404 u64 start2 = l2->offset;
405 u64 end2 = end_offset(start2, l2->length);
406
407 return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
408 (end2 == NFS4_MAX_UINT64 || end2 > start1);
409}
410
Fred Isaman4541d162011-01-06 11:36:23 +0000411static bool
Benny Halevy778b5502011-05-22 19:48:02 +0300412should_free_lseg(struct pnfs_layout_range *lseg_range,
413 struct pnfs_layout_range *recall_range)
Fred Isaman4541d162011-01-06 11:36:23 +0000414{
Benny Halevy778b5502011-05-22 19:48:02 +0300415 return (recall_range->iomode == IOMODE_ANY ||
416 lseg_range->iomode == recall_range->iomode) &&
417 lo_seg_intersecting(lseg_range, recall_range);
Fred Isaman4541d162011-01-06 11:36:23 +0000418}
419
Trond Myklebust24956802013-03-20 13:03:00 -0400420static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
421 struct list_head *tmp_list)
422{
423 if (!atomic_dec_and_test(&lseg->pls_refcount))
424 return false;
425 pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
426 list_add(&lseg->pls_list, tmp_list);
427 return true;
428}
429
Fred Isaman4541d162011-01-06 11:36:23 +0000430/* Returns 1 if lseg is removed from list, 0 otherwise */
431static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
432 struct list_head *tmp_list)
433{
434 int rv = 0;
435
436 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
437 /* Remove the reference keeping the lseg in the
438 * list. It will now be removed when all
439 * outstanding io is finished.
440 */
Fred Isamand684d2a2011-03-01 01:34:13 +0000441 dprintk("%s: lseg %p ref %d\n", __func__, lseg,
442 atomic_read(&lseg->pls_refcount));
Trond Myklebust24956802013-03-20 13:03:00 -0400443 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
Fred Isamand684d2a2011-03-01 01:34:13 +0000444 rv = 1;
Fred Isaman4541d162011-01-06 11:36:23 +0000445 }
446 return rv;
447}
448
449/* Returns count of number of matching invalid lsegs remaining in list
450 * after call.
451 */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000452int
Trond Myklebust49a85062012-09-18 20:43:31 -0400453pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
Fred Isaman4541d162011-01-06 11:36:23 +0000454 struct list_head *tmp_list,
Benny Halevy778b5502011-05-22 19:48:02 +0300455 struct pnfs_layout_range *recall_range)
Andy Adamson974cec82010-10-20 00:18:02 -0400456{
457 struct pnfs_layout_segment *lseg, *next;
Fred Isaman4541d162011-01-06 11:36:23 +0000458 int invalid = 0, removed = 0;
Andy Adamson974cec82010-10-20 00:18:02 -0400459
460 dprintk("%s:Begin lo %p\n", __func__, lo);
461
Trond Myklebust8006bfb2012-09-21 14:48:04 -0400462 if (list_empty(&lo->plh_segs))
Fred Isaman38511722011-02-03 18:28:50 +0000463 return 0;
Fred Isaman4541d162011-01-06 11:36:23 +0000464 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
Benny Halevy778b5502011-05-22 19:48:02 +0300465 if (!recall_range ||
466 should_free_lseg(&lseg->pls_range, recall_range)) {
Fred Isaman4541d162011-01-06 11:36:23 +0000467 dprintk("%s: freeing lseg %p iomode %d "
468 "offset %llu length %llu\n", __func__,
469 lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
470 lseg->pls_range.length);
471 invalid++;
472 removed += mark_lseg_invalid(lseg, tmp_list);
473 }
474 dprintk("%s:Return %i\n", __func__, invalid - removed);
475 return invalid - removed;
Andy Adamson974cec82010-10-20 00:18:02 -0400476}
477
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000478/* note free_me must contain lsegs from a single layout_hdr */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000479void
Fred Isaman4541d162011-01-06 11:36:23 +0000480pnfs_free_lseg_list(struct list_head *free_me)
Andy Adamson974cec82010-10-20 00:18:02 -0400481{
Fred Isaman4541d162011-01-06 11:36:23 +0000482 struct pnfs_layout_segment *lseg, *tmp;
Andy Adamson974cec82010-10-20 00:18:02 -0400483
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000484 if (list_empty(free_me))
485 return;
486
Fred Isaman4541d162011-01-06 11:36:23 +0000487 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
Fred Isaman566052c2011-01-06 11:36:20 +0000488 list_del(&lseg->pls_list);
Trond Myklebust905ca192012-09-20 20:46:49 -0400489 pnfs_free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400490 }
491}
492
Benny Halevye5e94012010-10-20 00:18:01 -0400493void
494pnfs_destroy_layout(struct nfs_inode *nfsi)
495{
496 struct pnfs_layout_hdr *lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400497 LIST_HEAD(tmp_list);
Benny Halevye5e94012010-10-20 00:18:01 -0400498
499 spin_lock(&nfsi->vfs_inode.i_lock);
500 lo = nfsi->layout;
501 if (lo) {
Fred Isaman38511722011-02-03 18:28:50 +0000502 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
Trond Myklebust49a85062012-09-18 20:43:31 -0400503 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Trond Myklebust3e621212012-09-24 13:07:16 -0400504 pnfs_get_layout_hdr(lo);
505 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
506 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
507 spin_unlock(&nfsi->vfs_inode.i_lock);
508 pnfs_free_lseg_list(&tmp_list);
509 pnfs_put_layout_hdr(lo);
510 } else
511 spin_unlock(&nfsi->vfs_inode.i_lock);
Benny Halevye5e94012010-10-20 00:18:01 -0400512}
Andy Adamson041245c2012-04-27 17:53:53 -0400513EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
Benny Halevye5e94012010-10-20 00:18:01 -0400514
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500515static bool
516pnfs_layout_add_bulk_destroy_list(struct inode *inode,
517 struct list_head *layout_list)
518{
519 struct pnfs_layout_hdr *lo;
520 bool ret = false;
521
522 spin_lock(&inode->i_lock);
523 lo = NFS_I(inode)->layout;
524 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
525 pnfs_get_layout_hdr(lo);
526 list_add(&lo->plh_bulk_destroy, layout_list);
527 ret = true;
528 }
529 spin_unlock(&inode->i_lock);
530 return ret;
531}
532
533/* Caller must hold rcu_read_lock and clp->cl_lock */
534static int
535pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
536 struct nfs_server *server,
537 struct list_head *layout_list)
538{
539 struct pnfs_layout_hdr *lo, *next;
540 struct inode *inode;
541
542 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
543 inode = igrab(lo->plh_inode);
544 if (inode == NULL)
545 continue;
546 list_del_init(&lo->plh_layouts);
547 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
548 continue;
549 rcu_read_unlock();
550 spin_unlock(&clp->cl_lock);
551 iput(inode);
552 spin_lock(&clp->cl_lock);
553 rcu_read_lock();
554 return -EAGAIN;
555 }
556 return 0;
557}
558
559static int
560pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
561 bool is_bulk_recall)
562{
563 struct pnfs_layout_hdr *lo;
564 struct inode *inode;
565 struct pnfs_layout_range range = {
566 .iomode = IOMODE_ANY,
567 .offset = 0,
568 .length = NFS4_MAX_UINT64,
569 };
570 LIST_HEAD(lseg_list);
571 int ret = 0;
572
573 while (!list_empty(layout_list)) {
574 lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
575 plh_bulk_destroy);
576 dprintk("%s freeing layout for inode %lu\n", __func__,
577 lo->plh_inode->i_ino);
578 inode = lo->plh_inode;
579 spin_lock(&inode->i_lock);
580 list_del_init(&lo->plh_bulk_destroy);
581 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
582 if (is_bulk_recall)
583 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
584 if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range))
585 ret = -EAGAIN;
586 spin_unlock(&inode->i_lock);
587 pnfs_free_lseg_list(&lseg_list);
588 pnfs_put_layout_hdr(lo);
589 iput(inode);
590 }
591 return ret;
592}
593
594int
595pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
596 struct nfs_fsid *fsid,
597 bool is_recall)
598{
599 struct nfs_server *server;
600 LIST_HEAD(layout_list);
601
602 spin_lock(&clp->cl_lock);
603 rcu_read_lock();
604restart:
605 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
606 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
607 continue;
608 if (pnfs_layout_bulk_destroy_byserver_locked(clp,
609 server,
610 &layout_list) != 0)
611 goto restart;
612 }
613 rcu_read_unlock();
614 spin_unlock(&clp->cl_lock);
615
616 if (list_empty(&layout_list))
617 return 0;
618 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
619}
620
621int
622pnfs_destroy_layouts_byclid(struct nfs_client *clp,
623 bool is_recall)
624{
625 struct nfs_server *server;
626 LIST_HEAD(layout_list);
627
628 spin_lock(&clp->cl_lock);
629 rcu_read_lock();
630restart:
631 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
632 if (pnfs_layout_bulk_destroy_byserver_locked(clp,
633 server,
634 &layout_list) != 0)
635 goto restart;
636 }
637 rcu_read_unlock();
638 spin_unlock(&clp->cl_lock);
639
640 if (list_empty(&layout_list))
641 return 0;
642 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
643}
644
Andy Adamson974cec82010-10-20 00:18:02 -0400645/*
646 * Called by the state manger to remove all layouts established under an
647 * expired lease.
648 */
649void
650pnfs_destroy_all_layouts(struct nfs_client *clp)
651{
Andy Adamsonc47abcf2011-06-15 17:52:40 -0400652 nfs4_deviceid_mark_client_invalid(clp);
653 nfs4_deviceid_purge_client(clp);
654
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500655 pnfs_destroy_layouts_byclid(clp, false);
Andy Adamson974cec82010-10-20 00:18:02 -0400656}
657
Trond Myklebust5a655032012-10-02 16:47:14 -0700658/*
659 * Compare 2 layout stateid sequence ids, to see which is newer,
660 * taking into account wraparound issues.
661 */
662static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
663{
664 return (s32)s1 - (s32)s2 > 0;
665}
666
Fred Isamanfd6002e2011-01-06 11:36:22 +0000667/* update lo->plh_stateid with new if is more recent */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000668void
669pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
670 bool update_barrier)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400671{
Trond Myklebust22aaf712012-10-04 16:32:22 -0700672 u32 oldseq, newseq, new_barrier;
673 int empty = list_empty(&lo->plh_segs);
Andy Adamson974cec82010-10-20 00:18:02 -0400674
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500675 oldseq = be32_to_cpu(lo->plh_stateid.seqid);
676 newseq = be32_to_cpu(new->seqid);
Trond Myklebust22aaf712012-10-04 16:32:22 -0700677 if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
Trond Myklebustf597c532012-03-04 18:13:56 -0500678 nfs4_stateid_copy(&lo->plh_stateid, new);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000679 if (update_barrier) {
Trond Myklebust22aaf712012-10-04 16:32:22 -0700680 new_barrier = be32_to_cpu(new->seqid);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000681 } else {
682 /* Because of wraparound, we want to keep the barrier
Trond Myklebust22aaf712012-10-04 16:32:22 -0700683 * "close" to the current seqids.
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000684 */
Trond Myklebust22aaf712012-10-04 16:32:22 -0700685 new_barrier = newseq - atomic_read(&lo->plh_outstanding);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000686 }
Trond Myklebust22aaf712012-10-04 16:32:22 -0700687 if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
688 lo->plh_barrier = new_barrier;
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000689 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400690}
691
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700692static bool
693pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
694 const nfs4_stateid *stateid)
695{
696 u32 seqid = be32_to_cpu(stateid->seqid);
697
698 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
699}
700
Fred Isamancf7d63f2011-01-06 11:36:25 +0000701/* lget is set to 1 if called from inside send_layoutget call chain */
702static bool
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700703pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo, int lget)
Fred Isamancf7d63f2011-01-06 11:36:25 +0000704{
Fred Isamanf7e89172011-01-06 11:36:32 +0000705 return lo->plh_block_lgets ||
706 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000707 (list_empty(&lo->plh_segs) &&
Fred Isamancf7d63f2011-01-06 11:36:25 +0000708 (atomic_read(&lo->plh_outstanding) > lget));
709}
710
Fred Isamanfd6002e2011-01-06 11:36:22 +0000711int
712pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
713 struct nfs4_state *open_state)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400714{
Fred Isamanfd6002e2011-01-06 11:36:22 +0000715 int status = 0;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400716
717 dprintk("--> %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000718 spin_lock(&lo->plh_inode->i_lock);
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700719 if (pnfs_layoutgets_blocked(lo, 1)) {
Fred Isamancf7d63f2011-01-06 11:36:25 +0000720 status = -EAGAIN;
721 } else if (list_empty(&lo->plh_segs)) {
Fred Isamanfd6002e2011-01-06 11:36:22 +0000722 int seq;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400723
Fred Isamanfd6002e2011-01-06 11:36:22 +0000724 do {
725 seq = read_seqbegin(&open_state->seqlock);
Trond Myklebustf597c532012-03-04 18:13:56 -0500726 nfs4_stateid_copy(dst, &open_state->stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000727 } while (read_seqretry(&open_state->seqlock, seq));
728 } else
Trond Myklebustf597c532012-03-04 18:13:56 -0500729 nfs4_stateid_copy(dst, &lo->plh_stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000730 spin_unlock(&lo->plh_inode->i_lock);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400731 dprintk("<-- %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000732 return status;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400733}
734
735/*
736* Get layout from server.
737* for now, assume that whole file layouts are requested.
738* arg->offset: 0
739* arg->length: all ones
740*/
Benny Halevye5e94012010-10-20 00:18:01 -0400741static struct pnfs_layout_segment *
742send_layoutget(struct pnfs_layout_hdr *lo,
743 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +0300744 struct pnfs_layout_range *range,
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400745 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400746{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000747 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400748 struct nfs_server *server = NFS_SERVER(ino);
749 struct nfs4_layoutget *lgp;
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400750 struct pnfs_layout_segment *lseg;
Benny Halevye5e94012010-10-20 00:18:01 -0400751
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400752 dprintk("--> %s\n", __func__);
753
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400754 lgp = kzalloc(sizeof(*lgp), gfp_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +0000755 if (lgp == NULL)
Andy Adamson974cec82010-10-20 00:18:02 -0400756 return NULL;
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400757
Benny Halevyfb3296e2011-05-22 19:47:26 +0300758 lgp->args.minlength = PAGE_CACHE_SIZE;
759 if (lgp->args.minlength > range->length)
760 lgp->args.minlength = range->length;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400761 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
Benny Halevyfb3296e2011-05-22 19:47:26 +0300762 lgp->args.range = *range;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400763 lgp->args.type = server->pnfs_curr_ld->id;
764 lgp->args.inode = ino;
765 lgp->args.ctx = get_nfs_open_context(ctx);
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400766 lgp->gfp_flags = gfp_flags;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400767
768 /* Synchronously retrieve layout information from server and
769 * store in lseg.
770 */
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400771 lseg = nfs4_proc_layoutget(lgp, gfp_flags);
772 if (IS_ERR(lseg)) {
773 switch (PTR_ERR(lseg)) {
774 case -ENOMEM:
775 case -ERESTARTSYS:
776 break;
777 default:
778 /* remember that LAYOUTGET failed and suspend trying */
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400779 pnfs_layout_io_set_failed(lo, range->iomode);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400780 }
781 return NULL;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400782 }
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400783
Andy Adamson974cec82010-10-20 00:18:02 -0400784 return lseg;
785}
786
Trond Myklebust24956802013-03-20 13:03:00 -0400787static void pnfs_clear_layoutcommit(struct inode *inode,
788 struct list_head *head)
789{
790 struct nfs_inode *nfsi = NFS_I(inode);
791 struct pnfs_layout_segment *lseg, *tmp;
792
793 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
794 return;
795 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
796 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
797 continue;
798 pnfs_lseg_dec_and_remove_zero(lseg, head);
799 }
800}
801
Andy Adamson293b3b02012-06-20 15:03:34 -0400802/*
803 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
804 * when the layout segment list is empty.
805 *
806 * Note that a pnfs_layout_hdr can exist with an empty layout segment
807 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
808 * deviceid is marked invalid.
809 */
Benny Halevycbe82602011-05-22 19:52:37 +0300810int
811_pnfs_return_layout(struct inode *ino)
812{
813 struct pnfs_layout_hdr *lo = NULL;
814 struct nfs_inode *nfsi = NFS_I(ino);
815 LIST_HEAD(tmp_list);
816 struct nfs4_layoutreturn *lrp;
817 nfs4_stateid stateid;
Andy Adamson293b3b02012-06-20 15:03:34 -0400818 int status = 0, empty;
Benny Halevycbe82602011-05-22 19:52:37 +0300819
Andy Adamson366d5052012-06-20 15:03:33 -0400820 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
Benny Halevycbe82602011-05-22 19:52:37 +0300821
822 spin_lock(&ino->i_lock);
823 lo = nfsi->layout;
Trond Myklebuste5929f32012-09-21 16:37:02 -0400824 if (!lo) {
Benny Halevycbe82602011-05-22 19:52:37 +0300825 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -0400826 dprintk("NFS: %s no layout to return\n", __func__);
827 goto out;
Benny Halevycbe82602011-05-22 19:52:37 +0300828 }
829 stateid = nfsi->layout->plh_stateid;
830 /* Reference matched in nfs4_layoutreturn_release */
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400831 pnfs_get_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -0400832 empty = list_empty(&lo->plh_segs);
Trond Myklebust24956802013-03-20 13:03:00 -0400833 pnfs_clear_layoutcommit(ino, &tmp_list);
Trond Myklebust49a85062012-09-18 20:43:31 -0400834 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Andy Adamson293b3b02012-06-20 15:03:34 -0400835 /* Don't send a LAYOUTRETURN if list was initially empty */
836 if (empty) {
837 spin_unlock(&ino->i_lock);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400838 pnfs_put_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -0400839 dprintk("NFS: %s no layout segments to return\n", __func__);
840 goto out;
841 }
Fred Isamanea0ded72011-06-15 12:31:02 -0400842 lo->plh_block_lgets++;
Benny Halevycbe82602011-05-22 19:52:37 +0300843 spin_unlock(&ino->i_lock);
844 pnfs_free_lseg_list(&tmp_list);
845
Benny Halevycbe82602011-05-22 19:52:37 +0300846 lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
847 if (unlikely(lrp == NULL)) {
848 status = -ENOMEM;
Trond Myklebust65857d52012-09-24 13:49:27 -0400849 spin_lock(&ino->i_lock);
850 lo->plh_block_lgets--;
851 spin_unlock(&ino->i_lock);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400852 pnfs_put_layout_hdr(lo);
Benny Halevycbe82602011-05-22 19:52:37 +0300853 goto out;
854 }
855
856 lrp->args.stateid = stateid;
857 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
858 lrp->args.inode = ino;
Trond Myklebusta56aaa02011-06-15 11:59:10 -0400859 lrp->args.layout = lo;
Benny Halevycbe82602011-05-22 19:52:37 +0300860 lrp->clp = NFS_SERVER(ino)->nfs_client;
861
862 status = nfs4_proc_layoutreturn(lrp);
863out:
864 dprintk("<-- %s status: %d\n", __func__, status);
865 return status;
866}
Andy Adamson0a57cda2012-04-27 17:53:50 -0400867EXPORT_SYMBOL_GPL(_pnfs_return_layout);
Benny Halevycbe82602011-05-22 19:52:37 +0300868
Fred Isamanf7e89172011-01-06 11:36:32 +0000869bool pnfs_roc(struct inode *ino)
870{
871 struct pnfs_layout_hdr *lo;
872 struct pnfs_layout_segment *lseg, *tmp;
873 LIST_HEAD(tmp_list);
874 bool found = false;
875
876 spin_lock(&ino->i_lock);
877 lo = NFS_I(ino)->layout;
878 if (!lo || !test_and_clear_bit(NFS_LAYOUT_ROC, &lo->plh_flags) ||
879 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
880 goto out_nolayout;
881 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
882 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
883 mark_lseg_invalid(lseg, &tmp_list);
884 found = true;
885 }
886 if (!found)
887 goto out_nolayout;
888 lo->plh_block_lgets++;
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400889 pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
Fred Isamanf7e89172011-01-06 11:36:32 +0000890 spin_unlock(&ino->i_lock);
891 pnfs_free_lseg_list(&tmp_list);
892 return true;
893
894out_nolayout:
895 spin_unlock(&ino->i_lock);
896 return false;
897}
898
899void pnfs_roc_release(struct inode *ino)
900{
901 struct pnfs_layout_hdr *lo;
902
903 spin_lock(&ino->i_lock);
904 lo = NFS_I(ino)->layout;
905 lo->plh_block_lgets--;
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400906 if (atomic_dec_and_test(&lo->plh_refcount)) {
907 pnfs_detach_layout_hdr(lo);
908 spin_unlock(&ino->i_lock);
909 pnfs_free_layout_hdr(lo);
910 } else
911 spin_unlock(&ino->i_lock);
Fred Isamanf7e89172011-01-06 11:36:32 +0000912}
913
914void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
915{
916 struct pnfs_layout_hdr *lo;
917
918 spin_lock(&ino->i_lock);
919 lo = NFS_I(ino)->layout;
Trond Myklebust0f35ad62012-10-04 16:28:17 -0700920 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
Fred Isamanf7e89172011-01-06 11:36:32 +0000921 lo->plh_barrier = barrier;
922 spin_unlock(&ino->i_lock);
923}
924
Trond Myklebust7fdab062012-09-20 20:15:57 -0400925bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
Fred Isamanf7e89172011-01-06 11:36:32 +0000926{
927 struct nfs_inode *nfsi = NFS_I(ino);
Trond Myklebust7fdab062012-09-20 20:15:57 -0400928 struct pnfs_layout_hdr *lo;
Fred Isamanf7e89172011-01-06 11:36:32 +0000929 struct pnfs_layout_segment *lseg;
Trond Myklebust7fdab062012-09-20 20:15:57 -0400930 u32 current_seqid;
Fred Isamanf7e89172011-01-06 11:36:32 +0000931 bool found = false;
932
933 spin_lock(&ino->i_lock);
934 list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
935 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
Trond Myklebust7fdab062012-09-20 20:15:57 -0400936 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
Fred Isamanf7e89172011-01-06 11:36:32 +0000937 found = true;
Trond Myklebust7fdab062012-09-20 20:15:57 -0400938 goto out;
Fred Isamanf7e89172011-01-06 11:36:32 +0000939 }
Trond Myklebust7fdab062012-09-20 20:15:57 -0400940 lo = nfsi->layout;
941 current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
Fred Isamanf7e89172011-01-06 11:36:32 +0000942
Trond Myklebust7fdab062012-09-20 20:15:57 -0400943 /* Since close does not return a layout stateid for use as
944 * a barrier, we choose the worst-case barrier.
945 */
946 *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
947out:
Fred Isamanf7e89172011-01-06 11:36:32 +0000948 spin_unlock(&ino->i_lock);
949 return found;
950}
951
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400952/*
953 * Compare two layout segments for sorting into layout cache.
954 * We want to preferentially return RW over RO layouts, so ensure those
955 * are seen first.
956 */
957static s64
Benny Halevyfb3296e2011-05-22 19:47:26 +0300958cmp_layout(struct pnfs_layout_range *l1,
959 struct pnfs_layout_range *l2)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400960{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300961 s64 d;
962
963 /* high offset > low offset */
964 d = l1->offset - l2->offset;
965 if (d)
966 return d;
967
968 /* short length > long length */
969 d = l2->length - l1->length;
970 if (d)
971 return d;
972
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400973 /* read > read/write */
Benny Halevyfb3296e2011-05-22 19:47:26 +0300974 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400975}
976
Andy Adamson974cec82010-10-20 00:18:02 -0400977static void
Trond Myklebust57036a32012-09-20 16:33:30 -0400978pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
Andy Adamson974cec82010-10-20 00:18:02 -0400979 struct pnfs_layout_segment *lseg)
980{
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400981 struct pnfs_layout_segment *lp;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400982
Andy Adamson974cec82010-10-20 00:18:02 -0400983 dprintk("%s:Begin\n", __func__);
984
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000985 list_for_each_entry(lp, &lo->plh_segs, pls_list) {
Benny Halevyfb3296e2011-05-22 19:47:26 +0300986 if (cmp_layout(&lseg->pls_range, &lp->pls_range) > 0)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400987 continue;
Fred Isaman566052c2011-01-06 11:36:20 +0000988 list_add_tail(&lseg->pls_list, &lp->pls_list);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400989 dprintk("%s: inserted lseg %p "
990 "iomode %d offset %llu length %llu before "
991 "lp %p iomode %d offset %llu length %llu\n",
Fred Isaman566052c2011-01-06 11:36:20 +0000992 __func__, lseg, lseg->pls_range.iomode,
993 lseg->pls_range.offset, lseg->pls_range.length,
994 lp, lp->pls_range.iomode, lp->pls_range.offset,
995 lp->pls_range.length);
Benny Halevyfb3296e2011-05-22 19:47:26 +0300996 goto out;
Andy Adamson974cec82010-10-20 00:18:02 -0400997 }
Benny Halevyfb3296e2011-05-22 19:47:26 +0300998 list_add_tail(&lseg->pls_list, &lo->plh_segs);
999 dprintk("%s: inserted lseg %p "
1000 "iomode %d offset %llu length %llu at tail\n",
1001 __func__, lseg, lseg->pls_range.iomode,
1002 lseg->pls_range.offset, lseg->pls_range.length);
1003out:
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001004 pnfs_get_layout_hdr(lo);
Andy Adamson974cec82010-10-20 00:18:02 -04001005
1006 dprintk("%s:Return\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001007}
1008
1009static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -04001010alloc_init_layout_hdr(struct inode *ino,
1011 struct nfs_open_context *ctx,
1012 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001013{
1014 struct pnfs_layout_hdr *lo;
1015
Benny Halevy636fb9c2011-05-22 19:51:33 +03001016 lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -04001017 if (!lo)
1018 return NULL;
Fred Isamancc6e5342011-01-06 11:36:28 +00001019 atomic_set(&lo->plh_refcount, 1);
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001020 INIT_LIST_HEAD(&lo->plh_layouts);
1021 INIT_LIST_HEAD(&lo->plh_segs);
Trond Myklebustfd9a8d72013-02-12 09:48:42 -05001022 INIT_LIST_HEAD(&lo->plh_bulk_destroy);
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001023 lo->plh_inode = ino;
Peng Tao9fa40752011-07-30 20:52:32 -04001024 lo->plh_lc_cred = get_rpccred(ctx->state->owner->so_cred);
Benny Halevye5e94012010-10-20 00:18:01 -04001025 return lo;
1026}
1027
1028static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -04001029pnfs_find_alloc_layout(struct inode *ino,
1030 struct nfs_open_context *ctx,
1031 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001032{
1033 struct nfs_inode *nfsi = NFS_I(ino);
1034 struct pnfs_layout_hdr *new = NULL;
1035
1036 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);
1037
Trond Myklebust251ec412012-10-02 15:41:05 -07001038 if (nfsi->layout != NULL)
1039 goto out_existing;
Benny Halevye5e94012010-10-20 00:18:01 -04001040 spin_unlock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001041 new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -04001042 spin_lock(&ino->i_lock);
1043
Trond Myklebust251ec412012-10-02 15:41:05 -07001044 if (likely(nfsi->layout == NULL)) { /* Won the race? */
Benny Halevye5e94012010-10-20 00:18:01 -04001045 nfsi->layout = new;
Trond Myklebust251ec412012-10-02 15:41:05 -07001046 return new;
Yanchuan Nian7175fe92012-10-31 16:05:48 +08001047 } else if (new != NULL)
1048 pnfs_free_layout_hdr(new);
Trond Myklebust251ec412012-10-02 15:41:05 -07001049out_existing:
1050 pnfs_get_layout_hdr(nfsi->layout);
Benny Halevye5e94012010-10-20 00:18:01 -04001051 return nfsi->layout;
1052}
1053
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001054/*
1055 * iomode matching rules:
1056 * iomode lseg match
1057 * ----- ----- -----
1058 * ANY READ true
1059 * ANY RW true
1060 * RW READ false
1061 * RW RW true
1062 * READ READ true
1063 * READ RW true
1064 */
1065static int
Benny Halevyfb3296e2011-05-22 19:47:26 +03001066is_matching_lseg(struct pnfs_layout_range *ls_range,
1067 struct pnfs_layout_range *range)
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001068{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001069 struct pnfs_layout_range range1;
1070
1071 if ((range->iomode == IOMODE_RW &&
1072 ls_range->iomode != IOMODE_RW) ||
1073 !lo_seg_intersecting(ls_range, range))
1074 return 0;
1075
1076 /* range1 covers only the first byte in the range */
1077 range1 = *range;
1078 range1.length = 1;
1079 return lo_seg_contained(ls_range, &range1);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001080}
1081
1082/*
1083 * lookup range in layout
1084 */
Benny Halevye5e94012010-10-20 00:18:01 -04001085static struct pnfs_layout_segment *
Benny Halevyfb3296e2011-05-22 19:47:26 +03001086pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1087 struct pnfs_layout_range *range)
Benny Halevye5e94012010-10-20 00:18:01 -04001088{
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001089 struct pnfs_layout_segment *lseg, *ret = NULL;
1090
1091 dprintk("%s:Begin\n", __func__);
1092
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001093 list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
Fred Isaman4541d162011-01-06 11:36:23 +00001094 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
Benny Halevyfb3296e2011-05-22 19:47:26 +03001095 is_matching_lseg(&lseg->pls_range, range)) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001096 ret = pnfs_get_lseg(lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001097 break;
1098 }
Benny Halevyd771e3a2011-06-14 16:30:16 -04001099 if (lseg->pls_range.offset > range->offset)
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001100 break;
1101 }
1102
1103 dprintk("%s:Return lseg %p ref %d\n",
Fred Isaman4541d162011-01-06 11:36:23 +00001104 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001105 return ret;
Benny Halevye5e94012010-10-20 00:18:01 -04001106}
1107
1108/*
Andy Adamsond23d61c2012-05-23 05:02:37 -04001109 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
1110 * to the MDS or over pNFS
1111 *
1112 * The nfs_inode read_io and write_io fields are cumulative counters reset
1113 * when there are no layout segments. Note that in pnfs_update_layout iomode
1114 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
1115 * WRITE request.
1116 *
1117 * A return of true means use MDS I/O.
1118 *
1119 * From rfc 5661:
1120 * If a file's size is smaller than the file size threshold, data accesses
1121 * SHOULD be sent to the metadata server. If an I/O request has a length that
1122 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
1123 * server. If both file size and I/O size are provided, the client SHOULD
1124 * reach or exceed both thresholds before sending its read or write
1125 * requests to the data server.
1126 */
1127static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
1128 struct inode *ino, int iomode)
1129{
1130 struct nfs4_threshold *t = ctx->mdsthreshold;
1131 struct nfs_inode *nfsi = NFS_I(ino);
1132 loff_t fsize = i_size_read(ino);
1133 bool size = false, size_set = false, io = false, io_set = false, ret = false;
1134
1135 if (t == NULL)
1136 return ret;
1137
1138 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
1139 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);
1140
1141 switch (iomode) {
1142 case IOMODE_READ:
1143 if (t->bm & THRESHOLD_RD) {
1144 dprintk("%s fsize %llu\n", __func__, fsize);
1145 size_set = true;
1146 if (fsize < t->rd_sz)
1147 size = true;
1148 }
1149 if (t->bm & THRESHOLD_RD_IO) {
1150 dprintk("%s nfsi->read_io %llu\n", __func__,
1151 nfsi->read_io);
1152 io_set = true;
1153 if (nfsi->read_io < t->rd_io_sz)
1154 io = true;
1155 }
1156 break;
1157 case IOMODE_RW:
1158 if (t->bm & THRESHOLD_WR) {
1159 dprintk("%s fsize %llu\n", __func__, fsize);
1160 size_set = true;
1161 if (fsize < t->wr_sz)
1162 size = true;
1163 }
1164 if (t->bm & THRESHOLD_WR_IO) {
1165 dprintk("%s nfsi->write_io %llu\n", __func__,
1166 nfsi->write_io);
1167 io_set = true;
1168 if (nfsi->write_io < t->wr_io_sz)
1169 io = true;
1170 }
1171 break;
1172 }
1173 if (size_set && io_set) {
1174 if (size && io)
1175 ret = true;
1176 } else if (size || io)
1177 ret = true;
1178
1179 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
1180 return ret;
1181}
1182
1183/*
Benny Halevye5e94012010-10-20 00:18:01 -04001184 * Layout segment is retreived from the server if not cached.
1185 * The appropriate layout segment is referenced and returned to the caller.
1186 */
Andy Adamson7c24d942011-06-13 18:22:38 -04001187struct pnfs_layout_segment *
Benny Halevye5e94012010-10-20 00:18:01 -04001188pnfs_update_layout(struct inode *ino,
1189 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +03001190 loff_t pos,
1191 u64 count,
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001192 enum pnfs_iomode iomode,
1193 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001194{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001195 struct pnfs_layout_range arg = {
1196 .iomode = iomode,
1197 .offset = pos,
1198 .length = count,
1199 };
Benny Halevy707ed5f2011-05-22 19:47:46 +03001200 unsigned pg_offset;
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001201 struct nfs_server *server = NFS_SERVER(ino);
1202 struct nfs_client *clp = server->nfs_client;
Benny Halevye5e94012010-10-20 00:18:01 -04001203 struct pnfs_layout_hdr *lo;
1204 struct pnfs_layout_segment *lseg = NULL;
Weston Andros Adamson30005122013-02-28 20:30:10 -05001205 bool first;
Benny Halevye5e94012010-10-20 00:18:01 -04001206
1207 if (!pnfs_enabled_sb(NFS_SERVER(ino)))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001208 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001209
1210 if (pnfs_within_mdsthreshold(ctx, ino, iomode))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001211 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001212
Benny Halevye5e94012010-10-20 00:18:01 -04001213 spin_lock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001214 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001215 if (lo == NULL) {
1216 spin_unlock(&ino->i_lock);
1217 goto out;
1218 }
Benny Halevye5e94012010-10-20 00:18:01 -04001219
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001220 /* Do we even need to bother with this? */
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001221 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001222 dprintk("%s matches recall, use MDS\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001223 goto out_unlock;
1224 }
1225
1226 /* if LAYOUTGET already failed once we don't try again */
Trond Myklebustb9e028f2012-09-18 16:41:18 -04001227 if (pnfs_layout_io_test_failed(lo, iomode))
Benny Halevye5e94012010-10-20 00:18:01 -04001228 goto out_unlock;
1229
Andy Adamson568e8c42011-03-01 01:34:22 +00001230 /* Check to see if the layout for the given range already exists */
Benny Halevyfb3296e2011-05-22 19:47:26 +03001231 lseg = pnfs_find_lseg(lo, &arg);
Andy Adamson568e8c42011-03-01 01:34:22 +00001232 if (lseg)
1233 goto out_unlock;
1234
Trond Myklebust19c54ab2012-10-05 16:56:58 -07001235 if (pnfs_layoutgets_blocked(lo, 0))
Fred Isamancf7d63f2011-01-06 11:36:25 +00001236 goto out_unlock;
1237 atomic_inc(&lo->plh_outstanding);
1238
Weston Andros Adamson30005122013-02-28 20:30:10 -05001239 first = list_empty(&lo->plh_layouts) ? true : false;
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001240 spin_unlock(&ino->i_lock);
Weston Andros Adamson30005122013-02-28 20:30:10 -05001241
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001242 if (first) {
Fred Isaman2130ff62011-01-06 11:36:26 +00001243 /* The lo must be on the clp list if there is any
1244 * chance of a CB_LAYOUTRECALL(FILE) coming in.
1245 */
1246 spin_lock(&clp->cl_lock);
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001247 list_add_tail(&lo->plh_layouts, &server->layouts);
Fred Isaman2130ff62011-01-06 11:36:26 +00001248 spin_unlock(&clp->cl_lock);
1249 }
Benny Halevye5e94012010-10-20 00:18:01 -04001250
Benny Halevy707ed5f2011-05-22 19:47:46 +03001251 pg_offset = arg.offset & ~PAGE_CACHE_MASK;
1252 if (pg_offset) {
1253 arg.offset -= pg_offset;
1254 arg.length += pg_offset;
1255 }
Andy Adamson7c24d942011-06-13 18:22:38 -04001256 if (arg.length != NFS4_MAX_UINT64)
1257 arg.length = PAGE_CACHE_ALIGN(arg.length);
Benny Halevy707ed5f2011-05-22 19:47:46 +03001258
Benny Halevyfb3296e2011-05-22 19:47:26 +03001259 lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +00001260 atomic_dec(&lo->plh_outstanding);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001261out_put_layout_hdr:
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001262 pnfs_put_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -04001263out:
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001264 dprintk("%s: inode %s/%llu pNFS layout segment %s for "
1265 "(%s, offset: %llu, length: %llu)\n",
1266 __func__, ino->i_sb->s_id,
1267 (unsigned long long)NFS_FILEID(ino),
1268 lseg == NULL ? "not found" : "found",
1269 iomode==IOMODE_RW ? "read/write" : "read-only",
1270 (unsigned long long)pos,
1271 (unsigned long long)count);
Benny Halevye5e94012010-10-20 00:18:01 -04001272 return lseg;
1273out_unlock:
1274 spin_unlock(&ino->i_lock);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001275 goto out_put_layout_hdr;
Benny Halevye5e94012010-10-20 00:18:01 -04001276}
Andy Adamson7c24d942011-06-13 18:22:38 -04001277EXPORT_SYMBOL_GPL(pnfs_update_layout);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001278
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001279struct pnfs_layout_segment *
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001280pnfs_layout_process(struct nfs4_layoutget *lgp)
1281{
1282 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
1283 struct nfs4_layoutget_res *res = &lgp->res;
1284 struct pnfs_layout_segment *lseg;
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001285 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001286 int status = 0;
1287
1288 /* Inject layout blob into I/O device driver */
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001289 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001290 if (!lseg || IS_ERR(lseg)) {
1291 if (!lseg)
1292 status = -ENOMEM;
1293 else
1294 status = PTR_ERR(lseg);
1295 dprintk("%s: Could not allocate layout: error %d\n",
1296 __func__, status);
1297 goto out;
1298 }
1299
1300 spin_lock(&ino->i_lock);
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001301 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001302 dprintk("%s forget reply due to recall\n", __func__);
1303 goto out_forget_reply;
1304 }
1305
Trond Myklebust19c54ab2012-10-05 16:56:58 -07001306 if (pnfs_layoutgets_blocked(lo, 1) ||
1307 pnfs_layout_stateid_blocked(lo, &res->stateid)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001308 dprintk("%s forget reply due to state\n", __func__);
1309 goto out_forget_reply;
1310 }
Trond Myklebust038d6492012-10-02 16:38:41 -07001311
1312 /* Done processing layoutget. Set the layout stateid */
1313 pnfs_set_layout_stateid(lo, &res->stateid, false);
1314
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001315 init_lseg(lo, lseg);
Fred Isaman566052c2011-01-06 11:36:20 +00001316 lseg->pls_range = res->range;
Trond Myklebust9369a432012-09-18 20:57:08 -04001317 pnfs_get_lseg(lseg);
Trond Myklebust57036a32012-09-20 16:33:30 -04001318 pnfs_layout_insert_lseg(lo, lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001319
Fred Isamanf7e89172011-01-06 11:36:32 +00001320 if (res->return_on_close) {
1321 set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
1322 set_bit(NFS_LAYOUT_ROC, &lo->plh_flags);
1323 }
1324
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001325 spin_unlock(&ino->i_lock);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001326 return lseg;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001327out:
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001328 return ERR_PTR(status);
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001329
1330out_forget_reply:
1331 spin_unlock(&ino->i_lock);
1332 lseg->pls_layout = lo;
1333 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1334 goto out;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001335}
1336
Trond Myklebustd8007d42011-06-10 13:30:23 -04001337void
1338pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
1339{
Peng Tao1fd937b2012-09-25 14:55:57 +08001340 u64 rd_size = req->wb_bytes;
1341
Trond Myklebustbc5a89b2012-10-15 14:58:04 -04001342 WARN_ON_ONCE(pgio->pg_lseg != NULL);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001343
Fred Isaman1825a0d2012-04-20 19:55:31 -04001344 if (req->wb_offset != req->wb_pgbase) {
1345 nfs_pageio_reset_read_mds(pgio);
1346 return;
1347 }
Peng Tao1fd937b2012-09-25 14:55:57 +08001348
1349 if (pgio->pg_dreq == NULL)
1350 rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
1351 else
1352 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
1353
Trond Myklebustd8007d42011-06-10 13:30:23 -04001354 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1355 req->wb_context,
1356 req_offset(req),
Peng Tao1fd937b2012-09-25 14:55:57 +08001357 rd_size,
Trond Myklebustd8007d42011-06-10 13:30:23 -04001358 IOMODE_READ,
1359 GFP_KERNEL);
Trond Myklebuste885de12011-06-10 13:30:23 -04001360 /* If no lseg, fall back to read through mds */
1361 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001362 nfs_pageio_reset_read_mds(pgio);
Trond Myklebuste885de12011-06-10 13:30:23 -04001363
Trond Myklebustd8007d42011-06-10 13:30:23 -04001364}
1365EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);
1366
1367void
Peng Tao62965562012-09-25 14:55:57 +08001368pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
1369 struct nfs_page *req, u64 wb_size)
Trond Myklebustd8007d42011-06-10 13:30:23 -04001370{
Trond Myklebustbc5a89b2012-10-15 14:58:04 -04001371 WARN_ON_ONCE(pgio->pg_lseg != NULL);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001372
Fred Isaman1825a0d2012-04-20 19:55:31 -04001373 if (req->wb_offset != req->wb_pgbase) {
1374 nfs_pageio_reset_write_mds(pgio);
1375 return;
1376 }
Peng Tao62965562012-09-25 14:55:57 +08001377
Trond Myklebustd8007d42011-06-10 13:30:23 -04001378 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1379 req->wb_context,
1380 req_offset(req),
Peng Tao62965562012-09-25 14:55:57 +08001381 wb_size,
Trond Myklebustd8007d42011-06-10 13:30:23 -04001382 IOMODE_RW,
1383 GFP_NOFS);
Trond Myklebuste885de12011-06-10 13:30:23 -04001384 /* If no lseg, fall back to write through mds */
1385 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001386 nfs_pageio_reset_write_mds(pgio);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001387}
1388EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
1389
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001390void
Fred Isaman061ae2e2012-04-20 14:47:48 -04001391pnfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, struct inode *inode,
1392 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust1751c362011-06-10 13:30:23 -04001393{
1394 struct nfs_server *server = NFS_SERVER(inode);
1395 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
1396
1397 if (ld == NULL)
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001398 nfs_pageio_init_read(pgio, inode, compl_ops);
1399 else
1400 nfs_pageio_init(pgio, inode, ld->pg_read_ops, compl_ops, server->rsize, 0);
Trond Myklebust1751c362011-06-10 13:30:23 -04001401}
1402
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001403void
Fred Isaman061ae2e2012-04-20 14:47:48 -04001404pnfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, struct inode *inode,
1405 int ioflags,
1406 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust1751c362011-06-10 13:30:23 -04001407{
1408 struct nfs_server *server = NFS_SERVER(inode);
1409 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
1410
1411 if (ld == NULL)
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001412 nfs_pageio_init_write(pgio, inode, ioflags, compl_ops);
1413 else
1414 nfs_pageio_init(pgio, inode, ld->pg_write_ops, compl_ops, server->wsize, ioflags);
Trond Myklebust1751c362011-06-10 13:30:23 -04001415}
1416
1417bool
Benny Halevydfed2062011-05-25 20:25:22 +03001418pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
1419 struct nfs_page *req)
Fred Isamanbae724e2011-03-01 01:34:15 +00001420{
Trond Myklebustd8007d42011-06-10 13:30:23 -04001421 if (pgio->pg_lseg == NULL)
1422 return nfs_generic_pg_test(pgio, prev, req);
Benny Halevy89a58e32011-05-25 20:54:40 +03001423
Trond Myklebust19982ba2011-06-10 13:30:23 -04001424 /*
1425 * Test if a nfs_page is fully contained in the pnfs_layout_range.
1426 * Note that this test makes several assumptions:
1427 * - that the previous nfs_page in the struct nfs_pageio_descriptor
1428 * is known to lie within the range.
1429 * - that the nfs_page being tested is known to be contiguous with the
1430 * previous nfs_page.
1431 * - Layout ranges are page aligned, so we only have to test the
1432 * start offset of the request.
1433 *
1434 * Please also note that 'end_offset' is actually the offset of the
1435 * first byte that lies outside the pnfs_layout_range. FIXME?
1436 *
1437 */
1438 return req_offset(req) < end_offset(pgio->pg_lseg->pls_range.offset,
1439 pgio->pg_lseg->pls_range.length);
Fred Isamanbae724e2011-03-01 01:34:15 +00001440}
Benny Halevy89a58e32011-05-25 20:54:40 +03001441EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
Fred Isamanbae724e2011-03-01 01:34:15 +00001442
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001443int pnfs_write_done_resend_to_mds(struct inode *inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001444 struct list_head *head,
Benny Halevy78f33272013-02-24 09:55:57 -05001445 const struct nfs_pgio_completion_ops *compl_ops,
1446 struct nfs_direct_req *dreq)
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001447{
1448 struct nfs_pageio_descriptor pgio;
1449 LIST_HEAD(failed);
1450
1451 /* Resend all requests through the MDS */
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001452 nfs_pageio_init_write(&pgio, inode, FLUSH_STABLE, compl_ops);
Benny Halevy78f33272013-02-24 09:55:57 -05001453 pgio.pg_dreq = dreq;
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001454 while (!list_empty(head)) {
1455 struct nfs_page *req = nfs_list_entry(head->next);
1456
1457 nfs_list_remove_request(req);
1458 if (!nfs_pageio_add_request(&pgio, req))
1459 nfs_list_add_request(req, &failed);
1460 }
1461 nfs_pageio_complete(&pgio);
1462
1463 if (!list_empty(&failed)) {
1464 /* For some reason our attempt to resend pages. Mark the
1465 * overall send request as having failed, and let
1466 * nfs_writeback_release_full deal with the error.
1467 */
1468 list_move(&failed, head);
1469 return -EIO;
1470 }
1471 return 0;
1472}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001473EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001474
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001475static void pnfs_ld_handle_write_error(struct nfs_write_data *data)
1476{
Fred Isamancd841602012-04-20 14:47:44 -04001477 struct nfs_pgio_header *hdr = data->header;
1478
1479 dprintk("pnfs write error = %d\n", hdr->pnfs_error);
1480 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001481 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001482 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001483 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04001484 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1485 data->task.tk_status = pnfs_write_done_resend_to_mds(hdr->inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001486 &hdr->pages,
Benny Halevy78f33272013-02-24 09:55:57 -05001487 hdr->completion_ops,
1488 hdr->dreq);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001489}
1490
Benny Halevyd20581a2011-05-22 19:52:03 +03001491/*
1492 * Called by non rpc-based layout drivers
1493 */
Peng Tao8ce160c2011-09-22 21:50:14 -04001494void pnfs_ld_write_done(struct nfs_write_data *data)
Fred Isaman94ad1c82011-03-01 01:34:14 +00001495{
Fred Isamancd841602012-04-20 14:47:44 -04001496 struct nfs_pgio_header *hdr = data->header;
1497
1498 if (!hdr->pnfs_error) {
Benny Halevyd20581a2011-05-22 19:52:03 +03001499 pnfs_set_layoutcommit(data);
Fred Isamancd841602012-04-20 14:47:44 -04001500 hdr->mds_ops->rpc_call_done(&data->task, data);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001501 } else
1502 pnfs_ld_handle_write_error(data);
Fred Isamancd841602012-04-20 14:47:44 -04001503 hdr->mds_ops->rpc_release(data);
Fred Isaman44b83792011-03-03 15:13:44 +00001504}
Benny Halevyd20581a2011-05-22 19:52:03 +03001505EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
Fred Isaman44b83792011-03-03 15:13:44 +00001506
Trond Myklebustdce81292011-07-13 15:59:19 -04001507static void
1508pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1509 struct nfs_write_data *data)
1510{
Fred Isamancd841602012-04-20 14:47:44 -04001511 struct nfs_pgio_header *hdr = data->header;
1512
Fred Isaman6c75dc02012-04-20 14:47:47 -04001513 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1514 list_splice_tail_init(&hdr->pages, &desc->pg_list);
1515 nfs_pageio_reset_write_mds(desc);
1516 desc->pg_recoalesce = 1;
1517 }
Trond Myklebustdce81292011-07-13 15:59:19 -04001518 nfs_writedata_release(data);
1519}
1520
1521static enum pnfs_try_status
Andy Adamson0382b742011-03-03 15:13:45 +00001522pnfs_try_to_write_data(struct nfs_write_data *wdata,
Trond Myklebustdce81292011-07-13 15:59:19 -04001523 const struct rpc_call_ops *call_ops,
1524 struct pnfs_layout_segment *lseg,
1525 int how)
Andy Adamson0382b742011-03-03 15:13:45 +00001526{
Fred Isamancd841602012-04-20 14:47:44 -04001527 struct nfs_pgio_header *hdr = wdata->header;
1528 struct inode *inode = hdr->inode;
Andy Adamson0382b742011-03-03 15:13:45 +00001529 enum pnfs_try_status trypnfs;
1530 struct nfs_server *nfss = NFS_SERVER(inode);
1531
Fred Isamancd841602012-04-20 14:47:44 -04001532 hdr->mds_ops = call_ops;
Andy Adamson0382b742011-03-03 15:13:45 +00001533
1534 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
1535 inode->i_ino, wdata->args.count, wdata->args.offset, how);
Andy Adamson0382b742011-03-03 15:13:45 +00001536 trypnfs = nfss->pnfs_curr_ld->write_pagelist(wdata, how);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001537 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson0382b742011-03-03 15:13:45 +00001538 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
Andy Adamson0382b742011-03-03 15:13:45 +00001539 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1540 return trypnfs;
1541}
1542
Trond Myklebustdce81292011-07-13 15:59:19 -04001543static void
1544pnfs_do_multiple_writes(struct nfs_pageio_descriptor *desc, struct list_head *head, int how)
1545{
1546 struct nfs_write_data *data;
1547 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1548 struct pnfs_layout_segment *lseg = desc->pg_lseg;
1549
1550 desc->pg_lseg = NULL;
1551 while (!list_empty(head)) {
1552 enum pnfs_try_status trypnfs;
1553
Fred Isaman6c75dc02012-04-20 14:47:47 -04001554 data = list_first_entry(head, struct nfs_write_data, list);
Trond Myklebustdce81292011-07-13 15:59:19 -04001555 list_del_init(&data->list);
1556
1557 trypnfs = pnfs_try_to_write_data(data, call_ops, lseg, how);
1558 if (trypnfs == PNFS_NOT_ATTEMPTED)
1559 pnfs_write_through_mds(desc, data);
1560 }
Trond Myklebust9369a432012-09-18 20:57:08 -04001561 pnfs_put_lseg(lseg);
Trond Myklebustdce81292011-07-13 15:59:19 -04001562}
1563
Fred Isaman6c75dc02012-04-20 14:47:47 -04001564static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
1565{
Trond Myklebust9369a432012-09-18 20:57:08 -04001566 pnfs_put_lseg(hdr->lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001567 nfs_writehdr_free(hdr);
1568}
Bryan Schumaker89d77c82012-07-30 16:05:25 -04001569EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001570
Trond Myklebustdce81292011-07-13 15:59:19 -04001571int
1572pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
1573{
Fred Isaman6c75dc02012-04-20 14:47:47 -04001574 struct nfs_write_header *whdr;
1575 struct nfs_pgio_header *hdr;
Trond Myklebustdce81292011-07-13 15:59:19 -04001576 int ret;
1577
Fred Isaman6c75dc02012-04-20 14:47:47 -04001578 whdr = nfs_writehdr_alloc();
1579 if (!whdr) {
Trond Myklebust9b5415b2012-04-27 14:31:47 -04001580 desc->pg_completion_ops->error_cleanup(&desc->pg_list);
Trond Myklebust9369a432012-09-18 20:57:08 -04001581 pnfs_put_lseg(desc->pg_lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001582 desc->pg_lseg = NULL;
1583 return -ENOMEM;
1584 }
1585 hdr = &whdr->header;
1586 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04001587 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001588 atomic_inc(&hdr->refcnt);
1589 ret = nfs_generic_flush(desc, hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04001590 if (ret != 0) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001591 pnfs_put_lseg(desc->pg_lseg);
Trond Myklebustdce81292011-07-13 15:59:19 -04001592 desc->pg_lseg = NULL;
Fred Isaman6c75dc02012-04-20 14:47:47 -04001593 } else
1594 pnfs_do_multiple_writes(desc, &hdr->rpc_list, desc->pg_ioflags);
1595 if (atomic_dec_and_test(&hdr->refcnt))
Fred Isaman061ae2e2012-04-20 14:47:48 -04001596 hdr->completion_ops->completion(hdr);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001597 return ret;
Trond Myklebustdce81292011-07-13 15:59:19 -04001598}
1599EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
1600
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001601int pnfs_read_done_resend_to_mds(struct inode *inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001602 struct list_head *head,
Benny Halevy78f33272013-02-24 09:55:57 -05001603 const struct nfs_pgio_completion_ops *compl_ops,
1604 struct nfs_direct_req *dreq)
Trond Myklebust62e4a762011-11-10 14:30:37 -05001605{
1606 struct nfs_pageio_descriptor pgio;
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001607 LIST_HEAD(failed);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001608
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001609 /* Resend all requests through the MDS */
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001610 nfs_pageio_init_read(&pgio, inode, compl_ops);
Benny Halevy78f33272013-02-24 09:55:57 -05001611 pgio.pg_dreq = dreq;
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001612 while (!list_empty(head)) {
1613 struct nfs_page *req = nfs_list_entry(head->next);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001614
1615 nfs_list_remove_request(req);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001616 if (!nfs_pageio_add_request(&pgio, req))
1617 nfs_list_add_request(req, &failed);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001618 }
1619 nfs_pageio_complete(&pgio);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001620
1621 if (!list_empty(&failed)) {
1622 list_move(&failed, head);
1623 return -EIO;
1624 }
1625 return 0;
1626}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001627EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001628
1629static void pnfs_ld_handle_read_error(struct nfs_read_data *data)
1630{
Fred Isamancd841602012-04-20 14:47:44 -04001631 struct nfs_pgio_header *hdr = data->header;
1632
1633 dprintk("pnfs read error = %d\n", hdr->pnfs_error);
1634 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001635 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001636 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001637 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001638 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1639 data->task.tk_status = pnfs_read_done_resend_to_mds(hdr->inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001640 &hdr->pages,
Benny Halevy78f33272013-02-24 09:55:57 -05001641 hdr->completion_ops,
1642 hdr->dreq);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001643}
1644
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001645/*
Benny Halevyd20581a2011-05-22 19:52:03 +03001646 * Called by non rpc-based layout drivers
1647 */
Peng Tao9b7eecd2011-09-22 21:50:15 -04001648void pnfs_ld_read_done(struct nfs_read_data *data)
Benny Halevyd20581a2011-05-22 19:52:03 +03001649{
Fred Isamancd841602012-04-20 14:47:44 -04001650 struct nfs_pgio_header *hdr = data->header;
1651
1652 if (likely(!hdr->pnfs_error)) {
Benny Halevyd20581a2011-05-22 19:52:03 +03001653 __nfs4_read_done_cb(data);
Fred Isamancd841602012-04-20 14:47:44 -04001654 hdr->mds_ops->rpc_call_done(&data->task, data);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001655 } else
1656 pnfs_ld_handle_read_error(data);
Fred Isamancd841602012-04-20 14:47:44 -04001657 hdr->mds_ops->rpc_release(data);
Benny Halevyd20581a2011-05-22 19:52:03 +03001658}
1659EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
1660
Trond Myklebust493292d2011-07-13 15:58:28 -04001661static void
1662pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
1663 struct nfs_read_data *data)
1664{
Fred Isamancd841602012-04-20 14:47:44 -04001665 struct nfs_pgio_header *hdr = data->header;
1666
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001667 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1668 list_splice_tail_init(&hdr->pages, &desc->pg_list);
1669 nfs_pageio_reset_read_mds(desc);
1670 desc->pg_recoalesce = 1;
1671 }
Trond Myklebust493292d2011-07-13 15:58:28 -04001672 nfs_readdata_release(data);
1673}
1674
Benny Halevyd20581a2011-05-22 19:52:03 +03001675/*
Andy Adamson64419a92011-03-01 01:34:16 +00001676 * Call the appropriate parallel I/O subsystem read function.
1677 */
Trond Myklebust493292d2011-07-13 15:58:28 -04001678static enum pnfs_try_status
Andy Adamson64419a92011-03-01 01:34:16 +00001679pnfs_try_to_read_data(struct nfs_read_data *rdata,
Trond Myklebust493292d2011-07-13 15:58:28 -04001680 const struct rpc_call_ops *call_ops,
1681 struct pnfs_layout_segment *lseg)
Andy Adamson64419a92011-03-01 01:34:16 +00001682{
Fred Isamancd841602012-04-20 14:47:44 -04001683 struct nfs_pgio_header *hdr = rdata->header;
1684 struct inode *inode = hdr->inode;
Andy Adamson64419a92011-03-01 01:34:16 +00001685 struct nfs_server *nfss = NFS_SERVER(inode);
1686 enum pnfs_try_status trypnfs;
1687
Fred Isamancd841602012-04-20 14:47:44 -04001688 hdr->mds_ops = call_ops;
Andy Adamson64419a92011-03-01 01:34:16 +00001689
1690 dprintk("%s: Reading ino:%lu %u@%llu\n",
1691 __func__, inode->i_ino, rdata->args.count, rdata->args.offset);
1692
1693 trypnfs = nfss->pnfs_curr_ld->read_pagelist(rdata);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001694 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson64419a92011-03-01 01:34:16 +00001695 nfs_inc_stats(inode, NFSIOS_PNFS_READ);
Andy Adamson64419a92011-03-01 01:34:16 +00001696 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1697 return trypnfs;
1698}
Andy Adamson863a3c62011-03-23 13:27:54 +00001699
Trond Myklebust493292d2011-07-13 15:58:28 -04001700static void
1701pnfs_do_multiple_reads(struct nfs_pageio_descriptor *desc, struct list_head *head)
1702{
1703 struct nfs_read_data *data;
1704 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1705 struct pnfs_layout_segment *lseg = desc->pg_lseg;
1706
1707 desc->pg_lseg = NULL;
1708 while (!list_empty(head)) {
1709 enum pnfs_try_status trypnfs;
1710
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001711 data = list_first_entry(head, struct nfs_read_data, list);
Trond Myklebust493292d2011-07-13 15:58:28 -04001712 list_del_init(&data->list);
1713
1714 trypnfs = pnfs_try_to_read_data(data, call_ops, lseg);
1715 if (trypnfs == PNFS_NOT_ATTEMPTED)
1716 pnfs_read_through_mds(desc, data);
1717 }
Trond Myklebust9369a432012-09-18 20:57:08 -04001718 pnfs_put_lseg(lseg);
Trond Myklebust493292d2011-07-13 15:58:28 -04001719}
1720
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001721static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
1722{
Trond Myklebust9369a432012-09-18 20:57:08 -04001723 pnfs_put_lseg(hdr->lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001724 nfs_readhdr_free(hdr);
1725}
Bryan Schumaker89d77c82012-07-30 16:05:25 -04001726EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001727
Trond Myklebust493292d2011-07-13 15:58:28 -04001728int
1729pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
1730{
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001731 struct nfs_read_header *rhdr;
1732 struct nfs_pgio_header *hdr;
Trond Myklebust493292d2011-07-13 15:58:28 -04001733 int ret;
1734
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001735 rhdr = nfs_readhdr_alloc();
1736 if (!rhdr) {
Fred Isaman061ae2e2012-04-20 14:47:48 -04001737 desc->pg_completion_ops->error_cleanup(&desc->pg_list);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001738 ret = -ENOMEM;
Trond Myklebust9369a432012-09-18 20:57:08 -04001739 pnfs_put_lseg(desc->pg_lseg);
Trond Myklebust493292d2011-07-13 15:58:28 -04001740 desc->pg_lseg = NULL;
1741 return ret;
1742 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001743 hdr = &rhdr->header;
1744 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04001745 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001746 atomic_inc(&hdr->refcnt);
1747 ret = nfs_generic_pagein(desc, hdr);
1748 if (ret != 0) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001749 pnfs_put_lseg(desc->pg_lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001750 desc->pg_lseg = NULL;
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001751 } else
1752 pnfs_do_multiple_reads(desc, &hdr->rpc_list);
1753 if (atomic_dec_and_test(&hdr->refcnt))
Fred Isaman061ae2e2012-04-20 14:47:48 -04001754 hdr->completion_ops->completion(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001755 return ret;
Trond Myklebust493292d2011-07-13 15:58:28 -04001756}
1757EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);
1758
Andy Adamson863a3c62011-03-23 13:27:54 +00001759/*
Peng Taoa9bae562011-07-30 20:52:33 -04001760 * There can be multiple RW segments.
Andy Adamson863a3c62011-03-23 13:27:54 +00001761 */
Peng Taoa9bae562011-07-30 20:52:33 -04001762static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
Andy Adamson863a3c62011-03-23 13:27:54 +00001763{
Peng Taoa9bae562011-07-30 20:52:33 -04001764 struct pnfs_layout_segment *lseg;
Andy Adamson863a3c62011-03-23 13:27:54 +00001765
Peng Taoa9bae562011-07-30 20:52:33 -04001766 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
1767 if (lseg->pls_range.iomode == IOMODE_RW &&
Trond Myklebusta073dbf2013-03-20 12:34:32 -04001768 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
Peng Taoa9bae562011-07-30 20:52:33 -04001769 list_add(&lseg->pls_lc_list, listp);
1770 }
Andy Adamson863a3c62011-03-23 13:27:54 +00001771}
1772
Trond Myklebusta073dbf2013-03-20 12:34:32 -04001773static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
1774{
1775 struct pnfs_layout_segment *lseg, *tmp;
1776 unsigned long *bitlock = &NFS_I(inode)->flags;
1777
1778 /* Matched by references in pnfs_set_layoutcommit */
1779 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
1780 list_del_init(&lseg->pls_lc_list);
1781 pnfs_put_lseg(lseg);
1782 }
1783
1784 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
1785 smp_mb__after_clear_bit();
1786 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
1787}
1788
Peng Tao1b0ae062011-09-22 21:50:12 -04001789void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
1790{
Trond Myklebustb9e028f2012-09-18 16:41:18 -04001791 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
Peng Tao1b0ae062011-09-22 21:50:12 -04001792}
1793EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
1794
Andy Adamson863a3c62011-03-23 13:27:54 +00001795void
1796pnfs_set_layoutcommit(struct nfs_write_data *wdata)
1797{
Fred Isamancd841602012-04-20 14:47:44 -04001798 struct nfs_pgio_header *hdr = wdata->header;
1799 struct inode *inode = hdr->inode;
1800 struct nfs_inode *nfsi = NFS_I(inode);
Vitaliy Gusev4b8ee2b2011-05-20 01:34:46 +04001801 loff_t end_pos = wdata->mds_offset + wdata->res.count;
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001802 bool mark_as_dirty = false;
Andy Adamson863a3c62011-03-23 13:27:54 +00001803
Fred Isamancd841602012-04-20 14:47:44 -04001804 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00001805 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001806 mark_as_dirty = true;
Andy Adamson863a3c62011-03-23 13:27:54 +00001807 dprintk("%s: Set layoutcommit for inode %lu ",
Fred Isamancd841602012-04-20 14:47:44 -04001808 __func__, inode->i_ino);
Andy Adamson863a3c62011-03-23 13:27:54 +00001809 }
Fred Isamancd841602012-04-20 14:47:44 -04001810 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &hdr->lseg->pls_flags)) {
Peng Taoa9bae562011-07-30 20:52:33 -04001811 /* references matched in nfs4_layoutcommit_release */
Trond Myklebust9369a432012-09-18 20:57:08 -04001812 pnfs_get_lseg(hdr->lseg);
Peng Taoa9bae562011-07-30 20:52:33 -04001813 }
Peng Taoacff58802011-07-30 20:52:31 -04001814 if (end_pos > nfsi->layout->plh_lwb)
1815 nfsi->layout->plh_lwb = end_pos;
Fred Isamancd841602012-04-20 14:47:44 -04001816 spin_unlock(&inode->i_lock);
Peng Taoacff58802011-07-30 20:52:31 -04001817 dprintk("%s: lseg %p end_pos %llu\n",
Fred Isamancd841602012-04-20 14:47:44 -04001818 __func__, hdr->lseg, nfsi->layout->plh_lwb);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001819
1820 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
1821 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
1822 if (mark_as_dirty)
Fred Isamancd841602012-04-20 14:47:44 -04001823 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001824}
1825EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
1826
Andy Adamsondb29c082011-07-30 20:52:38 -04001827void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
1828{
1829 struct nfs_server *nfss = NFS_SERVER(data->args.inode);
1830
1831 if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
1832 nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
Trond Myklebusta073dbf2013-03-20 12:34:32 -04001833 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
Andy Adamsondb29c082011-07-30 20:52:38 -04001834}
1835
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001836/*
1837 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
1838 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
1839 * data to disk to allow the server to recover the data if it crashes.
1840 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
1841 * is off, and a COMMIT is sent to a data server, or
1842 * if WRITEs to a data server return NFS_DATA_SYNC.
1843 */
Andy Adamson863a3c62011-03-23 13:27:54 +00001844int
Andy Adamsonef311532011-03-12 02:58:10 -05001845pnfs_layoutcommit_inode(struct inode *inode, bool sync)
Andy Adamson863a3c62011-03-23 13:27:54 +00001846{
1847 struct nfs4_layoutcommit_data *data;
1848 struct nfs_inode *nfsi = NFS_I(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001849 loff_t end_pos;
1850 int status = 0;
1851
1852 dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
1853
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001854 if (!test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1855 return 0;
1856
Andy Adamson863a3c62011-03-23 13:27:54 +00001857 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
1858 data = kzalloc(sizeof(*data), GFP_NOFS);
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001859 if (!data) {
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001860 status = -ENOMEM;
1861 goto out;
1862 }
Andy Adamson863a3c62011-03-23 13:27:54 +00001863
Peng Tao92407e72011-10-23 20:21:17 -07001864 if (!test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1865 goto out_free;
1866
1867 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
1868 if (!sync) {
1869 status = -EAGAIN;
1870 goto out_free;
1871 }
1872 status = wait_on_bit_lock(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING,
1873 nfs_wait_bit_killable, TASK_KILLABLE);
1874 if (status)
1875 goto out_free;
1876 }
1877
Peng Taoa9bae562011-07-30 20:52:33 -04001878 INIT_LIST_HEAD(&data->lseg_list);
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001879 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00001880 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Peng Tao92407e72011-10-23 20:21:17 -07001881 clear_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags);
Andy Adamson863a3c62011-03-23 13:27:54 +00001882 spin_unlock(&inode->i_lock);
Peng Tao92407e72011-10-23 20:21:17 -07001883 wake_up_bit(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING);
1884 goto out_free;
Andy Adamson863a3c62011-03-23 13:27:54 +00001885 }
Peng Taoa9bae562011-07-30 20:52:33 -04001886
1887 pnfs_list_write_lseg(inode, &data->lseg_list);
Andy Adamson863a3c62011-03-23 13:27:54 +00001888
Peng Taoacff58802011-07-30 20:52:31 -04001889 end_pos = nfsi->layout->plh_lwb;
Peng Taoacff58802011-07-30 20:52:31 -04001890 nfsi->layout->plh_lwb = 0;
Andy Adamson863a3c62011-03-23 13:27:54 +00001891
Trond Myklebustf597c532012-03-04 18:13:56 -05001892 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
Andy Adamson863a3c62011-03-23 13:27:54 +00001893 spin_unlock(&inode->i_lock);
1894
1895 data->args.inode = inode;
Peng Tao9fa40752011-07-30 20:52:32 -04001896 data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
Andy Adamson863a3c62011-03-23 13:27:54 +00001897 nfs_fattr_init(&data->fattr);
1898 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
1899 data->res.fattr = &data->fattr;
1900 data->args.lastbytewritten = end_pos - 1;
1901 data->res.server = NFS_SERVER(inode);
1902
1903 status = nfs4_proc_layoutcommit(data, sync);
1904out:
Peng Tao92407e72011-10-23 20:21:17 -07001905 if (status)
1906 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001907 dprintk("<-- %s status %d\n", __func__, status);
1908 return status;
Peng Tao92407e72011-10-23 20:21:17 -07001909out_free:
1910 kfree(data);
1911 goto out;
Andy Adamson863a3c62011-03-23 13:27:54 +00001912}
Andy Adamson82be4172012-05-23 05:02:35 -04001913
1914struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
1915{
1916 struct nfs4_threshold *thp;
1917
1918 thp = kzalloc(sizeof(*thp), GFP_NOFS);
1919 if (!thp) {
1920 dprintk("%s mdsthreshold allocation failed\n", __func__);
1921 return NULL;
1922 }
1923 return thp;
1924}