Branch data     Line data    Source code

       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : /*
       3                 :            :  * Copyright (c) 2016 Trond Myklebust
       4                 :            :  *
       5                 :            :  * I/O and data path helper functionality.
       6                 :            :  */
       7                 :            : 
       8                 :            : #include <linux/types.h>
       9                 :            : #include <linux/kernel.h>
      10                 :            : #include <linux/bitops.h>
      11                 :            : #include <linux/rwsem.h>
      12                 :            : #include <linux/fs.h>
      13                 :            : #include <linux/nfs_fs.h>
      14                 :            : 
      15                 :            : #include "internal.h"
      16                 :            : 
      17                 :            : /* Call with exclusively locked inode->i_rwsem */
      18                 :          0 : static void nfs_block_o_direct(struct nfs_inode *nfsi, struct inode *inode)
      19                 :            : {
      20         [ #  # ]:          0 :         if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) {
      21                 :          0 :                 clear_bit(NFS_INO_ODIRECT, &nfsi->flags);
      22                 :          0 :                 inode_dio_wait(inode);
      23                 :            :         }
      24                 :          0 : }
      25                 :            : 
      26                 :            : /**
      27                 :            :  * nfs_start_io_read - declare the file is being used for buffered reads
      28                 :            :  * @inode: file inode
      29                 :            :  *
      30                 :            :  * Declare that a buffered read operation is about to start, and ensure
      31                 :            :  * that we block all direct I/O.
      32                 :            :  * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset,
      33                 :            :  * and holds a shared lock on inode->i_rwsem to ensure that the flag
      34                 :            :  * cannot be changed.
      35                 :            :  * In practice, this means that buffered read operations are allowed to
      36                 :            :  * execute in parallel, thanks to the shared lock, whereas direct I/O
      37                 :            :  * operations need to wait to grab an exclusive lock in order to set
      38                 :            :  * NFS_INO_ODIRECT.
      39                 :            :  * Note that buffered writes and truncates both take a write lock on
      40                 :            :  * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
      41                 :            :  */
      42                 :            : void
      43                 :          0 : nfs_start_io_read(struct inode *inode)
      44                 :            : {
      45                 :          0 :         struct nfs_inode *nfsi = NFS_I(inode);
      46                 :            :         /* Be an optimist! */
      47                 :          0 :         down_read(&inode->i_rwsem);
      48         [ #  # ]:          0 :         if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0)
      49                 :            :                 return;
      50                 :          0 :         up_read(&inode->i_rwsem);
      51                 :            :         /* Slow path.... */
      52                 :          0 :         down_write(&inode->i_rwsem);
      53                 :          0 :         nfs_block_o_direct(nfsi, inode);
      54                 :          0 :         downgrade_write(&inode->i_rwsem);
      55                 :            : }
      56                 :            : 
      57                 :            : /**
      58                 :            :  * nfs_end_io_read - declare that the buffered read operation is done
      59                 :            :  * @inode: file inode
      60                 :            :  *
      61                 :            :  * Declare that a buffered read operation is done, and release the shared
      62                 :            :  * lock on inode->i_rwsem.
      63                 :            :  */
      64                 :            : void
      65                 :          0 : nfs_end_io_read(struct inode *inode)
      66                 :            : {
      67                 :          0 :         up_read(&inode->i_rwsem);
      68                 :          0 : }
      69                 :            : 
      70                 :            : /**
      71                 :            :  * nfs_start_io_write - declare the file is being used for buffered writes
      72                 :            :  * @inode: file inode
      73                 :            :  *
      74                 :            :  * Declare that a buffered read operation is about to start, and ensure
      75                 :            :  * that we block all direct I/O.
      76                 :            :  */
      77                 :            : void
      78                 :          0 : nfs_start_io_write(struct inode *inode)
      79                 :            : {
      80                 :          0 :         down_write(&inode->i_rwsem);
      81                 :          0 :         nfs_block_o_direct(NFS_I(inode), inode);
      82                 :          0 : }
      83                 :            : 
      84                 :            : /**
      85                 :            :  * nfs_end_io_write - declare that the buffered write operation is done
      86                 :            :  * @inode: file inode
      87                 :            :  *
      88                 :            :  * Declare that a buffered write operation is done, and release the
      89                 :            :  * lock on inode->i_rwsem.
      90                 :            :  */
      91                 :            : void
      92                 :          0 : nfs_end_io_write(struct inode *inode)
      93                 :            : {
      94                 :          0 :         up_write(&inode->i_rwsem);
      95                 :          0 : }
      96                 :            : 
      97                 :            : /* Call with exclusively locked inode->i_rwsem */
      98                 :            : static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode)
      99                 :            : {
     100                 :            :         if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) {
     101                 :            :                 set_bit(NFS_INO_ODIRECT, &nfsi->flags);
     102                 :            :                 nfs_sync_mapping(inode->i_mapping);
     103                 :            :         }
     104                 :            : }
     105                 :            : 
     106                 :            : /**
     107                 :            :  * nfs_end_io_direct - declare the file is being used for direct i/o
     108                 :            :  * @inode: file inode
     109                 :            :  *
     110                 :            :  * Declare that a direct I/O operation is about to start, and ensure
     111                 :            :  * that we block all buffered I/O.
     112                 :            :  * On exit, the function ensures that the NFS_INO_ODIRECT flag is set,
     113                 :            :  * and holds a shared lock on inode->i_rwsem to ensure that the flag
     114                 :            :  * cannot be changed.
     115                 :            :  * In practice, this means that direct I/O operations are allowed to
     116                 :            :  * execute in parallel, thanks to the shared lock, whereas buffered I/O
     117                 :            :  * operations need to wait to grab an exclusive lock in order to clear
     118                 :            :  * NFS_INO_ODIRECT.
     119                 :            :  * Note that buffered writes and truncates both take a write lock on
     120                 :            :  * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
     121                 :            :  */
     122                 :            : void
     123                 :          0 : nfs_start_io_direct(struct inode *inode)
     124                 :            : {
     125                 :          0 :         struct nfs_inode *nfsi = NFS_I(inode);
     126                 :            :         /* Be an optimist! */
     127                 :          0 :         down_read(&inode->i_rwsem);
     128         [ #  # ]:          0 :         if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0)
     129                 :            :                 return;
     130                 :          0 :         up_read(&inode->i_rwsem);
     131                 :            :         /* Slow path.... */
     132                 :          0 :         down_write(&inode->i_rwsem);
     133                 :          0 :         nfs_block_buffered(nfsi, inode);
     134                 :          0 :         downgrade_write(&inode->i_rwsem);
     135                 :            : }
     136                 :            : 
     137                 :            : /**
     138                 :            :  * nfs_end_io_direct - declare that the direct i/o operation is done
     139                 :            :  * @inode: file inode
     140                 :            :  *
     141                 :            :  * Declare that a direct I/O operation is done, and release the shared
     142                 :            :  * lock on inode->i_rwsem.
     143                 :            :  */
     144                 :            : void
     145                 :          0 : nfs_end_io_direct(struct inode *inode)
     146                 :            : {
     147                 :          0 :         up_read(&inode->i_rwsem);
     148                 :          0 : }