It will be more efficient to execute quick endio process(eg. non-sync overwriting case) under irq process rather than starting a worker to do it. Add a flag to control DIO to be finished inline(under irq context), which can be used for non-sync overwriting case. Besides, skip invalidating pages if DIO is finished inline, which will keep the same logic with dio_bio_end_aio in non-sync overwriting case.
Signed-off-by: Zhihao Cheng chengzhihao1@huawei.com --- fs/iomap/direct-io.c | 11 +++++++++-- include/linux/iomap.h | 6 ++++++ 2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index d4cf2481ecf4..892a4f8109e5 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -110,7 +110,8 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio) * zeros from unwritten extents. */ if (!dio->error && dio->size && - (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) { + (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages && + !(dio->flags & IOMAP_DIO_INLINE_COMP)) { int err; err = invalidate_inode_pages2_range(inode->i_mapping, offset >> PAGE_SHIFT, @@ -124,8 +125,10 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio) * If this is a DSYNC write, make sure we push it to stable storage now * that we've written data. */ - if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) + if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) { + WARN_ON_ONCE(dio->flags & IOMAP_DIO_INLINE_COMP); ret = generic_write_sync(iocb, ret); + }
kfree(dio);
@@ -488,6 +491,10 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, /* for data sync or sync, we need sync completion processing */ if (iocb->ki_flags & IOCB_DSYNC) dio->flags |= IOMAP_DIO_NEED_SYNC; + else if (dio_flags & IOMAP_DIO_MAY_INLINE_COMP) { + /* writes could complete inline */ + dio->flags |= IOMAP_DIO_INLINE_COMP; + }
/* * For datasync only writes, we optimistically try using FUA for diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 2476ec96b4e5..0965d5f12858 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -297,6 +297,12 @@ struct iomap_dio_ops { */ #define IOMAP_DIO_FORCE_WAIT (1 << 0)
+/* + * DIO will be completed inline unless sync operation is needed after io is + * finished. + */ +#define IOMAP_DIO_MAY_INLINE_COMP (1 << 3) + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags);