From: Jens Axboe axboe@kernel.dk
mainline inclusion from mainline-5.5-rc1 commit 4d7dd462971405c65bfb3821dbb6b9ce13b5e8d6 category: feature bugzilla: https://bugzilla.openeuler.org/show_bug.cgi?id=27 CVE: NA ---------------------------
We currently try and start the next link when we put the request, and only if we were going to free it. This means that the optimization to continue executing requests from the same context often fails, as we're not putting the final reference.
Add REQ_F_LINK_NEXT to keep track of this, and allow io_uring to find the next request more efficiently.
Signed-off-by: Jens Axboe axboe@kernel.dk Signed-off-by: Zhihao Cheng chengzhihao1@huawei.com Signed-off-by: yangerkun yangerkun@huawei.com Reviewed-by: zhangyi (F) yi.zhang@huawei.com Signed-off-by: Cheng Jian cj.chengjian@huawei.com --- fs/io_uring.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c index 79ee6964ff6c..3531ffbeacfc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -340,6 +340,7 @@ struct io_kiocb { #define REQ_F_NOWAIT 1 /* must not punt to workers */ #define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */ #define REQ_F_FIXED_FILE 4 /* ctx owns file */ +#define REQ_F_LINK_NEXT 8 /* already grabbed next link */ #define REQ_F_IO_DRAIN 16 /* drain existing IO first */ #define REQ_F_IO_DRAINED 32 /* drain done */ #define REQ_F_LINK 64 /* linked sqes */ @@ -878,6 +879,10 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr) struct io_kiocb *nxt; bool wake_ev = false;
+ /* Already got next link */ + if (req->flags & REQ_F_LINK_NEXT) + return; + /* * The list should never be empty when we are called here. But could * potentially happen if the chain is messed up, check to be on the @@ -914,6 +919,7 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr) break; }
+ req->flags |= REQ_F_LINK_NEXT; if (wake_ev) io_cqring_ev_posted(ctx); } @@ -950,12 +956,10 @@ static void io_fail_links(struct io_kiocb *req) io_cqring_ev_posted(ctx); }
-static void io_free_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt) +static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt) { - if (likely(!(req->flags & REQ_F_LINK))) { - __io_free_req(req); + if (likely(!(req->flags & REQ_F_LINK))) return; - }
/* * If LINK is set, we have dependent requests in this chain. If we @@ -981,7 +985,11 @@ static void io_free_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt) } else { io_req_link_next(req, nxt); } +}
+static void io_free_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt) +{ + io_req_find_next(req, nxt); __io_free_req(req); }
@@ -998,8 +1006,10 @@ static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr) { struct io_kiocb *nxt = NULL;
+ io_req_find_next(req, &nxt); + if (refcount_dec_and_test(&req->refs)) - io_free_req_find_next(req, &nxt); + __io_free_req(req);
if (nxt) { if (nxtptr)