1. export hash lock, alock and ocf_lru_* from OCF.
2. add weak attibute to ocf_engine_push_req_front and
ocf_engine_push_req_back to be overwritten with xcache queue handler.
3. add .flush_do_asynch_common to support flush metadata from
xcache.
4. overwritten IO, cleaner and metadata in OCF with xcache.
5. add xcache qos and evcting
Signed-off-by: Kemeng Shi <shikemeng(a)huaweicloud.com>
---
inc/ocf.h | 1 +
inc/ocf_def.h | 1 +
inc/xcache.h | 7 +
inc/xcache_cleaner.h | 6 +
inc/xcache_io.h | 90 ++++
src/cleaning/cleaning.c | 46 +-
src/cleaning/cleaning.h | 3 +
src/concurrency/ocf_metadata_concurrency.c | 4 +-
src/concurrency/ocf_metadata_concurrency.h | 7 +
src/engine/engine_common.c | 4 +-
src/engine/xcache_engine.c | 28 +
src/engine/xcache_engine.h | 8 +
src/engine/xcache_engine_common.c | 464 +++++++++++++++++
src/engine/xcache_engine_common.h | 216 ++++++++
src/engine/xcache_engine_flush.c | 140 +++++
src/engine/xcache_engine_flush.h | 25 +
src/engine/xcache_engine_rd.c | 366 +++++++++++++
src/engine/xcache_engine_rd.h | 6 +
src/engine/xcache_engine_wb.c | 211 ++++++++
src/engine/xcache_engine_wb.h | 6 +
src/engine/xcache_engine_wt.c | 210 ++++++++
src/engine/xcache_engine_wt.h | 6 +
src/evicting/deadline.c | 172 +++++++
src/evicting/deadline.h | 6 +
src/evicting/evicting.c | 3 +
src/evicting/evicting_helper.h | 32 ++
src/evicting/evicting_ops.h | 61 +++
src/metadata/metadata_raw.c | 87 +++-
src/metadata/metadata_raw.h | 40 ++
src/metadata/xcache_metadata.c | 88 ++++
src/metadata/xcache_metadata.h | 47 ++
src/mngt/ocf_mngt_cache.c | 11 +
src/ocf_cache_priv.h | 1 +
src/ocf_lru.c | 6 +-
src/ocf_lru.h | 7 +
src/ocf_queue.c | 11 +-
src/ocf_queue_priv.h | 2 +
src/qos/qos.c | 6 +
src/qos/qos.h | 27 +
src/qos/qos_lb.h | 143 ++++++
src/utils/utils_alock.c | 122 +++++
src/utils/utils_alock.h | 11 +
src/utils/utils_cache_line.c | 6 +-
src/xcache.c | 30 ++
src/xcache.h | 58 +++
src/xcache_cleaner.c | 572 +++++++++++++++++++++
src/xcache_cleaner.h | 69 +++
src/xcache_lru.c | 199 +++++++
src/xcache_lru.h | 9 +
src/xcache_ocf_core.c | 45 ++
src/xcache_queue.c | 337 ++++++++++++
src/xcache_queue.h | 35 ++
52 files changed, 4042 insertions(+), 56 deletions(-)
create mode 100644 inc/xcache.h
create mode 100644 inc/xcache_cleaner.h
create mode 100644 inc/xcache_io.h
create mode 100644 src/engine/xcache_engine.c
create mode 100644 src/engine/xcache_engine.h
create mode 100644 src/engine/xcache_engine_common.c
create mode 100644 src/engine/xcache_engine_common.h
create mode 100644 src/engine/xcache_engine_flush.c
create mode 100644 src/engine/xcache_engine_flush.h
create mode 100644 src/engine/xcache_engine_rd.c
create mode 100644 src/engine/xcache_engine_rd.h
create mode 100644 src/engine/xcache_engine_wb.c
create mode 100644 src/engine/xcache_engine_wb.h
create mode 100644 src/engine/xcache_engine_wt.c
create mode 100644 src/engine/xcache_engine_wt.h
create mode 100644 src/evicting/deadline.c
create mode 100644 src/evicting/deadline.h
create mode 100644 src/evicting/evicting.c
create mode 100644 src/evicting/evicting_helper.h
create mode 100644 src/evicting/evicting_ops.h
create mode 100644 src/metadata/xcache_metadata.c
create mode 100644 src/metadata/xcache_metadata.h
create mode 100644 src/qos/qos.c
create mode 100644 src/qos/qos.h
create mode 100644 src/qos/qos_lb.h
create mode 100644 src/xcache.c
create mode 100644 src/xcache.h
create mode 100644 src/xcache_cleaner.c
create mode 100644 src/xcache_cleaner.h
create mode 100644 src/xcache_lru.c
create mode 100644 src/xcache_lru.h
create mode 100644 src/xcache_ocf_core.c
create mode 100644 src/xcache_queue.c
create mode 100644 src/xcache_queue.h
diff --git a/inc/ocf.h b/inc/ocf.h
index 416d743..31137fc 100644
--- a/inc/ocf.h
+++ b/inc/ocf.h
@@ -32,5 +32,6 @@
#include "ocf_ctx.h"
#include "ocf_err.h"
#include "ocf_trace.h"
+#include "xcache.h"
#endif /* __OCF_H__ */
diff --git a/inc/ocf_def.h b/inc/ocf_def.h
index 89fb2e0..3466c3f 100644
--- a/inc/ocf_def.h
+++ b/inc/ocf_def.h
@@ -332,6 +332,7 @@ typedef enum {
*/
#define OCF_READ 0
#define OCF_WRITE 1
+#define OCF_FLUSH 2
/**
* @}
*/
diff --git a/inc/xcache.h b/inc/xcache.h
new file mode 100644
index 0000000..1080ba8
--- /dev/null
+++ b/inc/xcache.h
@@ -0,0 +1,7 @@
+#ifndef XCACHE_H__
+#define XCACHE_H__
+
+#include "xcache_io.h"
+#include "xcache_cleaner.h"
+
+#endif
diff --git a/inc/xcache_cleaner.h b/inc/xcache_cleaner.h
new file mode 100644
index 0000000..88c2760
--- /dev/null
+++ b/inc/xcache_cleaner.h
@@ -0,0 +1,6 @@
+#ifndef XCACHE_CLEANER_H__
+#define XCACHE_CLEANER_H__
+
+void xcache_cleaner_run(ocf_cleaner_t cleaner, ocf_queue_t queue);
+
+#endif
diff --git a/inc/xcache_io.h b/inc/xcache_io.h
new file mode 100644
index 0000000..5633a6c
--- /dev/null
+++ b/inc/xcache_io.h
@@ -0,0 +1,90 @@
+#ifndef XCACHE_IO_H__
+#define XCACHE_IO_H__
+
+#include "ocf_env.h"
+#include "ocf/ocf.h"
+
+enum entry_type {
+ XCACHE_IO_ENTRY,
+ XCACHE_BACKDEV_IO_ENTRY,
+ OCF_REQ_ENTRY,
+};
+
+struct queue_entry {
+ enum entry_type type;
+ struct list_head list;
+};
+
+#define INLINE_FLUSH_LINES 4
+struct xcache_io;
+typedef void (*xcache_io_end_fn)(struct xcache_io *io, int error);
+typedef int (*xcache_io_if)(struct xcache_io *io);
+struct xcache_io {
+ // queue_entry
+ enum entry_type type;
+ struct list_head queue_list;
+
+ xcache_io_end_fn end;
+ xcache_io_if io_if;
+ env_atomic remaining;
+ ctx_data_t *data;
+ int error;
+
+ ocf_queue_t io_queue;
+ ocf_cache_t cache;
+ ocf_core_t core;
+ uint64_t start_addr;
+ uint64_t size;
+ uint8_t rw;
+ uint8_t flags;
+
+ ocf_cache_line_t flush_lines[INLINE_FLUSH_LINES];
+ uint64_t flush_line_num;
+};
+
+struct backdev_io_end_arg {
+ uint64_t addr;
+ uint64_t size;
+ int error;
+};
+struct xcache_backdev_io;
+typedef int (*backdev_io_end_fn)(struct xcache_backdev_io *io, struct backdev_io_end_arg *arg);
+typedef void (*backdev_io_res_fn)(struct xcache_backdev_io *io);
+enum xcache_dir {
+ XCACHE_RD = 0,
+ XCACHE_WR,
+ XCACHE_FLUSH,
+};
+struct xcache_backdev_io {
+ // queue_entry
+ enum entry_type type;
+ struct list_head free_list;
+
+ struct xcache_io *xcache_io;
+ backdev_io_end_fn end;
+
+ ocf_cache_line_t line;
+ ctx_data_t *data;
+
+ union {
+ // for alock
+ struct {
+ backdev_io_res_fn io_res;
+ uint64_t addr;
+ uint64_t size;
+ };
+ // for io_end
+ struct {
+ int dev;
+ enum xcache_dir dir;
+ };
+ };
+
+ void *priv;
+};
+
+void xcache_backdev_io_end(struct xcache_backdev_io *bd_io, struct backdev_io_end_arg *arg);
+void xcache_submit_io(struct xcache_io *io);
+void spdk_backdev_submit_io(struct xcache_backdev_io *io_base, bool to_cache, uint64_t addr, uint64_t len, uint64_t offset, uint8_t dir);
+
+#endif
diff --git a/src/cleaning/cleaning.c b/src/cleaning/cleaning.c
index 94421f0..c7eef8f 100644
--- a/src/cleaning/cleaning.c
+++ b/src/cleaning/cleaning.c
@@ -70,47 +70,57 @@ static int _ocf_cleaner_run_check_dirty_inactive(ocf_cache_t cache)
return 1;
}
-static void ocf_cleaner_run_complete(ocf_cleaner_t cleaner, uint32_t interval)
+int ocf_cleaner_run_prepare(ocf_cleaner_t cleaner, ocf_queue_t queue)
{
ocf_cache_t cache = ocf_cleaner_get_cache(cleaner);
- ocf_mngt_cache_unlock(cache);
- ocf_queue_put(cleaner->io_queue);
- cleaner->end(cleaner, interval);
-}
-
-void ocf_cleaner_run(ocf_cleaner_t cleaner, ocf_queue_t queue)
-{
- ocf_cache_t cache;
-
- OCF_CHECK_NULL(cleaner);
- OCF_CHECK_NULL(queue);
-
- cache = ocf_cleaner_get_cache(cleaner);
-
/* Do not involve cleaning when cache is not running
* (error, etc.).
*/
if (!env_bit_test(ocf_cache_state_running, &cache->cache_state) ||
ocf_mngt_cache_is_locked(cache)) {
cleaner->end(cleaner, SLEEP_TIME_MS);
- return;
+ return -1;
}
/* Sleep in case there is management operation in progress. */
if (ocf_mngt_cache_trylock(cache)) {
cleaner->end(cleaner, SLEEP_TIME_MS);
- return;
+ return -1;
}
if (_ocf_cleaner_run_check_dirty_inactive(cache)) {
ocf_mngt_cache_unlock(cache);
cleaner->end(cleaner, SLEEP_TIME_MS);
- return;
+ return -1;
}
ocf_queue_get(queue);
cleaner->io_queue = queue;
+ return 0;
+}
+
+void ocf_cleaner_run_complete(ocf_cleaner_t cleaner, uint32_t interval)
+{
+ ocf_cache_t cache = ocf_cleaner_get_cache(cleaner);
+
+ ocf_mngt_cache_unlock(cache);
+ ocf_queue_put(cleaner->io_queue);
+ cleaner->end(cleaner, interval);
+}
+
+void ocf_cleaner_run(ocf_cleaner_t cleaner, ocf_queue_t queue)
+{
+ ocf_cache_t cache;
+
+ OCF_CHECK_NULL(cleaner);
+ OCF_CHECK_NULL(queue);
+
+ if (ocf_cleaner_run_prepare(cleaner, queue) != 0) {
+ return;
+ }
+
+ cache = ocf_cleaner_get_cache(cleaner);
ocf_cleaning_perform_cleaning(cache, ocf_cleaner_run_complete);
}
diff --git a/src/cleaning/cleaning.h b/src/cleaning/cleaning.h
index 007dac0..f514393 100644
--- a/src/cleaning/cleaning.h
+++ b/src/cleaning/cleaning.h
@@ -53,4 +53,7 @@ void ocf_kick_cleaner(ocf_cache_t cache);
void ocf_stop_cleaner(ocf_cache_t cache);
+int ocf_cleaner_run_prepare(ocf_cleaner_t cleaner, ocf_queue_t queue);
+void ocf_cleaner_run_complete(ocf_cleaner_t cleaner, uint32_t interval);
+
#endif
diff --git a/src/concurrency/ocf_metadata_concurrency.c b/src/concurrency/ocf_metadata_concurrency.c
index 52059a0..794e27f 100644
--- a/src/concurrency/ocf_metadata_concurrency.c
+++ b/src/concurrency/ocf_metadata_concurrency.c
@@ -212,7 +212,7 @@ void ocf_metadata_end_shared_access(struct ocf_metadata_lock *metadata_lock,
number. Preffered way to lock multiple hash buckets is to use
request lock rountines ocf_req_hash_(un)lock_(rd/wr).
*/
-static inline void ocf_hb_id_naked_lock(
+void ocf_hb_id_naked_lock(
struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash, int rw)
{
@@ -226,7 +226,7 @@ static inline void ocf_hb_id_naked_lock(
ENV_BUG();
}
-static inline void ocf_hb_id_naked_unlock(
+void ocf_hb_id_naked_unlock(
struct ocf_metadata_lock *metadata_lock,
ocf_cache_line_t hash, int rw)
{
diff --git a/src/concurrency/ocf_metadata_concurrency.h b/src/concurrency/ocf_metadata_concurrency.h
index 97262d8..43f4d1d 100644
--- a/src/concurrency/ocf_metadata_concurrency.h
+++ b/src/concurrency/ocf_metadata_concurrency.h
@@ -177,4 +177,11 @@ void ocf_collision_start_exclusive_access(struct ocf_metadata_lock *metadata_loc
uint32_t page);
void ocf_collision_end_exclusive_access(struct ocf_metadata_lock *metadata_lock,
uint32_t page);
+
+void ocf_hb_id_naked_lock(
+ struct ocf_metadata_lock *metadata_lock,
+ ocf_cache_line_t hash, int rw);
+void ocf_hb_id_naked_unlock(
+ struct ocf_metadata_lock *metadata_lock,
+ ocf_cache_line_t hash, int rw);
#endif
diff --git a/src/engine/engine_common.c b/src/engine/engine_common.c
index a789b13..5b30250 100644
--- a/src/engine/engine_common.c
+++ b/src/engine/engine_common.c
@@ -584,7 +584,7 @@ void ocf_engine_update_request_stats(struct ocf_request *req)
req->info.hit_no, req->core_line_count);
}
-void ocf_engine_push_req_back(struct ocf_request *req, bool allow_sync)
+void __attribute__((weak)) ocf_engine_push_req_back(struct ocf_request *req, bool allow_sync)
{
ocf_cache_t cache = req->cache;
ocf_queue_t q = NULL;
@@ -614,7 +614,7 @@ void ocf_engine_push_req_back(struct ocf_request *req, bool allow_sync)
ocf_queue_kick(q, allow_sync);
}
-void ocf_engine_push_req_front(struct ocf_request *req, bool allow_sync)
+void __attribute__((weak)) ocf_engine_push_req_front(struct ocf_request *req, bool allow_sync)
{
ocf_cache_t cache = req->cache;
ocf_queue_t q = NULL;
diff --git a/src/engine/xcache_engine.c b/src/engine/xcache_engine.c
new file mode 100644
index 0000000..089afa4
--- /dev/null
+++ b/src/engine/xcache_engine.c
@@ -0,0 +1,28 @@
+#include "ocf/ocf.h"
+
+#include "xcache_engine.h"
+#include "xcache_engine_common.h"
+#include "xcache_engine_rd.h"
+#include "xcache_engine_wt.h"
+#include "xcache_engine_wb.h"
+#include "xcache_engine_flush.h"
+
+void xcache_get_io_if(struct xcache_io *io, ocf_cache_mode_t mode)
+{
+ if (io->rw == OCF_WRITE) {
+ switch (mode) {
+ case ocf_cache_mode_wb:
+ io->io_if = xcache_wb;
+ return;
+ case ocf_cache_mode_wt:
+ io->io_if = xcache_wt;
+ return;
+ default:
+ return;
+ }
+ } else if (io->rw == OCF_READ) {
+ io->io_if = xcache_read_generic;
+ } else {
+ io->io_if = xcache_flush;
+ }
+}
diff --git a/src/engine/xcache_engine.h b/src/engine/xcache_engine.h
new file mode 100644
index 0000000..f20444a
--- /dev/null
+++ b/src/engine/xcache_engine.h
@@ -0,0 +1,8 @@
+#ifndef __XCACHE_ENGINE_H_
+#define __XCACHE_ENGINE_H_
+
+#include "ocf/xcache.h"
+
+void xcache_get_io_if(struct xcache_io *io, ocf_cache_mode_t mode);
+
+#endif
diff --git a/src/engine/xcache_engine_common.c b/src/engine/xcache_engine_common.c
new file mode 100644
index 0000000..687c54c
--- /dev/null
+++ b/src/engine/xcache_engine_common.c
@@ -0,0 +1,464 @@
+#include "ocf/ocf.h"
+#include "../ocf_cache_priv.h"
+#include "../ocf_queue_priv.h"
+#include "../utils/utils_cache_line.h"
+#include "../metadata/metadata.h"
+
+#include "../xcache.h"
+#include "xcache_engine_common.h"
+#include "../xcache_lru.h"
+#include "../xcache_queue.h"
+#include "../evicting/evicting_ops.h"
+
+uint64_t cache_line_to_addr(ocf_cache_t cache, ocf_cache_line_t line, uint64_t line_offset)
+{
+ uint64_t addr;
+
+ addr = ocf_metadata_map_lg2phy(cache, line);
+ addr = xcache_line_to_addr(cache, addr);
+ addr += cache->device->metadata_offset;
+ addr += line_offset;
+ return addr;
+}
+
+ocf_cache_line_t addr_to_cache_line(ocf_cache_t cache, uint64_t addr)
+{
+ ocf_cache_line_t line;
+
+ addr -= cache->device->metadata_offset;
+ line = xcache_addr_to_line(cache, addr);
+ line = ocf_metadata_map_phy2lg(cache, line);
+ return line;
+}
+
+ocf_cache_line_t xcache_engine_lookup_line(struct ocf_cache *cache,
+ ocf_core_id_t core_id, uint64_t core_line)
+{
+ ocf_cache_line_t line;
+ ocf_cache_line_t hash;
+
+ hash = ocf_metadata_hash_func(cache, core_line, core_id);
+ line = ocf_metadata_get_hash(cache, hash);
+
+ while (line != cache->device->collision_table_entries) {
+ ocf_core_id_t curr_core_id;
+ uint64_t curr_core_line;
+
+ ocf_metadata_get_core_info(cache, line, &curr_core_id,
+ &curr_core_line);
+
+ if (core_id == curr_core_id && curr_core_line == core_line) {
+ return line;
+ }
+
+ line = ocf_metadata_get_collision_next(cache, line);
+ }
+
+ return INVALID_LINE;
+}
+
+void xcache_map_cache_line(struct xcache_io_context *ctx,
+ ocf_cache_line_t cache_line)
+{
+ ocf_cache_t cache = xcache_ctx_cache(ctx);
+ ocf_core_id_t core_id = ocf_core_get_id(xcache_ctx_core(ctx));
+ unsigned int hash_index = ctx->hash;
+ uint64_t core_line = ctx->core_line;
+
+ /* Add the block to the corresponding collision list */
+ ocf_metadata_start_collision_shared_access(cache, cache_line);
+ ocf_metadata_add_to_collision(cache, core_id, core_line, hash_index,
+ cache_line);
+ ocf_metadata_end_collision_shared_access(cache, cache_line);
+}
+
+static void xcache_engine_prepare_wait(void *io)
+{
+ struct xcache_backdev_io *base_io = (struct xcache_backdev_io *)io;
+ struct xcache_io *ocf_io = base_io->xcache_io;
+
+ // avoid xcache_io finish when waiting for lock
+ xcache_io_get(ocf_io);
+}
+
+static void xcache_engine_prepare_wake(void *io)
+{
+ struct xcache_backdev_io *base_io = (struct xcache_backdev_io *)io;
+
+ xcache_queue_push_backdev_io_front(base_io, false);
+}
+
+static void xcache_get_core_range(struct xcache_io_context *ctx,
+ uint64_t start_core_line, uint64_t end_core_line,
+ uint64_t *addr, uint64_t *size)
+{
+ struct xcache_io *io = ctx->io;
+ ocf_cache_t cache = xcache_ctx_cache(ctx);
+ uint64_t start_addr, end_addr, end;
+
+ start_addr = xcache_io_start_addr(io);
+ end_addr = start_addr + xcache_io_size(io);
+
+ *addr = xcache_line_to_addr(cache, start_core_line);
+ end = xcache_line_to_addr(cache, end_core_line);
+
+ if (*addr < start_addr) {
+ *addr = start_addr;
+ }
+ if (end > end_addr) {
+ end = end_addr;
+ }
+
+ *size = end - *addr;
+}
+
+static inline int xcache_engine_lock_wr(struct xcache_io_context *ctx, struct ocf_alock *alock,
+ uint64_t core_line, ocf_cache_line_t line,
+ backdev_io_res_fn io_res)
+{
+ ocf_queue_t q = xcache_ctx_queue(ctx);
+ struct xcache_backdev_io *base_io = xcache_queue_alloc_backdev_io(q);
+ int ret;
+
+ if (base_io == NULL) {
+ ocf_cache_log(xcache_ctx_cache(ctx), log_err, "alloc base io failed\n");
+ return -ENOMEM;
+ }
+
+ base_io->xcache_io = ctx->io;
+ base_io->line = line;
+ base_io->io_res = io_res;
+ base_io->data = ctx->io->data;
+ xcache_get_core_range(ctx, core_line, core_line + 1, &base_io->addr, &base_io->size);
+
+ ret = xcache_lock_wr(alock, line, xcache_engine_prepare_wait,
+ xcache_engine_prepare_wake, base_io);
+ if (ret != OCF_LOCK_NOT_ACQUIRED) {
+ xcache_queue_free_backdev_io(q, base_io);
+ }
+
+ return ret;
+}
+
+static inline void xcache_hash_lock_rd(ocf_cache_t cache, ocf_cache_line_t hash, uint8_t lock_idx)
+{
+ ocf_metadata_start_shared_access(&cache->metadata.lock,
+ lock_idx);
+ ocf_hb_id_naked_lock(&cache->metadata.lock, hash,
+ OCF_METADATA_RD);
+}
+
+static inline void xcache_hash_unlock_rd(ocf_cache_t cache, ocf_cache_line_t hash, uint8_t lock_idx)
+{
+ ocf_hb_id_naked_unlock(&cache->metadata.lock, hash,
+ OCF_METADATA_RD);
+ ocf_metadata_end_shared_access(&cache->metadata.lock,
+ lock_idx);
+}
+
+static inline void xcache_hash_lock_upgrade(ocf_cache_t cache, ocf_cache_line_t hash)
+{
+ ocf_hb_id_naked_unlock(&cache->metadata.lock, hash,
+ OCF_METADATA_RD);
+ ocf_hb_id_naked_lock(&cache->metadata.lock, hash,
+ OCF_METADATA_WR);
+}
+
+static inline void xcache_hash_unlock_wr(ocf_cache_t cache, ocf_cache_line_t hash, uint8_t lock_idx)
+{
+ ocf_hb_id_naked_unlock(&cache->metadata.lock, hash,
+ OCF_METADATA_WR);
+ ocf_metadata_end_shared_access(&cache->metadata.lock,
+ lock_idx);
+}
+
+int xcache_engine_get_line(struct xcache_io_context *ctx,
+ uint64_t core_line, ocf_cache_line_t *line,
+ backdev_io_res_fn io_res)
+{
+ ocf_cache_t cache = xcache_ctx_cache(ctx);
+ ocf_core_t core = xcache_ctx_core(ctx);
+ ocf_queue_t q = xcache_ctx_queue(ctx);
+ ocf_core_id_t core_id = ocf_core_get_id(core);
+ unsigned lock_idx = ocf_metadata_concurrency_next_idx(q);
+ ocf_cache_line_t hash = ocf_metadata_hash_func(cache, core_line, core_id);
+ struct ocf_alock *alock = ocf_cache_line_concurrency(cache);
+
+ int lock;
+
+ xcache_hash_lock_rd(cache, hash, lock_idx);
+ *line = xcache_engine_lookup_line(cache, core_id, core_line);
+ if (*line != INVALID_LINE) {
+ lock = xcache_engine_lock_wr(ctx, alock, core_line, *line, io_res);
+ xcache_hash_unlock_rd(cache, hash, lock_idx);
+ goto found;
+ }
+
+ xcache_hash_lock_upgrade(cache, hash);
+ *line = xcache_engine_lookup_line(cache, core_id, core_line);
+ if (unlikely(*line) != INVALID_LINE) {
+ lock = xcache_engine_lock_wr(ctx, alock, core_line, *line, io_res);
+ xcache_hash_unlock_wr(cache, hash, lock_idx);
+ goto found;
+ }
+
+ ctx->core_line = core_line;
+ ctx->hash = hash;
+ *line = xcache_get_cline(ctx);
+ lock = OCF_LOCK_ACQUIRED;
+ xcache_hash_unlock_wr(cache, hash, lock_idx);
+ if (*line == INVALID_LINE) {
+ xcache_clean(ctx, 128);
+ return lock;
+ }
+
+found:
+ evicting_line_accessed(xcache_get_ctx(cache), *line);
+ return lock;
+}
+
+void xcache_init_io_ctx(struct xcache_io_context *ctx, struct xcache_io *io)
+{
+ uint64_t addr = xcache_io_start_addr(io);
+ uint64_t size = xcache_io_size(io);
+
+ ctx->io = io;
+ ctx->part_id = PARTITION_DEFAULT;
+ ctx->offset = 0;
+ ctx->start_miss_line = INVALID_CORE_LINE;
+ ctx->hit_no = 0;
+ ctx->cache_bytes = 0;
+ ctx->core_bytes = 0;
+ xcache_io_get_line_range(xcache_io_cache(io), addr, size,
+ &ctx->core_line_first, &ctx->core_line_last);
+}
+
+void xcache_update_stat(struct xcache_io_context *ctx)
+{
+ struct xcache_io *io = ctx->io;
+ uint64_t len = xcache_io_size(io);
+
+ ocf_core_stats_request_update(xcache_ctx_core(ctx), PARTITION_DEFAULT,
+ io->rw, ctx->hit_no, ctx->core_line_last - ctx->core_line_first + 1);
+ ocf_core_stats_vol_block_update(xcache_ctx_core(ctx), PARTITION_DEFAULT,
+ io->rw, len);
+ ocf_core_stats_core_block_update(xcache_ctx_core(ctx), PARTITION_DEFAULT,
+ io->rw, ctx->core_bytes);
+ ocf_core_stats_cache_block_update(xcache_ctx_core(ctx), PARTITION_DEFAULT,
+ io->rw, ctx->cache_bytes);
+}
+
+static void xcache_queue_miss_line(struct xcache_io_context *ctx, uint64_t core_line)
+{
+ if (ctx->start_miss_line == INVALID_CORE_LINE) {
+ ctx->start_miss_line = core_line;
+ }
+}
+
+static int xcache_submit_miss_line(struct xcache_io_context *ctx, uint64_t end_miss_line, xcache_line_miss_fn submit_func)
+{
+ struct xcache_io *io = ctx->io;
+
+ struct xcache_backdev_io *base_io;
+ uint64_t start_addr, end_addr;
+ uint64_t addr, end;
+
+ if (submit_func == NULL) {
+ ctx->start_miss_line = INVALID_CORE_LINE;
+ return 0;
+ }
+
+ if (ctx->start_miss_line == INVALID_CORE_LINE) {
+ return 0;
+ }
+
+ base_io = xcache_queue_alloc_backdev_io(xcache_ctx_queue(ctx));
+ if (base_io == NULL) {
+ ocf_cache_log(xcache_ctx_cache(ctx), log_err, "alloc base io failed\n");
+ io->error = -ENOMEM;
+ return -ENOMEM;
+ }
+ base_io->xcache_io = io;
+ base_io->data = io->data;
+
+ start_addr = xcache_io_start_addr(io);
+ end_addr = start_addr + xcache_io_size(io);
+ addr = xcache_line_to_addr(xcache_ctx_cache(ctx), ctx->start_miss_line);
+ end = xcache_line_to_addr(xcache_ctx_cache(ctx), end_miss_line);
+ if (addr < start_addr) {
+ addr = start_addr;
+ }
+ if (end > end_addr) {
+ end = end_addr;
+ }
+
+ submit_func(base_io, addr, end - addr, addr - start_addr);
+
+ ctx->start_miss_line = INVALID_CORE_LINE;
+ return 0;
+}
+
+int xcache_foreach_line(struct xcache_io_context *ctx, xcache_line_handle_func func, void *priv)
+{
+ int ret = 0;
+ struct xcache_io *io = ctx->io;
+ uint64_t start_addr = xcache_io_start_addr(io);
+ uint64_t len = xcache_io_size(io);
+
+ uint64_t addr, line_end, line_size, size;
+ uint8_t start_sector, end_sector, line_sectors;
+ uint64_t core_line_first, core_line_last, core_line_cur;
+
+ core_line_first = ctx->core_line_first;
+ core_line_last = ctx->core_line_last;
+ addr = start_addr;
+ line_size = ocf_line_size(xcache_ctx_cache(ctx));
+ line_sectors = BYTES_TO_SECTORS(line_size);
+ line_end = xcache_line_to_addr(xcache_ctx_cache(ctx), core_line_first + 1);
+ start_sector = xcache_sector_offset(xcache_ctx_cache(ctx), BYTES_TO_SECTORS(addr));
+
+ for (core_line_cur = core_line_first;
+ core_line_cur <= core_line_last;
+ core_line_cur++) {
+ if (core_line_cur == core_line_last) {
+ end_sector = xcache_sector_offset(xcache_ctx_cache(ctx), BYTES_TO_SECTORS(start_addr + len - 1));
+ size = start_addr + len - addr;
+ } else {
+ end_sector = line_sectors - 1;
+ size = line_end - addr;
+ }
+
+ ret = func(priv, core_line_cur,
+ addr, size, start_sector, end_sector);
+ if (ret) {
+ io->error = ret;
+ break;
+ }
+ ctx->offset += size;
+ addr += size;
+ line_end += line_size;
+ start_sector = 0;
+ }
+
+ return 0;
+}
+
+static int xcache_wr_lb_common_end(struct xcache_backdev_io *backdev_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = backdev_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_cache_line_t line = backdev_io->line;
+ uint8_t start_secotr, last_sector;
+
+ xcache_get_sectors_range(cache, cb_arg->addr, cb_arg->size, &start_secotr, &last_sector);
+ ocf_metadata_start_collision_shared_access(cache, line);
+ metadata_clear_valid_sec(cache, line, start_secotr, last_sector);
+ ocf_metadata_end_collision_shared_access(cache, line);
+
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), line);
+ xcache_queue_free_backdev_io(xcache_io_queue(backdev_io->xcache_io), backdev_io);
+ return 0;
+}
+
+void xcache_wr_lb_common(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t size)
+{
+ struct xcache_io *io = backdev_io->xcache_io;
+ uint64_t start_addr = xcache_io_start_addr(io);
+ uint64_t buf_offset = addr - start_addr;
+
+ backdev_io->line = line;
+ backdev_io->end = xcache_wr_lb_common_end;
+ xcache_backdev_submit_io(backdev_io, false, addr, size, buf_offset, OCF_WRITE);
+}
+
+static inline bool xcache_engine_need_lb(struct xcache_io_handler *handler, struct xcache_line_range *line_range)
+{
+ struct xcache_io_context *ctx = handler->ctx;
+ struct xcache_io *io = ctx->io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ xcache_context_t *xcache_ctx = xcache_get_ctx(cache);
+ struct backdev_io *backdev_io = NULL;
+
+ if (handler->lb_fn == NULL) {
+ return false;
+ }
+
+ if (handler->need_lb_fn != NULL &&
+ !handler->need_lb_fn(xcache_io_cache(io), line_range)) {
+ return false;
+ }
+
+ return xcache_qos_need_lb(&xcache_ctx->qos, xcache_io_dir(io), line_range->size);
+}
+
+static int xcache_handle_line(void *priv,
+ uint64_t core_line, uint64_t addr, uint64_t size,
+ uint8_t start_sector, uint8_t last_sector)
+{
+ struct xcache_io_handler *handler = (struct xcache_io_handler *)priv;
+ struct xcache_io_context *ctx = handler->ctx;
+ struct xcache_io *io = ctx->io;
+ ocf_cache_t cache = xcache_ctx_cache(ctx);
+ struct xcache_backdev_io *backdev_io;
+ struct xcache_line_range line_range = {
+ .addr = addr,
+ .size = size,
+ .start_sector = start_sector,
+ .last_sector = last_sector,
+ };
+
+ ocf_cache_line_t line;
+ int lock;
+
+ lock = xcache_engine_get_line(ctx, core_line, &line, handler->res_fn);
+ if (lock < 0) {
+ ocf_cache_log(cache, log_err, "try to wait for lock failed\n");
+ return -ENOMEM;
+ }
+
+ if (line == INVALID_LINE || (lock == OCF_LOCK_ACQUIRED &&
+ handler->valid_fn != NULL && handler->valid_fn(cache, line, start_sector, last_sector))) {
+ xcache_queue_miss_line(ctx, core_line);
+ ctx->core_bytes += size;
+ return 0;
+ }
+
+ xcache_submit_miss_line(ctx, core_line, handler->miss_fn);
+ if (lock == OCF_LOCK_NOT_ACQUIRED) {
+ ctx->hit_no++;
+ ctx->cache_bytes += size;
+ return 0;
+ }
+
+ backdev_io = xcache_alloc_backdev_io(io);
+ if (backdev_io == NULL) {
+ ocf_cache_log(cache, log_err, "alloc base io failed\n");
+ return -ENOMEM;
+ }
+
+ line_range.cache_line = line;
+ if (xcache_engine_need_lb(handler, &line_range))
+ {
+ ctx->core_bytes += size;
+ handler->lb_fn(backdev_io, line, addr, size);
+ return 0;
+ }
+
+ ctx->hit_no++;
+ ctx->cache_bytes += size;
+ handler->hit_fn(backdev_io, line, SECTORS_TO_BYTES(start_sector), size, ctx->offset);
+ return 0;
+}
+
+int xcache_handle_io(struct xcache_io_handler *handler)
+{
+ struct xcache_io_context *ctx = handler->ctx;
+ int ret;
+
+ ret = xcache_foreach_line(ctx, xcache_handle_line, (void *)handler);
+ if (ret != 0) {
+ return ret;
+ }
+ return xcache_submit_miss_line(ctx, ctx->core_line_last + 1, handler->miss_fn);
+}
diff --git a/src/engine/xcache_engine_common.h b/src/engine/xcache_engine_common.h
new file mode 100644
index 0000000..fd73519
--- /dev/null
+++ b/src/engine/xcache_engine_common.h
@@ -0,0 +1,216 @@
+#ifndef XCACHE_ENGINE_COMMON_H_
+#define XCACHE_ENGINE_COMMON_H_
+
+#include "../ocf_cache_priv.h"
+#include "../utils/utils_cache_line.h"
+#include "../ocf_def_priv.h"
+
+#include "../xcache.h"
+#include "../xcache_queue.h"
+
+#define INVALID_CORE_LINE ((uint64_t)-1)
+#define INVALID_LINE ((ocf_cache_line_t)-1)
+
+struct xcache_io_context {
+ struct xcache_io *io;
+ ocf_queue_t queue;
+ ocf_part_id_t part_id;
+ uint64_t offset;
+ uint64_t core_line_first;
+ uint64_t core_line_last;
+ uint64_t core_line;
+ ocf_cache_line_t hash;
+
+ uint64_t start_miss_line;
+
+ uint64_t hit_no;
+ uint64_t cache_bytes;
+ uint64_t core_bytes;
+};
+
+static inline uint8_t xcache_addr_offset(ocf_cache_t cache, uint64_t addr)
+{
+ return addr & (ocf_line_size(cache) - 1);
+}
+
+static inline uint8_t xcache_sector_offset(ocf_cache_t cache, uint64_t sector)
+{
+ return sector & (ocf_line_sectors(cache) - 1);
+}
+
+static inline void xcache_get_sectors_range(ocf_cache_t cache, uint64_t addr, uint64_t size, uint8_t *start_sector, uint8_t *last_sector)
+{
+ uint64_t offset = xcache_addr_offset(cache, addr);
+ *start_sector = BYTES_TO_SECTORS(offset);
+ *last_sector = BYTES_TO_SECTORS(offset + size - 1);
+}
+
+static inline void xcache_io_get_line_range(ocf_cache_t cache, uint64_t addr, uint64_t size,
+ uint64_t *line_first, uint64_t *line_last)
+{
+ *line_first = xcache_addr_to_line(cache, addr);
+ *line_last = xcache_addr_to_line(cache, addr + size - 1);
+}
+
+ocf_cache_line_t xcache_engine_lookup_line(struct ocf_cache *cache,
+ ocf_core_id_t core_id, uint64_t core_line);
+int xcache_engine_get_line(struct xcache_io_context *ctx,
+ uint64_t core_line, ocf_cache_line_t *line,
+ backdev_io_res_fn io_res);
+
+void xcache_map_cache_line(struct xcache_io_context *ctx,
+ ocf_cache_line_t cache_line);
+
+uint64_t cache_line_to_addr(ocf_cache_t cache, ocf_cache_line_t line, uint64_t line_offset);
+ocf_cache_line_t addr_to_cache_line(ocf_cache_t cache, uint64_t addr);
+
+struct xcache_line_range {
+ ocf_cache_line_t cache_line;
+ uint64_t core_line;
+ uint64_t addr;
+ uint64_t size;
+ uint64_t start_sector;
+ uint64_t last_sector;
+};
+
+typedef int (*xcache_line_handle_func)(void *priv,
+ uint64_t core_line, uint64_t addr, uint64_t size,
+ uint8_t start_sector, uint8_t last_sector);
+
+void xcache_init_io_ctx(struct xcache_io_context *ctx, struct xcache_io *io);
+int xcache_foreach_line(struct xcache_io_context *ctx, xcache_line_handle_func func, void *priv);
+void xcache_update_stat(struct xcache_io_context *ctx);
+
+static inline uint64_t xcache_io_start_addr(struct xcache_io *io)
+{
+ return io->start_addr;
+}
+
+static inline uint64_t xcache_io_size(struct xcache_io *io)
+{
+ return io->size;
+}
+
+static inline uint8_t xcache_io_flags(struct xcache_io *io)
+{
+ return io->flags;
+}
+
+static inline ocf_core_t xcache_io_core(struct xcache_io *io)
+{
+ return io->core;
+}
+
+static inline ocf_cache_t xcache_io_cache(struct xcache_io *io)
+{
+ return io->cache;
+}
+
+static inline ocf_queue_t xcache_io_queue(struct xcache_io *io)
+{
+ return io->io_queue;
+}
+
+static inline int xcache_io_dir(struct xcache_io *io)
+{
+ return io->rw;
+}
+
+static inline ocf_core_t xcache_ctx_core(struct xcache_io_context *ctx)
+{
+ return xcache_io_core(ctx->io);
+}
+
+static inline ocf_cache_t xcache_ctx_cache(struct xcache_io_context *ctx)
+{
+ return xcache_io_cache(ctx->io);
+}
+
+static inline ocf_queue_t xcache_ctx_queue(struct xcache_io_context *ctx)
+{
+ return xcache_io_queue(ctx->io);
+}
+
+static inline struct xcache_backdev_io *xcache_alloc_backdev_io(struct xcache_io *io)
+{
+ struct xcache_backdev_io *backdev_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io));
+
+ if (backdev_io == NULL) {
+ return NULL;
+ }
+
+ backdev_io->xcache_io = io;
+ backdev_io->data = io->data;
+ return backdev_io;
+}
+
+typedef int (*xcache_line_valid_fn)(ocf_cache_t cache, ocf_cache_line_t line,
+ uint8_t start_sector, uint8_t last_sector);
+typedef int (*xcache_line_hit_fn)(struct xcache_backdev_io *base_io, ocf_cache_line_t line,
+ uint64_t offset, uint64_t size, uint64_t buf_offset);
+typedef int (*xcache_line_miss_fn)(struct xcache_backdev_io *base_io, uint64_t addr,
+ uint64_t size, uint64_t buf_offset);
+typedef int (*xcache_line_need_lb_fn)(ocf_cache_t cache, struct xcache_line_range *line_range);
+typedef void (*xcache_line_lb_fn)(struct xcache_backdev_io *base_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t size);
+
+struct xcache_io_handler {
+ struct xcache_io_context *ctx;
+ xcache_line_valid_fn valid_fn;
+ xcache_line_hit_fn hit_fn;
+ xcache_line_miss_fn miss_fn;
+ backdev_io_res_fn res_fn;
+ xcache_line_need_lb_fn need_lb_fn;
+ xcache_line_lb_fn lb_fn;
+};
+int xcache_handle_io(struct xcache_io_handler *handler);
+
+void xcache_wr_lb_common(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t size);
+
+static inline void xcache_io_get(struct xcache_io *io)
+{
+ env_atomic_inc_return(&io->remaining);
+}
+
+static inline void xcache_io_end(struct xcache_io *io, int error)
+{
+ if (io->end)
+ io->end(io, error);
+
+}
+
+static inline void xcache_io_put(struct xcache_io *io)
+{
+ if (env_atomic_dec_return(&io->remaining))
+ return;
+
+ xcache_io_end(io, io->error);
+}
+
+static inline xcache_context_t *backdev_io_to_xcache_ctx(struct xcache_backdev_io *io_base)
+{
+ struct xcache_io *io = io_base->xcache_io;
+ ocf_queue_t q = io->io_queue;
+ ocf_cache_t cache = q->cache;
+ return xcache_get_ctx(cache);
+}
+
+static inline void xcache_backdev_submit_io(struct xcache_backdev_io *io_base, bool cached, uint64_t addr, uint64_t size, uint64_t buf_offset, uint8_t dir)
+{
+ struct xcache_io *io = io_base->xcache_io;
+ xcache_context_t *xcache_ctx = backdev_io_to_xcache_ctx(io_base);
+
+ io_base->dir = dir;
+ if (cached) {
+ io_base->dev = CACHE_DEV;
+ xcache_qos_load_add(&xcache_ctx->qos, CACHE_DEV, dir, size);
+ } else {
+ io_base->dev = CORE_DEV;
+ xcache_qos_load_add(&xcache_ctx->qos, CORE_DEV, dir, size);
+ }
+
+ xcache_io_get(io);
+ spdk_backdev_submit_io(io_base, cached, addr, size, buf_offset, dir);
+}
+#endif /* XCACHE_ENGINE_COMMON_H_ */
diff --git a/src/engine/xcache_engine_flush.c b/src/engine/xcache_engine_flush.c
new file mode 100644
index 0000000..6aaf28a
--- /dev/null
+++ b/src/engine/xcache_engine_flush.c
@@ -0,0 +1,140 @@
+#include "ocf/ocf.h"
+#include "../ocf_cache_priv.h"
+#include "../ocf_queue_priv.h"
+#include "../xcache_queue.h"
+
+#include "xcache_engine_common.h"
+#include "xcache_engine_flush.h"
+
+env_atomic g_need_flush = 0;
+
+static inline int xcache_do_flush(struct xcache_io *io);
+
+static inline int xcache_push_flush_io(struct xcache_io *io)
+{
+ ocf_queue_t q = xcache_io_queue(io);
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+
+ INIT_LIST_HEAD(&io->queue_list);
+ list_add_tail(&io->queue_list, &queue_ctx->flush_io_list);
+ env_atomic_inc(&queue_ctx->flush_io_no);
+ return env_atomic_read(&queue_ctx->flush_io_no);
+}
+
+static inline struct xcache_io *xcache_pop_flush_io(ocf_queue_t q)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+ struct xcache_io *flush_io = list_first_entry(&queue_ctx->flush_io_list,
+ struct xcache_io, queue_list);
+
+ env_atomic_dec(&queue_ctx->flush_io_no);
+ list_del(&flush_io->queue_list);
+
+ return flush_io;
+}
+
+static void xcache_flush_finish(struct xcache_io *io)
+{
+ ocf_queue_t q = io->io_queue;
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+ struct xcache_io *finish_io;
+ struct xcache_io *flush_io;
+
+ for (finish_io = xcache_pop_flush_io(q);
+ finish_io != io;
+ finish_io = xcache_pop_flush_io(q))
+ {
+ xcache_io_put(finish_io);
+ }
+ xcache_io_put(io);
+
+ if (env_atomic_read(&queue_ctx->flush_io_no) != 0) {
+ flush_io = list_entry(queue_ctx->flush_io_list.prev, struct xcache_io, queue_list);
+ xcache_do_flush(flush_io);
+ }
+}
+
+static int xcache_flush_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+
+ if (cb_arg->error != 0) {
+ ocf_cache_log(cache, log_err, "flush with error %d\n", cb_arg->error);
+ io->error = cb_arg->error;
+ }
+ xcache_queue_free_backdev_io(xcache_io_queue(io), base_io);
+
+ if (env_atomic_read(&io->remaining) == 2) {
+ xcache_flush_finish(io);
+ }
+ return 0;
+}
+
+static int xcache_flush_core(struct xcache_io *io)
+{
+ uint64_t addr = xcache_io_start_addr(io);
+ uint64_t size = xcache_io_size(io);
+ ocf_cache_t cache = xcache_io_cache(io);
+ struct xcache_backdev_io *base_io = NULL;
+
+ base_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io));
+ if (base_io == NULL) {
+ ocf_cache_log(cache, log_err, "alloc base io failed\n");
+ return -ENOMEM;
+ }
+
+ base_io->xcache_io = io;
+ base_io->end = xcache_flush_cb;
+ xcache_backdev_submit_io(base_io, false, addr, size, 0, OCF_FLUSH);
+ return 0;
+}
+
+static int xcache_flush_cache(struct xcache_io *io)
+{
+ ocf_cache_t cache = xcache_io_cache(io);
+ struct xcache_backdev_io *base_io = NULL;
+
+ base_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io));
+ if (base_io == NULL) {
+ ocf_cache_log(cache, log_err, "alloc base io failed\n");
+ return -ENOMEM;
+ }
+
+ base_io->xcache_io = io;
+ base_io->end = xcache_flush_cb;
+ xcache_backdev_submit_io(base_io, true, 0, 0, 0, OCF_FLUSH);
+ return 0;
+}
+
+static inline int xcache_do_flush(struct xcache_io *io)
+{
+ if (xcache_flush_core(io) != 0) {
+ io->error = -ENOMEM;
+ return -ENOMEM;
+ }
+
+ if (xcache_flush_cache(io) != 0) {
+ io->error = -ENOMEM;
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+int xcache_flush(struct xcache_io *io)
+{
+ io->error = 0;
+
+ if (!need_flush()) {
+ return 0;
+ }
+ clear_flush();
+
+ xcache_io_get(io);
+ if (xcache_push_flush_io(io) != 1) {
+ return 0;
+ }
+
+ return xcache_do_flush(io);
+}
diff --git a/src/engine/xcache_engine_flush.h b/src/engine/xcache_engine_flush.h
new file mode 100644
index 0000000..91738d8
--- /dev/null
+++ b/src/engine/xcache_engine_flush.h
@@ -0,0 +1,25 @@
+#ifndef XCACHE_ENGINE_FLUSH_H_
+#define XCACHE_ENGINE_FLUSH_H_
+
+#include "ocf/ocf.h"
+
+extern env_atomic g_need_flush;
+
+static inline void mark_flush(void)
+{
+ env_atomic_set(&g_need_flush, 1);
+}
+
+static inline void clear_flush(void)
+{
+ env_atomic_set(&g_need_flush, 0);
+}
+
+static inline bool need_flush(void)
+{
+ return (env_atomic_read(&g_need_flush) == 1);
+}
+
+int xcache_flush(struct xcache_io *io);
+
+#endif
diff --git a/src/engine/xcache_engine_rd.c b/src/engine/xcache_engine_rd.c
new file mode 100644
index 0000000..4b64975
--- /dev/null
+++ b/src/engine/xcache_engine_rd.c
@@ -0,0 +1,366 @@
+#include "ocf/ocf.h"
+#include "../ocf_cache_priv.h"
+#include "../concurrency/ocf_concurrency.h"
+#include "../utils/utils_cache_line.h"
+#include "../metadata/metadata.h"
+#include "../ocf_def_priv.h"
+
+#include "../xcache.h"
+#include "xcache_engine_rd.h"
+#include "xcache_engine_common.h"
+#include "../xcache_queue.h"
+
+static inline void xcache_read_bf_done(ocf_cache_t cache, struct xcache_backdev_io *base_io)
+{
+ ocf_queue_t q = xcache_io_queue(base_io->xcache_io);
+
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), base_io->line);
+ xcache_queue_free_line_data(q, cache, base_io->data);
+ base_io->data = NULL;
+ xcache_queue_free_backdev_io(q, base_io);
+}
+
+static int xcache_read_bf_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *bf_io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(bf_io);
+ ocf_core_t core = xcache_io_core(bf_io);
+
+ if (cb_arg->error) {
+ ocf_cache_log(cache, log_err, "read bf failed\n");
+ ocf_core_stats_cache_error_update(core, OCF_WRITE);
+ }
+ xcache_read_bf_done(cache, base_io);
+ return 0;
+}
+
+static int xcache_read_bf_update_metadata(struct xcache_backdev_io *base_io, uint64_t addr, uint64_t size)
+{
+ ocf_core_t core = xcache_io_core(base_io->xcache_io);
+ ocf_cache_t cache = ocf_core_get_cache(core);
+ ocf_cache_line_t line = base_io->line;
+ ocf_part_id_t part_id = PARTITION_DEFAULT;
+ uint8_t start_sector = xcache_sector_offset(cache, BYTES_TO_SECTORS(addr));
+ uint8_t end_sector = xcache_sector_offset(cache, BYTES_TO_SECTORS(addr + size - 1));
+
+ // set_cache_line_valid
+ ocf_metadata_start_collision_shared_access(cache, line);
+ if (unlikely(metadata_test_valid_sec(cache, line, start_sector, end_sector))) {
+ ocf_metadata_end_collision_shared_access(cache, line);
+ return -1;
+ }
+
+ if (metadata_set_valid_sec_changed(cache, line, start_sector, end_sector)) {
+ /*
+ * Update the number of cached data for that core object
+ */
+ env_atomic_inc(&core->runtime_meta->cached_clines);
+ env_atomic_inc(&core->runtime_meta->
+ part_counters[part_id].cached_clines);
+ }
+ ocf_metadata_end_collision_shared_access(cache, line);
+ return 0;
+}
+
+static ctx_data_t *xcache_get_bf_data(struct xcache_io *io, uint64_t addr, uint64_t size)
+{
+ ocf_cache_t cache = xcache_io_cache(io);
+ uint64_t start_byte = xcache_io_start_addr(io);
+ uint64_t from = addr - start_byte;
+ ctx_data_t *dst;
+
+ dst = xcache_queue_alloc_line_data(xcache_io_queue(io), cache);
+ if (dst == NULL) {
+ return NULL;
+ }
+
+ ctx_data_cpy(cache->owner, dst, io->data, 0, from, size);
+ return dst;
+}
+
+static int xcache_do_read_bf(struct xcache_backdev_io *base_io, uint64_t addr, uint64_t size)
+{
+ struct xcache_io *bf_io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(bf_io);
+ ocf_cache_line_t line = base_io->line;
+
+ int ret;
+ uint64_t cache_addr;
+
+ ret = xcache_read_bf_update_metadata(base_io, addr, size);
+ if (ret != 0) {
+ goto out;
+ }
+
+ cache_addr = cache_line_to_addr(cache, line, xcache_addr_offset(cache, addr));
+ base_io->end = xcache_read_bf_cb;
+ xcache_backdev_submit_io(base_io, true, cache_addr, size, 0, OCF_WRITE);
+ return 0;
+
+out:
+ xcache_read_bf_done(cache, base_io);
+ return ret;
+}
+
+static void xcache_read_bf_error(struct xcache_io *io, uint64_t addr, uint64_t size)
+{
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+ ocf_core_id_t core_id = ocf_core_get_id(core);
+ struct ocf_alock *alock = ocf_cache_line_concurrency(cache);
+
+ uint64_t core_line_first, core_line_last, core_line;
+ ocf_cache_line_t line;
+
+ xcache_io_get_line_range(cache, addr, size, &core_line_first, &core_line_last);
+ for (core_line = core_line_first; core_line <= core_line_last; core_line++) {
+ line = xcache_engine_lookup_line(cache, core_id, core_line);
+ if (line != INVALID_LINE) {
+ xcache_unlock_wr(alock, line);
+ }
+ }
+
+ ocf_core_stats_core_error_update(core, OCF_READ);
+}
+
+static void bf_io_end(struct xcache_io *bf_io, int error)
+{
+ xcache_queue_free_xcache_io(bf_io->io_queue, bf_io);
+}
+
+static struct xcache_io *xcache_get_bf_xcache_io(struct xcache_io *ori_io)
+{
+ struct xcache_io *bf_io = xcache_queue_alloc_xcache_io(ori_io->io_queue);
+
+ if (bf_io == NULL) {
+ ocf_cache_log(xcache_io_cache(ori_io), log_err, "alloc bf io failed\n");
+ return NULL;
+ }
+ bf_io->io_queue = ori_io->io_queue;
+ bf_io->cache = ori_io->cache;
+ bf_io->core = ori_io->core;
+ bf_io->error = 0;
+ bf_io->end = bf_io_end;
+ env_atomic_set(&bf_io->remaining, 1);
+ return bf_io;
+}
+
+static void xcache_free_bf_xcache_io(struct xcache_io *bf_io)
+{
+ xcache_queue_free_xcache_io(bf_io->io_queue, bf_io);
+}
+
+static int xcache_submit_read_bf_line(struct xcache_io *io, struct xcache_io *bf_io, uint64_t bf_addr, uint64_t bf_size, ocf_cache_line_t line)
+{
+ ocf_cache_t cache = xcache_io_cache(bf_io);
+ struct xcache_backdev_io *base_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io));
+ if (base_io == NULL) {
+ ocf_cache_log(cache, log_err, "alloc bf base_io failed\n");
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), line);
+ return -1;
+ }
+
+ base_io->data = xcache_get_bf_data(io, bf_addr, bf_size);
+ if (base_io->data == NULL) {
+ ocf_cache_log(cache, log_err, "alloc bf_data failed\n");
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), line);
+ xcache_queue_free_backdev_io(xcache_io_queue(io), base_io);
+ return -1;
+ }
+ base_io->xcache_io = bf_io;
+ base_io->line = line;
+
+ return xcache_do_read_bf(base_io, bf_addr, bf_size);
+}
+
+static void xcache_submit_read_bf(struct xcache_io *io, uint64_t addr, uint64_t size)
+{
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+ ocf_core_id_t core_id = ocf_core_get_id(core);
+ uint64_t line_size = ocf_line_size(cache);
+
+ uint64_t core_line_first, core_line_last, core_line;
+ ocf_cache_line_t line;
+ uint64_t bf_addr, bf_size;
+ struct xcache_backdev_io *base_io;
+ struct xcache_io *bf_io;
+
+ bf_io = xcache_get_bf_xcache_io(io);
+ if (bf_io == NULL) {
+ ocf_cache_log(cache, log_err, "alloc bf_io failed\n");
+ xcache_read_bf_error(io, addr, size);
+ return;
+ }
+
+ xcache_io_get_line_range(cache, addr, size, &core_line_first, &core_line_last);
+ bf_addr = addr;
+ bf_size = xcache_line_to_addr(cache, core_line_first + 1) - bf_addr;
+ for (core_line = core_line_first; core_line <= core_line_last;
+ core_line++, bf_addr += bf_size, bf_size = line_size) {
+ if (core_line == core_line_last) {
+ bf_size = (addr + size) - bf_addr;
+ }
+
+ line = xcache_engine_lookup_line(cache, core_id, core_line);
+ if (line == INVALID_LINE) {
+ continue;
+ }
+
+ if (xcache_submit_read_bf_line(io, bf_io, bf_addr, bf_size, line) != 0) {
+ ocf_cache_log(cache, log_err, "read bf line failed\n");
+ continue;
+ }
+ }
+
+ xcache_io_put(bf_io);
+}
+
+static int xcache_read_bf(struct xcache_backdev_io *parent_base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = parent_base_io->xcache_io;
+
+ xcache_queue_free_backdev_io(xcache_io_queue(parent_base_io->xcache_io), parent_base_io);
+
+ if (cb_arg->error) {
+ ocf_cache_log(xcache_io_cache(io), log_err, "read_miss failed with error %d\n",
+ cb_arg->error);
+
+ xcache_read_bf_error(io, cb_arg->addr, cb_arg->size);
+ io->error = cb_arg->error;
+ return cb_arg->error;
+ }
+
+ xcache_submit_read_bf(io, cb_arg->addr, cb_arg->size);
+
+ return 0;
+}
+
+static int xcache_read_miss(struct xcache_backdev_io *base_io,
+ uint64_t addr, uint64_t size, uint64_t buf_offset)
+{
+ base_io->end = xcache_read_bf;
+
+ xcache_backdev_submit_io(base_io, false, addr, size, buf_offset, OCF_READ);
+ return 0;
+}
+
+static int xcache_read_pt_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+
+ if (cb_arg->error) {
+ ocf_cache_log(cache, log_err, "read pt failed\n");
+ ocf_core_stats_core_error_update(core, OCF_READ);
+ }
+
+ xcache_queue_free_backdev_io(xcache_io_queue(base_io->xcache_io), base_io);
+ return 0;
+}
+
+static int xcache_read_hit_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+ ocf_cache_line_t line = base_io->line;
+
+ if (cb_arg->error) {
+ ocf_cache_log(cache, log_err, "read hit failed\n");
+ io->error = cb_arg->error;
+ ocf_core_stats_cache_error_update(core, OCF_READ);
+ }
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), line);
+ xcache_queue_free_backdev_io(xcache_io_queue(base_io->xcache_io), base_io);
+ return 0;
+}
+
+static int xcache_read_hit(struct xcache_backdev_io *base_io, ocf_cache_line_t line,
+ uint64_t offset, uint64_t size, uint64_t buf_offset)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ uint64_t addr = cache_line_to_addr(cache, line, offset);
+
+ base_io->end = xcache_read_hit_cb;
+ base_io->line = line;
+
+ xcache_backdev_submit_io(base_io, true, addr, size, buf_offset, OCF_READ);
+ return 0;
+}
+
+static void xcache_read_res(struct xcache_backdev_io *base_io)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ uint64_t start_addr = xcache_io_start_addr(io);
+
+ uint8_t start_sector, last_sector;
+ uint64_t buf_offset = base_io->addr - start_addr;
+
+ start_sector = BYTES_TO_SECTORS(xcache_addr_offset(cache, base_io->addr));
+ last_sector = start_sector + BYTES_TO_SECTORS(base_io->size - 1);
+
+ if (metadata_test_valid_sec(cache, base_io->line, start_sector, last_sector)) {
+ xcache_read_hit(base_io, base_io->line, SECTORS_TO_BYTES(start_sector),
+ base_io->size, buf_offset);
+ } else {
+ xcache_read_miss(base_io, base_io->addr, base_io->size, buf_offset);
+ }
+}
+
+static int xcache_read_line_valid(ocf_cache_t cache, ocf_cache_line_t line,
+ uint8_t start_sector, uint8_t last_sector)
+{
+ return !metadata_test_valid_sec(cache, line, start_sector, last_sector);
+}
+
+static int xcache_read_lb_cb(struct xcache_backdev_io *backdev_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = backdev_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), backdev_io->line);
+ xcache_queue_free_backdev_io(xcache_io_queue(backdev_io->xcache_io), backdev_io);
+ return 0;
+}
+
+static void xcache_read_lb(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t size)
+{
+ struct xcache_io *io = backdev_io->xcache_io;
+ uint64_t start_addr = xcache_io_start_addr(io);
+ uint64_t buf_offset = addr - start_addr;
+
+ backdev_io->end = xcache_read_lb_cb;
+ backdev_io->line = line;
+ xcache_backdev_submit_io(backdev_io, false, addr, size, buf_offset, OCF_READ);
+}
+
+int xcache_read_generic(struct xcache_io *io)
+{
+ int ret;
+ struct xcache_io_context ctx;
+ struct xcache_io_handler read_handler = {
+ .ctx = &ctx,
+ .res_fn = xcache_read_res,
+ .valid_fn = xcache_read_line_valid,
+ .miss_fn = xcache_read_miss,
+ .hit_fn = xcache_read_hit,
+ .need_lb_fn = NULL,
+ .lb_fn = xcache_read_lb,
+ };
+
+ xcache_init_io_ctx(&ctx, io);
+
+ ret = xcache_handle_io(&read_handler);
+ if (ret != 0) {
+ io->error = ret;
+ return ret;
+ }
+
+ xcache_update_stat(&ctx);
+ return 0;
+}
diff --git a/src/engine/xcache_engine_rd.h b/src/engine/xcache_engine_rd.h
new file mode 100644
index 0000000..3b939d4
--- /dev/null
+++ b/src/engine/xcache_engine_rd.h
@@ -0,0 +1,6 @@
+#ifndef XCACHE_ENGINE_RD_H_
+#define XCACHE_ENGINE_RD_H_
+
+int xcache_read_generic(struct xcache_io *io);
+
+#endif /* XCACHE_ENGINE_RD_H_ */
diff --git a/src/engine/xcache_engine_wb.c b/src/engine/xcache_engine_wb.c
new file mode 100644
index 0000000..cd07c96
--- /dev/null
+++ b/src/engine/xcache_engine_wb.c
@@ -0,0 +1,211 @@
+#include "ocf/ocf.h"
+#include "../ocf_cache_priv.h"
+#include "../metadata/metadata.h"
+#include "../utils/utils_cache_line.h"
+#include "../concurrency/ocf_cache_line_concurrency.h"
+
+#include "xcache_engine_common.h"
+#include "xcache_engine_wb.h"
+#include "xcache_engine_flush.h"
+#include "../xcache_queue.h"
+#include "../metadata/xcache_metadata.h"
+
+static bool xcache_wb_update_metadata(ocf_cache_t cache, ocf_core_t core,
+ ocf_cache_line_t line, uint8_t start_sector, uint8_t last_sector)
+{
+ ocf_part_id_t part_id = PARTITION_DEFAULT;
+ struct ocf_part *part = &cache->user_parts[part_id].part;
+ bool need_flush = false;
+
+ bool line_was_dirty;
+
+ ocf_metadata_start_collision_shared_access(cache, line);
+ if (metadata_set_valid_sec_changed(cache, line, start_sector, last_sector)) {
+ env_atomic_inc(&core->runtime_meta->cached_clines);
+ env_atomic_inc(&core->runtime_meta->
+ part_counters[part_id].cached_clines);
+ }
+
+ if (metadata_set_dirty_sec_changed(cache, line, start_sector, last_sector,
+ &line_was_dirty)) {
+ if (!line_was_dirty) {
+ /*
+ * If this is first dirty cline set dirty timestamp
+ */
+ if (!env_atomic64_read(&core->runtime_meta->dirty_since))
+ env_atomic64_cmpxchg(
+ &core->runtime_meta->dirty_since, 0,
+ env_ticks_to_secs(env_get_tick_count()));
+
+ /*
+ * Update the number of dirty cached data for that
+ * core object
+ */
+ env_atomic_inc(&core->runtime_meta->dirty_clines);
+
+ /*
+ * increment dirty clines statistic for given cline
+ */
+ env_atomic_inc(&core->runtime_meta->
+ part_counters[part_id].dirty_clines);
+
+ need_flush = true;
+ }
+ }
+ ocf_metadata_end_collision_shared_access(cache, line);
+ return need_flush;
+}
+
+static void xcache_wb_flush_metadata_end(void *flush_io, int error)
+{
+ struct xcache_io *io = (struct xcache_io *)flush_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ struct ocf_alock *lock = ocf_cache_line_concurrency(cache);
+ ocf_cache_line_t *flush_lines = xcache_io_get_flush_line(io);
+ uint64_t i;
+
+ for (i = 0; i < io->flush_line_num; i++) {
+ xcache_unlock_wr(lock, flush_lines[i]);
+ }
+ xcache_io_free_flush_line(io);
+
+ if (error) {
+ ocf_cache_log(cache, log_err, "wb flush metadata failed with error %d\n", error);
+ io->error = error;
+ }
+
+ xcache_io_put(io);
+}
+
+static int xcache_wb_hit_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ uint64_t addr = cb_arg->addr;
+ uint64_t len = cb_arg->size;
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_core_t core = xcache_io_core(io);
+ ocf_cache_t cache = ocf_core_get_cache(core);
+ int ret = 0;
+ ocf_cache_line_t line = addr_to_cache_line(cache, addr);
+
+ uint64_t offset;
+ uint8_t start_sector, last_sector;
+
+ xcache_queue_free_backdev_io(xcache_io_queue(base_io->xcache_io), base_io);
+
+ if (cb_arg->error) {
+ ocf_cache_log(cache, log_err, "wb write hit failed with error %d\n", cb_arg->error);
+ ret = cb_arg->error;
+ io->error = cb_arg->error;
+ ocf_core_stats_cache_error_update(core, OCF_WRITE);
+ goto out;
+ }
+
+ offset = xcache_addr_offset(cache, addr);
+ start_sector = BYTES_TO_SECTORS(offset);
+ last_sector = BYTES_TO_SECTORS(offset + len - 1);
+
+ if (xcache_wb_update_metadata(cache, core, line, start_sector, last_sector)) {
+ ret = xcache_io_add_flush_line(io, line);
+ if (ret != 0) {
+ ocf_cache_log(cache, log_err, "wb metadata add flush line failed with error %d\n", ret);
+ io->error = ret;
+ goto out;
+ }
+ if (xcache_metadata_should_flush(io)) {
+ xcache_io_metadata_flush(io, xcache_wb_flush_metadata_end);
+ }
+ return 0;
+ }
+
+out:
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), line);
+ return ret;
+}
+
+static int xcache_wb_hit(struct xcache_backdev_io *base_io, ocf_cache_line_t line,
+ uint64_t offset, uint64_t size, uint64_t buf_offset)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+
+ uint64_t addr;
+
+ addr = cache_line_to_addr(cache, line, offset);
+ base_io->end = xcache_wb_hit_cb;
+ xcache_backdev_submit_io(base_io, true, addr, size, buf_offset, OCF_WRITE);
+ return 0;
+}
+
+static int xcache_wb_miss_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+
+ if (cb_arg->error != 0) {
+ ocf_cache_log(cache, log_err, "wb miss with error %d\n", cb_arg->error);
+ io->error = cb_arg->error;
+ ocf_core_stats_core_error_update(core, OCF_WRITE);
+ }
+ xcache_queue_free_backdev_io(xcache_io_queue(base_io->xcache_io), base_io);
+ return 0;
+}
+
+static int xcache_wb_miss(struct xcache_backdev_io *base_io, uint64_t addr,
+ uint64_t size, uint64_t buf_offset)
+{
+ base_io->end = xcache_wb_miss_cb;
+ xcache_backdev_submit_io(base_io, false, addr, size, buf_offset, OCF_WRITE);
+ return 0;
+}
+
+static void xcache_wb_res(struct xcache_backdev_io *base_io)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ uint64_t offset = xcache_addr_offset(cache, base_io->addr);
+ uint64_t start_addr = xcache_io_start_addr(io);
+ uint64_t buf_offset = base_io->addr - start_addr;
+
+ xcache_wb_hit(base_io, base_io->line, offset, base_io->size, buf_offset);
+}
+
+// bypass dirty sectors to core will make additional cache io to update valid bit
+static int xcache_wb_need_lb(ocf_cache_t cache, struct xcache_line_range *range)
+{
+ return !metadata_test_dirty_sec(cache, range->cache_line, range->start_sector, range->last_sector);
+}
+
+static void xcache_wb_lb(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t size)
+{
+ xcache_wr_lb_common(backdev_io, line, addr, size);
+}
+
+int xcache_wb(struct xcache_io *io)
+{
+ int ret;
+ struct xcache_io_context ctx;
+ struct xcache_io_handler wb_handler = {
+ .ctx = &ctx,
+ .res_fn = xcache_wb_res,
+ .valid_fn = NULL,
+ .miss_fn = xcache_wb_miss,
+ .hit_fn = xcache_wb_hit,
+ .need_lb_fn = xcache_wb_need_lb,
+ .lb_fn = xcache_wb_lb,
+ };
+
+ mark_flush();
+
+ io->flush_line_num = 0;
+ xcache_init_io_ctx(&ctx, io);
+
+ ret = xcache_handle_io(&wb_handler);
+ if (ret != 0) {
+ return ret;
+ }
+
+ xcache_update_stat(&ctx);
+ return 0;
+}
diff --git a/src/engine/xcache_engine_wb.h b/src/engine/xcache_engine_wb.h
new file mode 100644
index 0000000..26dba02
--- /dev/null
+++ b/src/engine/xcache_engine_wb.h
@@ -0,0 +1,6 @@
+#ifndef XCACHE_ENGINE_WB_H_
+#define XCACHE_ENGINE_WB_H_
+
+int xcache_wb(struct xcache_io *io);
+
+#endif /* XCACHE_ENGINE_WB_H_ */
diff --git a/src/engine/xcache_engine_wt.c b/src/engine/xcache_engine_wt.c
new file mode 100644
index 0000000..2c08c5f
--- /dev/null
+++ b/src/engine/xcache_engine_wt.c
@@ -0,0 +1,210 @@
+#include "ocf/ocf.h"
+#include "../ocf_cache_priv.h"
+#include "../utils/utils_cache_line.h"
+#include "../metadata/metadata.h"
+
+#include "xcache_engine_wt.h"
+#include "xcache_engine_common.h"
+#include "xcache_engine_flush.h"
+#include "../xcache_queue.h"
+
+static void xcache_wt_udpate_metadata(ocf_cache_t cache, ocf_core_t core,
+ ocf_cache_line_t line, uint8_t start_sector, uint8_t last_sector)
+{
+ ocf_part_id_t part_id = PARTITION_DEFAULT;
+ struct ocf_part *part = &cache->user_parts[part_id].part;
+
+ bool line_is_clean;
+
+ ocf_metadata_start_collision_shared_access(cache, line);
+ // ocf_set_valid_map_info
+ if (metadata_set_valid_sec_changed(cache, line, start_sector, last_sector)) {
+ env_atomic_inc(&core->runtime_meta->cached_clines);
+ env_atomic_inc(&core->runtime_meta->
+ part_counters[part_id].cached_clines);
+ }
+
+ // set_cache_line_clean
+ if (metadata_clear_dirty_sec_changed(cache, line, start_sector, last_sector,
+ &line_is_clean)) {
+ if (line_is_clean) {
+ /*
+ * Update the number of dirty cached data for that
+ * core object
+ */
+ if (env_atomic_dec_and_test(&core->runtime_meta->
+ dirty_clines)) {
+ /*
+ * If this is last dirty cline reset dirty
+ * timestamp
+ */
+ env_atomic64_set(&core->runtime_meta->
+ dirty_since, 0);
+ }
+
+ /*
+ * decrement dirty clines statistic for given cline
+ */
+ env_atomic_dec(&core->runtime_meta->
+ part_counters[part_id].dirty_clines);
+ }
+ }
+ ocf_metadata_end_collision_shared_access(cache, line);
+}
+
+static int xcache_wt_hit_core_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_core_t core = xcache_io_core(io);
+ ocf_cache_t cache = ocf_core_get_cache(core);
+ int ret = 0;
+
+
+ if (cb_arg->error) {
+ ret = cb_arg->error;
+ ocf_cache_log(cache, log_err, "wt hit core failed with error %d\n", cb_arg->error);
+ io->error = cb_arg->error;
+ ocf_core_stats_core_error_update(core, OCF_WRITE);
+ }
+
+ xcache_queue_free_backdev_io(xcache_io_queue(io), base_io);
+ return ret;
+}
+
+static int xcache_wt_hit_cache_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ uint64_t addr = cb_arg->addr;
+ uint64_t len = cb_arg->size;
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+ ocf_cache_line_t line = base_io->line;
+ int ret = 0;
+
+ uint64_t offset;
+ uint8_t start_sector, last_sector;
+
+ if (cb_arg->error != 0) {
+ ocf_cache_log(cache, log_err, "wt hit cache with error %d\n", cb_arg->error);
+ io->error = cb_arg->error;
+ ocf_core_stats_cache_error_update(core, OCF_WRITE);
+ ret = cb_arg->error;
+ goto out;
+ }
+
+ offset = xcache_addr_offset(cache, addr);
+ start_sector = BYTES_TO_SECTORS(offset);
+ last_sector = BYTES_TO_SECTORS(offset + len - 1);
+
+ xcache_wt_udpate_metadata(cache, core, line, start_sector, last_sector);
+
+out:
+ xcache_unlock_wr(ocf_cache_line_concurrency(cache), line);
+ xcache_queue_free_backdev_io(xcache_io_queue(base_io->xcache_io), base_io);
+ return ret;
+}
+
+static int xcache_wt_hit_cache(struct xcache_backdev_io *base_io, ocf_cache_line_t line,
+ uint64_t offset, uint64_t size, uint64_t buf_offset)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ uint64_t addr = cache_line_to_addr(cache, line, offset);
+
+ base_io->line = line;
+ base_io->end = xcache_wt_hit_cache_cb;
+ xcache_backdev_submit_io(base_io, true, addr, size, buf_offset, OCF_WRITE);
+ return 0;
+}
+
+static inline int xcache_wt_hit(struct xcache_backdev_io *base_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t offset, uint64_t size, uint64_t buf_offset)
+{
+ return xcache_wt_hit_cache(base_io, line, offset, size, buf_offset);
+}
+
+static void xcache_wt_res(struct xcache_backdev_io *base_io)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ uint64_t offset = xcache_addr_offset(cache, base_io->addr);
+ uint64_t start_addr = xcache_io_start_addr(io);
+ uint64_t buf_offset = base_io->addr - start_addr;
+
+ xcache_wt_hit(base_io, base_io->line, base_io->addr, offset, base_io->size, buf_offset);
+}
+
+static int xcache_wt_miss_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_io *io = base_io->xcache_io;
+ ocf_cache_t cache = xcache_io_cache(io);
+ ocf_core_t core = xcache_io_core(io);
+
+ if (cb_arg->error != 0) {
+ ocf_cache_log(cache, log_err, "wt miss with error %d\n", cb_arg->error);
+ io->error = cb_arg->error;
+ ocf_core_stats_core_error_update(core, OCF_WRITE);
+ }
+ xcache_queue_free_backdev_io(xcache_io_queue(base_io->xcache_io), base_io);
+ return 0;
+}
+
+static int xcache_wt_core(struct xcache_io_context *ctx)
+{
+ struct xcache_io *io = ctx->io;
+ uint64_t addr = xcache_io_start_addr(io);
+ uint64_t size = xcache_io_size(io);
+ ocf_cache_t cache = xcache_io_cache(io);
+ struct xcache_backdev_io *base_io = NULL;
+
+ base_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io));
+ if (base_io == NULL) {
+ ocf_cache_log(cache, log_err, "alloc base io failed\n");
+ return -ENOMEM;
+ }
+ base_io->xcache_io = io;
+ base_io->data = io->data;
+ base_io->end = xcache_wt_miss_cb;
+ xcache_backdev_submit_io(base_io, false, addr, size, 0, OCF_WRITE);
+ return 0;
+}
+
+static void xcache_wt_lb(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line,
+ uint64_t addr, uint64_t size)
+{
+ xcache_wr_lb_common(backdev_io, line, addr, size);
+}
+
+int xcache_wt(struct xcache_io *io)
+{
+ struct xcache_io_context ctx;
+ struct xcache_io_handler wt_handler = {
+ .ctx = &ctx,
+ .res_fn = xcache_wt_res,
+ .valid_fn = NULL,
+ .miss_fn = NULL,
+ .hit_fn = xcache_wt_hit_cache,
+ .need_lb_fn = NULL,
+ .lb_fn = xcache_wt_lb,
+ };
+ int ret;
+
+ mark_flush();
+
+ xcache_init_io_ctx(&ctx, io);
+
+ if (xcache_wt_core(&ctx) != 0) {
+ io->error = -ENOMEM;
+ return -ENOMEM;
+ }
+
+ ret = xcache_handle_io(&wt_handler);
+ if (ret != 0) {
+ io->error = ret;
+ return ret;
+ }
+
+ ctx.core_bytes = xcache_io_size(io);
+ xcache_update_stat(&ctx);
+ return 0;
+}
diff --git a/src/engine/xcache_engine_wt.h b/src/engine/xcache_engine_wt.h
new file mode 100644
index 0000000..d2cc7e7
--- /dev/null
+++ b/src/engine/xcache_engine_wt.h
@@ -0,0 +1,6 @@
+#ifndef XCACHE_ENGINE_WT_H_
+#define XCACHE_ENGINE_WT_H_
+
+int xcache_wt(struct xcache_io *io);
+
+#endif /* XCACHE_ENGINE_WT_H_ */
diff --git a/src/evicting/deadline.c b/src/evicting/deadline.c
new file mode 100644
index 0000000..0050faf
--- /dev/null
+++ b/src/evicting/deadline.c
@@ -0,0 +1,172 @@
+#include <stdint.h>
+
+#include "deadline.h"
+#include "evicting_ops.h"
+#include "evicting_helper.h"
+#include "../xcache.h"
+
+#define STALE_SECS 600
+#define EVICT_BUNCH 10
+#define CLEAN_BUNCH 32
+
+struct deadline_ctrl {
+ uint32_t *access_tick;
+ uint64_t line_num;
+ xcache_line_t evict_iter_line;
+ xcache_line_t clean_iter_line;
+};
+
+struct deadline_iter {
+ xcache_line_t *lines;
+ int line_num;
+ int iter_bunch;
+ xcache_line_t *iter_start;
+ bool line_dirty;
+};
+
+static int deadline_init(xcache_context_t *ctx)
+{
+ struct deadline_ctrl *ctrl = env_malloc(sizeof(struct deadline_ctrl), 0);
+
+ if (ctrl == NULL) {
+ return -1;
+ }
+ ctrl->line_num = evicting_line_num(ctx);
+
+ ctrl->access_tick = env_zalloc(sizeof(uint32_t) * ctrl->line_num, 0);
+ if (ctrl->access_tick == NULL) {
+ goto alloc_tick_fail;
+ }
+
+ ctrl->evict_iter_line = 0;
+ ctrl->clean_iter_line = 0;
+ xcache_set_evicting(ctx, (void *)ctrl);
+ return 0;
+
+alloc_tick_fail:
+ env_free(ctrl);
+ return -1;
+}
+
+static void deadline_fini(xcache_context_t *ctx)
+{
+ struct deadline_ctrl *ctrl = (struct deadline_ctrl *)xcache_get_evicting(ctx);
+
+ if (ctrl == NULL) {
+ return;
+ }
+
+ xcache_set_evicting(ctx, NULL);
+
+ if (ctrl->access_tick != NULL) {
+ env_free(ctrl->access_tick);
+ ctrl->access_tick = NULL;
+ }
+
+ env_free(ctrl);
+}
+
+static void deadline_line_accessed(xcache_context_t *ctx, xcache_line_t line)
+{
+ struct deadline_ctrl *ctrl = (struct deadline_ctrl *)xcache_get_evicting(ctx);
+
+ ctrl->access_tick[line] = env_ticks_to_secs(env_get_tick_count());
+}
+
+static void deadline_line_dirty(xcache_context_t *ctx, xcache_line_t line)
+{
+ return;
+}
+
+static void deadline_line_clean(xcache_context_t *ctx, xcache_line_t line)
+{
+ return;
+}
+
+static int deadline_get_lines(xcache_context_t *ctx, struct deadline_iter *iter)
+{
+ struct deadline_ctrl *ctrl = xcache_get_evicting(ctx);
+ uint32_t stale = env_ticks_to_secs(env_get_tick_count()) - STALE_SECS;
+ uint64_t i, j;
+ xcache_line_t cline;
+
+ j = 0;
+ for (i = 0, cline = *(iter->iter_start);
+ i < iter->iter_bunch;
+ i++, cline = (cline + 1) % ctrl->line_num)
+ {
+ if (ctrl->access_tick[cline] > stale) {
+ continue;
+ }
+
+ if (!evicting_trylock_line(ctx, cline)) {
+ continue;
+ }
+
+ if (evicting_test_dirty(ctx, cline) != iter->line_dirty) {
+ evicting_unlock_line(ctx, cline);
+ continue;
+ }
+
+ iter->lines[j++] = cline;
+ if (j >= iter->line_num) {
+ *(iter->iter_start) = (cline + 1) % ctrl->line_num;
+ return j;
+ }
+ }
+
+ *(iter->iter_start) = cline;
+ return j;
+}
+
+static int deadline_line_to_clean(xcache_context_t *ctx, xcache_line_t *lines, int line_num)
+{
+ struct deadline_ctrl *ctrl = xcache_get_evicting(ctx);
+
+ struct deadline_iter iter = {
+ .lines = lines,
+ .line_num = line_num,
+ .iter_bunch = CLEAN_BUNCH,
+ .iter_start = &(ctrl->clean_iter_line),
+ .line_dirty = true,
+ };
+ return deadline_get_lines(ctx, &iter);
+}
+
+
+static int deadline_line_to_evict(xcache_context_t *ctx, xcache_line_t *lines, int line_num)
+{
+ struct deadline_ctrl *ctrl = xcache_get_evicting(ctx);
+
+ struct deadline_iter iter = {
+ .lines = lines,
+ .line_num = line_num,
+ .iter_bunch = EVICT_BUNCH,
+ .iter_start = &(ctrl->evict_iter_line),
+ .line_dirty = false,
+ };
+ return deadline_get_lines(ctx, &iter);
+}
+
+static void deadline_destroy(xcache_context_t *ctx)
+{
+ struct deadline_ctrl *ctrl = (struct deadline_ctrl *)ctx->xcache_evicting;
+
+ env_free(ctrl->access_tick);
+ env_free(ctrl);
+}
+
+static struct evicting_policy_ops g_deadline_ops = {
+ .init = deadline_init,
+ .fini = deadline_fini,
+ .line_accessed = deadline_line_accessed,
+ .line_dirty = deadline_line_dirty,
+ .line_clean = deadline_line_clean,
+ .line_to_clean = deadline_line_to_clean,
+ .line_to_evict = deadline_line_to_evict,
+};
+
+void set_deadline_policy(void)
+{
+ evicting_set(&g_deadline_ops);
+}
diff --git a/src/evicting/deadline.h b/src/evicting/deadline.h
new file mode 100644
index 0000000..887737b
--- /dev/null
+++ b/src/evicting/deadline.h
@@ -0,0 +1,6 @@
+#ifndef DEADLINE_H_
+#define DEADLINE_H_
+
+void set_deadline_policy(void);
+
+#endif
diff --git a/src/evicting/evicting.c b/src/evicting/evicting.c
new file mode 100644
index 0000000..542693f
--- /dev/null
+++ b/src/evicting/evicting.c
@@ -0,0 +1,3 @@
+#include "evicting_ops.h"
+
+struct evicting_policy_ops *g_evicting_policy = NULL;
diff --git a/src/evicting/evicting_helper.h b/src/evicting/evicting_helper.h
new file mode 100644
index 0000000..e6defa1
--- /dev/null
+++ b/src/evicting/evicting_helper.h
@@ -0,0 +1,32 @@
+#ifndef EVICTING_HELPER_H_
+#define EVICTING_HELPER_H_
+
+#include "../utils/utils_alock.h"
+#include "../concurrency/ocf_cache_line_concurrency.h"
+#include "../ocf_cache_priv.h"
+#include "../metadata/metadata_status.h"
+
+static inline bool evicting_test_dirty(xcache_context_t *ctx, xcache_line_t line)
+{
+ return metadata_test_dirty(ctx->cache, line);
+}
+
+static inline bool evicting_trylock_line(xcache_context_t *ctx, xcache_line_t line)
+{
+ struct ocf_alock *lock = ocf_cache_line_concurrency(ctx->cache);
+
+ return ocf_alock_trylock_entry_wr(lock, line);
+}
+
+static inline void evicting_unlock_line(xcache_context_t *ctx, xcache_line_t line)
+{
+ struct ocf_alock *lock = ocf_cache_line_concurrency(ctx->cache);
+
+ xcache_unlock_wr(lock, line);
+}
+
+static inline uint64_t evicting_line_num(xcache_context_t *ctx)
+{
+ return ctx->cache->device->collision_table_entries;
+}
+#endif
diff --git a/src/evicting/evicting_ops.h b/src/evicting/evicting_ops.h
new file mode 100644
index 0000000..bd9c3ec
--- /dev/null
+++ b/src/evicting/evicting_ops.h
@@ -0,0 +1,61 @@
+#ifndef EVICTING_OPS_H_
+#define EVICTING_OPS_H_
+
+#include <stddef.h>
+#include "../xcache.h"
+
+struct evicting_policy_ops {
+ int (*init)(xcache_context_t *ctx);
+ void (*line_accessed)(xcache_context_t *ctx, xcache_line_t line);
+ void (*line_dirty)(xcache_context_t *ctx, xcache_line_t line);
+ void (*line_clean)(xcache_context_t *ctx, xcache_line_t line);
+ int (*line_to_clean)(xcache_context_t *ctx, xcache_line_t *lines, int line_num);
+ int (*line_to_evict)(xcache_context_t *ctx, xcache_line_t *lines, int line_num);
+ void (*fini)(xcache_context_t *ctx);
+};
+
+extern struct evicting_policy_ops *g_evicting_policy;
+
+static inline void evicting_set(struct evicting_policy_ops *policy)
+{
+ g_evicting_policy = policy;
+}
+
+static inline void evicting_init(xcache_context_t *ctx)
+{
+ g_evicting_policy->init(ctx);
+}
+
+static inline void evicting_line_accessed(xcache_context_t *ctx, xcache_line_t line)
+{
+ g_evicting_policy->line_accessed(ctx, line);
+}
+
+static inline void evicting_line_dirty(xcache_context_t *ctx, xcache_line_t line)
+{
+ g_evicting_policy->line_dirty(ctx, line);
+}
+
+static inline void evicting_line_clean(xcache_context_t *ctx, xcache_line_t line)
+{
+ g_evicting_policy->line_clean(ctx, line);
+}
+
+static inline xcache_line_t evicting_line_to_clean(xcache_context_t *ctx,
+ xcache_line_t *lines, int line_num)
+{
+ return g_evicting_policy->line_to_clean(ctx, lines, line_num);
+}
+
+static inline xcache_line_t evicting_line_to_evict(xcache_context_t *ctx,
+ xcache_line_t *lines, int line_num)
+{
+ return g_evicting_policy->line_to_evict(ctx, lines, line_num);
+}
+
+static inline void evicting_fini(xcache_context_t *ctx)
+{
+ return g_evicting_policy->fini(ctx);
+}
+
+#endif
diff --git a/src/metadata/metadata_raw.c b/src/metadata/metadata_raw.c
index 15287e7..efef69c 100644
--- a/src/metadata/metadata_raw.c
+++ b/src/metadata/metadata_raw.c
@@ -349,6 +349,8 @@ struct _raw_ram_flush_ctx {
struct ocf_metadata_raw *raw;
struct ocf_request *req;
ocf_req_end_t complete;
+ void *io;
+ ocf_metadata_io_ctx_end_t io_end;
env_atomic flush_req_cnt;
int error;
};
@@ -369,8 +371,7 @@ static void _raw_ram_flush_do_asynch_io_complete(ocf_cache_t cache,
OCF_DEBUG_MSG(cache, "Asynchronous flushing complete");
/* Call metadata flush completed call back */
- ctx->req->error |= ctx->error;
- ctx->complete(ctx->req, ctx->error);
+ ctx->io_end(ctx->io, ctx->error);
env_free(ctx);
}
@@ -429,17 +430,17 @@ int _raw_ram_flush_do_page_cmp(const void *item1, const void *item2)
return 0;
}
-static void __raw_ram_flush_do_asynch_add_pages(struct ocf_request *req,
+static void __raw_ram_flush_do_asynch_add_pages(struct ocf_metadata_line_getter *getter,
uint32_t *pages_tab, struct ocf_metadata_raw *raw,
- int *pages_to_flush) {
+ int *pages_to_flush)
+{
int i, j = 0;
- int line_no = req->core_line_count;
- struct ocf_map_info *map;
+ int line_no = line_getter_line_num(getter);
+ ocf_cache_line_t line;
for (i = 0; i < line_no; i++) {
- map = &req->map[i];
- if (map->flush) {
- pages_tab[j] = _RAW_RAM_PAGE(raw, map->coll_idx);
+ if (line_getter_flush_line(getter, i, &line)) {
+ pages_tab[j] = _RAW_RAM_PAGE(raw, line);
j++;
}
}
@@ -447,37 +448,31 @@ static void __raw_ram_flush_do_asynch_add_pages(struct ocf_request *req,
*pages_to_flush = j;
}
-static int _raw_ram_flush_do_asynch(ocf_cache_t cache,
- struct ocf_request *req, struct ocf_metadata_raw *raw,
- ocf_req_end_t complete)
+static int _raw_ram_flush_asynch_common(ocf_cache_t cache,
+ struct ocf_metadata_raw *raw, struct ocf_metadata_io_context *io_ctx,
+ struct ocf_metadata_line_getter *line_getter)
{
int result = 0, i;
uint32_t __pages_tab[MAX_STACK_TAB_SIZE];
uint32_t *pages_tab;
- int line_no = req->core_line_count;
+ int line_no = line_getter_line_num(line_getter);
int pages_to_flush;
uint32_t start_page = 0;
uint32_t count = 0;
struct _raw_ram_flush_ctx *ctx;
- ENV_BUG_ON(!complete);
+ ENV_BUG_ON(!io_ctx->io_end);
OCF_DEBUG_TRACE(cache);
- if (!req->info.flush_metadata) {
- /* Nothing to flush call flush callback */
- complete(req, 0);
- return 0;
- }
-
ctx = env_zalloc(sizeof(*ctx), ENV_MEM_NOIO);
if (!ctx) {
- complete(req, -OCF_ERR_NO_MEM);
+ io_context_end(io_ctx, -OCF_ERR_NO_MEM);
return -OCF_ERR_NO_MEM;
}
- ctx->req = req;
- ctx->complete = complete;
+ ctx->io = io_ctx->io;
+ ctx->io_end = io_ctx->io_end;
ctx->raw = raw;
env_atomic_set(&ctx->flush_req_cnt, 1);
@@ -487,7 +482,7 @@ static int _raw_ram_flush_do_asynch(ocf_cache_t cache,
pages_tab = env_zalloc(sizeof(*pages_tab) * line_no, ENV_MEM_NOIO);
if (!pages_tab) {
env_free(ctx);
- complete(req, -OCF_ERR_NO_MEM);
+ io_context_end(io_ctx, -OCF_ERR_NO_MEM);
return -OCF_ERR_NO_MEM;
}
}
@@ -496,7 +491,7 @@ static int _raw_ram_flush_do_asynch(ocf_cache_t cache,
* to prevent freeing of asynchronous context
*/
- __raw_ram_flush_do_asynch_add_pages(req, pages_tab, raw,
+ __raw_ram_flush_do_asynch_add_pages(line_getter, pages_tab, raw,
&pages_to_flush);
env_sort(pages_tab, pages_to_flush, sizeof(*pages_tab),
@@ -526,9 +521,9 @@ static int _raw_ram_flush_do_asynch(ocf_cache_t cache,
env_atomic_inc(&ctx->flush_req_cnt);
- result |= metadata_io_write_i_asynch(cache, req->io_queue, ctx,
+ result |= metadata_io_write_i_asynch(cache, io_ctx->queue, ctx,
raw->ssd_pages_offset + start_page, count,
- req->ioi.io.flags,
+ io_ctx->io_flags,
_raw_ram_flush_do_asynch_fill,
_raw_ram_flush_do_asynch_io_complete,
raw->mio_conc);
@@ -547,6 +542,43 @@ static int _raw_ram_flush_do_asynch(ocf_cache_t cache,
return result;
}
+static int req_line_num(void *getter)
+{
+ struct ocf_request *req = (struct ocf_request *)getter;
+
+ return req->core_line_count;
+}
+
+static bool req_flush_line(void *getter, int index, ocf_cache_line_t *line)
+{
+ struct ocf_request *req = (struct ocf_request *)getter;
+
+ if (!req->map[index].flush) {
+ return false;
+ }
+
+ *line = req->map[index].coll_idx;
+ return true;
+}
+
+static int _raw_ram_flush_do_asynch(ocf_cache_t cache,
+ struct ocf_request *req, struct ocf_metadata_raw *raw,
+ ocf_req_end_t complete)
+{
+ struct ocf_metadata_io_context io_ctx = {
+ .io = (void *)req,
+ .io_flags = req->ioi.io.flags,
+ .io_end = (ocf_metadata_io_ctx_end_t)complete,
+ .queue = req->io_queue,
+ };
+ struct ocf_metadata_line_getter line_getter = {
+ .getter = (void *)req,
+ .get_line_num = req_line_num,
+ .get_flush_line = req_flush_line,
+ };
+ return _raw_ram_flush_asynch_common(cache, raw, &io_ctx, &line_getter);
+}
+
/*******************************************************************************
* RAW Interfaces definitions
******************************************************************************/
@@ -566,6 +598,7 @@ static const struct raw_iface IRAW[metadata_raw_type_max] = {
.flush_all = _raw_ram_flush_all,
.flush_mark = _raw_ram_flush_mark,
.flush_do_asynch = _raw_ram_flush_do_asynch,
+ .flush_do_asynch_common = _raw_ram_flush_asynch_common
},
[metadata_raw_type_dynamic] = {
.init = raw_dynamic_init,
diff --git a/src/metadata/metadata_raw.h b/src/metadata/metadata_raw.h
index 0357774..57f7b75 100644
--- a/src/metadata/metadata_raw.h
+++ b/src/metadata/metadata_raw.h
@@ -93,6 +93,35 @@ struct ocf_metadata_raw {
struct ocf_alock *mio_conc;
};
+struct ocf_metadata_line_getter {
+ void *getter;
+ int (*get_line_num)(void *getter);
+ bool (*get_flush_line)(void *getter, int index, ocf_cache_line_t *line);
+};
+
+static inline int line_getter_line_num(struct ocf_metadata_line_getter *getter)
+{
+ return getter->get_line_num(getter->getter);
+}
+
+static inline bool line_getter_flush_line(struct ocf_metadata_line_getter *getter,
+ int index, ocf_cache_line_t *line)
+{
+ return getter->get_flush_line(getter->getter, index, line);
+}
+
+typedef void (*ocf_metadata_io_ctx_end_t)(void *io, int error);
+struct ocf_metadata_io_context {
+ void *io;
+ int io_flags;
+ ocf_queue_t queue;
+ ocf_metadata_io_ctx_end_t io_end;
+};
+
+static inline void io_context_end(struct ocf_metadata_io_context *ctx, int error)
+{
+ ctx->io_end(ctx->io, error);
+}
/**
* RAW container interface
*/
@@ -137,8 +166,12 @@ struct raw_iface {
int (*flush_do_asynch)(ocf_cache_t cache, struct ocf_request *req,
struct ocf_metadata_raw *raw, ocf_req_end_t complete);
+
+ int (*flush_do_asynch_common)(ocf_cache_t cache, struct ocf_metadata_raw *raw,
+ struct ocf_metadata_io_context *io_ctx, struct ocf_metadata_line_getter *line_getter);
};
+
/**
* @brief Initialize RAW instance
*
@@ -288,6 +321,13 @@ static inline int ocf_metadata_raw_flush_do_asynch(ocf_cache_t cache,
return raw->iface->flush_do_asynch(cache, req, raw, complete);
}
+static inline int ocf_metadata_raw_flush_do_asynch_common(ocf_cache_t cache,
+ struct ocf_metadata_raw *raw, struct ocf_metadata_io_context *io_ctx,
+ struct ocf_metadata_line_getter *line_getter)
+{
+ return raw->iface->flush_do_asynch_common(cache, raw, io_ctx, line_getter);
+}
+
/*
* Check if line is valid for specified RAW descriptor
*/
diff --git a/src/metadata/xcache_metadata.c b/src/metadata/xcache_metadata.c
new file mode 100644
index 0000000..53f385c
--- /dev/null
+++ b/src/metadata/xcache_metadata.c
@@ -0,0 +1,88 @@
+#include "ocf/ocf.h"
+
+#include "xcache_metadata.h"
+#include "metadata.h"
+#include "metadata_internal.h"
+#include "../engine/xcache_engine_common.h" /* xcache_io_xxx */
+#include "../ocf_cache_priv.h"
+
+void xcache_metadata_flush_asynch(struct ocf_cache *cache,
+ struct ocf_metadata_io_context *io_ctx,
+ struct ocf_metadata_line_getter *line_getter)
+{
+ int result = 0;
+ struct ocf_metadata_ctrl *ctrl = NULL;
+
+ ctrl = (struct ocf_metadata_ctrl *) cache->metadata.priv;
+ result |= ocf_metadata_raw_flush_do_asynch_common(cache,
+ &(ctrl->raw_desc[metadata_segment_collision]),
+ io_ctx, line_getter);
+ if (result) {
+ ocf_metadata_error(cache);
+ ocf_cache_log(cache, log_err, "Metadata Flush ERROR\n");
+ }
+}
+
+static inline bool is_power_of_two(uint64_t n)
+{
+ return (((n - 1) & n) == 0);
+}
+
+int xcache_io_add_flush_line(struct xcache_io *io, ocf_cache_line_t line)
+{
+ ocf_cache_line_t *flush_lines;
+ int i;
+
+ if (io->flush_line_num < INLINE_FLUSH_LINES) {
+ flush_lines = io->flush_lines;
+ } else if (is_power_of_two(io->flush_line_num)) {
+ flush_lines = env_malloc(sizeof(ocf_cache_line_t) * (io->flush_line_num << 1), 0);
+ if (flush_lines == NULL) {
+ return -ENOMEM;
+ }
+ for (i = 0; i < io->flush_line_num; i++) {
+ flush_lines[i] = io->flush_lines[i];
+ }
+ *(ocf_cache_line_t **)io->flush_lines = flush_lines;
+ } else {
+ flush_lines = *(ocf_cache_line_t **)io->flush_lines;
+ }
+
+ flush_lines[io->flush_line_num++] = line;
+ return 0;
+}
+
+static int io_line_num(void *getter)
+{
+ struct xcache_io *io = (struct xcache_io *)getter;
+
+ return io->flush_line_num;
+}
+
+static bool io_flush_line(void *getter, int index, ocf_cache_line_t *line)
+{
+ struct xcache_io *io = (struct xcache_io *)getter;
+ ocf_cache_line_t *flush_lines;
+
+ flush_lines = xcache_io_get_flush_line(io);
+ *line = io->flush_lines[index];
+ return true;
+}
+
+void xcache_io_metadata_flush(struct xcache_io *io, ocf_metadata_io_ctx_end_t io_end)
+{
+ struct ocf_metadata_io_context io_ctx = {
+ .io = (void *)io,
+ .io_flags = xcache_io_flags(io),
+ .io_end = io_end,
+ .queue = xcache_io_queue(io),
+ };
+ struct ocf_metadata_line_getter line_getter = {
+ .getter = (void *)io,
+ .get_line_num = io_line_num,
+ .get_flush_line = io_flush_line,
+ };
+
+ xcache_io_get(io);
+ xcache_metadata_flush_asynch(xcache_io_cache(io), &io_ctx, &line_getter);
+}
diff --git a/src/metadata/xcache_metadata.h b/src/metadata/xcache_metadata.h
new file mode 100644
index 0000000..e82580e
--- /dev/null
+++ b/src/metadata/xcache_metadata.h
@@ -0,0 +1,47 @@
+#ifndef __XCACHE_METADATA_H__
+#define __XCACHE_METADATA_H__
+
+#include "ocf/ocf.h"
+#include "metadata_raw.h"
+
+#include "ocf/xcache.h"
+
+void xcache_metadata_flush_asynch(struct ocf_cache *cache,
+ struct ocf_metadata_io_context *io_ctx,
+ struct ocf_metadata_line_getter *line_getter);
+
+int xcache_io_add_flush_line(struct xcache_io *io, ocf_cache_line_t line);
+static inline void xcache_io_free_flush_line(struct xcache_io *io)
+{
+ if (io->flush_line_num <= INLINE_FLUSH_LINES) {
+ return;
+ }
+ env_free(*(ocf_cache_line_t **)io->flush_lines);
+}
+
+static inline ocf_cache_line_t *get_inline_flush_lines(struct xcache_io *io)
+{
+ return io->flush_lines;
+}
+
+static inline ocf_cache_line_t *get_alloc_flush_lines(struct xcache_io *io)
+{
+ return *(ocf_cache_line_t **)io->flush_lines;
+}
+
+static inline ocf_cache_line_t *xcache_io_get_flush_line(struct xcache_io *io)
+{
+ if (io->flush_line_num <= INLINE_FLUSH_LINES) {
+ return get_inline_flush_lines(io);
+ } else {
+ return get_alloc_flush_lines(io);
+ }
+}
+
+static inline bool xcache_metadata_should_flush(struct xcache_io *io)
+{
+ return (env_atomic_read(&io->remaining) == 1);
+}
+
+void xcache_io_metadata_flush(struct xcache_io *io, ocf_metadata_io_ctx_end_t io_end);
+#endif
diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c
index 54a2a67..591ffcd 100644
--- a/src/mngt/ocf_mngt_cache.c
+++ b/src/mngt/ocf_mngt_cache.c
@@ -124,6 +124,8 @@ struct ocf_cache_attach_context {
*/
bool concurrency_inited : 1;
+
+ bool xcache_inited : 1;
} flags;
struct {
@@ -975,6 +977,12 @@ static void _ocf_mngt_attach_prepare_metadata(ocf_pipeline_t pipeline,
context->flags.attached_metadata_inited = true;
+ ret = xcache_init(cache);
+ if (ret != 0) {
+ OCF_PL_FINISH_RET(pipeline, ret);
+ }
+ context->flags.xcache_inited = 1;
+
ret = ocf_concurrency_init(cache);
if (ret)
OCF_PL_FINISH_RET(pipeline, ret);
@@ -1126,6 +1134,9 @@ static void _ocf_mngt_attach_handle_error(
if (context->flags.device_alloc)
env_vfree(cache->device);
+ if (context->flags.xcache_inited)
+ xcache_fini(cache);
+
ocf_pipeline_destroy(cache->stop_pipeline);
}
diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h
index b0a6f77..07fcd63 100644
--- a/src/ocf_cache_priv.h
+++ b/src/ocf_cache_priv.h
@@ -139,6 +139,7 @@ struct ocf_cache {
} __attribute__((aligned(64)));
// This should be on it's own cacheline ideally
env_atomic last_access_ms;
+ void *xcache_ctx;
};
static inline ocf_core_t ocf_cache_get_core(ocf_cache_t cache,
diff --git a/src/ocf_lru.c b/src/ocf_lru.c
index e9c3882..8e323c0 100644
--- a/src/ocf_lru.c
+++ b/src/ocf_lru.c
@@ -221,7 +221,7 @@ void ocf_lru_init_cline(ocf_cache_t cache, ocf_cache_line_t cline)
node->next = end_marker;
}
-static struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part,
+struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part,
uint32_t lru_idx, bool clean)
{
if (part->id == PARTITION_FREELIST)
@@ -261,7 +261,7 @@ static inline void ocf_lru_move(ocf_cache_t cache, ocf_cache_line_t cline,
add_lru_head(cache, dst_list, cline);
}
-static void ocf_lru_repart_locked(ocf_cache_t cache, ocf_cache_line_t cline,
+void ocf_lru_repart_locked(ocf_cache_t cache, ocf_cache_line_t cline,
struct ocf_part *src_part, struct ocf_part *dst_part)
{
uint32_t lru_list = (cline % OCF_NUM_LRU_LISTS);
@@ -658,7 +658,7 @@ void ocf_lru_clean(ocf_cache_t cache, struct ocf_user_part *user_part,
ocf_cleaner_fire(cache, &attribs);
}
-static void ocf_lru_invalidate(ocf_cache_t cache, ocf_cache_line_t cline,
+void ocf_lru_invalidate(ocf_cache_t cache, ocf_cache_line_t cline,
ocf_core_id_t core_id, ocf_part_id_t part_id)
{
ocf_core_t core;
diff --git a/src/ocf_lru.h b/src/ocf_lru.h
index a71b5fd..ae50b5e 100644
--- a/src/ocf_lru.h
+++ b/src/ocf_lru.h
@@ -33,4 +33,11 @@ void ocf_lru_repart(ocf_cache_t cache, ocf_cache_line_t cline,
uint32_t ocf_lru_num_free(ocf_cache_t cache);
void ocf_lru_populate(ocf_cache_t cache, ocf_cache_line_t num_free_clines);
+void ocf_lru_repart_locked(ocf_cache_t cache, ocf_cache_line_t cline,
+ struct ocf_part *src_part, struct ocf_part *dst_part);
+struct ocf_lru_list *ocf_lru_get_list(struct ocf_part *part,
+ uint32_t lru_idx, bool clean);
+void ocf_lru_invalidate(ocf_cache_t cache, ocf_cache_line_t cline,
+ ocf_core_id_t core_id, ocf_part_id_t part_id);
+
#endif
diff --git a/src/ocf_queue.c b/src/ocf_queue.c
index a754d6e..9e89102 100644
--- a/src/ocf_queue.c
+++ b/src/ocf_queue.c
@@ -51,8 +51,15 @@ int ocf_queue_create(ocf_cache_t cache, ocf_queue_t *queue,
return result;
}
- list_add(&tmp_queue->list, &cache->io_queues);
+ result = xcache_queue_ctx_init(tmp_queue);
+ if (result) {
+ ocf_queue_seq_cutoff_deinit(tmp_queue);
+ ocf_mngt_cache_put(cache);
+ env_free(tmp_queue);
+ return result;
+ }
+ list_add(&tmp_queue->list, &cache->io_queues);
*queue = tmp_queue;
return 0;
@@ -91,7 +98,7 @@ void ocf_io_handle(struct ocf_io *io, void *opaque)
req->io_if->read(req);
}
-void ocf_queue_run_single(ocf_queue_t q)
+void __attribute__((weak)) ocf_queue_run_single(ocf_queue_t q)
{
struct ocf_request *io_req = NULL;
diff --git a/src/ocf_queue_priv.h b/src/ocf_queue_priv.h
index def5d08..62bf900 100644
--- a/src/ocf_queue_priv.h
+++ b/src/ocf_queue_priv.h
@@ -36,6 +36,8 @@ struct ocf_queue {
env_atomic ref_count;
env_spinlock io_list_lock;
+
+ void *priv1;
} __attribute__((__aligned__(64)));
static inline void ocf_queue_kick(ocf_queue_t queue, bool allow_sync)
diff --git a/src/qos/qos.c b/src/qos/qos.c
new file mode 100644
index 0000000..6ea2da9
--- /dev/null
+++ b/src/qos/qos.c
@@ -0,0 +1,6 @@
+#include "qos.h"
+
+void xcache_qos_init(struct xcache_qos *qos)
+{
+ qos_lb_init(&qos->qos_lb);
+}
diff --git a/src/qos/qos.h b/src/qos/qos.h
new file mode 100644
index 0000000..3b6a691
--- /dev/null
+++ b/src/qos/qos.h
@@ -0,0 +1,27 @@
+#ifndef __QOS_H__
+#define __QOS_H__
+
+#include "qos_lb.h"
+
+struct xcache_qos {
+ struct qos_lb qos_lb;
+};
+
+static inline void xcache_qos_load_add(struct xcache_qos *qos, int dev, enum xcache_dir dir, uint64_t bytes)
+{
+ qos_lb_load_add(&qos->qos_lb, dev, dir, bytes);
+}
+
+static inline void xcache_qos_load_sub(struct xcache_qos *qos, int dev, enum xcache_dir dir, uint64_t bytes)
+{
+ qos_lb_load_sub(&qos->qos_lb, dev, dir, bytes);
+}
+
+static inline bool xcache_qos_need_lb(struct xcache_qos *qos, enum xcache_dir dir, uint64_t bytes)
+{
+ return qos_need_lb(&qos->qos_lb, dir, bytes);
+}
+
+void xcache_qos_init(struct xcache_qos *qos);
+
+#endif
diff --git a/src/qos/qos_lb.h b/src/qos/qos_lb.h
new file mode 100644
index 0000000..bb3bfe4
--- /dev/null
+++ b/src/qos/qos_lb.h
@@ -0,0 +1,143 @@
+#ifndef __QOS_LB_H__
+#define __QOS_LB_H__
+
+#define CORE_DEFAULT_LOAD_WEIGHT 30
+#define CACHE_DEFAULT_LOAD_WEIGHT 1
+
+#define CORE_DEV 0
+#define CACHE_DEV 1
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "ocf/xcache.h"
+
+struct qos_dev_load {
+ env_atomic64 read_inflight_bytes;
+ env_atomic64 write_inflight_bytes;
+ uint32_t read_weight;
+ uint32_t write_weight;
+};
+
+static inline void qos_dev_load_init(struct qos_dev_load *load)
+{
+ env_atomic64_set(&load->read_inflight_bytes, 0);
+ env_atomic64_set(&load->write_inflight_bytes, 0);
+}
+
+static inline void qos_dev_load_add(struct qos_dev_load *load, enum xcache_dir dir, uint64_t bytes)
+{
+ switch (dir) {
+ case XCACHE_RD:
+ env_atomic64_add(bytes, &load->read_inflight_bytes);
+ break;
+ case XCACHE_WR:
+ env_atomic64_add(bytes, &load->write_inflight_bytes);
+ break;
+ default:
+ break;
+ }
+}
+
+static inline void qos_dev_load_sub(struct qos_dev_load *load, enum xcache_dir dir, uint64_t bytes)
+{
+ switch (dir) {
+ case XCACHE_RD:
+ env_atomic64_sub(bytes, &load->read_inflight_bytes);
+ break;
+ case XCACHE_WR:
+ env_atomic64_sub(bytes, &load->write_inflight_bytes);
+ break;
+ default:
+ break;
+ }
+}
+
+static inline uint64_t qos_dev_load_read(struct qos_dev_load *load, enum xcache_dir dir)
+{
+ switch (dir) {
+ case XCACHE_RD:
+ return env_atomic64_read(&load->read_inflight_bytes);
+ case XCACHE_WR:
+ return env_atomic64_read(&load->write_inflight_bytes);
+ default:
+ return 0;
+ }
+}
+
+static inline uint32_t qos_dev_load_weight(struct qos_dev_load *load, enum xcache_dir dir)
+{
+ switch (dir) {
+ case XCACHE_RD:
+ return load->read_weight;
+ case XCACHE_WR:
+ return load->write_weight;
+ default:
+ return 0;
+ }
+}
+
+static inline uint64_t do_cal_load(uint64_t bytes, uint32_t weight)
+{
+ return bytes * weight;
+}
+
+static inline uint64_t qos_dev_load_cal(struct qos_dev_load *load)
+{
+ uint64_t read_inflight_bytes = qos_dev_load_read(load, XCACHE_RD);
+ uint64_t write_inflight_bytes = qos_dev_load_read(load, XCACHE_WR);
+
+ return do_cal_load(read_inflight_bytes, load->read_weight) +
+ do_cal_load(write_inflight_bytes, load->write_weight);
+}
+
+struct qos_lb {
+ struct qos_dev_load cache_load;
+ struct qos_dev_load core_load;
+};
+
+static inline void qos_lb_init(struct qos_lb *qos_lb)
+{
+ qos_dev_load_init(&qos_lb->cache_load);
+ qos_dev_load_init(&qos_lb->core_load);
+ qos_lb->cache_load.read_weight = CACHE_DEFAULT_LOAD_WEIGHT;
+ qos_lb->cache_load.write_weight = CACHE_DEFAULT_LOAD_WEIGHT;
+ qos_lb->core_load.read_weight = CORE_DEFAULT_LOAD_WEIGHT;
+ qos_lb->core_load.write_weight = CORE_DEFAULT_LOAD_WEIGHT;
+}
+
+static inline void qos_lb_load_add(struct qos_lb *qos_lb, int dev, enum xcache_dir dir, uint64_t bytes)
+{
+ switch (dev) {
+ case CACHE_DEV:
+ qos_dev_load_add(&qos_lb->cache_load, dir, bytes);
+ break;
+ case CORE_DEV:
+ qos_dev_load_add(&qos_lb->core_load, dir, bytes);
+ break;
+ default:
+ break;
+ }
+}
+
+static inline void qos_lb_load_sub(struct qos_lb *qos_lb, int dev, enum xcache_dir dir, uint64_t bytes)
+{
+ switch (dev) {
+ case CACHE_DEV:
+ qos_dev_load_sub(&qos_lb->cache_load, dir, bytes);
+ break;
+ case CORE_DEV:
+ qos_dev_load_sub(&qos_lb->core_load, dir, bytes);
+ break;
+ default:
+ break;
+ }
+}
+
+static inline bool qos_need_lb(struct qos_lb *qos_lb, enum xcache_dir dir, uint64_t bytes)
+{
+ return qos_dev_load_cal(&qos_lb->cache_load) > qos_dev_load_cal(&qos_lb->core_load) +
+ do_cal_load(bytes, qos_dev_load_weight(&qos_lb->core_load, dir));
+}
+
+#endif
diff --git a/src/utils/utils_alock.c b/src/utils/utils_alock.c
index 25f41a6..7cecacf 100644
--- a/src/utils/utils_alock.c
+++ b/src/utils/utils_alock.c
@@ -799,3 +799,125 @@ uint32_t ocf_alock_waitlist_count(struct ocf_alock *alock)
{
return env_atomic_read(&alock->waiting);
}
+
+int xcache_lock_wr(struct ocf_alock *alock,
+ const ocf_cache_line_t entry,
+ ocf_io_lock_prepare_wait prepare_wait_fn,
+ ocf_io_lock_prepare_wake prepare_wake_fn,
+ void *io)
+{
+ struct ocf_alock_waiter *waiter;
+ unsigned long flags = 0;
+ int ret = OCF_LOCK_NOT_ACQUIRED;
+
+ if (ocf_alock_trylock_entry_wr(alock, entry)) {
+ return OCF_LOCK_ACQUIRED;
+ }
+
+ ocf_alock_waitlist_lock(alock, entry, flags);
+
+ /* At the moment list is protected, double check if the cache entry is
+ * unlocked
+ */
+ if (ocf_alock_trylock_entry_wr(alock, entry)) {
+ ret = OCF_LOCK_ACQUIRED;
+ goto unlock;
+ }
+
+ waiter = env_allocator_new(alock->allocator);
+ if (!waiter) {
+ ret = -OCF_ERR_NO_MEM;
+ goto unlock;
+ }
+
+ /* Setup waiters filed */
+ waiter->entry = entry;
+ waiter->req = (struct ocf_request *)io;
+ waiter->cmpl = prepare_wake_fn;
+ waiter->rw = OCF_WRITE;
+ INIT_LIST_HEAD(&waiter->item);
+
+ prepare_wait_fn(io);
+ /* Add to waiters list */
+ ocf_alock_waitlist_add(alock, entry, waiter);
+
+unlock:
+ ocf_alock_waitlist_unlock(alock, entry, flags);
+
+ return ret;
+}
+
+static inline void xcache_unlock_wr_common(struct ocf_alock *alock,
+ const ocf_cache_line_t entry)
+{
+ bool locked = false;
+ bool exchanged = true;
+
+ uint32_t idx = _WAITERS_LIST_ITEM(entry);
+ struct ocf_alock_waiters_list *lst = &alock->waiters_lsts[idx];
+ struct ocf_alock_waiter *waiter;
+
+ struct list_head *iter, *next;
+
+ /*
+ * Lock exchange scenario
+ * 1. WR -> IDLE
+ * 2. WR -> RD
+ * 3. WR -> WR
+ */
+
+ /* Check is requested page is on the list */
+ list_for_each_safe(iter, next, &lst->head) {
+ waiter = list_entry(iter, struct ocf_alock_waiter, item);
+
+ if (entry != waiter->entry)
+ continue;
+
+ if (exchanged) {
+ if (waiter->rw == OCF_WRITE)
+ locked = ocf_alock_trylock_entry_wr2wr(alock, entry);
+ else if (waiter->rw == OCF_READ)
+ locked = ocf_alock_trylock_entry_wr2rd(alock, entry);
+ else
+ ENV_BUG();
+ } else {
+ if (waiter->rw == OCF_WRITE)
+ locked = ocf_alock_trylock_entry_wr(alock, entry);
+ else if (waiter->rw == OCF_READ)
+ locked = ocf_alock_trylock_entry_rd(alock, entry);
+ else
+ ENV_BUG();
+ }
+
+ if (locked) {
+ exchanged = false;
+ list_del(iter);
+
+ waiter->cmpl(waiter->req);
+
+ env_allocator_del(alock->allocator, waiter);
+ } else {
+ break;
+ }
+ }
+
+ if (exchanged) {
+ /* No exchange, no waiters on the list, unlock and return
+ * WR -> IDLE
+ */
+ ocf_alock_unlock_entry_wr(alock, entry);
+ }
+}
+
+void xcache_unlock_wr(struct ocf_alock *alock,
+ const ocf_cache_line_t entry)
+{
+ unsigned long flags = 0;
+
+ OCF_DEBUG_CACHE(alock->cache, "Cache entry unlock one wr = %u", entry);
+
+ /* Lock waiters list */
+ ocf_alock_waitlist_lock(alock, entry, flags);
+ xcache_unlock_wr_common(alock, entry);
+ ocf_alock_waitlist_unlock(alock, entry, flags);
+}
diff --git a/src/utils/utils_alock.h b/src/utils/utils_alock.h
index 2d3df97..80188e3 100644
--- a/src/utils/utils_alock.h
+++ b/src/utils/utils_alock.h
@@ -87,4 +87,15 @@ void ocf_alock_waitlist_remove_entry(struct ocf_alock *alock,
bool ocf_alock_trylock_entry_rd_idle(struct ocf_alock *alock,
ocf_cache_line_t entry);
+typedef void (*ocf_io_lock_prepare_wait)(void *io);
+typedef void (*ocf_io_lock_prepare_wake)(void *io);
+int xcache_lock_wr(struct ocf_alock *alock,
+ const ocf_cache_line_t entry,
+ ocf_io_lock_prepare_wait prepare_wait_fn,
+ ocf_io_lock_prepare_wake prepare_wake_fn,
+ void *io);
+
+void xcache_unlock_wr(struct ocf_alock *alock,
+ const ocf_cache_line_t entry);
+
#endif
diff --git a/src/utils/utils_cache_line.c b/src/utils/utils_cache_line.c
index 281ff59..bc58054 100644
--- a/src/utils/utils_cache_line.c
+++ b/src/utils/utils_cache_line.c
@@ -119,8 +119,10 @@ void set_cache_line_clean(struct ocf_cache *cache, uint8_t start_bit,
*/
env_atomic_dec(&req->core->runtime_meta->
part_counters[part_id].dirty_clines);
- ocf_lru_clean_cline(cache, part, line);
- ocf_purge_cleaning_policy(cache, line);
+ if (cache->xcache_ctx == NULL) {
+ ocf_lru_clean_cline(cache, part, line);
+ ocf_purge_cleaning_policy(cache, line);
+ }
}
}
diff --git a/src/xcache.c b/src/xcache.c
new file mode 100644
index 0000000..0c6a2b8
--- /dev/null
+++ b/src/xcache.c
@@ -0,0 +1,30 @@
+#include "xcache.h"
+#include "evicting/evicting_ops.h"
+#include "evicting/deadline.h"
+#include "utils/utils_cache_line.h"
+
+int xcache_init(ocf_cache_t cache)
+{
+ xcache_context_t *ctx = env_malloc(sizeof(xcache_context_t), 0);
+
+ if (ctx == NULL) {
+ return -1;
+ }
+
+ ctx->cache = cache;
+ cache->xcache_ctx = ctx;
+
+ set_deadline_policy();
+ evicting_init(ctx);
+ ctx->line_size_shift = __builtin_ffsll(ocf_line_size(cache)) - 1;
+ xcache_qos_init(&ctx->qos);
+
+ return 0;
+}
+
+void xcache_fini(ocf_cache_t cache)
+{
+ evicting_fini(cache->xcache_ctx);
+ env_free(cache->xcache_ctx);
+ cache->xcache_ctx = NULL;
+}
diff --git a/src/xcache.h b/src/xcache.h
new file mode 100644
index 0000000..4fd7277
--- /dev/null
+++ b/src/xcache.h
@@ -0,0 +1,58 @@
+#ifndef XCACHE_H_
+#define XCACHE_H_
+
+#include <stdint.h>
+#include "ocf/ocf_types.h"
+#include "./ocf_cache_priv.h"
+
+#include "ocf/xcache.h"
+#include "xcache_cleaner.h"
+#include "qos/qos.h"
+
+typedef ocf_cache_line_t xcache_line_t;
+
+typedef struct xcache_context {
+ uint8_t line_size_shift;
+ void *xcache_evicting;
+ ocf_cache_t cache;
+ struct xcache_cleaning_ctx cleaning_ctx;
+ struct xcache_qos qos;
+} xcache_context_t;
+
+static inline xcache_context_t *xcache_get_ctx(ocf_cache_t cache)
+{
+ return cache->xcache_ctx;
+}
+
+static inline void *xcache_get_evicting(xcache_context_t *ctx)
+{
+ return ctx->xcache_evicting;
+}
+
+static inline void xcache_set_evicting(xcache_context_t *ctx, void *evicting)
+{
+ ctx->xcache_evicting = evicting;
+}
+
+static inline uint64_t xcache_addr_to_line(ocf_cache_t cache, uint64_t addr)
+{
+ xcache_context_t *ctx = cache->xcache_ctx;
+ return addr >> ctx->line_size_shift;
+}
+
+static inline uint64_t xcache_line_to_addr(ocf_cache_t cache, uint64_t line)
+{
+ xcache_context_t *ctx = cache->xcache_ctx;
+ return line << ctx->line_size_shift;
+}
+
+static inline struct xcache_cleaning_ctx *xcache_get_cleaning_ctx(ocf_cache_t cache)
+{
+ xcache_context_t *xcache_ctx = cache->xcache_ctx;
+ return &xcache_ctx->cleaning_ctx;
+}
+
+int xcache_init(ocf_cache_t cache);
+void xcache_fini(ocf_cache_t cache);
+
+#endif
diff --git a/src/xcache_cleaner.c b/src/xcache_cleaner.c
new file mode 100644
index 0000000..5de3369
--- /dev/null
+++ b/src/xcache_cleaner.c
@@ -0,0 +1,572 @@
+#include "metadata/metadata.h"
+#include "concurrency/ocf_concurrency.h"
+#include "utils/utils_cleaner.h"
+#include "utils/utils_cache_line.h"
+#include "ocf_queue_priv.h"
+#include "cleaning/cleaning.h"
+
+#include "ocf/xcache.h"
+
+#include "xcache_cleaner.h"
+#include "xcache.h"
+#include "engine/xcache_engine_common.h"
+#include "xcache_queue.h"
+#include "evicting/evicting_ops.h"
+#include "metadata/xcache_metadata.h"
+
+static inline uint8_t get_first_sector(ocf_cache_t cache, ocf_cache_line_t line, uint8_t start_sector, bool dirty)
+{
+ uint8_t sector;
+
+ for (sector = start_sector; sector < ocf_line_sectors(cache); sector++) {
+ if (metadata_test_dirty_one(cache, line, sector) == dirty) {
+ break;
+ }
+ }
+ return sector;
+}
+
+static inline uint8_t get_first_clean_sector(ocf_cache_t cache, ocf_cache_line_t line, uint8_t start_sector)
+{
+ return get_first_sector(cache, line, start_sector, false);
+}
+
+static inline uint8_t get_first_dirty_sector(ocf_cache_t cache, ocf_cache_line_t line, uint8_t start_sector)
+{
+ return get_first_sector(cache, line, start_sector, true);
+}
+
+static inline void get_dirty_sectors(ocf_cache_t cache, ocf_cache_line_t line,
+ uint8_t *start_sector, uint8_t *end_sector)
+{
+ bool dirty;
+
+ uint8_t sector = *start_sector;
+ uint8_t line_sectors = ocf_line_sectors(cache);
+
+ // fast path
+ if (*start_sector == 0 && metadata_test_dirty_all(cache, line)) {
+ *end_sector = line_sectors - 1;
+ return;
+ }
+
+ *start_sector = get_first_dirty_sector(cache, line, *start_sector);
+ if (*start_sector >= line_sectors) {
+ return;
+ }
+
+ *end_sector = get_first_clean_sector(cache, line, *start_sector + 1) - 1;
+}
+
+static inline void xcache_clean_unlock_line(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ struct clean_sort_data *data = cleaning_ctx->data;
+ struct ocf_alock *c = ocf_cache_line_concurrency(cleaning_ctx->cache);
+ int i;
+
+ for (i = 0; i < cleaning_ctx->count; i++) {
+ xcache_unlock_wr(c, data[i].line);
+ }
+}
+
+static inline void xcache_clean_cleanup(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ struct xcache_backdev_io *base_io = &cleaning_ctx->base_io;
+ ocf_cache_t cache = cleaning_ctx->cache;
+
+ int i;
+
+ if (base_io->data != NULL) {
+ ctx_data_free(cache->owner, base_io->data);
+ base_io->data = NULL;
+ }
+
+ xcache_cleaner_complete(cleaning_ctx);
+}
+
+static inline void xcache_clean_cb(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ if (cleaning_ctx->end) {
+ cleaning_ctx->end(cleaning_ctx->arg);
+ cleaning_ctx->end = NULL;
+ }
+}
+
+static void xcache_clean_end(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ xcache_clean_unlock_line(cleaning_ctx);
+ xcache_clean_cleanup(cleaning_ctx);
+ xcache_clean_cb(cleaning_ctx);
+}
+
+static void xcache_clean_update_metadata(ocf_cache_t cache, ocf_core_t core,
+ ocf_cache_line_t line)
+{
+ ocf_part_id_t part_id = PARTITION_DEFAULT;
+ struct ocf_part *part = &cache->user_parts[part_id].part;
+
+ ocf_metadata_start_collision_shared_access(cache, line);
+ metadata_clear_dirty(cache, line);
+
+ if (env_atomic_dec_and_test(&core->runtime_meta->
+ dirty_clines)) {
+ env_atomic64_set(&core->runtime_meta->
+ dirty_since, 0);
+ }
+
+ env_atomic_dec(&core->runtime_meta->
+ part_counters[part_id].dirty_clines);
+ ocf_metadata_end_collision_shared_access(cache,
+ line);
+}
+
+static int clean_line_num(void *clean_io)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = (struct xcache_cleaning_ctx *)clean_io;
+
+ return cleaning_ctx->count;
+}
+
+static bool clean_flush_line(void *clean_io, int index, ocf_cache_line_t *line)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = (struct xcache_cleaning_ctx *)clean_io;
+
+ *line = cleaning_ctx->data[index].line;
+ return true;
+}
+
+static void xcache_clean_flush_metadata_end(void *clean_io, int error)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = (struct xcache_cleaning_ctx *)clean_io;
+
+ xcache_clean_end(cleaning_ctx);
+}
+
+static void xcache_clean_metadata_flush(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ struct xcache_io *io = &cleaning_ctx->io;
+ struct ocf_metadata_io_context io_ctx = {
+ .io = (void *)cleaning_ctx,
+ .io_flags = 0,
+ .io_end = xcache_clean_flush_metadata_end,
+ .queue = xcache_io_queue(io),
+ };
+ struct ocf_metadata_line_getter line_getter = {
+ .getter = (void *)cleaning_ctx,
+ .get_line_num = clean_line_num,
+ .get_flush_line = clean_flush_line,
+ };
+
+ xcache_metadata_flush_asynch(cleaning_ctx->cache, &io_ctx, &line_getter);
+}
+
+static void xcache_clean_bf_end(struct xcache_io *io, int error)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = container_of(io, struct xcache_cleaning_ctx,
+ io);
+ struct clean_sort_data *data = cleaning_ctx->data;
+ ocf_cache_t cache = cleaning_ctx->cache;
+ struct ocf_alock *c = ocf_cache_line_concurrency(cleaning_ctx->cache);
+
+ ocf_core_t core;
+ ocf_cache_line_t line;
+ int i, j;
+
+ if (error != 0) {
+ xcache_clean_end(cleaning_ctx);
+ return;
+ }
+
+ for (i = 0, j = 0; i < cleaning_ctx->count; i++) {
+ // core_line write failed
+ if (data[i].core_line == ULLONG_MAX) {
+ xcache_unlock_wr(c, data[i].line);
+ continue;
+ }
+
+ core = ocf_cache_get_core(cache, data[i].core_id);
+ line = data[i].line;
+ xcache_clean_update_metadata(cache, core, line);
+ // record line to flush
+ data[j++].line = data[i].line;
+ }
+ cleaning_ctx->count = j;
+
+ if (j == 0) {
+ xcache_clean_end(cleaning_ctx);
+ } else {
+ xcache_clean_metadata_flush(cleaning_ctx);
+ }
+}
+
+static void remove_failed_line(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ int l, r;
+
+ l = 0;
+ for (r = 0; r < cleaning_ctx->count; r++) {
+ if (cleaning_ctx->data[r].line == INVALID_LINE) {
+ continue;
+ }
+ if (l != r) {
+ cleaning_ctx->data[l] = cleaning_ctx->data[r];
+ }
+ l++;
+ }
+ cleaning_ctx->count = l;
+}
+
+static int xcache_clean_write_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = container_of(base_io,
+ struct xcache_cleaning_ctx, base_io);
+ struct clean_sort_data *data = cleaning_ctx->data;
+ ocf_cache_t cache = cleaning_ctx->cache;
+
+ uint64_t first_line, last_line;
+ int i;
+ ocf_core_t core;
+
+ if (cb_arg->error == 0) {
+ return 0;
+ }
+
+ // core_line in data array is sorted in xcache_cleaner_clean
+ first_line = xcache_addr_to_line(cache, cb_arg->addr);
+ last_line = xcache_addr_to_line(cache, cb_arg->addr + cb_arg->size - 1);
+ for (i = 0; i < cleaning_ctx->count; i++) {
+ if (data[i].core_line >= first_line) {
+ break;
+ }
+ }
+
+ for (; i < cleaning_ctx->count; i++) {
+ if (data[i].core_line > last_line) {
+ break;
+ }
+ // cache line alock is still hold, don't touch data[i].line
+ data[i].core_line = ULLONG_MAX;
+ core = ocf_cache_get_core(cache, data[i].core_id);
+ ocf_core_stats_core_error_update(core, OCF_WRITE);
+ }
+
+ return 0;
+}
+
+struct dirty_range {
+ uint8_t start_sector;
+ uint8_t last_sector;
+ int idx;
+ uint64_t core_line;
+};
+
+struct clear_range {
+ uint8_t start_sector;
+ uint8_t last_sector;
+ int start_idx;
+ uint64_t start_line;
+ uint64_t last_line;
+};
+
+static void get_dirty_range(ocf_cache_t cache, struct xcache_cleaning_ctx *cleaning_ctx, struct dirty_range *range)
+{
+ struct clean_sort_data *data = cleaning_ctx->data;
+
+ for (; range->idx < cleaning_ctx->count; range->idx++) {
+ get_dirty_sectors(cache, data[range->idx].line, &range->start_sector, &range->last_sector);
+ if (range->start_sector < ocf_line_sectors(cache)) {
+ range->core_line = data[range->idx].core_line;
+ break;
+ }
+
+ range->start_sector = 0;
+ }
+}
+
+static void dirty_range_next(ocf_cache_t cache, struct dirty_range *range)
+{
+ if (range->start_sector >= ocf_line_sectors(cache)) {
+ range->start_sector = 0;
+ range->idx++;
+ } else {
+ range->start_sector++;
+ }
+}
+
+static int merge_dirty_range(ocf_cache_t cache, struct clear_range *clear_range, struct dirty_range *dirty_range)
+{
+ if (clear_range->last_sector != ocf_line_sectors(cache) - 1) {
+ return -1;
+ }
+
+ if (dirty_range->start_sector != 0) {
+ return -1;
+ }
+
+ if (dirty_range->core_line != clear_range->last_line + 1) {
+ return -1;
+ }
+
+ clear_range->last_line = dirty_range->core_line;
+ clear_range->last_sector = dirty_range->last_sector;
+ return 0;
+}
+
+static void submit_dirty_range(ocf_cache_t cache, struct xcache_cleaning_ctx *cleaning_ctx, struct clear_range *clear_range)
+{
+ struct xcache_backdev_io *base_io = &cleaning_ctx->base_io;
+ struct xcache_io *io = &cleaning_ctx->io;
+ uint64_t addr, size, buf_offset;
+
+ addr = xcache_line_to_addr(cache, clear_range->start_line) + SECTORS_TO_BYTES(clear_range->start_sector);
+ size = xcache_line_to_addr(cache, clear_range->last_line - clear_range->start_line) + SECTORS_TO_BYTES((int)clear_range->last_sector - (int)clear_range->start_sector + 1);
+ buf_offset = xcache_line_to_addr(cache, clear_range->start_idx) + SECTORS_TO_BYTES(clear_range->start_sector);
+ xcache_backdev_submit_io(base_io, false, addr, size, buf_offset, OCF_WRITE);
+}
+
+static void xcache_clean_bf(struct xcache_io *io, int error)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = container_of(io,
+ struct xcache_cleaning_ctx, io);
+ struct xcache_backdev_io *base_io = &cleaning_ctx->base_io;
+ struct clean_sort_data *data = cleaning_ctx->data;
+ ocf_cache_t cache = cleaning_ctx->cache;
+
+ struct dirty_range dirty_range;
+ struct clear_range clear_range;
+
+ remove_failed_line(cleaning_ctx);
+
+ if (error) {
+ xcache_clean_bf_end(io, error);
+ }
+
+ env_atomic_set(&io->remaining, 1);
+ io->end = xcache_clean_bf_end;
+ base_io->end = xcache_clean_write_cb;
+
+ dirty_range.start_sector = 0;
+ dirty_range.idx = 0;
+ get_dirty_range(cache, cleaning_ctx, &dirty_range);
+ while (dirty_range.idx < cleaning_ctx->count) {
+ clear_range.start_sector = dirty_range.start_sector;
+ clear_range.start_line = data[dirty_range.idx].core_line;
+ clear_range.start_idx = dirty_range.idx;
+ clear_range.last_sector = dirty_range.last_sector;
+ clear_range.last_line = data[dirty_range.idx].core_line;
+
+ for (dirty_range_next(cache, &dirty_range);
+ dirty_range.idx < cleaning_ctx->count;
+ dirty_range_next(cache, &dirty_range)) {
+ get_dirty_range(cache, cleaning_ctx, &dirty_range);
+ if (merge_dirty_range(cache, &clear_range, &dirty_range) != 0) {
+ break;
+ }
+ }
+ submit_dirty_range(cache, cleaning_ctx, &clear_range);
+ }
+
+ xcache_io_put(io);
+}
+
+static int xcache_clean_read_cb(struct xcache_backdev_io *base_io, struct backdev_io_end_arg *cb_arg)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = container_of(base_io,
+ struct xcache_cleaning_ctx, base_io);
+ struct ocf_alock *c = ocf_cache_line_concurrency(cleaning_ctx->cache);
+
+ ocf_cache_line_t line;
+ ocf_core_t core;
+ int i;
+
+ if (cb_arg->error == 0) {
+ return 0;
+ }
+
+ line = addr_to_cache_line(cleaning_ctx->cache, cb_arg->addr);
+ for (i = 0; i < cleaning_ctx->count; i++) {
+ if (cleaning_ctx->data[i].line == line) {
+ xcache_unlock_wr(c, line);
+ cleaning_ctx->data[i].line = INVALID_LINE;
+ core = ocf_cache_get_core(cleaning_ctx->cache, cleaning_ctx->data[i].core_id);
+ ocf_core_stats_cache_error_update(core, OCF_READ);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int xcache_clean_if(struct xcache_io *io)
+{
+ struct xcache_cleaning_ctx *cleaning_ctx = container_of(io,
+ struct xcache_cleaning_ctx, io);
+ struct xcache_backdev_io *base_io = &cleaning_ctx->base_io;
+ ocf_cache_t cache = cleaning_ctx->cache;
+ uint64_t line_size = ocf_line_size(cache);
+ uint64_t buf_offset = 0;
+
+ ocf_cache_line_t line;
+ uint64_t addr;
+ int i;
+
+ io->end = xcache_clean_bf;
+ base_io->end = xcache_clean_read_cb;
+ for (i = 0; i < cleaning_ctx->count; i++) {
+ line = cleaning_ctx->data[i].line;
+ addr = cache_line_to_addr(cache, line, 0);
+ xcache_backdev_submit_io(base_io, true, addr, line_size, buf_offset, OCF_READ);
+ buf_offset += line_size;
+ }
+ return 0;
+}
+
+static inline int sort_data_cmp(struct clean_sort_data *l, struct clean_sort_data *r)
+{
+ if (l->core_id != r->core_id) {
+ return l->core_id - r->core_id;
+ }
+ return l->core_line - r->core_line;
+}
+
+static inline void swap_sort_data(struct clean_sort_data *l, struct clean_sort_data *r)
+{
+ struct clean_sort_data tmp;
+
+ tmp = *l;
+ *l = *r;
+ *r = tmp;
+}
+
+static void clean_quick_sort(struct clean_sort_data *data, int start, int end)
+{
+ int i, j;
+
+ if (start >= end) {
+ return;
+ }
+
+ j = start;
+ for (i = start; i < end; i++) {
+ if (sort_data_cmp(&data[i], &data[end]) < 0) {
+ if (i != j) {
+ swap_sort_data(&data[i], &data[j]);
+ }
+ j++;
+ }
+ }
+ if (j != end) {
+ swap_sort_data(&data[j], &data[end]);
+ }
+
+ clean_quick_sort(data, start, j - 1);
+ clean_quick_sort(data, j + 1, end);
+}
+
+static int xcache_clean_sort(ocf_cache_t cache, struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ struct clean_sort_data *data = cleaning_ctx->data;
+ int count = cleaning_ctx->count;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ ocf_metadata_get_core_info(cache, data[i].line, &data[i].core_id,
+ &data[i].core_line);
+ }
+ clean_quick_sort(data, 0, count - 1);
+
+ return 0;
+}
+
+/*
+ * 1. we can't use spdk_io_to_* here
+ * 2. xcache_cleaner_prepare must be called
+ */
+int xcache_cleaner_clean(ocf_cache_t cache, ocf_queue_t q,
+ struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ struct xcache_io *io = &cleaning_ctx->io;
+ struct xcache_backdev_io *base_io = &cleaning_ctx->base_io;
+
+ cleaning_ctx->cache = cache;
+
+ if (cleaning_ctx->count == 0) {
+ goto err_out;
+ }
+
+ if (xcache_clean_sort(cache, cleaning_ctx) != 0) {
+ goto err_out;
+ }
+
+ base_io->data = ctx_data_alloc(cache->owner, ocf_line_pages(cache) * cleaning_ctx->count);
+ if (base_io->data == NULL) {
+ goto err_out;
+ }
+
+ base_io->xcache_io = io;
+ base_io->end = NULL;
+
+ io->io_if = xcache_clean_if;
+ io->io_queue = q;
+ io->error = 0;
+ env_atomic_set(&io->remaining, 1);
+ xcache_queue_push_xcache_io_back(io, true);
+ return 0;
+
+err_out:
+ xcache_clean_end(cleaning_ctx);
+ return -1;
+}
+
+#define CLEAN_INTERVAL 600000 // 600 seconds
+
+static void xcache_cleaner_run_complete(void *arg)
+{
+ ocf_cleaner_t cleaner = (ocf_cleaner_t)arg;
+
+ ocf_cleaner_run_complete(cleaner, CLEAN_INTERVAL);
+}
+
+void xcache_cleaner_run(ocf_cleaner_t cleaner, ocf_queue_t queue)
+{
+ ocf_cache_t cache;
+ struct xcache_cleaning_ctx *cleaning_ctx;
+ xcache_context_t *xcache_ctx;
+ ocf_cache_line_t cline[XCACHE_CLEAN_SIZE];
+ int cline_num;
+ unsigned lock_idx;
+
+ if (cleaner == NULL || queue == NULL) {
+ return;
+ }
+
+ cache = ocf_cleaner_get_cache(cleaner);
+ cleaning_ctx = xcache_get_cleaning_ctx(cache);
+ xcache_ctx = xcache_get_ctx(cache);
+
+ if (xcache_cleaner_prepare(cleaning_ctx) != 0) {
+ return;
+ }
+
+ if (ocf_cleaner_run_prepare(cleaner, queue) != 0) {
+ xcache_cleaner_complete(cleaning_ctx);
+ return;
+ }
+
+ lock_idx = ocf_metadata_concurrency_next_idx(queue);
+ ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx);
+
+ cline_num = evicting_line_to_clean(xcache_ctx, cline, XCACHE_CLEAN_SIZE);
+
+ ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx);
+
+ if (cline_num == 0) {
+ ocf_cleaner_run_complete(cleaner, CLEAN_INTERVAL);
+ xcache_cleaner_complete(cleaning_ctx);
+ return;
+ }
+
+ xcache_cleaner_set_end(cleaning_ctx, xcache_cleaner_run_complete, (void *)cleaner);
+ xcache_cleaner_fill(cleaning_ctx, cline, cline_num);
+ xcache_cleaner_clean(cache, queue, cleaning_ctx);
+}
diff --git a/src/xcache_cleaner.h b/src/xcache_cleaner.h
new file mode 100644
index 0000000..d7c9e4d
--- /dev/null
+++ b/src/xcache_cleaner.h
@@ -0,0 +1,69 @@
+#ifndef XCACHE_CLEANER_H_
+#define XCACHE_CLEANER_H_
+
+#include "ocf/ocf_io.h"
+#include "ocf/ocf_types.h"
+#include "utils/utils_refcnt.h"
+
+#include "ocf/xcache.h"
+
+#define XCACHE_CLEAN_SIZE 32
+
+struct clean_sort_data {
+ ocf_core_id_t core_id;
+ uint64_t core_line;
+ ocf_cache_line_t line;
+};
+
+typedef void (*cleaning_end_t)(void *arg);
+struct xcache_cleaning_ctx {
+ ocf_cache_t cache;
+ struct ocf_refcnt counter;
+ struct xcache_io io;
+ struct xcache_backdev_io base_io;
+ struct clean_sort_data data[XCACHE_CLEAN_SIZE];
+ int count;
+ cleaning_end_t end;
+ void *arg;
+};
+
+int xcache_cleaner_clean(ocf_cache_t cache, ocf_queue_t q,
+ struct xcache_cleaning_ctx *cleaning_ctx);
+
+static inline int xcache_cleaner_prepare(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ if (ocf_refcnt_inc(&cleaning_ctx->counter) == 1) {
+ return 0;
+ }
+
+ ocf_refcnt_dec(&cleaning_ctx->counter);
+ return -1;
+}
+
+static inline void xcache_cleaner_complete(struct xcache_cleaning_ctx *cleaning_ctx)
+{
+ ocf_refcnt_dec(&cleaning_ctx->counter);
+}
+
+static inline int xcache_cleaner_fill(struct xcache_cleaning_ctx *cleaning_ctx,
+ ocf_cache_line_t *clines, int line_num)
+{
+ struct clean_sort_data *data = cleaning_ctx->data;
+ int num = line_num < XCACHE_CLEAN_SIZE ? line_num : XCACHE_CLEAN_SIZE;
+ int i;
+
+ for (i = 0; i < num; i++) {
+ data[i].line = clines[i];
+ }
+ cleaning_ctx->count = num;
+ return num;
+}
+
+static inline void xcache_cleaner_set_end(struct xcache_cleaning_ctx *cleaning_ctx,
+ cleaning_end_t end, void *arg)
+{
+ cleaning_ctx->end = end;
+ cleaning_ctx->arg = arg;
+}
+
+#endif /* XCACHE_CLEANER_H_ */
diff --git a/src/xcache_lru.c b/src/xcache_lru.c
new file mode 100644
index 0000000..32f9605
--- /dev/null
+++ b/src/xcache_lru.c
@@ -0,0 +1,199 @@
+/*
+ * Copyright(c) 2012-2021 Intel Corporation
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ */
+
+#include "ocf_cache_priv.h"
+#include "ocf_lru.h"
+#include "metadata/metadata.h"
+#include "utils/utils_alock.h"
+#include "concurrency/ocf_cache_line_concurrency.h"
+
+#include "xcache_cleaner.h"
+#include "engine/xcache_engine_common.h"
+#include "xcache_lru.h"
+#include "evicting/evicting_ops.h"
+
+static inline bool xcache_trylock_hash(struct xcache_io_context *ctx,
+ ocf_core_id_t core_id, uint64_t core_line)
+{
+ if (ocf_metadata_hash_func(xcache_ctx_cache(ctx), core_line, core_id) ==
+ ctx->hash)
+ return true;
+
+ return ocf_hb_cline_naked_trylock_wr(
+ &xcache_ctx_cache(ctx)->metadata.lock,
+ core_id, core_line);
+}
+
+static inline void xcache_unlock_hash(struct xcache_io_context *ctx,
+ ocf_core_id_t core_id, uint64_t core_line)
+{
+ if (ocf_metadata_hash_func(xcache_ctx_cache(ctx), core_line, core_id) !=
+ ctx->hash) {
+ ocf_hb_cline_naked_unlock_wr(
+ &xcache_ctx_cache(ctx)->metadata.lock,
+ core_id, core_line);
+
+ }
+}
+
+static inline bool xcache_eviction_lock(struct xcache_io_context *ctx,
+ ocf_cache_line_t cache_line,
+ ocf_core_id_t *core_id, uint64_t *core_line)
+
+{
+ struct ocf_alock *lock = ocf_cache_line_concurrency(xcache_ctx_cache(ctx));
+
+ ocf_metadata_get_core_info(xcache_ctx_cache(ctx), cache_line,
+ core_id, core_line);
+
+ if (*core_id == ocf_core_get_id(xcache_ctx_core(ctx)) &&
+ *core_line >= ctx->core_line_first &&
+ *core_line <= ctx->core_line_last) {
+ xcache_unlock_wr(lock, cache_line);
+ return false;
+ }
+
+ if (!xcache_trylock_hash(ctx, *core_id, *core_line)) {
+ xcache_unlock_wr(lock, cache_line);
+ return false;
+ }
+
+ return true;
+}
+
+#define EVICT_RETRY_LOCK 3
+static inline ocf_cache_line_t xcache_evict_line(struct xcache_io_context *io_ctx)
+{
+ ocf_cache_t cache = xcache_ctx_cache(io_ctx);
+ xcache_context_t *ctx = xcache_get_ctx(cache);
+ ocf_core_id_t core_id;
+ uint64_t core_line;
+ ocf_cache_line_t cline;
+ int cline_num;
+ int i;
+
+ for (i = 0; i < EVICT_RETRY_LOCK; i++) {
+ cline_num = evicting_line_to_evict(ctx, &cline, 1);
+ if (cline_num == 0) {
+ break;
+ }
+ if (xcache_eviction_lock(io_ctx,
+ cline, &core_id, &core_line)) {
+ goto found;
+ }
+ }
+ return INVALID_LINE;
+
+found:
+ ocf_lru_invalidate(cache, cline, core_id, io_ctx->part_id);
+ xcache_unlock_hash(io_ctx, core_id, core_line);
+
+ xcache_map_cache_line(io_ctx, cline);
+ return cline;
+}
+
+static inline ocf_cache_line_t xcache_free_list_line(ocf_cache_t cache, uint32_t list_idx,
+ struct ocf_part *dst_part)
+{
+ struct ocf_part *free = &cache->free;
+ struct ocf_alock *lock = ocf_cache_line_concurrency(cache);
+ struct ocf_lru_list *list;
+ ocf_cache_line_t cline;
+
+ ocf_metadata_lru_wr_lock(&cache->metadata.lock, list_idx);
+
+ list = ocf_lru_get_list(free, list_idx, true);
+ for (cline = list->tail; cline != INVALID_LINE;
+ cline = ocf_metadata_get_lru(cache, cline)->prev) {
+ if (ocf_alock_trylock_entry_wr(lock, cline)) {
+ break;
+ }
+ }
+
+ if (cline != INVALID_LINE) {
+ ocf_lru_repart_locked(cache, cline, free, dst_part);
+ }
+
+ ocf_metadata_lru_wr_unlock(&cache->metadata.lock,
+ list_idx);
+
+ return cline;
+}
+
+static ocf_cache_line_t xcache_get_free_line(struct xcache_io_context *ctx)
+{
+ struct ocf_part *dst_part;
+ uint32_t start_idx, iter_idx;
+ ocf_cache_line_t cline;
+
+ dst_part = &xcache_ctx_cache(ctx)->user_parts[ctx->part_id].part;
+
+ start_idx = xcache_ctx_queue(ctx)->lru_idx++ % OCF_NUM_LRU_LISTS;
+ for (iter_idx = start_idx; iter_idx < OCF_NUM_LRU_LISTS; iter_idx++) {
+ cline = xcache_free_list_line(xcache_ctx_cache(ctx), iter_idx, dst_part);
+ if (cline != INVALID_LINE) {
+ goto found;
+ }
+ }
+
+ for (iter_idx = 0; iter_idx < start_idx; iter_idx++) {
+ cline = xcache_free_list_line(xcache_ctx_cache(ctx), iter_idx, dst_part);
+ if (cline != INVALID_LINE) {
+ goto found;
+ }
+ }
+
+ return INVALID_LINE;
+
+found:
+ ENV_BUG_ON(metadata_test_dirty(xcache_ctx_cache(ctx), cline));
+ xcache_map_cache_line(ctx, cline);
+ return cline;
+}
+
+ocf_cache_line_t xcache_get_cline(struct xcache_io_context *ctx)
+{
+ ocf_cache_t cache = xcache_ctx_cache(ctx);
+ ocf_cache_line_t line;
+
+ if (ocf_lru_num_free(cache) > 0) {
+ line = xcache_get_free_line(ctx);
+ if (line != INVALID_LINE) {
+ return line;
+ }
+ }
+
+ return xcache_evict_line(ctx);
+}
+
+void xcache_clean(struct xcache_io_context *io_ctx, uint32_t count)
+{
+ ocf_cache_t cache = xcache_ctx_cache(io_ctx);
+ struct xcache_cleaning_ctx *cleaning_ctx = xcache_get_cleaning_ctx(cache);
+ xcache_context_t *ctx = xcache_get_ctx(cache);
+ ocf_cache_line_t cline[XCACHE_CLEAN_SIZE];
+ int cline_num;
+
+ unsigned i;
+ unsigned lock_idx;
+
+ if (ocf_mngt_cache_is_locked(cache))
+ return;
+
+ if (xcache_cleaner_prepare(cleaning_ctx) != 0) {
+ return;
+ }
+
+ lock_idx = ocf_metadata_concurrency_next_idx(xcache_ctx_queue(io_ctx));
+ ocf_metadata_start_shared_access(&cache->metadata.lock, lock_idx);
+
+ count = count < XCACHE_CLEAN_SIZE ? count : XCACHE_CLEAN_SIZE;
+ cline_num = evicting_line_to_clean(ctx, cline, count);
+ xcache_cleaner_fill(cleaning_ctx, cline, cline_num);
+
+ ocf_metadata_end_shared_access(&cache->metadata.lock, lock_idx);
+
+ xcache_cleaner_clean(cache, xcache_ctx_queue(io_ctx), cleaning_ctx);
+}
diff --git a/src/xcache_lru.h b/src/xcache_lru.h
new file mode 100644
index 0000000..87e592e
--- /dev/null
+++ b/src/xcache_lru.h
@@ -0,0 +1,9 @@
+#ifndef __XCACHE_LRU_H__
+#define __XCACHE_LRU_H__
+
+#include "ocf/xcache.h"
+
+ocf_cache_line_t xcache_get_cline(struct xcache_io_context *ctx);
+void xcache_clean(struct xcache_io_context *ctx, uint32_t count);
+
+#endif
diff --git a/src/xcache_ocf_core.c b/src/xcache_ocf_core.c
new file mode 100644
index 0000000..bfd6619
--- /dev/null
+++ b/src/xcache_ocf_core.c
@@ -0,0 +1,45 @@
+#include "ocf/ocf.h"
+#include "ocf_cache_priv.h"
+#include "metadata/metadata.h"
+#include "engine/xcache_engine.h"
+#include "engine/xcache_engine_common.h"
+
+#include "ocf/xcache.h"
+#include "xcache_queue.h"
+#include "qos/qos.h"
+
+void xcache_submit_io(struct xcache_io *io)
+{
+ ocf_core_t core = xcache_io_core(io);
+ ocf_cache_t cache = xcache_io_cache(io);
+ int ret;
+
+ OCF_CHECK_NULL(io);
+
+ if (unlikely(!env_bit_test(ocf_cache_state_running,
+ &cache->cache_state))) {
+ xcache_io_end(io, -OCF_ERR_CACHE_NOT_AVAIL);
+ return;
+ }
+
+ xcache_get_io_if(io, cache->conf_meta->cache_mode);
+ if (io->io_if == NULL) {
+ xcache_io_end(io, -OCF_ERR_INVAL);
+ return;
+ }
+
+ env_atomic_set(&io->remaining, 1);
+ xcache_queue_push_xcache_io_back(io, true);
+}
+
+void xcache_backdev_io_end(struct xcache_backdev_io *bd_io, struct backdev_io_end_arg *arg)
+{
+ struct xcache_io *io = bd_io->xcache_io;
+ xcache_context_t *xcache_ctx = backdev_io_to_xcache_ctx(bd_io);
+
+ xcache_qos_load_sub(&xcache_ctx->qos, bd_io->dev, bd_io->dir, arg->size);
+
+ io->error |= arg->error;
+ bd_io->end(bd_io, arg);
+ xcache_io_put(io);
+}
diff --git a/src/xcache_queue.c b/src/xcache_queue.c
new file mode 100644
index 0000000..01e0445
--- /dev/null
+++ b/src/xcache_queue.c
@@ -0,0 +1,337 @@
+#include "ocf/ocf.h"
+#include "ocf_queue_priv.h"
+#include "ocf_priv.h"
+#include "ocf_cache_priv.h"
+
+#include "ocf/xcache.h"
+#include "xcache_queue.h"
+#include "engine/xcache_engine.h"
+#include "engine/xcache_engine_common.h"
+
+#define ENTRY_MASK ((uintptr_t)0x7)
+
+int xcache_queue_ctx_init(ocf_queue_t queue)
+{
+ struct xcache_queue_ctx *queue_ctx = env_malloc(sizeof(struct xcache_queue_ctx), 0);
+
+ if (queue_ctx == NULL) {
+ return -ENOMEM;
+ }
+
+ INIT_LIST_HEAD(&queue_ctx->backdev_io_list);
+ queue_ctx->backdev_io_no = 0;
+
+ INIT_LIST_HEAD(&queue_ctx->flush_io_list);
+ queue_ctx->flush_io_no = 0;
+
+ INIT_LIST_HEAD(&queue_ctx->xcache_io_list);
+ queue_ctx->xcache_io_no = 0;
+
+ queue_ctx->bf_data_num = 0;
+ queue->priv1 = (void *)queue_ctx;
+ return 0;
+}
+
+void xcache_queue_ctx_deinit(ocf_queue_t queue)
+{
+ env_free(queue->priv1);
+ queue->priv1 = NULL;
+}
+
+static inline void set_entry_type(struct queue_entry *entry,
+ enum entry_type type)
+{
+ if (type == OCF_REQ_ENTRY) {
+ entry->list.next = (struct list_head *)(((uintptr_t)entry->list.next) | (uintptr_t)1);
+ return;
+ }
+
+ entry->type = type;
+}
+
+static inline enum entry_type get_entry_type(struct queue_entry *entry)
+{
+ if (((uintptr_t)entry->list.next & ENTRY_MASK) != 0) {
+ return OCF_REQ_ENTRY;
+ }
+
+ return entry->type;
+}
+
+static inline void list_add_entry(struct list_head *list, struct list_head *head)
+{
+ list_add(list, head);
+}
+
+static inline void list_add_tail_entry(struct list_head *list, struct list_head *head)
+{
+ struct list_head *prev = head->prev;
+ uintptr_t bits = (uintptr_t)prev->next & ENTRY_MASK;
+
+ list->next = head;
+ list->prev = prev;
+
+ prev->next = (struct list_head *)((uintptr_t)list | bits);
+ head->prev = list;
+}
+
+static inline void list_pop_front_entry(struct queue_entry *entry)
+{
+ struct list_head *next = (struct list_head *)((uintptr_t)entry->list.next & ~ENTRY_MASK);
+
+ entry->list.prev->next = next;
+ next->prev = entry->list.prev;
+}
+
+static inline struct queue_entry *xcache_io_to_entry(struct xcache_io *io)
+{
+ return (struct queue_entry *)io;
+}
+
+static inline struct xcache_io *entry_to_xcache_io(struct queue_entry *entry)
+{
+ return (struct xcache_io *)entry;
+}
+
+static inline struct queue_entry *backdev_io_to_entry(struct xcache_backdev_io *base_io)
+{
+ return (struct queue_entry *)base_io;
+}
+
+static inline struct xcache_backdev_io *entry_to_backdev_io(struct queue_entry *entry)
+{
+ return (struct xcache_backdev_io *)entry;
+}
+
+static inline struct queue_entry *ocf_req_to_entry(struct ocf_request *req)
+{
+ return (struct queue_entry *)container_of(&req->list, struct queue_entry, list);
+}
+
+static inline struct ocf_request *entry_to_ocf_req(struct queue_entry *entry)
+{
+ return (struct ocf_request *)container_of(&(entry->list), struct ocf_request, list);
+}
+
+static void xcache_io_run(struct queue_entry *entry)
+{
+ struct xcache_io *io = entry_to_xcache_io(entry);
+
+ io->io_if(io);
+
+ xcache_io_put(io);
+}
+
+static void backdev_io_run(struct queue_entry *entry)
+{
+ struct xcache_backdev_io *base_io = entry_to_backdev_io(entry);
+ struct xcache_io *io = base_io->xcache_io;
+
+ base_io->io_res(base_io);
+
+ xcache_io_put(io);
+}
+
+static void ocf_req_run(struct queue_entry *entry)
+{
+ struct ocf_request *req = entry_to_ocf_req(entry);
+
+ if (req->ioi.io.handle)
+ req->ioi.io.handle(&req->ioi.io, req);
+ else
+ ocf_io_handle(&req->ioi.io, req);
+}
+
+static struct queue_entry_ops {
+ void (*entry_run)(struct queue_entry *entry);
+} queue_entry_ops[] = {
+ [XCACHE_IO_ENTRY] = {
+ .entry_run = xcache_io_run,
+ },
+ [XCACHE_BACKDEV_IO_ENTRY] = {
+ .entry_run = backdev_io_run,
+ },
+ [OCF_REQ_ENTRY] = {
+ .entry_run = ocf_req_run,
+ },
+};
+
+static inline void queue_entry_run(enum entry_type type, struct queue_entry *entry)
+{
+ queue_entry_ops[type].entry_run(entry);
+}
+
+static struct queue_entry *xcache_queue_pop_entry(ocf_queue_t q)
+{
+ unsigned long lock_flags = 0;
+ struct queue_entry *entry;
+
+ OCF_CHECK_NULL(q);
+
+ env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags);
+
+ if (list_empty(&q->io_list)) {
+ /* No items on the list */
+ env_spinlock_unlock_irqrestore(&q->io_list_lock,
+ lock_flags);
+ return NULL;
+ }
+
+ /* Get the first request and remove it from the list */
+ entry = list_first_entry(&q->io_list, struct queue_entry, list);
+
+ env_atomic_dec(&q->io_no);
+ list_pop_front_entry(entry);
+
+ env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags);
+
+ OCF_CHECK_NULL(entry);
+
+ return entry;
+}
+
+// overwrite ocf_queue_run_single
+void ocf_queue_run_single(ocf_queue_t q)
+{
+ struct queue_entry *entry = NULL;
+
+ OCF_CHECK_NULL(q);
+
+ entry = xcache_queue_pop_entry(q);
+
+ if (!entry)
+ return;
+
+ queue_entry_run(get_entry_type(entry), entry);
+}
+
+// only called by request in queue to avoid lock
+struct xcache_backdev_io *xcache_queue_alloc_backdev_io(ocf_queue_t q)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+ struct xcache_backdev_io *io;
+
+ if (queue_ctx->backdev_io_no == 0) {
+ return env_malloc(sizeof(struct xcache_backdev_io), 0);
+ }
+
+ io = list_first_entry(&queue_ctx->backdev_io_list, struct xcache_backdev_io, free_list);
+ list_del(&io->free_list);
+ queue_ctx->backdev_io_no--;
+ return io;
+}
+
+void xcache_queue_free_backdev_io(ocf_queue_t q, struct xcache_backdev_io *io)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+
+ if (queue_ctx->backdev_io_no > QUEUE_CACHE_SIZE) {
+ env_free(io);
+ return;
+ }
+
+ list_add_tail(&io->free_list, &queue_ctx->backdev_io_list);
+ queue_ctx->backdev_io_no++;
+}
+
+struct xcache_io *xcache_queue_alloc_xcache_io(ocf_queue_t q)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+ struct xcache_io *io;
+
+ if (queue_ctx->xcache_io_no == 0) {
+ return env_malloc(sizeof(struct xcache_io), 0);
+ }
+
+ io = list_first_entry(&queue_ctx->xcache_io_list, struct xcache_io, queue_list);
+ list_del(&io->queue_list);
+ queue_ctx->xcache_io_no--;
+ return io;
+}
+
+void xcache_queue_free_xcache_io(ocf_queue_t q, struct xcache_io *io)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+
+ if (queue_ctx->xcache_io_no > QUEUE_CACHE_SIZE) {
+ env_free(io);
+ return;
+ }
+
+ list_add_tail(&io->queue_list, &queue_ctx->xcache_io_list);
+ queue_ctx->xcache_io_no++;
+}
+
+ctx_data_t *xcache_queue_alloc_line_data(ocf_queue_t q, ocf_cache_t cache)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+
+ if (queue_ctx->bf_data_num > 0) {
+ return queue_ctx->bf_data[--queue_ctx->bf_data_num];
+ } else {
+ return ctx_data_alloc(cache->owner, (ocf_line_size(cache) + PAGE_SIZE - 1) / PAGE_SIZE);
+ }
+}
+
+void xcache_queue_free_line_data(ocf_queue_t q, ocf_cache_t cache, ctx_data_t *data)
+{
+ struct xcache_queue_ctx *queue_ctx = xcache_get_queue_ctx(q);
+
+ if (queue_ctx->bf_data_num < QUEUE_CACHE_SIZE) {
+ queue_ctx->bf_data[queue_ctx->bf_data_num++] = data;
+ } else {
+ ctx_data_free(cache->owner, data);
+ }
+}
+
+static void xcache_queue_push_entry(ocf_queue_t q, struct queue_entry *entry, bool at_head, bool allow_sync, enum entry_type type)
+{
+ ocf_cache_t cache = ocf_queue_get_cache(q);
+ unsigned long lock_flags = 0;
+
+ INIT_LIST_HEAD(&entry->list);
+
+ env_atomic_set(&cache->last_access_ms,
+ env_ticks_to_msecs(env_get_tick_count()));
+
+ env_spinlock_lock_irqsave(&q->io_list_lock, lock_flags);
+
+ if (at_head) {
+ list_add_entry(&entry->list, &q->io_list);
+ } else {
+ list_add_tail_entry(&entry->list, &q->io_list);
+ }
+ // type set must under lock
+ set_entry_type(entry, type);
+ env_atomic_inc(&q->io_no);
+
+ env_spinlock_unlock_irqrestore(&q->io_list_lock, lock_flags);
+
+ ocf_queue_kick(q, allow_sync);
+}
+
+void xcache_queue_push_xcache_io_back(struct xcache_io *io, bool allow_sync)
+{
+ struct queue_entry *entry = xcache_io_to_entry(io);
+ xcache_queue_push_entry(xcache_io_queue(io), entry, false, allow_sync, XCACHE_IO_ENTRY);
+}
+
+void xcache_queue_push_backdev_io_front(struct xcache_backdev_io *base_io, bool allow_sync)
+{
+ struct queue_entry *entry = backdev_io_to_entry(base_io);
+ xcache_queue_push_entry(xcache_io_queue(base_io->xcache_io), entry, true, allow_sync, XCACHE_BACKDEV_IO_ENTRY);
+}
+
+// overwrite ocf_engine_push_req_front
+void ocf_engine_push_req_front(struct ocf_request *req, bool allow_sync)
+{
+ struct queue_entry *entry = ocf_req_to_entry(req);
+ xcache_queue_push_entry(req->io_queue, entry, true, allow_sync, OCF_REQ_ENTRY);
+}
+
+// overwrite ocf_engine_push_req_back
+void ocf_engine_push_req_back(struct ocf_request *req, bool allow_sync)
+{
+ struct queue_entry *entry = ocf_req_to_entry(req);
+ xcache_queue_push_entry(req->io_queue, entry, false, allow_sync, OCF_REQ_ENTRY);
+}
diff --git a/src/xcache_queue.h b/src/xcache_queue.h
new file mode 100644
index 0000000..9a9fd23
--- /dev/null
+++ b/src/xcache_queue.h
@@ -0,0 +1,35 @@
+#ifndef XCACHE_QUEUE_H_
+#define XCACHE_QUEUE_H_
+
+#include "ocf/xcache.h"
+
+#define QUEUE_CACHE_SIZE 128
+
+struct xcache_queue_ctx {
+ struct list_head backdev_io_list;
+ int backdev_io_no;
+ struct list_head flush_io_list;
+ int flush_io_no;
+ struct list_head xcache_io_list;
+ int xcache_io_no;
+ void *bf_data[QUEUE_CACHE_SIZE];
+ int bf_data_num;
+};
+
+int xcache_queue_ctx_init(ocf_queue_t queue);
+void xcache_queue_ctx_deinit(ocf_queue_t queue);
+static inline struct xcache_queue_ctx *xcache_get_queue_ctx(ocf_queue_t queue)
+{
+ return (struct xcache_queue_ctx *)(queue->priv1);
+}
+
+struct xcache_backdev_io *xcache_queue_alloc_backdev_io(ocf_queue_t q);
+void xcache_queue_free_backdev_io(ocf_queue_t q, struct xcache_backdev_io *io);
+void xcache_queue_push_xcache_io_back(struct xcache_io *io, bool allow_sync);
+void xcache_queue_push_backdev_io_front(struct xcache_backdev_io *base_io, bool allow_sync);
+struct xcache_io *xcache_queue_alloc_xcache_io(ocf_queue_t q);
+void xcache_queue_free_xcache_io(ocf_queue_t q, struct xcache_io *io);
+
+ctx_data_t *xcache_queue_alloc_line_data(ocf_queue_t q, ocf_cache_t cache);
+void xcache_queue_free_line_data(ocf_queue_t q, ocf_cache_t cache, ctx_data_t *data);
+#endif
--
2.30.0
