- Kernel - mailweb.openeuler.org

[PATCH OLK-5.10] scsi/hiraid: Support New RAID features
by LinKun 04 Jun '26

04 Jun '26

From: 岳智超 <yuezhichao1(a)h-partners.com> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9101 CVE: NA -------------------------------- Fix high traffic IO disconnection issues Fix drive letter misalignment issue Fix timeout work mode Fix boot issue in single work mode Change adm timeout Change io queue depth Change prp、sgl Change stream detect Add linkdown fast recovery Add boot disk first scan Add flr Add CPU hot-plug Add ncq feature Delete driver class Signed-off-by: 岳智超 <yuezhichao1(a)h-partners.com> --- drivers/scsi/hisi_raid/hiraid.h | 81 ++- drivers/scsi/hisi_raid/hiraid_main.c | 850 ++++++++++++++------------- 2 files changed, 486 insertions(+), 445 deletions(-) diff --git a/drivers/scsi/hisi_raid/hiraid.h b/drivers/scsi/hisi_raid/hiraid.h index 17858dc..6684108 100644 --- a/drivers/scsi/hisi_raid/hiraid.h +++ b/drivers/scsi/hisi_raid/hiraid.h @@ -50,6 +50,10 @@ #define HIRAID_DEV_INFO_FLAG_VALID(flag) ((flag) & 0x01) #define HIRAID_DEV_INFO_FLAG_CHANGE(flag) ((flag) & 0x02) +#define HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(flag) ((flag) >> 2) +#define HIRAID_DEV_INFO_FLAG_DELETE_OR_ADD(flag, org_flag) \ + (HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(flag) != \ + HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(org_flag) ? 1 : 0) #define HIRAID_CAP_MQES(cap) ((cap) & 0xffff) #define HIRAID_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) @@ -78,6 +82,16 @@ #define HIRAID_SERVER_DEVICE_RAID_DID 0x3758 #define HIRAID_SERVER_DEVICE_RAIDS_DID 0x38D8 +#define HIRAID_SCSI_VPD_LEN SCSI_VPD_PG_LEN + +#define SCSI_RSVD_INIT_SHIFT 0XFC +#define NCQ_PRIO_SUPPORT_BYTE 213 +#define VPD_NCQ_PRIO_SUPPORT_BIT 4 + +#define HIRAID_CAP_CAC_STRIDE 8 + +#define HIRAID_MAX_PD_NUM (40 + 1) +#define HIRAID_MAX_STREAM_NUM 8 enum { HIRAID_SC_SUCCESS = 0x0, @@ -93,6 +107,7 @@ enum { enum { HIRAID_REG_CAP = 0x0000, + HIRAID_REG_VS = 0x0008, HIRAID_REG_CC = 0x0014, HIRAID_REG_CSTS = 0x001c, HIRAID_REG_AQA = 0x0024, @@ -239,6 +254,20 @@ enum { DISPATCH_BY_DISK, }; +enum hiraid_stream_type { + HIRAID_STREAM_TYPE_TOTAL, + HIRAID_STREAM_TYPE_WRITE, + HIRAID_STREAM_TYPE_READ, + HIRAID_STREAM_TYPE_CLEAN, + HIRAID_STREAM_TYPE_BOTTOM +}; + +enum hiraid_stream_io_operation_type { + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO = 1, + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST +}; + struct hiraid_completion { __le32 result; union { @@ -274,6 +303,19 @@ struct hiraid_ctrl_info { __u8 rsvd1[4020]; }; +struct hiraid_stream { + /* recog-window */ + u64 stream_lba; + u32 stream_len; + u16 did; + u16 type; + /* aging ctrl */ + int aging_credit; + int aging_grade; + u16 stream_id; + u16 stream_is_using; +}; + struct hiraid_dev { struct pci_dev *pdev; struct device *dev; @@ -321,6 +363,18 @@ struct hiraid_dev { u8 hdd_dispatch; struct request_queue *bsg_queue; + + u16 hiraid_stream_io_count; + u64 hiraid_stream_aging_time; + u64 hiraid_stream_sent_io_size[HIRAID_MAX_PD_NUM] + [HIRAID_MAX_STREAM_NUM]; + u16 hiraid_stream_io_num_per_pd[HIRAID_MAX_PD_NUM] + [HIRAID_STREAM_TYPE_BOTTOM + 1]; + spinlock_t hiraid_stream_array_lock; + struct hiraid_stream hiraid_stream_array[HIRAID_MAX_PD_NUM] + [HIRAID_MAX_STREAM_NUM]; + struct task_struct *hiraid_stream_submit_task; + u64 hiraid_stream_io_last_pull_time[HIRAID_MAX_PD_NUM]; }; struct hiraid_sgl_desc { @@ -759,38 +813,19 @@ struct hiraid_sdev_hostdata { u8 flag; u8 rg_id; u8 hwq; + u8 sata_ncq_prio_enable; + u8 rsvd[3]; u16 pend_count; }; -enum stream_type { - TYPE_TOTAL, - TYPE_WRITE, - TYPE_READ, - TYPE_CLEAN, - TYPE_BOTTOM -}; - -struct HIRAID_STREAM_S { - /* recog-window */ - u64 stream_lba; - u32 stream_len; - u16 did; - u16 type; - /* aging ctrl */ - int aging_credit; - int aging_grade; - u16 stream_id; - u16 using; -}; - -struct IO_LIST_S { +struct hiraid_stream_io_list { struct list_head list; struct hiraid_scsi_io_cmd io_cmd; struct hiraid_queue *submit_queue; unsigned int sector_size; }; -struct mutex_list_head_s { +struct mutex_list_head { struct list_head list; struct mutex lock; }; diff --git a/drivers/scsi/hisi_raid/hiraid_main.c b/drivers/scsi/hisi_raid/hiraid_main.c index 2deadba..ea9b847 100644 --- a/drivers/scsi/hisi_raid/hiraid_main.c +++ b/drivers/scsi/hisi_raid/hiraid_main.c @@ -26,8 +26,13 @@ #include <linux/bsg-lib.h> #include <asm/unaligned.h> #include <linux/sort.h> +#include <linux/kthread.h> +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/sched/prio.h> #include <target/target_core_backend.h> +#include <linux/blk-mq-pci.h> #include <scsi/scsi.h> #include <scsi/scsi_cmnd.h> #include <scsi/scsi_device.h> @@ -35,14 +40,10 @@ #include <scsi/scsi_transport.h> #include <scsi/scsi_dbg.h> #include <scsi/sg.h> -#include <linux/kthread.h> -#include <linux/mutex.h> -#include <linux/sched.h> -#include <linux/sched/prio.h> #include "hiraid.h" -static u32 admin_tmout = 60; +static u32 admin_tmout = 180; module_param(admin_tmout, uint, 0644); MODULE_PARM_DESC(admin_tmout, "admin commands timeout (seconds)"); @@ -101,7 +102,7 @@ static const struct kernel_param_ops io_queue_depth_ops = { .get = param_get_uint, }; -static u32 io_queue_depth = 1024; +static u32 io_queue_depth = 4096; module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); @@ -142,8 +143,6 @@ MODULE_PARM_DESC(extra_pool_num, "set prp extra pool num, default 4, MAX 16"); static void hiraid_handle_async_notice(struct hiraid_dev *hdev, u32 result); static void hiraid_handle_async_vs(struct hiraid_dev *hdev, u32 result, u32 result1); -static struct class *hiraid_class; - #define HIRAID_CAP_TIMEOUT_UNIT_MS (HZ / 2) static struct workqueue_struct *work_queue; @@ -154,7 +153,7 @@ static struct workqueue_struct *work_queue; __func__, ##__VA_ARGS__); \ } while (0) -#define HIRAID_DRV_VERSION "1.1.0.2" +#define HIRAID_DRV_VERSION "2.1.0.1" #define ADMIN_TIMEOUT (admin_tmout * HZ) #define USRCMD_TIMEOUT (180 * HZ) @@ -177,8 +176,12 @@ static struct workqueue_struct *work_queue; #define MIN_CREDIT 0 #define MAX_CREDIT 64 #define CREDIT_THRES 32 +#ifndef MIN #define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif +#ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif enum SENSE_STATE_CODE { SENSE_STATE_OK = 0, @@ -354,91 +357,6 @@ static u32 hiraid_get_max_cmd_size(struct hiraid_dev *hdev) return sizeof(struct hiraid_mapmange) + alloc_size; } -static int hiraid_build_passthru_prp(struct hiraid_dev *hdev, struct hiraid_mapmange *mapbuf) -{ - struct scatterlist *sg = mapbuf->sgl; - __le64 *phy_regpage, *prior_list; - u64 buf_addr = sg_dma_address(sg); - int buf_length = sg_dma_len(sg); - u32 page_size = hdev->page_size; - int offset = buf_addr & (page_size - 1); - void **list = hiraid_mapbuf_list(mapbuf); - int maplen = mapbuf->len; - struct dma_pool *pool; - dma_addr_t buffer_phy; - int i; - - maplen -= (page_size - offset); - if (maplen <= 0) { - mapbuf->first_dma = 0; - return 0; - } - - buf_length -= (page_size - offset); - if (buf_length) { - buf_addr += (page_size - offset); - } else { - sg = sg_next(sg); - buf_addr = sg_dma_address(sg); - buf_length = sg_dma_len(sg); - } - - if (maplen <= page_size) { - mapbuf->first_dma = buf_addr; - return 0; - } - - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; - - phy_regpage = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!phy_regpage) { - dev_err_ratelimited(hdev->dev, "allocate first admin prp_list memory failed\n"); - mapbuf->first_dma = buf_addr; - mapbuf->page_cnt = -1; - return -ENOMEM; - } - list[0] = phy_regpage; - mapbuf->first_dma = buffer_phy; - i = 0; - for (;;) { - if (i == page_size / PRP_ENTRY_SIZE) { - prior_list = phy_regpage; - - phy_regpage = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!phy_regpage) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th admin prp list memory failed\n", - mapbuf->page_cnt + 1); - return -ENOMEM; - } - list[mapbuf->page_cnt++] = phy_regpage; - phy_regpage[0] = prior_list[i - 1]; - prior_list[i - 1] = cpu_to_le64(buffer_phy); - i = 1; - } - phy_regpage[i++] = cpu_to_le64(buf_addr); - buf_addr += page_size; - buf_length -= page_size; - maplen -= page_size; - if (maplen <= 0) - break; - if (buf_length > 0) - continue; - if (unlikely(buf_length < 0)) - goto bad_admin_sgl; - sg = sg_next(sg); - buf_addr = sg_dma_address(sg); - buf_length = sg_dma_len(sg); - } - - return 0; - -bad_admin_sgl: - dev_err(hdev->dev, "setup prps, invalid admin SGL for payload[%d] nents[%d]\n", - mapbuf->len, mapbuf->sge_cnt); - return -EIO; -} - static int hiraid_build_prp(struct hiraid_dev *hdev, struct hiraid_mapmange *mapbuf) { struct scatterlist *sg = mapbuf->sgl; @@ -641,66 +559,8 @@ static void hiraid_sgl_set_seg(struct hiraid_sgl_desc *sge, dma_addr_t buffer_ph } } -static int hiraid_build_passthru_sgl(struct hiraid_dev *hdev, - struct hiraid_admin_command *admin_cmd, - struct hiraid_mapmange *mapbuf) -{ - struct hiraid_sgl_desc *sg_list, *link, *old_sg_list; - struct scatterlist *sg = mapbuf->sgl; - void **list = hiraid_mapbuf_list(mapbuf); - struct dma_pool *pool; - int nsge = mapbuf->sge_cnt; - dma_addr_t buffer_phy; - int i = 0; - - admin_cmd->common.flags |= SQE_FLAG_SGL_METABUF; - - if (nsge == 1) { - hiraid_sgl_set_data(&admin_cmd->common.dptr.sgl, sg); - return 0; - } - - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; - - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate first admin sgl_list failed\n"); - mapbuf->page_cnt = -1; - return -ENOMEM; - } - - list[0] = sg_list; - mapbuf->first_dma = buffer_phy; - hiraid_sgl_set_seg(&admin_cmd->common.dptr.sgl, buffer_phy, nsge); - do { - if (i == SGES_PER_PAGE) { - old_sg_list = sg_list; - link = &old_sg_list[SGES_PER_PAGE - 1]; - - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th admin sgl_list failed\n", - mapbuf->page_cnt + 1); - return -ENOMEM; - } - list[mapbuf->page_cnt++] = sg_list; - - i = 0; - memcpy(&sg_list[i++], link, sizeof(*link)); - hiraid_sgl_set_seg(link, buffer_phy, nsge); - } - - hiraid_sgl_set_data(&sg_list[i++], sg); - sg = sg_next(sg); - } while (--nsge > 0); - - return 0; -} - - static int hiraid_build_sgl(struct hiraid_dev *hdev, struct hiraid_scsi_io_cmd *io_cmd, - struct hiraid_mapmange *mapbuf) + struct hiraid_mapmange *mapbuf) { struct hiraid_sgl_desc *sg_list, *link, *old_sg_list; struct scatterlist *sg = mapbuf->sgl; @@ -760,102 +620,52 @@ static int hiraid_build_sgl(struct hiraid_dev *hdev, struct hiraid_scsi_io_cmd * return 0; } -#define MAX_PD_NUM (40 + 1) -#define MAX_STREAM_NUM 8 -#define PER_MB (1024 * 1024) -#define MAX_IO_NUM (200 * PER_MB) -#define STREAM_LEN (4 * PER_MB) +#define MAX_IO_NUM (200 * 1024 * 1024) +#define STREAM_LEN (4 * 1024 * 1024) #define MAX_IO_NUM_ONCE 100 #define IO_SUBMIT_TIME_OUT 100 -#define MAX_AGING_NUM 100 - +#define MAX_AGING_NUM 5000 +#define MAX_AGING_TIME 16 +#define AGING_DEGRADE (-2) #define MIN_IO_SEND_TIME 10 #define MAX_IO_SEND_TIME 50 -enum io_operation_type { - TYPE_DELETE_SINGLE_IO = 1, - TYPE_DELETE_SINGLE_IO_LIST, - TYPE_DELETE_ALL_IO_LIST -}; - -struct HIRAID_STREAM_S stream_array[MAX_PD_NUM][MAX_STREAM_NUM] = {0}; -struct mutex_list_head_s io_heads_per_stream[MAX_PD_NUM * MAX_STREAM_NUM]; -spinlock_t stream_array_lock; +struct mutex_list_head hiraid_io_heads_per_stream[ + HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM]; DEFINE_MUTEX(g_stream_operation_mutex); -u64 g_io_transport_num[MAX_PD_NUM][MAX_STREAM_NUM] = {0}; -u16 g_io_stream_num[MAX_PD_NUM][TYPE_BOTTOM] = {0}; -u16 g_io_count = 1; - -void hiraid_inc_io_transport_num(u16 disk_id, u16 streamd_id, u16 nlb) -{ - g_io_transport_num[disk_id][streamd_id] += nlb; -} - -void hiraid_refresh_io_transport_num(u16 disk_id, u16 streamd_id) -{ - g_io_transport_num[disk_id][streamd_id] = 0; -} - -void hiraid_inc_stream_num(u16 disk_id) -{ - spin_lock(&stream_array_lock); - g_io_stream_num[disk_id][TYPE_TOTAL]++; - spin_unlock(&stream_array_lock); -} - -void hiraid_dec_stream_num(u16 disk_id) -{ - spin_lock(&stream_array_lock); - if (g_io_stream_num[disk_id][TYPE_TOTAL] > 0) - g_io_stream_num[disk_id][TYPE_TOTAL]--; - spin_unlock(&stream_array_lock); -} - -static bool hiraid_io_recog_check_stream_exceed(u16 disk_id) +static inline bool hiraid_io_recog_check_stream_exceed(struct hiraid_dev *hdev, + u16 disk_id) { bool exceed_flag; - spin_lock(&stream_array_lock); - exceed_flag = (g_io_stream_num[disk_id][TYPE_TOTAL] >= MAX_STREAM_NUM); - spin_unlock(&stream_array_lock); + spin_lock(&hdev->hiraid_stream_array_lock); + exceed_flag = + (hdev->hiraid_stream_io_num_per_pd[disk_id][HIRAID_STREAM_TYPE_TOTAL] >= + HIRAID_MAX_STREAM_NUM); + spin_unlock(&hdev->hiraid_stream_array_lock); return exceed_flag; } -static u16 hiraid_get_stream_num(u16 disk_id) -{ - return g_io_stream_num[disk_id][TYPE_TOTAL]; -} - -static inline struct HIRAID_STREAM_S *hiraid_get_stream(u16 disk_id, - u16 stream_id) -{ - return &stream_array[disk_id][stream_id]; -} - -static inline struct mutex_list_head_s *hiraid_get_io_head(u16 disk_id) -{ - return &(io_heads_per_stream[disk_id]); -} - -static bool hiraid_recognition_acknowledge(const struct HIRAID_STREAM_S *stream) +static inline bool hiraid_recognition_acknowledge( + const struct hiraid_stream *stream) { return (stream->aging_credit >= CREDIT_THRES) ? true : false; } -void hiraid_io_recognition_init(void) +void hiraid_io_recognition_init(struct hiraid_dev *hdev) { u16 i; - spin_lock_init(&stream_array_lock); - for (i = 0; i < (MAX_PD_NUM * MAX_STREAM_NUM); i++) { - INIT_LIST_HEAD(&hiraid_get_io_head(i)->list); - mutex_init(&hiraid_get_io_head(i)->lock); + spin_lock_init(&hdev->hiraid_stream_array_lock); + for (i = 0; i < (HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM); i++) { + INIT_LIST_HEAD(&hiraid_io_heads_per_stream[i].list); + mutex_init(&hiraid_io_heads_per_stream[i].lock); } } -static void hiraid_io_recognition_iterator(struct HIRAID_STREAM_S *stream, - int direction) +static void hiraid_io_recognition_iterator(struct hiraid_stream *stream, + int direction) { stream->aging_grade = stream->aging_grade + direction * INC_GRADE; stream->aging_grade = MAX(stream->aging_grade, MAX_DECREASE_GRADE); @@ -865,46 +675,52 @@ static void hiraid_io_recognition_iterator(struct HIRAID_STREAM_S *stream, stream->aging_credit = MIN(stream->aging_credit, MAX_CREDIT); } -struct HIRAID_STREAM_S *hiraid_io_pick_stream( +struct hiraid_stream *hiraid_io_pick_stream(struct hiraid_dev *hdev, struct hiraid_scsi_rw_cmd *req, u16 type, u16 actual_id) { - struct HIRAID_STREAM_S *first_hit_stream = NULL; - struct HIRAID_STREAM_S *temp_stream = NULL; - u16 pick_flag = 0; + struct hiraid_stream *first_hit_stream = NULL; + struct hiraid_stream *temp_stream = NULL; + bool pick_flag = false; u8 i; - for (i = 0; i < MAX_STREAM_NUM; i++) { - temp_stream = &stream_array[actual_id][i]; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_stream = &hdev->hiraid_stream_array[actual_id][i]; temp_stream->stream_id = i; if (req->slba < temp_stream->stream_lba || req->slba >= temp_stream->stream_lba + temp_stream->stream_len || - temp_stream->type != type) { + temp_stream->type != type || + !temp_stream->stream_is_using) { continue; } - if (!pick_flag) { + if (pick_flag == false) { temp_stream->stream_lba = req->slba; first_hit_stream = temp_stream; - pick_flag = 1; + pick_flag = true; continue; } - hiraid_dec_stream_num(actual_id); - memset(temp_stream, 0, - sizeof(struct HIRAID_STREAM_S)); // 去重影 + spin_lock(&hdev->hiraid_stream_array_lock); + if (hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL] > 0) + hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL]--; + + memset(temp_stream, 0, sizeof(struct hiraid_stream)); + spin_unlock(&hdev->hiraid_stream_array_lock); } return first_hit_stream; } -static struct HIRAID_STREAM_S *hiraid_init_flow_stream( +static struct hiraid_stream *hiraid_init_flow_stream(struct hiraid_dev *hdev, struct hiraid_scsi_rw_cmd *req, u16 type, u16 actual_id) { int i; - struct HIRAID_STREAM_S *stream = NULL; + struct hiraid_stream *stream = NULL; - for (i = 0; i < MAX_STREAM_NUM; i++) { - stream = hiraid_get_stream(actual_id, i); - if (!stream->using) { - stream->using = 1; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + stream = &hdev->hiraid_stream_array[actual_id][i]; + if (!stream->stream_is_using) { + stream->stream_is_using = 1; stream->stream_id = i; break; } @@ -918,82 +734,97 @@ static struct HIRAID_STREAM_S *hiraid_init_flow_stream( return stream; } -static struct HIRAID_STREAM_S *hiraid_stream_detect(struct hiraid_dev *hdev, - struct hiraid_scsi_rw_cmd *io_cmd, u16 actual_id) +static struct hiraid_stream *hiraid_stream_detect(struct hiraid_dev *hdev, + struct hiraid_scsi_rw_cmd *io_cmd, u16 actual_id) { - u16 type = io_cmd->opcode == HIRAID_CMD_WRITE ? TYPE_WRITE : TYPE_READ; - struct HIRAID_STREAM_S *stream = hiraid_io_pick_stream(io_cmd, type, - actual_id); + u16 type = HIRAID_STREAM_TYPE_READ; + struct hiraid_stream *stream = NULL; + + if (io_cmd->opcode == HIRAID_CMD_WRITE) + type = HIRAID_STREAM_TYPE_WRITE; - if (stream != NULL) { /* 可以命中一个stream */ + stream = hiraid_io_pick_stream(hdev, + io_cmd, type, actual_id); + + if (stream != NULL) { return stream; } - if (hiraid_io_recog_check_stream_exceed(actual_id)) + if (hiraid_io_recog_check_stream_exceed(hdev, actual_id)) return NULL; - stream = hiraid_init_flow_stream(io_cmd, type, actual_id); - hiraid_inc_stream_num(actual_id); + + stream = hiraid_init_flow_stream(hdev, io_cmd, type, actual_id); + spin_lock(&hdev->hiraid_stream_array_lock); + hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL]++; + spin_unlock(&hdev->hiraid_stream_array_lock); return stream; } -u64 g_io_last_pull_time[MAX_PD_NUM] = {0}; - static u16 hiraid_get_submit_io_stream(u16 did, struct hiraid_dev *hdev) { u64 temp_num, i; - static u16 stream_num[MAX_PD_NUM] = {0}; + static u16 stream_num[HIRAID_MAX_PD_NUM] = {0}; - if (g_io_last_pull_time[did] == 0) - g_io_last_pull_time[did] = jiffies_to_msecs(jiffies); + if (hdev->hiraid_stream_io_last_pull_time[did] == 0) + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); - for (i = 0; i < MAX_STREAM_NUM; i++) { - temp_num = g_io_transport_num[did][i]; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_num = hdev->hiraid_stream_sent_io_size[did][i]; if (temp_num != 0) { if ((temp_num < MAX_IO_NUM) && - ((jiffies_to_msecs(jiffies) - g_io_last_pull_time[did]) - < IO_SUBMIT_TIME_OUT)) { + ((jiffies_to_msecs(jiffies) - + hdev->hiraid_stream_io_last_pull_time[did]) < + IO_SUBMIT_TIME_OUT)) { stream_num[did] = i; return i; } - g_io_last_pull_time[did] = jiffies_to_msecs(jiffies); - hiraid_refresh_io_transport_num(did, i); - stream_num[did] = ((i+1) % MAX_STREAM_NUM); - return ((i+1) % MAX_STREAM_NUM); + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + hdev->hiraid_stream_sent_io_size[did][i] = 0; + stream_num[did] = ((i+1) % + HIRAID_MAX_STREAM_NUM); + return ((i+1) % HIRAID_MAX_STREAM_NUM); + } } - g_io_last_pull_time[did] = jiffies_to_msecs(jiffies); - return ((stream_num[did]++) % MAX_STREAM_NUM); + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + return ((stream_num[did]++) % HIRAID_MAX_STREAM_NUM); } static void hiraid_submit_io_stream(u16 hdid, struct hiraid_dev *hdev) { - struct mutex_list_head_s *io_slist = NULL; + struct mutex_list_head *io_slist = NULL; struct list_head *node = NULL; struct list_head *next_node = NULL; struct list_head temp_header; - struct hiraid_scsi_io_cmd io_cmd = {0}; + struct hiraid_scsi_io_cmd io_cmd; u16 submit_stream_id; - struct IO_LIST_S *temp_io_stream = NULL; + struct hiraid_stream_io_list *temp_io_stream = NULL; u16 count = 0; INIT_LIST_HEAD(&temp_header); submit_stream_id = hiraid_get_submit_io_stream(hdid, hdev); - io_slist = hiraid_get_io_head(hdid * MAX_STREAM_NUM + submit_stream_id); + io_slist = &hiraid_io_heads_per_stream[hdid * HIRAID_MAX_STREAM_NUM + + submit_stream_id]; mutex_lock(&io_slist->lock); list_for_each_safe(node, next_node, &io_slist->list) { list_del(node); list_add_tail(node, &temp_header); - if (++count >= MAX_IO_NUM_ONCE) + if (count++ >= MAX_IO_NUM_ONCE) break; } mutex_unlock(&io_slist->lock); list_for_each_safe(node, next_node, &temp_header) { - temp_io_stream = list_entry(node, struct IO_LIST_S, list); + temp_io_stream = list_entry(node, + struct hiraid_stream_io_list, list); io_cmd = temp_io_stream->io_cmd; hiraid_submit_cmd(temp_io_stream->submit_queue, &io_cmd); - hiraid_inc_io_transport_num(hdid, submit_stream_id, - io_cmd.rw.nlb * temp_io_stream->sector_size); + hdev->hiraid_stream_sent_io_size[hdid][submit_stream_id] += + io_cmd.rw.nlb * temp_io_stream->sector_size; list_del(node); kfree(temp_io_stream); } @@ -1001,59 +832,90 @@ static void hiraid_submit_io_stream(u16 hdid, struct hiraid_dev *hdev) list_del_init(&temp_header); } -static u8 hiraid_detect_if_aging(void) +static u8 hiraid_aging_detect(struct hiraid_dev *hdev) { - if (++g_io_count == MAX_AGING_NUM) { - g_io_count = 0; + if (hdev->hiraid_stream_aging_time == 0) + hdev->hiraid_stream_aging_time = jiffies_to_msecs(jiffies); + if (++hdev->hiraid_stream_io_count > MAX_AGING_NUM) { + hdev->hiraid_stream_io_count = 0; + if ((jiffies_to_msecs(jiffies) - + hdev->hiraid_stream_aging_time) < + MAX_AGING_TIME) + return 0; + hdev->hiraid_stream_aging_time = jiffies_to_msecs(jiffies); return 1; } return 0; } +static void hiraid_sync_using_stream(struct hiraid_dev *hdev, u16 hdid) +{ + u8 i; + struct hiraid_stream *temp_stream = NULL; + + hdev->hiraid_stream_io_num_per_pd[hdid][HIRAID_STREAM_TYPE_TOTAL] = 0; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_stream = &hdev->hiraid_stream_array[hdid][i]; + if (temp_stream->stream_is_using) + hdev->hiraid_stream_io_num_per_pd[hdid] + [HIRAID_STREAM_TYPE_TOTAL]++; + } +} + static void hiraid_aging(struct hiraid_dev *hdev) { - struct HIRAID_STREAM_S *temp_stream = NULL; + struct hiraid_stream *temp_stream = NULL; int i = 0; int j = 0; - - for (i = 0; i < MAX_PD_NUM; i++) { - for (j = 0; j < MAX_STREAM_NUM; j++) { - temp_stream = hiraid_get_stream(i, j); - if (temp_stream->using) { - hiraid_io_recognition_iterator(temp_stream, -1); + u16 *tmp_io_num = NULL; + + for (i = 0; i < HIRAID_MAX_PD_NUM; i++) { + for (j = 0; j < HIRAID_MAX_STREAM_NUM; j++) { + temp_stream = &hdev->hiraid_stream_array[i][j]; + if (temp_stream->stream_is_using) { + hiraid_io_recognition_iterator(temp_stream, + AGING_DEGRADE); + spin_lock(&hdev->hiraid_stream_array_lock); + tmp_io_num = &hdev->hiraid_stream_io_num_per_pd + [i][HIRAID_STREAM_TYPE_TOTAL]; if (temp_stream->aging_credit <= 0) { - hiraid_dec_stream_num(i); + if (*tmp_io_num > 0) + (*tmp_io_num)--; memset(temp_stream, - 0, sizeof(struct HIRAID_STREAM_S)); // 老化 + 0, sizeof(struct hiraid_stream)); } + spin_unlock(&hdev->hiraid_stream_array_lock); } } + hiraid_sync_using_stream(hdev, i); } } -static u8 hiraid_io_list_operation(u32 hdid, u16 cid, u16 hwq, u8 operation) +static u8 hiraid_stream_delete_io_list(struct hiraid_dev *hdev, u32 hdid, + u16 cid, u16 hwq, u8 delete_operation) { int i, j; - - struct mutex_list_head_s *io_slist = NULL; + struct mutex_list_head *io_slist = NULL; struct list_head *node = NULL; struct list_head *next_node = NULL; struct hiraid_scsi_io_cmd *io_cmd = NULL; struct hiraid_queue *hiraidq = NULL; - struct IO_LIST_S *temp_io_stream = NULL; + struct hiraid_stream_io_list *temp_io_stream = NULL; + u8 max_hd_num = delete_operation == + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST ? HIRAID_MAX_PD_NUM : hdid + 1; - u8 max_hd_num = operation == TYPE_DELETE_ALL_IO_LIST ? - MAX_PD_NUM : hdid + 1; for (i = hdid; i < max_hd_num; i++) { - for (j = 0; j < MAX_STREAM_NUM; j++) { - io_slist = hiraid_get_io_head(i * MAX_STREAM_NUM + j); + for (j = 0; j < HIRAID_MAX_STREAM_NUM; j++) { + io_slist = + &hiraid_io_heads_per_stream[i * HIRAID_MAX_STREAM_NUM + j]; mutex_lock(&io_slist->lock); list_for_each_safe(node, next_node, &io_slist->list) { - temp_io_stream = list_entry(node, - struct IO_LIST_S, list); + temp_io_stream = + list_entry(node, struct hiraid_stream_io_list, list); io_cmd = &(temp_io_stream->io_cmd); hiraidq = temp_io_stream->submit_queue; - if (operation >= TYPE_DELETE_SINGLE_IO_LIST) { + if (delete_operation >= + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST) { list_del_init(node); kfree(temp_io_stream); temp_io_stream = NULL; @@ -1073,54 +935,27 @@ static u8 hiraid_io_list_operation(u32 hdid, u16 cid, u16 hwq, u8 operation) return 0; } -static u8 hiraid_check_io_list(u32 hdid, u16 cid, u16 hwq) -{ - u8 ret; - - mutex_lock(&g_stream_operation_mutex); - ret = hiraid_io_list_operation(hdid, cid, hwq, TYPE_DELETE_SINGLE_IO); - mutex_unlock(&g_stream_operation_mutex); - return ret; -} - -static u8 hiraid_delete_single_pd_io_list(u32 hdid) +static u8 hiraid_add_io_to_list(struct hiraid_dev *hdev, + struct hiraid_queue *submit_queue, struct hiraid_stream *tmp_stream, + struct hiraid_scsi_io_cmd io_cmd, unsigned int sector_size, + u16 actual_id) { - u8 ret; + struct mutex_list_head *io_slist = NULL; + struct hiraid_stream_io_list *new_io_node = NULL; + u16 temp_io_id = actual_id * HIRAID_MAX_STREAM_NUM + + tmp_stream->stream_id; - mutex_lock(&g_stream_operation_mutex); - ret = hiraid_io_list_operation(hdid, 0, 0, TYPE_DELETE_SINGLE_IO_LIST); - mutex_unlock(&g_stream_operation_mutex); - return ret; -} - -static u8 hiraid_delete_all_io_list(void) -{ - u8 ret; - - mutex_lock(&g_stream_operation_mutex); - ret = hiraid_io_list_operation(0, 0, 0, TYPE_DELETE_ALL_IO_LIST); - mutex_unlock(&g_stream_operation_mutex); - return ret; -} - -static u8 hiraid_add_io_to_list(struct hiraid_queue *submit_queue, - struct HIRAID_STREAM_S *tmp_stream, struct hiraid_scsi_io_cmd io_cmd, - unsigned int sector_size, u16 actual_id) -{ - struct mutex_list_head_s *io_slist = NULL; - struct IO_LIST_S *new_io_node = NULL; - - new_io_node = kmalloc(sizeof(struct IO_LIST_S), GFP_KERNEL); + new_io_node = kmalloc(sizeof(struct hiraid_stream_io_list), GFP_KERNEL); if (!new_io_node) return 0; new_io_node->io_cmd = io_cmd; new_io_node->submit_queue = submit_queue; new_io_node->sector_size = sector_size; - io_slist = hiraid_get_io_head(actual_id * - MAX_STREAM_NUM + tmp_stream->stream_id); + io_slist = &hiraid_io_heads_per_stream[temp_io_id]; mutex_lock(&io_slist->lock); INIT_LIST_HEAD(&(new_io_node->list)); - list_add_tail(&(new_io_node->list), &io_slist->list); + list_add_tail(&(new_io_node->list), + &hiraid_io_heads_per_stream[temp_io_id].list); mutex_unlock(&io_slist->lock); return 1; } @@ -1130,10 +965,8 @@ static void hiraid_submit_io_threading(struct hiraid_dev *hdev) int i = 0; while (!kthread_should_stop()) { - mutex_lock(&g_stream_operation_mutex); - for (i = 0; i < MAX_PD_NUM; i++) + for (i = 0; i < HIRAID_MAX_PD_NUM; i++) hiraid_submit_io_stream(i, hdev); - mutex_unlock(&g_stream_operation_mutex); usleep_range(MIN_IO_SEND_TIME, MAX_IO_SEND_TIME); } } @@ -1142,16 +975,18 @@ static void hiraid_destroy_io_stream_resource(struct hiraid_dev *hdev) { u16 i; - for (i = 0; i < (MAX_PD_NUM * MAX_STREAM_NUM); i++) - list_del_init(&hiraid_get_io_head(i)->list); + for (i = 0; i < (HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM); i++) { + list_del_init(&hiraid_io_heads_per_stream[i].list); + mutex_destroy(&hiraid_io_heads_per_stream[i].lock); + } } -struct task_struct *g_hiraid_submit_task; static void hiraid_init_io_stream(struct hiraid_dev *hdev) { - hiraid_io_recognition_init(); - g_hiraid_submit_task = kthread_run((void *)hiraid_submit_io_threading, - hdev, "hiraid_submit_thread"); + hiraid_io_recognition_init(hdev); + hdev->hiraid_stream_submit_task = + kthread_run((void *)hiraid_submit_io_threading, + hdev, "hiraid_stream_submit_task"); } #define HIRAID_RW_FUA BIT(14) @@ -1365,7 +1200,7 @@ static int hiraid_io_map_data(struct hiraid_dev *hdev, struct hiraid_mapmange *m ret = scsi_dma_map(scmd); if (unlikely(ret < 0)) - return ret; + return SCSI_MLQUEUE_HOST_BUSY; mapbuf->sge_cnt = ret; /* No data to DMA, it may be scsi no-rw command */ @@ -1446,6 +1281,35 @@ static inline void hiraid_query_scmd_tag(struct scsi_cmnd *scmd, u16 *qid, u16 * *cid = blk_mq_unique_tag_to_tag(tag); } +static void hiraid_set_ncq_prio(struct scsi_cmnd *scmd, + struct hiraid_scsi_io_cmd *io_cmd, + struct hiraid_sdev_hostdata *hostdata, + struct hiraid_dev *hdev) +{ + struct request *rq = scmd->request; + int class = 0; + + if (hostdata->sata_ncq_prio_enable) { + class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); + switch (class) { + case IOPRIO_CLASS_NONE: + break; + case IOPRIO_CLASS_RT: + io_cmd->rw.rsvd2 &= SCSI_RSVD_INIT_SHIFT; + io_cmd->rw.rsvd2 |= (1 << IOPRIO_CLASS_RT); + break; + case IOPRIO_CLASS_BE: + break; + case IOPRIO_CLASS_IDLE: + break; + default: + break; + } + } + dev_log_dbg(hdev->dev, "blk_ncq_prio = %d, scmd_ncq_prio = 0x%02x\n", + class, io_cmd->rw.rsvd2); +} + static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) { struct hiraid_mapmange *mapbuf = scsi_cmd_priv(scmd); @@ -1454,7 +1318,7 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) struct hiraid_sdev_hostdata *hostdata; struct hiraid_scsi_io_cmd io_cmd; struct hiraid_queue *ioq; - struct HIRAID_STREAM_S *tmp_stm = NULL; + struct hiraid_stream *tmp_stream; u16 hwq, cid; int ret; @@ -1484,6 +1348,8 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) io_cmd.rw.hdid = cpu_to_le32(hostdata->hdid); io_cmd.rw.cmd_id = cpu_to_le16(cid); + hiraid_set_ncq_prio(scmd, &io_cmd, hostdata, hdev); + ret = hiraid_setup_io_cmd(hdev, &io_cmd, scmd, mapbuf); if (unlikely(ret)) { if (ret == RW_LENGTH_ZERO) { @@ -1514,6 +1380,10 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) mapbuf->cid = cid; ret = hiraid_io_map_data(hdev, mapbuf, scmd, &io_cmd); if (unlikely(ret)) { + if (ret == SCSI_MLQUEUE_HOST_BUSY) { + atomic_dec(&ioq->inflight); + return ret; + } dev_err(hdev->dev, "io map data err\n"); set_host_byte(scmd, DID_ERROR); scmd->scsi_done(scmd); @@ -1522,20 +1392,20 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) } WRITE_ONCE(mapbuf->state, CMD_FLIGHT); - if (hiraid_is_rw_scmd(scmd) && hiraid_disk_is_hdd_rawdrive(hostdata->attr)) { - if (hiraid_detect_if_aging()) + if (hiraid_aging_detect(hdev)) hiraid_aging(hdev); - tmp_stm = hiraid_stream_detect(hdev, &(io_cmd.rw), sdev->id); - if (tmp_stm != NULL) { - hiraid_io_recognition_iterator(tmp_stm, 1); - if (hiraid_recognition_acknowledge(tmp_stm) && - (hiraid_get_stream_num(sdev->id) > 1)) { - if (hiraid_add_io_to_list(ioq, - tmp_stm, io_cmd, sdev->sector_size, sdev->id)) { + tmp_stream = hiraid_stream_detect(hdev, &(io_cmd.rw), sdev->id); + if (tmp_stream != NULL) { + hiraid_io_recognition_iterator(tmp_stream, 1); + if (hiraid_recognition_acknowledge(tmp_stream) && + (sdev->id < HIRAID_MAX_PD_NUM) && + (hdev->hiraid_stream_io_num_per_pd[sdev->id][HIRAID_STREAM_TYPE_TOTAL] > + 1)) { + if (hiraid_add_io_to_list(hdev, ioq, tmp_stream, + io_cmd, sdev->sector_size, sdev->id)) return 0; - } } } } @@ -1671,6 +1541,7 @@ static int hiraid_slave_configure(struct scsi_device *sdev) static void hiraid_shost_init(struct hiraid_dev *hdev) { struct pci_dev *pdev = hdev->pdev; + struct scsi_host_template *hostt; u8 domain, bus; u32 dev_func; @@ -1679,7 +1550,10 @@ static void hiraid_shost_init(struct hiraid_dev *hdev) dev_func = pdev->devfn; hdev->shost->nr_hw_queues = work_mode ? 1 : hdev->online_queues - 1; - hdev->shost->can_queue = hdev->scsi_qd; + hdev->shost->can_queue = min(hdev->scsi_qd, + le16_to_cpu(hdev->ctrl_info->max_cmds) - + HIRAID_AQ_DEPTH - + HIRAID_TOTAL_PTCMDS(hdev->online_queues - 1)); hdev->shost->sg_tablesize = le16_to_cpu(hdev->ctrl_info->max_num_sge); /* 512B per sector */ @@ -1692,7 +1566,13 @@ static void hiraid_shost_init(struct hiraid_dev *hdev) hdev->shost->this_id = -1; hdev->shost->unique_id = (domain << 16) | (bus << 8) | dev_func; hdev->shost->max_cmd_len = MAX_CDB_LEN; - hdev->shost->hostt->cmd_size = hiraid_get_max_cmd_size(hdev); + + hostt = (struct scsi_host_template *)hdev->shost->hostt; + hostt->cmd_size = hiraid_get_max_cmd_size(hdev); + + dev_info(hdev->dev, "nr_hw_queues[%u] can_queue[%u] unique_id[%u] cmd_size[%u]\n", + hdev->shost->nr_hw_queues, hdev->shost->can_queue, + hdev->shost->unique_id, hdev->shost->hostt->cmd_size); } static int hiraid_alloc_queue(struct hiraid_dev *hdev, u16 qid, u16 depth) @@ -2481,6 +2361,14 @@ static int hiraid_get_queue_cnt(struct hiraid_dev *hdev, u32 *cnt) return 0; } +static bool hiraid_pci_is_present(struct hiraid_dev *hdev) +{ + if (!pci_device_is_present(hdev->pdev) || + readl(hdev->bar + HIRAID_REG_VS) == 0Xffffffff) + return false; + return true; +} + static int hiraid_setup_io_queues(struct hiraid_dev *hdev) { struct hiraid_queue *adminq = &hdev->queues[0]; @@ -2552,7 +2440,7 @@ static void hiraid_delete_io_queues(struct hiraid_dev *hdev) u8 opcode = HIRAID_ADMIN_DELETE_SQ; u16 i, pass; - if (!pci_device_is_present(hdev->pdev)) { + if (!hiraid_pci_is_present(hdev)) { dev_err(hdev->dev, "pci_device is not present, skip disable io queues\n"); return; } @@ -2591,7 +2479,7 @@ static void hiraid_disable_admin_queue(struct hiraid_dev *hdev, bool shutdown) struct hiraid_queue *adminq = &hdev->queues[0]; u16 start, end; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { if (shutdown) hiraid_shutdown_control(hdev); else @@ -2809,11 +2697,16 @@ static int hiraid_luntarget_sort(const void *l, const void *r) const struct hiraid_dev_info *rn = r; int l_attr = HIRAID_DEV_INFO_ATTR_BOOT(ln->attr); int r_attr = HIRAID_DEV_INFO_ATTR_BOOT(rn->attr); - /* boot first */ if (l_attr != r_attr) return (r_attr - l_attr); + l_attr = HIRAID_DEV_INFO_ATTR_VD(ln->attr); + r_attr = HIRAID_DEV_INFO_ATTR_VD(rn->attr); + /* vd second */ + if (l_attr != r_attr) + return (r_attr - l_attr); + if (ln->channel == rn->channel) return le16_to_cpu(ln->target) - le16_to_cpu(rn->target); @@ -2850,6 +2743,19 @@ static void hiraid_scan_work(struct work_struct *work) if (HIRAID_DEV_INFO_FLAG_VALID(flag)) { if (!HIRAID_DEV_INFO_FLAG_VALID(org_flag)) { + down_write(&hdev->dev_rwsem); + memcpy(&old_dev[i], &dev[i], + sizeof(struct hiraid_dev_info)); + memcpy(&new_dev[count++], &dev[i], + sizeof(struct hiraid_dev_info)); + up_write(&hdev->dev_rwsem); + } else if (HIRAID_DEV_INFO_FLAG_DELETE_OR_ADD(flag, + org_flag)) { + down_write(&hdev->dev_rwsem); + old_dev[i].flag &= 0xfe; + up_write(&hdev->dev_rwsem); + hiraid_delete_device(hdev, &old_dev[i]); + down_write(&hdev->dev_rwsem); memcpy(&old_dev[i], &dev[i], sizeof(struct hiraid_dev_info)); @@ -2997,7 +2903,7 @@ static void hiraid_bsg_buf_unmap(struct hiraid_dev *hdev, struct bsg_job *job) } static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, - struct hiraid_admin_command *cmd) + struct hiraid_admin_command *cmd, u16 qid) { struct hiraid_bsg_request *bsg_req = job->request; struct request *rq = blk_mq_rq_from_pdu(job); @@ -3010,6 +2916,7 @@ static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, mapbuf->sgl = job->request_payload.sg_list; mapbuf->len = job->request_payload.payload_len; mapbuf->page_cnt = -1; + mapbuf->hiraidq = &hdev->queues[qid]; if (unlikely(mapbuf->sge_cnt == 0)) goto out; @@ -3019,13 +2926,15 @@ static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, mapbuf->use_sgl = !hiraid_is_prp(hdev, mapbuf->sgl, mapbuf->sge_cnt); - if ((mapbuf->use_sgl == (bool)true) && (bsg_req->msgcode == HIRAID_BSG_IOPTHRU) && - (hdev->ctrl_info->pt_use_sgl != (bool)false)) { - ret = hiraid_build_passthru_sgl(hdev, cmd, mapbuf); + if ((mapbuf->use_sgl == true) && + (bsg_req->msgcode == HIRAID_BSG_IOPTHRU) && + (hdev->ctrl_info->pt_use_sgl != false)) { + ret = hiraid_build_sgl(hdev, (struct hiraid_scsi_io_cmd *)cmd, + mapbuf); } else { mapbuf->use_sgl = false; - ret = hiraid_build_passthru_prp(hdev, mapbuf); + ret = hiraid_build_prp(hdev, mapbuf); cmd->common.dptr.prp1 = cpu_to_le64(sg_dma_address(mapbuf->sgl)); cmd->common.dptr.prp2 = cpu_to_le64(mapbuf->first_dma); } @@ -3103,8 +3012,7 @@ static int hiraid_init_control_info(struct hiraid_dev *hdev) if (hdev->ctrl_info->asynevent > HIRAID_ASYN_COMMANDS) hdev->ctrl_info->asynevent = HIRAID_ASYN_COMMANDS; - hdev->scsi_qd = work_mode ? - le16_to_cpu(hdev->ctrl_info->max_cmds) : (hdev->ioq_depth - HIRAID_PTHRU_CMDS_PERQ); + hdev->scsi_qd = hdev->ioq_depth - HIRAID_PTHRU_CMDS_PERQ; return 0; } @@ -3137,7 +3045,7 @@ static int hiraid_user_send_admcmd(struct hiraid_dev *hdev, struct bsg_job *job) admin_cmd.common.cdw14 = cpu_to_le32(ptcmd->cdw14); admin_cmd.common.cdw15 = cpu_to_le32(ptcmd->cdw15); - status = hiraid_bsg_buf_map(hdev, job, &admin_cmd); + status = hiraid_bsg_buf_map(hdev, job, &admin_cmd, 0); if (status) { dev_err(hdev->dev, "err, map data failed\n"); return status; @@ -3149,7 +3057,7 @@ static int hiraid_user_send_admcmd(struct hiraid_dev *hdev, struct bsg_job *job) memcpy(job->reply, result, sizeof(result)); } if (status) - dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] result0[0x%x];" + dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] result0[0x%x] " "result1[0x%x]\n", ptcmd->opcode, ptcmd->info_0.subopcode, status, result[0], result[1]); @@ -3194,18 +3102,12 @@ static void hiraid_free_io_ptcmds(struct hiraid_dev *hdev) } static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, struct hiraid_scsi_io_cmd *io_cmd, - u32 *result, u32 *reslen, u32 timeout) + struct hiraid_cmd *pt_cmd, u32 *result, u32 *reslen, u32 timeout) { int ret; dma_addr_t buffer_phy; struct hiraid_queue *ioq; void *sense_addr = NULL; - struct hiraid_cmd *pt_cmd = hiraid_get_cmd(hdev, HIRAID_CMD_PTHRU); - - if (!pt_cmd) { - dev_err(hdev->dev, "err, get ioq cmd failed\n"); - return -EFAULT; - } timeout = timeout ? timeout : ADMIN_TIMEOUT; @@ -3235,8 +3137,6 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, struct hiraid_scs (le32_to_cpu(io_cmd->common.cdw3[0]) & 0xffff)); hiraid_admin_timeout(hdev, pt_cmd); - - hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); return -ETIME; } @@ -3247,8 +3147,6 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, struct hiraid_scs } } - hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); - return pt_cmd->status; } @@ -3257,6 +3155,7 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) struct hiraid_bsg_request *bsg_req = (struct hiraid_bsg_request *)(job->request); struct hiraid_passthru_io_cmd *cmd = &(bsg_req->pthrucmd); struct hiraid_scsi_io_cmd pthru_cmd; + struct hiraid_cmd *pt_cmd; int status = 0; u32 timeout = msecs_to_jiffies(cmd->timeout_ms); // data len is 4k before use sgl, now len is 1M @@ -3298,20 +3197,30 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) pthru_cmd.common.cdw26[2] = cpu_to_le32(cmd->cdw26[2]); pthru_cmd.common.cdw26[3] = cpu_to_le32(cmd->cdw26[3]); - status = hiraid_bsg_buf_map(hdev, job, (struct hiraid_admin_command *)&pthru_cmd); + pt_cmd = hiraid_get_cmd(hdev, HIRAID_CMD_PTHRU); + if (pt_cmd == NULL) { + dev_err(hdev->dev, "err, get ioq cmd failed\n"); + return -EFAULT; + } + + status = hiraid_bsg_buf_map(hdev, job, + (struct hiraid_admin_command *)&pthru_cmd, pt_cmd->qid); if (status) { dev_err(hdev->dev, "err, map data failed\n"); - return status; + goto ret; } - status = hiraid_put_io_sync_request(hdev, &pthru_cmd, job->reply, &job->reply_len, timeout); - + status = hiraid_put_io_sync_request(hdev, &pthru_cmd, pt_cmd, + job->reply, &job->reply_len, timeout); if (status) dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] replylen[%d]\n", cmd->opcode, cmd->info_1.subopcode, status, job->reply_len); hiraid_bsg_buf_unmap(hdev, job); +ret: + hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); + return status; } @@ -3332,20 +3241,26 @@ static bool hiraid_check_scmd_finished(struct scsi_cmnd *scmd) return false; } -static enum blk_eh_timer_return hiraid_timed_out(struct scsi_cmnd *scmd) +#define EH_TIMED_RET_TP enum blk_eh_timer_return +#define EH_TIMED_RET_DONE BLK_EH_DONE +#define EH_TIMED_RET_NOT_HANDLED BLK_EH_DONE +#define EH_TIMED_RET_RESET_TIMER BLK_EH_RESET_TIMER +static EH_TIMED_RET_TP hiraid_timed_out(struct scsi_cmnd *scmd) { struct hiraid_mapmange *mapbuf = scsi_cmd_priv(scmd); unsigned int timeout = scmd->device->request_queue->rq_timeout; if (hiraid_check_scmd_finished(scmd)) - goto out; + return EH_TIMED_RET_DONE; - if (time_after(jiffies, scmd->jiffies_at_alloc + timeout)) { + if (time_after_eq(jiffies, scmd->jiffies_at_alloc + timeout)) { + scmd_printk(KERN_WARNING, scmd, "timeout is work[%d], need abort\n", + mapbuf->state); if (cmpxchg(&mapbuf->state, CMD_FLIGHT, CMD_TIMEOUT) == CMD_FLIGHT) - return BLK_EH_DONE; + return EH_TIMED_RET_NOT_HANDLED; } -out: - return BLK_EH_RESET_TIMER; + + return EH_TIMED_RET_RESET_TIMER; } /* send abort command by admin queue temporary */ @@ -3468,7 +3383,7 @@ static int hiraid_dev_disable(struct hiraid_dev *hdev, bool shutdown) struct hiraid_queue *adminq = &hdev->queues[0]; u16 start, end; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { if (shutdown) hiraid_shutdown_control(hdev); else @@ -3663,11 +3578,19 @@ static int hiraid_abort(struct scsi_cmnd *scmd) hiraid_check_scmd_finished(scmd)) return SUCCESS; + if (hdev->state != DEV_LIVE) + return SUCCESS; + hostdata = scmd->device->hostdata; cid = mapbuf->cid; hwq = mapbuf->hiraidq->qid; - if (hiraid_check_io_list(hostdata->hdid, cid, hwq)) { + mutex_lock(&g_stream_operation_mutex); + ret = hiraid_stream_delete_io_list(hdev, hostdata->hdid, + cid, hwq, HIRAID_STREAM_TYPE_DELETE_SINGLE_IO); + mutex_unlock(&g_stream_operation_mutex); + + if (ret == 1) { dev_warn(hdev->dev, "find cid[%d] qid[%d] in host, abort succ\n", cid, hwq); return SUCCESS; @@ -3704,7 +3627,10 @@ static int hiraid_scsi_reset(struct scsi_cmnd *scmd, enum hiraid_rst_type rst) ret = hiraid_send_reset_cmd(hdev, rst, hostdata->hdid); if ((ret == 0) || (ret == FW_EH_DEV_NONE && rst == HIRAID_RESET_TARGET)) { if (rst == HIRAID_RESET_TARGET) { - hiraid_delete_single_pd_io_list(hostdata->hdid); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, hostdata->hdid, + 0, 0, HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); ret = wait_tgt_reset_io_done(scmd); if (ret) { dev_warn(hdev->dev, "sdev[%d:%d] target has %d peding cmd, target reset failed\n", @@ -3742,9 +3668,13 @@ static int hiraid_host_reset(struct scsi_cmnd *scmd) dev_warn(hdev->dev, "sdev[%d:%d] send host reset\n", scmd->device->channel, scmd->device->id); - hiraid_delete_all_io_list(); - if (hiraid_reset_work_sync(hdev) == -EBUSY) + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + if (hiraid_reset_work_sync(hdev) == -EBUSY) { flush_work(&hdev->reset_work); + } if (hdev->state != DEV_LIVE) { dev_warn(hdev->dev, "sdev[%d:%d] host reset failed\n", @@ -3776,7 +3706,10 @@ static pci_ers_result_t hiraid_pci_error_detected(struct pci_dev *pdev, scsi_block_requests(hdev->shost); hiraid_dev_state_trans(hdev, DEV_RESETTING); - hiraid_delete_all_io_list(); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); return PCI_ERS_RESULT_NEED_RESET; case pci_channel_io_perm_failure: @@ -3812,16 +3745,31 @@ static void hiraid_reset_pci_finish(struct pci_dev *pdev) { struct hiraid_dev *hdev = pci_get_drvdata(pdev); + if (hiraid_reset_work_sync(hdev) == -EBUSY) + flush_work(&hdev->reset_work); + dev_info(hdev->dev, "enter hiraid reset finish\n"); } +static void hiraid_reset_pci_prepare(struct pci_dev *pdev) +{ + struct hiraid_dev *hdev = pci_get_drvdata(pdev); + + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + msleep(100); + dev_info(hdev->dev, "exit hiraid reset prepare\n"); +} + static ssize_t csts_pp_show(struct device *cdev, struct device_attribute *attr, char *buf) { struct Scsi_Host *shost = class_to_shost(cdev); struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_PP_MASK); ret >>= HIRAID_CSTS_PP_SHIFT; } @@ -3835,7 +3783,7 @@ static ssize_t csts_shst_show(struct device *cdev, struct device_attribute *attr struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_SHST_MASK); ret >>= HIRAID_CSTS_SHST_SHIFT; } @@ -3849,7 +3797,7 @@ static ssize_t csts_cfs_show(struct device *cdev, struct device_attribute *attr, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_CFS_MASK); ret >>= HIRAID_CSTS_CFS_SHIFT; } @@ -3863,7 +3811,7 @@ static ssize_t csts_rdy_show(struct device *cdev, struct device_attribute *attr, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) + if (hiraid_pci_is_present(hdev)) ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_RDY); return snprintf(buf, PAGE_SIZE, "%d\n", ret); @@ -4108,22 +4056,80 @@ static ssize_t dispatch_hwq_store(struct device *dev, struct device_attribute *a return strlen(buf); } +static ssize_t sata_ncq_prio_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hiraid_sdev_hostdata *hostdata; + + hostdata = to_scsi_device(dev)->hostdata; + return snprintf(buf, PAGE_SIZE, "%d\n", hostdata->sata_ncq_prio_enable); +} + +bool scsi_ncq_prio_support(struct scsi_device *sdev, struct hiraid_dev *hdev) +{ + unsigned char *buf; + bool ncq_prio_supp = false; + + if (!scsi_device_supports_vpd(sdev)) { + dev_err(hdev->dev, "scsi device not support vpd\n"); + return ncq_prio_supp; + } + + buf = kzalloc(HIRAID_SCSI_VPD_LEN, GFP_KERNEL); + if (!buf) { + dev_err(hdev->dev, "alloc scsi vpd data memeroy failed\n"); + return ncq_prio_supp; + } + + if (!scsi_get_vpd_page(sdev, 0x89, buf, HIRAID_SCSI_VPD_LEN)) + ncq_prio_supp = (buf[NCQ_PRIO_SUPPORT_BYTE] >> + VPD_NCQ_PRIO_SUPPORT_BIT) & 1; + + kfree(buf); + return ncq_prio_supp; +} + +static ssize_t sata_ncq_prio_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct scsi_device *sdev = to_scsi_device(dev); + struct hiraid_sdev_hostdata *hostdata = sdev->hostdata; + struct hiraid_dev *hdev = shost_priv(sdev->host); + bool ncq_prio_enable = 0; + + if (kstrtobool(buf, &ncq_prio_enable)) + return -EINVAL; + + if (!scsi_ncq_prio_support(sdev, hdev)) { + dev_err(hdev->dev, "sdev[%d:%d] not support ncq prio\n", + sdev->channel, sdev->id); + return -EINVAL; + } + + hostdata->sata_ncq_prio_enable = ncq_prio_enable; + + return strlen(buf); +} + static DEVICE_ATTR_RO(raid_level); static DEVICE_ATTR_RO(raid_state); static DEVICE_ATTR_RO(raid_resync); static DEVICE_ATTR_RW(dispatch_hwq); +static DEVICE_ATTR_RW(sata_ncq_prio_enable); static struct device_attribute *hiraid_dev_attrs[] = { &dev_attr_raid_state, &dev_attr_raid_level, &dev_attr_raid_resync, &dev_attr_dispatch_hwq, + &dev_attr_sata_ncq_prio_enable, NULL, }; static struct pci_error_handlers hiraid_err_handler = { .error_detected = hiraid_pci_error_detected, .slot_reset = hiraid_pci_slot_reset, + .reset_prepare = hiraid_reset_pci_prepare, .reset_done = hiraid_reset_pci_finish, }; @@ -4148,6 +4154,16 @@ static int hiraid_scan_finished(struct Scsi_Host *shost, unsigned long time) return 1; } +static int hiraid_map_queues(struct Scsi_Host *shost) +{ + struct hiraid_dev *hdev = shost_priv(shost); + + if (shost->nr_hw_queues == 1) + return 0; + + return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT], + hdev->pdev, 1); +} static struct scsi_host_template hiraid_driver_template = { .module = THIS_MODULE, .name = "hiraid", @@ -4164,6 +4180,7 @@ static struct scsi_host_template hiraid_driver_template = { .eh_host_reset_handler = hiraid_host_reset, .change_queue_depth = scsi_change_queue_depth, .this_id = -1, + .map_queues = hiraid_map_queues, .unchecked_isa_dma = 0, .shost_attrs = hiraid_host_attrs, .sdev_attrs = hiraid_dev_attrs, @@ -4382,14 +4399,17 @@ static void hiraid_remove(struct pci_dev *pdev) dev_info(hdev->dev, "enter hiraid remove\n"); - kthread_stop(g_hiraid_submit_task); - hiraid_delete_all_io_list(); + kthread_stop(hdev->hiraid_stream_submit_task); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); hiraid_destroy_io_stream_resource(hdev); hiraid_dev_state_trans(hdev, DEV_DELETING); flush_work(&hdev->reset_work); - if (!pci_device_is_present(pdev)) + if (!hiraid_pci_is_present(hdev)) hiraid_flush_running_cmds(hdev); hiraid_unregist_bsg(hdev); @@ -4435,22 +4455,9 @@ static int __init hiraid_init(void) if (!work_queue) return -ENOMEM; - hiraid_class = class_create(THIS_MODULE, "hiraid"); - if (IS_ERR(hiraid_class)) { - ret = PTR_ERR(hiraid_class); - goto destroy_wq; - } - ret = pci_register_driver(&hiraid_driver); if (ret < 0) - goto destroy_class; - - return 0; - -destroy_class: - class_destroy(hiraid_class); -destroy_wq: - destroy_workqueue(work_queue); + destroy_workqueue(work_queue); return ret; } @@ -4458,7 +4465,6 @@ destroy_wq: static void __exit hiraid_exit(void) { pci_unregister_driver(&hiraid_driver); - class_destroy(hiraid_class); destroy_workqueue(work_queue); } -- 2.45.1.windows.1

2 3

[PATCH OLK-6.6] scsi/hiraid: Support New RAID features
by LinKun 04 Jun '26

04 Jun '26

From: 岳智超 <yuezhichao1(a)h-partners.com> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9101 CVE: NA -------------------------------- Fix high traffic IO disconnection issues Fix drive letter misalignment issue Fix timeout work mode Fix boot issue in single work mode Change adm timeout Change io queue depth Change prp、sgl Add linkdown fast recovery Add boot disk first scan Add stream detect Add flr Add CPU hot-plug Add ncq feature Delete driver class Signed-off-by: 岳智超 <yuezhichao1(a)h-partners.com> --- drivers/scsi/hisi_raid/hiraid.h | 68 ++ drivers/scsi/hisi_raid/hiraid_main.c | 888 ++++++++++++++++++++------- 2 files changed, 739 insertions(+), 217 deletions(-) diff --git a/drivers/scsi/hisi_raid/hiraid.h b/drivers/scsi/hisi_raid/hiraid.h index a9d2b15..9116c99 100644 --- a/drivers/scsi/hisi_raid/hiraid.h +++ b/drivers/scsi/hisi_raid/hiraid.h @@ -48,6 +48,10 @@ #define HIRAID_DEV_INFO_FLAG_VALID(flag) ((flag) & 0x01) #define HIRAID_DEV_INFO_FLAG_CHANGE(flag) ((flag) & 0x02) +#define HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(flag) ((flag) >> 2) +#define HIRAID_DEV_INFO_FLAG_DELETE_OR_ADD(flag, org_flag) \ + (HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(flag) != \ + HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(org_flag) ? 1 : 0) #define HIRAID_CAP_MQES(cap) ((cap) & 0xffff) #define HIRAID_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) @@ -76,6 +80,15 @@ #define HIRAID_SERVER_DEVICE_RAID_DID 0x3758 #define HIRAID_SERVER_DEVICE_RAIDS_DID 0x38D8 +#define HIRAID_SCSI_VPD_LEN SCSI_DEFAULT_VPD_LEN +#define SCSI_RSVD_INIT_SHIFT 0XFC +#define NCQ_PRIO_SUPPORT_BYTE 213 +#define VPD_NCQ_PRIO_SUPPORT_BIT 4 + +#define HIRAID_CAP_CAC_STRIDE 8 + +#define HIRAID_MAX_PD_NUM (40 + 1) +#define HIRAID_MAX_STREAM_NUM 8 enum { HIRAID_SC_SUCCESS = 0x0, @@ -91,6 +104,7 @@ enum { enum { HIRAID_REG_CAP = 0x0000, + HIRAID_REG_VS = 0x0008, HIRAID_REG_CC = 0x0014, HIRAID_REG_CSTS = 0x001c, HIRAID_REG_AQA = 0x0024, @@ -237,6 +251,20 @@ enum { DISPATCH_BY_DISK, }; +enum hiraid_stream_type { + HIRAID_STREAM_TYPE_TOTAL, + HIRAID_STREAM_TYPE_WRITE, + HIRAID_STREAM_TYPE_READ, + HIRAID_STREAM_TYPE_CLEAN, + HIRAID_STREAM_TYPE_BOTTOM +}; + +enum hiraid_stream_io_operation_type { + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO = 1, + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST +}; + struct hiraid_completion { __le32 result; union { @@ -272,6 +300,19 @@ struct hiraid_ctrl_info { __u8 rsvd1[4020]; }; +struct hiraid_stream { + /* recog-window */ + u64 stream_lba; + u32 stream_len; + u16 did; + u16 type; + /* aging ctrl */ + int aging_credit; + int aging_grade; + u16 stream_id; + u16 stream_is_using; +}; + struct hiraid_dev { struct pci_dev *pdev; struct device *dev; @@ -319,6 +360,18 @@ struct hiraid_dev { u8 hdd_dispatch; struct request_queue *bsg_queue; + + u16 hiraid_stream_io_count; + u64 hiraid_stream_aging_time; + u64 hiraid_stream_sent_io_size[HIRAID_MAX_PD_NUM] + [HIRAID_MAX_STREAM_NUM]; + u16 hiraid_stream_io_num_per_pd[HIRAID_MAX_PD_NUM] + [HIRAID_STREAM_TYPE_BOTTOM + 1]; + spinlock_t hiraid_stream_array_lock; + struct hiraid_stream hiraid_stream_array[HIRAID_MAX_PD_NUM] + [HIRAID_MAX_STREAM_NUM]; + struct task_struct *hiraid_stream_submit_task; + u64 hiraid_stream_io_last_pull_time[HIRAID_MAX_PD_NUM]; }; struct hiraid_sgl_desc { @@ -686,6 +739,7 @@ struct hiraid_queue { atomic_t inflight; void *sense_buffer_virt; dma_addr_t sense_buffer_phy; + s32 pci_irq; struct dma_pool *prp_small_pool; }; @@ -757,8 +811,22 @@ struct hiraid_sdev_hostdata { u8 flag; u8 rg_id; u8 hwq; + u8 sata_ncq_prio_enable; + u8 rsvd[3]; u16 pend_count; }; +struct hiraid_stream_io_list { + struct list_head list; + struct hiraid_scsi_io_cmd io_cmd; + struct hiraid_queue *submit_queue; + unsigned int sector_size; +}; + +struct mutex_list_head { + struct list_head list; + struct mutex lock; +}; + #endif diff --git a/drivers/scsi/hisi_raid/hiraid_main.c b/drivers/scsi/hisi_raid/hiraid_main.c index f687c93..5add277 100644 --- a/drivers/scsi/hisi_raid/hiraid_main.c +++ b/drivers/scsi/hisi_raid/hiraid_main.c @@ -26,8 +26,13 @@ #include <linux/bsg-lib.h> #include <asm/unaligned.h> #include <linux/sort.h> +#include <linux/kthread.h> +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/sched/prio.h> #include <target/target_core_backend.h> +#include <linux/blk-mq-pci.h> #include <scsi/scsi.h> #include <scsi/scsi_cmnd.h> #include <scsi/scsi_device.h> @@ -38,7 +43,7 @@ #include "hiraid.h" -static u32 admin_tmout = 60; +static u32 admin_tmout = 180; module_param(admin_tmout, uint, 0644); MODULE_PARM_DESC(admin_tmout, "admin commands timeout (seconds)"); @@ -99,7 +104,7 @@ static const struct kernel_param_ops io_queue_depth_ops = { .get = param_get_uint, }; -static u32 io_queue_depth = 1024; +static u32 io_queue_depth = 4096; module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); @@ -140,8 +145,6 @@ MODULE_PARM_DESC(extra_pool_num, "set prp extra pool num, default 4, MAX 16"); static void hiraid_handle_async_notice(struct hiraid_dev *hdev, u32 result); static void hiraid_handle_async_vs(struct hiraid_dev *hdev, u32 result, u32 result1); -static struct class *hiraid_class; - #define HIRAID_CAP_TIMEOUT_UNIT_MS (HZ / 2) static struct workqueue_struct *work_queue; @@ -152,7 +155,7 @@ static struct workqueue_struct *work_queue; __func__, ##__VA_ARGS__); \ } while (0) -#define HIRAID_DRV_VERSION "1.1.0.0" +#define HIRAID_DRV_VERSION "2.1.0.1" #define ADMIN_TIMEOUT (admin_tmout * HZ) #define USRCMD_TIMEOUT (180 * HZ) @@ -171,6 +174,19 @@ static struct workqueue_struct *work_queue; #define MAX_CAN_QUEUE (4096 - 1) #define MIN_CAN_QUEUE (1024 - 1) +#define MAX_DECREASE_GRADE (-8) +#define MAX_INCREASE_GRADE 8 +#define INC_GRADE 1 +#define MIN_CREDIT 0 +#define MAX_CREDIT 64 +#define CREDIT_THRES 32 +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif +#ifndef MAX +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif + enum SENSE_STATE_CODE { SENSE_STATE_OK = 0, SENSE_STATE_NEED_CHECK, @@ -344,91 +360,6 @@ static u32 hiraid_get_max_cmd_size(struct hiraid_dev *hdev) return sizeof(struct hiraid_mapmange) + alloc_size; } -static int hiraid_build_passthru_prp(struct hiraid_dev *hdev, struct hiraid_mapmange *mapbuf) -{ - struct scatterlist *sg = mapbuf->sgl; - __le64 *phy_regpage, *prior_list; - u64 buf_addr = sg_dma_address(sg); - int buf_length = sg_dma_len(sg); - u32 page_size = hdev->page_size; - int offset = buf_addr & (page_size - 1); - void **list = hiraid_mapbuf_list(mapbuf); - int maplen = mapbuf->len; - struct dma_pool *pool; - dma_addr_t buffer_phy; - int i; - - maplen -= (page_size - offset); - if (maplen <= 0) { - mapbuf->first_dma = 0; - return 0; - } - - buf_length -= (page_size - offset); - if (buf_length) { - buf_addr += (page_size - offset); - } else { - sg = sg_next(sg); - buf_addr = sg_dma_address(sg); - buf_length = sg_dma_len(sg); - } - - if (maplen <= page_size) { - mapbuf->first_dma = buf_addr; - return 0; - } - - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; - - phy_regpage = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!phy_regpage) { - dev_err_ratelimited(hdev->dev, "allocate first admin prp_list memory failed\n"); - mapbuf->first_dma = buf_addr; - mapbuf->page_cnt = -1; - return -ENOMEM; - } - list[0] = phy_regpage; - mapbuf->first_dma = buffer_phy; - i = 0; - for (;;) { - if (i == page_size / PRP_ENTRY_SIZE) { - prior_list = phy_regpage; - - phy_regpage = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!phy_regpage) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th admin prp list memory failed\n", - mapbuf->page_cnt + 1); - return -ENOMEM; - } - list[mapbuf->page_cnt++] = phy_regpage; - phy_regpage[0] = prior_list[i - 1]; - prior_list[i - 1] = cpu_to_le64(buffer_phy); - i = 1; - } - phy_regpage[i++] = cpu_to_le64(buf_addr); - buf_addr += page_size; - buf_length -= page_size; - maplen -= page_size; - if (maplen <= 0) - break; - if (buf_length > 0) - continue; - if (unlikely(buf_length < 0)) - goto bad_admin_sgl; - sg = sg_next(sg); - buf_addr = sg_dma_address(sg); - buf_length = sg_dma_len(sg); - } - - return 0; - -bad_admin_sgl: - dev_err(hdev->dev, "setup prps, invalid admin SGL for payload[%d] nents[%d]\n", - mapbuf->len, mapbuf->sge_cnt); - return -EIO; -} - static int hiraid_build_prp(struct hiraid_dev *hdev, struct hiraid_mapmange *mapbuf) { struct scatterlist *sg = mapbuf->sgl; @@ -631,9 +562,8 @@ static void hiraid_sgl_set_seg(struct hiraid_sgl_desc *sge, dma_addr_t buffer_ph } } -static int hiraid_build_passthru_sgl(struct hiraid_dev *hdev, - struct hiraid_admin_command *admin_cmd, - struct hiraid_mapmange *mapbuf) +static int hiraid_build_sgl(struct hiraid_dev *hdev, + struct hiraid_scsi_io_cmd *io_cmd, struct hiraid_mapmange *mapbuf) { struct hiraid_sgl_desc *sg_list, *link, *old_sg_list; struct scatterlist *sg = mapbuf->sgl; @@ -643,26 +573,31 @@ static int hiraid_build_passthru_sgl(struct hiraid_dev *hdev, dma_addr_t buffer_phy; int i = 0; - admin_cmd->common.flags |= SQE_FLAG_SGL_METABUF; + io_cmd->common.flags |= SQE_FLAG_SGL_METABUF; if (nsge == 1) { - hiraid_sgl_set_data(&admin_cmd->common.dptr.sgl, sg); + hiraid_sgl_set_data(&io_cmd->common.dptr.sgl, sg); return 0; } - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; + if (nsge <= (EXTRA_POOL_SIZE / sizeof(struct hiraid_sgl_desc))) { + pool = mapbuf->hiraidq->prp_small_pool; + mapbuf->page_cnt = 0; + } else { + pool = hdev->prp_page_pool; + mapbuf->page_cnt = 1; + } sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate first admin sgl_list failed\n"); + dev_err_ratelimited(hdev->dev, "allocate first sgl_list failed\n"); mapbuf->page_cnt = -1; return -ENOMEM; } list[0] = sg_list; mapbuf->first_dma = buffer_phy; - hiraid_sgl_set_seg(&admin_cmd->common.dptr.sgl, buffer_phy, nsge); + hiraid_sgl_set_seg(&io_cmd->common.dptr.sgl, buffer_phy, nsge); do { if (i == SGES_PER_PAGE) { old_sg_list = sg_list; @@ -670,8 +605,8 @@ static int hiraid_build_passthru_sgl(struct hiraid_dev *hdev, sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th admin sgl_list failed\n", - mapbuf->page_cnt + 1); + dev_err_ratelimited(hdev->dev, "allocate [%d]th sgl_list failed\n", + mapbuf->page_cnt + 1); return -ENOMEM; } list[mapbuf->page_cnt++] = sg_list; @@ -688,68 +623,374 @@ static int hiraid_build_passthru_sgl(struct hiraid_dev *hdev, return 0; } +#define MAX_IO_NUM (200 * 1024 * 1024) +#define STREAM_LEN (4 * 1024 * 1024) +#define MAX_IO_NUM_ONCE 100 +#define IO_SUBMIT_TIME_OUT 100 +#define MAX_AGING_NUM 5000 +#define MAX_AGING_TIME 16 +#define AGING_DEGRADE (-2) +#define MIN_IO_SEND_TIME 10 +#define MAX_IO_SEND_TIME 50 -static int hiraid_build_sgl(struct hiraid_dev *hdev, struct hiraid_scsi_io_cmd *io_cmd, - struct hiraid_mapmange *mapbuf) +struct mutex_list_head hiraid_io_heads_per_stream[ + HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM]; +DEFINE_MUTEX(g_stream_operation_mutex); + +static inline bool hiraid_io_recog_check_stream_exceed(struct hiraid_dev *hdev, + u16 disk_id) { - struct hiraid_sgl_desc *sg_list, *link, *old_sg_list; - struct scatterlist *sg = mapbuf->sgl; - void **list = hiraid_mapbuf_list(mapbuf); - struct dma_pool *pool; - int nsge = mapbuf->sge_cnt; - dma_addr_t buffer_phy; - int i = 0; + bool exceed_flag; - io_cmd->common.flags |= SQE_FLAG_SGL_METABUF; + spin_lock(&hdev->hiraid_stream_array_lock); + exceed_flag = + (hdev->hiraid_stream_io_num_per_pd[disk_id][HIRAID_STREAM_TYPE_TOTAL] >= + HIRAID_MAX_STREAM_NUM); + spin_unlock(&hdev->hiraid_stream_array_lock); + return exceed_flag; +} - if (nsge == 1) { - hiraid_sgl_set_data(&io_cmd->common.dptr.sgl, sg); - return 0; +static inline bool hiraid_recognition_acknowledge( + const struct hiraid_stream *stream) +{ + return (stream->aging_credit >= CREDIT_THRES) ? true : false; +} + +void hiraid_io_recognition_init(struct hiraid_dev *hdev) +{ + u16 i; + + spin_lock_init(&hdev->hiraid_stream_array_lock); + for (i = 0; i < (HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM); i++) { + INIT_LIST_HEAD(&hiraid_io_heads_per_stream[i].list); + mutex_init(&hiraid_io_heads_per_stream[i].lock); } +} - if (nsge <= (EXTRA_POOL_SIZE / sizeof(struct hiraid_sgl_desc))) { - pool = mapbuf->hiraidq->prp_small_pool; - mapbuf->page_cnt = 0; - } else { - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; +static void hiraid_io_recognition_iterator(struct hiraid_stream *stream, + int direction) +{ + stream->aging_grade = stream->aging_grade + direction * INC_GRADE; + stream->aging_grade = MAX(stream->aging_grade, MAX_DECREASE_GRADE); + stream->aging_grade = MIN(stream->aging_grade, MAX_INCREASE_GRADE); + stream->aging_credit = stream->aging_credit + stream->aging_grade; + stream->aging_credit = MAX(stream->aging_credit, MIN_CREDIT); + stream->aging_credit = MIN(stream->aging_credit, MAX_CREDIT); +} + +struct hiraid_stream *hiraid_io_pick_stream(struct hiraid_dev *hdev, + struct hiraid_scsi_rw_cmd *req, u16 type, u16 actual_id) +{ + struct hiraid_stream *first_hit_stream = NULL; + struct hiraid_stream *temp_stream = NULL; + bool pick_flag = false; + u8 i; + + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_stream = &hdev->hiraid_stream_array[actual_id][i]; + temp_stream->stream_id = i; + if (req->slba < temp_stream->stream_lba || + req->slba >= temp_stream->stream_lba + + temp_stream->stream_len || + temp_stream->type != type || + !temp_stream->stream_is_using) { + continue; + } + if (pick_flag == false) { + temp_stream->stream_lba = req->slba; + first_hit_stream = temp_stream; + pick_flag = true; + continue; + } + spin_lock(&hdev->hiraid_stream_array_lock); + if (hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL] > 0) + hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL]--; + + memset(temp_stream, 0, sizeof(struct hiraid_stream)); + spin_unlock(&hdev->hiraid_stream_array_lock); } + return first_hit_stream; +} - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate first sgl_list failed\n"); - mapbuf->page_cnt = -1; - return -ENOMEM; +static struct hiraid_stream *hiraid_init_flow_stream(struct hiraid_dev *hdev, + struct hiraid_scsi_rw_cmd *req, u16 type, u16 actual_id) +{ + int i; + struct hiraid_stream *stream = NULL; + + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + stream = &hdev->hiraid_stream_array[actual_id][i]; + if (!stream->stream_is_using) { + stream->stream_is_using = 1; + stream->stream_id = i; + break; + } } + stream->stream_lba = req->slba; + stream->did = actual_id; + stream->type = type; + stream->aging_credit = 0; + stream->aging_grade = 0; + stream->stream_len = STREAM_LEN; + return stream; +} - list[0] = sg_list; - mapbuf->first_dma = buffer_phy; - hiraid_sgl_set_seg(&io_cmd->common.dptr.sgl, buffer_phy, nsge); - do { - if (i == SGES_PER_PAGE) { - old_sg_list = sg_list; - link = &old_sg_list[SGES_PER_PAGE - 1]; +static struct hiraid_stream *hiraid_stream_detect(struct hiraid_dev *hdev, + struct hiraid_scsi_rw_cmd *io_cmd, u16 actual_id) +{ + u16 type = HIRAID_STREAM_TYPE_READ; - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th sgl_list failed\n", - mapbuf->page_cnt + 1); - return -ENOMEM; + if (io_cmd->opcode == HIRAID_CMD_WRITE) + type = HIRAID_STREAM_TYPE_WRITE; + + struct hiraid_stream *stream = hiraid_io_pick_stream(hdev, + io_cmd, type, actual_id); + + if (stream != NULL) { + return stream; + } + + if (hiraid_io_recog_check_stream_exceed(hdev, actual_id)) + return NULL; + + stream = hiraid_init_flow_stream(hdev, io_cmd, type, actual_id); + spin_lock(&hdev->hiraid_stream_array_lock); + hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL]++; + spin_unlock(&hdev->hiraid_stream_array_lock); + return stream; +} + +static u16 hiraid_get_submit_io_stream(u16 did, struct hiraid_dev *hdev) +{ + u64 temp_num, i; + static u16 stream_num[HIRAID_MAX_PD_NUM] = {0}; + + if (hdev->hiraid_stream_io_last_pull_time[did] == 0) + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_num = hdev->hiraid_stream_sent_io_size[did][i]; + if (temp_num != 0) { + if ((temp_num < MAX_IO_NUM) && + ((jiffies_to_msecs(jiffies) - + hdev->hiraid_stream_io_last_pull_time[did]) < + IO_SUBMIT_TIME_OUT)) { + stream_num[did] = i; + return i; } - list[mapbuf->page_cnt++] = sg_list; + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + hdev->hiraid_stream_sent_io_size[did][i] = 0; + stream_num[did] = ((i+1) % + HIRAID_MAX_STREAM_NUM); + return ((i+1) % HIRAID_MAX_STREAM_NUM); - i = 0; - memcpy(&sg_list[i++], link, sizeof(*link)); - hiraid_sgl_set_seg(link, buffer_phy, nsge); } + } + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + return ((stream_num[did]++) % HIRAID_MAX_STREAM_NUM); +} - hiraid_sgl_set_data(&sg_list[i++], sg); - sg = sg_next(sg); - } while (--nsge > 0); +static void hiraid_submit_io_stream(u16 hdid, struct hiraid_dev *hdev) +{ + struct mutex_list_head *io_slist = NULL; + struct list_head *node = NULL; + struct list_head *next_node = NULL; + struct list_head temp_header; + struct hiraid_scsi_io_cmd io_cmd; + u16 submit_stream_id; + struct hiraid_stream_io_list *temp_io_stream = NULL; + u16 count = 0; + + INIT_LIST_HEAD(&temp_header); + submit_stream_id = hiraid_get_submit_io_stream(hdid, hdev); + io_slist = &hiraid_io_heads_per_stream[hdid * HIRAID_MAX_STREAM_NUM + + submit_stream_id]; + mutex_lock(&io_slist->lock); + list_for_each_safe(node, next_node, &io_slist->list) { + list_del(node); + list_add_tail(node, &temp_header); + if (count++ >= MAX_IO_NUM_ONCE) + break; + } + mutex_unlock(&io_slist->lock); + list_for_each_safe(node, next_node, &temp_header) { + temp_io_stream = list_entry(node, + struct hiraid_stream_io_list, list); + io_cmd = temp_io_stream->io_cmd; + hiraid_submit_cmd(temp_io_stream->submit_queue, &io_cmd); + hdev->hiraid_stream_sent_io_size[hdid][submit_stream_id] += + io_cmd.rw.nlb * temp_io_stream->sector_size; + list_del(node); + kfree(temp_io_stream); + } + temp_io_stream = NULL; + list_del_init(&temp_header); +} + +static u8 hiraid_aging_detect(struct hiraid_dev *hdev) +{ + if (hdev->hiraid_stream_aging_time == 0) + hdev->hiraid_stream_aging_time = jiffies_to_msecs(jiffies); + if (++hdev->hiraid_stream_io_count > MAX_AGING_NUM) { + hdev->hiraid_stream_io_count = 0; + if ((jiffies_to_msecs(jiffies) - + hdev->hiraid_stream_aging_time) < + MAX_AGING_TIME) + return 0; + hdev->hiraid_stream_aging_time = jiffies_to_msecs(jiffies); + return 1; + } return 0; } +static void hiraid_sync_using_stream(struct hiraid_dev *hdev, u16 hdid) +{ + u8 i; + struct hiraid_stream *temp_stream = NULL; + + hdev->hiraid_stream_io_num_per_pd[hdid][HIRAID_STREAM_TYPE_TOTAL] = 0; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_stream = &hdev->hiraid_stream_array[hdid][i]; + if (temp_stream->stream_is_using) + hdev->hiraid_stream_io_num_per_pd[hdid] + [HIRAID_STREAM_TYPE_TOTAL]++; + } +} + +static void hiraid_aging(struct hiraid_dev *hdev) +{ + struct hiraid_stream *temp_stream = NULL; + int i = 0; + int j = 0; + u16 *tmp_io_num = NULL; + + for (i = 0; i < HIRAID_MAX_PD_NUM; i++) { + for (j = 0; j < HIRAID_MAX_STREAM_NUM; j++) { + temp_stream = &hdev->hiraid_stream_array[i][j]; + if (temp_stream->stream_is_using) { + hiraid_io_recognition_iterator(temp_stream, + AGING_DEGRADE); + spin_lock(&hdev->hiraid_stream_array_lock); + tmp_io_num = &hdev->hiraid_stream_io_num_per_pd + [i][HIRAID_STREAM_TYPE_TOTAL]; + if (temp_stream->aging_credit <= 0) { + if (*tmp_io_num > 0) + (*tmp_io_num)--; + memset(temp_stream, + 0, sizeof(struct hiraid_stream)); + } + spin_unlock(&hdev->hiraid_stream_array_lock); + } + } + hiraid_sync_using_stream(hdev, i); + } +} + +static u8 hiraid_stream_delete_io_list(struct hiraid_dev *hdev, u32 hdid, + u16 cid, u16 hwq, u8 delete_operation) +{ + int i, j; + struct mutex_list_head *io_slist = NULL; + struct list_head *node = NULL; + struct list_head *next_node = NULL; + struct hiraid_scsi_io_cmd *io_cmd = NULL; + struct hiraid_queue *hiraidq = NULL; + struct hiraid_stream_io_list *temp_io_stream = NULL; + u8 max_hd_num = delete_operation == + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST ? HIRAID_MAX_PD_NUM : hdid + 1; + + for (i = hdid; i < max_hd_num; i++) { + for (j = 0; j < HIRAID_MAX_STREAM_NUM; j++) { + io_slist = + &hiraid_io_heads_per_stream[i * HIRAID_MAX_STREAM_NUM + j]; + mutex_lock(&io_slist->lock); + list_for_each_safe(node, next_node, &io_slist->list) { + temp_io_stream = + list_entry(node, struct hiraid_stream_io_list, list); + io_cmd = &(temp_io_stream->io_cmd); + hiraidq = temp_io_stream->submit_queue; + if (delete_operation >= + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST) { + list_del_init(node); + kfree(temp_io_stream); + temp_io_stream = NULL; + } else { + if ((io_cmd->rw.cmd_id == cid) && + (hiraidq->qid == hwq)) { + list_del_init(node); + mutex_unlock(&io_slist->lock); + kfree(temp_io_stream); + return 1; + } + } + } + mutex_unlock(&io_slist->lock); + } + } + return 0; +} + +static u8 hiraid_add_io_to_list(struct hiraid_dev *hdev, + struct hiraid_queue *submit_queue, struct hiraid_stream *tmp_stream, + struct hiraid_scsi_io_cmd io_cmd, unsigned int sector_size, + u16 actual_id) +{ + struct mutex_list_head *io_slist = NULL; + struct hiraid_stream_io_list *new_io_node = NULL; + u16 temp_io_id = actual_id * HIRAID_MAX_STREAM_NUM + + tmp_stream->stream_id; + + new_io_node = kmalloc(sizeof(struct hiraid_stream_io_list), GFP_KERNEL); + if (!new_io_node) + return 0; + new_io_node->io_cmd = io_cmd; + new_io_node->submit_queue = submit_queue; + new_io_node->sector_size = sector_size; + io_slist = &hiraid_io_heads_per_stream[temp_io_id]; + mutex_lock(&io_slist->lock); + INIT_LIST_HEAD(&(new_io_node->list)); + list_add_tail(&(new_io_node->list), + &hiraid_io_heads_per_stream[temp_io_id].list); + mutex_unlock(&io_slist->lock); + return 1; +} + +static void hiraid_submit_io_threading(struct hiraid_dev *hdev) +{ + int i = 0; + + while (!kthread_should_stop()) { + for (i = 0; i < HIRAID_MAX_PD_NUM; i++) + hiraid_submit_io_stream(i, hdev); + usleep_range(MIN_IO_SEND_TIME, MAX_IO_SEND_TIME); + } +} + +static void hiraid_destroy_io_stream_resource(struct hiraid_dev *hdev) +{ + u16 i; + + for (i = 0; i < (HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM); i++) { + list_del_init(&hiraid_io_heads_per_stream[i].list); + mutex_destroy(&hiraid_io_heads_per_stream[i].lock); + } +} + +static void hiraid_init_io_stream(struct hiraid_dev *hdev) +{ + hiraid_io_recognition_init(hdev); + hdev->hiraid_stream_submit_task = + kthread_run((void *)hiraid_submit_io_threading, + hdev, "hiraid_stream_submit_task"); +} + #define HIRAID_RW_FUA BIT(14) #define RW_LENGTH_ZERO (67) @@ -872,6 +1113,30 @@ static int hiraid_setup_nonrw_cmd(struct hiraid_dev *hdev, return 0; } +static bool hiraid_disk_is_hdd(u8 attr) +{ + switch (HIRAID_DEV_DISK_TYPE(attr)) { + case HIRAID_SAS_HDD_VD: + case HIRAID_SATA_HDD_VD: + case HIRAID_SAS_HDD_PD: + case HIRAID_SATA_HDD_PD: + return true; + default: + return false; + } +} + +static bool hiraid_disk_is_hdd_rawdrive(u8 attr) +{ + switch (HIRAID_DEV_DISK_TYPE(attr)) { + case HIRAID_SAS_HDD_PD: + case HIRAID_SATA_HDD_PD: + return true; + default: + return false; + } +} + static int hiraid_setup_io_cmd(struct hiraid_dev *hdev, struct hiraid_scsi_io_cmd *io_cmd, struct scsi_cmnd *scmd, struct hiraid_mapmange *mapbuf) @@ -937,7 +1202,7 @@ static int hiraid_io_map_data(struct hiraid_dev *hdev, struct hiraid_mapmange *m ret = scsi_dma_map(scmd); if (unlikely(ret < 0)) - return ret; + return SCSI_MLQUEUE_HOST_BUSY; mapbuf->sge_cnt = ret; /* No data to DMA, it may be scsi no-rw command */ @@ -1018,6 +1283,35 @@ static inline void hiraid_query_scmd_tag(struct scsi_cmnd *scmd, u16 *qid, u16 * *cid = blk_mq_unique_tag_to_tag(tag); } +static void hiraid_set_ncq_prio(struct scsi_cmnd *scmd, + struct hiraid_scsi_io_cmd *io_cmd, + struct hiraid_sdev_hostdata *hostdata, + struct hiraid_dev *hdev) +{ + struct request *rq = scsi_cmd_to_rq(scmd); + int class = 0; + + if (hostdata->sata_ncq_prio_enable) { + class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); + switch (class) { + case IOPRIO_CLASS_NONE: + break; + case IOPRIO_CLASS_RT: + io_cmd->rw.rsvd2 &= SCSI_RSVD_INIT_SHIFT; + io_cmd->rw.rsvd2 |= (1 << IOPRIO_CLASS_RT); + break; + case IOPRIO_CLASS_BE: + break; + case IOPRIO_CLASS_IDLE: + break; + default: + break; + } + } + dev_log_dbg(hdev->dev, "blk_ncq_prio = %d, scmd_ncq_prio = 0x%02x\n", + class, io_cmd->rw.rsvd2); +} + static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) { struct hiraid_mapmange *mapbuf = scsi_cmd_priv(scmd); @@ -1026,6 +1320,7 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) struct hiraid_sdev_hostdata *hostdata; struct hiraid_scsi_io_cmd io_cmd; struct hiraid_queue *ioq; + struct hiraid_stream *tmp_stream; u16 hwq, cid; int ret; @@ -1055,6 +1350,8 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) io_cmd.rw.hdid = cpu_to_le32(hostdata->hdid); io_cmd.rw.cmd_id = cpu_to_le16(cid); + hiraid_set_ncq_prio(scmd, &io_cmd, hostdata, hdev); + ret = hiraid_setup_io_cmd(hdev, &io_cmd, scmd, mapbuf); if (unlikely(ret)) { if (ret == RW_LENGTH_ZERO) { @@ -1085,6 +1382,10 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) mapbuf->cid = cid; ret = hiraid_io_map_data(hdev, mapbuf, scmd, &io_cmd); if (unlikely(ret)) { + if (ret == SCSI_MLQUEUE_HOST_BUSY) { + atomic_dec(&ioq->inflight); + return ret; + } dev_err(hdev->dev, "io map data err\n"); set_host_byte(scmd, DID_ERROR); scsi_done(scmd); @@ -1093,6 +1394,23 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) } WRITE_ONCE(mapbuf->state, CMD_FLIGHT); + if (hiraid_is_rw_scmd(scmd) && + hiraid_disk_is_hdd_rawdrive(hostdata->attr)) { + if (hiraid_aging_detect(hdev)) + hiraid_aging(hdev); + tmp_stream = hiraid_stream_detect(hdev, &(io_cmd.rw), sdev->id); + if (tmp_stream != NULL) { + hiraid_io_recognition_iterator(tmp_stream, 1); + if (hiraid_recognition_acknowledge(tmp_stream) && + (sdev->id < HIRAID_MAX_PD_NUM) && + (hdev->hiraid_stream_io_num_per_pd[sdev->id][HIRAID_STREAM_TYPE_TOTAL] > + 1)) { + if (hiraid_add_io_to_list(hdev, ioq, tmp_stream, + io_cmd, sdev->sector_size, sdev->id)) + return 0; + } + } + } hiraid_submit_cmd(ioq, &io_cmd); return 0; @@ -1142,19 +1460,6 @@ static int hiraid_disk_qd(u8 attr) } } -static bool hiraid_disk_is_hdd(u8 attr) -{ - switch (HIRAID_DEV_DISK_TYPE(attr)) { - case HIRAID_SAS_HDD_VD: - case HIRAID_SATA_HDD_VD: - case HIRAID_SAS_HDD_PD: - case HIRAID_SATA_HDD_PD: - return true; - default: - return false; - } -} - static int hiraid_slave_alloc(struct scsi_device *sdev) { struct hiraid_sdev_hostdata *hostdata; @@ -1238,6 +1543,7 @@ static int hiraid_slave_configure(struct scsi_device *sdev) static void hiraid_shost_init(struct hiraid_dev *hdev) { struct pci_dev *pdev = hdev->pdev; + struct scsi_host_template *hostt; u8 domain, bus; u32 dev_func; @@ -1246,8 +1552,10 @@ static void hiraid_shost_init(struct hiraid_dev *hdev) dev_func = pdev->devfn; hdev->shost->nr_hw_queues = work_mode ? 1 : hdev->online_queues - 1; - hdev->shost->can_queue = hdev->scsi_qd; - + hdev->shost->can_queue = min(hdev->scsi_qd, + le16_to_cpu(hdev->ctrl_info->max_cmds) - + HIRAID_AQ_DEPTH - + HIRAID_TOTAL_PTCMDS(hdev->online_queues - 1)); hdev->shost->sg_tablesize = le16_to_cpu(hdev->ctrl_info->max_num_sge); /* 512B per sector */ hdev->shost->max_sectors = (1U << ((hdev->ctrl_info->mdts) * 1U) << 12) / 512; @@ -1259,6 +1567,13 @@ static void hiraid_shost_init(struct hiraid_dev *hdev) hdev->shost->this_id = -1; hdev->shost->unique_id = (domain << 16) | (bus << 8) | dev_func; hdev->shost->max_cmd_len = MAX_CDB_LEN; + + hostt = (struct scsi_host_template *)hdev->shost->hostt; + hostt->cmd_size = hiraid_get_max_cmd_size(hdev); + + dev_info(hdev->dev, "nr_hw_queues[%u] can_queue[%u] unique_id[%u] cmd_size[%u]\n", + hdev->shost->nr_hw_queues, hdev->shost->can_queue, + hdev->shost->unique_id, hdev->shost->hostt->cmd_size); } static int hiraid_alloc_queue(struct hiraid_dev *hdev, u16 qid, u16 depth) @@ -2046,6 +2361,14 @@ static int hiraid_get_queue_cnt(struct hiraid_dev *hdev, u32 *cnt) return 0; } +static bool hiraid_pci_is_present(struct hiraid_dev *hdev) +{ + if (!pci_device_is_present(hdev->pdev) || + readl(hdev->bar + HIRAID_REG_VS) == 0Xffffffff) + return false; + return true; +} + static int hiraid_setup_io_queues(struct hiraid_dev *hdev) { struct hiraid_queue *adminq = &hdev->queues[0]; @@ -2117,7 +2440,7 @@ static void hiraid_delete_io_queues(struct hiraid_dev *hdev) u8 opcode = HIRAID_ADMIN_DELETE_SQ; u16 i, pass; - if (!pci_device_is_present(hdev->pdev)) { + if (!hiraid_pci_is_present(hdev)) { dev_err(hdev->dev, "pci_device is not present, skip disable io queues\n"); return; } @@ -2154,7 +2477,7 @@ static void hiraid_disable_admin_queue(struct hiraid_dev *hdev, bool shutdown) struct hiraid_queue *adminq = &hdev->queues[0]; u16 start, end; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { if (shutdown) hiraid_shutdown_control(hdev); else @@ -2372,11 +2695,16 @@ static int hiraid_luntarget_sort(const void *l, const void *r) const struct hiraid_dev_info *rn = r; int l_attr = HIRAID_DEV_INFO_ATTR_BOOT(ln->attr); int r_attr = HIRAID_DEV_INFO_ATTR_BOOT(rn->attr); - /* boot first */ if (l_attr != r_attr) return (r_attr - l_attr); + l_attr = HIRAID_DEV_INFO_ATTR_VD(ln->attr); + r_attr = HIRAID_DEV_INFO_ATTR_VD(rn->attr); + /* vd second */ + if (l_attr != r_attr) + return (r_attr - l_attr); + if (ln->channel == rn->channel) return le16_to_cpu(ln->target) - le16_to_cpu(rn->target); @@ -2413,6 +2741,19 @@ static void hiraid_scan_work(struct work_struct *work) if (HIRAID_DEV_INFO_FLAG_VALID(flag)) { if (!HIRAID_DEV_INFO_FLAG_VALID(org_flag)) { + down_write(&hdev->dev_rwsem); + memcpy(&old_dev[i], &dev[i], + sizeof(struct hiraid_dev_info)); + memcpy(&new_dev[count++], &dev[i], + sizeof(struct hiraid_dev_info)); + up_write(&hdev->dev_rwsem); + } else if (HIRAID_DEV_INFO_FLAG_DELETE_OR_ADD(flag, + org_flag)) { + down_write(&hdev->dev_rwsem); + old_dev[i].flag &= 0xfe; + up_write(&hdev->dev_rwsem); + hiraid_delete_device(hdev, &old_dev[i]); + down_write(&hdev->dev_rwsem); memcpy(&old_dev[i], &dev[i], sizeof(struct hiraid_dev_info)); @@ -2560,7 +2901,7 @@ static void hiraid_bsg_buf_unmap(struct hiraid_dev *hdev, struct bsg_job *job) } static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, - struct hiraid_admin_command *cmd) + struct hiraid_admin_command *cmd, u16 qid) { struct hiraid_bsg_request *bsg_req = job->request; struct request *rq = blk_mq_rq_from_pdu(job); @@ -2573,6 +2914,7 @@ static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, mapbuf->sgl = job->request_payload.sg_list; mapbuf->len = job->request_payload.payload_len; mapbuf->page_cnt = -1; + mapbuf->hiraidq = &hdev->queues[qid]; if (unlikely(mapbuf->sge_cnt == 0)) goto out; @@ -2582,13 +2924,15 @@ static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, mapbuf->use_sgl = !hiraid_is_prp(hdev, mapbuf->sgl, mapbuf->sge_cnt); - if ((mapbuf->use_sgl == (bool)true) && (bsg_req->msgcode == HIRAID_BSG_IOPTHRU) && - (hdev->ctrl_info->pt_use_sgl != (bool)false)) { - ret = hiraid_build_passthru_sgl(hdev, cmd, mapbuf); + if ((mapbuf->use_sgl == true) && + (bsg_req->msgcode == HIRAID_BSG_IOPTHRU) && + (hdev->ctrl_info->pt_use_sgl != false)) { + ret = hiraid_build_sgl(hdev, (struct hiraid_scsi_io_cmd *)cmd, + mapbuf); } else { mapbuf->use_sgl = false; - ret = hiraid_build_passthru_prp(hdev, mapbuf); + ret = hiraid_build_prp(hdev, mapbuf); cmd->common.dptr.prp1 = cpu_to_le64(sg_dma_address(mapbuf->sgl)); cmd->common.dptr.prp2 = cpu_to_le64(mapbuf->first_dma); } @@ -2666,8 +3010,7 @@ static int hiraid_init_control_info(struct hiraid_dev *hdev) if (hdev->ctrl_info->asynevent > HIRAID_ASYN_COMMANDS) hdev->ctrl_info->asynevent = HIRAID_ASYN_COMMANDS; - hdev->scsi_qd = work_mode ? - le16_to_cpu(hdev->ctrl_info->max_cmds) : (hdev->ioq_depth - HIRAID_PTHRU_CMDS_PERQ); + hdev->scsi_qd = hdev->ioq_depth - HIRAID_PTHRU_CMDS_PERQ; return 0; } @@ -2700,7 +3043,7 @@ static int hiraid_user_send_admcmd(struct hiraid_dev *hdev, struct bsg_job *job) admin_cmd.common.cdw14 = cpu_to_le32(ptcmd->cdw14); admin_cmd.common.cdw15 = cpu_to_le32(ptcmd->cdw15); - status = hiraid_bsg_buf_map(hdev, job, &admin_cmd); + status = hiraid_bsg_buf_map(hdev, job, &admin_cmd, 0); if (status) { dev_err(hdev->dev, "err, map data failed\n"); return status; @@ -2712,7 +3055,7 @@ static int hiraid_user_send_admcmd(struct hiraid_dev *hdev, struct bsg_job *job) memcpy(job->reply, result, sizeof(result)); } if (status) - dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] result0[0x%x];" + dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] result0[0x%x] " "result1[0x%x]\n", ptcmd->opcode, ptcmd->info_0.subopcode, status, result[0], result[1]); @@ -2757,18 +3100,12 @@ static void hiraid_free_io_ptcmds(struct hiraid_dev *hdev) } static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, struct hiraid_scsi_io_cmd *io_cmd, - u32 *result, u32 *reslen, u32 timeout) + struct hiraid_cmd *pt_cmd, u32 *result, u32 *reslen, u32 timeout) { int ret; dma_addr_t buffer_phy; struct hiraid_queue *ioq; void *sense_addr = NULL; - struct hiraid_cmd *pt_cmd = hiraid_get_cmd(hdev, HIRAID_CMD_PTHRU); - - if (!pt_cmd) { - dev_err(hdev->dev, "err, get ioq cmd failed\n"); - return -EFAULT; - } timeout = timeout ? timeout : ADMIN_TIMEOUT; @@ -2798,8 +3135,6 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, struct hiraid_scs (le32_to_cpu(io_cmd->common.cdw3[0]) & 0xffff)); hiraid_admin_timeout(hdev, pt_cmd); - - hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); return -ETIME; } @@ -2810,8 +3145,6 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, struct hiraid_scs } } - hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); - return pt_cmd->status; } @@ -2820,6 +3153,7 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) struct hiraid_bsg_request *bsg_req = (struct hiraid_bsg_request *)(job->request); struct hiraid_passthru_io_cmd *cmd = &(bsg_req->pthrucmd); struct hiraid_scsi_io_cmd pthru_cmd; + struct hiraid_cmd *pt_cmd; int status = 0; u32 timeout = msecs_to_jiffies(cmd->timeout_ms); // data len is 4k before use sgl, now len is 1M @@ -2861,20 +3195,30 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) pthru_cmd.common.cdw26[2] = cpu_to_le32(cmd->cdw26[2]); pthru_cmd.common.cdw26[3] = cpu_to_le32(cmd->cdw26[3]); - status = hiraid_bsg_buf_map(hdev, job, (struct hiraid_admin_command *)&pthru_cmd); + pt_cmd = hiraid_get_cmd(hdev, HIRAID_CMD_PTHRU); + if (pt_cmd == NULL) { + dev_err(hdev->dev, "err, get ioq cmd failed\n"); + return -EFAULT; + } + + status = hiraid_bsg_buf_map(hdev, job, + (struct hiraid_admin_command *)&pthru_cmd, pt_cmd->qid); if (status) { dev_err(hdev->dev, "err, map data failed\n"); - return status; + goto ret; } - status = hiraid_put_io_sync_request(hdev, &pthru_cmd, job->reply, &job->reply_len, timeout); - + status = hiraid_put_io_sync_request(hdev, &pthru_cmd, pt_cmd, + job->reply, &job->reply_len, timeout); if (status) dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] replylen[%d]\n", cmd->opcode, cmd->info_1.subopcode, status, job->reply_len); hiraid_bsg_buf_unmap(hdev, job); +ret: + hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); + return status; } @@ -2895,20 +3239,26 @@ static bool hiraid_check_scmd_finished(struct scsi_cmnd *scmd) return false; } -static enum scsi_timeout_action hiraid_timed_out(struct scsi_cmnd *scmd) +#define EH_TIMED_RET_TP enum scsi_timeout_action +#define EH_TIMED_RET_DONE SCSI_EH_DONE +#define EH_TIMED_RET_NOT_HANDLED SCSI_EH_NOT_HANDLED +#define EH_TIMED_RET_RESET_TIMER SCSI_EH_RESET_TIMER +static EH_TIMED_RET_TP hiraid_timed_out(struct scsi_cmnd *scmd) { struct hiraid_mapmange *mapbuf = scsi_cmd_priv(scmd); unsigned int timeout = scmd->device->request_queue->rq_timeout; if (hiraid_check_scmd_finished(scmd)) - goto out; + return EH_TIMED_RET_DONE; - if (time_after(jiffies, scmd->jiffies_at_alloc + timeout)) { + if (time_after_eq(jiffies, scmd->jiffies_at_alloc + timeout)) { + scmd_printk(KERN_WARNING, scmd, "timeout is work[%d], need abort\n", + mapbuf->state); if (cmpxchg(&mapbuf->state, CMD_FLIGHT, CMD_TIMEOUT) == CMD_FLIGHT) - return SCSI_EH_DONE; + return EH_TIMED_RET_NOT_HANDLED; } -out: - return SCSI_EH_RESET_TIMER; + + return EH_TIMED_RET_RESET_TIMER; } /* send abort command by admin queue temporary */ @@ -3031,7 +3381,7 @@ static int hiraid_dev_disable(struct hiraid_dev *hdev, bool shutdown) struct hiraid_queue *adminq = &hdev->queues[0]; u16 start, end; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { if (shutdown) hiraid_shutdown_control(hdev); else @@ -3226,10 +3576,24 @@ static int hiraid_abort(struct scsi_cmnd *scmd) hiraid_check_scmd_finished(scmd)) return SUCCESS; + if (hdev->state != DEV_LIVE) + return SUCCESS; + hostdata = scmd->device->hostdata; cid = mapbuf->cid; hwq = mapbuf->hiraidq->qid; + mutex_lock(&g_stream_operation_mutex); + ret = hiraid_stream_delete_io_list(hdev, hostdata->hdid, + cid, hwq, HIRAID_STREAM_TYPE_DELETE_SINGLE_IO); + mutex_unlock(&g_stream_operation_mutex); + + if (ret == 1) { + dev_warn(hdev->dev, "find cid[%d] qid[%d] in host, abort succ\n", + cid, hwq); + return SUCCESS; + } + dev_warn(hdev->dev, "cid[%d] qid[%d] timeout, send abort\n", cid, hwq); ret = hiraid_send_abort_cmd(hdev, hostdata->hdid, hwq, cid); if (ret != -ETIME) { @@ -3261,6 +3625,10 @@ static int hiraid_scsi_reset(struct scsi_cmnd *scmd, enum hiraid_rst_type rst) ret = hiraid_send_reset_cmd(hdev, rst, hostdata->hdid); if ((ret == 0) || (ret == FW_EH_DEV_NONE && rst == HIRAID_RESET_TARGET)) { if (rst == HIRAID_RESET_TARGET) { + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, hostdata->hdid, + 0, 0, HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); ret = wait_tgt_reset_io_done(scmd); if (ret) { dev_warn(hdev->dev, "sdev[%d:%d] target has %d peding cmd, target reset failed\n", @@ -3298,8 +3666,13 @@ static int hiraid_host_reset(struct scsi_cmnd *scmd) dev_warn(hdev->dev, "sdev[%d:%d] send host reset\n", scmd->device->channel, scmd->device->id); - if (hiraid_reset_work_sync(hdev) == -EBUSY) + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + if (hiraid_reset_work_sync(hdev) == -EBUSY) { flush_work(&hdev->reset_work); + } if (hdev->state != DEV_LIVE) { dev_warn(hdev->dev, "sdev[%d:%d] host reset failed\n", @@ -3331,6 +3704,10 @@ static pci_ers_result_t hiraid_pci_error_detected(struct pci_dev *pdev, scsi_block_requests(hdev->shost); hiraid_dev_state_trans(hdev, DEV_RESETTING); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); return PCI_ERS_RESULT_NEED_RESET; case pci_channel_io_perm_failure: @@ -3366,16 +3743,31 @@ static void hiraid_reset_pci_finish(struct pci_dev *pdev) { struct hiraid_dev *hdev = pci_get_drvdata(pdev); + if (hiraid_reset_work_sync(hdev) == -EBUSY) + flush_work(&hdev->reset_work); + dev_info(hdev->dev, "enter hiraid reset finish\n"); } +static void hiraid_reset_pci_prepare(struct pci_dev *pdev) +{ + struct hiraid_dev *hdev = pci_get_drvdata(pdev); + + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + msleep(100); + dev_info(hdev->dev, "exit hiraid reset prepare\n"); +} + static ssize_t csts_pp_show(struct device *cdev, struct device_attribute *attr, char *buf) { struct Scsi_Host *shost = class_to_shost(cdev); struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_PP_MASK); ret >>= HIRAID_CSTS_PP_SHIFT; } @@ -3389,7 +3781,7 @@ static ssize_t csts_shst_show(struct device *cdev, struct device_attribute *attr struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_SHST_MASK); ret >>= HIRAID_CSTS_SHST_SHIFT; } @@ -3403,7 +3795,7 @@ static ssize_t csts_cfs_show(struct device *cdev, struct device_attribute *attr, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_CFS_MASK); ret >>= HIRAID_CSTS_CFS_SHIFT; } @@ -3417,7 +3809,7 @@ static ssize_t csts_rdy_show(struct device *cdev, struct device_attribute *attr, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) + if (hiraid_pci_is_present(hdev)) ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_RDY); return snprintf(buf, PAGE_SIZE, "%d\n", ret); @@ -3671,16 +4063,73 @@ static ssize_t dispatch_hwq_store(struct device *dev, struct device_attribute *a return strlen(buf); } +static ssize_t sata_ncq_prio_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hiraid_sdev_hostdata *hostdata; + + hostdata = to_scsi_device(dev)->hostdata; + return snprintf(buf, PAGE_SIZE, "%d\n", hostdata->sata_ncq_prio_enable); +} + +bool scsi_ncq_prio_support(struct scsi_device *sdev, struct hiraid_dev *hdev) +{ + unsigned char *buf; + bool ncq_prio_supp = false; + + if (!scsi_device_supports_vpd(sdev)) { + dev_err(hdev->dev, "scsi device not support vpd\n"); + return ncq_prio_supp; + } + + buf = kzalloc(HIRAID_SCSI_VPD_LEN, GFP_KERNEL); + if (!buf) { + dev_err(hdev->dev, "alloc scsi vpd data memeroy failed\n"); + return ncq_prio_supp; + } + + if (!scsi_get_vpd_page(sdev, 0x89, buf, HIRAID_SCSI_VPD_LEN)) + ncq_prio_supp = (buf[NCQ_PRIO_SUPPORT_BYTE] >> + VPD_NCQ_PRIO_SUPPORT_BIT) & 1; + + kfree(buf); + return ncq_prio_supp; +} + +static ssize_t sata_ncq_prio_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct scsi_device *sdev = to_scsi_device(dev); + struct hiraid_sdev_hostdata *hostdata = sdev->hostdata; + struct hiraid_dev *hdev = shost_priv(sdev->host); + bool ncq_prio_enable = 0; + + if (kstrtobool(buf, &ncq_prio_enable)) + return -EINVAL; + + if (!scsi_ncq_prio_support(sdev, hdev)) { + dev_err(hdev->dev, "sdev[%d:%d] not support ncq prio\n", + sdev->channel, sdev->id); + return -EINVAL; + } + + hostdata->sata_ncq_prio_enable = ncq_prio_enable; + + return strlen(buf); +} + static DEVICE_ATTR_RO(raid_level); static DEVICE_ATTR_RO(raid_state); static DEVICE_ATTR_RO(raid_resync); static DEVICE_ATTR_RW(dispatch_hwq); +static DEVICE_ATTR_RW(sata_ncq_prio_enable); static struct attribute *hiraid_dev_attrs[] = { &dev_attr_raid_state.attr, &dev_attr_raid_level.attr, &dev_attr_raid_resync.attr, &dev_attr_dispatch_hwq.attr, + &dev_attr_sata_ncq_prio_enable.attr, NULL, }; @@ -3696,6 +4145,7 @@ const struct attribute_group *hiraid_dev_groups[] = { static struct pci_error_handlers hiraid_err_handler = { .error_detected = hiraid_pci_error_detected, .slot_reset = hiraid_pci_slot_reset, + .reset_prepare = hiraid_reset_pci_prepare, .reset_done = hiraid_reset_pci_finish, }; @@ -3720,7 +4170,17 @@ static int hiraid_scan_finished(struct Scsi_Host *shost, unsigned long time) return 1; } -static const struct scsi_host_template hiraid_driver_template = { +static void hiraid_map_queues(struct Scsi_Host *shost) +{ + struct hiraid_dev *hdev = shost_priv(shost); + + if (shost->nr_hw_queues == 1) + return; + + blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT], + hdev->pdev, 1); +} +static struct scsi_host_template hiraid_driver_template = { .module = THIS_MODULE, .name = "hiraid", .proc_name = "hiraid", @@ -3736,10 +4196,10 @@ static const struct scsi_host_template hiraid_driver_template = { .eh_host_reset_handler = hiraid_host_reset, .change_queue_depth = scsi_change_queue_depth, .this_id = -1, + .map_queues = hiraid_map_queues, .shost_groups = hiraid_host_attrs_groups, .sdev_groups = hiraid_dev_groups, .host_reset = hiraid_sysfs_host_reset, - .cmd_size = sizeof(struct hiraid_mapmange) + HIRAID_ALLOC_SIZE_MAX, }; static void hiraid_shutdown(struct pci_dev *pdev) @@ -3919,6 +4379,7 @@ static int hiraid_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto unregist_bsg; scsi_scan_host(hdev->shost); + hiraid_init_io_stream(hdev); return 0; @@ -3950,10 +4411,17 @@ static void hiraid_remove(struct pci_dev *pdev) dev_info(hdev->dev, "enter hiraid remove\n"); + kthread_stop(hdev->hiraid_stream_submit_task); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + hiraid_destroy_io_stream_resource(hdev); + hiraid_dev_state_trans(hdev, DEV_DELETING); flush_work(&hdev->reset_work); - if (!pci_device_is_present(pdev)) + if (!hiraid_pci_is_present(hdev)) hiraid_flush_running_cmds(hdev); hiraid_unregist_bsg(hdev); @@ -3999,22 +4467,9 @@ static int __init hiraid_init(void) if (!work_queue) return -ENOMEM; - hiraid_class = class_create("hiraid"); - if (IS_ERR(hiraid_class)) { - ret = PTR_ERR(hiraid_class); - goto destroy_wq; - } - ret = pci_register_driver(&hiraid_driver); if (ret < 0) - goto destroy_class; - - return 0; - -destroy_class: - class_destroy(hiraid_class); -destroy_wq: - destroy_workqueue(work_queue); + destroy_workqueue(work_queue); return ret; } @@ -4022,7 +4477,6 @@ destroy_wq: static void __exit hiraid_exit(void) { pci_unregister_driver(&hiraid_driver); - class_destroy(hiraid_class); destroy_workqueue(work_queue); } -- 2.45.1.windows.1

2 1

[PATCH openEuler-1.0-LTS] scsi/hiraid: Support New RAID features
by LinKun 04 Jun '26

04 Jun '26

From: 岳智超 <yuezhichao1(a)h-partners.com> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9101 CVE: NA -------------------------------- Fix high traffic IO disconnection issues Fix drive letter misalignment issue Fix timeout work mode Fix boot issue in single work mode Change adm timeout Change io queue depth Change prp、sgl Change stream detect Add linkdown fast recovery Add boot disk first scan Add flr Add ncq feature Delete driver class Signed-off-by: 岳智超 <yuezhichao1(a)h-partners.com> --- drivers/scsi/hisi_raid/hiraid.h | 87 ++- drivers/scsi/hisi_raid/hiraid_main.c | 920 ++++++++++++++------------- 2 files changed, 538 insertions(+), 469 deletions(-) diff --git a/drivers/scsi/hisi_raid/hiraid.h b/drivers/scsi/hisi_raid/hiraid.h index cb90d3d..25df43b 100644 --- a/drivers/scsi/hisi_raid/hiraid.h +++ b/drivers/scsi/hisi_raid/hiraid.h @@ -50,6 +50,10 @@ #define HIRAID_DEV_INFO_FLAG_VALID(flag) ((flag) & 0x01) #define HIRAID_DEV_INFO_FLAG_CHANGE(flag) ((flag) & 0x02) +#define HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(flag) ((flag) >> 2) +#define HIRAID_DEV_INFO_FLAG_DELETE_OR_ADD(flag, org_flag) \ + (HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(flag) != \ + HIRAID_DEV_INFO_FLAG_DEV_MAGIC_DEV(org_flag) ? 1 : 0) #define HIRAID_CAP_MQES(cap) ((cap) & 0xffff) #define HIRAID_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) @@ -74,7 +78,19 @@ #define PCI_VENDOR_ID_HUAWEI_LOGIC 0x19E5 #define HIRAID_SERVER_DEVICE_HBA_DID 0x3858 +#define HIRAID_SERVER_DEVICE_HBAS_DID 0x3918 #define HIRAID_SERVER_DEVICE_RAID_DID 0x3758 +#define HIRAID_SERVER_DEVICE_RAIDS_DID 0x38D8 + +#define HIRAID_SCSI_VPD_LEN SCSI_VPD_PG_LEN +#define SCSI_RSVD_INIT_SHIFT 0XFC +#define NCQ_PRIO_SUPPORT_BYTE 213 +#define VPD_NCQ_PRIO_SUPPORT_BIT 4 + +#define HIRAID_CAP_CAC_STRIDE 8 + +#define HIRAID_MAX_PD_NUM (40 + 1) +#define HIRAID_MAX_STREAM_NUM 8 enum { HIRAID_SC_SUCCESS = 0x0, @@ -90,6 +106,7 @@ enum { enum { HIRAID_REG_CAP = 0x0000, + HIRAID_REG_VS = 0x0008, HIRAID_REG_CC = 0x0014, HIRAID_REG_CSTS = 0x001c, HIRAID_REG_AQA = 0x0024, @@ -236,6 +253,20 @@ enum { DISPATCH_BY_DISK, }; +enum hiraid_stream_type { + HIRAID_STREAM_TYPE_TOTAL, + HIRAID_STREAM_TYPE_WRITE, + HIRAID_STREAM_TYPE_READ, + HIRAID_STREAM_TYPE_CLEAN, + HIRAID_STREAM_TYPE_BOTTOM +}; + +enum hiraid_stream_io_operation_type { + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO = 1, + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST +}; + struct hiraid_completion { __le32 result; union { @@ -271,6 +302,19 @@ struct hiraid_ctrl_info { __u8 rsvd1[4020]; }; +struct hiraid_stream { + /* recog-window */ + u64 stream_lba; + u32 stream_len; + u16 did; + u16 type; + /* aging ctrl */ + int aging_credit; + int aging_grade; + u16 stream_id; + u16 stream_is_using; +}; + struct hiraid_dev { struct pci_dev *pdev; struct device *dev; @@ -318,6 +362,18 @@ struct hiraid_dev { u8 hdd_dispatch; struct request_queue *bsg_queue; + + u16 hiraid_stream_io_count; + u64 hiraid_stream_aging_time; + u64 hiraid_stream_sent_io_size[HIRAID_MAX_PD_NUM] + [HIRAID_MAX_STREAM_NUM]; + u16 hiraid_stream_io_num_per_pd[HIRAID_MAX_PD_NUM] + [HIRAID_STREAM_TYPE_BOTTOM + 1]; + spinlock_t hiraid_stream_array_lock; + struct hiraid_stream hiraid_stream_array[HIRAID_MAX_PD_NUM] + [HIRAID_MAX_STREAM_NUM]; + struct task_struct *hiraid_stream_submit_task; + u64 hiraid_stream_io_last_pull_time[HIRAID_MAX_PD_NUM]; }; struct hiraid_sgl_desc { @@ -696,6 +752,10 @@ struct hiraid_mapmange { u32 sge_cnt; u32 len; bool use_sgl; + /* This field is used in the I/O read/write process. + * It indicates the TRANSFER LENGTH field in the CDB. + */ + u32 cdb_data_len; dma_addr_t first_dma; void *sense_buffer_virt; dma_addr_t sense_buffer_phy; @@ -755,38 +815,19 @@ struct hiraid_sdev_hostdata { u8 flag; u8 rg_id; u8 hwq; + u8 sata_ncq_prio_enable; + u8 rsvd[3]; u16 pend_count; }; -enum stream_type { - TYPE_TOTAL, - TYPE_WRITE, - TYPE_READ, - TYPE_CLEAN, - TYPE_BOTTOM -}; - -struct HIRAID_STREAM_S { - /* recog-window */ - u64 stream_lba; - u32 stream_len; - u16 did; - u16 type; - /* aging ctrl */ - int aging_credit; - int aging_grade; - u16 stream_id; - u16 using; -}; - -struct IO_LIST_S { +struct hiraid_stream_io_list { struct list_head list; struct hiraid_scsi_io_cmd io_cmd; struct hiraid_queue *submit_queue; unsigned int sector_size; }; -struct mutex_list_head_s { +struct mutex_list_head { struct list_head list; struct mutex lock; }; diff --git a/drivers/scsi/hisi_raid/hiraid_main.c b/drivers/scsi/hisi_raid/hiraid_main.c index cdba466..81f0736 100644 --- a/drivers/scsi/hisi_raid/hiraid_main.c +++ b/drivers/scsi/hisi_raid/hiraid_main.c @@ -26,6 +26,10 @@ #include <linux/bsg-lib.h> #include <asm/unaligned.h> #include <linux/sort.h> +#include <linux/kthread.h> +#include <linux/mutex.h> +#include <linux/sched.h> +#include <linux/sched/prio.h> #include <target/target_core_backend.h> #include <scsi/scsi.h> @@ -35,14 +39,10 @@ #include <scsi/scsi_transport.h> #include <scsi/scsi_dbg.h> #include <scsi/sg.h> -#include <linux/kthread.h> -#include <linux/mutex.h> -#include <linux/sched.h> -#include <linux/sched/prio.h> #include "hiraid.h" -static u32 admin_tmout = 60; +static u32 admin_tmout = 180; module_param(admin_tmout, uint, 0644); MODULE_PARM_DESC(admin_tmout, "admin commands timeout (seconds)"); @@ -101,7 +101,7 @@ static const struct kernel_param_ops io_queue_depth_ops = { .get = param_get_uint, }; -static u32 io_queue_depth = 1024; +static u32 io_queue_depth = 4096; module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); @@ -143,8 +143,6 @@ static void hiraid_handle_async_notice(struct hiraid_dev *hdev, u32 result); static void hiraid_handle_async_vs(struct hiraid_dev *hdev, u32 result, u32 result1); -static struct class *hiraid_class; - #define HIRAID_CAP_TIMEOUT_UNIT_MS (HZ / 2) static struct workqueue_struct *work_queue; @@ -155,7 +153,7 @@ static struct workqueue_struct *work_queue; __func__, ##__VA_ARGS__); \ } while (0) -#define HIRAID_DRV_VERSION "1.1.0.1" +#define HIRAID_DRV_VERSION "2.1.0.1" #define ADMIN_TIMEOUT (admin_tmout * HZ) #define USRCMD_TIMEOUT (180 * HZ) @@ -178,8 +176,12 @@ static struct workqueue_struct *work_queue; #define MIN_CREDIT 0 #define MAX_CREDIT 64 #define CREDIT_THRES 32 +#ifndef MIN #define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif +#ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#endif enum SENSE_STATE_CODE { SENSE_STATE_OK = 0, @@ -362,95 +364,8 @@ static u32 hiraid_get_max_cmd_size(struct hiraid_dev *hdev) return sizeof(struct hiraid_mapmange) + alloc_size; } -static int hiraid_build_passthru_prp(struct hiraid_dev *hdev, - struct hiraid_mapmange *mapbuf) -{ - struct scatterlist *sg = mapbuf->sgl; - __le64 *phy_regpage, *prior_list; - u64 buf_addr = sg_dma_address(sg); - int buf_length = sg_dma_len(sg); - u32 page_size = hdev->page_size; - int offset = buf_addr & (page_size - 1); - void **list = hiraid_mapbuf_list(mapbuf); - int maplen = mapbuf->len; - struct dma_pool *pool; - dma_addr_t buffer_phy; - int i; - - maplen -= (page_size - offset); - if (maplen <= 0) { - mapbuf->first_dma = 0; - return 0; - } - - buf_length -= (page_size - offset); - if (buf_length) { - buf_addr += (page_size - offset); - } else { - sg = sg_next(sg); - buf_addr = sg_dma_address(sg); - buf_length = sg_dma_len(sg); - } - - if (maplen <= page_size) { - mapbuf->first_dma = buf_addr; - return 0; - } - - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; - - phy_regpage = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!phy_regpage) { - dev_err_ratelimited(hdev->dev, "allocate first admin prp_list memory failed\n"); - mapbuf->first_dma = buf_addr; - mapbuf->page_cnt = -1; - return -ENOMEM; - } - list[0] = phy_regpage; - mapbuf->first_dma = buffer_phy; - i = 0; - for (;;) { - if (i == page_size / PRP_ENTRY_SIZE) { - prior_list = phy_regpage; - - phy_regpage = dma_pool_alloc(pool, GFP_ATOMIC, - &buffer_phy); - if (!phy_regpage) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th admin prp list memory failed\n", - mapbuf->page_cnt + 1); - return -ENOMEM; - } - list[mapbuf->page_cnt++] = phy_regpage; - phy_regpage[0] = prior_list[i - 1]; - prior_list[i - 1] = cpu_to_le64(buffer_phy); - i = 1; - } - phy_regpage[i++] = cpu_to_le64(buf_addr); - buf_addr += page_size; - buf_length -= page_size; - maplen -= page_size; - if (maplen <= 0) - break; - if (buf_length > 0) - continue; - if (unlikely(buf_length < 0)) - goto bad_admin_sgl; - sg = sg_next(sg); - buf_addr = sg_dma_address(sg); - buf_length = sg_dma_len(sg); - } - - return 0; - -bad_admin_sgl: - dev_err(hdev->dev, "setup prps, invalid admin SGL for payload[%d] nents[%d]\n", - mapbuf->len, mapbuf->sge_cnt); - return -EIO; -} - static int hiraid_build_prp(struct hiraid_dev *hdev, - struct hiraid_mapmange *mapbuf) + struct hiraid_mapmange *mapbuf) { struct scatterlist *sg = mapbuf->sgl; __le64 *phy_regpage, *prior_list; @@ -656,67 +571,9 @@ static void hiraid_sgl_set_seg(struct hiraid_sgl_desc *sge, } } -static int hiraid_build_passthru_sgl(struct hiraid_dev *hdev, - struct hiraid_admin_command *admin_cmd, - struct hiraid_mapmange *mapbuf) -{ - struct hiraid_sgl_desc *sg_list, *link, *old_sg_list; - struct scatterlist *sg = mapbuf->sgl; - void **list = hiraid_mapbuf_list(mapbuf); - struct dma_pool *pool; - int nsge = mapbuf->sge_cnt; - dma_addr_t buffer_phy; - int i = 0; - - admin_cmd->common.flags |= SQE_FLAG_SGL_METABUF; - - if (nsge == 1) { - hiraid_sgl_set_data(&admin_cmd->common.dptr.sgl, sg); - return 0; - } - - pool = hdev->prp_page_pool; - mapbuf->page_cnt = 1; - - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate first admin sgl_list failed\n"); - mapbuf->page_cnt = -1; - return -ENOMEM; - } - - list[0] = sg_list; - mapbuf->first_dma = buffer_phy; - hiraid_sgl_set_seg(&admin_cmd->common.dptr.sgl, buffer_phy, nsge); - do { - if (i == SGES_PER_PAGE) { - old_sg_list = sg_list; - link = &old_sg_list[SGES_PER_PAGE - 1]; - - sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &buffer_phy); - if (!sg_list) { - dev_err_ratelimited(hdev->dev, "allocate [%d]th admin sgl_list failed\n", - mapbuf->page_cnt + 1); - return -ENOMEM; - } - list[mapbuf->page_cnt++] = sg_list; - - i = 0; - memcpy(&sg_list[i++], link, sizeof(*link)); - hiraid_sgl_set_seg(link, buffer_phy, nsge); - } - - hiraid_sgl_set_data(&sg_list[i++], sg); - sg = sg_next(sg); - } while (--nsge > 0); - - return 0; -} - - static int hiraid_build_sgl(struct hiraid_dev *hdev, - struct hiraid_scsi_io_cmd *io_cmd, - struct hiraid_mapmange *mapbuf) + struct hiraid_scsi_io_cmd *io_cmd, + struct hiraid_mapmange *mapbuf) { struct hiraid_sgl_desc *sg_list, *link, *old_sg_list; struct scatterlist *sg = mapbuf->sgl; @@ -776,102 +633,52 @@ static int hiraid_build_sgl(struct hiraid_dev *hdev, return 0; } -#define MAX_PD_NUM (40 + 1) -#define MAX_STREAM_NUM 8 -#define PER_MB (1024 * 1024) -#define MAX_IO_NUM (200 * PER_MB) -#define STREAM_LEN (4 * PER_MB) +#define MAX_IO_NUM (200 * 1024 * 1024) +#define STREAM_LEN (4 * 1024 * 1024) #define MAX_IO_NUM_ONCE 100 #define IO_SUBMIT_TIME_OUT 100 -#define MAX_AGING_NUM 100 - +#define MAX_AGING_NUM 5000 +#define MAX_AGING_TIME 16 +#define AGING_DEGRADE (-2) #define MIN_IO_SEND_TIME 10 #define MAX_IO_SEND_TIME 50 -enum io_operation_type { - TYPE_DELETE_SINGLE_IO = 1, - TYPE_DELETE_SINGLE_IO_LIST, - TYPE_DELETE_ALL_IO_LIST -}; - -struct HIRAID_STREAM_S stream_array[MAX_PD_NUM][MAX_STREAM_NUM] = {0}; -struct mutex_list_head_s io_heads_per_stream[MAX_PD_NUM * MAX_STREAM_NUM]; -spinlock_t stream_array_lock; +struct mutex_list_head hiraid_io_heads_per_stream[ + HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM]; DEFINE_MUTEX(g_stream_operation_mutex); -u64 g_io_transport_num[MAX_PD_NUM][MAX_STREAM_NUM] = {0}; -u16 g_io_stream_num[MAX_PD_NUM][TYPE_BOTTOM] = {0}; -u16 g_io_count = 1; - -void hiraid_inc_io_transport_num(u16 disk_id, u16 streamd_id, u16 nlb) -{ - g_io_transport_num[disk_id][streamd_id] += nlb; -} - -void hiraid_refresh_io_transport_num(u16 disk_id, u16 streamd_id) -{ - g_io_transport_num[disk_id][streamd_id] = 0; -} - -void hiraid_inc_stream_num(u16 disk_id) -{ - spin_lock(&stream_array_lock); - g_io_stream_num[disk_id][TYPE_TOTAL]++; - spin_unlock(&stream_array_lock); -} - -void hiraid_dec_stream_num(u16 disk_id) -{ - spin_lock(&stream_array_lock); - if (g_io_stream_num[disk_id][TYPE_TOTAL] > 0) - g_io_stream_num[disk_id][TYPE_TOTAL]--; - spin_unlock(&stream_array_lock); -} - -static bool hiraid_io_recog_check_stream_exceed(u16 disk_id) +static inline bool hiraid_io_recog_check_stream_exceed(struct hiraid_dev *hdev, + u16 disk_id) { bool exceed_flag; - spin_lock(&stream_array_lock); - exceed_flag = (g_io_stream_num[disk_id][TYPE_TOTAL] >= MAX_STREAM_NUM); - spin_unlock(&stream_array_lock); + spin_lock(&hdev->hiraid_stream_array_lock); + exceed_flag = + (hdev->hiraid_stream_io_num_per_pd[disk_id][HIRAID_STREAM_TYPE_TOTAL] >= + HIRAID_MAX_STREAM_NUM); + spin_unlock(&hdev->hiraid_stream_array_lock); return exceed_flag; } -static u16 hiraid_get_stream_num(u16 disk_id) -{ - return g_io_stream_num[disk_id][TYPE_TOTAL]; -} - -static inline struct HIRAID_STREAM_S *hiraid_get_stream(u16 disk_id, - u16 stream_id) -{ - return &stream_array[disk_id][stream_id]; -} - -static inline struct mutex_list_head_s *hiraid_get_io_head(u16 disk_id) -{ - return &(io_heads_per_stream[disk_id]); -} - -static bool hiraid_recognition_acknowledge(const struct HIRAID_STREAM_S *stream) +static inline bool hiraid_recognition_acknowledge( + const struct hiraid_stream *stream) { return (stream->aging_credit >= CREDIT_THRES) ? true : false; } -void hiraid_io_recognition_init(void) +void hiraid_io_recognition_init(struct hiraid_dev *hdev) { u16 i; - spin_lock_init(&stream_array_lock); - for (i = 0; i < (MAX_PD_NUM * MAX_STREAM_NUM); i++) { - INIT_LIST_HEAD(&hiraid_get_io_head(i)->list); - mutex_init(&hiraid_get_io_head(i)->lock); + spin_lock_init(&hdev->hiraid_stream_array_lock); + for (i = 0; i < (HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM); i++) { + INIT_LIST_HEAD(&hiraid_io_heads_per_stream[i].list); + mutex_init(&hiraid_io_heads_per_stream[i].lock); } } -static void hiraid_io_recognition_iterator(struct HIRAID_STREAM_S *stream, - int direction) +static void hiraid_io_recognition_iterator(struct hiraid_stream *stream, + int direction) { stream->aging_grade = stream->aging_grade + direction * INC_GRADE; stream->aging_grade = MAX(stream->aging_grade, MAX_DECREASE_GRADE); @@ -881,46 +688,52 @@ static void hiraid_io_recognition_iterator(struct HIRAID_STREAM_S *stream, stream->aging_credit = MIN(stream->aging_credit, MAX_CREDIT); } -struct HIRAID_STREAM_S *hiraid_io_pick_stream( +struct hiraid_stream *hiraid_io_pick_stream(struct hiraid_dev *hdev, struct hiraid_scsi_rw_cmd *req, u16 type, u16 actual_id) { - struct HIRAID_STREAM_S *first_hit_stream = NULL; - struct HIRAID_STREAM_S *temp_stream = NULL; - u16 pick_flag = 0; + struct hiraid_stream *first_hit_stream = NULL; + struct hiraid_stream *temp_stream = NULL; + bool pick_flag = false; u8 i; - for (i = 0; i < MAX_STREAM_NUM; i++) { - temp_stream = &stream_array[actual_id][i]; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_stream = &hdev->hiraid_stream_array[actual_id][i]; temp_stream->stream_id = i; if (req->slba < temp_stream->stream_lba || req->slba >= temp_stream->stream_lba + temp_stream->stream_len || - temp_stream->type != type) { + temp_stream->type != type || + !temp_stream->stream_is_using) { continue; } - if (!pick_flag) { + if (pick_flag == false) { temp_stream->stream_lba = req->slba; first_hit_stream = temp_stream; - pick_flag = 1; + pick_flag = true; continue; } - hiraid_dec_stream_num(actual_id); - memset(temp_stream, 0, - sizeof(struct HIRAID_STREAM_S)); // 去重影 + spin_lock(&hdev->hiraid_stream_array_lock); + if (hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL] > 0) + hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL]--; + + memset(temp_stream, 0, sizeof(struct hiraid_stream)); + spin_unlock(&hdev->hiraid_stream_array_lock); } return first_hit_stream; } -static struct HIRAID_STREAM_S *hiraid_init_flow_stream( +static struct hiraid_stream *hiraid_init_flow_stream(struct hiraid_dev *hdev, struct hiraid_scsi_rw_cmd *req, u16 type, u16 actual_id) { int i; - struct HIRAID_STREAM_S *stream = NULL; + struct hiraid_stream *stream = NULL; - for (i = 0; i < MAX_STREAM_NUM; i++) { - stream = hiraid_get_stream(actual_id, i); - if (!stream->using) { - stream->using = 1; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + stream = &hdev->hiraid_stream_array[actual_id][i]; + if (!stream->stream_is_using) { + stream->stream_is_using = 1; stream->stream_id = i; break; } @@ -934,82 +747,97 @@ static struct HIRAID_STREAM_S *hiraid_init_flow_stream( return stream; } -static struct HIRAID_STREAM_S *hiraid_stream_detect(struct hiraid_dev *hdev, - struct hiraid_scsi_rw_cmd *io_cmd, u16 actual_id) +static struct hiraid_stream *hiraid_stream_detect(struct hiraid_dev *hdev, + struct hiraid_scsi_rw_cmd *io_cmd, u16 actual_id) { - u16 type = io_cmd->opcode == HIRAID_CMD_WRITE ? TYPE_WRITE : TYPE_READ; - struct HIRAID_STREAM_S *stream = hiraid_io_pick_stream(io_cmd, type, - actual_id); + u16 type = HIRAID_STREAM_TYPE_READ; + struct hiraid_stream *stream = NULL; + + if (io_cmd->opcode == HIRAID_CMD_WRITE) + type = HIRAID_STREAM_TYPE_WRITE; + + stream = hiraid_io_pick_stream(hdev, + io_cmd, type, actual_id); - if (stream != NULL) { /* 可以命中一个stream */ + if (stream != NULL) { return stream; } - if (hiraid_io_recog_check_stream_exceed(actual_id)) + if (hiraid_io_recog_check_stream_exceed(hdev, actual_id)) return NULL; - stream = hiraid_init_flow_stream(io_cmd, type, actual_id); - hiraid_inc_stream_num(actual_id); + + stream = hiraid_init_flow_stream(hdev, io_cmd, type, actual_id); + spin_lock(&hdev->hiraid_stream_array_lock); + hdev->hiraid_stream_io_num_per_pd[actual_id] + [HIRAID_STREAM_TYPE_TOTAL]++; + spin_unlock(&hdev->hiraid_stream_array_lock); return stream; } -u64 g_io_last_pull_time[MAX_PD_NUM] = {0}; - static u16 hiraid_get_submit_io_stream(u16 did, struct hiraid_dev *hdev) { u64 temp_num, i; - static u16 stream_num[MAX_PD_NUM] = {0}; + static u16 stream_num[HIRAID_MAX_PD_NUM] = {0}; - if (g_io_last_pull_time[did] == 0) - g_io_last_pull_time[did] = jiffies_to_msecs(jiffies); + if (hdev->hiraid_stream_io_last_pull_time[did] == 0) + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); - for (i = 0; i < MAX_STREAM_NUM; i++) { - temp_num = g_io_transport_num[did][i]; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_num = hdev->hiraid_stream_sent_io_size[did][i]; if (temp_num != 0) { if ((temp_num < MAX_IO_NUM) && - ((jiffies_to_msecs(jiffies) - g_io_last_pull_time[did]) - < IO_SUBMIT_TIME_OUT)) { + ((jiffies_to_msecs(jiffies) - + hdev->hiraid_stream_io_last_pull_time[did]) < + IO_SUBMIT_TIME_OUT)) { stream_num[did] = i; return i; } - g_io_last_pull_time[did] = jiffies_to_msecs(jiffies); - hiraid_refresh_io_transport_num(did, i); - stream_num[did] = ((i+1) % MAX_STREAM_NUM); - return ((i+1) % MAX_STREAM_NUM); + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + hdev->hiraid_stream_sent_io_size[did][i] = 0; + stream_num[did] = ((i+1) % + HIRAID_MAX_STREAM_NUM); + return ((i+1) % HIRAID_MAX_STREAM_NUM); + } } - g_io_last_pull_time[did] = jiffies_to_msecs(jiffies); - return ((stream_num[did]++) % MAX_STREAM_NUM); + hdev->hiraid_stream_io_last_pull_time[did] = + jiffies_to_msecs(jiffies); + return ((stream_num[did]++) % HIRAID_MAX_STREAM_NUM); } static void hiraid_submit_io_stream(u16 hdid, struct hiraid_dev *hdev) { - struct mutex_list_head_s *io_slist = NULL; + struct mutex_list_head *io_slist = NULL; struct list_head *node = NULL; struct list_head *next_node = NULL; struct list_head temp_header; - struct hiraid_scsi_io_cmd io_cmd = {0}; + struct hiraid_scsi_io_cmd io_cmd; u16 submit_stream_id; - struct IO_LIST_S *temp_io_stream = NULL; + struct hiraid_stream_io_list *temp_io_stream = NULL; u16 count = 0; INIT_LIST_HEAD(&temp_header); submit_stream_id = hiraid_get_submit_io_stream(hdid, hdev); - io_slist = hiraid_get_io_head(hdid * MAX_STREAM_NUM + submit_stream_id); + io_slist = &hiraid_io_heads_per_stream[hdid * HIRAID_MAX_STREAM_NUM + + submit_stream_id]; mutex_lock(&io_slist->lock); list_for_each_safe(node, next_node, &io_slist->list) { list_del(node); list_add_tail(node, &temp_header); - if (++count >= MAX_IO_NUM_ONCE) + if (count++ >= MAX_IO_NUM_ONCE) break; } mutex_unlock(&io_slist->lock); list_for_each_safe(node, next_node, &temp_header) { - temp_io_stream = list_entry(node, struct IO_LIST_S, list); + temp_io_stream = list_entry(node, + struct hiraid_stream_io_list, list); io_cmd = temp_io_stream->io_cmd; hiraid_submit_cmd(temp_io_stream->submit_queue, &io_cmd); - hiraid_inc_io_transport_num(hdid, submit_stream_id, - io_cmd.rw.nlb * temp_io_stream->sector_size); + hdev->hiraid_stream_sent_io_size[hdid][submit_stream_id] += + io_cmd.rw.nlb * temp_io_stream->sector_size; list_del(node); kfree(temp_io_stream); } @@ -1017,59 +845,90 @@ static void hiraid_submit_io_stream(u16 hdid, struct hiraid_dev *hdev) list_del_init(&temp_header); } -static u8 hiraid_detect_if_aging(void) +static u8 hiraid_aging_detect(struct hiraid_dev *hdev) { - if (++g_io_count == MAX_AGING_NUM) { - g_io_count = 0; + if (hdev->hiraid_stream_aging_time == 0) + hdev->hiraid_stream_aging_time = jiffies_to_msecs(jiffies); + if (++hdev->hiraid_stream_io_count > MAX_AGING_NUM) { + hdev->hiraid_stream_io_count = 0; + if ((jiffies_to_msecs(jiffies) - + hdev->hiraid_stream_aging_time) < + MAX_AGING_TIME) + return 0; + hdev->hiraid_stream_aging_time = jiffies_to_msecs(jiffies); return 1; } return 0; } +static void hiraid_sync_using_stream(struct hiraid_dev *hdev, u16 hdid) +{ + u8 i; + struct hiraid_stream *temp_stream = NULL; + + hdev->hiraid_stream_io_num_per_pd[hdid][HIRAID_STREAM_TYPE_TOTAL] = 0; + for (i = 0; i < HIRAID_MAX_STREAM_NUM; i++) { + temp_stream = &hdev->hiraid_stream_array[hdid][i]; + if (temp_stream->stream_is_using) + hdev->hiraid_stream_io_num_per_pd[hdid] + [HIRAID_STREAM_TYPE_TOTAL]++; + } +} + static void hiraid_aging(struct hiraid_dev *hdev) { - struct HIRAID_STREAM_S *temp_stream = NULL; + struct hiraid_stream *temp_stream = NULL; int i = 0; int j = 0; - - for (i = 0; i < MAX_PD_NUM; i++) { - for (j = 0; j < MAX_STREAM_NUM; j++) { - temp_stream = hiraid_get_stream(i, j); - if (temp_stream->using) { - hiraid_io_recognition_iterator(temp_stream, -1); + u16 *tmp_io_num = NULL; + + for (i = 0; i < HIRAID_MAX_PD_NUM; i++) { + for (j = 0; j < HIRAID_MAX_STREAM_NUM; j++) { + temp_stream = &hdev->hiraid_stream_array[i][j]; + if (temp_stream->stream_is_using) { + hiraid_io_recognition_iterator(temp_stream, + AGING_DEGRADE); + spin_lock(&hdev->hiraid_stream_array_lock); + tmp_io_num = &hdev->hiraid_stream_io_num_per_pd + [i][HIRAID_STREAM_TYPE_TOTAL]; if (temp_stream->aging_credit <= 0) { - hiraid_dec_stream_num(i); + if (*tmp_io_num > 0) + (*tmp_io_num)--; memset(temp_stream, - 0, sizeof(struct HIRAID_STREAM_S)); // 老化 + 0, sizeof(struct hiraid_stream)); } + spin_unlock(&hdev->hiraid_stream_array_lock); } } + hiraid_sync_using_stream(hdev, i); } } -static u8 hiraid_io_list_operation(u32 hdid, u16 cid, u16 hwq, u8 operation) +static u8 hiraid_stream_delete_io_list(struct hiraid_dev *hdev, u32 hdid, + u16 cid, u16 hwq, u8 delete_operation) { int i, j; - - struct mutex_list_head_s *io_slist = NULL; + struct mutex_list_head *io_slist = NULL; struct list_head *node = NULL; struct list_head *next_node = NULL; struct hiraid_scsi_io_cmd *io_cmd = NULL; struct hiraid_queue *hiraidq = NULL; - struct IO_LIST_S *temp_io_stream = NULL; + struct hiraid_stream_io_list *temp_io_stream = NULL; + u8 max_hd_num = delete_operation == + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST ? HIRAID_MAX_PD_NUM : hdid + 1; - u8 max_hd_num = operation == TYPE_DELETE_ALL_IO_LIST ? - MAX_PD_NUM : hdid + 1; for (i = hdid; i < max_hd_num; i++) { - for (j = 0; j < MAX_STREAM_NUM; j++) { - io_slist = hiraid_get_io_head(i * MAX_STREAM_NUM + j); + for (j = 0; j < HIRAID_MAX_STREAM_NUM; j++) { + io_slist = + &hiraid_io_heads_per_stream[i * HIRAID_MAX_STREAM_NUM + j]; mutex_lock(&io_slist->lock); list_for_each_safe(node, next_node, &io_slist->list) { - temp_io_stream = list_entry(node, - struct IO_LIST_S, list); + temp_io_stream = + list_entry(node, struct hiraid_stream_io_list, list); io_cmd = &(temp_io_stream->io_cmd); hiraidq = temp_io_stream->submit_queue; - if (operation >= TYPE_DELETE_SINGLE_IO_LIST) { + if (delete_operation >= + HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST) { list_del_init(node); kfree(temp_io_stream); temp_io_stream = NULL; @@ -1089,54 +948,27 @@ static u8 hiraid_io_list_operation(u32 hdid, u16 cid, u16 hwq, u8 operation) return 0; } -static u8 hiraid_check_io_list(u32 hdid, u16 cid, u16 hwq) -{ - u8 ret; - - mutex_lock(&g_stream_operation_mutex); - ret = hiraid_io_list_operation(hdid, cid, hwq, TYPE_DELETE_SINGLE_IO); - mutex_unlock(&g_stream_operation_mutex); - return ret; -} - -static u8 hiraid_delete_single_pd_io_list(u32 hdid) +static u8 hiraid_add_io_to_list(struct hiraid_dev *hdev, + struct hiraid_queue *submit_queue, struct hiraid_stream *tmp_stream, + struct hiraid_scsi_io_cmd io_cmd, unsigned int sector_size, + u16 actual_id) { - u8 ret; + struct mutex_list_head *io_slist = NULL; + struct hiraid_stream_io_list *new_io_node = NULL; + u16 temp_io_id = actual_id * HIRAID_MAX_STREAM_NUM + + tmp_stream->stream_id; - mutex_lock(&g_stream_operation_mutex); - ret = hiraid_io_list_operation(hdid, 0, 0, TYPE_DELETE_SINGLE_IO_LIST); - mutex_unlock(&g_stream_operation_mutex); - return ret; -} - -static u8 hiraid_delete_all_io_list(void) -{ - u8 ret; - - mutex_lock(&g_stream_operation_mutex); - ret = hiraid_io_list_operation(0, 0, 0, TYPE_DELETE_ALL_IO_LIST); - mutex_unlock(&g_stream_operation_mutex); - return ret; -} - -static u8 hiraid_add_io_to_list(struct hiraid_queue *submit_queue, - struct HIRAID_STREAM_S *tmp_stream, struct hiraid_scsi_io_cmd io_cmd, - unsigned int sector_size, u16 actual_id) -{ - struct mutex_list_head_s *io_slist = NULL; - struct IO_LIST_S *new_io_node = NULL; - - new_io_node = kmalloc(sizeof(struct IO_LIST_S), GFP_KERNEL); + new_io_node = kmalloc(sizeof(struct hiraid_stream_io_list), GFP_KERNEL); if (!new_io_node) return 0; new_io_node->io_cmd = io_cmd; new_io_node->submit_queue = submit_queue; new_io_node->sector_size = sector_size; - io_slist = hiraid_get_io_head(actual_id * - MAX_STREAM_NUM + tmp_stream->stream_id); + io_slist = &hiraid_io_heads_per_stream[temp_io_id]; mutex_lock(&io_slist->lock); INIT_LIST_HEAD(&(new_io_node->list)); - list_add_tail(&(new_io_node->list), &io_slist->list); + list_add_tail(&(new_io_node->list), + &hiraid_io_heads_per_stream[temp_io_id].list); mutex_unlock(&io_slist->lock); return 1; } @@ -1146,10 +978,8 @@ static void hiraid_submit_io_threading(struct hiraid_dev *hdev) int i = 0; while (!kthread_should_stop()) { - mutex_lock(&g_stream_operation_mutex); - for (i = 0; i < MAX_PD_NUM; i++) + for (i = 0; i < HIRAID_MAX_PD_NUM; i++) hiraid_submit_io_stream(i, hdev); - mutex_unlock(&g_stream_operation_mutex); usleep_range(MIN_IO_SEND_TIME, MAX_IO_SEND_TIME); } } @@ -1158,27 +988,34 @@ static void hiraid_destroy_io_stream_resource(struct hiraid_dev *hdev) { u16 i; - for (i = 0; i < (MAX_PD_NUM * MAX_STREAM_NUM); i++) - list_del_init(&hiraid_get_io_head(i)->list); + for (i = 0; i < (HIRAID_MAX_PD_NUM * HIRAID_MAX_STREAM_NUM); i++) { + list_del_init(&hiraid_io_heads_per_stream[i].list); + mutex_destroy(&hiraid_io_heads_per_stream[i].lock); + } } -struct task_struct *g_hiraid_submit_task; static void hiraid_init_io_stream(struct hiraid_dev *hdev) { - hiraid_io_recognition_init(); - g_hiraid_submit_task = kthread_run((void *)hiraid_submit_io_threading, - hdev, "hiraid_submit_thread"); + hiraid_io_recognition_init(hdev); + hdev->hiraid_stream_submit_task = + kthread_run((void *)hiraid_submit_io_threading, + hdev, "hiraid_stream_submit_task"); } #define HIRAID_RW_FUA BIT(14) +#define RW_LENGTH_ZERO (67) static int hiraid_setup_rw_cmd(struct hiraid_dev *hdev, struct hiraid_scsi_rw_cmd *io_cmd, - struct scsi_cmnd *scmd) + struct scsi_cmnd *scmd, + struct hiraid_mapmange *mapbuf) { + u32 ret = 0; u32 start_lba_lo, start_lba_hi; u32 datalength = 0; u16 control = 0; + struct scsi_device *sdev = scmd->device; + u32 buf_len = cpu_to_le32(scsi_bufflen(scmd)); start_lba_lo = 0; start_lba_hi = 0; @@ -1187,6 +1024,8 @@ static int hiraid_setup_rw_cmd(struct hiraid_dev *hdev, io_cmd->opcode = HIRAID_CMD_WRITE; } else if (scmd->sc_data_direction == DMA_FROM_DEVICE) { io_cmd->opcode = HIRAID_CMD_READ; + } else if (scmd->sc_data_direction == DMA_NONE) { + ret = RW_LENGTH_ZERO; } else { dev_err(hdev->dev, "invalid RW_IO for unsupported data direction[%d]\n", scmd->sc_data_direction); @@ -1194,6 +1033,9 @@ static int hiraid_setup_rw_cmd(struct hiraid_dev *hdev, return -EINVAL; } + if (ret == RW_LENGTH_ZERO) + return ret; + /* 6-byte READ(0x08) or WRITE(0x0A) cdb */ if (scmd->cmd_len == 6) { datalength = (u32)(scmd->cmnd[4] == 0 ? @@ -1230,18 +1072,32 @@ static int hiraid_setup_rw_cmd(struct hiraid_dev *hdev, control |= HIRAID_RW_FUA; } - if (unlikely(datalength > U16_MAX || datalength == 0)) { - dev_err(hdev->dev, "invalid IO for err trans data length[%u]\n", + if (unlikely(datalength > U16_MAX)) { + dev_err(hdev->dev, "invalid IO for illegal transfer data length[%u]\n", datalength); WARN_ON(1); return -EINVAL; } + if (unlikely(datalength == 0)) + return RW_LENGTH_ZERO; + io_cmd->slba = cpu_to_le64(((u64)start_lba_hi << 32) | start_lba_lo); /* 0base for nlb */ io_cmd->nlb = cpu_to_le16((u16)(datalength - 1)); io_cmd->control = cpu_to_le16(control); + mapbuf->cdb_data_len = (u32)((io_cmd->nlb + 1) * sdev->sector_size); + if (mapbuf->cdb_data_len > buf_len) { + /* return DID_ERROR */ + dev_err(hdev->dev, "error: buf len[0x%x] is smaller than actual length[0x%x] sectorsize[0x%x]\n", + buf_len, mapbuf->cdb_data_len, sdev->sector_size); + return -EINVAL; + } else if (mapbuf->cdb_data_len < buf_len) { + dev_warn(hdev->dev, "warn: buf_len[0x%x] cdb_data_len[0x%x] nlb[0x%x] sectorsize[0x%x]\n", + buf_len, mapbuf->cdb_data_len, + io_cmd->nlb, sdev->sector_size); + } return 0; } @@ -1296,14 +1152,17 @@ static bool hiraid_disk_is_hdd_rawdrive(u8 attr) } static int hiraid_setup_io_cmd(struct hiraid_dev *hdev, - struct hiraid_scsi_io_cmd *io_cmd, - struct scsi_cmnd *scmd) + struct hiraid_scsi_io_cmd *io_cmd, struct scsi_cmnd *scmd, + struct hiraid_mapmange *mapbuf) { memcpy(io_cmd->common.cdb, scmd->cmnd, scmd->cmd_len); io_cmd->common.cdb_len = scmd->cmd_len; + /* init cdb_data_len */ + mapbuf->cdb_data_len = cpu_to_le32(scsi_bufflen(scmd)); + if (hiraid_is_rw_scmd(scmd)) - return hiraid_setup_rw_cmd(hdev, &io_cmd->rw, scmd); + return hiraid_setup_rw_cmd(hdev, &io_cmd->rw, scmd, mapbuf); else return hiraid_setup_nonrw_cmd(hdev, &io_cmd->nonrw, scmd); } @@ -1360,7 +1219,7 @@ static int hiraid_io_map_data(struct hiraid_dev *hdev, ret = scsi_dma_map(scmd); if (unlikely(ret < 0)) - return ret; + return SCSI_MLQUEUE_HOST_BUSY; mapbuf->sge_cnt = ret; /* No data to DMA, it may be scsi no-rw command */ @@ -1390,7 +1249,12 @@ static void hiraid_check_status(struct hiraid_mapmange *mapbuf, struct scsi_cmnd *scmd, struct hiraid_completion *cqe) { - scsi_set_resid(scmd, 0); + u32 datalength = cpu_to_le32(scsi_bufflen(scmd)); + + if (datalength > mapbuf->cdb_data_len) + scsi_set_resid(scmd, datalength - mapbuf->cdb_data_len); + else + scsi_set_resid(scmd, 0); switch ((le16_to_cpu(cqe->status) >> 1) & 0x7f) { case SENSE_STATE_OK: @@ -1443,6 +1307,35 @@ static inline void hiraid_query_scmd_tag(struct scsi_cmnd *scmd, u16 *qid, *cid = blk_mq_unique_tag_to_tag(tag); } +static void hiraid_set_ncq_prio(struct scsi_cmnd *scmd, + struct hiraid_scsi_io_cmd *io_cmd, + struct hiraid_sdev_hostdata *hostdata, + struct hiraid_dev *hdev) +{ + struct request *rq = scmd->request; + int class = 0; + + if (hostdata->sata_ncq_prio_enable) { + class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); + switch (class) { + case IOPRIO_CLASS_NONE: + break; + case IOPRIO_CLASS_RT: + io_cmd->rw.rsvd2 &= SCSI_RSVD_INIT_SHIFT; + io_cmd->rw.rsvd2 |= (1 << IOPRIO_CLASS_RT); + break; + case IOPRIO_CLASS_BE: + break; + case IOPRIO_CLASS_IDLE: + break; + default: + break; + } + } + dev_log_dbg(hdev->dev, "blk_ncq_prio = %d, scmd_ncq_prio = 0x%02x\n", + class, io_cmd->rw.rsvd2); +} + static int hiraid_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) { @@ -1452,7 +1345,7 @@ static int hiraid_queue_command(struct Scsi_Host *shost, struct hiraid_sdev_hostdata *hostdata; struct hiraid_scsi_io_cmd io_cmd; struct hiraid_queue *ioq; - struct HIRAID_STREAM_S *tmp_stm = NULL; + struct hiraid_stream *tmp_stream; u16 hwq, cid; int ret; @@ -1482,9 +1375,16 @@ static int hiraid_queue_command(struct Scsi_Host *shost, io_cmd.rw.hdid = cpu_to_le32(hostdata->hdid); io_cmd.rw.cmd_id = cpu_to_le16(cid); - ret = hiraid_setup_io_cmd(hdev, &io_cmd, scmd); + hiraid_set_ncq_prio(scmd, &io_cmd, hostdata, hdev); + + ret = hiraid_setup_io_cmd(hdev, &io_cmd, scmd, mapbuf); if (unlikely(ret)) { - set_host_byte(scmd, DID_ERROR); + if (ret == RW_LENGTH_ZERO) { + scsi_set_resid(scmd, scsi_bufflen(scmd)); + set_host_byte(scmd, DID_OK); + } else { + set_host_byte(scmd, DID_ERROR); + } scmd->scsi_done(scmd); atomic_dec(&ioq->inflight); return 0; @@ -1507,6 +1407,10 @@ static int hiraid_queue_command(struct Scsi_Host *shost, mapbuf->cid = cid; ret = hiraid_io_map_data(hdev, mapbuf, scmd, &io_cmd); if (unlikely(ret)) { + if (ret == SCSI_MLQUEUE_HOST_BUSY) { + atomic_dec(&ioq->inflight); + return ret; + } dev_err(hdev->dev, "io map data err\n"); set_host_byte(scmd, DID_ERROR); scmd->scsi_done(scmd); @@ -1515,20 +1419,20 @@ static int hiraid_queue_command(struct Scsi_Host *shost, } WRITE_ONCE(mapbuf->state, CMD_FLIGHT); - if (hiraid_is_rw_scmd(scmd) && hiraid_disk_is_hdd_rawdrive(hostdata->attr)) { - if (hiraid_detect_if_aging()) + if (hiraid_aging_detect(hdev)) hiraid_aging(hdev); - tmp_stm = hiraid_stream_detect(hdev, &(io_cmd.rw), sdev->id); - if (tmp_stm != NULL) { - hiraid_io_recognition_iterator(tmp_stm, 1); - if (hiraid_recognition_acknowledge(tmp_stm) && - (hiraid_get_stream_num(sdev->id) > 1)) { - if (hiraid_add_io_to_list(ioq, - tmp_stm, io_cmd, sdev->sector_size, sdev->id)) { + tmp_stream = hiraid_stream_detect(hdev, &(io_cmd.rw), sdev->id); + if (tmp_stream != NULL) { + hiraid_io_recognition_iterator(tmp_stream, 1); + if (hiraid_recognition_acknowledge(tmp_stream) && + (sdev->id < HIRAID_MAX_PD_NUM) && + (hdev->hiraid_stream_io_num_per_pd[sdev->id][HIRAID_STREAM_TYPE_TOTAL] > + 1)) { + if (hiraid_add_io_to_list(hdev, ioq, tmp_stream, + io_cmd, sdev->sector_size, sdev->id)) return 0; - } } } } @@ -1666,6 +1570,7 @@ static int hiraid_slave_configure(struct scsi_device *sdev) static void hiraid_shost_init(struct hiraid_dev *hdev) { struct pci_dev *pdev = hdev->pdev; + struct scsi_host_template *hostt; u8 domain, bus; u32 dev_func; @@ -1674,8 +1579,10 @@ static void hiraid_shost_init(struct hiraid_dev *hdev) dev_func = pdev->devfn; hdev->shost->nr_hw_queues = work_mode ? 1 : hdev->online_queues - 1; - hdev->shost->can_queue = hdev->scsi_qd; - + hdev->shost->can_queue = min(hdev->scsi_qd, + le16_to_cpu(hdev->ctrl_info->max_cmds) - + HIRAID_AQ_DEPTH - + HIRAID_TOTAL_PTCMDS(hdev->online_queues - 1)); hdev->shost->sg_tablesize = le16_to_cpu(hdev->ctrl_info->max_num_sge); /* 512B per sector */ hdev->shost->max_sectors = @@ -1689,7 +1596,13 @@ static void hiraid_shost_init(struct hiraid_dev *hdev) hdev->shost->this_id = -1; hdev->shost->unique_id = (domain << 16) | (bus << 8) | dev_func; hdev->shost->max_cmd_len = MAX_CDB_LEN; - hdev->shost->hostt->cmd_size = hiraid_get_max_cmd_size(hdev); + + hostt = (struct scsi_host_template *)hdev->shost->hostt; + hostt->cmd_size = hiraid_get_max_cmd_size(hdev); + + dev_info(hdev->dev, "nr_hw_queues[%u] can_queue[%u] unique_id[%u] cmd_size[%u]\n", + hdev->shost->nr_hw_queues, hdev->shost->can_queue, + hdev->shost->unique_id, hdev->shost->hostt->cmd_size); } static int hiraid_alloc_queue(struct hiraid_dev *hdev, u16 qid, u16 depth) @@ -2505,6 +2418,14 @@ static int hiraid_get_queue_cnt(struct hiraid_dev *hdev, u32 *cnt) return 0; } +static bool hiraid_pci_is_present(struct hiraid_dev *hdev) +{ + if (!pci_device_is_present(hdev->pdev) || + readl(hdev->bar + HIRAID_REG_VS) == 0Xffffffff) + return false; + return true; +} + static int hiraid_setup_io_queues(struct hiraid_dev *hdev) { struct hiraid_queue *adminq = &hdev->queues[0]; @@ -2579,7 +2500,7 @@ static void hiraid_delete_io_queues(struct hiraid_dev *hdev) u8 opcode = HIRAID_ADMIN_DELETE_SQ; u16 i, pass; - if (!pci_device_is_present(hdev->pdev)) { + if (!hiraid_pci_is_present(hdev)) { dev_err(hdev->dev, "pci_device is not present, skip disable io queues\n"); return; } @@ -2619,7 +2540,7 @@ static void hiraid_disable_admin_queue(struct hiraid_dev *hdev, struct hiraid_queue *adminq = &hdev->queues[0]; u16 start, end; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { if (shutdown) hiraid_shutdown_control(hdev); else @@ -2850,11 +2771,16 @@ static int hiraid_luntarget_sort(const void *l, const void *r) const struct hiraid_dev_info *rn = r; int l_attr = HIRAID_DEV_INFO_ATTR_BOOT(ln->attr); int r_attr = HIRAID_DEV_INFO_ATTR_BOOT(rn->attr); - /* boot first */ if (l_attr != r_attr) return (r_attr - l_attr); + l_attr = HIRAID_DEV_INFO_ATTR_VD(ln->attr); + r_attr = HIRAID_DEV_INFO_ATTR_VD(rn->attr); + /* vd second */ + if (l_attr != r_attr) + return (r_attr - l_attr); + if (ln->channel == rn->channel) return le16_to_cpu(ln->target) - le16_to_cpu(rn->target); @@ -2892,6 +2818,19 @@ static void hiraid_scan_work(struct work_struct *work) if (HIRAID_DEV_INFO_FLAG_VALID(flag)) { if (!HIRAID_DEV_INFO_FLAG_VALID(org_flag)) { + down_write(&hdev->dev_rwsem); + memcpy(&old_dev[i], &dev[i], + sizeof(struct hiraid_dev_info)); + memcpy(&new_dev[count++], &dev[i], + sizeof(struct hiraid_dev_info)); + up_write(&hdev->dev_rwsem); + } else if (HIRAID_DEV_INFO_FLAG_DELETE_OR_ADD(flag, + org_flag)) { + down_write(&hdev->dev_rwsem); + old_dev[i].flag &= 0xfe; + up_write(&hdev->dev_rwsem); + hiraid_delete_device(hdev, &old_dev[i]); + down_write(&hdev->dev_rwsem); memcpy(&old_dev[i], &dev[i], sizeof(struct hiraid_dev_info)); @@ -3044,7 +2983,7 @@ static void hiraid_bsg_buf_unmap(struct hiraid_dev *hdev, struct bsg_job *job) } static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, - struct hiraid_admin_command *cmd) + struct hiraid_admin_command *cmd, u16 qid) { struct hiraid_bsg_request *bsg_req = job->request; struct request *rq = blk_mq_rq_from_pdu(job); @@ -3058,24 +2997,26 @@ static int hiraid_bsg_buf_map(struct hiraid_dev *hdev, struct bsg_job *job, mapbuf->sgl = job->request_payload.sg_list; mapbuf->len = job->request_payload.payload_len; mapbuf->page_cnt = -1; + mapbuf->hiraidq = &hdev->queues[qid]; if (unlikely(mapbuf->sge_cnt == 0)) goto out; - mapbuf->use_sgl = !hiraid_is_prp(hdev, mapbuf->sgl, mapbuf->sge_cnt); - ret = dma_map_sg_attrs(hdev->dev, mapbuf->sgl, mapbuf->sge_cnt, - dma_dir, DMA_ATTR_NO_WARN); + dma_dir, DMA_ATTR_NO_WARN); if (!ret) goto out; - if ((mapbuf->use_sgl == (bool)true) && + mapbuf->use_sgl = !hiraid_is_prp(hdev, mapbuf->sgl, mapbuf->sge_cnt); + + if ((mapbuf->use_sgl == true) && (bsg_req->msgcode == HIRAID_BSG_IOPTHRU) && - (hdev->ctrl_info->pt_use_sgl != (bool)false)) { - ret = hiraid_build_passthru_sgl(hdev, cmd, mapbuf); + (hdev->ctrl_info->pt_use_sgl != false)) { + ret = hiraid_build_sgl(hdev, (struct hiraid_scsi_io_cmd *)cmd, + mapbuf); } else { mapbuf->use_sgl = false; - ret = hiraid_build_passthru_prp(hdev, mapbuf); + ret = hiraid_build_prp(hdev, mapbuf); cmd->common.dptr.prp1 = cpu_to_le64(sg_dma_address(mapbuf->sgl)); cmd->common.dptr.prp2 = cpu_to_le64(mapbuf->first_dma); @@ -3156,9 +3097,7 @@ static int hiraid_init_control_info(struct hiraid_dev *hdev) if (hdev->ctrl_info->asynevent > HIRAID_ASYN_COMMANDS) hdev->ctrl_info->asynevent = HIRAID_ASYN_COMMANDS; - hdev->scsi_qd = work_mode ? - le16_to_cpu(hdev->ctrl_info->max_cmds) : - (hdev->ioq_depth - HIRAID_PTHRU_CMDS_PERQ); + hdev->scsi_qd = hdev->ioq_depth - HIRAID_PTHRU_CMDS_PERQ; return 0; } @@ -3191,7 +3130,7 @@ static int hiraid_user_send_admcmd(struct hiraid_dev *hdev, struct bsg_job *job) admin_cmd.common.cdw14 = cpu_to_le32(ptcmd->cdw14); admin_cmd.common.cdw15 = cpu_to_le32(ptcmd->cdw15); - status = hiraid_bsg_buf_map(hdev, job, &admin_cmd); + status = hiraid_bsg_buf_map(hdev, job, &admin_cmd, 0); if (status) { dev_err(hdev->dev, "err, map data failed\n"); return status; @@ -3251,19 +3190,13 @@ static void hiraid_free_io_ptcmds(struct hiraid_dev *hdev) } static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, - struct hiraid_scsi_io_cmd *io_cmd, - u32 *result, u32 *reslen, u32 timeout) + struct hiraid_scsi_io_cmd *io_cmd, struct hiraid_cmd *pt_cmd, + u32 *result, u32 *reslen, u32 timeout) { int ret; dma_addr_t buffer_phy; struct hiraid_queue *ioq; void *sense_addr = NULL; - struct hiraid_cmd *pt_cmd = hiraid_get_cmd(hdev, HIRAID_CMD_PTHRU); - - if (!pt_cmd) { - dev_err(hdev->dev, "err, get ioq cmd failed\n"); - return -EFAULT; - } timeout = timeout ? timeout : ADMIN_TIMEOUT; @@ -3271,8 +3204,9 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, ioq = &hdev->queues[pt_cmd->qid]; if (work_mode) { - ret = ((pt_cmd->qid - 1) * HIRAID_PTHRU_CMDS_PERQ + - pt_cmd->cid) * SCSI_SENSE_BUFFERSIZE; + ret = ((pt_cmd->qid - 1) * + HIRAID_PTHRU_CMDS_PERQ + pt_cmd->cid) * + SCSI_SENSE_BUFFERSIZE; sense_addr = hdev->sense_buffer_virt + ret; buffer_phy = hdev->sense_buffer_phy + ret; } else { @@ -3293,8 +3227,6 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, (le32_to_cpu(io_cmd->common.cdw3[0]) & 0xffff)); hiraid_admin_timeout(hdev, pt_cmd); - - hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); return -ETIME; } @@ -3305,8 +3237,6 @@ static int hiraid_put_io_sync_request(struct hiraid_dev *hdev, } } - hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); - return pt_cmd->status; } @@ -3316,6 +3246,7 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) (struct hiraid_bsg_request *)(job->request); struct hiraid_passthru_io_cmd *cmd = &(bsg_req->pthrucmd); struct hiraid_scsi_io_cmd pthru_cmd; + struct hiraid_cmd *pt_cmd; int status = 0; u32 timeout = msecs_to_jiffies(cmd->timeout_ms); // data len is 4k before use sgl, now len is 1M @@ -3358,16 +3289,21 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) pthru_cmd.common.cdw26[2] = cpu_to_le32(cmd->cdw26[2]); pthru_cmd.common.cdw26[3] = cpu_to_le32(cmd->cdw26[3]); + pt_cmd = hiraid_get_cmd(hdev, HIRAID_CMD_PTHRU); + if (pt_cmd == NULL) { + dev_err(hdev->dev, "err, get ioq cmd failed\n"); + return -EFAULT; + } + status = hiraid_bsg_buf_map(hdev, job, - (struct hiraid_admin_command *)&pthru_cmd); + (struct hiraid_admin_command *)&pthru_cmd, pt_cmd->qid); if (status) { dev_err(hdev->dev, "err, map data failed\n"); - return status; + goto ret; } - status = hiraid_put_io_sync_request(hdev, &pthru_cmd, job->reply, - &job->reply_len, timeout); - + status = hiraid_put_io_sync_request(hdev, &pthru_cmd, pt_cmd, + job->reply, &job->reply_len, timeout); if (status) dev_info(hdev->dev, "opcode[0x%x] subopcode[0x%x] status[0x%x] replylen[%d]\n", cmd->opcode, cmd->info_1.subopcode, @@ -3375,6 +3311,9 @@ static int hiraid_user_send_ptcmd(struct hiraid_dev *hdev, struct bsg_job *job) hiraid_bsg_buf_unmap(hdev, job); +ret: + hiraid_put_cmd(hdev, pt_cmd, HIRAID_CMD_PTHRU); + return status; } @@ -3396,23 +3335,28 @@ static bool hiraid_check_scmd_finished(struct scsi_cmnd *scmd) return false; } -static enum blk_eh_timer_return hiraid_timed_out(struct scsi_cmnd *scmd) +#define EH_TIMED_RET_TP enum blk_eh_timer_return +#define EH_TIMED_RET_DONE BLK_EH_DONE +#define EH_TIMED_RET_NOT_HANDLED BLK_EH_DONE +#define EH_TIMED_RET_RESET_TIMER BLK_EH_RESET_TIMER +static EH_TIMED_RET_TP hiraid_timed_out(struct scsi_cmnd *scmd) { struct hiraid_mapmange *mapbuf = scsi_cmd_priv(scmd); unsigned int timeout = scmd->device->request_queue->rq_timeout; if (hiraid_check_scmd_finished(scmd)) - goto out; + return EH_TIMED_RET_DONE; - if (time_after(jiffies, scmd->jiffies_at_alloc + timeout)) { + if (time_after_eq(jiffies, scmd->jiffies_at_alloc + timeout)) { + scmd_printk(KERN_WARNING, scmd, "timeout is work[%d], need abort\n", + mapbuf->state); if (cmpxchg(&mapbuf->state, CMD_FLIGHT, CMD_TIMEOUT) == CMD_FLIGHT) - return BLK_EH_DONE; + return EH_TIMED_RET_NOT_HANDLED; } -out: - return BLK_EH_RESET_TIMER; -} + return EH_TIMED_RET_RESET_TIMER; +} /* send abort command by admin queue temporary */ static int hiraid_send_abort_cmd(struct hiraid_dev *hdev, u32 hdid, u16 qid, u16 cid) @@ -3535,7 +3479,7 @@ static int hiraid_dev_disable(struct hiraid_dev *hdev, bool shutdown) struct hiraid_queue *adminq = &hdev->queues[0]; u16 start, end; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { if (shutdown) hiraid_shutdown_control(hdev); else @@ -3733,11 +3677,19 @@ static int hiraid_abort(struct scsi_cmnd *scmd) hiraid_check_scmd_finished(scmd)) return SUCCESS; + if (hdev->state != DEV_LIVE) + return SUCCESS; + hostdata = scmd->device->hostdata; cid = mapbuf->cid; hwq = mapbuf->hiraidq->qid; - if (hiraid_check_io_list(hostdata->hdid, cid, hwq)) { + mutex_lock(&g_stream_operation_mutex); + ret = hiraid_stream_delete_io_list(hdev, hostdata->hdid, + cid, hwq, HIRAID_STREAM_TYPE_DELETE_SINGLE_IO); + mutex_unlock(&g_stream_operation_mutex); + + if (ret == 1) { dev_warn(hdev->dev, "find cid[%d] qid[%d] in host, abort succ\n", cid, hwq); return SUCCESS; @@ -3749,14 +3701,14 @@ static int hiraid_abort(struct scsi_cmnd *scmd) ret = hiraid_wait_io_completion(mapbuf); if (ret) { dev_warn(hdev->dev, "cid[%d] qid[%d] abort failed, not found\n", - cid, hwq); + cid, hwq); return FAILED; } dev_warn(hdev->dev, "cid[%d] qid[%d] abort succ\n", cid, hwq); return SUCCESS; } dev_warn(hdev->dev, "cid[%d] qid[%d] abort failed, timeout\n", - cid, hwq); + cid, hwq); return FAILED; } @@ -3778,7 +3730,10 @@ static int hiraid_scsi_reset(struct scsi_cmnd *scmd, enum hiraid_rst_type rst) if ((ret == 0) || (ret == FW_EH_DEV_NONE && rst == HIRAID_RESET_TARGET)) { if (rst == HIRAID_RESET_TARGET) { - hiraid_delete_single_pd_io_list(hostdata->hdid); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, hostdata->hdid, + 0, 0, HIRAID_STREAM_TYPE_DELETE_SINGLE_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); ret = wait_tgt_reset_io_done(scmd); if (ret) { dev_warn(hdev->dev, "sdev[%d:%d] target has %d peding cmd, target reset failed\n", @@ -3818,9 +3773,13 @@ static int hiraid_host_reset(struct scsi_cmnd *scmd) dev_warn(hdev->dev, "sdev[%d:%d] send host reset\n", scmd->device->channel, scmd->device->id); - hiraid_delete_all_io_list(); - if (hiraid_reset_work_sync(hdev) == -EBUSY) + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + if (hiraid_reset_work_sync(hdev) == -EBUSY) { flush_work(&hdev->reset_work); + } if (hdev->state != DEV_LIVE) { dev_warn(hdev->dev, "sdev[%d:%d] host reset failed\n", @@ -3852,7 +3811,10 @@ static pci_ers_result_t hiraid_pci_error_detected(struct pci_dev *pdev, scsi_block_requests(hdev->shost); hiraid_dev_state_trans(hdev, DEV_RESETTING); - hiraid_delete_all_io_list(); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); return PCI_ERS_RESULT_NEED_RESET; case pci_channel_io_perm_failure: @@ -3888,9 +3850,24 @@ static void hiraid_reset_pci_finish(struct pci_dev *pdev) { struct hiraid_dev *hdev = pci_get_drvdata(pdev); + if (hiraid_reset_work_sync(hdev) == -EBUSY) + flush_work(&hdev->reset_work); + dev_info(hdev->dev, "enter hiraid reset finish\n"); } +static void hiraid_reset_pci_prepare(struct pci_dev *pdev) +{ + struct hiraid_dev *hdev = pci_get_drvdata(pdev); + + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); + msleep(100); + dev_info(hdev->dev, "exit hiraid reset prepare\n"); +} + static ssize_t csts_pp_show(struct device *cdev, struct device_attribute *attr, char *buf) { @@ -3898,7 +3875,7 @@ static ssize_t csts_pp_show(struct device *cdev, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_PP_MASK); ret >>= HIRAID_CSTS_PP_SHIFT; @@ -3914,7 +3891,7 @@ static ssize_t csts_shst_show(struct device *cdev, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_SHST_MASK); ret >>= HIRAID_CSTS_SHST_SHIFT; @@ -3930,7 +3907,7 @@ static ssize_t csts_cfs_show(struct device *cdev, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) { + if (hiraid_pci_is_present(hdev)) { ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_CFS_MASK); ret >>= HIRAID_CSTS_CFS_SHIFT; @@ -3946,7 +3923,7 @@ static ssize_t csts_rdy_show(struct device *cdev, struct hiraid_dev *hdev = shost_priv(shost); int ret = -1; - if (pci_device_is_present(hdev->pdev)) + if (hiraid_pci_is_present(hdev)) ret = (readl(hdev->bar + HIRAID_REG_CSTS) & HIRAID_CSTS_RDY); return snprintf(buf, PAGE_SIZE, "%d\n", ret); @@ -4239,22 +4216,80 @@ static ssize_t dispatch_hwq_store(struct device *dev, return strlen(buf); } +static ssize_t sata_ncq_prio_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hiraid_sdev_hostdata *hostdata; + + hostdata = to_scsi_device(dev)->hostdata; + return snprintf(buf, PAGE_SIZE, "%d\n", hostdata->sata_ncq_prio_enable); +} + +bool scsi_ncq_prio_support(struct scsi_device *sdev, struct hiraid_dev *hdev) +{ + unsigned char *buf; + bool ncq_prio_supp = false; + + if (!scsi_device_supports_vpd(sdev)) { + dev_err(hdev->dev, "scsi device not support vpd\n"); + return ncq_prio_supp; + } + + buf = kzalloc(HIRAID_SCSI_VPD_LEN, GFP_KERNEL); + if (!buf) { + dev_err(hdev->dev, "alloc scsi vpd data memeroy failed\n"); + return ncq_prio_supp; + } + + if (!scsi_get_vpd_page(sdev, 0x89, buf, HIRAID_SCSI_VPD_LEN)) + ncq_prio_supp = (buf[NCQ_PRIO_SUPPORT_BYTE] >> + VPD_NCQ_PRIO_SUPPORT_BIT) & 1; + + kfree(buf); + return ncq_prio_supp; +} + +static ssize_t sata_ncq_prio_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct scsi_device *sdev = to_scsi_device(dev); + struct hiraid_sdev_hostdata *hostdata = sdev->hostdata; + struct hiraid_dev *hdev = shost_priv(sdev->host); + bool ncq_prio_enable = 0; + + if (kstrtobool(buf, &ncq_prio_enable)) + return -EINVAL; + + if (!scsi_ncq_prio_support(sdev, hdev)) { + dev_err(hdev->dev, "sdev[%d:%d] not support ncq prio\n", + sdev->channel, sdev->id); + return -EINVAL; + } + + hostdata->sata_ncq_prio_enable = ncq_prio_enable; + + return strlen(buf); +} + static DEVICE_ATTR_RO(raid_level); static DEVICE_ATTR_RO(raid_state); static DEVICE_ATTR_RO(raid_resync); static DEVICE_ATTR_RW(dispatch_hwq); +static DEVICE_ATTR_RW(sata_ncq_prio_enable); static struct device_attribute *hiraid_dev_attrs[] = { &dev_attr_raid_state, &dev_attr_raid_level, &dev_attr_raid_resync, &dev_attr_dispatch_hwq, + &dev_attr_sata_ncq_prio_enable, NULL, }; static struct pci_error_handlers hiraid_err_handler = { .error_detected = hiraid_pci_error_detected, .slot_reset = hiraid_pci_slot_reset, + .reset_prepare = hiraid_reset_pci_prepare, .reset_done = hiraid_reset_pci_finish, }; @@ -4518,14 +4553,17 @@ static void hiraid_remove(struct pci_dev *pdev) dev_info(hdev->dev, "enter hiraid remove\n"); - kthread_stop(g_hiraid_submit_task); - hiraid_delete_all_io_list(); + kthread_stop(hdev->hiraid_stream_submit_task); + mutex_lock(&g_stream_operation_mutex); + hiraid_stream_delete_io_list(hdev, 0, 0, 0, + HIRAID_STREAM_TYPE_DELETE_ALL_IO_LIST); + mutex_unlock(&g_stream_operation_mutex); hiraid_destroy_io_stream_resource(hdev); hiraid_dev_state_trans(hdev, DEV_DELETING); flush_work(&hdev->reset_work); - if (!pci_device_is_present(pdev)) + if (!hiraid_pci_is_present(hdev)) hiraid_flush_running_cmds(hdev); hiraid_unregist_bsg(hdev); @@ -4548,8 +4586,12 @@ static void hiraid_remove(struct pci_dev *pdev) static const struct pci_device_id hiraid_hw_card_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI_LOGIC, HIRAID_SERVER_DEVICE_HBA_DID) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI_LOGIC, + HIRAID_SERVER_DEVICE_HBAS_DID) }, { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI_LOGIC, HIRAID_SERVER_DEVICE_RAID_DID) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI_LOGIC, + HIRAID_SERVER_DEVICE_RAIDS_DID) }, { 0, } }; MODULE_DEVICE_TABLE(pci, hiraid_hw_card_ids); @@ -4572,22 +4614,9 @@ static int __init hiraid_init(void) if (!work_queue) return -ENOMEM; - hiraid_class = class_create(THIS_MODULE, "hiraid"); - if (IS_ERR(hiraid_class)) { - ret = PTR_ERR(hiraid_class); - goto destroy_wq; - } - ret = pci_register_driver(&hiraid_driver); if (ret < 0) - goto destroy_class; - - return 0; - -destroy_class: - class_destroy(hiraid_class); -destroy_wq: - destroy_workqueue(work_queue); + destroy_workqueue(work_queue); return ret; } @@ -4595,7 +4624,6 @@ destroy_wq: static void __exit hiraid_exit(void) { pci_unregister_driver(&hiraid_driver); - class_destroy(hiraid_class); destroy_workqueue(work_queue); } -- 2.45.1.windows.1

2 1

[PATCH openEuler-1.0-LTS 0/3] *** revert CVE-2026-31527 ***
by Lin Ruifeng 04 Jun '26

04 Jun '26

*** revert CVE-2026-31527 *** Lin Ruifeng (3): Revert "driver/core: Fix kabi broken of platform_device/device/bus_type" Revert "driver core: platform: use generic driver_override infrastructure" Revert "driver core: generalize driver_override in struct device" drivers/base/bus.c | 49 +-------------------- drivers/base/core.c | 3 -- drivers/base/dd.c | 61 --------------------------- drivers/base/platform.c | 57 ++++++++++++++++++++++--- drivers/slimbus/qcom-ngd-ctrl.c | 12 +----- include/linux/device.h | 75 --------------------------------- include/linux/platform_device.h | 7 +-- 7 files changed, 55 insertions(+), 209 deletions(-) -- 2.43.0

2 4

[PATCH OLK-5.10] xfrm: defensively unhash xfrm_state lists in __xfrm_state_delete
by superdcc97＠163.com 04 Jun '26

04 Jun '26

From: Michal Kosiorek <mkosiorek121(a)gmail.com> mainline inclusion from mainline-v7.1-rc3 commit 14acf9652e5690de3c7486c6db5fb8dafd0a32a3 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/15365 CVE: CVE-2026-46116 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?… -------------------------------- KASAN reproduces a slab-use-after-free in __xfrm_state_delete()'s hlist_del_rcu calls under syzkaller load on linux-6.12.y stable (reproduced on 6.12.47, also reachable via the same code path on torvalds/master and on the ipsec tree). Nine unique signatures cluster in the xfrm_state lifecycle, the load-bearing one being: BUG: KASAN: slab-use-after-free in __hlist_del include/linux/list.h:990 [inline] BUG: KASAN: slab-use-after-free in hlist_del_rcu include/linux/rculist.h:516 [inline] BUG: KASAN: slab-use-after-free in __xfrm_state_delete net/xfrm/xfrm_state.c Write of size 8 at addr ffff8881198bcb70 by task kworker/u8:9/435 Workqueue: netns cleanup_net Call Trace: __hlist_del / hlist_del_rcu __xfrm_state_delete xfrm_state_delete xfrm_state_flush xfrm_state_fini ops_exit_list cleanup_net The other observed signatures hit the same slab object from __xfrm_state_lookup, xfrm_alloc_spi, __xfrm_state_insert and an OOB write variant of __xfrm_state_delete, all on the byseq/byspi hash chains. __xfrm_state_delete() guards its byseq and byspi unhashes with value-based predicates: if (x->km.seq) hlist_del_rcu(&x->byseq); if (x->id.spi) hlist_del_rcu(&x->byspi); while everywhere else in the file (e.g. state_cache, state_cache_input) the safer hlist_unhashed() check is used. xfrm_alloc_spi() sets x->id.spi = newspi inside xfrm_state_lock and then immediately inserts into byspi, but a path that observes x->id.spi != 0 outside of xfrm_state_lock can still skip-or-hit the byspi unhash inconsistently with whether x is actually on the list. The same holds for x->km.seq versus byseq, and the bydst/bysrc unhashes have no predicate at all, so a second __xfrm_state_delete() on the same object writes through LIST_POISON pprev. The defensive change here: - Use hlist_del_init_rcu() instead of hlist_del_rcu() on bydst, bysrc, byseq and byspi so a second deletion is a no-op rather than a write through LIST_POISON pprev. The byseq/byspi nodes are already initialised in xfrm_state_alloc(). - Test hlist_unhashed() rather than the value predicate for byseq/byspi, so the unhash decision tracks list state rather than mutable scalar fields. Empirical verification: applied this patch on top of v6.12.47, rebuilt, and re-ran the same syzkaller harness for 1h16m on a previously-crashy configuration that produced ~100 hits each of slab-use-after-free Read in xfrm_alloc_spi / Read in __xfrm_state_lookup / Write in __xfrm_state_delete. After the patch, 7.1M execs across 32 VMs at ~1550 exec/sec produced zero xfrm_state UAF/OOB hits. /proc/slabinfo confirms the xfrm_state slab is actively allocated and freed during the run (~143 KiB resident), so the fuzzer is still exercising those code paths -- they just no longer crash. Reproduction: - Linux 6.12.47 x86_64 + KASAN_GENERIC + KASAN_INLINE + KCOV - syzkaller @ 746545b8b1e4c3a128db8652b340d3df90ce61db - 32 QEMU/KVM VMs x 2 vCPU on AWS c5.metal bare metal - 9 unique signatures collected in ~9h, all within xfrm_state lifecycle Fixes: fe9f1d8779cb ("xfrm: add state hashtable keyed by seq") Fixes: 7b4dc3600e48 ("[XFRM]: Do not add a state whose SPI is zero to the SPI hash.") Reported-by: Michal Kosiorek <mkosiorek121(a)gmail.com> Tested-by: Michal Kosiorek <mkosiorek121(a)gmail.com> Cc: stable(a)vger.kernel.org Signed-off-by: Michal Kosiorek <mkosiorek121(a)gmail.com> Signed-off-by: Steffen Klassert <steffen.klassert(a)secunet.com> Conflicts: net/xfrm/xfrm_state.c [The OLK-5.10 target lacks commit 3f01d9df5408 ("xfrm: insert a new per-socket policy to the per-socket list after the link is established") which introduced state_cache/state_cache_input unhashing in __xfrm_state_delete(). OLK-5.10 also lacks commit fe9f1d8779cb ("xfrm: add state hashtable keyed by seq") which introduced byseq hash and its unhashing. OLK-5.10 also lacks xfrm_nat_keepalive_state_updated() call. Adapted to only apply the defensive unhashing changes to bydst, bysrc, and byspi lists which are present in OLK-5.10.] Signed-off-by: Dong Chenchen <dongchenchen2(a)huawei.com> --- net/xfrm/xfrm_state.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index e3e96606976b..35d26e410738 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -672,10 +672,10 @@ int __xfrm_state_delete(struct xfrm_state *x) x->km.state = XFRM_STATE_DEAD; spin_lock(&net->xfrm.xfrm_state_lock); list_del(&x->km.all); - hlist_del_rcu(&x->bydst); - hlist_del_rcu(&x->bysrc); - if (x->id.spi) - hlist_del_rcu(&x->byspi); + hlist_del_init_rcu(&x->bydst); + hlist_del_init_rcu(&x->bysrc); + if (!hlist_unhashed(&x->byspi)) + hlist_del_init_rcu(&x->byspi); net->xfrm.state_num--; spin_unlock(&net->xfrm.xfrm_state_lock); -- 2.43.0

2 1

[PATCH OLK-6.6 0/4] backport cpufreq patches from linux mainline.
by Lifeng Zheng 04 Jun '26

04 Jun '26

From: Hongye Lin <linhongye(a)h-partners.com> driver inclusion category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/9163 ---------------------------------------------------------------------- Lifeng Zheng (1): cpufreq: conservative: Simplify frequency limit handling Pengjie Zhang (1): arm64: smp: Do not mark secondary CPUs possible under nosmp Viresh Kumar (2): cpufreq: Fix typo in comment cpufreq: Avoid redundant target() calls for unchanged limits arch/arm64/kernel/smp.c | 14 ++++++--- drivers/cpufreq/cpufreq.c | 31 +++++++++++++------ drivers/cpufreq/cpufreq_conservative.c | 12 +------ include/linux/cpufreq.h | 5 ++- 4 files changed, 37 insertions(+), 25 deletions(-) -- 2.33.0

2 5

[PATCH OLK-5.10] RDMA/mlx4: Fix resource leak on error in mlx4_ib_create_srq()
by Chen Jinghuang 04 Jun '26

04 Jun '26

From: Jason Gunthorpe <jgg(a)nvidia.com> stable inclusion from stable-v5.10.258 commit 53fd4c03558672ccb167754fbacbf045c7ab335c category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/15428 CVE: CVE-2026-46178 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id… -------------------------------- commit c54c7e4cb679c0aaa1cb489b9c3f2cd98e63a44c upstream. Sashiko points out that mlx4_srq_alloc() was not undone during error unwind, add the missing call to mlx4_srq_free(). Cc: stable(a)vger.kernel.org Fixes: 225c7b1feef1 ("IB/mlx4: Add a driver Mellanox ConnectX InfiniBand adapters") Link: https://sashiko.dev/#/patchset/0-v1-e911b76a94d1%2B65d95-rdma_udata_rep_jgg… Link: https://patch.msgid.link/r/11-v1-41f3135e5565+9d2-rdma_ai_fixes1_jgg@nvidia… Signed-off-by: Jason Gunthorpe <jgg(a)nvidia.com> Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org> Signed-off-by: Chen Jinghuang <chenjinghuang2(a)huawei.com> --- drivers/infiniband/hw/mlx4/srq.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index bf618529e734..dd509ed35d95 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -189,13 +189,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, if (udata) if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { err = -EFAULT; - goto err_wrid; + goto err_srq; } init_attr->attr.max_wr = srq->msrq.max - 1; return 0; +err_srq: + mlx4_srq_free(dev->dev, &srq->msrq); err_wrid: if (udata) mlx4_ib_db_unmap_user(ucontext, &srq->db); -- 2.34.1

2 2

[PATCH openEuler-1.0-LTS 0/3] *** revert CVE-2026-31527 ***
by Lin Ruifeng 04 Jun '26

04 Jun '26

*** revert CVE-2026-31527 *** Lin Ruifeng (3): Revert "driver/core: Fix kabi broken of platform_device/device/bus_type" Revert "driver core: platform: use generic driver_override infrastructure" Revert "driver core: generalize driver_override in struct device" drivers/base/bus.c | 49 +-------------------- drivers/base/core.c | 3 -- drivers/base/dd.c | 61 --------------------------- drivers/base/platform.c | 57 ++++++++++++++++++++++--- drivers/slimbus/qcom-ngd-ctrl.c | 12 +----- include/linux/device.h | 75 --------------------------------- include/linux/platform_device.h | 7 +-- 7 files changed, 55 insertions(+), 209 deletions(-) -- 2.43.0

2 5

[PATCH OLK-5.10] drm/gem: Fix inconsistent plane dimension calculation in drm_gem_fb_init_with_funcs()
by Hongtao Zhang 04 Jun '26

04 Jun '26

From: Ashutosh Desai <ashutoshdesai993(a)gmail.com> stable inclusion from stable-v6.6.140 commit 6b992591e04f2cce813bcf239b354f375bbf84d3 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/15462 CVE: CVE-2026-46209 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id… -------------------------------- commit 3d4c2268bd7243c3780fe32bf24ff876da272acf upstream. drm_gem_fb_init_with_funcs() computes sub-sampled plane dimensions using plain integer division: unsigned int width = mode_cmd->width / (i ? info->hsub : 1); unsigned int height = mode_cmd->height / (i ? info->vsub : 1); However, the ioctl-level framebuffer_check() in drm_framebuffer.c uses drm_format_info_plane_width/height() which round up dimensions via DIV_ROUND_UP(). This inconsistency corrupts the subsequent GEM object size check for certain pixel format and dimension combinations. For example, with NV12 (vsub=2) and a 1-pixel-tall framebuffer the GEM size validation path sees height=0 instead of height=1. The expression (height - 1) then wraps to UINT_MAX as an unsigned int, causing min_size to overflow and wrap back to a small value. A tiny GEM object therefore passes the size guard, yet when the GPU accesses the chroma plane it will read or write memory beyond the object's bounds. Fix by replacing the open-coded divisions with drm_format_info_plane_width() and drm_format_info_plane_height(), which use DIV_ROUND_UP() and match the calculation already used in framebuffer_check(). Fixes: 4c3dbb2c312c ("drm: Add GEM backed framebuffer library") Cc: stable(a)vger.kernel.org # v4.14+ Reviewed-by: Thomas Zimmermann <tzimmermann(a)suse.de> Signed-off-by: Ashutosh Desai <ashutoshdesai993(a)gmail.com> Signed-off-by: Thomas Zimmermann <tzimmermann(a)suse.de> Link: https://patch.msgid.link/20260420013637.457751-1-ashutoshdesai993@gmail.com Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org> Conflicts: drivers/gpu/drm/drm_gem_framebuffer_helper.c [Not merge bf39607c1614 and f2b816d78a94] Signed-off-by: Zhang Hongtao <zhanghongtao35(a)huawei.com> --- drivers/gpu/drm/drm_gem_framebuffer_helper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c index 109d11fb4cd4..1253e7815d9b 100644 --- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c +++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c @@ -159,8 +159,8 @@ int drm_gem_fb_init_with_funcs(struct drm_device *dev, return -EINVAL; for (i = 0; i < info->num_planes; i++) { - unsigned int width = mode_cmd->width / (i ? info->hsub : 1); - unsigned int height = mode_cmd->height / (i ? info->vsub : 1); + unsigned int width = drm_format_info_plane_width(info, mode_cmd->width, i); + unsigned int height = drm_format_info_plane_height(info, mode_cmd->height, i); unsigned int min_size; objs[i] = drm_gem_object_lookup(file, mode_cmd->handles[i]); -- 2.43.0

2 3

[PATCH OLK-5.10] ipv6: xfrm6: release dst on error in xfrm6_rcv_encap()
by superdcc97＠163.com 04 Jun '26

04 Jun '26

From: Yilin Zhu <zylzyl2333(a)gmail.com> mainline inclusion from mainline-v7.1-rc3 commit bc0fcb9823cd0894934cf968b525c575833d7078 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/15406 CVE: CVE-2026-46172 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?… -------------------------------- xfrm6_rcv_encap() performs an IPv6 route lookup when the skb does not already have a dst attached. ip6_route_input_lookup() returns a referenced dst entry even when the lookup resolves to an error route. If dst->error is set, xfrm6_rcv_encap() drops the skb without attaching the dst to the skb and without releasing the reference returned by the lookup. Repeated packets hitting this path therefore leak dst entries. Release the dst before jumping to the drop path. Fixes: 0146dca70b87 ("xfrm: add support for UDPv6 encapsulation of ESP") Cc: stable(a)kernel.org Reported-by: Yifan Wu <yifanwucs(a)gmail.com> Reported-by: Juefei Pu <tomapufckgml(a)gmail.com> Co-developed-by: Yuan Tan <yuantan098(a)gmail.com> Signed-off-by: Yuan Tan <yuantan098(a)gmail.com> Suggested-by: Xin Liu <bird(a)lzu.edu.cn> Tested-by: Ruide Cao <caoruide123(a)gmail.com> Signed-off-by: Yilin Zhu <zylzyl2333(a)gmail.com> Signed-off-by: Ren Wei <n05ec(a)lzu.edu.cn> Reviewed-by: Simon Horman <horms(a)kernel.org> Signed-off-by: Steffen Klassert <steffen.klassert(a)secunet.com> Signed-off-by: Dong Chenchen <dongchenchen2(a)huawei.com> --- net/ipv6/xfrm6_protocol.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/ipv6/xfrm6_protocol.c b/net/ipv6/xfrm6_protocol.c index ea2f805d3b01..9b586fcec485 100644 --- a/net/ipv6/xfrm6_protocol.c +++ b/net/ipv6/xfrm6_protocol.c @@ -88,8 +88,10 @@ int xfrm6_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi, dst = ip6_route_input_lookup(dev_net(skb->dev), skb->dev, &fl6, skb, flags); - if (dst->error) + if (dst->error) { + dst_release(dst); goto drop; + } skb_dst_set(skb, dst); } -- 2.43.0

2 1