- Kernel - mailweb.openeuler.org

[PATCH OLK-5.10] Bluetooth: L2CAP: Fix use-after-free in l2cap_sock_new_connection_cb()
by Chen Jinghuang 03 Aug '26

03 Aug '26

From: Siwei Zhang <oss(a)fourdim.xyz> mainline inclusion from mainline-v7.2-rc3 commit 6fef032af0092ed5ccb767239a9ac1bc38c08a40 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16859 CVE: CVE-2026-64557 Reference: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/comm… -------------------------------- l2cap_sock_new_connection_cb() returned l2cap_pi(sk)->chan after release_sock(parent). Once the parent lock is dropped the newly enqueued child socket sk is reachable via the accept queue, so another task can accept and free it before the callback dereferences sk, resulting in a use-after-free. Rework the ->new_connection() op so the core, rather than the callback, owns the child channel's lifetime. The op now receives a pre-allocated new_chan and returns an errno instead of allocating and returning a channel. l2cap_new_connection() allocates the child channel and links it into the conn list via __l2cap_chan_add() before invoking the callback, so the conn-list reference keeps the channel alive once release_sock(parent) exposes the socket to other tasks. Channel configuration that was duplicated in l2cap_sock_init() and the various new_connection callbacks is consolidated into l2cap_chan_set_defaults(), which now inherits from the parent channel when one is supplied. Fixes: 8ffb929098a5 ("Bluetooth: Remove parent socket usage from l2cap_core.c") Cc: stable(a)kernel.org Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Siwei Zhang <oss(a)fourdim.xyz> Signed-off-by: Luiz Augusto von Dentz <luiz.von.dentz(a)intel.com> Conflicts: include/net/bluetooth/l2cap.h net/bluetooth/6lowpan.c net/bluetooth/l2cap_sock.c net/bluetooth/l2cap_core.c [context conflicts, not merged 0a120d961663("Bluetooth: L2CAP: Fix null-ptr-deref in l2cap_sock_new_connection_cb()")] Signed-off-by: Chen Jinghuang <chenjinghuang2(a)huawei.com> --- include/net/bluetooth/l2cap.h | 10 ++-- net/bluetooth/6lowpan.c | 18 +----- net/bluetooth/l2cap_core.c | 78 ++++++++++++++++++++----- net/bluetooth/l2cap_sock.c | 103 ++++++++++++++++------------------ net/bluetooth/smp.c | 27 ++------- 5 files changed, 126 insertions(+), 110 deletions(-) diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 678394d95321..541c3c5c6079 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -650,7 +650,8 @@ struct l2cap_chan { struct l2cap_ops { char *name; - struct l2cap_chan *(*new_connection) (struct l2cap_chan *chan); + int (*new_connection)(struct l2cap_chan *chan, + struct l2cap_chan *new_chan); int (*recv) (struct l2cap_chan * chan, struct sk_buff *skb); void (*teardown) (struct l2cap_chan *chan, int err); @@ -909,9 +910,10 @@ static inline __u16 __next_seq(struct l2cap_chan *chan, __u16 seq) return (seq + 1) % (chan->tx_win_max + 1); } -static inline struct l2cap_chan *l2cap_chan_no_new_connection(struct l2cap_chan *chan) +static inline int l2cap_chan_no_new_connection(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - return NULL; + return -EOPNOTSUPP; } static inline int l2cap_chan_no_recv(struct l2cap_chan *chan, struct sk_buff *skb) @@ -986,7 +988,7 @@ int l2cap_chan_reconfigure(struct l2cap_chan *chan, __u16 mtu); int l2cap_chan_send(struct l2cap_chan *chan, struct msghdr *msg, size_t len); void l2cap_chan_busy(struct l2cap_chan *chan, int busy); int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator); -void l2cap_chan_set_defaults(struct l2cap_chan *chan); +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan); int l2cap_ertm_init(struct l2cap_chan *chan); void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index 7601ce9143c1..6ee874654794 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -650,7 +650,7 @@ static struct l2cap_chan *chan_create(void) if (!chan) return NULL; - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); chan->chan_type = L2CAP_CHAN_CONN_ORIENTED; chan->mode = L2CAP_MODE_LE_FLOWCTL; @@ -762,21 +762,6 @@ static inline void chan_ready_cb(struct l2cap_chan *chan) ifup(dev->netdev); } -static inline struct l2cap_chan *chan_new_conn_cb(struct l2cap_chan *pchan) -{ - struct l2cap_chan *chan; - - chan = chan_create(); - if (!chan) - return NULL; - - chan->ops = pchan->ops; - - BT_DBG("chan %p pchan %p", chan, pchan); - - return chan; -} - static void delete_netdev(struct work_struct *work) { struct lowpan_btle_dev *entry = container_of(work, @@ -895,7 +880,6 @@ static long chan_get_sndtimeo_cb(struct l2cap_chan *chan) static const struct l2cap_ops bt_6lowpan_chan_ops = { .name = "L2CAP 6LoWPAN channel", - .new_connection = chan_new_conn_cb, .recv = chan_recv_cb, .close = chan_close_cb, .state_change = chan_state_change_cb, diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 3b6e354551c1..b7b4e24f78e1 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -537,7 +537,10 @@ void l2cap_chan_put(struct l2cap_chan *c) } EXPORT_SYMBOL_GPL(l2cap_chan_put); -void l2cap_chan_set_defaults(struct l2cap_chan *chan) +/* Initialise @chan with default values, inheriting from the parent channel + * @pchan when it is given. + */ +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan) { chan->fcs = L2CAP_FCS_CRC16; chan->max_tx = L2CAP_DEFAULT_MAX_TX; @@ -551,6 +554,31 @@ void l2cap_chan_set_defaults(struct l2cap_chan *chan) chan->retrans_timeout = L2CAP_DEFAULT_RETRANS_TO; chan->monitor_timeout = L2CAP_DEFAULT_MONITOR_TO; + if (pchan) { + BT_DBG("chan %p pchan %p", chan, pchan); + + chan->chan_type = pchan->chan_type; + chan->imtu = pchan->imtu; + chan->omtu = pchan->omtu; + chan->mode = pchan->mode; + chan->fcs = pchan->fcs; + chan->max_tx = pchan->max_tx; + chan->tx_win = pchan->tx_win; + chan->tx_win_max = pchan->tx_win_max; + chan->sec_level = pchan->sec_level; + chan->conf_state = pchan->conf_state; + chan->flags = pchan->flags; + chan->tx_credits = pchan->tx_credits; + chan->rx_credits = pchan->rx_credits; + + if (chan->chan_type == L2CAP_CHAN_FIXED) { + chan->scid = pchan->scid; + chan->dcid = pchan->scid; + } + + return; + } + chan->conf_state = 0; set_bit(CONF_NOT_COMPLETE, &chan->conf_state); @@ -4126,6 +4154,38 @@ static inline int l2cap_command_rej(struct l2cap_conn *conn, return 0; } +/* Allocate and initialise a channel for an incoming connection. + * + * The channel inherits its configuration from @pchan and is linked into @conn + * before ->new_connection() runs, so the conn list reference keeps it alive if + * the callback exposes it (e.g. via the socket accept queue) before this + * returns. The l2cap_chan_create() reference is taken over by the subsystem on + * success and dropped here on failure. + */ +static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn, + struct l2cap_chan *pchan) +{ + struct l2cap_chan *chan; + + chan = l2cap_chan_create(); + if (!chan) + return NULL; + + l2cap_chan_set_defaults(chan, pchan); + chan->ops = pchan->ops; + + __l2cap_chan_add(conn, chan); + + if (pchan->ops->new_connection && + pchan->ops->new_connection(pchan, chan) < 0) { + l2cap_chan_del(chan, 0); + l2cap_chan_put(chan); + return NULL; + } + + return chan; +} + static struct l2cap_chan *l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u8 *data, u8 rsp_code, u8 amp_id) @@ -4173,7 +4233,7 @@ static struct l2cap_chan *l2cap_connect(struct l2cap_conn *conn, goto response; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) goto response; @@ -4192,8 +4252,6 @@ static struct l2cap_chan *l2cap_connect(struct l2cap_conn *conn, chan->dcid = scid; chan->local_amp_id = amp_id; - __l2cap_chan_add(conn, chan); - dcid = chan->scid; __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); @@ -5881,7 +5939,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; goto response_unlock; @@ -5896,8 +5954,6 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_le_flowctl_init(chan, __le16_to_cpu(req->credits)); dcid = chan->scid; @@ -6099,7 +6155,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, continue; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; continue; @@ -6114,8 +6170,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_ecred_init(chan, __le16_to_cpu(req->credits)); /* Init response */ @@ -8352,14 +8406,12 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status) goto next; l2cap_chan_lock(pchan); - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (chan) { bacpy(&chan->src, &hcon->src); bacpy(&chan->dst, &hcon->dst); chan->src_type = bdaddr_src_type(hcon); chan->dst_type = dst_type; - - __l2cap_chan_add(conn, chan); } l2cap_chan_unlock(pchan); diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 644ec4bf06f0..cc06be98f3c0 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -44,7 +44,8 @@ static struct bt_sock_list l2cap_sk_list = { static const struct proto_ops l2cap_sock_ops; static void l2cap_sock_init(struct sock *sk, struct sock *parent); static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern); + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan); static void l2cap_sock_cleanup_listen(struct sock *parent); bool l2cap_is_socket(struct socket *sock) @@ -1215,6 +1216,23 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg, return err; } +/* Release the sock's ref on chan and clear the pointer so that the ref is + * dropped exactly once even if both l2cap_sock_kill() and + * l2cap_sock_destruct() run. Setting chan->data to NULL first stops any other + * task from dereferencing the now-dead sock pointer. + */ +static void l2cap_sock_put_chan(struct sock *sk) +{ + struct l2cap_chan *chan = l2cap_pi(sk)->chan; + + if (!chan) + return; + + chan->data = NULL; + l2cap_pi(sk)->chan = NULL; + l2cap_chan_put(chan); +} + /* Kill socket (only if zapped and orphan) * Must be called on unlocked socket, with l2cap channel lock. */ @@ -1225,13 +1243,9 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); - /* Sock is dead, so set chan data to NULL, avoid other task use invalid - * sock pointer. - */ - l2cap_pi(sk)->chan->data = NULL; - /* Kill poor orphan */ + l2cap_sock_put_chan(sk); - l2cap_chan_put(l2cap_pi(sk)->chan); + /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); sock_put(sk); } @@ -1473,7 +1487,8 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) } } -static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) +static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { struct sock *sk, *parent = chan->data; @@ -1483,25 +1498,28 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); release_sock(parent); - return NULL; + return -ENOBUFS; } sk = l2cap_sock_alloc(sock_net(parent), NULL, BTPROTO_L2CAP, - GFP_ATOMIC, 0); + GFP_ATOMIC, 0, new_chan); if (!sk) { release_sock(parent); - return NULL; - } + return -ENOMEM; + } bt_sock_reclassify_lock(sk, BTPROTO_L2CAP); l2cap_sock_init(sk, parent); + /* The conn list reference taken by l2cap_new_connection() keeps new_chan + * alive once release_sock() lets another task free this socket. + */ bt_accept_enqueue(parent, sk, false); release_sock(parent); - return l2cap_pi(sk)->chan; + return 0; } static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) @@ -1770,10 +1788,7 @@ static void l2cap_sock_destruct(struct sock *sk) { BT_DBG("sk %p", sk); - if (l2cap_pi(sk)->chan) { - l2cap_pi(sk)->chan->data = NULL; - l2cap_chan_put(l2cap_pi(sk)->chan); - } + l2cap_sock_put_chan(sk); if (l2cap_pi(sk)->rx_busy_skb) { kfree_skb(l2cap_pi(sk)->rx_busy_skb); @@ -1804,30 +1819,12 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) BT_DBG("sk %p", sk); if (parent) { - struct l2cap_chan *pchan = l2cap_pi(parent)->chan; - sk->sk_type = parent->sk_type; bt_sk(sk)->flags = bt_sk(parent)->flags; - chan->chan_type = pchan->chan_type; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->conf_state = pchan->conf_state; - chan->mode = pchan->mode; - chan->fcs = pchan->fcs; - chan->max_tx = pchan->max_tx; - chan->tx_win = pchan->tx_win; - chan->tx_win_max = pchan->tx_win_max; - chan->sec_level = pchan->sec_level; - chan->flags = pchan->flags; - chan->tx_credits = pchan->tx_credits; - chan->rx_credits = pchan->rx_credits; - - if (chan->chan_type == L2CAP_CHAN_FIXED) { - chan->scid = pchan->scid; - chan->dcid = pchan->scid; - } - + /* Channel configuration is inherited from the parent by + * l2cap_new_connection(). + */ security_sk_clone(parent, sk); } else { switch (sk->sk_type) { @@ -1853,7 +1850,7 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) chan->mode = L2CAP_MODE_BASIC; } - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); } /* Default config options */ @@ -1870,10 +1867,10 @@ static struct proto l2cap_proto = { }; static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern) + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan) { struct sock *sk; - struct l2cap_chan *chan; sk = bt_sock_alloc(net, sock, &l2cap_proto, proto, prio, kern); if (!sk) @@ -1882,18 +1879,7 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, sk->sk_destruct = l2cap_sock_destruct; sk->sk_sndtimeo = L2CAP_CONN_TIMEOUT; - chan = l2cap_chan_create(); - if (!chan) { - sk_free(sk); - if (sock) - sock->sk = NULL; - return NULL; - } - - l2cap_chan_hold(chan); - - l2cap_pi(sk)->chan = chan; - + /* The sock takes ownership of the caller's reference on chan. */ return sk; } @@ -1901,6 +1887,7 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, int kern) { struct sock *sk; + struct l2cap_chan *chan; BT_DBG("sock %p", sock); @@ -1915,10 +1902,16 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, sock->ops = &l2cap_sock_ops; - sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern); - if (!sk) + chan = l2cap_chan_create(); + if (!chan) return -ENOMEM; + sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern, chan); + if (!sk) { + l2cap_chan_put(chan); + return -ENOMEM; + } + l2cap_sock_init(sk, NULL); bt_sock_link(&l2cap_sk_list, sk); return 0; diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 081b08c3a6bc..e556550eb139 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -3236,34 +3236,19 @@ static const struct l2cap_ops smp_chan_ops = { .get_sndtimeo = l2cap_chan_no_get_sndtimeo, }; -static inline struct l2cap_chan *smp_new_conn_cb(struct l2cap_chan *pchan) +static inline int smp_new_conn_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - struct l2cap_chan *chan; - - BT_DBG("pchan %p", pchan); - - chan = l2cap_chan_create(); - if (!chan) - return NULL; - - chan->chan_type = pchan->chan_type; - chan->ops = &smp_chan_ops; - chan->scid = pchan->scid; - chan->dcid = chan->scid; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->mode = pchan->mode; + new_chan->ops = &smp_chan_ops; /* Other L2CAP channels may request SMP routines in order to * change the security level. This means that the SMP channel * lock must be considered in its own category to avoid lockdep * warnings. */ - atomic_set(&chan->nesting, L2CAP_NESTING_SMP); - - BT_DBG("created chan %p", chan); + atomic_set(&new_chan->nesting, L2CAP_NESTING_SMP); - return chan; + return 0; } static const struct l2cap_ops smp_root_chan_ops = { @@ -3334,7 +3319,7 @@ static struct l2cap_chan *smp_add_cid(struct hci_dev *hdev, u16 cid) l2cap_add_scid(chan, cid); - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); if (cid == L2CAP_CID_SMP) { u8 bdaddr_type; -- 2.34.1

2 1

[PATCH OLK-6.6] infiniband/hroce5: Add hiroce5 driver
by Maogangpin 03 Aug '26

03 Aug '26

From: maogangpin <maogangpin(a)huawei.com<mailto:maogangpin@huawei.com>> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9708 CVE: NA --------------------------------- Add hiroce5 driver to support hi1825 chip. Signed-off-by: maogangpin <maogangpin(a)huawei.com<mailto:maogangpin@huawei.com>> Reviewed-by: Qiujun <roland.qiu(a)huawei.com<mailto:roland.qiu@huawei.com>> Reviewed-by: chiqijun <chiqijun(a)huawei.com<mailto:chiqijun@huawei.com>> --- .../hw/hiroce5/drv_intf/roce_drv_msg_intf.h | 14 + .../hw/hiroce5/kernel/CMakeLists.txt | 62 + drivers/infiniband/hw/hiroce5/kernel/Makefile | 543 ++++ .../hw/hiroce5/kernel/base/cq/roce_cq.h | 230 ++ .../hiroce5/kernel/base/cq/roce_cq_common.c | 299 ++ .../hw/hiroce5/kernel/base/cq/roce_cq_cqe.c | 763 ++++++ .../hiroce5/kernel/base/cq/roce_cq_create.c | 636 +++++ .../hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c | 964 +++++++ .../hiroce5/kernel/base/cq/roce_cq_destroy.c | 304 +++ .../hiroce5/kernel/base/cq/roce_cq_format.h | 29 + .../hiroce5/kernel/base/cq/roce_cq_intern.c | 98 + .../hiroce5/kernel/base/cq/roce_cq_intern.h | 24 + .../hw/hiroce5/kernel/base/event/roce_event.c | 437 +++ .../hw/hiroce5/kernel/base/event/roce_event.h | 25 + .../base/extension/roce_event_extension.c | 56 + .../base/extension/roce_main_extension.c | 170 ++ .../kernel/base/extension/roce_mr_extension.c | 40 + .../kernel/base/extension/roce_qp_extension.c | 222 ++ .../base/extension/roce_srq_extension.c | 55 + .../hw/hiroce5/kernel/base/mr/roce_mr.c | 1281 +++++++++ .../hw/hiroce5/kernel/base/mr/roce_mr.h | 113 + .../hw/hiroce5/kernel/base/qp/roce_post.h | 171 ++ .../hw/hiroce5/kernel/base/qp/roce_qp.h | 317 +++ .../hiroce5/kernel/base/qp/roce_qp_create.c | 1506 +++++++++++ .../kernel/base/qp/roce_qp_ctx_format.h | 278 ++ .../hiroce5/kernel/base/qp/roce_qp_destroy.c | 342 +++ .../hw/hiroce5/kernel/base/qp/roce_qp_exp.h | 80 + .../hw/hiroce5/kernel/base/qp/roce_qp_ext.c | 39 + .../hw/hiroce5/kernel/base/qp/roce_qp_ext.h | 19 + .../hw/hiroce5/kernel/base/qp/roce_qp_inner.h | 17 + .../hiroce5/kernel/base/qp/roce_qp_modify.c | 2398 +++++++++++++++++ .../kernel/base/qp/roce_qp_post_recv.c | 238 ++ .../kernel/base/qp/roce_qp_post_send.c | 1205 +++++++++ .../hw/hiroce5/kernel/base/qp/roce_qp_query.c | 543 ++++ .../hw/hiroce5/kernel/base/rdma/rdma_comp.c | 22 + .../hw/hiroce5/kernel/base/rdma/rdma_comp.h | 166 ++ .../hiroce5/kernel/base/rdma/rdma_comp_gid.c | 489 ++++ .../hiroce5/kernel/base/rdma/rdma_comp_init.c | 443 +++ .../kernel/base/rdma/rdma_comp_mw_mr.c | 120 + .../kernel/base/rdma/rdma_comp_mw_mr.h | 12 + .../hiroce5/kernel/base/rdma/rdma_comp_pd.c | 51 + .../hiroce5/kernel/base/rdma/rdma_comp_res.c | 140 + .../hw/hiroce5/kernel/base/rdma/roce_com.h | 78 + .../hw/hiroce5/kernel/base/rdma/roce_mix.c | 1594 +++++++++++ .../hw/hiroce5/kernel/base/rdma/roce_mix.h | 269 ++ .../hw/hiroce5/kernel/base/rdma/roce_pd.c | 138 + .../hw/hiroce5/kernel/base/rdma/roce_pd.h | 15 + .../hw/hiroce5/kernel/base/srq/roce_srq.h | 191 ++ .../hiroce5/kernel/base/srq/roce_srq_comm.c | 116 + .../hiroce5/kernel/base/srq/roce_srq_create.c | 712 +++++ .../hiroce5/kernel/base/srq/roce_srq_ctrl.c | 635 +++++ .../hiroce5/kernel/base/srq/roce_srq_format.h | 36 + .../hw/hiroce5/kernel/base/uld/roce_uld_api.c | 729 +++++ .../hiroce5/kernel/base/uld/roce_uld_inner.h | 77 + .../hw/hiroce5/kernel/base/xrc/roce_xrc.c | 169 ++ .../hw/hiroce5/kernel/base/xrc/roce_xrc.h | 17 + .../hw/hiroce5/kernel/dev/card/roce_card.c | 162 ++ .../hw/hiroce5/kernel/dev/card/roce_card.h | 27 + .../hw/hiroce5/kernel/dev/cmd/roce_cmd.c | 1270 +++++++++ .../hw/hiroce5/kernel/dev/cmd/roce_cmd.h | 82 + .../hw/hiroce5/kernel/dev/db/roce_db.c | 86 + .../hw/hiroce5/kernel/dev/db/roce_db.h | 30 + .../hw/hiroce5/kernel/dev/init/roce_main.c | 1701 ++++++++++++ .../hiroce5/kernel/dev/netdev/roce_netdev.c | 1082 ++++++++ .../hiroce5/kernel/dev/netdev/roce_netdev.h | 70 + .../hiroce5/kernel/dev/netlink/roce_netlink.c | 413 +++ .../hiroce5/kernel/dev/netlink/roce_netlink.h | 53 + .../kernel/dev/netlink/roce_netlink_process.c | 832 ++++++ .../kernel/dev/netlink/roce_netlink_process.h | 133 + .../hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c | 74 + .../hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h | 21 + .../kernel/dev/sysfs/roce_multipath_sysfs.c | 240 ++ .../kernel/dev/sysfs/roce_multipath_sysfs.h | 30 + .../hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c | 2157 +++++++++++++++ .../hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h | 241 ++ .../hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c | 472 ++++ .../hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h | 73 + .../extern/chip_ext/roce_main_chip_ext.c | 13 + .../extern/chip_ext/roce_main_chip_ext.h | 17 + .../kernel/extern/chip_ext/roce_mr_chip_ext.c | 348 +++ .../kernel/extern/chip_ext/roce_mr_chip_ext.h | 24 + .../kernel/extern/chip_ext/roce_qp_chip_ext.c | 44 + .../extern/chip_ext/roce_srq_chip_ext.c | 26 + .../extern/hyper_roce/hyper_roce_common.h | 13 + .../extern/hyper_roce/hyper_roce_netlink.c | 511 ++++ .../extern/hyper_roce/hyper_roce_netlink.h | 82 + .../kernel/extern/hyper_roce/hyper_roce_qp.c | 29 + .../kernel/extern/hyper_roce/hyper_roce_qp.h | 35 + .../extern/hyper_roce/tool/hyper_roce_tool.c | 60 + .../extern/hyper_roce/tool/hyper_roce_tool.h | 21 + .../hyper_roce/tool/hyper_roce_tool_pktdrop.c | 142 + .../hyper_roce/tool/hyper_roce_tool_pktdrop.h | 23 + .../hw/hiroce5/kernel/include/roce.h | 665 +++++ .../hw/hiroce5/kernel/include/roce_compat.h | 87 + .../kernel/include/roce_event_extension.h | 17 + .../hw/hiroce5/kernel/include/roce_gid.h | 48 + .../hw/hiroce5/kernel/include/roce_infra.h | 77 + .../kernel/include/roce_kernel_compat_gen.h | 6 + .../kernel/include/roce_main_extension.h | 77 + .../kernel/include/roce_mr_extension.h | 14 + .../hiroce5/kernel/include/roce_os_compat.h | 237 ++ .../kernel/include/roce_qp_extension.h | 61 + .../hw/hiroce5/kernel/include/roce_restore.h | 88 + .../kernel/include/roce_srq_extension.h | 38 + .../hw/hiroce5/kernel/include/roce_user.h | 100 + .../kernel/include/roce_verbs_compat.h | 24 + .../hw/hiroce5/kernel/util/bond/roce_bond.h | 79 + .../hiroce5/kernel/util/bond/roce_bond_cfm.c | 490 ++++ .../kernel/util/bond/roce_bond_device.c | 490 ++++ .../kernel/util/bond/roce_bond_internal.h | 97 + .../hiroce5/kernel/util/bond/roce_bond_mac.c | 260 ++ .../kernel/util/bond/roce_bond_slave.c | 459 ++++ .../hiroce5/kernel/util/bond/roce_bond_tx.c | 94 + .../hw/hiroce5/kernel/util/debug/roce_debug.h | 25 + .../hw/hiroce5/kernel/util/dfx/roce_dfx.c | 375 +++ .../hw/hiroce5/kernel/util/dfx/roce_dfx.h | 378 +++ .../hiroce5/kernel/util/dfx/roce_dfx_attack.c | 828 ++++++ .../hiroce5/kernel/util/dfx/roce_dfx_debug.c | 35 + .../hiroce5/kernel/util/dfx/roce_dfx_query.c | 851 ++++++ .../kernel/util/obj_ext/roce_obj_ext.c | 83 + .../kernel/util/obj_ext/roce_obj_ext.h | 40 + .../hiroce5/kernel/util/option/roce_option.c | 125 + .../hiroce5/kernel/util/option/roce_option.h | 30 + 123 files changed, 36652 insertions(+) create mode 100644 drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt create mode 100644 drivers/infiniband/hw/hiroce5/kernel/Makefile create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c create mode 100644 drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h diff --git a/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h b/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h new file mode 100644 index 000000000..4875a1f1a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/drv_intf/roce_drv_msg_intf.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +#ifndef ROCE_DRV_MSG_INTF_H +#define ROCE_DRV_MSG_INTF_H + +#include "udk_usrnl.h" + +typedef struct { + usrnl_drv_msg_hdr_s common_hdr; + unsigned long long card_id; +} roce5_drv_msg_hdr_s; + +#endif /* ROCE_DRV_MSG_INTF_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt b/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt new file mode 100644 index 000000000..e67756615 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/CMakeLists.txt @@ -0,0 +1,62 @@ +option(ENABLE_DEBUG "Enable debug in RoCE kernel driver" OFF) +option(ENABLE_HYPER_ROCE "Enable hyper roce" OFF) + +if("${BUILD_VERSION}" MATCHES "_debug$") + set(ENABLE_DEBUG ON) +endif() + +# ubx kernel hyper_roce使能宏 +if (DEFINED ENV{build_roce_mode}) + if ($ENV{build_roce_mode} STREQUAL "hyperroce") + set(ENABLE_HYPER_ROCE ON) + endif() +endif() + +if("${BUILD_VERSION}" MATCHES "ub_ascend") + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core-v1) +elseif("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(UBUS_DIR ${TOP_DIR}/drivers/ubus) + set(KDIR ${KERNEL_WORK_DIR}/../linux-4.19) + set(UMMU_CORE_BUILD_DIR ${UBUS_DIR}/kernel/ummu-core-v1) +else() + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core) +endif() + +set(UBUS_UBC_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ubus) +set(HISDK5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/sdk/knldk/lld) +set(HINIC5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/nic/linux) +set(HIROCE5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/roce/linux/kernel) + +if("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(hiroce5_depends hisdk hinic) + add_device_ko(LOCAL_MODULE hiroce5 + KO_SRC_FOLDER ${CMAKE_CURRENT_SOURCE_DIR} + MAKE_ARGS "IS_ASCEND=true" + TARGETE_DPENDS "${hiroce5_depends}") +else() + add_custom_target(hiroce5_ko + COMMENT echo "build ${CMAKE_CURRENT_SOURCE_DIR} start." + COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/Makefile ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${MAKE} -j64 + -C ${KDIR} + M=${CMAKE_CURRENT_BINARY_DIR} + src=${CMAKE_CURRENT_SOURCE_DIR} + UBUS_UBC_BUILD_DIR=${UBUS_UBC_BUILD_DIR} + HI1823_TRUNK_DIR=${TOP_DIR}/ChipSolution + HI1823_BUILD_DIR=${TOP_DIR}/ChipSolution/build + CONFIG_UBUS_DEVICE=y + HI1823_OS_TYPE=openEuler24.03 + VERBOSE=1 + HISDK5_SYMVERS=${HISDK5_BUILD_DIR}/Module.symvers + HINIC5_SYMVERS=${HINIC5_BUILD_DIR}/Module.symvers + CHIP_VERSION=hi1825v100 + VERSION=ROCE_STANDARD + ENABLE_DEBUG=${ENABLE_DEBUG} + ENABLE_HYPER_ROCE=${ENABLE_HYPER_ROCE} + COMMAND cp -f *.ko ${CMAKE_INSTALL_PREFIX}/ko + DEPENDS kernel + ) + add_dependencies(hiroce5_ko hisdk5_ko) + add_dependencies(hiroce5_ko hinic5_ko) + add_dependencies(hiroce5_ko ubus_ko) +endif() \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/Makefile b/drivers/infiniband/hw/hiroce5/kernel/Makefile new file mode 100644 index 000000000..0527b1b8a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/Makefile @@ -0,0 +1,543 @@ +ifeq ($(KERNELRELEASE), ) + +.PHONY: all kernel install_kernel install clean clean_kernel + +CWD=$(shell pwd) +TOP_DIR=$(CWD)/../../../../../../.. +INSTALL_MOD_PATH=$(TOP_DIR)/ci/bin + +KERN_VERSION=$(shell uname -r) +KSRC=/lib/modules/$(shell uname -r)/build +ifeq ($(CONFIG_UB_UNIFIED_UBUS), y) + KERN_VERSION ?= 6.6.0-132.0.0.111.oe2403sp3.aarch64 + KSRC=${UBUS_BUILD_KERNEL_DIR} +endif + +ENABLE_DEBUG ?= OFF +ENABLE_HYPER_ROCE ?= ON + +PANGEA_EULER_2107=4.19.90-vhulk2107.1.0.h699.eulerosv2r10.aarch64 +PANGEA_EULER_2108=4.19.90-vhulk2108.6.0.h832.eulerosv2r10.aarch64 +PANGEA_EULER_2202=4.19.90-vhulk2202.2.0.h1064.eulerosv2r10.aarch64 +EULER_V2R12_H1032=5.10.0-136.12.0.86.h1032.eulerosv2r12.x86_64 +EULER_V2R12_ARM_H1032=5.10.0-136.12.0.86.h1032.eulerosv2r12.aarch64 +EULER_V2R12_ARM_H1258=5.10.0-136.12.0.86.h1258.eulerosv2r12.aarch64 +EULER_V2R11_X86=5.10.0-60.18.0.50.h509.eulerosv2r11.x86_64 +EULER_V2R11_ARM=5.10.0-60.18.0.50.h665.eulerosv2r11.aarch64 +HCE_2_ARM_R865_35=5.10.0-60.18.0.50.r865_35.hce2.aarch64 +HCE_2_ARM_R1526_92=5.10.0-136.12.0.86.r1526_92.hce2.aarch64 +HCE_3_ARM_R1040_72=6.6.0-72.0.0.76.r1040_72.hce3.aarch64 +HCE_3_ARM_R1245_84=6.6.0-72.0.0.76.r1245_84.hce3.aarch64 +OPEN_EULER_153_2203_ARM=5.10.0-153.12.0.92.oe2203sp2.aarch64 +OPEN_EULER_136_2203_ARM=5.10.0-136.12.0.86.oe2203sp1.aarch64 +OPEN_EULER_2403_X86=6.6.0-28.0.0.34.oe2403.x86_64 +OPEN_EULER_2403_ARM=6.6.0-28.0.0.34.oe2403.aarch64 +OPEN_EULER_2403_SP1_ARM=6.6.0-72.0.0.76.oe2403sp1.aarch64 +OPEN_EULER_2403_SP2_ARM=6.6.0-98.0.0.103.oe2403sp2.aarch64 +OPEN_EULER_2403_SP3_ARM=6.6.0-132.0.0.111.oe2403sp3.aarch64 +EULER_V2R9_ARM=4.19.90-vhulk2009.2.0.h310.eulerosv2r9.aarch64 + +OS_NAME := $(shell grep -E '^NAME=' /etc/os-release | cut -d= -f2 | tr -d '"') +OS_VERSION_ID := $(shell grep -E '^VERSION_ID=' /etc/os-release | cut -d= -f2 | tr -d '"') +OS_VERSION := $(shell grep -E '^VERSION=' /etc/os-release | cut -d= -f2 | tr -d '"') + +KBUILD_EXTRA_SYMBOLS+=$(TOP_DIR)/ci/lib/SDK_Module.symvers +KBUILD_EXTRA_SYMBOLS+=$(TOP_DIR)/ci/lib/NIC_Module.symvers + +# Set CFLAGS based on public macro definations +CFLAGS += -DHW_CONVERT_ENDIAN +CFLAGS += -DROCE_SERVICE +CFLAGS += -DROCE_DRIVER_COMPILE +CFLAGS += -D__ROCE_DFX__ +CFLAGS += -DROCE_CC_EN +CFLAGS += -DRDMA_ROCE_MODULE +CFLAGS += $(HI1823_FLAGS) +CFLAGS += -Werror -Wall +CFLAGS += -DROCE_DWQE_EN + +# Set CFLAGS, KBUILD_EXTRA_SYMBOLS and other THIRDPARTY* variables +# from OS and kernel version information +ifeq ($(OFED_VERSION), MLNX_OFED) + THIRDPARTY = /opt/buildtools/mlnx-ofa_kernel-5.8 + KBUILD_EXTRA_SYMBOLS+=/opt/buildtools/mlnx-ofa_kernel-5.8/Module.symvers +else ifeq ($(KERN_VERSION),$(filter $(KERN_VERSION), $(PANGEA_EULER_2107) $(PANGEA_EULER_2108) $(PANGEA_EULER_2202))) + KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/pangea/output/aarch64/euleros/ofed_knl/include/Module.symvers + THIRDPARTY_INC = $(TOP_DIR)/pangea/output/aarch64/euleros/ofed_knl/ + CFLAGS += -DCONFIG_INFINIBAND_USER_MEM + CFLAGS += -DHAVE_GET_USER_PAGES_GUP_FLAGS + CFLAGS += -DCONFIG_MODULE_UPGRADE + CFLAGS += -DCONFIG_NVMF_AA +else ifeq ($(KERN_VERSION),$(filter $(KERN_VERSION), $(EULER_V2R12_H1032) $(EULER_V2R12_ARM_H1032) $(EULER_V2R12_ARM_H1258) \ + $(EULER_V2R11_ARM) $(OPEN_EULER_153_2203_ARM) $(HCE_2_ARM_R865_35) $(HCE_2_ARM_R1526_92) $(OPEN_EULER_136_2203_ARM) \ + $(EULER_V2R9_ARM) $(OPEN_EULER_2403_X86) $(OPEN_EULER_2403_ARM) $(OPEN_EULER_2403_SP1_ARM) $(OPEN_EULER_2403_SP2_ARM) $(OPEN_EULER_2403_SP3_ARM) \ + $(HCE_3_ARM_R1040_72) $(HCE_3_ARM_R1245_84))) + THIRDPARTY_INC = /usr/src/kernels/$(KERN_VERSION)/include +else ifeq ($(OS_NAME),openEuler) + ifeq ($(OS_VERSION_ID),24.03) + ifneq (,$(findstring SP4,$(OS_VERSION))) + THIRDPARTY_INC = /usr/src/kernels/$(KERN_VERSION)/include + endif + endif +else ifeq ($(KERN_VERSION), $(filter $(KERN_VERSION), $(EULER_V2R11_X86))) + THIRDPARTY =/usr/src/compat-rdma-4.17 + KBUILD_EXTRA_SYMBOLS+=/usr/src/compat-rdma/Module.symvers + THIRDPARTY_INC = /usr/src/compat-rdma +else + ifeq ($(CROSS_COMPILE),) + THIRDPARTY =/usr/src/compat-rdma-4.17 + KBUILD_EXTRA_SYMBOLS+=/usr/src/compat-rdma/Module.symvers + THIRDPARTY_INC = /usr/src/compat-rdma +# centos compat 417 need 6 para + CFLAGS += -DHAVE_NLA_PARSE_6_PARAMS + else + THIRDPARTY =/arm/cross_compile/install/sysroot/usr/src/compat-rdma-4.8 + KBUILD_EXTRA_SYMBOLS+=/arm/cross_compile/install/sysroot/usr/src/compat-rdma/Module.symvers + THIRDPARTY_INC = /arm/cross_compile/install/sysroot/usr/src/compat-rdma + endif +endif + +# Set CFLAGS from compiled driver version +ifeq ($(VERSION),ROCE_STANDARD) + CFLAGS += -DROCE_STANDARD + CFLAGS += -DROCE_BONDING_EN + CFLAGS += -DSIGN_MTT_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond + # CFLAGS += -DROCE_LATENCY_PROF_EN # 时延统计工具，需手动开启 +else ifeq ($(VERSION),ROCE_VBS) + CFLAGS += -DROCE_STANDARD + CFLAGS += -DROCE_BONDING_EN + CFLAGS += -DSIGN_MTT_EN + CFLAGS += -DROCE_VBS_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond + CFLAGS += -I$(TOP_DIR)/test/at_test/host/service/vbs/extend/roce_kernel/ +else ifeq ($(VERSION),CHIP_TEST) + CFLAGS += -DROCE_CHIP_TEST + CFLAGS += -DSIGN_MTT_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/chip_ext +else ifeq ($(VERSION),ROCE_COMPUTE) + CFLAGS += -DROCE_STANDARD + CFLAGS += -DROCE_COMPUTE +else ifeq ($(VERSION),ROCE_VROCE) + CFLAGS += -DROCE_VROCE_EN + CFLAGS += -I$(TOP_DIR)/src/dpu_extended_library/host/service/roce/linux/kernel/extern/vroce/include +endif + +ifeq ($(ENABLE_DEBUG),ON) + CFLAGS += -DENABLE_DEBUG +endif + +ifeq ($(ENABLE_HYPER_ROCE),ON) + CFLAGS += -DENABLE_HYPER_ROCE + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce/tool + CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/cqm +endif + +# Set CFLAGS from default file directories +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce/hi182x +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/drv_intf +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/cq +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/event +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/mr +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/qp +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/rdma +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/srq +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/xrc +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/uld +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/card +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/cmd +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/db +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netdev +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netlink +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sdk_api +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sysfs +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/dfx +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/option +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/debug +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/obj_ext +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_cfm_intf/hmm +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/urts +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/crm +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/rdma +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ccp +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include +CFLAGS += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg +CFLAGS += -I$(TOP_DIR)/platform/huawei_secure_c/include +CFLAGS += -I$(TOP_DIR)/platform/huawei_secure_c/src +CFLAGS += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/roce +CFLAGS += -DSECUREC_EXPORT_KERNEL_SYMBOL=0 + +##Safe compilation options +ifneq ($(HI1823_OS_TYPE), BCLINUX) + CFLAGS += -fstack-protector-all -Wl,-z,noexecstack +endif + +# Set CFLAGS from Driver version info, +# Currently not in use. +CFLAGS += -D HIROCE5_DRV_VER=\"$(HIROCE5_DRV_VERSION)\" + +export KERN_VERSION +export KBUILD_EXTRA_SYMBOLS + +DEPMOD = /sbin/depmod +INSTALL_MOD_DIR ?= $(shell test -f /etc/redhat-release && echo extra/ofa_kernel || echo updates) + +##Define compilation target +all: kernel + +install: install_kernel +install_kernel: install_modules + +##Test whether there are autoconf.h, kconfig.h files in the kernel directory, and if so, refer to the header file later +autoconf_h=$(shell /bin/ls -1 $(KSRC)/include/*/autoconf.h 2> /dev/null | head -1) +kconfig_h=$(shell /bin/ls -1 $(KSRC)/include/*/kconfig.h 2> /dev/null | head -1) + +ifneq ($(kconfig_h),) + KCONFIG_H = -include $(kconfig_h) +endif + +##Test whether V variable is defined. If so, "make" will print some information during subsequent compilation +V ?= 0 + +##KBUILD_NOCMDDEP=1, when "make" again after "make", it will not recompile +##LINUXINCLUDE, reconfigure kernel module compilation options +##__OFED_BUILD__, use macro definition configured by OFED instead of that configured by kernel + +linux_include := -include $(autoconf_h) \ + $(KCONFIG_H) \ + $(BACKPORT_INCLUDES) \ + -I$(THIRDPARTY)/include \ + -I$(THIRDPARTY)/include/uapi \ + $$(if $$(CONFIG_XEN),-I$$(srctree)/arch/x86/include/mach-xen) \ + $$(if $$(CONFIG_XEN),-D__XEN_INTERFACE_VERSION__=$$(CONFIG_XEN_INTERFACE_VERSION)) \ + -I$$(srctree)/arch/$$(SRCARCH)/include \ + -I$$(srctree)/arch/$$(SRCARCH)/include/uapi \ + -Iarch/$$(SRCARCH)/include/generated \ + -Iarch/$$(SRCARCH)/include/generated/uapi \ + -Iinclude \ + $$(if $$(KBUILD_SRC),-Iinclude2 -I$$(srctree)/include) \ + -I$$(srctree)/include/uapi \ + -I$$(srctree)/arch/$$(SRCARCH)/include \ + -Iinclude/generated/uapi \ + -Iarch/$$(SRCARCH)/include/generated + +######################### +# make kernel # +######################### +#NB: The LINUXINCLUDE value comes from main kernel Makefile +# with local directories prepended. This eventually affects +# CPPFLAGS in the kernel Makefile +kernel: $(COMPAT_CONFIG) $(COMPAT_AUTOCONF) + @echo '============== INFO ==============' + @echo 'Building kernel modules' + @echo 'Driver Version: $(VERSION)' + @echo 'Kernel Version: $(KERN_VERSION)' + @echo 'Modules Directory: $(INSTALL_MOD_PATH)/$(MODULES_DIR)' + @echo 'Kernel Sources: $(KSRC)' + @echo 'Thirdparty Source: $(THIRDPARTY)' + @echo 'Thirdparty Symbols: $(KBUILD_EXTRA_SYMBOLS)' + @echo 'TOP_DIR: $(TOP_DIR)' + @echo 'HI1823_TRUNK_DIR: $(HI1823_TRUNK_DIR)' + @echo '==================================' + $(MAKE) -C $(KSRC) M="$(CWD)" V=$(V) KBUILD_NOCMDDEP=1 $(WITH_MAKE_PARAMS) EXTRA_CFLAGS="$(CFLAGS)" \ + LINUXINCLUDE='$(linux_include)' ENABLE_DEBUG=$(ENABLE_DEBUG) ENABLE_HYPER_ROCE=$(ENABLE_HYPER_ROCE) modules + +######################### +# Install kernel # +######################### +install_modules: + @echo "Installing kernel modules" + + $(MAKE) -C $(KSRC) M="$(CWD)" \ + INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) \ + INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \ + $(WITH_MAKE_PARAMS) modules_install; + if [ ! -n "$(INSTALL_MOD_PATH)" ]; then $(DEPMOD) $(KERN_VERSION);fi; + +clean: clean_kernel + +clean_kernel: + @echo "clean kernel modules" + $(MAKE) -C $(KSRC) M="$(CWD)" $(WITH_MAKE_PARAMS) clean + @/bin/rm -f $(clean-files) + +clean-files := Module.symvers modules.order Module.markers compat/modules.order +clean-files += $(COMPAT_CONFIG) $(COMPAT_AUTOCONF) + +help: + @echo + @echo "kernel: build kernel modules" + @echo "all: build kernel modules" + @echo + @echo "install_kernel: install kernel modules under $(INSTALL_MOD_PATH)/$(MODULES_DIR)" + @echo "install: run install_kernel" + @echo + @echo "clean: delete kernel modules binaries" + @echo "clean_kernel: delete kernel modules binaries" + @echo + +else + +obj-m += hiroce5.o + +ENABLE_DEBUG ?= OFF +$(info kernel ENABLE_DEBUG: $(ENABLE_DEBUG)) + +ifeq ($(CONFIG_UBUS_DEVICE), y) +TOP_DIR=$(HI1823_TRUNK_DIR) + +GLOBAL_VERSION=$(shell cat $(HI1823_TRUNK_DIR)/src/GLOBAL_VERSION_NEW | grep driver | awk -F ':' '{print $$2}') +ccflags-y += -DGLOBAL_VERSION_STR=\"$(GLOBAL_VERSION)\" +ccflags-y += -DHW_CONVERT_ENDIAN +ccflags-y += -DROCE_SERVICE +ccflags-y += -DROCE_DRIVER_COMPILE +ccflags-y += -D__ROCE_DFX__ +ccflags-y += -DROCE_CC_EN +ccflags-y += -DRDMA_ROCE_MODULE +ccflags-y += $(HI1823_FLAGS) +# ccflags-y += -Werror -Wall +ccflags-y += -Wall +ccflags-y += -DROCE_DWQE_EN + +KBUILD_EXTRA_SYMBOLS += $(HISDK5_SYMVERS) +KBUILD_EXTRA_SYMBOLS += $(HINIC5_SYMVERS) +# ccflags-y = /usr/src/kernels/$(KERN_VERSION)/include + +ccflags-y += -DROCE_STANDARD +ccflags-y += -DROCE_BONDING_EN +ccflags-y += -DSIGN_MTT_EN + +ifeq ($(ENABLE_DEBUG),ON) + ccflags-y += -DENABLE_DEBUG +endif +ifeq ($(ENABLE_HYPER_ROCE),ON) + ccflags-y += -DENABLE_HYPER_ROCE + ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce + ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/extern/hyper_roce/tool + ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/cqm +endif + +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/bond +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce/hi182x +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/drv_intf +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/cq +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/event +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/mr +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/qp +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/rdma +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/srq +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/xrc +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/base/uld +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/card +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/cmd +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/db +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netdev +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/netlink +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sdk_api +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/dev/sysfs +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/dfx +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/option +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/debug +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/service/roce/linux/kernel/util/obj_ext +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_cfm_intf/hmm +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/urts +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/hwsdk +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/ +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/rdma +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/crm +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/sdk/knldk/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ccp +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/roce +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg +ccflags-y += -I$(TOP_DIR)/platform/huawei_secure_c/include +ccflags-y += -I$(TOP_DIR)/platform/huawei_secure_c/src +ccflags-y += -I$(TOP_DIR)/src/dpu_platform_library/host/knldk/rdma +ccflags-y += -I$(TOP_DIR)/..//opensource/openeuler/kernel/olk-6.6/include/linux/ +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface +ccflags-y += -I$(TOP_DIR)/../opensource/openeuler/kernel/olk-6.6/include +ccflags-y += -nostdinc +endif + +# List files of release here. +ccflags-y += -DSECUREC_EXPORT_KERNEL_SYMBOL=0 +hiroce5-y := ../../../../../../../platform/huawei_secure_c/src/securecutil.o \ + ../../../../../../../platform/huawei_secure_c/src/secureinput_a.o \ + ../../../../../../../platform/huawei_secure_c/src/secureprintoutput_a.o \ + ../../../../../../../platform/huawei_secure_c/src/memset_s.o \ + ../../../../../../../platform/huawei_secure_c/src/memcpy_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strncpy_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strncat_s.o \ + ../../../../../../../platform/huawei_secure_c/src/sprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/vsprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/vsnprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/snprintf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/vsscanf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/sscanf_s.o \ + ../../../../../../../platform/huawei_secure_c/src/memmove_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strcpy_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strcat_s.o \ + ../../../../../../../platform/huawei_secure_c/src/strtok_s.o \ + ../../../../cfm/hmm/hmm_buddy.o \ + ../../../../cfm/hmm/hmm_common.o \ + ../../../../cfm/hmm/hmm_em.o \ + ../../../../cfm/hmm/hmm_init.o \ + ../../../../cfm/hmm/hmm_mpt.o \ + ../../../../cfm/hmm/hmm_mr.o \ + ../../../../cfm/hmm/hmm_mtt.o \ + ../../../../cfm/hmm/hmm_umem.o \ + ../../../../cfm/rdma/rdma_bitmap.o \ + base/cq/roce_cq_common.o \ + base/cq/roce_cq_cqe.o \ + base/cq/roce_cq_create.o \ + base/cq/roce_cq_ctrl.o \ + base/cq/roce_cq_destroy.o \ + base/cq/roce_cq_intern.o \ + base/extension/roce_event_extension.o \ + base/extension/roce_main_extension.o \ + base/extension/roce_mr_extension.o \ + base/extension/roce_qp_extension.o \ + base/extension/roce_srq_extension.o \ + base/mr/roce_mr.o \ + base/qp/roce_qp_create.o \ + base/qp/roce_qp_destroy.o \ + base/qp/roce_qp_ext.o \ + base/qp/roce_qp_modify.o \ + base/qp/roce_qp_post_recv.o \ + base/qp/roce_qp_post_send.o \ + base/qp/roce_qp_query.o \ + base/rdma/rdma_comp.o \ + base/rdma/rdma_comp_res.o \ + base/rdma/rdma_comp_gid.o \ + base/rdma/rdma_comp_init.o \ + base/rdma/rdma_comp_pd.o \ + base/rdma/rdma_comp_mw_mr.o \ + base/rdma/roce_pd.o \ + base/rdma/roce_mix.o \ + base/srq/roce_srq_comm.o \ + base/srq/roce_srq_create.o \ + base/srq/roce_srq_ctrl.o \ + base/xrc/roce_xrc.o \ + base/event/roce_event.o \ + base/uld/roce_uld_api.o \ + dev/card/roce_card.o \ + dev/db/roce_db.o \ + dev/cmd/roce_cmd.o \ + dev/init/roce_main.o \ + dev/netdev/roce_netdev.o \ + dev/netlink/roce_netlink_process.o \ + dev/netlink/roce_netlink.o \ + dev/sdk_api/roce_cqm_cmd.o \ + dev/sysfs/roce_sysfs.o \ + dev/sysfs/roce_ucc_sysfs.o \ + dev/sysfs/roce_multipath_sysfs.o \ + util/dfx/roce_dfx.o \ + util/dfx/roce_dfx_query.o \ + util/option/roce_option.o \ + util/obj_ext/roce_obj_ext.o + +ifeq ($(ENABLE_DEBUG),ON) + # List debug files here. + hiroce5-y += util/dfx/roce_dfx_debug.o + hiroce5-y += util/dfx/roce_dfx_attack.o +endif + +ifeq ($(ENABLE_HYPER_ROCE),ON) + # List hyper roce files here. + hiroce5-y += extern/hyper_roce/hyper_roce_netlink.o + hiroce5-y += extern/hyper_roce/hyper_roce_qp.o + hiroce5-y += extern/hyper_roce/tool/hyper_roce_tool.o + hiroce5-y += extern/hyper_roce/tool/hyper_roce_tool_pktdrop.o +endif + +# Add hiroce5 objects from compiled driver version +# from dpu_extended_library directories +ifeq ($(VERSION),ROCE_STANDARD) + hiroce5-y +=util/bond/roce_bond_mac.o + hiroce5-y +=util/bond/roce_bond_slave.o + hiroce5-y +=util/bond/roce_bond_cfm.o + hiroce5-y +=util/bond/roce_bond_device.o + hiroce5-y +=util/bond/roce_bond_tx.o +else ifeq ($(VERSION),ROCE_COMPUTE) + # hiroce5-y +=util/bond/roce_bond_common.o +else ifeq ($(VERSION),CHIP_TEST) +# CHIP_TEST or ROCE_VBS, as VBS driver currently runs on chip test + hiroce5-y +=extern/chip_ext/roce_main_chip_ext.o + hiroce5-y +=extern/chip_ext/roce_mr_chip_ext.o + hiroce5-y +=extern/chip_ext/roce_qp_chip_ext.o + hiroce5-y +=extern/chip_ext/roce_srq_chip_ext.o +else ifeq ($(VERSION),ROCE_VROCE) + hiroce5-y +=util/bond/roce_bond_mac.o + hiroce5-y +=util/bond/roce_bond_slave.o + hiroce5-y +=util/bond/roce_bond_cfm.o + hiroce5-y +=util/bond/roce_bond_device.o + hiroce5-y +=util/bond/roce_bond_tx.o + hiroce5-y +=../../../../../../../src/dpu_extended_library/host/roce/vroce/roce_cmd_ext.o +else ifeq ($(VERSION),ROCE_VBS) + hiroce5-y +=util/bond/roce_bond_mac.o + hiroce5-y +=util/bond/roce_bond_slave.o + hiroce5-y +=util/bond/roce_bond_cfm.o + hiroce5-y +=util/bond/roce_bond_device.o + hiroce5-y +=util/bond/roce_bond_tx.o + hiroce5-y +=../../../../../../../test/at_test/host/service/vbs/extend/roce_kernel/roce_vbs_qp.o +endif + +# Driver Extension +# DPU_HIROCE5_DRV_EXTEND_MK, DPU_DRV_MK_DIR, DPU_DRV_HIROCE5_EXTEND_OBJS对外提供用户使用，修改需要评审 +ifneq ($(DPU_HIROCE5_DRV_EXTEND_MK),) + $(info Using roce extension $(DPU_HIROCE5_DRV_EXTEND_MK)) + export DPU_DRV_MK_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + include $(DPU_HIROCE5_DRV_EXTEND_MK) + hiroce5-y += $(DPU_DRV_HIROCE5_EXTEND_OBJS) +endif + +build_info: + mkdir -p build + @echo "CURDIR=$(CURDIR)" > build/build_src.txt + @for obj in $(hiroce5-y); do \ + echo $$(realpath $$obj) >> build/build_src.txt; \ + done + +endif + +include $(HI1823_BUILD_DIR)/host/linux/Makefile.ko \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h new file mode 100644 index 000000000..37562da9c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq.h @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQ_H +#define ROCE_CQ_H + +#include <rdma/ib_verbs.h> +#include <linux/slab.h> + +#include "hinic5_hw.h" +#include "hinic5_rdma.h" + +#include "roce.h" +#include "roce_srq.h" + +#include "roce_option.h" +#include "hinic5_hmm.h" + +#include "securec.h" + +/* DB type of ARM_CQ */ +enum { ROCE_CQ_DB_REQ_NOT_SOL = 1, ROCE_CQ_DB_REQ_NOT = 2 }; + +#define CQ_DW_TO_BYTE 4 +#define ROCE_CQE_UP_THRESHOLD 1024 + +/* +1.Same type of operation as SQ WQE +8'h00-Send +8'h01-Send with Invalidate +8'h02-Send with Immediate Data +8'h03-reserved + +8'h04-RDMA Write +8'h05-RDMA Write with Immediate Data +8'h06-reserved +8'h07-reserved + +8'h08-RDMA Read +8'h09-reserved +8'h0a-reserved +8'h0b-reserved + +8'h0c-Atomic Compare & Swap +8'h0d-Atomic Fetch & Add +8'h0e-Atomic Masked Compare & Swap (Extended Atomic operation) +8'h0f-Atomic Masked Fetch & Add (Extended Atomic operation) + +8'h10-Fast Register PMR +8'h11-Local Invalidate +8'h12-Bind Memory Window Type1/2 +8'h13-Local opreation(Extended for further local opreation) +other-Reserved + +2.Receive +00000 - RDMA Write with Immediate +00001 - Send +00010 - Send with Immediate +00011 - Send & Invalidate + +3.The following are general +11110 Error coding +10110 Resize coding +*/ +enum roce5_cqe_send_opcode { + ROCE_OPCODE_SEND = 0x0, + ROCE_OPCODE_SEND_WITH_INV = 0x1, + ROCE_OPCODE_SEND_WITH_IMM = 0x2, + /* 0x3 reserved */ + + ROCE_OPCODE_RDMA_WRITE = 0x4, + ROCE_OPCODE_RDMA_WRITE_WITH_IMM = 0x5, + /* 0x6 and 0x7 reserved */ + + ROCE_OPCODE_RDMA_READ = 0x8, + /* 0x9~0xb reserved */ + + ROCE_OPCODE_ATOMIC_COMP_AND_SWP = 0xc, + ROCE_OPCODE_ATOMIC_FETCH_AND_ADD = 0xd, + ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP = 0xe, + ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD = 0xf, + + ROCE_OPCODE_FAST_REG_PMR = 0x10, + ROCE_OPCODE_LOCAL_INV = 0x11, + ROCE_OPCODE_BIND_TYPE2_MW = 0x12, + ROCE_OPCODE_REG_SIG_MR = 0x13, + + ROCE_OPCODE_RESIZE_CQE = 0x16, + ROCE_OPCODE_ERR = 0x1e, + ROCE_OPCODE_CQE_UNUSED = 0x1f /* Be used in new CQ buf when reszie cq */ +}; + +enum roce5_cqe_recv_opcode { + ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM = 0x0, + ROCE_RECV_OPCODE_SEND = 0x1, + ROCE_RECV_OPCODE_SEND_WITH_IMM = 0x2, + ROCE_RECV_OPCODE_SEND_WITH_INV = 0x3 +}; + +/* Define the state type of the CQ */ +enum cq_state { + ROCE_CQ_STATE_INVALID = 0x0, + ROCE_CQ_STATE_ERR = 0x1, + ROCE_CQ_STATE_OVERFLOW = 0x2, + ROCE_CQ_STATE_VALID = 0xf, + ROCE_CQ_STATE_MEM_INIT = 0xa /* Initial value of Host Memory */ +}; + +#define ROCE_CQ_TIME_OUT_CHECK_VALUE 0xe +#define ROCE_CQ_STATE_CHECK_VALUE 0x0 + +#define ROCE_CQE_RQ_INLINE 1 +#define ROCE_CQE_RQ_NORMAL 0 +#define ROCE_CQE_SEND_COMP 1 +#define ROCE_CQE_RECV_COMP 0 + +#define ATOMIC_DATA_LEN 8 /* Specified as 8B by protocol */ + +#define ROCE_CQE_INVALID_VALUE 0xff +#define ROCE_CQ_RESIZE_POLL_TIMES 100 + +/* roce Commands, bufs and data structures related to cq */ +struct roce5_cq_buf { + struct tag_cqm_buf *buf; /* pointer to describe the buf structure b cqm */ + struct mtt + mtt; /* the mtt struct used by kernel mode and user mode to discribe buf */ + int entry_size; /* the size of cqe */ + int buf_size; /* the size of cq_buf */ +}; + +struct roce5_cq_resize_buf { + struct roce5_cq_buf buf; /* the size of buf that was resized */ + int cqe; /* the number of resized cqe */ +}; + +#define ROCE_RCQ_ULD_DEF_LEN 64 +#define ROCE_RCQ_MAGIC 0x5a5ae0e0 +struct roce5_rcq_uld_def { + u8 uld_def[ROCE_RCQ_ULD_DEF_LEN]; +}; + +struct roce5_cq { + struct ib_cq ibcq; + cqm_queue_s *cqm_cq; /* Save the handle obtained from cqm */ + struct roce5_db + db; /* The address information of the software DB that stores the CI user mode/kernel mode */ + __be32 *set_ci_db; /* Kernel-mode software DB */ + u32 cons_index; /* consumer pointer */ + + u32 arm_sn; /* the serial number of arm */ + int arm_flag; /* Used to determine whether the arm command has been sent, to avoid repeated sending */ + u32 cqn; + unsigned int vector; /* associated to the eq used */ + struct roce5_cq_buf buf; /* pointer describing the buf structure of cqm */ + struct roce5_cq_resize_buf *resize_buf; /* resize buf struction */ + + spinlock_t lock; /* Need to lock when operating cq */ + struct mutex resize_mutex; /* resize the mutex of cq */ + struct ib_umem *umem; /* record the infomation mapped by User-mode buf */ + struct ib_umem * + resize_umem; /* record the infomation mapped by buf which the User-mode resized */ + + struct list_head send_qp_list; /* send queue qp */ + struct list_head recv_qp_list; /* receive queue qp */ + + int reset_notify_added; + struct list_head reset_notify; + u32 cqe_mask; /* mask of the cq size */ + + /* + * 'left_num' and 'wqebb_index' are only valid in the merged CQE within the RQ. + * 'left_num' records the remaining count of the last merged CQE. + * 'wqebb_index' records the wqebb index of the current CQE. + */ + u32 left_num; + u32 wqebb_index; + struct roce5_option *opt; + u8 shadow; + u32 shadow_vfid; + + void (*reset_flow_comp)(struct roce5_cq *); + + struct roce5_rcq_uld_def + uld_def; /* must placed at cq_magic before, preventstop uld access memory stomping, support up to 64B space */ + u32 rcq_magic; +}; + +/* Used when destroy QP */ +void roce5_cq_clean(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq); +void roce5_cq_async_event(struct roce5_device *rdev, struct roce5_cq *cq, + int type); +void roce5_cq_clean_process(struct roce5_cq *cq, u32 qpn, + struct roce5_srq *srq); +void roce5_cq_put_umem(struct roce5_device *rdev, struct roce5_cq_buf *buf, + struct ib_umem **umem, u8 shadow_cq); +int roce5_create_cq_common(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce5_cq *rcq, + u32 index); +int roce5_cq_get_umem(struct roce5_device *rdev, struct ib_ucontext *context, + struct roce5_cq *rcq, u64 buf_addr, int cqe, + bool is_resize); +void *roce5_get_sw_cqe(struct roce5_cq *cq, unsigned int n); +void *roce5_get_cqe_from_buf(struct roce5_cq_buf *buf, unsigned int n); +void roce5_cq_set_ci(struct roce5_cq *cq); +void roce5_cq_buf_init(struct roce5_cq_buf *buf); +void *roce5_get_cqe(struct roce5_cq *cq, unsigned int n); +void roce5_reset_flow_comp(struct roce5_cq *rcq); + +void roce5_lock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __acquires(&roce5_send_cq->lock) __acquires(&roce5_recv_cq->lock); +void roce5_unlock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __releases(&roce5_send_cq->lock) __releases(&roce5_recv_cq->lock); + +struct roce5_cq *roce5_get_rcq_by_cqn(struct roce5_device *rdev, u32 cqn); +void roce5_cq_completion(void *svc_hd, u32 cqn, void *cq_handler); + +static inline void roce5_rcq_put(struct roce5_cq *rcq) +{ + if (rcq == NULL) { + return; + } + roce5_option_put_value(rcq->opt); + cqm5_object_put(&rcq->cqm_cq->object); +} + +#endif // ROCE_CQ_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c new file mode 100644 index 000000000..f212acecc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_common.c @@ -0,0 +1,299 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "hinic5_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_user.h" +#include "roce_cq.h" +#include "roce_xqe_format.h" +#include "roce_option.h" +#include "hinic5_hmm.h" +#include "securec.h" +#include "roce_os_compat.h" +#include "roce_aeq_format.h" + +/* **************************************************************************** + Prototype : roce5_cq_async_event + Description : roce5_cq_async_event + Input : struct roce5_device *rdev + struct roce5_cq *cq + int type + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_async_event(struct roce5_device *rdev, struct roce5_cq *cq, + int type) +{ + struct ib_cq *ibcq = &cq->ibcq; + struct ib_event event; + int ret; + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset event.(ret:%d)", + ret); + } + if (type != ROCE_EVENT_TYPE_CQ_ERROR) { + ROCE_DEV_WARN_LIMIT( + rdev, + "Unexpected event type(0x%x) on CQ(%06x), func_id(%d)", + type, cq->cqn, rdev->glb_func_id); + return; + } + + if (ibcq->event_handler) { + event.device = ibcq->device; + event.event = IB_EVENT_CQ_ERR; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } +} + +/* **************************************************************************** + Prototype : roce5_cq_buf_init + Description : roce5_cq_buf_init + Input : struct roce5_cq_buf *buf + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_buf_init(struct roce5_cq_buf *buf) +{ + /* optype was initialized into ROCE_OPCODE_CQE_UNUSED(0x1f) */ + int ret; + ret = memset_s(buf->buf->direct.va, + (unsigned long)((unsigned)buf->buf_size), + ROCE_CQE_INVALID_VALUE, + (unsigned long)((unsigned)buf->buf_size)); + if (ret != 0) { + ROCE_ERR("Failed to memset va.(ret:%d)", ret); + } +} + +/* **************************************************************************** + Prototype : roce5_cq_put_umem + Description : Release umem and corresponding mtt, corresponding to put + Input : struct roce5_device *rdev + struct roce5_cq_buf *buf + struct ib_umem **umem + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_put_umem(struct roce5_device *rdev, struct roce5_cq_buf *buf, + struct ib_umem **umem, u8 shadow_cq) +{ + if (shadow_cq != 0) { + return; + } + /* free MTT of Buffer */ + hmm_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + + /* release umem */ + ib_umem_release(*umem); +} + +static void *roce5_cq_buf_offset(struct tag_cqm_buf *buf, unsigned int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce5_get_cqe_from_buf(struct roce5_cq_buf *buf, unsigned int n) +{ + return roce5_cq_buf_offset(buf->buf, (n * buf->entry_size)); +} + +void *roce5_get_cqe(struct roce5_cq *cq, unsigned int n) +{ + return roce5_get_cqe_from_buf(&cq->buf, n); +} + +/* **************************************************************************** + Prototype : roce5_get_sw_cqe + Description : roce5_get_sw_cqe + Input : struct roce5_cq *cq + unsigned int n + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void *roce5_get_sw_cqe(struct roce5_cq *cq, unsigned int n) +{ + struct roce_cqe *cqe = NULL; + struct roce_cqe tmp_cqe; + unsigned int tmp_val; + + cqe = (struct roce_cqe *)roce5_get_cqe( + cq, (n & ((unsigned int)cq->cqe_mask))); + if (cqe == NULL) { + ROCE_ERR("Get sw cqe failed."); + return NULL; + } + tmp_cqe.dw0.value = roce5_convert_cpu32(cqe->dw0.value); + tmp_cqe.dw1.value = roce5_convert_cpu32(cqe->dw1.value); + + /* Add judgment condition: the optype of CQE cannot be UNUSED, UNUSED means that it has been initialized in resize + cq */ + tmp_val = ((n & ((unsigned int)cq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) == + 0) ? + 1 : + 0; + if ((ROCE_LIKELY(tmp_cqe.dw1.bs.op_type != ROCE_OPCODE_CQE_UNUSED)) && + ((tmp_cqe.dw0.bs.owner ^ tmp_val) != 0)) { + return cqe; + } + + return NULL; +} + +int roce5_cq_get_umem(struct roce5_device *rdev, struct ib_ucontext *context, + struct roce5_cq *rcq, u64 buf_addr, int cqe, + bool is_resize) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + int cqe_size = (int)rdev->rdma_cap.cqe_size; + struct roce5_cq_buf *buf = NULL; + struct ib_umem **umem = NULL; + + if (rcq->shadow != 0) { + return 0; + } + if (is_resize) { + buf = &rcq->resize_buf->buf; + umem = &rcq->resize_umem; + } else { + buf = &rcq->buf; + umem = &rcq->umem; + } + + *umem = roce5_umem_get(&rdev->ib_dev, context, buf_addr, + (unsigned long)(cqe * cqe_size), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(*umem)) { + ROCE_DEV_ERR(rdev, "Failed to get ib_umem, func_id(%d)", + rdev->glb_func_id); + return (int)PTR_ERR(*umem); + } + + page_shift = (int)roce5_umem_get_buf_page_shift(rdev, *umem, buf_addr); + npages = roce5_umem_get_mtt_npages(rdev, *umem, page_shift); + + buf->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &buf->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma_mtt, func_id(%d)", + rdev->glb_func_id); + goto err_buf; + } + + ret = roce5_umem_write_mtt(rdev, &buf->mtt, *umem, true); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt for umem, func_id(%d)", + rdev->glb_func_id); + goto err_mtt; + } + + return 0; + +err_mtt: + hmm_mtt_free(rdev->hwdev, &buf->mtt, SERVICE_T_ROCE); + +err_buf: + ib_umem_release(*umem); + + return ret; +} + +void roce5_reset_flow_comp(struct roce5_cq *rcq) +{ + struct ib_cq *ibcq = &rcq->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +/** + * @brief Get roce cq by cqn. When querying succeeded, you have to call roce5_rcq_put() to free + the cqm_obj and opt. But there is no need to call roce5_rcq_put() if querying failed. + * @return Success: return the roce cq. Failure: return NULL. + */ +struct roce5_cq *roce5_get_rcq_by_cqn(struct roce5_device *rdev, u32 cqn) +{ + cqm_object_s *cqm_obj = NULL; + cqm_queue_s *cqm_queue = NULL; + struct roce5_option *opt = NULL; + struct roce5_cq *rcq = NULL; + + cqm_obj = cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find rcq according to cqn(0x%x), func_id(%d)", + cqn, rdev->glb_func_id); + goto err_out; + } + + cqm_queue = container_of(cqm_obj, cqm_queue_s, object); + opt = (struct roce5_option *)cqm_queue->priv; + if (opt == NULL) { + ROCE_DEV_ERR(rdev, "Failed to get option, priv ptr is NULL."); + goto err_out_cqm_put; + } + + rcq = (struct roce5_cq *)roce5_option_get_value(opt); + if (rcq == NULL) { + ROCE_DEV_ERR(rdev, "Failed to get rcq from option."); + goto err_out_cqm_put; + } + + return rcq; + +err_out_cqm_put: + cqm5_object_put(cqm_obj); +err_out: + return NULL; +} + +void roce5_cq_completion(void *svc_hd, u32 cqn, void *cq_handler) +{ + struct roce5_option *opt = NULL; + struct roce5_cq *rcq = NULL; + + if (cq_handler == NULL) { + ROCE_ERR("Cq_handler is null"); + return; + } + + opt = (struct roce5_option *)cq_handler; + rcq = (struct roce5_cq *)roce5_option_get_value(opt); + if (rcq == NULL) { + ROCE_WARN("Failed to get rcq from option."); + return; + } + + ++rcq->arm_sn; + rcq->arm_flag = 0; + + rcq->ibcq.comp_handler(&rcq->ibcq, rcq->ibcq.cq_context); + + roce5_option_put_value(opt); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c new file mode 100644 index 000000000..b79aad2dc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_cqe.c @@ -0,0 +1,763 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "hinic5_hw.h" +#include "securec.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" +#include "roce_restore.h" +#include "roce_aeq_format.h" +#include "roce_xqe_format.h" + +#include "roce_main_extension.h" + +/* **************************************************************************** + Prototype : roce5_next_cqe_sw + Description : roce5_next_cqe_sw + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static struct roce_cqe *roce5_next_cqe_sw(struct roce5_cq *cq) +{ + return (struct roce_cqe *)roce5_get_sw_cqe(cq, cq->cons_index); +} + +static int g_error_cqe_to_wc[ROCE_COMPLETE_EVENT_MAX] = { + [ROCE_COMPLETE_EVENT_LOC_LEN_ERR] = IB_WC_LOC_LEN_ERR, + [ROCE_COMPLETE_EVENT_LOC_QP_OPERATION_ERR] = IB_WC_LOC_QP_OP_ERR, + [ROCE_COMPLETE_EVENT_LOC_PROTECTION_ERR] = IB_WC_LOC_PROT_ERR, + [ROCE_COMPLETE_EVENT_WR_FLUSH_ERR] = IB_WC_WR_FLUSH_ERR, + + [ROCE_COMPLETE_EVENT_MW_BIND_ERR] = IB_WC_MW_BIND_ERR, + [ROCE_COMPLETE_EVENT_BAD_RESP_ERR] = IB_WC_BAD_RESP_ERR, + + [ROCE_COMPLETE_EVENT_LOC_ACCESS_ERR] = IB_WC_LOC_ACCESS_ERR, + [ROCE_COMPLETE_EVENT_REM_INV_REQ_ERR] = IB_WC_REM_INV_REQ_ERR, + [ROCE_COMPLETE_EVENT_REM_ACCESS_ERR] = IB_WC_REM_ACCESS_ERR, + [ROCE_COMPLETE_EVENT_REM_OPERATION_ERR] = IB_WC_REM_OP_ERR, + [ROCE_COMPLETE_EVENT_RETRY_CTR_EXCEED_ERR] = IB_WC_RETRY_EXC_ERR, + + [ROCE_COMPLETE_EVENT_RNR_RETRY_CTR_EXCEED_ERR] = + IB_WC_RNR_RETRY_EXC_ERR, + + [ROCE_COMPLETE_EVENT_REM_ABORTED_ERR] = IB_WC_REM_ABORT_ERR, + [ROCE_COMPLETE_EVENT_XRC_VIOLATION_ERR] = IB_WC_REM_INV_RD_REQ_ERR, +}; + +/* **************************************************************************** + Prototype : roce5_handle_error_cqe + Description : roce5_handle_error_cqe + Input : struct roce_err_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_error_cqe(struct roce_err_cqe *cqe, struct ib_wc *wc) +{ + if (cqe->dw7.bs.syndrome >= ROCE_COMPLETE_EVENT_MAX) { + wc->status = IB_WC_GENERAL_ERR; + } else if (cqe->dw7.bs.syndrome > 0) { + wc->status = g_error_cqe_to_wc[cqe->dw7.bs.syndrome]; + if (wc->status == 0) { + wc->status = IB_WC_GENERAL_ERR; + } + } + + wc->vendor_err = cqe->dw7.bs.vendor_err; +} + +static void roce5_get_local_opcode_from_cqe(const struct roce_cqe *cqe, + struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case ROCE_OPCODE_REG_SIG_MR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + ROCE_WARN("Unknown cqe optype"); + break; + } +} + +static void roce5_get_opcode_type_part_1(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + + case ROCE_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_IMM); + break; + + case ROCE_OPCODE_SEND: + wc->opcode = IB_WC_SEND; + break; + + case ROCE_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + break; + default: + roce5_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +static void roce5_get_opcode_type_part_2(struct roce_cqe *cqe, struct ib_wc *wc) +{ + switch (cqe->dw1.bs.op_type) { + case ROCE_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = cqe->byte_cnt; + break; + + case ROCE_OPCODE_ATOMIC_COMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_COMP_AND_SWP: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = ATOMIC_DATA_LEN; + break; + + case ROCE_OPCODE_ATOMIC_MASKED_FETCH_AND_ADD: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = ATOMIC_DATA_LEN; + break; + case ROCE_OPCODE_FAST_REG_PMR: + wc->opcode = IB_WC_REG_MR; + break; + + default: + roce5_get_local_opcode_from_cqe(cqe, wc); + break; + } +} + +/* **************************************************************************** + Prototype : roce5_get_opcode_from_scqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_get_opcode_from_scqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->wc_flags = 0; + + if (cqe->dw1.bs.op_type < ROCE_OPCODE_RDMA_READ) { + roce5_get_opcode_type_part_1(cqe, wc); + } else { + roce5_get_opcode_type_part_2(cqe, wc); + } +} + +/* **************************************************************************** + Prototype : roce5_get_opcode_from_rcqe + Description : Set the opcode/flag/byte_len of wc according to the opcode of send_cqe + Input : struct roce_cqe *cqe + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_get_opcode_from_rcqe(struct roce_cqe *cqe, struct ib_wc *wc) +{ + wc->byte_len = cqe->byte_cnt; + + switch (cqe->dw1.bs.op_type) { + case ROCE_RECV_OPCODE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + /* The driver does not perform big or small endian conversion for immediate data, but the incoming CQE + has been converted to the CPU endian, so it needs to be converted back */ + wc->ex.imm_data = roce5_convert_be32(cqe->imm_invalid_rkey); + break; + + case ROCE_RECV_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = + roce5_convert_be32(cqe->imm_invalid_rkey); + break; + + case ROCE_RECV_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + + case ROCE_RECV_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + /* The driver does not perform big or small endian conversion for immediate data, but the incoming CQE + has been converted to the CPU endian, so it needs to be converted back */ + wc->ex.imm_data = roce5_convert_be32(cqe->imm_invalid_rkey); + break; + + default: + ROCE_WARN("Not supported"); + break; + } +} + +/* **************************************************************************** + Prototype : roce5_cq_poll_and_resize + Description : roce5_cq_poll_and_resize + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_poll_and_resize(struct roce5_device *rdev, + struct roce5_cq *cq) +{ + if (ROCE_LIKELY(cq->resize_buf != NULL)) { + /* Release the original Buffer of CQ */ + cqm5_object_resize_free_old(&cq->cqm_cq->object); + cq->buf = cq->resize_buf->buf; + cq->ibcq.cqe = cq->resize_buf->cqe; + + kfree(cq->resize_buf); + cq->resize_buf = NULL; + } +} + +/* **************************************************************************** + Prototype : roce5_cq_get_cur_qp + Description : roce5_cq_get_cur_qp + Input : struct roce5_cq *cq + struct roce5_qp **cur_qp + struct roce_cqe *cqe + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_get_cur_qp(struct roce5_cq *cq, struct roce5_qp **cur_qp, + struct roce_cqe *cqe) +{ + struct roce5_device *rdev = to_roce5_dev(cq->ibcq.device); + + if ((*cur_qp == NULL) || (cqe->dw0.bs.qpn != (u32)(*cur_qp)->qpn)) { + if (*cur_qp != NULL) { + roce5_rqp_put(*cur_qp); + } + + *cur_qp = roce5_get_rqp_by_qpn(rdev, cqe->dw0.bs.qpn); + if (*cur_qp == NULL) { + ROCE_DEV_ERR( + rdev, + "CQ(%06x) with entry for unknown QPN(%06x), func_id(%d)", + cq->cqn, cqe->dw0.bs.qpn, rdev->glb_func_id); + return -EINVAL; + } + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_get_xrc_srq + Description : roce5_cq_get_xrc_srq + Input : struct roce5_cq *cq + struct roce5_srq **srq + struct roce_cqe *cqe + u32 qp_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_get_xrc_srq(struct roce5_cq *cq, struct roce5_srq **srq, + struct roce_cqe *cqe, u32 qp_type) +{ + u32 srqn = cqe->dw6.bs.srqn_rqpn; + struct roce5_device *rdev = to_roce5_dev(cq->ibcq.device); + + if (qp_type == IB_QPT_XRC_TGT) { + if (*srq != NULL) { + roce5_rsrq_put(*srq); + } + + *srq = roce5_get_rsrq_by_srqn(rdev, cqe->dw6.bs.srqn_rqpn); + if (*srq == NULL) { + ROCE_DEV_ERR( + rdev, + "CQ(%06x) with entry for unknown SRQN(%06x), func_id(%d)", + cq->cqn, srqn, rdev->glb_func_id); + return -EINVAL; + } + } + + return 0; +} + +static int roce5_poll_recv_cqe(struct roce5_cq *rcq, struct roce5_qp *cur_qp, + struct roce5_srq *srq, struct ib_wc *wc) +{ + u32 wqe_ctr = rcq->wqebb_index; + unsigned int tail = 0; + struct roce5_srq *srq_tmp = NULL; + + if (cur_qp->ibqp.srq) { + srq_tmp = to_roce5_srq(cur_qp->ibqp.srq); + /* Determine whether to overflow */ + if (wqe_ctr > (int)srq_tmp->max_depth) { + ROCE_ERR("Get wqe index(0x%x) from cqe", wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq_tmp->wrid[wqe_ctr]; + roce5_free_srq_wqe(srq_tmp, wqe_ctr); + } else if (srq != NULL) { + /* Determine whether to overflow */ + if (wqe_ctr > srq->max_depth) { + ROCE_ERR("Get wqe index(0x%x) from cqe", wqe_ctr); + return -EINVAL; + } + + wc->wr_id = srq->wrid[wqe_ctr]; + roce5_free_srq_wqe(srq, wqe_ctr); + } else { + struct roce5_wq *wq = &(cur_qp->rq); + tail = (u32)(wq->tail & ((unsigned int)wq->wqebb_cnt - 1)); + wc->wr_id = wq->wrid[tail]; + ++wq->tail; + } + + return 0; +} + +static void roce5_poll_move_tail(const struct roce5_qp *cur_qp, + struct roce5_wq *wq, + struct roce_cqe *trans_cqe) +{ + u16 wqe_ctr = 0; + if (cur_qp->sq_signal_bits == 0) { + wqe_ctr = trans_cqe->dw7.bs.wqe_cnt; + wq->tail += (u16)(wqe_ctr - (u16)wq->tail); + } +} + +static void roce5_poll_one_get_av_info(const struct roce5_qp *cur_qp, + struct ib_wc *wc, + struct roce_cqe trans_cqe, + struct roce_cqe *cqe) +{ + int ret; + + if ((cur_qp->qp_type == IB_QPT_UD) || (cur_qp->qp_type == IB_QPT_GSI)) { + wc->sl = trans_cqe.dw4.bs.vlan_pri; + wc->src_qp = trans_cqe.dw6.bs.srqn_rqpn; + wc->network_hdr_type = trans_cqe.dw6.bs.stp; + wc->smac[0] = + (u8)(trans_cqe.dw4.bs.smac_h >> + 8); /* Take the first Byte data, shift 8bit */ + wc->smac[1] = (u8)(trans_cqe.dw4.bs.smac_h & 0xff); + ret = memcpy_s(&wc->smac[2], sizeof(cqe->smac_l), &cqe->smac_l, + sizeof(cqe->smac_l)); //2 : smac array idx + if (ret != 0) { + ROCE_ERR("Failed to memcpy smac.(ret:%d)", ret); + } + wc->wc_flags = (int)((u32)wc->wc_flags | IB_WC_WITH_SMAC); + if (trans_cqe.dw6.bs.vlan_pre != 0) { + wc->vlan_id = trans_cqe.dw4.bs.vlan_id; + wc->wc_flags = + (int)((u32)wc->wc_flags | IB_WC_WITH_VLAN); + } + } +} + +static int roce5_poll_one_get_qp_and_srq(struct roce5_cq *cq, + struct roce5_qp **cur_qp, + struct ib_wc *wc, + struct roce_cqe *trans_cqe, + struct roce5_srq **srq) +{ + int ret = 0; + ret = roce5_cq_get_cur_qp(cq, cur_qp, trans_cqe); + if (ret != 0) { + ROCE_ERR("Failed to get current qp"); + return ret; + } + + wc->qp = &((*cur_qp)->ibqp); + + ret = roce5_cq_get_xrc_srq(cq, srq, trans_cqe, wc->qp->qp_type); + if (ret != 0) { + ROCE_ERR("Failed to get xrc srq"); + return ret; + } + + return ret; +} + +static void roce5_bytes_trans(u32 *addr, int dw_num) +{ + u32 *dw = addr; + int i = 0; + + for (i = 0; i < dw_num; i++) { + *dw = roce5_convert_cpu32(*dw); + dw++; + } +} + +static int roce5_get_and_trans_cqe(struct roce5_device *rdev, + struct roce5_cq *cq, + struct roce_cqe *trans_cqe, int *is_send, + struct roce_cqe **cqe) +{ + int ret; + + do { + *cqe = roce5_next_cqe_sw(cq); + if (*cqe == NULL) { + return -EAGAIN; + } + + /* Later, it is necessary to perform big and small end conversion on CQE. + In order to avoid turning back after processing, copy a copy */ + ret = memcpy_s((void *)trans_cqe, sizeof(struct roce_cqe), + (void *)*cqe, sizeof(struct roce_cqe)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy trans_cqe.(ret:%d).", ret); + return -ENOMEM; + } + + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + /* Convert the CQE into the CPU endian corresponding to the running environment */ + roce5_bytes_trans( + (u32 *)(void *)trans_cqe, + (int)(sizeof(struct roce_cqe) / CQ_DW_TO_BYTE)); + + *is_send = trans_cqe->dw1.bs.s_r; + /* Resize CQ operation in progress */ + if (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == + ROCE_OPCODE_RESIZE_CQE)) { + roce5_cq_poll_and_resize(rdev, cq); + } + } while (ROCE_UNLIKELY(trans_cqe->dw1.bs.op_type == + ROCE_OPCODE_RESIZE_CQE)); + + return 0; +} + +static void roce5_poll_send_cqe(struct roce5_qp *cur_qp, + struct roce_cqe *trans_cqe, struct ib_wc *wc) +{ + struct roce5_wq *wq = &(cur_qp->sq); + /* QP of type IB_SIGNAL_REQ_WR, tail needs to skip several WRs that do not generate CQE */ + roce5_poll_move_tail(cur_qp, wq, trans_cqe); + + wc->wr_id = wq->wrid[wq->tail & ((unsigned int)wq->wqebb_cnt - 1)]; + ++wq->tail; + + if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce5_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe, + wc); + return; + } + + roce5_get_opcode_from_scqe(trans_cqe, wc); +} + +static void roce5_init_wc(struct roce_cqe *trans_cqe, struct ib_wc *wc) +{ + roce5_get_opcode_from_rcqe(trans_cqe, wc); + wc->wc_flags = (int)((unsigned int)wc->wc_flags | IB_WC_GRH); + wc->pkey_index = 0; + wc->vlan_id = 0xffff; + wc->sl = 0; + /* avoid cm_req_handler()->ib_lid_be16() trigger call trace */ + wc->slid = 0; + return; +} + +static int roce5_poll_one_cqe(struct roce5_cq *rcq, struct roce5_qp *cur_qp, + int is_send, struct roce_cqe *trans_cqe, + struct roce_cqe *cqe, struct roce5_srq *srq, + struct ib_wc *wc) +{ + int ret = 0; + + wc->status = IB_WC_SUCCESS; + if (is_send != 0) { + roce5_poll_send_cqe(cur_qp, trans_cqe, wc); + return 0; + } + + ret = roce5_poll_recv_cqe(rcq, cur_qp, srq, wc); + if (ret != 0) { + return ret; + } + + if (trans_cqe->dw1.bs.op_type == ROCE_OPCODE_ERR) { + roce5_handle_error_cqe((struct roce_err_cqe *)(void *)trans_cqe, + wc); + return 0; + } + + roce5_init_wc(trans_cqe, wc); + + roce5_poll_one_get_av_info((cur_qp), wc, *trans_cqe, cqe); + + /* Kernel mode does not support receiving inline */ + if (trans_cqe->dw1.bs.inline_r == ROCE_CQE_RQ_INLINE) { + ROCE_ERR("Receive inline not supported in kernel space"); + return -EINVAL; + } + + return 0; +} + +static int roce5_update_cons_index(struct roce5_device *rdev, + struct roce5_cq *rcq, + struct roce_cqe *trans_cqe) +{ + if (trans_cqe->dw1.bs.flush_op != 0) { + if (trans_cqe->dw1.bs.s_r == ROCE_CQE_SEND_COMP) { + ROCE_DEV_ERR(rdev, "Unexpected flush operation in SQ."); + ++rcq->cons_index; + return -EBADE; + } + + if (rcq->left_num == 0) { + rcq->left_num = trans_cqe->wqe_num; + rcq->wqebb_index = trans_cqe->dw1.bs.wqebb_cnt; + } + } else { + rcq->left_num = 1; + rcq->wqebb_index = trans_cqe->dw1.bs.wqebb_cnt; + } + + if (rcq->left_num == 1) { + ++rcq->cons_index; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_poll_one + Description : roce5_poll_one + Input : struct roce5_cq *cq + struct roce5_qp **cur_qp + struct ib_wc *wc + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_poll_one(struct roce5_device *rdev, struct roce5_cq *cq, + struct roce5_qp **cur_qp, struct ib_wc *wc) +{ + int ret = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe trans_cqe; + struct roce5_srq *srq = NULL; + int is_send = 0; + bool need_poll = true; + + while (need_poll) { + ret = roce5_get_and_trans_cqe(rdev, cq, &trans_cqe, &is_send, + &cqe); + if (ret != 0) { + goto err_out; + } + + ret = roce5_update_cons_index(rdev, cq, &trans_cqe); + if (ret != 0) { + goto err_out; + } + + ret = roce5_poll_one_get_qp_and_srq(cq, cur_qp, wc, &trans_cqe, + &srq); + if (ret != 0) { + goto err_out; + } + + /* Poll cq exception handling, generally cq judges user state information, xrc judges wrid */ + if ((*cur_qp) == NULL) { + ROCE_ERR("cur_qp is NULL when poll cq"); + ret = -EACCES; + goto err_out; + } + if ((*cur_qp)->umem || + ((*cur_qp)->ibqp.xrcd && srq != NULL && !(srq->wrid))) { + ROCE_ERR( + "qp(%u) create in user space , but poll cq in kernel. NOT PERMIT!", + (*cur_qp)->qpn); + ret = -EACCES; + goto err_out; + } + + if (trans_cqe.dw1.bs.fake != 0) { + ROCE_INFO("Fake cqe go repoll."); + continue; + } + + ret = roce5_poll_one_cqe(cq, *cur_qp, is_send, &trans_cqe, cqe, + srq, wc); + if (ret != 0) { + goto err_out; + } + + --cq->left_num; + need_poll = false; + } + +err_out: + roce5_rsrq_put(srq); + return ret; +} + +/* the format of cq_ci DB:the bits of ci less than 24bit */ +void roce5_cq_set_ci(struct roce5_cq *cq) +{ + *cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff); +} + +int roce5_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct roce5_device *rdev = NULL; + struct roce5_cq *rcq = to_roce5_cq(ibcq); + struct roce5_qp *cur_qp = NULL; + unsigned long flags = 0; /* type required by kernel API */ + int npolled = 0; + int ret = 0; + + spin_lock_irqsave(&rcq->lock, flags); + + rdev = to_roce5_dev(ibcq->device); + + for (npolled = 0; npolled < num_entries; ++npolled) { + ret = roce5_poll_one(rdev, rcq, &cur_qp, wc + npolled); + if (ret != 0) { + break; + } + } + + roce5_cq_set_ci(rcq); + + spin_unlock_irqrestore(&rcq->lock, flags); + + if ((ret == 0) || (ret == -EAGAIN)) { + ret = npolled; + goto err_out; + } + +err_out: + roce5_rqp_put(cur_qp); + return ret; +} + +static void roce5_cq_arm(struct roce5_cq *cq, u32 cmd, void __iomem *uar_page) +{ + struct roce_db_cq_arm db_value; + int ret; + + ret = memset_s(&db_value, sizeof(db_value), 0, sizeof(db_value)); + if (ret != 0) { + ROCE_ERR("Failed to memset db_value.(ret:%d)", ret); + } + + db_value.dw0.bs.type = 3; + db_value.dw0.bs.cos = 0; /* arm_cq don't need cos */ + db_value.dw0.bs.cp = 1; /* 1 for cp */ + db_value.dw0.bs.non_filter = 1; + db_value.dw0.bs.cqc_type = 0; + db_value.dw0.bs.cqn = cq->cqn; + db_value.dw0.value = roce5_convert_be32(db_value.dw0.value); + + db_value.dw1.bs.cmd_sn = cq->arm_sn; + db_value.dw1.bs.cmd = cmd; + db_value.dw1.bs.ci = cq->cons_index; + db_value.dw1.value = roce5_convert_be32(db_value.dw1.value); + + /* + * Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + wmb(); + + roce5_write64((u32 *)(void *)&db_value, uar_page); +} + +int roce5_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + unsigned long lock_flags = 0; /* type required by kernel API */ + struct roce5_cq *rcq = NULL; + + if (CHECK_2LAYER_IS_NULL(ibcq, device)) { + return -EINVAL; + } + + rcq = to_roce5_cq(ibcq); + + spin_lock_irqsave(&rcq->lock, lock_flags); + if (rcq->arm_flag != 0) { + spin_unlock_irqrestore(&rcq->lock, lock_flags); + return 0; + } + + rcq->arm_flag = 1; + spin_unlock_irqrestore(&rcq->lock, lock_flags); + + /* Only 64-bit is supported, 64-bit writes are atomic, no need to lock */ + roce5_cq_arm(rcq, + (((unsigned int)flags & IB_CQ_SOLICITED_MASK) == + IB_CQ_SOLICITED) ? + ROCE_CQ_DB_REQ_NOT_SOL : + ROCE_CQ_DB_REQ_NOT, + to_roce5_dev(ibcq->device)->kernel_db_map); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c new file mode 100644 index 000000000..0df5d34e4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_create.c @@ -0,0 +1,636 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "comm_defs.h" +#include "roce_cq.h" +#include "roce_npu_cmd.h" +#include "roce_cqm_cmd.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_os_compat.h" +#include "roce_user.h" +#include "roce_option.h" +#include "roce_cq_intern.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" + +#define ROCE_CQE_BASE_SIZE_BYTES 16 + +/* **************************************************************************** + Prototype :roce5_cq_attr_assign + Description : Assign value to CQC + Input : struct roce_cq_attr *cq_attr + struct roce5_cq *rcq + int eqn + int page_shift + Output : None + + 1.Date : 2019/10/13 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_attr_assign(roce_verbs_cq_attr_s *cq_attr, + struct roce5_cq *rcq, int eqn, int page_shift) +{ + /* Assign value to CQ */ + /* dw0 */ + cq_attr->dw0.bs.size = (u32)ROCE_ILOG2( + (unsigned)(rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)); + cq_attr->dw0.bs.page_size = (u32)page_shift; + cq_attr->dw0.bs.cqe_size = (u32)ROCE_ILOG2( + (unsigned)(rcq->buf.entry_size / ROCE_CQE_BASE_SIZE_BYTES)); + cq_attr->dw0.bs.mtt_page_size = + rcq->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + cq_attr->dw0.bs.tss_timer_num = + 7; /* 7 : The maximum number of timers supported by cq */ + cq_attr->dw0.bs.arm_timer_en = 0; + cq_attr->dw0.bs.timer_mode = 0; + cq_attr->dw0.bs.cnt_adjust_en = 1; + cq_attr->dw0.bs.cnt_clear_en = 1; + cq_attr->dw0.bs.ci_on_chip = 0; + cq_attr->dw0.value = cpu_to_be32(cq_attr->dw0.value); + + /* dw1 */ + cq_attr->dw1.bs.dma_attr_idx = 0; + cq_attr->dw1.bs.so_ro = 0; + cq_attr->dw1.bs.state = ROCE_CQ_STATE_VALID; + cq_attr->dw1.value = cpu_to_be32(cq_attr->dw1.value); + + /* dw2 */ + cq_attr->dw2.bs.idle_max_count = 0; + cq_attr->dw2.bs.cqecnt_lth = 10; /* update ci threshhold: 2^10=1024 */ + cq_attr->dw2.bs.cqecnt_rctl_en = 0; + cq_attr->dw2.bs.ceqe_en = 1; + cq_attr->dw2.bs.arm_ceqe_en = 1; + cq_attr->dw2.bs.ceqn = (u8)eqn; + cq_attr->dw2.value = cpu_to_be32(cq_attr->dw2.value); + + /* dw3 */ + cq_attr->dw3.bs.timeout = + 0; /* The timeout mechanism is disabled by default when CQ is created */ + cq_attr->dw3.bs.max_cnt = + 0; /* The overtime mechanism is disabled by default when CQ is created */ + cq_attr->dw3.value = cpu_to_be32(cq_attr->dw3.value); + + /* dw4 - dw5 */ + cq_attr->cqc_l0mtt_gpa = rcq->buf.mtt.mtt_paddr; + cq_attr->cqc_l0mtt_gpa = cpu_to_be64(cq_attr->cqc_l0mtt_gpa); + + /* dw6 - dw7 */ + cq_attr->ci_record_gpa_at_hop_num = cpu_to_be64( + (rcq->db.dma & (~0x3uLL)) | rcq->buf.mtt.mtt_layers); +} + +static int roce5_cq_fill_create_inbuf(struct roce5_device *rdev, + struct roce5_cq *rcq, int vector, + int page_shift, + cqm_cmd_buf_s *cqm_cmd_inbuf) +{ + int eqn = 0; + roce_verbs_cq_attr_s *cq_attr = NULL; + roce_uni_cmd_create_cq_s *cq_sw2hw_inbuf = NULL; + + cq_sw2hw_inbuf = (roce_uni_cmd_create_cq_s *)cqm_cmd_inbuf->buf; + cq_sw2hw_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (rcq->shadow != 0) { + cq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_sw2hw_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rcq->shadow_vfid); + } + cq_attr = &cq_sw2hw_inbuf->cq_attr; + + /* Get the EQN of the CEQ based on the Vector index */ + eqn = hinic5_vector_to_eqn(rdev->hwdev, SERVICE_T_ROCE, vector); + if (eqn < 0) { + ROCE_DEV_ERR(rdev, + "Failed to get eqn from hinic vector, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + roce5_cq_attr_assign(cq_attr, rcq, eqn, page_shift); + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_sw2hw + Description : Send the cqc configuration command + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int vector + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_sw2hw(struct roce5_device *rdev, struct roce5_cq *rcq, + int vector, int page_shift) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_create_cq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + ret = roce5_cq_fill_create_inbuf(rdev, rcq, vector, page_shift, + cqm_cmd_inbuf); + if (ret != 0) { + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_SW2HW_CQ, + cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send SW2HW_CQ command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + + /* CMDq times out or CMDq does not work, update the device status, notify the PCIe module to reset + the device through OFED */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto err_send_cmd; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + rcq->cons_index = 0; + rcq->arm_sn = 1; + rcq->arm_flag = 0; + rcq->vector = (u32)vector; + + return 0; + +err_send_cmd: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_create_cq_check + Description : roce5_create_cq_check + Input : struct ib_device *ibdev + int entries + int vector + struct ib_ucontext *ibcontext + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_cq_check(struct ib_device *ibdev, int entries, + int vector) +{ + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + ROCE_DEV_ERR( + rdev, + "Cqe number invalid, entries(%d), max_cqes(%d), func_id(%d)", + entries, (int)rdev->rdma_cap.max_cqes, + rdev->glb_func_id); + return -EINVAL; + } + + if (vector > (int)rdev->rdma_cap.num_comp_vectors) { + ROCE_DEV_ERR(rdev, "Vector over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +static int roce5_check_cq_create_flags(u32 flags) +{ + /* + * It returns non-zero value for unsupported CQ + * create flags, otherwise it returns zero. + */ + return (int)(flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN | + IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)); +} + +static int roce5_cq_cqc_cfg(struct roce5_device *rdev, struct roce5_cq *rcq, + int vector, struct ib_udata *udata, u64 cq_buf_addr) +{ + int ret = 0; + int page_shift = (int)roce5_umem_get_buf_page_shift(rdev, rcq->umem, + cq_buf_addr) - + PAGE_SHIFT_4K; + + /* configurate CQC */ + ret = roce5_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to handle cq sw2hw, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + /* User mode requires outgoing CQN */ + if (ib_copy_to_udata(udata, &rcq->cqn, sizeof(u32)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + return ret; + } + + return 0; +} + +static int roce5_create_user_cq(struct roce5_device *rdev, struct roce5_cq *rcq, + int cqe_num, const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret = 0; + int vector = attr->comp_vector; + struct roce5_create_cq_cmd ucmd = { 0 }; + roce_user_handle_t hdl = { 0 }; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + goto err_out; + } + + ROCE_CONSTUCT_ULD_HDL_RCQ(&hdl, rdev, rcq, udata, &rcq->buf.mtt, + &rcq->db.dma, cqe_num); + ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rcq->shadow, &rcq->shadow_vfid, + NULL); + ret = roce5_uld_cq_create((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_ERR("Failed to execute uld cq create, ret(%d)", ret); + goto err_out; + } + /* Entries has been decreased by one when it was input by User Mode, and it is the expected value exactly + after increased by one by Kernel Mode */ + ret = roce5_cq_get_umem(rdev, ibcontext, rcq, ucmd.buf_addr, cqe_num, + false); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get umem, func_id(%d)", + rdev->glb_func_id); + goto err_destroy_uld; + } + + ret = roce5_db_map_user(to_roce5_ucontext(ibcontext), + (unsigned long)ucmd.db_addr, &rcq->db, + rcq->shadow); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to map kernel_mem to user_mem, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + ret = roce5_cq_cqc_cfg(rdev, rcq, vector, udata, ucmd.buf_addr); + if (ret != 0) { + goto err_unmap_db; + } + + return 0; + +err_unmap_db: + roce5_db_unmap_user(to_roce5_ucontext(ibcontext), &rcq->db, + rcq->shadow); + +err_free_mtt: + roce5_cq_put_umem(rdev, &rcq->buf, &rcq->umem, rcq->shadow); + rcq->umem = NULL; + +err_destroy_uld: + roce5_uld_cq_destroy((roce_uld_handle)&hdl); + +err_out: + + return ret; +} + +static void roce5_fill_rcq_info(struct roce5_cq *rcq) +{ + rcq->cqn = rcq->cqm_cq->index; + rcq->buf.buf = &rcq->cqm_cq->q_room_buf_1; + + /* Initialize buf to unused */ + roce5_cq_buf_init(&rcq->buf); + + /* Software DB assignment */ + rcq->set_ci_db = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr + ->doorbell_record); + *rcq->set_ci_db = 0; + rcq->db.db_record = (__be32 *)(void *)(&rcq->cqm_cq->q_header_vaddr + ->doorbell_record); + rcq->db.dma = rcq->cqm_cq->q_header_paddr; +} + +/* **************************************************************************** + Prototype : roce5_create_kernel_cq + Description : roce5_create_kernel_cq + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + int vector + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_kernel_cq(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries, int vector, + u32 index) +{ + int ret = 0; + int page_shift = 0; + + rcq->buf.buf_size = entries * (int)rdev->rdma_cap.cqe_size; + rcq->cqm_cq = cqm5_object_rdma_queue_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_SCQ, + (u32)rcq->buf.buf_size, rcq->opt, true, index, 0, 0); + if (rcq->cqm_cq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma_queue from cqm_object, func_id(%d), index(%d)", + rdev->glb_func_id, index); + return (-ENOMEM); + } + + /* Buffer is obtained from room1 when the queue is just created */ + if (rcq->cqm_cq->current_q_room != CQM_RDMA_Q_ROOM_1) { + ROCE_DEV_ERR(rdev, "Not use CQM_RDMA_Q_ROOM_1, func_id(%d)", + rdev->glb_func_id); + ret = -EINVAL; + goto err_free_cqm_cq; + } + + roce5_fill_rcq_info(rcq); + + page_shift = ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size); + + /* allocate MTT */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, rcq->buf.buf->buf_number, + (u32)page_shift, &rcq->buf.mtt, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_cqm_cq; + } + + /* Write the PA of the CQ Buffer to the MTT */ + ret = roce5_buf_write_mtt(rdev, &rcq->buf.mtt, rcq->buf.buf); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + page_shift = (int)ROCE_ILOG2(rcq->cqm_cq->q_room_buf_1.buf_size) - + PAGE_SHIFT_4K; + + /* configurate CQC */ + ret = roce5_cq_sw2hw(rdev, rcq, vector, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to configure CQC, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + return 0; + +err_free_mtt: + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_free_cqm_cq: + cqm5_object_delete(&rcq->cqm_cq->object); + + return ret; +} + +static int roce5_cq_depth_init(struct roce5_device *rdev, struct roce5_cq *rcq, + const struct ib_cq_init_attr *attr, int *cqe_num) +{ + u32 entries = (u32)attr->cqe; + + /* The CQE queue should reserve four special CQE for the HOST CI can be updated to the CQC CI */ + entries += ROCE_CQE_UP_THRESHOLD; + entries = ROCE_ROUNDUP_POW_OF_TWO(entries) & 0xffffffff; + /* Chip Constraints: Minimum queue depth needs to be page-aligned */ + if ((entries * rdev->rdma_cap.cqe_size) < PAGE_SIZE) { + entries = (PAGE_SIZE >> ROCE_ILOG2(rdev->rdma_cap.cqe_size)); + } + if (entries == 0) { + ROCE_DEV_ERR(rdev, "Invalid entries after align, func_id(%u)", + rdev->glb_func_id); + return -EINVAL; + } + rcq->cqe_mask = entries - 1; + /* Check if max spec is exceeded */ + if (rdev->rdma_cap.max_cqes > UINT_MAX - ROCE_CQE_UP_THRESHOLD) { + ROCE_DEV_ERR( + rdev, + "max_cqes overflow detected, max_cqes(%u), func_id(%u)", + rdev->rdma_cap.max_cqes, rdev->glb_func_id); + return -EINVAL; + } + if (entries > (rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR( + rdev, + "Over range after align, entries(%u), max_cqes(%u), func_id(%u)", + entries, + rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD, + rdev->glb_func_id); + return -EINVAL; + } + + if ((int)entries > INT_MAX) { + ROCE_DEV_ERR(rdev, "Bug detected: Int Overflow!"); + return -EINVAL; + } + + *cqe_num = (int)entries; + rcq->ibcq.cqe = (int)entries - ROCE_CQE_UP_THRESHOLD; + return 0; +} + +static int roce5_do_create_cq(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce5_cq *rcq, + u32 index) +{ + struct roce5_device *rdev = NULL; + struct roce5_ucontext *context = NULL; + int ret = 0; + int vector = attr->comp_vector; + int cqe_num = 0; + + rdev = to_roce5_dev(ibdev); + ret = roce5_cq_depth_init(rdev, rcq, attr, &cqe_num); + if (ret != 0) { + return ret; + } + + mutex_init(&rcq->resize_mutex); + spin_lock_init(&rcq->lock); + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + rcq->buf.entry_size = (int)rdev->rdma_cap.cqe_size; + rcq->rcq_magic = ROCE_RCQ_MAGIC; + + rcq->opt = roce5_some(rcq); + if (rcq->opt == NULL) { + ROCE_DEV_ERR(rdev, "Failed to init some, cqn(%u).", rcq->cqn); + return -ENOMEM; + } + + INIT_LIST_HEAD(&rcq->send_qp_list); + INIT_LIST_HEAD(&rcq->recv_qp_list); + if (udata != NULL) { + ret = roce5_create_cq_alloc_cqn(rdev, rcq, index); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm cq resource, func_id(%d)", + rdev->glb_func_id); + return ret; + } + context = roce5_queue_to_ucontext((void *)(&rcq->ibcq), udata, + ROCE_Q_TYPE_CQ); + ret = roce5_create_user_cq(rdev, rcq, cqe_num, attr, + &context->ibucontext, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create user_cq, func_id(%d)", + rdev->glb_func_id); + cqm5_object_delete(&rcq->cqm_cq->object); + return ret; + } + } else { + ret = roce5_create_kernel_cq(rdev, rcq, cqe_num, vector, index); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create kernel_cq, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + + rcq->reset_flow_comp = roce5_reset_flow_comp; + + return 0; +} + +int roce5_create_cq_common(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata, struct roce5_cq *rcq, + u32 index) +{ + int ret; + int vector = attr->comp_vector; + int entries = (int)attr->cqe; + + if (roce5_check_cq_create_flags(attr->flags) != 0) { + ROCE_ERR("Not support the cq_create flag(%x)", attr->flags); + return -EOPNOTSUPP; + } + + ret = roce5_create_cq_check(ibdev, entries, vector); + if (ret != 0) { + ROCE_ERR("Failed to check cq information"); + return ret; + } + + return roce5_do_create_cq(ibdev, attr, udata, rcq, index); +} + +int roce5_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) +{ + struct roce5_cq *rcq = NULL; + + if ((ibcq == NULL) || (attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rcq = to_roce5_cq(ibcq); + return roce5_create_cq_common(ibcq->device, attr, udata, rcq, + ROCE_CQN_INVLD); +} + +struct ib_cq *roce5_create_cq_ofa(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret; + struct roce5_cq *rcq = NULL; + struct ib_uobject uobject; + + if ((ibdev == NULL) || (attr == NULL)) { + ROCE_ERR("ibdev or attr is NULL"); + return ERR_PTR(-EINVAL); + } + + uobject.context = ibcontext; + + rcq = (struct roce5_cq *)kzalloc(sizeof(*rcq), GFP_KERNEL); + if (rcq == NULL) { + ret = -ENOMEM; + ROCE_ERR("Failed to alloc rcq"); + goto out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + rcq->ibcq.device = ibdev; +#if defined(IB_CQ_HAVE_UOBJECT) + rcq->ibcq.uobject = ((udata != NULL) ? &uobject : NULL); +#endif + ret = roce5_create_cq(&rcq->ibcq, attr, udata); + if (ret != 0) { + kfree(rcq); + rcq = NULL; + goto out; + } + +out: + /* disassociate anyway */ + if (rcq != NULL) { + rcq->ibcq.uobject = NULL; + } + return ((ret != 0) ? (struct ib_cq *)ERR_PTR((long)ret) : &rcq->ibcq); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c new file mode 100644 index 000000000..eaf8e22f4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_ctrl.c @@ -0,0 +1,964 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "comm_defs.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_npu_cmd.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_infra.h" +#include "roce_compat.h" +#include "roce_os_compat.h" +#include "roce_cqe_format.h" +#include "roce_user.h" +#include "roce_com.h" + +/* and above next is V100 adaptation data structure */ +typedef struct tag_roce_cmd_resize_cq { + roce_verbs_cmd_com_s com; + + u32 rsvd; + u32 page_size; /* Size of the resize buf page. */ + u32 log_cq_size; /* Cq depth after resize */ + u32 mtt_layer_num; /* Number of mtt levels after resize */ + /* DW4~5 */ + union { + u64 mtt_base_addr; /* Start address of mr or mw */ + u32 cqc_l0mtt_gpa[2]; + }; + u32 mtt_page_size; /* Size of the mtt page after resize. */ + roce_xq_mtt_info_s mtt_info; +} roce_cmd_resize_cq_s; + +typedef struct tag_roce_cmd_modify_cq { + roce_verbs_cmd_com_s com; + u32 max_cnt; + u32 timeout; +} roce_cmd_modify_cq_s; + +static int roce5_uni_cmd_cq_modify(struct roce5_device *rdev, + struct roce5_cq *cq, u32 cnt, u32 period) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_modify_cq_s *cq_modify_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_cq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_modify_inbuf = (roce_uni_cmd_modify_cq_s *)cqm_cmd_inbuf->buf; + cq_modify_inbuf->com.index = cpu_to_be32(cq->cqn); + cq_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); + if (cq->shadow != 0) { + cq_modify_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_modify_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(cq->shadow_vfid); + } + cq_modify_inbuf->cq_modify.max_cnt = cpu_to_be32(cnt); + cq_modify_inbuf->cq_modify.timeout = cpu_to_be32(period); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_MODIFY_CQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_CQ, cqn(0x%x), ret(%d), func_id(%hu)", + cq->cqn, ret, rdev->glb_func_id); + /* CMDq may return a positive number, so its return value cannot be used directly */ + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int roce5_cq_modify(struct roce5_device *rdev, struct roce5_cq *cq, + u32 cnt, u32 period) +{ + if (ROCE5_SUPPORT_CQ_MODIFY_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_cq_modify(rdev, cq, cnt, period); + } + return -EINVAL; +} + +/* **************************************************************************** + Prototype : roce5_modify_cq + Description : OFED_3_12 + Input : struct ib_cq *ibcq + u16 cq_count + u16 cq_period + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period) +{ + int ret = 0; + struct roce5_cq *cq = NULL; + struct roce5_device *rdev = NULL; + + if (ibcq == NULL) { + ROCE_ERR(" Ibcq is null"); + return -EINVAL; + } + + cq = to_roce5_cq(ibcq); + rdev = to_roce5_dev(ibcq->device); + + ret = roce5_cq_modify(rdev, cq, cq_count, cq_period); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to modify cq, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_alloc_resize_buf + Description : Kernel mode applies for resize_buf, and its corresponding MTT + When there is an error in execution, this function ensures that all intermediate resources are released; + when the execution is successful, this function can confirm that all resources are acquired. + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_alloc_resize_buf(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries) +{ + int ret = 0; + int page_shift = 0; + + if (rcq->resize_buf != NULL) { + ROCE_DEV_ERR(rdev, "Resize buffer is busy, func_id(%d)", + rdev->glb_func_id); + return -EBUSY; + } + + /* Apply for resize_buf and assign it */ + rcq->resize_buf = (struct roce5_cq_resize_buf *)kmalloc( + sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc resize buffer, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + ret = cqm5_object_resize_alloc_new(&rcq->cqm_cq->object, + (u32)rcq->resize_buf->buf.buf_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to resize cq buffer, func_id(%d)", + rdev->glb_func_id); + goto err_resize_buf_alloc; + } + + /* If the application is successful, record the buffer information of the new buffer */ + rcq->resize_buf->cqe = entries - ROCE_CQE_UP_THRESHOLD; + rcq->resize_buf->buf.buf = &rcq->cqm_cq->q_room_buf_2; + + /* Initialize new buf to unused state */ + roce5_cq_buf_init(&rcq->resize_buf->buf); + + page_shift = (int)ROCE_ILOG2(rcq->resize_buf->buf.buf->buf_size); + + /* Apply MTT for resize_buf */ + rcq->buf.mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, rcq->resize_buf->buf.buf->buf_number, + (u32)page_shift, &rcq->resize_buf->buf.mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_mtt_alloc; + } + + /* configurate MTT, write PA of resize_buf to MTT */ + ret = roce5_buf_write_mtt(rdev, &rcq->resize_buf->buf.mtt, + rcq->resize_buf->buf.buf); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_mtt_write; + } + + return 0; + +err_mtt_write: + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + +err_mtt_alloc: + /* free resize_buf */ + cqm5_object_resize_free_new(&rcq->cqm_cq->object); + +err_resize_buf_alloc: + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_cq_free_resize_buf + Description : Kernel Mode releases resize_buf and its corresponding MTT + Input : struct roce5_device *rdev + struct roce5_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_free_resize_buf(struct roce5_device *rdev, + struct roce5_cq *rcq) +{ + /* Release the MTT of resize_buf first */ + hmm_mtt_free(rdev->hwdev, &(rcq->resize_buf->buf.mtt), SERVICE_T_ROCE); + + /* free resize_buf */ + cqm5_object_resize_free_new(&(rcq->cqm_cq->object)); + + /* free the resize_buf pointer itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* **************************************************************************** + Prototype : roce5_cq_alloc_resize_umem + Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_alloc_resize_umem(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_resize_cq_cmd ucmd = { 0 }; + struct roce5_ucontext *context = roce5_queue_to_ucontext( + (void *)(&rcq->ibcq), udata, ROCE_Q_TYPE_CQ); + + /* If a resize is being executed, it is not allowed to execute another resize at the same time */ + if (rcq->resize_umem != NULL) { + ROCE_DEV_ERR(rdev, "Resize_umem is busy, func_id(%d)", + rdev->glb_func_id); + return -EBUSY; + } + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + return -EFAULT; + } + + rcq->resize_buf = (struct roce5_cq_resize_buf *)kmalloc( + sizeof(*rcq->resize_buf), GFP_ATOMIC); + if (rcq->resize_buf == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to alloc cq resize buffer, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + ret = roce5_cq_get_umem(rdev, &context->ibucontext, rcq, ucmd.buf_addr, + entries, true); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get umem, func_id(%d)", + rdev->glb_func_id); + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + + return ret; + } + + rcq->resize_buf->buf.entry_size = rcq->buf.entry_size; + rcq->resize_buf->buf.buf_size = entries * rcq->buf.entry_size; + rcq->resize_buf->cqe = entries - ROCE_CQE_UP_THRESHOLD; + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_free_resize_umem + Description : User mode applies for umem of resize_buf, and MTT corresponding to user mode buf + Input : struct roce5_device *rdev + struct roce5_cq *rcq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_cq_free_resize_umem(struct roce5_device *rdev, + struct roce5_cq *rcq) +{ + /* Free MTT and umem of resize_buf */ + roce5_cq_put_umem(rdev, &rcq->resize_buf->buf, &rcq->resize_umem, 0); + rcq->resize_umem = NULL; + + /* free resize_buf itself */ + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; +} + +/* **************************************************************************** + Prototype : roce5_cq_get_outstanding_cqes + Description : roce5_cq_get_outstanding_cqes + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_get_outstanding_cqes(struct roce5_cq *cq) +{ + u32 i = 0; + + i = cq->cons_index; + while (roce5_get_sw_cqe(cq, i)) { + ++i; + } + + return (int)(i - cq->cons_index); +} + +static int roce5_cq_get_next_cqe(struct roce_cqe **cqe, struct roce5_cq *cq, + unsigned int *i, u32 *times, + const struct roce_cqe *start_cqe) +{ + do { + *cqe = (struct roce_cqe *)roce5_get_sw_cqe(cq, ++(*i)); + if (*cqe == NULL) { + ROCE_DELAY(); + --(*times); + } + } while ((*cqe == NULL) && (*times != 0)); + + if ((*cqe == start_cqe) || (*cqe == NULL)) { + ROCE_ERR("Failed to get resize CQE, CQN(0x%x)", cq->cqn); + return -ENOMEM; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_cq_resize_copy_cqes + Description : roce5_cq_resize_copy_cqes + Input : struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_cq_resize_copy_cqes(struct roce5_cq *cq) +{ + struct roce_cqe *cqe = NULL; + struct roce_cqe *new_cqe = NULL; + struct roce_cqe *start_cqe = NULL; + u32 times; + unsigned int i = 0; + int ret = 0; + + i = cq->cons_index; + + times = ROCE_CQ_RESIZE_POLL_TIMES; + do { + cqe = (struct roce_cqe *)roce5_get_sw_cqe(cq, i); + if (cqe == NULL) { + ROCE_DELAY(); + --times; + } + } while ((cqe == NULL) && (times != 0)); + + if (cqe == NULL) { + ROCE_ERR("Failed to get resize CQE, CQN(0x%x)", cq->cqn); + return -ENOMEM; + } + + start_cqe = cqe; + + /* r_cqe:resize_cqe + CI + +--------+-----+-----+-----+----------+ + old_buf: | | CQE | CQE |R_CQE| | + +--------+-----+-----+-----+----------+ + | | + | | + +--------+-----+-----+-----+----------+ + new_buf: | | CQE | CQE | | | + +--------+-----+-----+-----+----------+ + */ + /* Convert the fields needed by CQE to little endian */ + cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value); + while (cqe->dw1.bs.op_type != ROCE_OPCODE_RESIZE_CQE) { + /* The resized PI can be inherited, and the index of the original CQE remains unchanged in resize_buf */ + new_cqe = (struct roce_cqe *)roce5_get_cqe_from_buf( + &cq->resize_buf->buf, + (i & ((unsigned)cq->resize_buf->cqe))); + + ret = memcpy_s((void *)new_cqe, sizeof(struct roce_cqe), + (void *)cqe, sizeof(struct roce_cqe)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy new_cqe.(ret:%d)", ret); + return -ENOMEM; + } + + /* If the CQE has been wrapped in resize_buf, the corresponding Owner bit needs to be modified + to be owned by the software. + * The rule is: when the owner bit is owned by CQE software, the O bit is opposite to the high bit of ci; + when the owner bit is owned by hardware, the O bit is the same as the high bit of CI. + */ + new_cqe->dw0.bs.owner = ((i & ((unsigned)cq->resize_buf->cqe + + ROCE_CQE_UP_THRESHOLD)) != 0) ? + 1 : + 0; + + /* After processing, turn the DW0/DW1 of CQE back to big endian */ + cqe->dw0.value = roce5_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_be32(cqe->dw1.value); + + /* Convert DW0/DW1 of the CQE in the new queue back to big endian */ + new_cqe->dw0.value = roce5_convert_be32(new_cqe->dw0.value); + new_cqe->dw1.value = roce5_convert_be32(new_cqe->dw1.value); + + /* Get the next CQE */ + ret = roce5_cq_get_next_cqe(&cqe, cq, &i, &times, start_cqe); + if (ret != 0) { + return ret; + } + + /* Convert DW0/DW1 of CQE to little endian */ + cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value); + } + + return 0; +} + +static int roce5_uni_cmd_cq_resize_inbuf_fill(struct roce5_device *rdev, + struct roce5_cq *rcq, + int page_shift, + cqm_cmd_buf_s **cqm_cmd_inbuf) +{ + int ret = 0; + struct dev_roce_svc_own_cap *rdma_own_cap = NULL; + u32 cq_size = 0; + u32 mtt_layer = 0; + u64 mtt_paddr = 0; + u32 mtt_page_size = 0; + roce_uni_cmd_resize_cq_s *cq_resize_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_resize_cq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + rdma_own_cap = &rdev->rdma_cap.dev_rdma_cap.roce_own_cap; + cq_size = (u32)rcq->resize_buf->cqe + ROCE_CQE_UP_THRESHOLD; + cq_size = (u32)ROCE_ILOG2(cq_size); + mtt_layer = rcq->resize_buf->buf.mtt.mtt_layers; + mtt_paddr = rcq->resize_buf->buf.mtt.mtt_paddr; + mtt_page_size = rcq->resize_buf->buf.mtt.mtt_page_shift - PAGE_SHIFT_4K; + + cq_resize_inbuf = (roce_uni_cmd_resize_cq_s *)(*cqm_cmd_inbuf)->buf; + cq_resize_inbuf->com.index = cpu_to_be32(rcq->cqn); + cq_resize_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (rcq->shadow != 0) { + cq_resize_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_resize_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rcq->shadow_vfid); + } + cq_resize_inbuf->cq_resize.page_size = cpu_to_be32((u32)page_shift); + cq_resize_inbuf->cq_resize.log_cq_size = cpu_to_be32(cq_size); + cq_resize_inbuf->cq_resize.mtt_layer_num = cpu_to_be32(mtt_layer); + cq_resize_inbuf->cq_resize.mtt_base_addr = mtt_paddr; + cq_resize_inbuf->cq_resize.mtt_base_addr = + cpu_to_be64(cq_resize_inbuf->cq_resize.mtt_base_addr); + cq_resize_inbuf->cq_resize.mtt_page_size = cpu_to_be32(mtt_page_size); + + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_flags = 0; + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_num = 0; + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_own_cap->cmtt_cl_start); + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_own_cap->cmtt_cl_end); + cq_resize_inbuf->cq_resize.cmtt_cache.mtt_cache_line_size = + cpu_to_be32(rdma_own_cap->cmtt_cl_sz); + return 0; +} + +static int roce5_cq_resize(struct roce5_device *rdev, struct roce5_cq *rcq, + int page_shift) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + if (ROCE5_SUPPORT_IB_CMD_RESIZE_CQ_MSG_V2(rdev) != 0) { + ret = roce5_uni_cmd_cq_resize_inbuf_fill(rdev, rcq, page_shift, + &cqm_cmd_inbuf); + } + + if (ret != 0) { + return ret; + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_RESIZE_CQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RESIZE_CQ command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + + /* When CMDq times out or CMDq cannot work, update the device status and notify the PCIe module to reset + the device through OFED */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +/* **************************************************************************** + Prototype : roce5_user_cq_resize + Description : roce5_user_cq_resize + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_user_cq_resize(struct roce5_device *rdev, struct roce5_cq *rcq, + int entries, struct ib_udata *udata) +{ + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "Over range after align, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + ret = roce5_cq_alloc_resize_umem(rdev, rcq, entries, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc resize_umem, func_id(%d)", + rdev->glb_func_id); + return ret; + } +#if defined(UMEM_HAVE_PAGE_SHIFT) + page_shift = rcq->resize_umem->page_shift - PAGE_SHIFT_4K; +#else + page_shift = PAGE_SHIFT - PAGE_SHIFT_4K; +#endif + + /* Send the cq_resize command to configure CQC. After the configuration is successful, the new CQE is written to + resize_buf, and the old buffer may still retain the old CQE. */ + ret = roce5_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to resize cq, func_id(%d)", + rdev->glb_func_id); + roce5_cq_free_resize_umem(rdev, rcq); + return ret; + } + + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_resize_user_cq + Description : roce5_resize_user_cq + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_resize_user_cq(struct roce5_device *rdev, struct roce5_cq *rcq, + int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_resize_cq_cmd ucmd = { 0 }; + int cqe_entries = entries; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + return -EFAULT; + } + + mutex_lock(&rcq->resize_mutex); + + if (ucmd.stage == 1) { + goto release_flow; + } + + /* The CQE queue should reserve four special CQE for the HOST CI can be updated to the CQC CI */ + cqe_entries += ROCE_CQE_UP_THRESHOLD; + cqe_entries = + (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); + //Minimum queue depth needs to be aligned by page + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) { + cqe_entries = + (int)(PAGE_SIZE >> (unsigned int)ROCE_ILOG2( + rdev->rdma_cap.cqe_size)); + } + if (cqe_entries == 0) { + ROCE_DEV_ERR(rdev, + "Invalid cqe_entries after align, func_id(%d)", + rdev->glb_func_id); + ret = -EINVAL; + goto out; + } + rcq->cqe_mask = cqe_entries - 1; + + /* The new depth of CQ cannot be the same as the old depth. Four special CQE space is reserved when CQ is created, + so the value of ibcq->cqe is 4 smaller than the actual value */ + if (cqe_entries == (rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "No need to resize cq, func_id(%d)", + rdev->glb_func_id); + ret = 0; + goto out; + } + + ret = roce5_user_cq_resize(rdev, rcq, cqe_entries, udata); + if (ret != 0) { + goto out; + } + + mutex_unlock(&rcq->resize_mutex); + + return 0; + +release_flow: + /* free old MTT */ + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + + rcq->buf = rcq->resize_buf->buf; + ib_umem_release(rcq->umem); + rcq->umem = rcq->resize_umem; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + rcq->resize_umem = NULL; + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_kernel_cq_resize + Description : roce5_kernel_cq_resize + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_kernel_cq_resize(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries) +{ + int outst_cqe = 0; + int page_shift = 0; + int ret = 0; + + /* Cannot exceed max size after power-of-2 alignment */ + if (entries > ((int)rdev->rdma_cap.max_cqes + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "Over range after align, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + /* The number of resized CQEs cannot be smaller than the number of outstanding CQEs */ + outst_cqe = roce5_cq_get_outstanding_cqes(rcq); + if (entries < (outst_cqe + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR( + rdev, + "Can't resize, because smaller than the number of outstanding CQES, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + ret = roce5_cq_alloc_resize_buf(rdev, rcq, entries); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc resize buffer, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + page_shift = (int)ROCE_ILOG2(rcq->cqm_cq->q_room_buf_2.buf_size) - + PAGE_SHIFT_4K; + + /* Send the cq_resize command to configure CQC. After the configuration is successful, the new CQE is written to + resize_buf, and the old buffer may still retain the old CQE. */ + ret = roce5_cq_resize(rdev, rcq, page_shift); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to resize cq, func_id(%d)", + rdev->glb_func_id); + roce5_cq_free_resize_buf(rdev, rcq); + return ret; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_resize_kernel_cq + Description : roce5_resize_kernel_cq + Input : struct roce5_device *rdev + struct roce5_cq *rcq + int entries + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_resize_kernel_cq(struct roce5_device *rdev, + struct roce5_cq *rcq, int entries) +{ + int ret = 0; + int old_cqe = 0; + struct mtt old_mtt; + int cqe_entries = entries; + + mutex_lock(&rcq->resize_mutex); + + cqe_entries += ROCE_CQE_UP_THRESHOLD; + cqe_entries = + (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)cqe_entries) & 0xffffffff); + /* Minimum queue depth needs to be aligned by page */ + if ((u32)(cqe_entries * (int)rdev->rdma_cap.cqe_size) < PAGE_SIZE) { + cqe_entries = (PAGE_SIZE >> (unsigned int)ROCE_ILOG2( + rdev->rdma_cap.cqe_size)); + } + + /* The new depth of CQ cannot be the same as the old depth. Four special CQE space is reserved when CQ is created, + so the value of ibcq->cqe is four smaller than the actual value */ + if (cqe_entries == (rcq->ibcq.cqe + ROCE_CQE_UP_THRESHOLD)) { + ROCE_DEV_ERR(rdev, "No need to resize cq, func_id(%d)", + rdev->glb_func_id); + mutex_unlock(&rcq->resize_mutex); + return 0; + } + + ret = roce5_kernel_cq_resize(rdev, rcq, cqe_entries); + if (ret != 0) { + goto out; + } + + /* free old MTT */ + old_mtt = rcq->buf.mtt; + + /* When copying CQE from the old buffer to resize_buf, the user may be polling cqe from the old buf, so + it needs to be locked.If CQE of the old buffer has been polled in the polling process, polling process will + free old buffer and switch to new buffer. + */ + spin_lock_irq(&rcq->lock); + + /* If the new buf has been switched to, nothing needs to be done */ + if (rcq->resize_buf != NULL) { + ret = roce5_cq_resize_copy_cqes(rcq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Have not polled resize cqe(fuc:%d, cqn:%u, ret:%d)", + rdev->glb_func_id, rcq->cqn, ret); + roce5_cq_free_resize_buf(rdev, rcq); + spin_unlock_irq(&rcq->lock); + mutex_unlock(&rcq->resize_mutex); + return ret; + } + old_cqe = rcq->ibcq.cqe; + rcq->buf = rcq->resize_buf->buf; + rcq->ibcq.cqe = rcq->resize_buf->cqe; + + kfree(rcq->resize_buf); + rcq->resize_buf = NULL; + } + + spin_unlock_irq(&rcq->lock); + + /* Non-0 means that the above resize_buf non-empty branch has been entered, the old buffer needs to be released */ + if (old_cqe != 0) { + hmm_mtt_free(rdev->hwdev, &old_mtt, SERVICE_T_ROCE); + cqm5_object_resize_free_old(&rcq->cqm_cq->object); + } + +out: + mutex_unlock(&rcq->resize_mutex); + + return ret; +} + +static int roce5_resize_cq_check(struct ib_cq *ibcq, int entries, + const struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + + if (ibcq == NULL) { + ROCE_ERR("Ibcq is null"); + return -EINVAL; + } + + if ((ibcq->uobject != NULL) && (udata == NULL)) { + ROCE_ERR("Udata is null, but uobject is not null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibcq->device); + if ((entries < 1) || (entries > (int)rdev->rdma_cap.max_cqes)) { + ROCE_DEV_ERR(rdev, + "Resize CQEs invalid. entries(%d), func_id(%d)", + entries, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_resize_cq + Description : roce5_resize_cq + Input : struct ib_cq *ibcq + int entries + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_cq *rcq = NULL; + struct roce5_device *rdev = NULL; + + ret = roce5_resize_cq_check(ibcq, entries, udata); + if (ret != 0) { + ROCE_ERR("Failed to check resize_cq"); + return ret; + } + + rcq = to_roce5_cq(ibcq); + rdev = to_roce5_dev(ibcq->device); + + if (ibcq->uobject) { + ret = roce5_resize_user_cq(rdev, rcq, entries, udata); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to resize user cq, func_id(%u) ret(%d)", + (u32)rdev->glb_func_id, ret); + return ret; + } + } else { + ret = roce5_resize_kernel_cq(rdev, rcq, entries); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to resize kernel cq, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +void roce5_lock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __acquires(&roce5_send_cq->lock) __acquires(&roce5_recv_cq->lock) +{ + if (roce5_send_cq == roce5_recv_cq) { + spin_lock_irq(&roce5_send_cq->lock); + __acquire(&roce5_recv_cq->lock); + } else if (roce5_send_cq->cqn < roce5_recv_cq->cqn) { + spin_lock_irq(&roce5_send_cq->lock); + spin_lock_nested(&roce5_recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&roce5_recv_cq->lock); + spin_lock_nested(&roce5_send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +void roce5_unlock_cqs(struct roce5_cq *roce5_send_cq, + struct roce5_cq *roce5_recv_cq) + __releases(&roce5_send_cq->lock) __releases(&roce5_recv_cq->lock) +{ + if (roce5_send_cq == roce5_recv_cq) { + __release(&roce5_recv_cq->lock); + spin_unlock_irq(&roce5_send_cq->lock); + } else if (roce5_send_cq->cqn < roce5_recv_cq->cqn) { + spin_unlock(&roce5_recv_cq->lock); + spin_unlock_irq(&roce5_send_cq->lock); + } else { + spin_unlock(&roce5_send_cq->lock); + spin_unlock_irq(&roce5_recv_cq->lock); + } +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c new file mode 100644 index 000000000..c4e442598 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_destroy.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" + +#include "roce.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_mix.h" +#include "roce_xrc.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_npu_cmd.h" +#include "hinic5_hmm.h" +#include "securec.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_compat.h" +#include "roce_os_compat.h" +#include "roce_cq_intern.h" +#include "roce_com.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" + +/* and above next is V100 adaptation data structure */ +typedef struct tag_roce_cmd_cq_cache_invalidate { + roce_verbs_cmd_com_s com; + roce_xq_mtt_info_s mtt_info; +} roce_cmd_cq_cache_invalidate_s; +#define CQC_CHECK_RESCHED_LOOP_TIMES 0xffffU +static int roce5_check_cqc_data_state(struct roce5_device *rdev, + const struct roce5_cq *cq, + struct roce5_cq_context *cqc_data, + u32 check_state) +{ + int read_back_flag = 0; + u32 times = (u32)rdev->try_times; + struct roce5_cq_context check_cqc_data; + while ((times--) != 0) { + check_cqc_data.dw2.value = be32_to_cpu(cqc_data->dw2.value); + if (check_cqc_data.dw2.bs.state == check_state) { + read_back_flag = 1; + break; + } + ROCE_DELAY(); + + // limit schedule cond_resched count, preventstop thread schedule frequency past high + if ((times & CQC_CHECK_RESCHED_LOOP_TIMES) == 0) { + cond_resched(); + } + } + + if (read_back_flag == 0) { + ROCE_DEV_ERR( + rdev, + "Failed to read back after try %d times, Cqn(0x%x), CQ state(0x%x), func_id(%u)", + rdev->try_times, cq->cqn, cqc_data->dw2.value, + (u32)rdev->glb_func_id); + return -1; + } + + return 0; +} +/* **************************************************************************** + Prototype : roce5_cq_hw2sw + Description : roce5_cq_hw2sw + Input : struct roce5_device *rdev + struct roce5_cq *cq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_cq_hw2sw(struct roce5_device *rdev, struct roce5_cq *cq) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + struct roce5_cq_context *cqc_data = NULL; + roce_uni_cmd_cq_hw2sw_s *cq_hw2sw_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_cq_hw2sw_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + cq_hw2sw_inbuf = (roce_uni_cmd_cq_hw2sw_s *)cqm_cmd_inbuf->buf; + cq_hw2sw_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_CQ_BITMASK); + if (cq->shadow != 0) { + cq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_hw2sw_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(cq->shadow_vfid); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_HW2SW_CQ, + cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send HW2SW_CQ command, func_id(%u), cqn(0x%x), (ret:%d)", + (u32)rdev->glb_func_id, cq->cqn, ret); + + ret = (-EINVAL); + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + + /* CMDq may return a positive number, so its return value cannot be used directly */ + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + /* HI1825_V100 will retry load cqc in cmdq */ + if (ROCE5_SUPPORT_RETRY_LOAD_CQC_MSG_FEATURE(rdev) != 0) { + cqc_data = (struct roce5_cq_context *)((void *)cq->cqm_cq + ->q_ctx_vaddr); + return roce5_check_cqc_data_state(rdev, cq, cqc_data, + ROCE_CQ_TIME_OUT_CHECK_VALUE); + } + return 0; +} + +static int roce5_uni_cmd_cq_cache_out(struct roce5_device *rdev, + struct roce5_cq *cq, + cqm_cmd_buf_s **cqm_cmd_inbuf) +{ + int ret = 0; + struct dev_roce_svc_own_cap *roce_own_cap = NULL; + roce_uni_cmd_cq_hw2sw_s *cq_cache_invld_inbuf = NULL; + + /* Send the cache out command */ + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_cq_hw2sw_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce_own_cap = &rdev->rdma_cap.dev_rdma_cap.roce_own_cap; + cq_cache_invld_inbuf = (roce_uni_cmd_cq_hw2sw_s *)(*cqm_cmd_inbuf)->buf; + cq_cache_invld_inbuf->com.index = cpu_to_be32((u32)cq->cqn); + cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (cq->shadow != 0) { + cq_cache_invld_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_cache_invld_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(cq->shadow_vfid); + } + cq_cache_invld_inbuf->cmtt_cache.mtt_flags = 0; + cq_cache_invld_inbuf->cmtt_cache.mtt_num = 0; + cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_start = + cpu_to_be32(roce_own_cap->cmtt_cl_start); + cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_end = + cpu_to_be32(roce_own_cap->cmtt_cl_end); + cq_cache_invld_inbuf->cmtt_cache.mtt_cache_line_size = + cpu_to_be32(roce_own_cap->cmtt_cl_sz); + + return 0; +} + +static int roce5_cq_cache_out(struct roce5_device *rdev, struct roce5_cq *cq) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + if (ROCE5_SUPPORT_CQ_CACHE_OUT_MSG_V2(rdev) != 0) { + ret = roce5_uni_cmd_cq_cache_out(rdev, cq, &cqm_cmd_inbuf); + } + + if (ret != 0) { + return ret; + } + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_MISC_CQ_CACHE_INVLD, cqm_cmd_inbuf, + NULL, NULL, ROCE_CMD_TIME_CLASS_A, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send CQ_CACHE_INVLD command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + return 0; + } + + return -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +int roce5_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + struct roce5_cq *rcq = NULL; + struct roce5_ucontext *ucontext = NULL; + char waiting_msg[ROCE_WAITING_MSG_MAX_SIZE]; + roce_user_handle_t hdl = { 0 }; + + if (ibcq == NULL) { + ROCE_ERR("Ibcq is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibcq->device); + rcq = to_roce5_cq(ibcq); + + ret = sprintf_s(waiting_msg, ROCE_WAITING_MSG_MAX_SIZE, + "Waiting for roce cq(%u) release...", rcq->cqn); + if (ret < 0) { + ROCE_DEV_WARN(rdev, "Failed to setup waiting message, ret(%d).", + ret); + } + + roce5_option_unset(rcq->opt, waiting_msg, sizeof(waiting_msg)); + + /* Modify CQC to be owned by the software */ + ret = roce5_cq_hw2sw(rdev, rcq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify CQC, ret(%d), func_id(%d), cqn(0x%x)", + ret, rdev->glb_func_id, rcq->cqn); + return ret; + } + + /* Send the cache out command */ + ret = roce5_cq_cache_out(rdev, rcq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send cache out command, ret(%d), func_id(%hu)", + ret, rdev->glb_func_id); + return ret; + } + + ROCE_CONSTUCT_ULD_HDL_RCQ(&hdl, rdev, rcq, NULL, NULL, NULL, 0); + roce5_uld_cq_destroy((roce_uld_handle)&hdl); + + /* If it is user mode, you also need to cancel the mapping of the corresponding software DB */ + if (ibcq->uobject != NULL) { + ucontext = roce5_queue_to_ucontext((void *)ibcq, udata, + ROCE_Q_TYPE_CQ); + roce5_db_unmap_user(ucontext, &rcq->db, rcq->shadow); + roce5_cq_put_umem(rdev, &rcq->buf, &rcq->umem, rcq->shadow); + } else { + hmm_mtt_free(rdev->hwdev, &rcq->buf.mtt, SERVICE_T_ROCE); + } + + roce5_destroy_cq_dealloc_cqn(rdev, rcq); + return 0; +} + +int roce5_destroy_cq_ofa(struct ib_cq *ibcq) +{ + int ret = 0; + struct roce5_cq *rcq = NULL; + + if (ibcq == NULL) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + ret = roce5_destroy_cq(ibcq, NULL); + if (ret == 0) { + rcq = to_roce5_cq(ibcq); + kfree(rcq); + } + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h new file mode 100644 index 000000000..3c1cb6640 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_format.h @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQ_FORMAT_H +#define ROCE_CQ_FORMAT_H + +#include <linux/types.h> + +#include "roce_qp_ctx_format.h" + +typedef struct roce5_cq_context { + u32 rsvd[2]; + union { + struct { + u32 rsvd : 28; + u32 state : 4; + } bs; + u32 value; + } dw2; + + u32 rsvd1[12]; + + struct roce5_qpc_timer_seg timer_seg; + + u32 rsvd2[8]; +} roce5_cq_context_s; + +#endif /* ROCE_CQ_FORMAT_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c new file mode 100644 index 000000000..119fafba9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.c @@ -0,0 +1,98 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "hinic5_hw.h" +#include "securec.h" + +#include "roce.h" +#include "roce_user.h" +#include "roce_cq.h" +#include "roce_os_compat.h" +#include "roce_cq_intern.h" + +/* + * high performance cq, based on reserved of cq, first to firstget get. + * 特 point: supports 23V200 & 25V100 two types NIC, according to device type different to allocate different of cq. + * later get of cq so bind qp performance will has so next decrease. +*/ + +#define ROCE_CQN_PERF_BASE 1 + +int roce5_create_cq_alloc_cqn(struct roce5_device *rdev, struct roce5_cq *rcq, + u32 index) +{ + u32 cqn = index; + u32 offset = ROCE_CQN_PERF_BASE; + u32 step = rdev->cqn_ctrl.rsvd_cq_step; + u32 rsvd_cqs = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs; + + mutex_lock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + if (rdev->cqn_ctrl.alloc_cq_cnt == rdev->cqn_ctrl.high_perf_cq_cnt) { + /* The reserved high performance CQs have been fully allocated. */ + goto cqn_alloc; + } + + while (rdev->cqn_ctrl.alloc_cq_cnt < rdev->cqn_ctrl.high_perf_cq_cnt && + (offset < rsvd_cqs)) { + if (test_and_set_bit(offset, &rdev->cqn_ctrl.cq_bitmap) != 0) { + offset += step; + continue; + } + cqn = offset; + rdev->cqn_ctrl.alloc_cq_cnt++; + ROCE_DEV_INFO( + rdev, + "Alloc High performance CQ, func_id(%d), total_high_perf_cq(%u), cqn(%d)", + rdev->glb_func_id, rdev->cqn_ctrl.alloc_cq_cnt, cqn); + break; + } + +cqn_alloc: + mutex_unlock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + rcq->cqm_cq = cqm5_object_rdma_queue_create(rdev->hwdev, SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SCQ, 0, + rcq->opt, false, cqn, 0, 0); + if (rcq->cqm_cq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue from cqm object, func_id(%d), index(%d)", + rdev->glb_func_id, index); + return -ENOMEM; + } + + rcq->cqn = rcq->cqm_cq->index; + + return 0; +} + +void roce5_destroy_cq_dealloc_cqn(struct roce5_device *rdev, + struct roce5_cq *rcq) +{ + u32 cqn = rcq->cqn; + u32 rsvd_cqs = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs; + /* Call the CQM interface to release CQN and CQC. Since there is no cq_buf and software DB in user mode, + it does not need to be released; since kernel mode has both, it needs to be released */ + cqm5_object_delete(&rcq->cqm_cq->object); + kfree(rcq->opt); + rcq->opt = NULL; + + mutex_lock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + if ((cqn < rsvd_cqs) && + test_and_clear_bit(cqn, &rdev->cqn_ctrl.cq_bitmap) != 0 && + rdev->cqn_ctrl.alloc_cq_cnt > 0) { + rdev->cqn_ctrl.alloc_cq_cnt--; + ROCE_DEV_INFO( + rdev, + "Dealloc High performance CQ, func_id(%d), total_high_perf_cq(%u), cqn(%d)", + rdev->glb_func_id, rdev->cqn_ctrl.alloc_cq_cnt, cqn); + } + mutex_unlock(&rdev->cqn_ctrl.cq_ctrl_mutex); + + return; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h new file mode 100644 index 000000000..a2cb56943 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/cq/roce_cq_intern.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQ_INTERN_H +#define ROCE_CQ_INTERN_H + +#include <rdma/ib_verbs.h> + +#include "hinic5_hw.h" +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_cq.h" + +#define ROCE_CQN_INVLD CQM_INDEX_INVALID + +int roce5_create_cq_alloc_cqn(struct roce5_device *rdev, struct roce5_cq *rcq, + u32 index); +void roce5_destroy_cq_dealloc_cqn(struct roce5_device *rdev, + struct roce5_cq *rcq); + +#endif //ROCE_CQ_INTERN_H \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c new file mode 100644 index 000000000..0ce3ffb25 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.c @@ -0,0 +1,437 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_event_extension.h" +#include "roce_aeq_format.h" +#include "hinic5_mt.h" +#include "roce_restore.h" +#include "roce_infra.h" +#include "roce_cq.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif +#include "roce_event.h" + +static void roce5_event_report_common(const struct roce5_device *rdev, + int event_str_index) +{ + switch (event_str_index) { + case OFED_ET_PATH_MIG: + ROCE_DEV_INFO_LIMIT(rdev, + "OFED event: Path mig. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_COMM_EST: + ROCE_INFO_ONCE( + "OFED event: Communication establish. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_SQ_DRAINED: + ROCE_DEV_ERR_LIMIT(rdev, + "OFED event: Sq drained. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_SRQ_QP_LAST_WQE: + ROCE_ERR_ONCE("OFED event: Srq/qp last wqe. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_WQ_CATAS_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Wq catas error. function id: %hu", + rdev->glb_func_id); + break; + case OFED_ET_PATH_MIG_FAILED: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Path mig failed. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_WQ_INVAL_REQ_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Wq inval req error. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_WQ_ACCESS_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Wq access error. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_CQ_ERR: + ROCE_DEV_ERR_LIMIT(rdev, + "OFED event: Cq error. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_SRQ_LIMIT: + ROCE_DEV_ERR_LIMIT(rdev, + "OFED event: Srq limit. function id: %u", + rdev->glb_func_id); + break; + case OFED_ET_SRQ_CATAS_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, "OFED event: Srq catas error. function id: %u", + rdev->glb_func_id); + break; + case NON_OFED_ET_QPC_LOOKUP_ERR: + ROCE_DEV_ERR_LIMIT( + rdev, + "[non ofed event type] Qpc lookup error. function id: %u", + rdev->glb_func_id); + break; + case NON_OFED_ET_OTHER_TYPE_ERR: + ROCE_ERR_ONCE( + "[non ofed event type] other type error. function id: %u", + rdev->glb_func_id); + break; + default: + break; + } +} + +/* **************************************************************************** + Prototype : roce5_event_report + Description : roce5_event_report + Input : struct roce5_device *rdev + int event_str_index + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_event_report(struct roce5_device *rdev, u8 *val, + int event_str_index) +{ + if (event_str_index <= NON_OFED_ET_OTHER_TYPE_ERR) { + roce5_event_report_common(rdev, event_str_index); + } else { + roce5_event_report_extend(rdev, val, event_str_index); + } +} + +/* **************************************************************************** + Prototype : to_qp_event_str_index + Description : to_qp_event_str_index + Input : u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int to_qp_event_str_index(u8 event_type) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + return OFED_ET_PATH_MIG; + + case ROCE_EVENT_TYPE_COMM_EST: + return OFED_ET_COMM_EST; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + return OFED_ET_SQ_DRAINED; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + return OFED_ET_SRQ_QP_LAST_WQE; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + return OFED_ET_WQ_CATAS_ERR; + + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + return OFED_ET_PATH_MIG_FAILED; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + return OFED_ET_WQ_INVAL_REQ_ERR; + + default: + return OFED_ET_WQ_ACCESS_ERR; + } +} + +/* **************************************************************************** + Prototype : roce5_handle_qp_async_event + Description : roce5_handle_qp_async_event + Input : struct roce5_device *rdev + u32 qpn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_qp_async_event(struct roce5_device *rdev, u32 qpn, + u8 event_type) +{ + struct roce5_qp *rqp = NULL; + cqm_object_s *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm5_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, + qpn, false); + if (cqm_obj_queue == NULL) { + ROCE_DEV_ERR_LIMIT( + rdev, + "Failed to get cqm obj queue, func_id(%d), qpn:%u, event_type:%u", + rdev->glb_func_id, qpn, event_type); + return; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_queue); + roce5_qp_async_event(rdev, rqp, (int)event_type); + cqm5_object_put(cqm_obj_queue); +} + +/* **************************************************************************** + Prototype : roce5_handle_cq_async_event + Description : roce5_handle_cq_async_event + Input : struct roce5_device *rdev + u32 cqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_cq_async_event(struct roce5_device *rdev, u32 cqn, + u8 event_type) +{ + struct roce5_cq *rcq = NULL; + + rcq = roce5_get_rcq_by_cqn(rdev, cqn); + if (rcq == NULL) { + ROCE_DEV_ERR(rdev, "Failed to get roce cq by cqn(%u).", cqn); + return; + } + + roce5_cq_async_event(rdev, rcq, (int)event_type); + roce5_rcq_put(rcq); +} + +/* **************************************************************************** + Prototype : roce5_handle_srq_async_event + Description : roce5_handle_srq_async_event + Input : struct roce5_device *rdev + u32 srqn + u8 event_type + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static void roce5_handle_srq_async_event(struct roce5_device *rdev, u32 srqn, + u8 event_type) +{ + struct roce5_srq *rsrq = NULL; + cqm_object_s *cqm_obj_queue = NULL; + + cqm_obj_queue = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_queue == NULL) { + ROCE_DEV_ERR_LIMIT( + rdev, + "Failed to get cqm obj queue, func_id(%d), srqn:%u, event_type:%u", + rdev->glb_func_id, srqn, event_type); + return; + } + + rsrq = cqmobj_to_roce5_srq(cqm_obj_queue); + roce5_srq_async_event(rdev, rsrq, (int)event_type); + cqm5_object_put(cqm_obj_queue); +} + +static bool roce5_async_event_need_reset(u8 event_status) +{ + if ((event_status == API_FORMAT_ERROR) || + (event_status == UNKNOWN_RESERVED_ERROR1)) { + return true; + } + + return false; +} + +u8 roce5_async_event_level(void *svc_hd, u8 event_type, u8 *val) +{ + u8 event_level = FAULT_LEVEL_MAX; + u8 event_status; + u64 param = *((u64 *)val); + + if (svc_hd == NULL) { + ROCE_ERR_LIMIT("Svc_hd is null"); + return FAULT_LEVEL_SUGGESTION; + } + + event_status = param & 0xff; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_SRQ_LIMIT: + case ROCE_EVENT_TYPE_DEBUG: + event_level = FAULT_LEVEL_SUGGESTION; + break; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: //ROCE_AEQE_EVENT_QP_REQ_ERR + if (event_status == API_FORMAT_ERROR) { + return FAULT_LEVEL_SERIOUS_RESET; + } + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: //ROCE_AEQE_EVENT_WQE_CATAS + if (roce5_async_event_need_reset(event_status)) { + return FAULT_LEVEL_SERIOUS_RESET; + } + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: //CQ ERR + if (roce5_async_event_need_reset(event_status)) { + return FAULT_LEVEL_SERIOUS_RESET; + } + break; + + default: + event_level = FAULT_LEVEL_GENERAL; + break; + } + + return event_level; +} + +static int roce5_async_event_handle_common(u8 event_type, u8 *val, + struct roce5_device *rdev) +{ + u32 xid = 0; + int event_str_index = 0; + + switch (event_type) { + case ROCE_EVENT_TYPE_PATH_MIG: + case ROCE_EVENT_TYPE_COMM_EST: + case ROCE_EVENT_TYPE_SQ_DRAINED: + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case ROCE_EVENT_TYPE_PATH_MIG_FAILED: + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + xid = *((u32 *)val); + roce5_handle_qp_async_event(rdev, xid, event_type); + event_str_index = to_qp_event_str_index(event_type); + break; + + case ROCE_EVENT_TYPE_CQ_ERROR: + xid = *((u32 *)val); + roce5_handle_cq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_CQ_ERR; + break; + + case ROCE_EVENT_TYPE_SRQ_LIMIT: + xid = *((u32 *)val); + roce5_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_LIMIT; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + xid = *((u32 *)val); + roce5_handle_srq_async_event(rdev, xid, event_type); + event_str_index = OFED_ET_SRQ_CATAS_ERR; + break; + + default: + event_str_index = to_unkown_event_str_index(event_type, val); + break; + } + + return event_str_index; +} + +/* **************************************************************************** + Prototype : roce5_async_event + Description : roce5_async_event + Input : void *svc_hd + u8 event_type + u64 val + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_async_event(void *svc_hd, u8 event_type, u8 *val) +{ + int event_str_index = 0; + struct roce5_device *rdev = NULL; + + if (svc_hd == NULL) { + ROCE_ERR_LIMIT("Svc_hd is null"); + return; + } + + rdev = (struct roce5_device *)svc_hd; + if (event_type <= ROCE_EVENT_TYPE_ODP_PAGE_FAULT) { + event_str_index = + roce5_async_event_handle_common(event_type, val, rdev); + } else { + event_str_index = + roce5_async_event_handle_extend(event_type, val, rdev); + } + + roce5_event_report(rdev, val, event_str_index); +} + +#ifdef ROCE_BONDING_EN +void roce5_handle_bonded_port_state_event(struct roce5_device *rdev) +{ + struct net_device *master = netdev_master_upper_dev_get_rcu(rdev->ndev); + enum ib_port_state bonded_port_state = IB_PORT_NOP; + struct roce5_bond_device *bond_dev = NULL; + enum ib_port_state curr_port_state; + enum ib_event_type event; + struct roce5_bond_slave *slave = NULL; + u32 i; + + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + ROCE_DEV_ERR(rdev, "No bond_dev found"); + return; + } + + if (master == NULL) { + bonded_port_state = IB_PORT_DOWN; + } else { + if (netif_running(master) != 0) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->netdev == NULL) { + continue; + } + + curr_port_state = + (netif_running(slave->netdev) && + netif_carrier_ok(slave->netdev)) ? + IB_PORT_ACTIVE : + IB_PORT_DOWN; + bonded_port_state = + (bonded_port_state != IB_PORT_ACTIVE) ? + curr_port_state : + IB_PORT_ACTIVE; + } + mutex_unlock(&bond_dev->slave_lock); + } else { + bonded_port_state = IB_PORT_DOWN; + } + } + + if (rdev->port_state != bonded_port_state) { + event = (bonded_port_state == IB_PORT_ACTIVE) ? + IB_EVENT_PORT_ACTIVE : + IB_EVENT_PORT_ERR; + if (rdev->port_state != IB_PORT_NOP) { + roce5_ifconfig_up_down_event_report(rdev, event); + } + rdev->port_state = bonded_port_state; + } +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h new file mode 100644 index 000000000..b5acc9b15 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/event/roce_event.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_EVENT_H +#define ROCE_EVENT_H + +#include <linux/types.h> + +#define ROCE_M_DELAY 100 +#define API_FORMAT_ERROR 0x2 +#define UNKNOWN_RESERVED_ERROR1 0xe +#define QPC_LOOKUP_ERR_VALUE 0x300d00016 + +struct roce5_aeqe_event_data { + u32 xid; + + //microcode side by ROCE_AEQE_EVENT construct, driver side byte order differs from microcode side + u32 err_code : 8; + u32 type : 12; + u32 subtype : 8; + u32 module_type : 4; +}; + +#endif /* ROCE_EVENT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c new file mode 100644 index 000000000..c33bab11a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_event_extension.c @@ -0,0 +1,56 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#include <linux/types.h> + +#include "roce_event_extension.h" +#include "roce_aeq_format.h" + +int to_unkown_event_str_index(uint8_t event_type, const uint8_t *val) +{ + switch (event_type) { + case ROCE_EVENT_TYPE_OFED_NO_DEF: + if (*((u64 *)val) == QPC_LOOKUP_ERR_VALUE) { + return NON_OFED_ET_QPC_LOOKUP_ERR; + } + return NON_OFED_ET_OTHER_TYPE_ERR; + + case ROCE_EVENT_TYPE_LOCAL_CATAS_ERROR: + return NON_OFED_ET_OTHER_TYPE_ERR; + + case ROCE_EVENT_TYPE_DEBUG: + return NON_OFED_ET_DEBUG_ERR; + + default: + return INVAL_ET_ERR; + } +} + +void roce5_event_report_extend(const struct roce5_device *rdev, u8 *val, + int event_str_index) +{ + if (event_str_index == NON_OFED_ET_DEBUG_ERR) { + struct roce5_aeqe_event_data *data = + (struct roce5_aeqe_event_data *)val; + /* debug subtype according to ROCE_AEQE_DEBUG_SUBTYPE_E */ + ROCE_ERR_ONCE( + "[non ofed event type] Debug extend event subtype 0x%x. function id: %u", + data->subtype, rdev->glb_func_id); + return; + } + + ROCE_ERR_ONCE( + "[non ofed event type] Invalid extend event type error. function id: %u", + rdev->glb_func_id); +} + +int roce5_async_event_handle_extend(u8 event_type, u8 *val, + struct roce5_device *rdev) +{ + int event_str_index = 0; + + event_str_index = to_unkown_event_str_index(event_type, val); + + return event_str_index; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c new file mode 100644 index 000000000..759ba598f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_main_extension.c @@ -0,0 +1,170 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_main_extension.h" +#include "roce_infra.h" +#include "roce_os_compat.h" +#include "roce_kernel_compat_gen.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" + +static struct mutex g_rdev_mutex; +#endif + +void roce5_service_init_pre(void) +{ +#ifdef ROCE_BONDING_EN + mutex_init(&g_rdev_mutex); +#endif + + return; +} + +void roce5_lock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_lock(&g_rdev_mutex); +#endif +} + +void roce5_unlock_rdev(void) +{ +#ifdef ROCE_BONDING_EN + mutex_unlock(&g_rdev_mutex); +#endif +} + +int roce5_get_rdev_by_uld(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, + struct hinic5_event_info *event) +{ +#ifdef ROCE_BONDING_EN + int ret; + + ret = roce5_bond_event_cfg_rdev(lld_dev, uld_dev, rdev, false); + if (ret != 0) { + return -1; + } +#else + *rdev = (struct roce5_device *)uld_dev; +#endif + return 0; +} + +#ifndef ROCE_CHIP_TEST +void roce5_init_dev_ext_handlers(struct roce5_device *rdev) +{ + return; +} +#endif + +int roce5_board_cfg_check(struct roce5_device *rdev) +{ + int ret = 0; + int port_num = 0; + int port_speed = 0; + + port_num = rdev->board_info.port_num; + port_speed = rdev->board_info.port_speed; + if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_100G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_100G_HCA; + } else if ((port_num == ROCE5_4_PORT_NUM) && + (port_speed == ROCE5_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_4_25G_HCA; + } else if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_25G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_25G_HCA; + } else if ((port_num == ROCE5_1_PORT_NUM) && + (port_speed == ROCE5_100G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_1_100G_HCA; + } else if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_200G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_200G_HCA; + } else if ((port_num == ROCE5_2_PORT_NUM) && + (port_speed == ROCE5_400G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_2_400G_HCA; + } else if ((port_num == ROCE5_4_PORT_NUM) && + (port_speed == ROCE5_200G_PORT_SPEED)) { + rdev->hw_info.hca_type = ROCE5_4_200G_HCA; + } else { + ROCE_DEV_ERR( + rdev, + "Invalid fw cfg, port_num(%d), port_speed(%d), hca_type(%d)", + port_num, port_speed, rdev->hw_info.hca_type); + ret = (-EINVAL); + } + + return ret; +} + +void *roce5_rdev_alloc_ext(void) +{ +#if defined(HAVE__IB_ALLOC_DEVICE) + return (void *)ib_alloc_device(roce5_device, ib_dev); +#else + return (void *)ib_alloc_device(sizeof(struct roce5_device)); +#endif +} + +int roce5_set_comm_event(const struct roce5_device *rdev, + const struct hinic5_event_info *event) +{ + ROCE_DEV_INFO(rdev, "Comm event unsupported, func_id(%d), event(%d)", + rdev->glb_func_id, event->type); + return -1; +} + +struct roce5_ucontext * +roce5_queue_to_ucontext(void *ibq, struct ib_udata *udata, int q_type) +{ + struct roce5_ucontext *rctx = NULL; + +#if defined(UVERBS_IOCTL_HAVE_RDMA_UDATA_TO_DRV_CONTEXT) + rctx = rdma_udata_to_drv_context(udata, struct roce5_ucontext, + ibucontext); +#else + struct ib_qp *ibqp = NULL; + struct ib_cq *ibcq = NULL; + struct ib_srq *ibsrq = NULL; + switch (q_type) { + case ROCE_Q_TYPE_QP: + ibqp = (struct ib_qp *)ibq; + rctx = to_roce5_ucontext(ibqp->uobject->context); + break; + case ROCE_Q_TYPE_CQ: + ibcq = (struct ib_cq *)ibq; + rctx = to_roce5_ucontext(ibcq->uobject->context); + break; + case ROCE_Q_TYPE_SRQ: + ibsrq = (struct ib_srq *)ibq; + rctx = to_roce5_ucontext(ibsrq->uobject->context); + break; + default: + ROCE_ERR("Unknown queue type(%d)", q_type); + break; + } +#endif + return rctx; +} + +#ifndef ROCE_VROCE_EN +void roce5_fill_gid_entry_ext(const struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + return; +} + +void roce5_cfg_info_update_ext(struct roce5_device *rdev, + roce_get_cfg_info_resp_s *resp) +{ + return; +} + +int roce5_add_check_ext(const struct hinic5_lld_dev *lld_dev) +{ + return 0; +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c new file mode 100644 index 000000000..7f2e14669 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_mr_extension.c @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + +#include "roce_mr.h" +#include "roce_kernel_compat_gen.h" +#include "roce_main_extension.h" +#include "roce_mr_extension.h" + +#ifndef ROCE_CHIP_TEST +int roce5_check_alloc_mr_type(enum ib_mr_type mr_type) +{ + int ret = 0; + + if (mr_type != IB_MR_TYPE_MEM_REG) { + ret = -EINVAL; + ROCE_ERR("mr_type is invalid. mr_type:%u", mr_type); + } + + return ret; +} + +enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type) +{ + switch (ib_mr_type) { +#if defined(IB_MR_TYPE_HAVE_USER) && defined(IB_MR_TYPE_HAVE_INTEGRITY) + case IB_MR_TYPE_MEM_REG: + return RDMA_FRMR; + case IB_MR_TYPE_USER: + return RDMA_USER_MR; + case IB_MR_TYPE_INTEGRITY: + return RDMA_SIG_MR; +#else + case IB_MR_TYPE_MEM_REG: + return RDMA_DMA_MR; +#endif + default: + return RDMA_DMA_MR; + } +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c new file mode 100644 index 000000000..0e0170e9d --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_qp_extension.c @@ -0,0 +1,222 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_qp_extension.h" +#include "roce_ctx_format.h" +#include "roce_restore.h" + +#ifdef ROCE_EXTEND +#include "roce_pd.h" +#endif + +int to_roce5_qp_type(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return ROCE_QP_ST_RC; + + case IB_QPT_UC: + return ROCE_QP_ST_UC; + + case IB_QPT_UD: + return ROCE_QP_ST_UD; + + case IB_QPT_XRC_INI: + case IB_QPT_XRC_TGT: + return ROCE_QP_ST_XRC; + + case IB_QPT_GSI: + return ROCE_QP_ST_UD; + + default: + return -1; + } +} + +bool roce5_check_qp_modify_ok(enum ib_qp_state cur_state, + enum ib_qp_state next_state, enum ib_qp_type type, + enum ib_qp_attr_mask mask, + enum rdma_link_layer ll) +{ +#if defined(IB_MODIFY_QP_IS_OK_HAVE_RDMA_LINK_LAYER) + return (bool)ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll); +#else + return (bool)ib_modify_qp_is_ok(cur_state, next_state, type, mask); +#endif +} + +int roce5_is_qp_normal(struct roce5_qp *rqp, struct ib_qp_init_attr *init_attr) +{ + return 1; +} + +#ifdef ROCE_EXTEND +static struct roce5_qp * +roce5_cdev_lookup_and_check_rqp(struct roce5_device *rdev, u32 qpn) +{ + struct roce5_qp *rqp = NULL; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Can't find rqp according to qpn(0x%x), func_id(%d)", + qpn, rdev->glb_func_id); + return NULL; + } + + if (rqp->qpn >= QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC) { + ROCE_DEV_ERR(rdev, + "qpn[%u] more than sqpc num offset(%d) for sqpc.", + rqp->qpn, QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + roce5_rqp_put(rqp); + return NULL; + } + + return rqp; +} + +static int roce5_alloc_sqpc(struct roce5_device *rdev, struct roce5_qp *rqp, + struct roce5_vbs_qp *vbs_rqp) +{ + cqm_qpc_mpt_s *qpc_info = NULL; + + /* Call the CQM interface to allocate QPNs and QPCs */ + /* For SQPC, do not need to allocate QPN, QPN is directly spcified in rsvd segment */ + qpc_info = cqm5_object_qpc_mpt_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, NULL, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, 0, 0); + if (qpc_info == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create qpc by cqm object, func_id(%d), qpn(%u)", + rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC); + return -ENOMEM; + } + + if (qpc_info->xid != (rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC)) { + ROCE_DEV_ERR( + rdev, + "Create qpc error, func_id(%d), expect qpn(%d), actual qpn(%d)", + rdev->glb_func_id, + rqp->qpn + QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC, + qpc_info->xid); + cqm5_object_delete(&(qpc_info->object)); + return -EFAULT; + } + + vbs_rqp->vbs_sqpc_info = qpc_info; + rqp->vbs_qp_ptr = (void *)vbs_rqp; + + return 0; +} + +long roce5_vbs_create_sqpc(struct roce5_device *rdev, void *buf) +{ + struct roce5_modify_qp_vbs_cmd cmd; + struct roce5_vbs_qp *vbs_rqp = NULL; + struct roce5_qp *rqp = NULL; + struct roce5_pd *pd = NULL; + int ret; + + ret = (int)copy_from_user(&cmd, buf, sizeof(cmd)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to copy data from user"); + return (long)ret; + } + + rqp = roce5_cdev_lookup_and_check_rqp(rdev, cmd.qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + vbs_rqp = (struct roce5_vbs_qp *)kzalloc(sizeof(*vbs_rqp), GFP_KERNEL); + if (vbs_rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc vbs_rqp, func_id(%d)", + rdev->glb_func_id); + ret = -ENOMEM; + goto put_rqp; + } + + pd = roce5_get_pd(rqp); + ret = roce5_db_map_user(to_roce5_ucontext(pd->ibpd.uobject->context), + cmd.ci_record_addr, &vbs_rqp->db, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to map db page to user, func_id(%d)", + rdev->glb_func_id); + goto free_rqp; + } + + ret = roce5_alloc_sqpc(rdev, rqp, vbs_rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc sqpc, func_id(%d)", + rdev->glb_func_id); + goto free_rqp; + } + + roce5_rqp_put(rqp); + return 0; + +free_rqp: + kfree(vbs_rqp); +put_rqp: + roce5_rqp_put(rqp); + return (long)ret; +} +#endif + +#ifndef ROCE_CHIP_TEST +void roce5_set_qp_dif_attr(struct roce5_qp *rqp, + const struct ib_qp_init_attr *init_attr, + const struct roce5_device *rdev) +{ + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EN) != 0) { + rqp->signature_en = true; + ROCE_DEV_INFO(rdev, + "func(%d) qp(%u) roce5_create_qp signature_en.", + rdev->glb_func_id, rqp->qpn); + } + return; +} +#endif + +#ifndef ROCE_VBS_EN +void roce5_set_attr_ext(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, int attr_mask) +{ + return; +} + +void roce5_transmit_ci_record_addr_ext(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr) +{ + return; +} +#endif + +#ifndef ROCE_VBS_EN +int roce5_modify_qp_pre_ext(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + return 0; +} + +void roce5_destroy_pcqc_ext(struct roce5_qp *rqp) +{ + return; +} + +#endif + +#ifndef ROCE_VROCE_EN + +void roce5_cos_update_ext(struct roce5_device *rdev, u8 *cos, + enum ib_qp_type qp_type) +{ + *cos = *cos & 0x7; +} +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c new file mode 100644 index 000000000..7067e8c80 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/extension/roce_srq_extension.c @@ -0,0 +1,55 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include "roce_srq_extension.h" + +u8 roce5_get_container_sz(u32 container_mode) +{ + switch (container_mode) { + case ROCE_SRQ_MODE_0: + return ROCE_SRQ_CONT_NUM_MODE0; + case ROCE_SRQ_MODE_1: + return ROCE_SRQ_CONT_NUM_MODE1; + case ROCE_SRQ_MODE_2: + return ROCE_SRQ_CONT_NUM_MODE2; + case ROCE_SRQ_MODE_3: + return ROCE_SRQ_CONT_NUM_MODE3; + default: + return ROCE_SRQ_CONT_NUM_MODE3; + } +} + +u8 roce5_calculate_cont_th(u32 srq_limit) +{ + u8 cont_th = 0; + u32 srq_limit_tmp = srq_limit; + + srq_limit_tmp >>= 1; + while (srq_limit_tmp != 0) { + srq_limit_tmp >>= 1; + ++cont_th; + } + + return cont_th; +} + +#ifndef ROCE_CHIP_TEST +void roce5_srq_container_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, struct roce5_device *rdev) +{ + rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC); + + if (rsrq->xrc_en != 0) { + rsrq->container_flag = rdev->cfg_info.srq_container_en; + } + rsrq->container_mode = (rsrq->xrc_en != 0) ? + rdev->cfg_info.xrc_srq_container_mode : + rdev->cfg_info.srq_container_mode; + rsrq->container_warn_th = + roce5_calculate_cont_th(init_attr->attr.srq_limit); + rsrq->rqe_cnt_th = rdev->cfg_info.warn_th; + rsrq->container_size = roce5_get_container_sz(rsrq->container_mode); +} +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c new file mode 100644 index 000000000..f7f9117f4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.c @@ -0,0 +1,1281 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_mr.h" +#include "roce_mr_extension.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "rdma_comp_mw_mr.h" +#include "hmm_common.h" +#include "roce_os_compat.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" + +/* **************************************************************************** + Prototype : get_key_from_index + Description : get_key_from_index + Input : u32 mpt_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static u32 get_key_from_index(u32 mpt_index) +{ + /* Algorithm of mr key is obtained by index shift calculation */ + return (mpt_index >> MR_KEY_RIGHT_SHIFT_OFS) | + (mpt_index << MR_KEY_LEFT_SHIFT_OFS); +} + +/* **************************************************************************** + Prototype : convert_ib_access + Description : convert ib access to rdma comp access + Input : int access_flag + Output : None + + 1.Date : 2015/7/16 + Modification : Created function + +**************************************************************************** */ +static u32 convert_ib_access(int access_flag) +{ + u32 u_access_flag = (u32)access_flag; + u32 access; + + access = (u32)((((u_access_flag & IB_ACCESS_REMOTE_ATOMIC) != 0) ? + RDMA_IB_ACCESS_REMOTE_ATOMIC : + 0) | + (((u_access_flag & IB_ACCESS_REMOTE_WRITE) != 0) ? + RDMA_IB_ACCESS_REMOTE_WRITE : + 0) | + (((u_access_flag & IB_ACCESS_REMOTE_READ) != 0) ? + RDMA_IB_ACCESS_REMOTE_READ : + 0) | + (((u_access_flag & IB_ACCESS_LOCAL_WRITE) != 0) ? + RDMA_IB_ACCESS_LOCAL_WRITE : + 0) | + (((u_access_flag & IB_ACCESS_MW_BIND) != 0) ? + RDMA_IB_ACCESS_MW_BIND : + 0) | + (((u_access_flag & IB_ZERO_BASED) != 0) ? + RDMA_IB_ACCESS_ZERO_BASED : + 0)); + + return access; +} + +/* **************************************************************************** + Prototype : check_ib_access + Description : check remote access without local write + Input : int access_flag + Output : None + + 1.Date : 2017/11/18 + Modification : Created function + +**************************************************************************** */ +static int check_ib_access(int access_flag) +{ + /* + * Local write permission is required if remote write or + * remote atomic permission is also requested. + */ + + if (((((u32)access_flag) & + (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE)) != 0) && + ((((u32)access_flag) & IB_ACCESS_LOCAL_WRITE) == 0)) { + return -EINVAL; + } + + return 0; +} + +u32 roce5_get_mpt_xid_mode(struct roce5_device *rdev) +{ + u8 mode = rdev->mpt_xid_mode; + + if (mode == ROCE_MPT_XID_SEQUENTIAL) + return CQM_INDEX_INVALID; + + return CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, CQM_XID_LOW_BIT_0_0_1, + (u32)mode); +} + +/* **************************************************************************** + Prototype : roce5_alloc_tpt + Description : alloc mpt and mtt + Input : struct roce5_device *rdev + struct hmm_rdma *mr + u32 npages + u32 page_shift + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_alloc_tpt(struct roce5_device *rdev, struct hmm_rdma *mr, u32 npages, + u32 page_shift, u32 mpt_xid_mode) +{ + int ret = 0; + + ret = hmm_mpt_alloc_templated(rdev->hwdev, &mr->mpt, SERVICE_T_ROCE, + mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + return ret; + } + + mr->enabled = RDMA_MPT_EN_SW; + + /* npages = 0 is a legal case, when npages = 0 or npages = 1, MTT does not need to do address translation */ + ret = hmm_mtt_alloc(rdev->hwdev, npages, page_shift, &mr->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mtt, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto err_alloc_mtt; + } + + return 0; + +err_alloc_mtt: + hmm_mpt_free(rdev->hwdev, &mr->mpt); + mr->enabled = RDMA_MPT_DISABLED; + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_free_tpt + Description : free mpt and mtt + Input : struct roce5_device *rdev + struct hmm_rdma *mr + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +void roce5_free_tpt(struct roce5_device *rdev, struct hmm_rdma *mr) +{ + hmm_mtt_free(rdev->hwdev, &mr->mtt, SERVICE_T_ROCE); + + hmm_mpt_free(rdev->hwdev, &mr->mpt); + + mr->enabled = RDMA_MPT_DISABLED; +} + +/* **************************************************************************** + Prototype : roce5_set_rdma_mr + Description : set the member of rdma_mr + Input : struct hmm_rdma *mr + enum rdma_mr_type mr_type + u32 pdn + u64 iova + u64 size + u32 access + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +void roce5_set_rdma_mr(struct hmm_rdma *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access) +{ + mr->iova = iova; + mr->size = size; + mr->pdn = pdn; + mr->access = convert_ib_access(( + int)access); /* Convert ib permissions to rdma component permissions */ + mr->key = get_key_from_index( + mr->mpt.mpt_index); /* Convert from mpt index to key */ + mr->type = roce5_mr_type2hmm(mr_type); +} + +/* **************************************************************************** + Prototype : roce5_set_rdma_mw + Description : set the member of rdma_mw + Input : struct rdma_mw *mw + u32 pdn + enum ib_mw_type type + Output : None + + 1.Date : 2016/6/25 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_rdma_mw(struct rdma_mw *mw, u32 pdn, enum ib_mw_type type) +{ + mw->enabled = RDMA_MPT_EN_SW; + mw->pdn = pdn; + mw->key = get_key_from_index(mw->mpt.mpt_index); + mw->type = (type == IB_MW_TYPE_1 ? RDMA_MW_TYPE_1 : RDMA_MW_TYPE_2); +} +#ifdef __ROCE_DFX__ +static void roce5_store_mr_list(struct roce5_device *rdev, + struct list_head *node) +{ + struct roce5_list *roce_list = NULL; + + if (rdev->device_res == NULL) { + ROCE_INFO("Hasn't been created resource, don't need store."); + return; + } + + roce_list = &rdev->device_res->mr_list; + mutex_lock(&roce_list->node_lock); + list_add(node, &roce_list->node); + mutex_unlock(&roce_list->node_lock); +} + +static void roce5_delete_mr_list(struct roce5_device *rdev, + struct list_head *node) +{ + struct roce5_list *roce_list = NULL; + + if (rdev->device_res == NULL) { + ROCE_INFO("Resource has been release, don't need delete."); + return; + } + + roce_list = &rdev->device_res->mr_list; + mutex_lock(&roce_list->node_lock); + list_del(node); + mutex_unlock(&roce_list->node_lock); +} +#endif //__ROCE_DFX__ + +/* **************************************************************************** + Prototype : roce5_get_dma_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + int access + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +struct ib_mr *roce5_get_dma_mr(struct ib_pd *ibpd, int access) +{ + int ret = 0; + u32 mpt_xid_mode; + struct roce5_mr *mr = NULL; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if ((ibpd == NULL) || (check_ib_access(access) != 0)) { + ret = -EINVAL; + ROCE_ERR("Invalid Param.p1:%d", access); + goto err_out; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + mr = (struct roce5_mr *)kzalloc(sizeof *mr, GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, + "Failed to alloc memory for dma mr, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, 0, 0, mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, func_id(%d)", + rdev->glb_func_id); + goto err_alloc_tpt; + } + + /* Set the content in rdma_mr */ + roce5_set_rdma_mr(&mr->rdmamr, RDMA_DMA_MR, pd->pdn, 0ULL, + ROCE_DMA_MR_SIZE, (u32)access); + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(mr->rdmamr), + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of DMA mr, func_id(%d)", + rdev->glb_func_id); + goto err_enable_mpt; + } +#ifdef __ROCE_DFX__ + roce5_store_mr_list(rdev, &mr->node); +#endif + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + mr->ibmr.device = ibpd->device; + + ROCE_DEV_INFO(rdev, "Roce reg dma mr success, mr mpt index is %u", + mr->rdmamr.mpt.mpt_index); + + return &mr->ibmr; + +err_enable_mpt: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce5_alloc_priv_pages(struct ib_device *ibdev, struct roce5_mr *rmr, + u32 max_pages) +{ + int ret; + + rmr->page_map_size = + roundup(max_pages * sizeof(u64), ROCE5_MR_PAGES_ALIGN); + rmr->pages = (__be64 *)(void *)(uintptr_t)get_zeroed_page(GFP_KERNEL); + if (rmr->pages == NULL) { + ROCE_ERR("Failed to alloc rmr->pages"); + ret = -ENOMEM; + goto err_out; + } + + rmr->page_map = dma_map_single(ibdev->dev.parent, rmr->pages, + rmr->page_map_size, DMA_TO_DEVICE); + if (dma_mapping_error(ibdev->dev.parent, rmr->page_map) != 0) { + ROCE_ERR("Failed to do dma mapping"); + ret = -ENOMEM; + goto err_free_pages; + } + + return 0; + +err_free_pages: + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + +err_out: + return ret; +} + +void roce5_free_frmr_umem(struct ib_umem **fumem, int fumem_nr) +{ + int i; + + ROCE_INFO("%d frmr umems to be free.", fumem_nr); + for (i = 0; i < fumem_nr; i++) { + ib_umem_release(fumem[i]); + } +} + +static void roce5_free_priv_pages(struct roce5_mr *rmr) +{ + struct ib_device *ib_dev = rmr->ibmr.device; + + if (rmr->pages != NULL) { + ib_dev = rmr->ibmr.device; + dma_unmap_single(ib_dev->dev.parent, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + free_page((unsigned long)(uintptr_t)(void *)rmr->pages); + rmr->pages = NULL; + } +} + +static int roce5_alloc_mr_param_validate(const struct ib_pd *ibpd, + enum ib_mr_type mr_type, + u32 max_num_sg) +{ + int ret = 0; + + if (ibpd == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibpd is null"); + goto err_out; + } + + ret = roce5_check_alloc_mr_type(mr_type); + if (ret != 0) { + ret = -EINVAL; + ROCE_ERR("mr_type is invalid"); + goto err_out; + } + + if (max_num_sg >= ROCE_FRMR_MAX_PAGES) { + ROCE_ERR("Invalid max_num_sg(%d)", max_num_sg); + ret = -EINVAL; + goto err_out; + } + + return 0; + +err_out: + return ret; +} + +static int roce5_alloc_mr_param_check(struct roce5_device *rdev, + struct roce5_mr **mr, u32 max_num_sg) +{ + *mr = (struct roce5_mr *)kzalloc(sizeof(struct roce5_mr), GFP_KERNEL); + if ((*mr) == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc mr memory, func(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + (*mr)->max_pages = max_num_sg; + return 0; +} +/* **************************************************************************** + Prototype : roce5_alloc_mr + Description : register DMA_MR + Input : struct ib_pd *ibpd + enum ib_mr_type mr_type + u32 max_num_sg + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +struct ib_mr *roce5_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, + u32 max_num_sg) +{ + int ret = 0; + u16 sign_val = 0; + u32 mpt_xid_mode; + struct roce5_mr *mr = NULL; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + ret = roce5_alloc_mr_param_validate(ibpd, mr_type, max_num_sg); + if (ret != 0) { + goto err_out; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + ret = roce5_alloc_mr_param_check(rdev, &mr, max_num_sg); + if (ret != 0) { + goto err_out; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, max_num_sg, 0, + mpt_xid_mode); //alloc mptc and mtt + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc tpt, func(%d)", + rdev->glb_func_id); + goto err_alloc_tpt; + } + + ret = roce5_alloc_priv_pages(ibpd->device, mr, + max_num_sg); //alloc memory to store pa + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc memory, func(%u)", + rdev->glb_func_id); + goto err_alloc_priv_pages; + } + + roce5_set_rdma_mr(&mr->rdmamr, roce5_get_mrtype(mr_type), pd->pdn, 0ULL, + max_num_sg * (1 << PAGE_SHIFT), + ROCE5_MR_DEFAULT_ACCESS); + +#ifdef SIGN_MTT_EN + /* + MTT table entry of bit[11:2] used for fill in addr_signature, to hardware used for check MTT table entry complete validity. [1:1] is vmr, indicates current page whether be swap out. + bit[0:0] is vld bit indicates pointed by of next one level DMTT or Data Buffer of GPA whether valid. use sign_val record [11:0] this 12bit data, exist mr in. + send down frmr wqe after chip will do dma use pbl table fill MTT table, here need in advance calculate check value, in roce5_set_page() in write pbl table. + */ + sign_val = (u16)(hmm_gen_mtt_sign(mr->rdmamr.mtt.mtt_paddr, + mr->rdmamr.mtt.mtt_type) & + 0xffff); +#endif + mr->sign_val = ((sign_val << 1) | MTT_PA_VALID); + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(mr->rdmamr), + HINIC5_CHANNEL_ROCE); // send to chip by cmdq + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to enable mpt, func(%d)", + rdev->glb_func_id); + goto err_enale_mpt; + } + +#ifdef __ROCE_DFX__ + roce5_store_mr_list(rdev, &mr->node); +#endif + + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + return &mr->ibmr; + +err_enale_mpt: + roce5_free_priv_pages(mr); + +err_alloc_priv_pages: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +static int roce5_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct roce5_mr *rmr = NULL; + + if (ibmr == NULL) { + ROCE_ERR("Ibmr is null"); + return -EINVAL; + } + + rmr = to_roce5_mr(ibmr); + if (ROCE_UNLIKELY(rmr->npages >= rmr->max_pages)) { + ROCE_ERR("Invalid npages(0x%x), can't set page", rmr->npages); + return -ENOMEM; + } + + rmr->pages[rmr->npages] = cpu_to_be64(addr | rmr->sign_val); + rmr->npages++; + + return 0; +} + +/* RoCE supports FRMR, for at privileged user, can via post send WR of way for at at free status of MR perform fast register. + here need in advance via input parameter of sgl available physical page 整理 in pbl table 里, later via frmr wqe send down. */ +int roce5_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + int ret; + struct roce5_mr *rmr = NULL; + + /* sg_offset can be null */ + if ((ibmr == NULL) || (sg == NULL)) { + ROCE_ERR("Ibmr or sg is null"); + return -EINVAL; + } + + rmr = to_roce5_mr(ibmr); + rmr->npages = 0; + + if (ROCE_UNLIKELY(((u32)sg_nents) > rmr->max_pages)) { + ROCE_ERR("Invalid sg_nents(0x%x), max(0x%x)", sg_nents, + rmr->max_pages); + return -EINVAL; + } + + ib_dma_sync_single_for_cpu(ibmr->device, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + + ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, roce5_set_page); + + ib_dma_sync_single_for_device(ibmr->device, rmr->page_map, + rmr->page_map_size, DMA_TO_DEVICE); + + return ret; +} + +static int roce5_umem_write_mtt_check(const struct roce5_device *rdev, + const struct mtt *mtt, + const struct ib_umem *umem) +{ + if ((rdev == NULL) || (mtt == NULL) || (umem == NULL)) { + ROCE_ERR("Rdev or mtt or umem is null"); + return -EINVAL; + } + + return 0; +} + +static int roce5_umem_write_mtt_update(struct roce5_device *rdev, + struct mtt *mtt, struct ib_umem *umem, + u64 *page_list, bool use_huge) +{ + int ret = 0; + u32 npages = 0; /* The number of recorded pages */ + u32 start_index = 0; /* page need to be written to mtt */ + struct ib_block_iter biter; + + rdma_umem_for_each_dma_block(umem, &biter, BIT(mtt->buf_page_shift)) { + page_list[npages++] = rdma_block_iter_dma_address(&biter); + /* The size of the memory pointed to by page_list is one page, which can store up to PAGE_SIZE / sizeof(u64) + pas. After a page is full, mtt needs to be written */ + if (npages == BIT(mtt->mtt_page_shift) / PA_SIZE) { + ret = hmm_mtt_write(rdev->hwdev, mtt, start_index, + npages, page_list, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to write mtt, ret(%d), start_index(%u), func_id(%u).", + ret, start_index, rdev->glb_func_id); + goto out; + } + start_index += npages; + npages = 0; + } + } + + if (npages != 0) { + ret = hmm_mtt_write(rdev->hwdev, mtt, start_index, npages, + page_list, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to write mtt, ret(%d), start_index(%u), func_id(%u).", + ret, start_index, rdev->glb_func_id); + goto out; + } + } + +out: + return ret; +} + +int roce5_umem_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct ib_umem *umem, bool use_huge) +{ + int ret = 0; + u64 *page_list = NULL; /* page_list to write to mtt */ + + ret = roce5_umem_write_mtt_check(rdev, mtt, umem); + if (ret != 0) { + return ret; + } + + page_list = (u64 *)kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc memory for page list, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + ret = roce5_umem_write_mtt_update(rdev, mtt, umem, page_list, use_huge); + if (ret != 0) { + ROCE_ERR("Failed to write mtt."); + } + + kfree(page_list); + return ret; +} + +/* **************************************************************************** + Prototype : roce5_buf_write_mtt + Description : roce5_buf_write_mtt + Input : struct roce5_device *rdev + struct mtt *mtt + struct tag_cqm_buf *buf + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +int roce5_buf_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct tag_cqm_buf *buf) +{ + u64 *page_list = NULL; + int ret = 0; + u32 i = 0; + + if ((rdev == NULL) || (mtt == NULL) || (buf == NULL)) { + ROCE_ERR("Rdev or mtt or buf is null"); + return (-EINVAL); + } + + page_list = + (u64 *)kzalloc(buf->buf_number * sizeof *page_list, GFP_KERNEL); + if (page_list == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc memory for page list, func_id(%d)", + rdev->glb_func_id); + return (-ENOMEM); + } + + /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */ + for (i = 0; i < buf->buf_number; ++i) { + page_list[i] = buf->buf_list[i].pa; + } + + ret = hmm_mtt_write(rdev->hwdev, mtt, 0, buf->buf_number, page_list, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + rdev->glb_func_id); + } + + kfree(page_list); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_is_umem_dma_contiguous + Description : Determine whether the input ib_umem occupies a contiguous + region in DMA address space, i.e., all scatter-gather entries + are adjacent in DMA addresses and the total length matches + umem->length. + Input : struct roce5_device *rdev + struct ib_umem *umem + Output : bool - true if the umem is DMA-contiguous, false otherwise + 1.Date : 2026/3/12 + Modification : Created function +**************************************************************************** */ +static inline bool roce5_is_umem_dma_contiguous(struct roce5_device *rdev, + struct ib_umem *umem) +{ + struct scatterlist *sg = NULL; + int i; + dma_addr_t expected_dma, dma; + uint64_t total_len = 0, len; + +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + sg = umem->sgt_append.sgt.sgl; +#else + sg = umem->sg_head.sgl; +#endif + + expected_dma = sg_dma_address(sg); + +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + for_each_sg(umem->sgt_append.sgt.sgl, sg, umem->sgt_append.sgt.nents, i) +#else + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) +#endif + { + dma = sg_dma_address(sg); + len = sg_dma_len(sg); + + if (i > 0 && dma != expected_dma) { + ROCE_DEV_WARN( + rdev, + "Enable huge page, but discontinuity at sg[%d]: expected 0x%llx, got 0x%llx\n", + i, (u64)expected_dma, (u64)dma); + return false; + } + + total_len += len; + expected_dma = dma + len; + } + + /* Check the total length. */ + if (total_len != umem->length) { + ROCE_DEV_WARN(rdev, "Total length %llu != umem length(%zu)\n", + total_len, umem->length); + return false; + } + + return true; +} + +static int roce5_get_page_info(struct roce5_device *rdev, struct ib_umem *umem, + u32 *npages, u32 *page_shift, u64 virt_addr) +{ + *page_shift = roce5_umem_get_buf_page_shift(rdev, umem, virt_addr); + *npages = roce5_umem_get_mtt_npages(rdev, umem, *page_shift); + + if (rdev->huge_page_en && *page_shift > PAGE_SHIFT) { + if (*page_shift != PAGE_SHIFT_2MB) { + ROCE_DEV_WARN( + rdev, + "RoCE reg huge MR expect page_size(2MB), but found %lu.", + BIT(*page_shift)); + } + ROCE_DEV_INFO(rdev, + "RoCE reg huge MR: npages(%u), page_shift(%u).", + *npages, *page_shift); + return ROCE_USE_HUGEPAGE; + } + + return ROCE_NOT_USE_HUGEPAGE; +} + +int roce5_user_mr_reg(struct roce5_device *rdev, struct roce5_mr *mr, u32 pdn, + u64 virt_addr, u64 length, int access) +{ + int ret = 0; + u32 npages = 0; + u32 page_shift = PAGE_SHIFT; + u32 mpt_xid_mode; + int use_huge; + struct hinic5_hwdev *hwdev = (struct hinic5_hwdev *)rdev->hwdev; + + if (hwdev == NULL) { + ROCE_ERR("hwdev is null"); + return 0; + } + + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + + use_huge = roce5_get_page_info(rdev, mr->umem, &npages, &page_shift, + virt_addr); + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift, + mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, func_id(%d)", + hinic5_global_func_id(hwdev)); + goto err_alloc_tpt; + } + + roce5_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, pdn, virt_addr, length, + (u32)access); + + ret = roce5_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem, use_huge); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + hinic5_global_func_id(hwdev)); + goto err_write_mtt; + } + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, hwdev, &mr->rdmamr, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of user mr, func_id(%d)", + hinic5_global_func_id(hwdev)); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + return ret; +} + +/* **************************************************************************** + Prototype : roce5_reg_user_mr + Description : register MR for user + Input : struct ib_pd *ibpd + u64 start + u64 length + u64 virt_addr + int access + struct ib_udata *udata + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +struct ib_mr *roce5_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, + u64 virt_addr, int access, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_mr *mr = NULL; + struct roce5_device *rdev = NULL; + struct roce5_pd *rpd = to_roce5_pd(ibpd); + roce_user_handle_t *handle = NULL; + + if (ibpd == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibpd is null"); + goto err_out; + } + + rdev = to_roce5_dev(ibpd->device); + + mr = (struct roce5_mr *)kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, + "Failed to alloc memory for roce mr, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } +#if defined(IB_UMEM_GET_PEER) + mr->umem = roce5_mr_umem_get_peer(rdev, (size_t)length, access, start, + udata, true); +#else + mr->umem = roce5_umem_get(&rdev->ib_dev, ibpd->uobject->context, start, + (size_t)length, access, 0); +#endif + if (IS_ERR(mr->umem)) { + handle = (roce_user_handle_t *)kzalloc( + sizeof(roce_user_handle_t), GFP_KERNEL); + if (handle == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR( + rdev, + "Failed to alloc memory for handle, func_id(%d)", + rdev->glb_func_id); + goto err_get_umem; + } + handle->magic = ROCE_ULD_HANDLE_MAGIC_NUM; + handle->rdev = rdev; + handle->mr = mr; + handle->virt_addr = virt_addr; + handle->length = length; + handle->pdn = rpd->pdn; + handle->access = access; + + mr->david_en = true; + mr->private_data = (void *)handle; + + /* call ULD interface */ + ret = roce5_uld_reg_user_mr((roce_uld_handle)handle); + if (ret != 0) { + ROCE_DEV_WARN( + rdev, + "Failed to reg user mr uld, func_id(%d), ret(%d), err(%d)", + rdev->glb_func_id, ret, (int)PTR_ERR(mr->umem)); + ret = (int)PTR_ERR(mr->umem); + kfree(handle); + mr->private_data = NULL; + goto err_get_umem; + } + + mr->umem = NULL; + } else { + ret = roce5_user_mr_reg(rdev, mr, rpd->pdn, virt_addr, length, + access); + if (ret != 0) { + goto err_mr_update; + } + } +#ifdef __ROCE_DFX__ + roce5_store_mr_list(rdev, &mr->node); +#endif + mr->ibmr.lkey = mr->rdmamr.key; + mr->ibmr.rkey = mr->rdmamr.key; + + ROCE_DEV_INFO(rdev, "Roce reg user mr success, mr mpt index is %u", + mr->rdmamr.mpt.mpt_index); + + return &mr->ibmr; + +err_mr_update: + ib_umem_release(mr->umem); + +err_get_umem: + kfree(mr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +int roce5_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_mr *mr = NULL; + struct roce5_device *rdev = NULL; + + if (ibmr == NULL) { + ROCE_ERR("Ibmr is null"); + return -EINVAL; + } + + mr = to_roce5_mr(ibmr); + rdev = to_roce5_dev(ibmr->device); + + if (mr->rdmamr.type == HMM_RDMA_FRMR) { + roce5_free_priv_pages(mr); + } + + if (mr->david_en) { + if (mr->private_data == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid parameter, private data is NULL."); + return -EINVAL; + } + + ret = roce5_uld_dereg_user_mr( + (roce_uld_handle)mr->private_data); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "roce dereg user mr uld failed, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + kfree(mr->private_data); + mr->private_data = NULL; + } else { + ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, + &mr->rdmamr, SERVICE_T_ROCE, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Mr may has mw bind on it, ret(%d), mr_key(%#x), func_id(%d)", + ret, mr->rdmamr.key, rdev->glb_func_id); + if (ret == (-CMDQ_TIMEOUT)) { + rdev->dev_status_to_ofed = + ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + return ret; + } + + roce5_free_tpt(rdev, &mr->rdmamr); + + ib_umem_release(mr->umem); + } + + if (mr->fumem_nr != 0) { + roce5_free_frmr_umem(mr->fumem, mr->fumem_nr); + mr->fumem_nr = 0; + } +#ifdef __ROCE_DFX__ + roce5_delete_mr_list(rdev, &mr->node); +#endif + + kfree(mr); + + return 0; +} + +int roce5_dereg_mr_ofa(struct ib_mr *ibmr) +{ + return roce5_dereg_mr(ibmr, NULL); +} + +static void roce5_err_enable_mpt_handler(void *hwdev, struct roce5_mw *mw) +{ + hmm_mpt_free(hwdev, &mw->rdmamw.mpt); + mw->rdmamw.enabled = RDMA_MPT_DISABLED; + return; +} + +static void roce5_mw2hmm_adapter(const struct rdma_mw *rdmamw, + struct hmm_rdma *hmmrdma) +{ + hmmrdma->mpt = rdmamw->mpt; + hmmrdma->key = rdmamw->key; + hmmrdma->pdn = rdmamw->pdn; + hmmrdma->type = (rdmamw->type == RDMA_MW_TYPE_1) ? HMM_RDMA_MW_TYPE_1 : + HMM_RDMA_MW_TYPE_2; + hmmrdma->enabled = rdmamw->enabled; +} + +int roce5_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) +{ + int ret; + u32 mpt_xid_mode; + struct roce5_mw *mw = NULL; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + struct hmm_rdma hmmrdma = { 0 }; + + if (ibmw == NULL) { + ROCE_ERR("Ibmw is null"); + return -EINVAL; + } + + mw = to_roce5_mw(ibmw); + pd = to_roce5_pd(ibmw->pd); + rdev = to_roce5_dev(ibmw->device); + + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + ret = hmm_mpt_alloc_templated(rdev->hwdev, &mw->rdmamw.mpt, + SERVICE_T_ROCE, mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt, func_id(%d)", + rdev->glb_func_id); + goto err; + } + + roce5_set_rdma_mw(&mw->rdmamw, pd->pdn, ibmw->type); + roce5_mw2hmm_adapter(&mw->rdmamw, &hmmrdma); + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &hmmrdma, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of mw, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + if (ret == (-CMDQ_TIMEOUT)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + goto err_enable_mpt; + } + + mw->rdmamw.enabled = hmmrdma.enabled; + mw->ibmw.rkey = mw->rdmamw.key; + return 0; +err_enable_mpt: + roce5_err_enable_mpt_handler(rdev->hwdev, mw); +err: + return ret; +} + +struct ib_mw *roce5_alloc_mw_ofa(struct ib_pd *ibpd, enum ib_mw_type type, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_mw *mw = NULL; + + if (ibpd == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibpd is null"); + goto out; + } + + mw = (struct roce5_mw *)kzalloc(sizeof *mw, GFP_KERNEL); + if (mw == NULL) { + ret = -ENOMEM; + ROCE_ERR("Failed to alloc memory for roce mw"); + goto out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + mw->ibmw.pd = ibpd; + mw->ibmw.device = ibpd->device; + mw->ibmw.type = type; + ret = roce5_alloc_mw(&mw->ibmw, udata); + if (ret != 0) { + kfree(mw); + } + +out: + return (ret != 0 ? (struct ib_mw *)ERR_PTR((long)ret) : &mw->ibmw); +} + +/* **************************************************************************** + Prototype : roce5_dealloc_mw + Description : dealloc mw and release related resource + Input : struct ib_mw *ibmw + Output : None + + 1.Date : 2015/4/24 + Modification : Created function + +**************************************************************************** */ +int roce5_dealloc_mw(struct ib_mw *ibmw) +{ + int ret = 0; + struct roce5_mw *mw = NULL; + struct roce5_device *rdev = NULL; + struct hmm_rdma hmmrdma; + + if (ibmw == NULL) { + ROCE_ERR("Ibmw is null"); + return -EINVAL; + } + + mw = to_roce5_mw(ibmw); + rdev = to_roce5_dev(ibmw->device); + roce5_mw2hmm_adapter(&mw->rdmamw, &hmmrdma); + + ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, &hmmrdma, + SERVICE_T_ROCE, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to disable mpt of mw, func_id(%d)", + rdev->glb_func_id); + if (ret == (-CMDQ_TIMEOUT)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + return ret; + } + mw->rdmamw.enabled = hmmrdma.enabled; + + hmm_mpt_free(rdev->hwdev, &mw->rdmamw.mpt); + mw->rdmamw.enabled = RDMA_MPT_DISABLED; + + return ret; +} + +int roce5_dealloc_mw_ofa(struct ib_mw *ibmw) +{ + int ret = 0; + struct roce5_mw *mw = NULL; + + ret = roce5_dealloc_mw(ibmw); + if (ret == 0) { + mw = to_roce5_mw(ibmw); + kfree(mw); + } + + return ret; +} + +int roce5_check_mr_status(struct ib_mr *ibmr, u32 check_mask, + struct ib_mr_status *mr_status) +{ + int ret = 0; + struct roce5_mr *mr = NULL; + + if ((ibmr == NULL) || (mr_status == NULL)) { + ROCE_ERR("ibmr or mr_status is NULL ptr."); + return -EINVAL; + } + + mr = to_roce5_mr(ibmr); + if ((check_mask & (~IB_MR_CHECK_SIG_STATUS)) != 0) { + ROCE_ERR("check_mask(%#x) invalid", check_mask); + return -EINVAL; + } + + if (mr->signature_en == 0) { + ROCE_ERR("mr not configure dif info"); + ret = -EINVAL; + return ret; + } + + mr->sig.sig_status_checked = true; + mr_status->fail_status = 0; + + /* If err does not exist, return 0 */ + if (!mr->sig.sig_err_exists) { + return 0; + } + + ret = memcpy_s((void *)&mr_status->sig_err, sizeof(struct ib_sig_err), + (void *)&mr->sig.sig_err_item, + sizeof(struct ib_sig_err)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy sig_err.(ret:%d)", ret); + return -ENOMEM; + } + mr->sig.sig_err_exists = false; + mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; + + return 0; +} + +/** + * @brief Get mpt by index. When querying succeeded, you have to call roce5_mpt_put() to free + the cqm_obj_mpt. But there is no need to call roce5_mpt_put() if querying failed. + * @return Success: return the mpt. Failure: return NULL. + */ +struct mpt *roce5_get_mpt_by_index(struct roce5_device *rdev, u32 mpt_index) +{ + cqm_object_s *cqm_obj_mpt = NULL; + cqm_qpc_mpt_s *cqm_mpt = NULL; + struct mpt *mpt = NULL; + + cqm_obj_mpt = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_MPT, mpt_index, false); + if (cqm_obj_mpt == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find mpt according to index(0x%x), func_id(%d)", + mpt_index, rdev->glb_func_id); + return NULL; + } + + cqm_mpt = container_of(cqm_obj_mpt, cqm_qpc_mpt_s, object); + mpt = (struct mpt *)cqm_mpt->priv; + if (mpt == NULL) { + ROCE_DEV_ERR(rdev, "Failed to convert obj to mpt."); + cqm5_object_put(cqm_obj_mpt); + } + + return mpt; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h new file mode 100644 index 000000000..3124c2470 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/mr/roce_mr.h @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MR_H +#define ROCE_MR_H + +#include <rdma/ib_verbs.h> +#include <rdma/ib_umem.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/vmalloc.h> + +#include "roce.h" + +#include "hinic5_hmm.h" +#include "hinic5_rdma.h" +#include "rdma_comp.h" + +#include "securec.h" + +#define ROCE5_MR_DEFAULT_ACCESS \ + (IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE | \ + IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_ATOMIC) + +#define ROCE5_MR_PAGES_ALIGN 64U +#define ROCE5_FRMR_UMEM_NUM 512 +#define ROCE5_SIZE_2M (2 * 1024 * 1024) + +struct roce5_sig_ctx { + struct ib_sig_attrs sig_attrs; /* recorded when submitting WR */ + struct ib_sig_err + sig_err_item; /* Get information configuration from CQE when polling CQE */ + bool sig_status_checked; /* Indicates whether the status is checked */ + bool sig_err_exists; /* Indicates if an error exists */ + u32 sig_err_count; /* Count the number of wrong CQEs */ +}; + +/* uld service can extension structure space */ +#define ROCE_MR_ULD_DEF_LEN 24 +#define ROCE_NOT_USE_HUGEPAGE 0 +#define ROCE_USE_HUGEPAGE 1 + +/* used for verify uld access whether memory stomping */ +#define ROCE_RMR_MAGIC 0x1317F73a +struct roce5_mr_uld_def { + u8 self_def[ROCE_MR_ULD_DEF_LEN]; +}; + +struct roce5_mr { + struct ib_mr ibmr; /* mr structure provided by ofed */ + struct ib_umem * + umem; /* Maintain the structure where the user-mode allocated buffer */ + struct ib_umem **fumem; /* used to frmr only */ + struct hmm_rdma rdmamr; /* mr structure provided by rdma component */ + struct roce5_sig_ctx sig; /* Record signature related information */ + u16 signature_en; /* Whether the logo contains signature information */ + u16 sign_val; /* MTT entry [11:0] */ + __be64 *pages; + u32 npages; + u32 fumem_nr; + u32 max_pages; + size_t page_map_size; + dma_addr_t page_map; +#ifdef __ROCE_DFX__ + struct list_head node; +#endif // __ROCE_DFX__ + bool david_en; /* Indicate whether the mr is a david mr */ + void *private_data; /* private data for driver */ + u32 user_data_len; /* length of user data which stored in struct hmm_rdma */ + struct roce5_mr_uld_def uld_def; + u32 rmr_magic; /* preventstop uld access memory stomping, support up to 24B space */ +}; + +struct roce5_mw { + struct ib_mw ibmw; /* mw structure provided by ofed */ + struct rdma_mw rdmamw; /* mw structure provided by rdma component */ +}; + +int roce5_alloc_tpt(struct roce5_device *rdev, struct hmm_rdma *mr, u32 npages, + u32 page_shift, u32 mpt_xid_mode); +void roce5_free_tpt(struct roce5_device *rdev, struct hmm_rdma *mr); +void roce5_set_rdma_mr(struct hmm_rdma *mr, enum rdma_mr_type mr_type, u32 pdn, + u64 iova, u64 size, u32 access); +void roce5_free_frmr_umem(struct ib_umem **fumem, int fumem_nr); +struct mpt *roce5_get_mpt_by_index(struct roce5_device *rdev, u32 mpt_index); +static inline void roce5_mpt_put(struct mpt *rmpt) +{ + if (rmpt == NULL) { + return; + } + cqm5_object_put(&((cqm_qpc_mpt_s *)(rmpt->mpt_object))->object); +} + +static inline int roce5_mr_type2hmm(enum rdma_mr_type mr_type) +{ + int mr_type_convert_table[] = { + [RDMA_DMA_MR] = HMM_RDMA_DMA_MR, + [RDMA_USER_MR] = HMM_RDMA_USER_MR, + [RDMA_FRMR] = HMM_RDMA_FRMR, + [RDMA_FMR] = HMM_RDMA_FMR, + [RDMA_PHYS_MR] = HMM_RDMA_PHYS_MR, + [RDMA_RSVD_LKEY] = HMM_RDMA_RSVD_LKEY, + [RDMA_SIG_MR] = HMM_RDMA_SIG_MR, + [RDMA_INDIRECT_MR] = HMM_RDMA_INDIRECT_MR, + [RDMA_ODP_IMPLICIT_MR] = HMM_RDMA_ODP_IMPLICIT_MR, + [RDMA_ODP_EXPLICIT_MR] = HMM_RDMA_ODP_EXPLICIT_MR, + }; + + return mr_type_convert_table[mr_type]; +} + +#endif // __ROCE_MR_H_ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h new file mode 100644 index 000000000..28a092b29 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_post.h @@ -0,0 +1,171 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#ifndef ROCE_POST_H +#define ROCE_POST_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "roce_wqe_format.h" + +#define ROCE_SQ_DOORBELL_TYPE_2 2 +#define ROCE_SQ_DOORBELL_TYPE_23 23 +#define ROCE_SQ_DOORBELL_TYPE_21 21 + +#define ROCE_DWQE_DDB_TYPE 3 /* RoCE direct wqe of DDB type */ + +#define ROCE_IMM_EXT_LEN 4 +#define ROCE_TASK_SEG_ALIGN 8 + +enum roce_tsl_8_byte_aligned_size_e { + ROCE_SEND_LOCAL_WQE_TSL = 2, + ROCE_RDMA_WQE_TSL = 4, + ROCE_ATOMIC_CWP_WQE_TSL = 6, + ROCE_UD_WQE_COM_TSL = 8 +}; + +enum { + ROCE_WQE_OPCODE_SEND = 0x00, + ROCE_WQE_OPCODE_SEND_INVAL = 0x01, + ROCE_WQE_OPCODE_SEND_IMM = 0x02, + + ROCE_WQE_OPCODE_RDMA_WRITE = 0x04, + ROCE_WQE_OPCODE_RDMA_WRITE_IMM = 0x05, + + ROCE_WQE_OPCODE_RDMA_READ = 0x08, + + ROCE_WQE_OPCODE_ATOMIC_CMP_SWP = 0x0c, + ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD = 0x0d, + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP = 0x0e, + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD = 0x0f, + + ROCE_WQE_OPCODE_FRMR = 0x10, + ROCE_WQE_OPCODE_LOCAL_INVAL = 0x11, + ROCE_WQE_OPCODE_BIND_MW = 0x12, + ROCE_WQE_OPCODE_REG_SIG_MR = + 0x13, /* Extended for further local opreation */ +}; + +#define ROCE_KERNEL_OPCODE_NUM (ROCE_WQE_OPCODE_REG_SIG_MR + 1) + +enum { + ROCE_DWQE_DB_SUBTYPE_SEND = 0x1, + ROCE_DWQE_DB_SUBTYPE_SEND_IMM = 0x2, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE = 0x3, + ROCE_DWQE_DB_SUBTYPE_RDMA_WRITE_IMM = 0x4, + ROCE_DWQE_DB_SUBTYPE_RDMA_READ = 0x5, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_CMP_SWP = 0x6, + ROCE_DWQE_DB_SUBTYPE_ATOMIC_FETCH_ADD = 0x7 +}; + +/* UD send WQE task seg1 */ +struct roce_wqe_ud_tsk_seg_cycle1 { + roce_wqe_tsk_com_seg_u common; + + /* DW0 */ + u32 data_len; + + /* DW1 */ + u32 immdata_invkey; + + /* DW2 */ + union { + struct { + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; /* 0: No limit on the static rate (100% port speed) + 1-6: reserved + 7: 2.5 Gb/s. 8: 10 Gb/s. 9: 30 Gb/s. 10: 5 Gb/s. 11: 20 Gb/s. + 12: 40 Gb/s. 13: 60 Gb/s. 14: 80 Gb/s.15: 120 Gb/s. */ + u32 rsvd1 : 3; + u32 fl : 1; + } bs; + u32 value; + } dw2; + + /* DW3 */ + union { + struct { + u32 hop_limit : 8; + u32 sgid_idx : 7; + u32 rsvd0 : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tc : 8; + } bs; + u32 value; + } dw3; + + /* DW4 */ + union { + struct { + u32 flow_label : 20; + u32 rsvd0 : 4; + u32 smac_index : 3; + u32 rsvd1 : 5; + } bs; + u32 value; + } dw4; + + /* DW5~8 */ + u8 dgid[16]; + + /* DW9 */ + union { + struct { + u32 dst_qp : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw9; + + /* DW10 */ + u32 qkey; +}; + +/* UD send WQE task seg2; */ +struct roce_wqe_ud_tsk_seg_cycle2 { + /* DW0 */ + union { + struct { + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 cfi : 1; + u32 vlan_prio : 3; + } bs; + u32 value; + } dw0; + + /* DW1 */ + u32 dmac_l32; +}; + +struct roce5_post_send_normal_param { + struct roce5_wqe_ctrl_seg *ctrl_seg; + union roce5_wqe_tsk_com_seg *tsk_com_seg; + u32 wqe_size; + u8 *wqe; + struct roce_wqe_data_seg *dseg; + u32 wr_num; /* record posted WR numbers */ + u32 index; /* WQEBB id */ + int inline_flag; /* Inline flag */ + u32 data_len; + u32 *data_len_addr; + u32 sq_rmd_size; + s32 opcode; + s32 cycle; + u32 sq_wqe_ssn; + struct roce5_wqe_ctrl_seg ctrl_seg_tmp; + + unsigned long flags; /* type required by kernel API */ +}; + +typedef struct tag_roce_opcode_attr { + uint32_t rsvd : 31; + uint32_t remote_opcode : 1; +} roce_opcode_attr_s; + +#endif /* ROCE_POST_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h new file mode 100644 index 000000000..7088cd550 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp.h @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_H +#define ROCE_QP_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_cq.h" +#include "roce_db.h" +#include "roce_pd.h" +#include "roce_compat.h" +#include "roce_npu_cmd_qp_defs.h" +#include "roce_wqe_base_format.h" +#include "roce_post.h" +#include "roce_mix.h" +#include "roce_xqe_format.h" +#include "roce_qp_ctx_format.h" +#include "roce_cq_format.h" +#include "roce_srq_format.h" +#include "roce_com.h" +#include "roce_obj_ext.h" +#include "roce_kernel_compat_gen.h" + +#define ROCE_RQ_MIN_SGE 4 +#define ROCE_RQ_MID_SGE 8 +#define ROCE_RQ_MAX_SGE 16 + +#define ROCE_WQE_NEXT_SGE_INVALID (1UL << 31) + +#define ROCE_RDMARC_DESTROY_CHECK_VALUE 0xcece5a5a + +#define RDMARC_NUM_PER_CACHELINE 8 +#define RDMARC_TABLE_ENTRY_SIZE 32 +#define ROCE_QP_MAX_PFETCH_MTT_LAYER 3 +#define ROCE_QP_MAX_TIMER_NUM 5 + +#define ROCE_QP_SMI_QP_NUM 0x0 +#define ROCE_QP_GSI_QP_NUM 0x1 +#define ROCE_QP_INVLID_QP_NUM CQM_INDEX_INVALID + +#define ROCE_QP_MAX_DWQE_SIZE 256 +#define ROCE_QP_DEFAULT_WQE_SHIFT 6 + +#define ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE 512 + +#define ROCE_QP_STATE_MEM_INIT 0xa + +#define ROCE_UD_MAX_INLINE_LEN_SUB 48 + +#define ROCE_QP_GPA_SIG_LEN 3 +#define ROCE_WR_MIN_NUM 2 + +#define ROCE_QPC_PMTU_TRANSLATE(pmtu) (((1U << ((pmtu)-1)) - 1) & 0xf) + +#define ROCE_QP_OPTPAR_ALT_ADDR_PATH (1 << 0) +#define ROCE_QP_OPTPAR_RRE (1 << 1) +#define ROCE_QP_OPTPAR_RAE (1 << 2) +#define ROCE_QP_OPTPAR_RWE (1 << 3) +#define ROCE_QP_OPTPAR_PKEY_INDEX (1 << 4) +#define ROCE_QP_OPTPAR_Q_KEY (1 << 5) +#define ROCE_QP_OPTPAR_RNR_TIMEOUT (1 << 6) +#define ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH (1 << 7) +#define ROCE_QP_OPTPAR_SRA_MAX (1 << 8) +#define ROCE_QP_OPTPAR_RRA_MAX (1 << 9) +#define ROCE_QP_OPTPAR_PM_STATE (1 << 10) +#define ROCE_QP_OPTPAR_RETRY_COUNT (1 << 11) +#define ROCE_QP_OPTPAR_RNR_RETRY (1 << 12) +#define ROCE_QP_OPTPAR_ACK_TIMEOUT (1 << 13) +#define ROCE_QP_OPTPAR_SCHED_QUEUE (1 << 14) +#define ROCE_QP_OPTPAR_COUNTER_INDEX (1 << 15) + +#define roce5_LB1_MASK_1825V100 0x7 + +enum roce_qp_state { + ROCE_QP_STATE_RST = 0, + ROCE_QP_STATE_INIT = 1, + ROCE_QP_STATE_RTR = 2, + ROCE_QP_STATE_RTS = 3, + ROCE_QP_STATE_SQER = 4, + ROCE_QP_STATE_SQD = 5, + ROCE_QP_STATE_ERR = 6, + ROCE_QP_STATE_SQ_DRAINING = 7, + ROCE_QP_STATE_NUM +}; + +enum { + ROCE_QP_PM_MIGRATED = 0x3, + ROCE_QP_PM_ARMED = 0x0, + ROCE_QP_PM_REARM = 0x1 +}; + +enum { ROCE_QP_NO_SRQ, ROCE_QP_HAS_SRQ }; + +enum { ROCE_QP = 0, ROCE_QP_EXT }; + +struct roce5_wq { + u64 *wrid; + spinlock_t lock; + u32 wqebb_cnt; + u32 max_post; + u32 max_sge; + u32 offset; + u32 wqe_shift; + u32 head; + u32 tail; +}; + +struct roce5_sqp_attr { + int pkey_index; + u32 qkey; + u32 send_psn; +}; + +#define ROCE_RQP_ULD_DEF_LEN 64 +#define ROCE_RQP_MAGIC 0x5a5b5c5d +struct roce5_rqp_uld_def { + u8 uld_def[ROCE_RQP_ULD_DEF_LEN]; +}; + +struct roce5_qp { + struct ib_qp ibqp; + u32 qpn; + struct roce5_db db; + cqm_queue_s *qp_buf_info; + cqm_qpc_mpt_s *qpc_info; + struct roce5_wq rq; + u32 sq_signal_bits; + unsigned int sq_next_wqe; + u32 sq_max_wqes_per_wr; + struct roce5_wq sq; + struct ib_umem *umem; + struct mtt mtt; + struct roce5_buf buf; + int buf_size; + struct mutex mutex; + u32 xrcdn; + u8 port; + u8 atomic_rd_en; + u8 qp_state; + u8 qp_ext; + u32 qp_type; + u32 max_inline_data; + int max_dwqe_size; + u32 sq_wqe_ssn; + struct rdma_rdmarc rdmarc; + u16 rsp_depth; + u8 sl; + bool has_rq; + u8 *sq_head_addr; + u8 *sq_tail_addr; + bool signature_en; + u8 pi_on_chip; + u8 double_sgl_mode; + u8 ext_mtu; + u8 ext_mtu_mode; + + u8 db_sgid_index; + u8 db_path_mtu; + u8 db_cos; + u8 shadow; + + struct roce5_sqp_attr *sqp_attr; + + u32 qid; + u32 local_comm_id; /* used by NOF AA to reuse qpn when disconnection happens */ + u32 remote_comm_id; + void *wb_va; /* used for xrc */ + dma_addr_t wb_dma; /* used for xrc */ + u32 shadow_vfid; + + /* hot-plug record */ + struct list_head qps_list; + struct list_head cq_recv_list; + struct list_head cq_send_list; +#if defined(ROCE_VBS_EN) || defined(ROCE_EXTEND) + void *vbs_qp_ptr; +#endif +#ifdef ROCE_BONDING_EN + u32 tx_hash_value; +#endif + + struct roce5_obj_ext obj_ext; + + struct roce5_rqp_uld_def + uld_def; /* must placed at qp_magic before, preventstop uld access memory stomping, support up to 64B space */ + u32 rqp_magic; +}; + +typedef struct tag_roce_modify_qp_param { + u32 optpar; + roce_verbs_qp_attr_s qp_attr; + roce_verbs_cmd_qp_modify_rsp_s modify_rsp; +} roce_modify_qp_param_s; + +void roce5_qp_async_event(struct roce5_device *rdev, struct roce5_qp *qp, + int type); + +void roce5_free_opt_rdmarc(struct roce5_qp *rqp); +void roce5_set_opt_rdmarc(struct rdma_rdmarc *rdmarc, u16 depth, + roce_verbs_qp_attr_s *qp_attr); +void roce5_get_cqs(struct roce5_qp *rqp, struct roce5_cq **send_cq, + struct roce5_cq **recv_cq); +int roce5_qp_modify_2rst_cmd(struct roce5_device *rdev, u32 qpn); +int roce5_qp_cache_out_cmd(struct roce5_device *rdev, struct roce5_qp *rqp); +struct roce5_pd *roce5_get_pd(struct roce5_qp *rqp); +bool roce5_sqp_check(const struct roce5_qp *rqp); +void *roce5_get_wqe(struct roce5_qp *rqp, u32 offset); +int roce5_wq_overflow(struct roce5_wq *wq, u32 wr_num, struct ib_cq *ibcq); +void roce5_set_data_seg(struct roce_wqe_data_seg *dseg, struct ib_sge *sge); +void roce5_qpc_to_be(roce_verbs_qp_attr_s *qp_attr, struct roce5_qp *rqp, + u32 *be_ctx); +void roce5_qp_rst2init(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr); + +int roce5_send_qp_lb_cmd(u32 qpn, struct roce5_device *rdev, u8 cmd, + cqm_cmd_buf_s *buf_in, cqm_cmd_buf_s *buf_out, + u32 timeout); + +int roce5_qp_query(struct roce5_device *rdev, u32 qpn, u32 *context, + int qpc_size); +int roce5_post_send_standard(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +void qpc_seg_to_le32(struct roce5_qp_context *be_ctx, + struct roce5_qp_context *le_ctx, u32 srq_vld); +void roce5_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size); + +int roce5_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); + +int roce5_post_send_ofa(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr); +struct roce5_qp *roce5_get_rqp_by_qpn(struct roce5_device *rdev, u32 qpn); +static inline void roce5_rqp_put(struct roce5_qp *rqp) +{ + if (rqp == NULL) { + return; + } + cqm5_object_put(&rqp->qpc_info->object); +} + +#define ROCE_UD_MTU_SHIFT 4 /* 4K mtu */ +static inline void roce5_set_sq_db(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct roce_sq_db_seg *db, + unsigned int index) +{ + db->dw0.bs.qpn = rqp->qpn & 0xfffff; + db->dw0.bs.ctx_size = 1; /* 512B */ + db->dw0.bs.cp_flag = 0; + db->dw0.bs.cos = rqp->db_cos; + if (ROCE5_SUPPORT_SQ_DB_TYPE_21(rdev) != 0) { + db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_21; + } else if (ROCE5_SUPPORT_SQ_DB_TYPE_MSG_V2(rdev) != 0) { + db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_23; + } else { + db->dw0.bs.type = ROCE_SQ_DOORBELL_TYPE_2; + } + + db->dw1.bs.pi = ((index & 0xffff) >> 8) & 0xff; /* 8bits */ + db->dw1.bs.sub_type = 0; + db->dw1.bs.mtu_shift = rqp->db_path_mtu & 0x7; + db->dw1.bs.sgid_index = rqp->db_sgid_index; + if (rqp->qp_type == IB_QPT_RC) { + } else if (rqp->qp_type == IB_QPT_UD || rqp->qp_type == IB_QPT_GSI) { + db->dw1.bs.mtu_shift = ROCE_UD_MTU_SHIFT; + } else if (rqp->qp_type == IB_QPT_XRC_INI || + rqp->qp_type == IB_QPT_XRC_TGT) { + db->dw1.bs.xrc_vld = 1; + } + + db->dw0.value = roce5_convert_be32(db->dw0.value); + db->dw1.value = roce5_convert_be32(db->dw1.value); +} + +static inline int roce5_write_sq_db(struct roce5_device *rdev, + struct roce5_qp *rqp, u32 index, + struct ib_udata *udata) +{ + int ret = 0; + union roce_sq_db sq_db; + + ret = memset_s(&sq_db.sq_db_val, sizeof(u64), 0, sizeof(u64)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memset sq_db.(ret:%d)", ret); + return -ENOMEM; + } + + roce5_set_sq_db(rdev, rqp, &sq_db.sq_db_seg, index); + + if (udata == NULL) { + *(rqp->db.db_record) = cpu_to_be32(index & 0xffff); + } + + wmb(); + ret = cqm5_ring_hardware_db(rdev->hwdev, SERVICE_T_ROCE, index & 0xff, + sq_db.sq_db_val); + + /* + * Make sure doorbells don't leak out of SQ spinlock + * and reach the HCA out of order. + */ + wmb(); + return ret; +} + +#if defined(IB_QP_CREATE_FLAGS_HAVE_INTEGRITY_EN) +#define IB_QP_CREATE_SIGNATURE_EN IB_QP_CREATE_INTEGRITY_EN +#endif +#endif // ROCE_QP_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c new file mode 100644 index 000000000..6adbe787c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_create.c @@ -0,0 +1,1506 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_infra.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" +#include "roce_aeq_format.h" +#include "roce_restore.h" +#include "roce_os_compat.h" +#include "roce_cc_format.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_qp_inner.h" +#include "roce_com.h" +#include "roce_qp_ext.h" + +#include "hinic5_hmm.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#include "securec.h" + +/* **************************************************************************** + Prototype : roce5_check_rq_size + Description : roce5_check_rq_size + Input : struct roce5_device *rdev + struct roce5_qp *rqp + struct ib_qp_cap *cap + bool has_rq + Output : None + + 1.Date : 2017/4/27 + Modification : Created function +**************************************************************************** */ +static int roce5_check_rq_size(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr, bool has_rq) +{ + u32 max_sge_num; + struct ib_qp_cap *cap = &init_attr->cap; + + max_sge_num = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg; + if ((cap->max_recv_wr > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) || + (cap->max_recv_sge > max_sge_num)) { + ROCE_DEV_ERR( + rdev, + "Failed to check rq size, over range, func_id(%d), attr max_recv_wr(%d), cap max_wqes(%d), \ + attr max_recv_sge(%d), cap max_sge_num(%d)", + rdev->glb_func_id, cap->max_recv_wr, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + cap->max_recv_sge, max_sge_num); + return -EINVAL; + } + + if (has_rq && ((cap->max_recv_wr == 0) || (cap->max_recv_sge == 0))) { + ROCE_DEV_ERR( + rdev, + "Has rq, cap->max_recv_wr(%d), cap->max_recv_sge(%d), func_id(%d)", + cap->max_recv_wr, cap->max_recv_sge, rdev->glb_func_id); + return -EINVAL; + } + + if ((!has_rq) && (cap->max_recv_wr != 0)) { + ROCE_DEV_ERR(rdev, + "Not has rq, cap->max_recv_wr(%d), func_id(%d).", + cap->max_recv_wr, rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_set_rq_size + Description : roce5_set_rq_size + Input : struct roce5_device *rdev + struct roce5_qp *rqp + struct ib_qp_cap *cap + bool is_user + bool has_rq + Output : None + + 1.Date : 2015/4/29 + Modification : Created function +**************************************************************************** */ +/* **************************************************************************** +* +* 1 there is SQ or RQ +* SQ/RQ depth must be power of 2 +* if max_send_wr=0, no need to alloc buf for SQ/RQ + +* RQ WQE SIZE equals 64 or128 +* SQ WQE SIZE equals multi of wqebb size + +* SQ max_gs, max 8 and min 0 +* RQ max_gs, can be 4 or 8, but can not be 0. + +* 2 there is no SQ or RQ +* all related params should be 0 +* + +**************************************************************************** */ +static int roce5_set_rq_size(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr, bool is_user, + bool has_rq) +{ + int ret; + u32 wqebb_num = 0; + u32 sge_total_len = 0; + u32 page_size; + struct ib_qp_cap *cap = &init_attr->cap; + + page_size = roce5_get_page_size(rdev, true); + + ret = roce5_check_rq_size(rdev, rqp, init_attr, has_rq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check rq size, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (has_rq) { + wqebb_num = ROCE_MAX(2U, cap->max_recv_wr); + rqp->rq.wqebb_cnt = + (u32)(ROCE_ROUNDUP_POW_OF_TWO(wqebb_num) & 0xffffffff); + + /* RQ SGE range [0,4] select 4,[5,8] select 8 */ + rqp->rq.max_sge = + (cap->max_recv_sge <= ROCE_RQ_MIN_SGE) ? + ROCE_RQ_MIN_SGE : + ((cap->max_recv_sge <= ROCE_RQ_MID_SGE) ? + ROCE_RQ_MID_SGE : + ROCE_RQ_MAX_SGE); + + sge_total_len = (u32)((u32)rqp->rq.max_sge * + sizeof(struct roce_wqe_data_seg)); + rqp->rq.wqe_shift = (u32)ROCE_ILOG2(sge_total_len); + + if ((u32)((u32)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift) < page_size) { + rqp->rq.wqebb_cnt = + (page_size >> (unsigned)rqp->rq.wqe_shift); + } + + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + rqp->rq.max_post = + ROCE_MIN(rdev->rdma_cap.dev_rdma_cap + .roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + cap->max_recv_sge = (u32)rqp->rq.max_sge; + } else { + rqp->rq.max_post = + ROCE_MIN(rdev->rdma_cap.dev_rdma_cap + .roce_own_cap.max_wqes, + rqp->rq.wqebb_cnt); + cap->max_recv_wr = (u32)rqp->rq.max_post; + cap->max_recv_sge = + ROCE_MIN((u32)rqp->rq.max_sge, + rdev->rdma_cap.dev_rdma_cap + .roce_own_cap.max_rq_sg); + } + } else { + rqp->rq.wqebb_cnt = 0; + rqp->rq.max_sge = 0; + rqp->rq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->rq.max_post = 0; + cap->max_recv_sge = 0; + } + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + return 0; +} + +static int roce5_set_user_sq_size_check(struct roce5_device *rdev, + const struct ib_qp_cap *cap, + const struct roce5_qp *rqp) +{ + u32 ud_max_inline_size; + + if (cap->max_send_wr > + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes)) { + ROCE_DEV_ERR(rdev, "Log_sq_bb_count over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (rqp->qp_type == IB_QPT_UD) { + ud_max_inline_size = + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz > + ROCE_UD_MAX_INLINE_LEN_SUB) ? + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz - + ROCE_UD_MAX_INLINE_LEN_SUB) : + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz; + } else { + ud_max_inline_size = rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_sq_inline_data_sz; + } + + /* RC:192B inline data, UD:176B */ + if (cap->max_inline_data > ud_max_inline_size) { + ROCE_DEV_ERR(rdev, "SQ max inline data over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} +/* **************************************************************************** + Prototype : roce5_set_user_sq_size + Description : roce5_set_user_sq_size + Input : struct roce5_device *rdev + struct ib_qp_cap *cap + struct roce5_qp *rqp + struct roce5_create_qp *ucmd + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +static int roce5_set_user_sq_size(struct roce5_device *rdev, + struct ib_qp_cap *cap, struct roce5_qp *rqp, + struct create_qp_cmd *ucmd) +{ + u32 tmp_max_sq_wqe_sz = 0; + u32 sq_buf_sz = 0; + int ret = 0; + u32 page_size; + + page_size = roce5_get_page_size(rdev, true); + + ret = roce5_set_user_sq_size_check(rdev, cap, rqp); + if (ret != 0) { + return ret; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + /* between max and min */ + tmp_max_sq_wqe_sz = + (u32)(ROCE_ROUNDUP_POW_OF_TWO( + rdev->rdma_cap.max_sq_desc_sz) & + 0xffffffff); + if ((ucmd->log_sq_stride > ROCE_ILOG2(tmp_max_sq_wqe_sz)) || + (ucmd->log_sq_stride < + ROCE_ILOG2(rdev->rdma_cap.wqebb_size))) { + ROCE_DEV_ERR( + rdev, + "WQE size invalid, func_id(%d) log_sq_stride:%d wqebb_size:%d tmp_max_sq_wqe_sz:%d", + rdev->glb_func_id, ucmd->log_sq_stride, + rdev->rdma_cap.wqebb_size, tmp_max_sq_wqe_sz); + return -EINVAL; + } + + rqp->sq.wqebb_cnt = (u32)(1U << ucmd->log_sq_bb_count); + rqp->sq.wqe_shift = ucmd->log_sq_stride; + + if ((u32)((u64)rqp->sq.wqebb_cnt + << (unsigned)rqp->sq.wqe_shift) < page_size) { + rqp->sq.wqebb_cnt = + (page_size >> (unsigned)rqp->sq.wqe_shift); + } + + sq_buf_sz = (u32)ROCE_ALIGN( + (u64)rqp->sq.wqebb_cnt << (unsigned)rqp->sq.wqe_shift, + page_size); + rqp->buf_size = + (int)ROCE_ALIGN(((unsigned)sq_buf_sz + + ((unsigned)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift)), + page_size); + + rqp->sq.offset = 0; + rqp->rq.offset = sq_buf_sz; + + rqp->max_inline_data = (u32)cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ucmd->log_sq_stride; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_send_wqe_overhead + Description : roce5_send_wqe_overhead + Input : enum ib_qp_type qp_type + Output : None + + 1.Date : 2015/5/7 + Modification : Created function + +**************************************************************************** */ +static int roce5_send_wqe_overhead(enum ib_qp_type qp_type) +{ + switch (qp_type) { + case IB_QPT_RC: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_frmr_tsk_seg); + + case IB_QPT_UC: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_send_tsk_seg); + + case IB_QPT_UD: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_ud_tsk_seg); + + case IB_QPT_GSI: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_ud_tsk_seg); + + case IB_QPT_XRC_TGT: + return 0; + + case IB_QPT_XRC_INI: + return sizeof(struct roce5_wqe_ctrl_seg) + + sizeof(struct roce_wqe_frmr_tsk_seg); + + default: + ROCE_ERR("Not supported this qp_type(%d)", qp_type); + return -1; + } +} + +static int roce5_set_kernel_sq_size_check(const struct roce5_device *rdev, + const struct ib_qp_cap *cap, + int *sq_max_wqe_size, + enum ib_qp_type qp_type) +{ + int wqe_overhead = 0; + int ret = 0; + + if (cap->max_send_wr > + (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes) { + ROCE_DEV_ERR(rdev, "SQ WR over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_send_sge > rdev->rdma_cap.max_sq_sg) { + ROCE_DEV_ERR(rdev, "SQ SGE over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (cap->max_inline_data > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz) { + ROCE_DEV_ERR(rdev, "SQ max inline data over range, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + wqe_overhead = roce5_send_wqe_overhead(qp_type); + if (wqe_overhead < 0) { + ROCE_DEV_ERR(rdev, "Invalid service type(%d), func_id(%d)", + (int)qp_type, rdev->glb_func_id); + return -EINVAL; + } + + *sq_max_wqe_size = ROCE_MAX((int)(cap->max_send_sge * + sizeof(struct roce_wqe_data_seg)), + (int)cap->max_inline_data) + + wqe_overhead; + if (*sq_max_wqe_size > (int)rdev->rdma_cap.max_sq_desc_sz) { + ROCE_DEV_ERR( + rdev, + "SQ WQE size over range, func_id(%d) max_send_sge:%u,max_inline_data:%u," + "wqe_overhead:%d,sq_max_wqe_size:%d,max_sq_desc_sz:%u", + rdev->glb_func_id, cap->max_send_sge, + cap->max_inline_data, wqe_overhead, *sq_max_wqe_size, + rdev->rdma_cap.max_sq_desc_sz); + return -EINVAL; + } + + return ret; +} +/* **************************************************************************** + Prototype : roce5_set_kernel_sq_size + Description : roce5_set_kernel_sq_size + Input : struct roce5_device *rdev + struct ib_qp_cap *cap + enum roce5_qp_type qp_type + struct roce5_qp *rqp + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +static int roce5_set_kernel_sq_size(struct roce5_device *rdev, + struct ib_qp_cap *cap, + enum ib_qp_type qp_type, + struct roce5_qp *rqp) + +{ + u32 sq_buf_sz = 0; + int sq_max_wqe_size = 0; + int ret = 0; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + ret = roce5_set_kernel_sq_size_check(rdev, cap, + &sq_max_wqe_size, qp_type); + if (ret != 0) { + return ret; + } + + rqp->sq.wqe_shift = (u32)ROCE_ILOG2(rdev->rdma_cap.wqebb_size); + rqp->sq_max_wqes_per_wr = (u32)ROCE_DIV_ROUND_UP( + (unsigned)sq_max_wqe_size, + (u32)(1UL << (unsigned)rqp->sq.wqe_shift)); + + /* + * As one wqebb ought to be reserved for invalidate wqebb, 1 is added to sq.wqebb_cnt here + * to satisfy the actual IB_MAD_QP_SEND_SIZE set by applications + */ + rqp->sq.wqebb_cnt = + (u32)((u32)cap->max_send_wr * rqp->sq_max_wqes_per_wr + + 1); + rqp->sq.wqebb_cnt = + (u32)(ROCE_ROUNDUP_POW_OF_TWO(rqp->sq.wqebb_cnt) & + 0xffffffff); + + if ((u32)((u64)rqp->sq.wqebb_cnt << (u32)rqp->sq.wqe_shift) < + PAGE_SIZE) { + rqp->sq.wqebb_cnt = + (PAGE_SIZE >> ((u32)rqp->sq.wqe_shift)); + } + + rqp->sq.max_sge = + (u32)(((u64)rqp->sq_max_wqes_per_wr + << (unsigned)rqp->sq.wqe_shift) - + (unsigned)roce5_send_wqe_overhead(qp_type)); + rqp->sq.max_sge = (u32)ROCE_MIN(rdev->rdma_cap.max_sq_desc_sz, + rqp->sq.max_sge) / + (u32)sizeof(struct roce_wqe_data_seg); + + sq_buf_sz = (u32)ALIGN((u64)rqp->sq.wqebb_cnt + << (unsigned)rqp->sq.wqe_shift, + PAGE_SIZE); + rqp->buf_size = (int)ALIGN(((unsigned)sq_buf_sz + + ((unsigned)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift)), + PAGE_SIZE); + + rqp->sq.offset = 0u; + rqp->rq.offset = sq_buf_sz; + + if (((rqp->sq.wqebb_cnt - 1) / rqp->sq_max_wqes_per_wr) == 1) { + ROCE_DEV_ERR(rdev, + "Only one WR,please check,func_id(%d)", + rdev->glb_func_id); + rqp->sq.max_post = ROCE_WR_MIN_NUM; + cap->max_send_wr = rqp->sq.max_post; + } else { + rqp->sq.max_post = (rqp->sq.wqebb_cnt - 1) / + rqp->sq_max_wqes_per_wr; + cap->max_send_wr = (u32)rqp->sq.max_post; + } + + cap->max_send_sge = (u32)ROCE_MIN(rqp->sq.max_sge, + rdev->rdma_cap.max_sq_sg); + rqp->max_inline_data = (u32)cap->max_inline_data; + } else { + rqp->sq.wqebb_cnt = 0; + rqp->sq.wqe_shift = ROCE_QP_DEFAULT_WQE_SHIFT; + rqp->buf_size = 0; + rqp->sq.offset = 0; + rqp->rq.offset = 0; + rqp->max_inline_data = 0; + cap->max_inline_data = 0; + } + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_qp_async_event + Description : roce5_qp_async_event + Input : struct roce5_device *rdev + struct roce5_qp *qp + int type + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +void roce5_qp_async_event(struct roce5_device *rdev, struct roce5_qp *qp, + int type) +{ + struct ib_event event; + struct ib_qp *ibqp = &qp->ibqp; + int ret; + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset dmac.(ret:%d)", ret); + return; + } + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + + switch (type) { + case ROCE_EVENT_TYPE_COMM_EST: + event.event = IB_EVENT_COMM_EST; + break; + + case ROCE_EVENT_TYPE_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + break; + + case ROCE_EVENT_TYPE_SRQ_QP_LAST_WQE: + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + + case ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + + case ROCE_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + + case ROCE_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + + default: + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static void *roce5_buf_offset(struct roce5_buf *buf, u32 offset) +{ + return (void *)((char *)buf->direct.buf + offset); +} + +void *roce5_get_wqe(struct roce5_qp *rqp, u32 offset) +{ + return roce5_buf_offset(&rqp->buf, offset); +} + +static void *roce5_get_send_wqe_head(struct roce5_qp *rqp) +{ + return roce5_get_wqe(rqp, rqp->sq.offset); +} + +/* **************************************************************************** + Prototype : roce5_get_send_wqe_tail + Description : roce5_get_send_wqe_tail + Input : struct roce5_qp *rqp + Output : None + + 1.Date : 2015/8/8 + Modification : Created function + +**************************************************************************** */ +static void *roce5_get_send_wqe_tail(struct roce5_qp *rqp) +{ + return roce5_get_wqe(rqp, rqp->sq.offset + (rqp->sq.wqebb_cnt + << (u32)rqp->sq.wqe_shift)); +} + +/* **************************************************************************** + Prototype : roce5_sq_buf_init + Description : roce5_sq_buf_init + Input : struct roce5_qp *qp + Output : None + + 1.Date : 2015/11/9 + Modification : Created function + +**************************************************************************** */ +static void roce5_sq_buf_init(struct roce5_qp *qp) +{ + u32 entries = qp->rq.offset / (1U << qp->sq.wqe_shift); + u8 *wqe = NULL; + struct roce5_wqe_ctrl_seg *wqe_ctrl = NULL; + u32 i; + + for (i = 0; i < entries; i++) { + wqe = (u8 *)qp->buf.direct.buf + + (u32)(i * (1U << ((u32)qp->sq.wqe_shift))); + wqe_ctrl = (struct roce5_wqe_ctrl_seg *)((void *)wqe); + wqe_ctrl->dw0.value = be32_to_cpu(wqe_ctrl->dw0.value); + wqe_ctrl->dw0.bs.owner = 1; + wqe_ctrl->dw0.value = cpu_to_be32(wqe_ctrl->dw0.value); + } +} + +static int roce5_check_qp_udata(struct roce5_device *rdev, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct roce5_qp *rqp, + struct create_qp_cmd *ucmd) +{ + int ret = 0; + + if ((udata == NULL) || + (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0)) { + ret = -EFAULT; + return ret; + } + + ret = roce5_set_user_sq_size(rdev, &init_attr->cap, rqp, ucmd); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set user sq_size, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return ret; +} + +static bool +roce5_need_qpn_lb1_consistent_srqn(const struct roce5_qp *rqp, + const struct roce5_device *rdev, + const struct ib_qp_init_attr *init_attr) +{ + if ((init_attr->srq == NULL) || (rdev->cfg_info.lb_en == 0)) { + return false; + } + + if (ROCE5_SUPPORT_QP_SRQN_LB_MSG_V2(rdev) != 0) { + return true; + } + + return false; +} + +static bool +roce5_need_qpn_lb1_consistent_scqn(struct roce5_device *rdev, + struct ib_qp_init_attr *init_attr) +{ + if ((init_attr->send_cq == NULL) || (rdev->cfg_info.lb_en == 0)) { + return false; + } + + if (ROCE5_SUPPORT_ALLOC_QPC_MSG_FEATURE(rdev) != 0) { + return false; + } + return true; +} + +static int roce5_find_free_lb_qpn(struct roce5_device *rdev) +{ + int idx = 0; + for (; idx < ROCE_LB_QPN_MAX; idx++) { + if (rdev->qp_lb_scqn_ctrl.qp[idx] == 0) { + return idx; + } + } + return -1; +} + +static int roce5_alloc_qpc(struct roce5_qp *rqp, struct roce5_device *rdev, + u32 qpn, struct ib_qp_init_attr *init_attr) +{ + struct roce5_srq *rsrq = NULL; + struct roce5_cq *rcq = NULL; + u32 qpn_tmp = qpn; + u32 roce_lb_mask = 0; + u32 cqm_lb_mode = 0; + int idx = -1; + + if ((rqp == NULL) || (rdev == NULL) || (init_attr == NULL)) { + ROCE_ERR("rqp, rdev or init_attr is NULL"); + return -EINVAL; + } + + if (ROCE_IS_DEV_1825V100(rdev)) { + cqm_lb_mode = CQM_XID_LOW_BIT_1_1_1; + roce_lb_mask = roce5_LB1_MASK_1825V100; + } else { + ROCE_DEV_ERR(rdev, "Unknown device type(%u), func_id(%u).", + rdev->device_type, rdev->glb_func_id); + return -EINVAL; + } + mutex_lock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + + if (roce5_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + rsrq = to_roce5_srq(init_attr->srq); + qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, cqm_lb_mode, + rsrq->srqn); + } else if ((qpn != ROCE_QP_GSI_QP_NUM) && + roce5_need_qpn_lb1_consistent_scqn(rdev, init_attr)) { + rcq = to_roce5_cq(init_attr->send_cq); + //if not found free of qpn, idx return -1, not need do LB + idx = roce5_find_free_lb_qpn(rdev); + if (idx >= 0) { + qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_ALL, + cqm_lb_mode, rcq->cqn); + } + } + + rqp->qpc_info = cqm5_object_qpc_mpt_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, rqp, + qpn_tmp, 0, 0); + + if (rqp->qpc_info == NULL && rdev->full_qp_spec) { + /* For subsequent QPs, XID alignment is not mandatory. */ + rqp->qpc_info = cqm5_object_qpc_mpt_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_SERVICE_CTX, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.qpc_entry_sz, + rqp, qpn, 0, 0); + } else if (rqp->qpc_info != NULL) { + if (idx >= 0) { //idx >= 0 indicates has emptyremaining of lb qpn bit position + rdev->qp_lb_scqn_ctrl.qp[idx] = rqp->qpc_info->xid; + ROCE_DEV_INFO( + rdev, + "RoCE create lb qpn func_id(%d), qpn(%d), idx(%d)", + rdev->glb_func_id, rqp->qpc_info->xid, idx); + } + } + mutex_unlock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + if (rqp->qpc_info == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create qpc mpt by cqm object, func_id(%d), qpn(%u)", + rdev->glb_func_id, qpn_tmp); + return -ENOMEM; + } + + rqp->qpn = rqp->qpc_info->xid; + if (roce5_need_qpn_lb1_consistent_srqn(rqp, rdev, init_attr)) { + if (rsrq != NULL && ((rsrq->srqn & roce_lb_mask) != + (rqp->qpn & roce_lb_mask))) { + ROCE_DEV_ERR( + rdev, + "Failed to create qpc LB srqn, func_id(%d), qpn(%u) srqn(%u) mask:0x%x", + rdev->glb_func_id, rqp->qpn, rsrq->srqn, + roce_lb_mask); + } + } + + return 0; +} + +static int roce5_create_qp_user_umem_mtt(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct ib_pd *ibpd, + struct create_qp_cmd *ucmd) +{ + u32 npages = 0; + int ret; + + if (rqp->shadow != 0) { + return 0; + } + rqp->umem = roce5_umem_get(&rdev->ib_dev, ibpd->uobject->context, + ucmd->buf_addr, (size_t)rqp->buf_size, 0, 0); + if (IS_ERR(rqp->umem)) { + ROCE_DEV_ERR(rdev, "Failed to get ib_umem, func_id(%d)", + rdev->glb_func_id); + ret = (int)PTR_ERR(rqp->umem); + return ret; + } + + rqp->buf.page_shift = (int)roce5_umem_get_buf_page_shift( + rdev, rqp->umem, ucmd->buf_addr); + npages = roce5_umem_get_mtt_npages(rdev, rqp->umem, + (u32)rqp->buf.page_shift); + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)rqp->buf.page_shift, + &rqp->mtt, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc rdma_mtt, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_umem; + } + + ret = roce5_umem_write_mtt(rdev, &rqp->mtt, rqp->umem, 1); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +err_umem: + ib_umem_release(rqp->umem); + return ret; +} + +void roce5_destroy_qp_user_umem_mtt(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + if (rqp->shadow != 0) { + return; + } + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + ib_umem_release(rqp->umem); +} + +/* **************************************************************************** + Prototype : roce5_create_qp_user + Description : create qp resource for user space qp + Input : struct roce5_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + struct ib_udata *udata + struct roce5_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_qp_user(struct roce5_device *rdev, struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct roce5_qp *rqp) +{ + int ret = 0; + struct roce5_ucontext *roce5_uctx = NULL; + struct create_qp_cmd ucmd = { 0 }; + u32 qpn = ROCE_QP_INVLID_QP_NUM; + + roce5_uctx = to_roce5_ucontext(ibpd->uobject->context); + + ret = roce5_check_qp_udata(rdev, init_attr, udata, rqp, &ucmd); + if (ret != 0) { + return ret; + } + + ret = roce5_create_qp_user_umem_mtt(rdev, rqp, ibpd, &ucmd); + if (ret != 0) { + return ret; + } + + if (init_attr->qp_type != IB_QPT_XRC_TGT) { + ret = roce5_db_map_user(roce5_uctx, ucmd.db_addr, &rqp->db, + rqp->shadow); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to map db page to user, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_map_db; + } + } + + ret = roce5_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) { + goto err_alloc_qpc; + } + + return 0; + +err_alloc_qpc: + if (init_attr->qp_type != IB_QPT_XRC_TGT) { + roce5_db_unmap_user(roce5_uctx, &rqp->db, rqp->shadow); + } + +err_map_db: + roce5_destroy_qp_user_umem_mtt(rdev, rqp); + + return ret; +} + +static void roce5_free_wrid(struct roce5_qp *rqp) +{ + if (rqp->rq.wrid) { + kvfree(rqp->rq.wrid); + } + + if (rqp->sq.wrid) { + kvfree(rqp->sq.wrid); + } +} + +static int roce5_get_wrid(struct roce5_qp *rqp, struct roce5_device *rdev) +{ + int ret = 0; + + rqp->sq.wrid = (u64 *)kzalloc((unsigned)rqp->sq.wqebb_cnt * sizeof(u64), + GFP_KERNEL); + if (rqp->sq.wrid == NULL) { + rqp->sq.wrid = (u64 *)(void *)vzalloc( + (size_t)(rqp->sq.wqebb_cnt * sizeof(u64))); + } + + rqp->rq.wrid = (u64 *)kzalloc((unsigned)rqp->rq.wqebb_cnt * sizeof(u64), + GFP_KERNEL); + if (rqp->rq.wrid == NULL) { + rqp->rq.wrid = (u64 *)(void *)vzalloc( + (size_t)(rqp->rq.wqebb_cnt * sizeof(u64))); + } + + if ((rqp->sq.wrid == NULL) || (rqp->rq.wrid == NULL)) { + ROCE_DEV_ERR(rdev, + "Failed to alloc wrid of sq or rq, func_id(%d)", + rdev->glb_func_id); + ret = -ENOMEM; + } + + return ret; +} + +static void roce5_create_qp_kernel_set_attr(struct roce5_qp *rqp, + cqm_queue_s *qp_buf_info) +{ + int ret; + ret = memset_s(qp_buf_info->q_room_buf_1.direct.va, + (size_t)((unsigned)rqp->buf_size), 0, + (size_t)((unsigned)rqp->buf_size)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_buf_info.(ret:%d)", ret); + return; + } + rqp->buf.direct.buf = qp_buf_info->q_room_buf_1.direct.va; + rqp->buf.direct.map = qp_buf_info->q_room_buf_1.direct.pa; + rqp->buf.page_shift = + (int)ROCE_ILOG2(qp_buf_info->q_room_buf_1.buf_size); + + roce5_sq_buf_init(rqp); + + rqp->db.db_record = (__be32 *)(void *)(&qp_buf_info->q_header_vaddr + ->doorbell_record); + rqp->db.dma = qp_buf_info->q_header_paddr; + + qp_buf_info->q_header_vaddr->doorbell_record = 0; + + rqp->qp_buf_info = qp_buf_info; + + rqp->sq_head_addr = (u8 *)roce5_get_send_wqe_head(rqp); + rqp->sq_tail_addr = (u8 *)roce5_get_send_wqe_tail(rqp); + rqp->max_dwqe_size = 0; + rqp->sq_wqe_ssn = 0; + + if (rqp->max_inline_data != 0) { + rqp->max_dwqe_size = ROCE_QP_MAX_DWQE_SIZE; + } +} + +static int roce5_create_qp_kernel_normal(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + int ret; + cqm_queue_s *qp_buf_info = NULL; + + qp_buf_info = cqm5_object_rdma_queue_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_QP, + (u32)rqp->buf_size, rqp, true, ROCE_QP_INVLID_QP_NUM, 0, 0); + if (qp_buf_info == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue by cqm object, func_id(%d)", + rdev->glb_func_id); + return -1; + } + + roce5_create_qp_kernel_set_attr(rqp, qp_buf_info); + + rqp->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, qp_buf_info->q_room_buf_1.buf_number, + (u32)rqp->buf.page_shift, &rqp->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_alloc_mtt; + } + ret = roce5_buf_write_mtt(rdev, &rqp->mtt, + &rqp->qp_buf_info->q_room_buf_1); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write rdma mtt, func_id(%d)", + rdev->glb_func_id); + goto err_write_mtt; + } + + ret = roce5_get_wrid(rqp, rdev); + if (ret != 0) { + goto err_alloc_wrid; + } + + return 0; + +err_alloc_wrid: + roce5_free_wrid(rqp); + +err_write_mtt: + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + +err_alloc_mtt: + cqm5_object_delete(&rqp->qp_buf_info->object); + + return ret; +} + +static void roce5_destroy_qp_kernel_normal(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + roce5_free_wrid(rqp); + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); + cqm5_object_delete(&rqp->qp_buf_info->object); +} + +/* **************************************************************************** + Prototype : roce5_create_qp_kernel + Description : create qp resource for kernel space qp + Input : struct roce5_device *rdev + struct ib_qp_init_attr *init_attr + struct roce5_qp *rqp + u32 qpn + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_qp_kernel(struct roce5_device *rdev, + struct ib_qp_init_attr *init_attr, + struct roce5_qp *rqp, u32 qpn) +{ + int ret = 0; + + ret = roce5_set_kernel_sq_size(rdev, &init_attr->cap, + (enum ib_qp_type)rqp->qp_type, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set sq buffer size in kernel, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + ret = roce5_create_qp_kernel_normal(rdev, rqp); + if (ret != 0) { + return ret; + } + } + + ret = roce5_alloc_qpc(rqp, rdev, qpn, init_attr); + if (ret != 0) { + goto err_alloc_qpc; + } + + return 0; + +err_alloc_qpc: + if (rqp->qp_type != IB_QPT_XRC_TGT) { + roce5_destroy_qp_kernel_normal(rdev, rqp); + } + + return ret; +} + +static void roce5_qp_add_cq_list(struct ib_qp_init_attr *init_attr, + struct roce5_qp *rqp, + struct roce5_device *rdev) +{ + struct roce5_cq *send_rcq = to_roce5_cq(init_attr->send_cq); + struct roce5_cq *recv_rcq = to_roce5_cq(init_attr->recv_cq); + unsigned long flags = 0; /* type required by kernel API */ + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce5_lock_cqs(send_rcq, recv_rcq); + list_add_tail(&rqp->qps_list, &rdev->qp_list); + list_add_tail(&rqp->cq_send_list, &send_rcq->send_qp_list); + list_add_tail(&rqp->cq_recv_list, &recv_rcq->recv_qp_list); + roce5_unlock_cqs(send_rcq, recv_rcq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); +} + +static int roce5_alloc_xrc_extra_dma(const struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + dma_addr_t wb_dma = 0; + void *wb_va = NULL; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + return 0; + } + + wb_va = dma_alloc_coherent(rdev->dev, (size_t)4UL, &wb_dma, GFP_KERNEL); + if (wb_va == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc wb_va, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + rqp->wb_dma = wb_dma; + rqp->wb_va = wb_va; + + return 0; +} + +#define ROCE_MAX_PI_ON_CHIP_QPN 1024 /* temporarily when set PI is 1024(count word), later debug add */ +static int roce5_set_pi_on_chip(const struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + /* shadow scenario next, actual use of is vf, and pi on chip function in shadow scenario not already verify, default disable */ + if (ROCE5_SUPPORT_SET_PI_ON_CHIP_MSG_V2(rdev) != 0 && + rqp->shadow == 0) { + /* becauseslice on store pi resource has limit, disable VF of pi_on_chip, such as need enabled, need modify configuration file in chip store store PI data, Note 审monitorslice on resource whether satisfy requirement */ + rqp->pi_on_chip = (roce5_func_is_vf(rdev->hwdev) == false) && + (rqp->qpn < ROCE_MAX_PI_ON_CHIP_QPN); + } + + return 0; +} + +static int roce5_do_create_qp(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret = 0; + + mutex_init(&rqp->mutex); + + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + + rqp->qp_state = IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + + ret = roce5_alloc_xrc_extra_dma(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc xrc extra dma, func_id(%d), ret(%d).", + rdev->glb_func_id, ret); + return ret; + } + + if (ibpd != NULL && ibpd->uobject) { + ret = roce5_set_rq_size(rdev, rqp, init_attr, true, + rqp->has_rq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set rq size, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + + ret = roce5_create_qp_user(rdev, ibpd, init_attr, udata, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to create qp user, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + } else { + ret = roce5_set_rq_size(rdev, rqp, init_attr, false, + rqp->has_rq); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set rq size, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + + ret = roce5_create_qp_kernel(rdev, init_attr, rqp, + ROCE_QP_INVLID_QP_NUM); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to create qp kernel, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto create_qp_err; + } + + roce5_qp_add_cq_list(init_attr, rqp, rdev); + } + + ret = roce5_set_pi_on_chip(rdev, rqp); + if (ret != 0) { + goto create_qp_err; + } + + ret = roce5_create_qp_ext_init(rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init qp extension, ret(%d).", + ret); + goto create_qp_err; + } + + return ret; +create_qp_err: + if (rqp->wb_va != NULL) { + dma_free_coherent(rdev->dev, sizeof(u32), rqp->wb_va, + rqp->wb_dma); + } + return ret; +} + +/* **************************************************************************** + Prototype : roce5_create_qp_gsi + Description : roce5_create_qp_gsi + Input : struct roce5_device *rdev + struct ib_pd *ibpd + struct ib_qp_init_attr *init_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_qp_gsi(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr) +{ + int ret = 0; + + rqp->sqp_attr = (struct roce5_sqp_attr *)kzalloc( + sizeof(struct roce5_sqp_attr), GFP_KERNEL); + if (rqp->sqp_attr == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to allocate memory for sqp attribute."); + return -ENOMEM; + } + + mutex_init(&rqp->mutex); + + spin_lock_init(&rqp->sq.lock); + spin_lock_init(&rqp->rq.lock); + + rqp->qp_state = (u8)IB_QPS_RESET; + rqp->qp_type = init_attr->qp_type; + rqp->sq_signal_bits = (u32)(init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); + rqp->has_rq = !init_attr->srq; + + ret = roce5_set_rq_size(rdev, rqp, init_attr, false, rqp->has_rq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set rq size, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_out; + } + + ret = roce5_create_qp_kernel(rdev, init_attr, rqp, ROCE_QP_GSI_QP_NUM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create qp kernel, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_out; + } + + rqp->qpn = ROCE_QP_GSI_QP_NUM; + + roce5_qp_add_cq_list(init_attr, rqp, rdev); + + return 0; + +err_out: + kfree(rqp->sqp_attr); + + return ret; +} + +static int roce5_create_check_init_attr(const struct ib_pd *ibpd, + const struct ib_qp_init_attr *init_attr, + const struct ib_udata *udata) +{ + if (init_attr == NULL) { + ROCE_ERR("init_attr is null"); + return (-EINVAL); + } + + if ((ibpd == NULL) && (init_attr->qp_type != IB_QPT_XRC_TGT)) { + ROCE_ERR( + "Ibpd is null and qp_type is not IB_QPT_XRC_TGT, qp_type(0x%x)", + init_attr->qp_type); + return (-EINVAL); + } + + if ((udata) && (init_attr->qp_type == IB_QPT_GSI)) { + ROCE_ERR("Udata is not null and qp_type is IB_QPT_GSI"); + return (-EINVAL); + } + + return 0; +} + +static int roce5_set_qp_attr(struct ib_qp_init_attr *init_attr, + struct roce5_qp *rqp, struct roce5_device *rdev, + struct ib_pd **ibpd, int qp_ext) +{ + u16 xrcdn = 0; + + rqp->qp_ext = (u8)qp_ext; + if (init_attr->qp_type == IB_QPT_XRC_TGT) { + *ibpd = to_roce5_xrcd(init_attr->xrcd)->pd; + xrcdn = (u16)to_roce5_xrcd(init_attr->xrcd)->xrcdn; + init_attr->send_cq = to_roce5_xrcd(init_attr->xrcd)->cq; + init_attr->recv_cq = init_attr->send_cq; + + rqp->has_rq = false; + rqp->xrcdn = xrcdn; + } else if (init_attr->qp_type == IB_QPT_XRC_INI) { + init_attr->recv_cq = init_attr->send_cq; + rqp->has_rq = false; + } else if ((init_attr->qp_type == IB_QPT_RC) || + (init_attr->qp_type == IB_QPT_UC) || + (init_attr->qp_type == IB_QPT_UD)) { + rqp->has_rq = !init_attr->srq; + } else { + ROCE_DEV_ERR(rdev, "Unknown QP Type(%u), func_id(%d)", + init_attr->qp_type, rdev->glb_func_id); + return (-EINVAL); + } + + return 0; +} + +static int roce5_create_gsi_qp_result(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr) +{ + int ret; + + ret = roce5_create_qp_gsi(rdev, rqp, init_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create gsi qp, func_id(%d), ret(%d).", + rdev->glb_func_id, ret); + return ret; + } + + rqp->ibqp.qp_num = rqp->qpn; + return 0; +} + +static int roce5_create_qp_common(struct roce5_qp *rqp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, int qp_ext) +{ + int ret; + int uld_ret; + //when init_attr->qp_type == IB_QPT_XRC_TGT, ibpd is NULL. + struct ib_pd *ibpd = rqp->ibqp.pd; + struct roce5_device *rdev = + to_roce5_dev(ibpd ? ibpd->device : init_attr->xrcd->device); + roce_user_handle_t hdl = { 0 }; + + if (rdev->ecn_ctx.cc_algo == ROCE_CC_LDCP_ALGO && + (init_attr->qp_type == IB_QPT_XRC_TGT || + init_attr->qp_type == IB_QPT_XRC_INI)) { + ROCE_DEV_ERR( + rdev, + "XRC QP cant be created when LDCP is enabled, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + if (init_attr->qp_type == IB_QPT_GSI) { + return roce5_create_gsi_qp_result(rdev, rqp, init_attr); + } + + rqp->rqp_magic = ROCE_RQP_MAGIC; + ret = roce5_set_qp_attr(init_attr, rqp, rdev, &ibpd, qp_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set qp attribute, ret(%d).", ret); + goto err_set_attr; + } + + ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, NULL, udata, &rqp->mtt, + &rqp->db.dma); + ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rqp->shadow, &rqp->shadow_vfid, + NULL); + ret = roce5_uld_qp_create((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_ERR("Failed to execute uld qp creation, ret(%d)", ret); + goto err_set_attr; + } + ret = roce5_do_create_qp(rdev, rqp, ibpd, init_attr, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create qp in common process, ret(%d).", + ret); + goto err_create_qp; + } + + roce5_set_qp_dif_attr(rqp, init_attr, rdev); + rqp->ibqp.qp_num = rqp->qpn; + rqp->rsp_depth = 0; + + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.alloc_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + + return 0; + +err_create_qp: + uld_ret = roce5_uld_qp_destroy((roce_uld_handle)&hdl); + if (uld_ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to destroy uld qp in common process, uld_ret(%d).", + uld_ret); + } +err_set_attr: + return ret; +} + +struct ib_qp *roce5_create_qp_compat(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct roce5_qp *rqp = NULL; + int ret; + + ret = roce5_create_check_init_attr(ibpd, init_attr, udata); + if (ret != 0) { + ROCE_ERR("Failed to check init attr"); + return (struct ib_qp *)ERR_PTR((long)ret); + } + + rqp = (struct roce5_qp *)kzalloc(sizeof(*rqp), GFP_KERNEL); + if (rqp == NULL) { + ROCE_ERR("Failed to alloc roce qp."); + return (struct ib_qp *)ERR_PTR((long)-ENOMEM); + } + + rqp->ibqp.pd = ibpd; + + ret = roce5_create_qp_common(rqp, init_attr, udata, ROCE_QP); + if (ret != 0) { + ROCE_ERR("Failed to execute common qp creation, ret(%d)", ret); + goto err_create_qp; + } + + return &(rqp->ibqp); +err_create_qp: + kfree(rqp); + return (struct ib_qp *)ERR_PTR((long)-ENOMEM); +} + +int roce5_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_qp *rqp = NULL; + + if (ibqp == NULL) { + ROCE_ERR("Invalid parameter, ibqp is NULL."); + return -EINVAL; + } + + ret = roce5_create_check_init_attr(ibqp->pd, init_attr, udata); + if (ret != 0) { + ROCE_ERR("Failed to check init attr"); + return ret; + } + + rqp = to_roce5_qp(ibqp); + ret = roce5_create_qp_common(rqp, init_attr, udata, ROCE_QP); + if (ret != 0) { + ROCE_ERR("Failed to execute common qp creation, ret(%d)", ret); + return ret; + } + + return 0; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h new file mode 100644 index 000000000..c435e7090 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ctx_format.h @@ -0,0 +1,278 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_CTX_FOMRAT_H +#define ROCE_QP_CTX_FOMRAT_H + +#include <linux/types.h> + +/* Align each field with 4bytes. */ +#pragma pack(push) +#pragma pack(4) + +struct roce5_chip_seg_rrwc { + u32 dw0; + + union { + u64 rrw_curt_sge_va; + struct { + u32 rrw_curt_sge_va_hi; + u32 rrw_curt_sge_va_lo; + } dw1; + }; + + u32 rsvd[5]; +}; + +struct roce5_chip_seg_rcc { + u32 dw0; + + union { + u64 rc_curt_sge_va; + struct { + u32 rc_curt_sge_va_hi; + u32 rc_curt_sge_va_lo; + } dw1; + }; + + u32 rsvd[5]; +}; + +struct roce5_chip_seg_sqc { + u32 rsvd[9]; + + union { + u64 sq_curt_sge_va; + struct { + u32 sq_curt_sge_va_hi; + u32 sq_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_sqac { + u32 rsvd[9]; + + union { + u64 sqa_curt_sge_va; + struct { + u32 sqa_curt_sge_va_hi; + u32 sqa_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_rqc { + u32 rsvd[9]; + + union { + u64 rq_curt_sge_va; + struct { + u32 rq_curt_sge_va_hi; + u32 rq_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_srqc { + u32 rsvd[9]; + + union { + u64 srq_curt_sge_va; + struct { + u32 srq_curt_sge_va_hi; + u32 srq_curt_sge_va_lo; + } dw9; + }; + + u32 rsvd1[5]; +}; + +struct roce5_chip_seg_qpcc { + union { + u64 sq_rq_l0mtt_gpa; + struct { + u32 sq_rq_l0mtt_gpa_hi; + u32 sq_rq_l0mtt_gpa_lo; + } dw0; + }; + + union { + u64 sq_rq_pi_record_gpa_at_hop_num; + struct { + u32 sq_rq_pi_record_gpa_hi; + u32 sq_rq_pi_record_gpa_lo_at_hop_num; + } dw2; + }; + + u32 dw4; + + union { + struct { + u32 rsvd1 : 22; + u32 qp_rre : 1; + u32 qp_rae : 1; + u32 qp_rwe : 1; + u32 rsvd : 7; + } bs; + u32 value; + } dw5; + + u32 rsvd[2]; +}; + +typedef struct roce5_qpc_chip_seg { + struct roce5_chip_seg_rrwc rrwc; + struct roce5_chip_seg_rcc rcc; + struct roce5_chip_seg_sqc sqc; + struct roce5_chip_seg_sqac sqac; + union { + struct roce5_chip_seg_rqc rqc; + struct roce5_chip_seg_srqc srqc; + }; + struct roce5_chip_seg_qpcc qpcc; +} roce5_qpc_chip_seg_s; + +struct roce5_qpc_timer_seg { + u32 rsvd[8]; +}; + +typedef struct roce5_qpc_drv_seg { + union { + struct { + u32 dest_qp : 24; + u32 state : 4; + u32 rsvd : 4; + } bs; + u32 value; + } dw0; + + u32 dw1; + + union { + struct { + u32 pmtu : 3; + u32 rsvd : 13; + u32 to_retry_limit : 3; + u32 rnr_retry_limit : 3; + u32 min_rnr_nak : 5; + u32 ack_to : 5; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 rsvd : 26; + u32 rra_max : 3; + u32 sra_max : 3; + } bs; + u32 value; + } dw3; +} roce5_qpc_drv_seg_s; + +typedef struct roce5_drv_path_info { + u32 rsvd[2]; + + u8 dgid[16]; + + union { + struct { + u32 flow_label : 20; + u32 tclass : 8; + u32 rsvd : 4; + } bs; + u32 value; + } dw6; + + union { + struct { + u32 hoplmt : 8; + u32 sgid_index : 7; + u32 rsvd : 14; + u32 sl : 3; + } bs; + u32 value; + } dw7; +} roce5_drv_path_info_s; + +typedef struct roce5_ucode_common_ctx { + union { + struct { + u32 rsvd1 : 20; + u32 port : 2; + u32 rsvd : 10; + } bs; + u32 value; + } dw0; + + u32 dw1; + + union { + u32 qkey; + u32 value; + } dw2; + + u32 dw3; +} roce5_ucode_common_ctx_s; + +typedef struct roce5_ucode_sq_ctx { + union { + struct { + u32 next_send_psn : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw8; + + u32 rsvd[12]; +} roce5_ucode_sq_ctx_s; + +typedef struct roce5_ucode_rq_ctx { + union { + struct { + u32 next_rcv_psn : 24; + u32 rsvd : 8; + } bs; + u32 value; + } dw20; + + u32 rsvd[9]; +} roce5_ucode_rq_ctx_s; + +typedef struct roce5_drv_ucode_all_info { + /* DW0~DW3 */ + roce5_ucode_common_ctx_s common; + + /* DW4~DW16 */ + roce5_ucode_sq_ctx_s sq_ctx; + + /* DW17~DW26 */ + roce5_ucode_rq_ctx_s rq_ctx; + + u32 rsvd; +} roce5_drv_ucode_all_info_s; + +typedef struct roce5_qpc_sw_seg { + roce5_qpc_drv_seg_s drv_seg; + roce5_drv_path_info_s path_seg; + roce5_drv_ucode_all_info_s ucode_seg; + u32 rsvd[8]; +} roce5_qpc_sw_seg_s; + +typedef struct roce5_qp_context { + struct roce5_qpc_chip_seg chip_seg; + struct roce5_qpc_timer_seg timer_seg; + struct roce5_qpc_sw_seg sw_seg; +} roce5_qp_context_s; + +#pragma pack(pop) + +#endif /* ROCE_QP_CTX_FOMRAT_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c new file mode 100644 index 000000000..71a77ada0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_destroy.c @@ -0,0 +1,342 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_qp_ext.h" +#include "roce_qp_extension.h" +#include "roce_restore.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif +#include "hinic5_hmm.h" +#include "securec.h" +#include "roce_os_compat.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_qp_inner.h" + +static void roce5_clear_lb_qpn(struct roce5_device *rdev, u32 qpn) +{ + int idx = 0; + mutex_lock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + for (; idx < ROCE_LB_QPN_MAX; idx++) { + if (rdev->qp_lb_scqn_ctrl.qp[idx] == qpn) { + rdev->qp_lb_scqn_ctrl.qp[idx] = 0; + ROCE_DEV_INFO( + rdev, + "RoCE destroy lb qpn func_id(%d), qpn(%d), idx(%d)", + rdev->glb_func_id, qpn, idx); + break; + } + } + mutex_unlock(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); +} + +static void roce5_clean_qp_user(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_udata *udata) +{ + struct roce5_ucontext *ucontext = NULL; + + roce5_free_opt_rdmarc(rqp); + + cqm5_object_delete(&(rqp->qpc_info->object)); + + roce5_clear_lb_qpn(rdev, rqp->qpn); + + roce5_destroy_pcqc_ext(rqp); + + ucontext = roce5_queue_to_ucontext((void *)(&rqp->ibqp), udata, + ROCE_Q_TYPE_QP); + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + roce5_db_unmap_user(ucontext, &rqp->db, rqp->shadow); + } + + roce5_destroy_qp_user_umem_mtt(rdev, rqp); +} + +/* **************************************************************************** + Prototype : roce5_cq_clean_process + Description : roce5_cq_clean_process + Input : struct roce5_cq *cq + u32 qpn + struct roce5_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_cq_clean_process(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq) +{ + u32 prod_index = 0; + int nfreed = 0; + struct roce_cqe *cqe = NULL; + struct roce_cqe *dest = NULL; + u8 owner_bit = 0; + int ret; + + /* + * First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->cons_index; roce5_get_sw_cqe(cq, prod_index); + ++prod_index) { + if (prod_index == (u32)((int)cq->cons_index + cq->ibcq.cqe)) { + break; + } + } + + /* + * Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + /* xcqe:qpn's cqe. + CI + +--------+------+------+------+----------+ + b4_clean: | | cqe1 | xcqe2| cqe3 | | + +--------+------+------+------+----------+ + \ + \ + +---------------+------+------+----------+ + af_clean: | | cqe1 | cqe3 | | + +---------------+------+------+----------+ + */ + while ((int)--prod_index - (int)cq->cons_index >= 0) { + cqe = (struct roce_cqe *)roce5_get_cqe( + cq, prod_index & (u32)cq->cqe_mask); + + cqe->dw0.value = roce5_convert_cpu32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_cpu32(cqe->dw1.value); + cqe->dw7.value = roce5_convert_cpu32(cqe->dw7.value); + + if (cqe->dw0.bs.qpn == qpn) { + if (srq && (cqe->dw1.bs.s_r == ROCE_CQE_RECV_COMP)) { + roce5_free_srq_wqe(srq, cqe->dw7.bs.wqe_cnt); + } + + ++nfreed; + } else if (nfreed != 0) { + dest = (struct roce_cqe *)roce5_get_cqe( + cq, (unsigned int)((int)prod_index + nfreed) & + (unsigned int)cq->cqe_mask); + dest->dw0.value = roce5_convert_cpu32(dest->dw0.value); + owner_bit = dest->dw0.bs.owner; + ret = memcpy_s((void *)dest, sizeof(struct roce_cqe), + (void *)cqe, sizeof(struct roce_cqe)); + if (ret != 0) { + ROCE_ERR("[Failed to memcpy dest.(ret:%d)", + ret); + return; + } + dest->dw0.bs.owner = owner_bit; + dest->dw0.value = roce5_convert_be32(dest->dw0.value); + dest->dw1.value = roce5_convert_be32(dest->dw1.value); + dest->dw7.value = roce5_convert_be32(dest->dw7.value); + } else { + ROCE_INFO("Nothing need to do"); + } + + cqe->dw0.value = roce5_convert_be32(cqe->dw0.value); + cqe->dw1.value = roce5_convert_be32(cqe->dw1.value); + cqe->dw7.value = roce5_convert_be32(cqe->dw7.value); + } + + if (nfreed != 0) { + cq->cons_index += (u32)nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + + roce5_cq_set_ci(cq); + } +} + +/* **************************************************************************** + Prototype : roce5_clean_qp_kernel + Description : roce5_clean_qp_kernel + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Created function + +**************************************************************************** */ +static void roce5_clean_qp_kernel(struct roce5_device *rdev, + struct roce5_qp *rqp) +{ + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + unsigned long flags = 0; /* type required by kernel API */ + + roce5_free_opt_rdmarc(rqp); + + roce5_get_cqs(rqp, &send_cq, &recv_cq); + + spin_lock_irqsave(&rdev->reset_flow_resource_lock, flags); + roce5_lock_cqs(send_cq, recv_cq); + /* del from lists under both locks above to protect reset flow paths */ + list_del(&rqp->qps_list); + list_del(&rqp->cq_send_list); + list_del(&rqp->cq_recv_list); + + roce5_cq_clean_process(recv_cq, rqp->qpn, + rqp->ibqp.srq ? to_roce5_srq(rqp->ibqp.srq) : + NULL); + + if (send_cq != recv_cq) { + roce5_cq_clean_process(send_cq, rqp->qpn, NULL); + } + + roce5_unlock_cqs(send_cq, recv_cq); + spin_unlock_irqrestore(&rdev->reset_flow_resource_lock, flags); + + cqm5_object_delete(&(rqp->qpc_info->object)); + + roce5_clear_lb_qpn(rdev, rqp->qpn); + + if (rqp->sq.wrid) { + kvfree(rqp->sq.wrid); + rqp->sq.wrid = NULL; + } + + if (rqp->rq.wrid) { + kvfree(rqp->rq.wrid); + rqp->rq.wrid = NULL; + } + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + cqm5_object_delete(&(rqp->qp_buf_info->object)); + } + + hmm_mtt_free(rdev->hwdev, &rqp->mtt, SERVICE_T_ROCE); +} + +static int roce5_qp_destroy(struct roce5_qp *rqp, struct roce5_device *rdev) +{ + int ret = 0; + + ret = roce5_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) to RESET, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce5_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "QP(0x%06x) cache invalid, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + return 0; +} + +int roce5_destroy_qp_common(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_pd *pd = roce5_get_pd(rqp); + roce_user_handle_t hdl = { 0 }; + + roce5_destroy_qp_ext_deinit(rqp); + + if (rqp->wb_va != NULL) { + dma_free_coherent(rdev->dev, sizeof(u32), rqp->wb_va, + rqp->wb_dma); + rqp->wb_va = NULL; + } + + if (rqp->qp_state != IB_QPS_RESET) { + ret = roce5_qp_destroy(rqp, rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to destroy qp, ret(%d).", + ret); + return ret; + } + + rqp->qp_state = IB_QPS_RESET; + } + + if (pd->ibpd.uobject) { + roce5_clean_qp_user(rdev, rqp, udata); + } else { + roce5_clean_qp_kernel(rdev, rqp); + + if (roce5_sqp_check(rqp) != 0 && rqp->sqp_attr != NULL) { + kfree(rqp->sqp_attr); + rqp->sqp_attr = NULL; + } + } + + ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, NULL, NULL, NULL, NULL); + ret = roce5_uld_qp_destroy((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to destroy uld qp, ret(%d).", ret); + return ret; + } + + mutex_lock(&rdev->qp_cnt.cur_qps_mutex); + rdev->qp_cnt.del_qp_cnt++; + mutex_unlock(&rdev->qp_cnt.cur_qps_mutex); + return 0; +} + +int roce5_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + int ret; + + if (ibqp == NULL) { + ROCE_ERR("Ibqp is null"); + return -EINVAL; + } + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + ret = roce5_destroy_qp_common(rdev, rqp, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Error occur when destroy qp! ret(%d).", + ret); + return ret; + } + +#if !defined(IB_DEVICE_OPS_HAVE_IB_QP_OBJ) + kfree(rqp); +#endif + return 0; +} + +int roce5_destroy_qp_ofa(struct ib_qp *ibqp) +{ + return roce5_destroy_qp(ibqp, NULL); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h new file mode 100644 index 000000000..646717c2a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_exp.h @@ -0,0 +1,80 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_EXP_H +#define ROCE_QP_EXP_H + +#include <linux/io-mapping.h> +#include <linux/list.h> + +#include <rdma/ib_verbs.h> + +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_db.h" +#include "roce_cq.h" +#include "roce_qp.h" + +#define EXP_OK (0) +#define EXP_PARAM_ERR (1) +#define EXP_FUNC_ERR (2) + +#define QPC_MIN_SIZE (512) +#define CONTX_SZ (sizeof(struct roce_qp_context) / QPC_MIN_SIZE) + +#define BUFFER_SZ (sizeof(struct roce_qp_context) / sizeof(u32)) + +#define TYPE_LEN (8) +#define DEV_NUM (32) +#define QPN_MAX (0xFFFFF) +#define QPN_MIN (0) +#define IB_QP_EXP_CMD (1 << 23) +#define ROCE_OPTPAR_EXP (1 << 16) +#define VA_OFFSET (8) + +struct roce5_device_list { + struct ib_device *ib_dev[DEV_NUM]; + int ib_dev_num; + struct mutex mutex; +}; + +struct ib_qp_info { + dma_addr_t sq_pg_base_addr[64]; + dma_addr_t hw_doorbell_addr; + dma_addr_t sw_doorbell_addr; + + u32 sq_depth; + u32 qpn; + u8 sq_pg_cnt; + u8 qp_cos; + u8 cntx_sz; + u8 sq_wqe_size; +}; + +struct ib_qp_attr_data { + u32 data_type; //plog:1 + u32 data_len; + u8 *data_buf; +}; + +struct ib_qp_attr_exp { + struct ib_qp_attr qp_attr; + struct ib_qp_attr_data qp_attr_data; +}; + +struct roce5_device_list *roce5_get_plog_device_info(void); + +int ib_get_qp_info(char *dev_name, int qpn, struct ib_qp_info *qp_info); +struct ib_qp *ib_get_qp(const char *dev_name, int qpn); +int ib_put_qp(struct ib_qp *ibqp); +int roce5_qp_modify_exp(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state, + u16 vlan_id); +int roce5_modify_extend_qp(struct ib_qp_attr *attr, int attr_mask, + struct ib_qp *ibqp); + +#endif //ROCE_QP_EXP_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c new file mode 100644 index 000000000..a548667ee --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.c @@ -0,0 +1,39 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "roce_qp.h" +#include "roce_infra.h" +#include "roce_obj_ext.h" +#ifdef ENABLE_HYPER_ROCE +#include "hyper_roce_qp.h" +#endif + +const struct roce5_ext_reg g_rqp_exts[] = { +#ifdef ENABLE_HYPER_ROCE + { + .ext_id = ROCE_EXT_HYPER_ROCE, + .data_size = sizeof(struct roce5_qp_hyper_extend), + .init_func = roce5_hyper_qp_init, + .cleanup_func = roce5_hyper_qp_cleanup, + }, +#endif +}; + +const u8 g_rqp_exts_num = sizeof(g_rqp_exts) / sizeof(struct roce5_ext_reg); + +int roce5_create_qp_ext_init(struct roce5_qp *rqp) +{ + int ret = roce5_obj_ext_init((void *)rqp, g_rqp_exts, g_rqp_exts_num, + &rqp->obj_ext); + if (ret != 0) { + ROCE_ERR("Failed to init qp extension, ret(%d).", ret); + return ret; + } + return 0; +} + +void roce5_destroy_qp_ext_deinit(struct roce5_qp *rqp) +{ + roce5_obj_ext_deinit((void *)rqp, g_rqp_exts, g_rqp_exts_num, + &rqp->obj_ext); +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h new file mode 100644 index 000000000..8b3e95565 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_ext.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_QP_EXT_H +#define ROCE_QP_EXT_H + +#include "roce_qp.h" +#include "roce_obj_ext.h" + +static inline void *roce5_qp_get_ext_buf(struct roce5_qp *rqp, + enum ROCE_EXT_ID ext_id) +{ + return roce5_obj_ext_get_buf(&rqp->obj_ext, ext_id); +} + +int roce5_create_qp_ext_init(struct roce5_qp *rqp); +void roce5_destroy_qp_ext_deinit(struct roce5_qp *rqp); + +#endif /* ROCE_QP_EXT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h new file mode 100644 index 000000000..c757ce2ba --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_inner.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef ROCE_QP_INNER_H +#define ROCE_QP_INNER_H + +#include <rdma/ib_verbs.h> +#include "roce.h" +#include "roce_qp.h" + +int roce5_destroy_qp_common(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_udata *udata); +void roce5_destroy_qp_user_umem_mtt(struct roce5_device *rdev, + struct roce5_qp *rqp); + +#endif /* ROCE_QP_INNER_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c new file mode 100644 index 000000000..5a6a4ec7b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_modify.c @@ -0,0 +1,2398 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> +#include <linux/sched.h> + +#include "comm_defs.h" +#include "roce_compat.h" +#include "hinic5_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_qp_exp.h" +#include "roce_qp_extension.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "securec.h" +#include "roce_ctx_format.h" +#include "roce_npu_cmd.h" +#include "roce_npu_cmd_qp_defs.h" +#include "roce_npu_cmd_ext_defs.h" +#include "roce_cmd.h" +#include "roce_uld_kernel_api.h" +#include "roce_uld_inner.h" +#include "roce_netdev.h" +#include "roce_os_compat.h" + +#ifdef ROCE_ANTI_ATTACK +#include "roce_anti_attack.h" +#endif + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#define ROCE_MTT_GPA_VALID 0x1 +#define ROCE_WQE_PREFETCH_MAXNUM 5 //5: 8(count word) WQEBB +#define ROCE_SQ_WQEBB_SIZE_SHIFT 4 +#define ROCE_SQ_WQEBB_SIZE_MASK 0x7 +#define ROCE_SQ_PD_MASK 0x3ffff +#define ROCE_SQ_PAGE_SIZE_MASK 0xf +#define ROCE_SQ_WQECNT_LTH 4 +#define ROCE_SQ_PREFETCH_MINNUM 2 +#define ROCE_SQ_PREFETCH_MAXNUM 4 +#define ROCE_SQ_SIZE_MASK 0x1f +#define ROCE_SQ_CQN_MASK 0xfffff +#define ROCE_RQ_WQECNT_LTH 0xe +#define ROCE_RQ_MTT_PREFETCH_MAXLEN0 3 +#define ROCE_RQ_MTT_PREFETCH_MAXWQE 7 +#define ROCE_RQ_WQE_PREFETCH_MAXNUM 7 +#define ROCE_SRQ_INIT_STATE 0xf +#define ROCE_MAX_PI_ON_CHIP_QPN 1024 + +enum { + BIT_LENGTH_IB_MTU_256 = 8, + BIT_LENGTH_IB_MTU_512, + BIT_LENGTH_IB_MTU_1024, + BIT_LENGTH_IB_MTU_2048, + BIT_LENGTH_IB_MTU_4096, + BIT_LENGTH_IB_MTU_8192, +}; + +enum { + CACHE_INVLAID_ALLOC, + CACHE_INVLAID_CMDQ, + CACHE_INVLAID_RDMARC_CHECK, + CACHE_INVLAID_QP_CHECK, + MODIFY_2RST_CMDQ, + CMDQ_SDKCALL_BUTT +}; + +#define ROCE_CMDQ_TIMEOUT 1000 +#define ROCE_CMDQ_TIMEOUT_10MS 10 + +#ifdef ROCE_SIGN_EN +enum mtt_check_type_e { MTT_CHECK_TYPE_0 = 0, MTT_CHECK_TYPE_1 }; + +#define ROCE_QP_QPCC_SIGN_WIDTH 5 +#define ROCE_QP_QPCC_SIGN_SPLITNUM 4 +#define ROCE_QP_QPCC_SIGN_CHECKBITS \ + (ROCE_QP_QPCC_SIGN_WIDTH * ROCE_QP_QPCC_SIGN_SPLITNUM) + +#define ROCE_QP_SUB_SIGN_WIDTH 3 +#define ROCE_QP_SUB_SIGN_SPLITNUM 5 +#define ROCE_QP_SUB_SIGN_CHECKBITS \ + (ROCE_QP_SUB_SIGN_WIDTH * ROCE_QP_SUB_SIGN_SPLITNUM) + +#define ROCE_QP_GPA_SIGN_WIDTH 3 +#define ROCE_QP_GPA_SIGN_SPLITNUM 11 +#define ROCE_QP_GPA_SIGN_CHECKBITS \ + (ROCE_QP_GPA_SIGN_WIDTH * ROCE_QP_GPA_SIGN_SPLITNUM) + +#define ROCE_SQ_WQE_SIGN_WIDTH 8 +#define ROCE_SQ_WQE_SIGN_SPLITNUM 9 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_1 32 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_2 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_12 \ + (ROCE_SQ_WQE_SIGN_CHECKBITS_2 + ROCE_SQ_WQE_SIGN_CHECKBITS_1) +#define ROCE_SQ_WQE_SIGN_CHECKBITS_3 20 +#define ROCE_SQ_WQE_SIGN_CHECKBITS_123 \ + (ROCE_SQ_WQE_SIGN_CHECKBITS_3 + ROCE_SQ_WQE_SIGN_CHECKBITS_12) +#define ROCE_SQ_WQE_SIGN_CHECKBITS \ + (ROCE_SQ_WQE_SIGN_WIDTH * ROCE_SQ_WQE_SIGN_SPLITNUM) + +#define ROCE_MTT_SIGN_WIDTH 11 +#define ROCE_MTT_SIGN_SPLITNUM 5 +#define ROCE_MTT_SIGN_CHECKBITS_1 22 +#define ROCE_MTT_SIGN_CHECKBITS_2 33 +#define ROCE_MTT_SIGN_CHECKBITS_12 \ + (ROCE_MTT_SIGN_CHECKBITS_1 + ROCE_MTT_SIGN_CHECKBITS_2) +#define ROCE_MTT_SIGN_CHECKBITS (ROCE_MTT_SIGN_WIDTH * ROCE_MTT_SIGN_SPLITNUM) + +#define ROCE_MTT_BLOCK_SIGN_WIDTH 5 +#define ROCE_MTT_BLOCK_SIGN_SPLITNUM 11 + +/* **************************************************************************** + Prototype : roce5_calculate_sign_bit + Description : roce5_calculate_signature + Input : u32 *check_bit + u32 sign_width + u32 sign_split + u32 *sign_bit + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static void roce5_calculate_sign_bit(u32 *check_bit, u32 sign_width, + u32 sign_split, u32 *sign_bit) +{ + u32 i = 0; + u32 j = 0; + + for (i = 0; i < sign_width; i++) { + sign_bit[i] = 0; + for (j = 0; j < sign_split; j++) { + sign_bit[i] = sign_bit[i] ^ + check_bit[i + sign_width * j]; + } + + sign_bit[i] = (~sign_bit[i]) & 0x1; + } +} + +/* **************************************************************************** + Prototype : roce5_calculate_signature + Description : roce5_calculate_signature + Input : u32 *sign_bit + u32 sign_width + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_calculate_signature(u32 *sign_bit, u32 sign_width) +{ + u32 i = 0; + u32 signature = 0; + + for (i = 0; i < sign_width; i++) { + signature |= sign_bit[i] << i; + } + + return (signature & ((1U << sign_width) - 1)); +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_rcc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_rcc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_sqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_sqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_sqac_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_sqac_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_rqc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_rqc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_qp_rrwc_sign + Description : Signature 3bit + CheckedBits = QPn[14:0] + Input : u32 qpn + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_qp_rrwc_sign(u32 qpn) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_QP_SUB_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_QP_SUB_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_QP_SUB_SIGN_CHECKBITS; i++) { + if (qpn & (1U << i)) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_QP_SUB_SIGN_WIDTH, + ROCE_QP_SUB_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_QP_SUB_SIGN_WIDTH); + + return signature; +} + +/* **************************************************************************** + Prototype : roce5_gen_sq_wqe_sign + Description : Signature 8bit + CheckedBits = {QPn[19:0], WQECI[19:0], WQE.Header[31:0]} + Input : u32 qpn + u32 wqe_ci + u32 wqe_header + Output : None + + 1.Date : 2017/4/1 + Modification : Created function + +**************************************************************************** */ +static u32 roce5_gen_sq_wqe_sign(u32 qpn, u32 wqe_ci, u32 wqe_header) +{ + u32 i; + u32 signature = 0; + u32 sign_bit[ROCE_SQ_WQE_SIGN_WIDTH] = { 0 }; + u32 check_bit[ROCE_SQ_WQE_SIGN_CHECKBITS] = { 0 }; + + for (i = 0; i < ROCE_SQ_WQE_SIGN_CHECKBITS_1; i++) { + if (wqe_header & (1U << i)) { + check_bit[i] = 1; + } + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_1; + i < ROCE_SQ_WQE_SIGN_CHECKBITS_12; i++) { + if (wqe_ci & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_1))) { + check_bit[i] = 1; + } + } + + for (i = ROCE_SQ_WQE_SIGN_CHECKBITS_12; + i < ROCE_SQ_WQE_SIGN_CHECKBITS_123; i++) { + if (qpn & (1U << (i - ROCE_SQ_WQE_SIGN_CHECKBITS_12))) { + check_bit[i] = 1; + } + } + + roce5_calculate_sign_bit(check_bit, ROCE_SQ_WQE_SIGN_WIDTH, + ROCE_SQ_WQE_SIGN_SPLITNUM, sign_bit); + + signature = roce5_calculate_signature(sign_bit, ROCE_SQ_WQE_SIGN_WIDTH); + + return signature; +} +#endif + +#define ROCE_BASE_GID_IDX \ + 1 /* current IP mode supports precise push GID, reserved this field compatible old version firmware */ +static void roce5_qp_set_path(const struct roce5_qp *rqp, + struct rdma_ah_attr *ah_attr, + roce_verbs_qpc_attr_path_s *path_info) +{ + u8 *dmac = ah_attr->roce.dmac; + int ret; + + path_info->dw7.bs.hoplmt = (u8)ah_attr->grh.hop_limit; + path_info->dw6.bs.tclass = + (u8)(ah_attr->grh.traffic_class | IP_ECN_PERMIT); + path_info->dw6.bs.flow_label = ah_attr->grh.flow_label & + 0xfffff; /* flow_label: 20bit */ + path_info->dw7.bs.sgid_index = ah_attr->grh.sgid_index & + 0x7f; /* sgid_index: 7bit */ + + path_info->dw7.bs.base_sgid_n = + (path_info->dw7.bs.sgid_index != ROCE_BASE_GID_IDX); + + ret = memcpy_s((void *)path_info->dgid, sizeof(path_info->dgid), + (void *)ah_attr->grh.dgid.raw, sizeof(path_info->dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + } + + ret = memcpy_s((void *)&path_info->dmac_l32, + sizeof(path_info->dmac_l32), + (void *)&dmac[ROCE_RAH_DMAC_L32_START], + sizeof(path_info->dmac_l32)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac_l32.(ret:%d)", ret); + } + path_info->dmac_l32 = cpu_to_be32(path_info->dmac_l32); + path_info->dw0.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | + dmac[1]; //shift dmac[0] left by 8 bits + path_info->dw7.bs.sl = (ah_attr->sl & 0x7); /* sl: 3bit */ + path_info->dw0.bs.udp_src_port_h = + ((rqp->qpn >> ROCE_UDP_SRC_PORT_H8_SHIFT) & 0xff); + path_info->dw7.bs.udp_src_port = (rqp->qpn & 0xff); +#ifdef ROCE_BONDING_EN + path_info->dw0.bs.bond_tx_hash_value = (rqp->tx_hash_value & 0xf); +#endif +} + +struct roce5_pd *roce5_get_pd(struct roce5_qp *rqp) +{ + if (rqp->qp_type == IB_QPT_XRC_TGT) { + return to_roce5_pd(to_roce5_xrcd(rqp->ibqp.xrcd)->pd); + } else { + return to_roce5_pd(rqp->ibqp.pd); + } +} + +void roce5_get_cqs(struct roce5_qp *rqp, struct roce5_cq **send_cq, + struct roce5_cq **recv_cq) +{ + switch (rqp->qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = to_roce5_cq(to_roce5_xrcd(rqp->ibqp.xrcd)->cq); + *recv_cq = *send_cq; + break; + + case IB_QPT_XRC_INI: + *send_cq = to_roce5_cq(rqp->ibqp.send_cq); + *recv_cq = *send_cq; + break; + + default: + *send_cq = to_roce5_cq(rqp->ibqp.send_cq); + *recv_cq = to_roce5_cq(rqp->ibqp.recv_cq); + break; + } +} + +/* **************************************************************************** + Prototype : roce5_set_access_flags + Description : roce5_set_access_flags + Input : struct roce5_qp *rqp + roce_verbs_qp_attr_s *qp_attr + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + 2.Date : 2016/2/26 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_access_flags(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr, + struct ib_qp_attr *attr, int attr_mask) +{ + u16 dest_rd_atomic = 0; + u32 access_flags = 0; + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + dest_rd_atomic = attr->max_dest_rd_atomic; + } else { + dest_rd_atomic = rqp->rsp_depth; + } + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) { + access_flags = (u32)attr->qp_access_flags; + } else { + access_flags = rqp->atomic_rd_en; + } + + if (dest_rd_atomic == 0) { + access_flags &= IB_ACCESS_REMOTE_WRITE; + } + + if ((access_flags & IB_ACCESS_REMOTE_READ) != 0) { + qp_attr->com_info.dw0.bs.rre = 1; /* rc_rre: 1bits */ + } + + if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) != 0) { + qp_attr->com_info.dw0.bs.rae = 1; /* rc_rae: 1bits */ + } + + if ((access_flags & IB_ACCESS_REMOTE_WRITE) != 0) { + qp_attr->com_info.dw0.bs.rwe = 1; /* rrw_rwe: 1bits */ + } +} + +bool roce5_sqp_check(const struct roce5_qp *rqp) +{ + if (((rqp->qp_type == IB_QPT_GSI) && + (rqp->qpn == ROCE_QP_SMI_QP_NUM)) || + (rqp->qpn == ROCE_QP_GSI_QP_NUM)) { + return true; + } + + return false; +} + +/* **************************************************************************** + Prototype : store_sqp_attrs + Description : save the special + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + int attr_mask + Output : int status + + 1.Date : 2015/7/24 + Modification : Created function + +**************************************************************************** */ +static int store_sqp_attrs(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + int attr_mask) +{ + if (rqp->sqp_attr == NULL) { + ROCE_ERR("Error! sqp attribute is NULL, qpn(%u).", rqp->qpn); + return -EINVAL; + } + + if (((u32)attr_mask & IB_QP_PKEY_INDEX) != 0) { + rqp->sqp_attr->pkey_index = attr->pkey_index; + } + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) { + rqp->sqp_attr->qkey = attr->qkey; + } + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) { + rqp->sqp_attr->send_psn = attr->sq_psn; + } + + return 0; +} + +/* **************************************************************************** + Prototype : qpc_seg_to_be + Input : struct roce_qp_context *be_ctx + struct roce_qp_context *le_ctx + struct roce5_qp *rqp + Output : None + + 1.Date : 2015/7/29 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +static void qpc_seg_to_be(roce_verbs_qp_attr_s *be_ctx, + roce_verbs_qp_attr_s *le_ctx, struct roce5_qp *rqp) +{ + /* DRV Seg */ + int ret; + + ret = memcpy_s((void *)be_ctx->path_info.dgid, + sizeof(le_ctx->path_info.dgid), + (void *)le_ctx->path_info.dgid, + sizeof(le_ctx->path_info.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + } + + /* CHIP Seg */ + be_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa = + cpu_to_be64(le_ctx->chip_seg.dw0.sq_rq_l0mtt_gpa); + + be_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = cpu_to_be64( + le_ctx->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num); +} + +/* **************************************************************************** + Prototype : roce5_qpc_to_be + Description : translate big endian + Input : roce_verbs_qp_attr_s *qp_attr + struct roce5_qp *rqp + u32 *be_ctx + Output : None + + 1.Date : 2015/7/29 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +void roce5_qpc_to_be(roce_verbs_qp_attr_s *qp_attr, struct roce5_qp *rqp, + u32 *be_ctx) +{ + u32 *ctx = NULL; + u32 *ctx1 = NULL; + u32 i = 0; + u32 ctx_size = 0; + + ctx = be_ctx; + ctx1 = (u32 *)qp_attr; + ctx_size = sizeof(roce_verbs_qp_attr_s) / sizeof(u32); + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) { + *ctx = cpu_to_be32(*ctx1); + } + + qpc_seg_to_be((roce_verbs_qp_attr_s *)((void *)be_ctx), qp_attr, rqp); +} + +/* **************************************************************************** + Prototype : roce5_alloc_opt_rdmarc + Description : alloc rdmarc when modify qp + Input : struct roce5_qp *rqp + struct ib_qp_attr *attr + int attr_mask + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static int roce5_alloc_opt_rdmarc(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + u32 response_depth = 0; + + rdev = to_roce5_dev(rqp->ibqp.device); + response_depth = (attr->max_dest_rd_atomic == 0) ? + 1 : + (u32)ROCE_ROUNDUP_POW_OF_TWO( + (u32)attr->max_dest_rd_atomic); + if (rqp->rsp_depth != response_depth) { + if (rqp->rsp_depth > 0) { + roce5_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + } + + ret = roce5_rdma_rdmarc_alloc(rdev->hwdev, response_depth, + &rqp->rdmarc); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc rdma rdmarc, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + rqp->rsp_depth = (u16)response_depth; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_free_opt_rdmarc + Description : free rdmarc when modify qp failed or destory qp + Input : struct roce5_qp *rqp + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +void roce5_free_opt_rdmarc(struct roce5_qp *rqp) +{ + struct roce5_device *rdev = NULL; + + rdev = to_roce5_dev(rqp->ibqp.device); + + if (rqp->rsp_depth > 0) { + roce5_rdma_rdmarc_free(rdev->hwdev, &rqp->rdmarc); + } + + rqp->rsp_depth = 0; +} + +/* **************************************************************************** + Prototype : roce5_set_opt_rdmarc + Description : set opt rdmarc field + Input : struct roce5_device *rdev + struct roce5_qp *rqp + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** */ +void roce5_set_opt_rdmarc(struct rdma_rdmarc *rdmarc, u16 depth, + roce_verbs_qp_attr_s *qp_attr) +{ + qp_attr->chip_seg.dw10.bs.rc_page_gpa_h = + (rdmarc->dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & + 0xffffff; /* high:24bit, 64-24= 40 */ + qp_attr->chip_seg.rc_page_gpa_l = + (u32)(rdmarc->dma_addr >> + ROCE5_RDMARC_GPA_L_SHIFT); //bit[39:8] + qp_attr->chip_seg.dw4.bs.rc_size = rdmarc->order; + qp_attr->com_info.dw1.bs.rra_max = + ((u8)fls(depth - 1)) & + ROCE5_RDMARC_RRA_MAX_MASK; /* rra_max:3bit */ + qp_attr->chip_seg.dw4.bs.rc_max_size = + rdmarc->ext_order - ROCE5_RDMARC_EXT_ORDER_SHIFT; +} + +static int roce5_set_opt_mtu(struct roce5_device *rdev, + roce_verbs_qp_attr_s *qp_attr, + const struct ib_qp_attr *attr) +{ + enum roce5_mtu rmtu = (enum roce5_mtu)attr->path_mtu; + if (rmtu == ROCE_MTU_8192 && ROCE5_SUPPORT_8K_MTU(rdev) == 0) { + ROCE_DEV_ERR(rdev, "Not support 8K MTU."); + return -EINVAL; + } + + qp_attr->com_info.dw3.bs.pmtu = (u32)rmtu & 0x7; /* 1,2,3,4,5 */ + qp_attr->com_info.dw5.bs.mtu_code = + ROCE_QPC_PMTU_TRANSLATE(rmtu); /* 0,1,3,7,15 */ + if (qp_attr->com_info.dw3.bs.pmtu == IB_MTU_1024) { + qp_attr->com_info.dw3.bs.base_mtu_n = 0; + } else { + qp_attr->com_info.dw3.bs.base_mtu_n = 1; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_set_opt_sra + Description : set opt sra + Input : roce_verbs_qp_attr_s *qp_attr + struct ib_qp_attr *attr + Output : None + + 1.Date : 2017/04/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_opt_sra(roce_verbs_qp_attr_s *qp_attr, + struct ib_qp_attr *attr) +{ + u8 initiator_depth; + + initiator_depth = + (attr->max_rd_atomic == 0) ? + 1 : + (u8)ROCE_ROUNDUP_POW_OF_TWO((u32)attr->max_rd_atomic); + qp_attr->com_info.dw1.bs.sra_max = + ((u32)ROCE_FLS(initiator_depth - 1)) & 0x7; +} + +static void roce5_set_opt_attr_av(struct roce5_qp *rqp, + struct roce5_device *rdev, + struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, u32 *optpar) +{ + u8 cos; + roce5_get_cos_inbuf_s inbuf = { 0 }; + + roce5_qp_set_path(rqp, &attr->ah_attr, &qp_attr->path_info); + qp_attr->com_info.dw3.bs.ext_mtu = rqp->ext_mtu & 0x1; + qp_attr->com_info.dw3.bs.ext_md = rqp->ext_mtu_mode & 0x1; + *optpar |= ROCE_QP_OPTPAR_PRIMARY_ADDR_PATH; + + inbuf.sl = attr->ah_attr.sl; + inbuf.sgid_index = attr->ah_attr.grh.sgid_index; + inbuf.port_num = attr->ah_attr.port_num; + inbuf.traffic_class = attr->ah_attr.grh.traffic_class; + + (void)roce5_get_dcb_cfg_cos(rdev, &inbuf, &cos, rqp->qp_type); + + qp_attr->com_info.dw5.bs.cos = cos; + + rqp->db_sgid_index = attr->ah_attr.grh.sgid_index & 0x7f; + rqp->db_cos = cos; +} + +static void roce5_set_opt_field_attr(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + int attr_mask, + roce_verbs_qp_attr_s *qp_attr, u32 *optpar) +{ + /* RTR2RTS/SQD2SQD RETRY_CNT */ + if ((((u32)attr_mask) & IB_QP_RETRY_CNT) != 0) { + qp_attr->com_info.dw1.bs.to_retry_limit = + attr->retry_cnt & 0x7; /* to_retry_limit:3 */ + *optpar |= ROCE_QP_OPTPAR_RETRY_COUNT; + } + + /* RTR2RTS/SQD2SQD RNR_RETRY */ + if ((((u32)attr_mask) & IB_QP_RNR_RETRY) != 0) { + qp_attr->com_info.dw1.bs.rnr_retry_limit = + attr->rnr_retry & 0x7; /* rnr_retry_limit:3bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_RETRY; + } + + /* INIT2RTR/SQD2SQD/RTR2RTS/RTS2RTS MIN_RNR_TIMER */ + if ((((u32)attr_mask) & IB_QP_MIN_RNR_TIMER) != 0) { + qp_attr->com_info.dw2.bs.min_rnr_nak = + attr->min_rnr_timer & 0x1f; /* min_rnr_nak:5bits */ + *optpar |= ROCE_QP_OPTPAR_RNR_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_SQ_PSN) != 0) { + qp_attr->com_info.dw8.bs.next_send_psn = + attr->sq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_RQ_PSN) != 0) { + qp_attr->com_info.dw9.bs.next_rcv_psn = + attr->rq_psn & 0xffffff; /* next_rcv_psn:24bit */ + } + + if ((((u32)attr_mask) & IB_QP_DEST_QPN) != 0) { + qp_attr->com_info.dw0.bs.dest_qp = attr->dest_qp_num & + 0xffffff; /* dest_qp:24bit */ + } + + roce5_set_attr_ext(rqp, attr, qp_attr, attr_mask); +} + +static s32 roce5_set_db_path_mtu(const struct ib_qp_attr *attr, + struct roce5_qp *rqp, + struct roce5_device *rdev) +{ + switch ((enum roce5_mtu)attr->path_mtu) { + case ROCE_MTU_256: + case ROCE_MTU_512: + ROCE_DEV_ERR( + rdev, + "Only support mtu larger than 1K.func_id(%d) mtu:%u", + rdev->glb_func_id, attr->path_mtu); + return -EINVAL; + case ROCE_MTU_1024: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_1024; + break; + case ROCE_MTU_2048: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_2048; + break; + case ROCE_MTU_4096: + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_4096; + break; + case ROCE_MTU_8192: + if (ROCE5_SUPPORT_8K_MTU(rdev) != 0) { + rqp->db_path_mtu = BIT_LENGTH_IB_MTU_8192; + break; + } + + ROCE_DEV_ERR(rdev, "Not support 8K MTU."); + return -EINVAL; + default: + return -EINVAL; + } + + return 0; +} + +static int roce5_set_opt_field(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, + roce_modify_qp_param_s *modify_para) +{ + int ret = 0; + roce_verbs_qp_attr_s *qp_attr = &modify_para->qp_attr; + u32 *optpar = &modify_para->optpar; + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + + if ((((u32)attr_mask) & IB_QP_QKEY) != 0) { + qp_attr->com_info.q_key = attr->qkey; + *optpar |= ROCE_QP_OPTPAR_Q_KEY; + } + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + roce5_set_opt_attr_av(rqp, rdev, attr, qp_attr, optpar); + } + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) { + qp_attr->com_info.dw5.bs.port = attr->port_num & + 0x7; /* port: 3bits */ + *optpar |= ROCE_QP_OPTPAR_SCHED_QUEUE; + } + + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + roce5_set_opt_rdmarc(&rqp->rdmarc, rqp->rsp_depth, qp_attr); + *optpar |= ROCE_QP_OPTPAR_RRA_MAX; + } + + /* RTR2RTS/SQD2SQD SRA_MAX */ + if ((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) { + roce5_set_opt_sra(qp_attr, attr); + *optpar |= ROCE_QP_OPTPAR_SRA_MAX; + } + + /* RWE/RRE/RAE,RST2INIT */ + if ((((u32)attr_mask) & + (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) != 0) { + roce5_set_access_flags(rqp, qp_attr, attr, attr_mask); + *optpar |= ROCE_QP_OPTPAR_RWE | ROCE_QP_OPTPAR_RRE | + ROCE_QP_OPTPAR_RAE; + } + + /* RTR2RTS/SQD2SQD ACK TIMEOUT */ + if ((((u32)attr_mask) & IB_QP_TIMEOUT) != 0) { + qp_attr->com_info.dw2.bs.ack_to = attr->timeout & + 0x1f; /* ack_to:5bits */ + *optpar |= ROCE_QP_OPTPAR_ACK_TIMEOUT; + } + + if ((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) { + ret = roce5_set_db_path_mtu(attr, rqp, rdev); + if (ret != 0) { + return ret; + } + ret = roce5_set_opt_mtu(rdev, qp_attr, attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set MTU."); + return ret; + } + } + + roce5_set_opt_field_attr(rqp, attr, attr_mask, qp_attr, optpar); + return ret; +} + +static void roce5_set_qp_srq_attr(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr, + struct roce5_pd *pd, + const struct roce5_cq *recv_cq) +{ + struct roce5_srq *rsrq = to_roce5_srq(rqp->ibqp.srq); + struct roce5_srq_context temp; + + if (rsrq->container_flag != 0) { + struct roce5_srq_context *srqc = + (struct roce5_srq_context *)((void *)rsrq->cqm_srq + ->q_ctx_vaddr); + temp.dw0.value = be32_to_cpu(srqc->dw0.value); + qp_attr->chip_seg.dw12.bs.srq_pd = temp.dw0.bs.pdn & + 0x3ffff; /* srq_pd:18bits */ + qp_attr->chip_seg.dw12.bs.srq_wqebb_size = + temp.dw0.bs.wqebb_size; + qp_attr->chip_seg.dw12.bs.srq_page_size = temp.dw0.bs.page_size; + qp_attr->chip_seg.dw12.bs.srq_size = temp.dw0.bs.size; + + temp.dw1.value = be32_to_cpu(srqc->dw1.value); + qp_attr->com_info.dw5.bs.so_ro = temp.dw1.bs_c.so_ro; + qp_attr->com_info.dw5.bs.dma_attr_idx = + temp.dw1.bs_c.dma_attr_idx; + + qp_attr->chip_seg.dw9.bs.container_sz = + MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode; + qp_attr->chip_seg.dw9.bs.container_en = 1; + + qp_attr->com_info.dw3.bs.srq_container = 1; + + if (rsrq->rqe_cnt_th > 0) { + qp_attr->chip_seg.dw9.bs.srq_warth_flag = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = + rsrq->rqe_cnt_th & + 0xf; /* srq_rqecnt_th:4bits */ + } + + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & + 0x3ffff; /* srqn:18bits */ + } else { + qp_attr->chip_seg.dw7.bs.srqn = rsrq->srqn & + 0x3ffff; /* srqn:18bits */ + } + + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw0.bs.rkey_en = 1; + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & + 0xfffff; /* rq_cqn:20bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; +} + +static void roce5_set_qp_rst2init_attr(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr, + struct roce5_pd *pd, + struct roce5_cq *recv_cq) +{ + qp_attr->chip_seg.dw4.bs.rq_base_ci = + (u32)(ROCE_ILOG2(rqp->rq.offset >> rqp->rq.wqe_shift)); + qp_attr->chip_seg.dw19.bs.qp_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + qp_attr->com_info.dw3.bs.dsgl_en = rqp->double_sgl_mode & 0x1; + + if (rqp->ibqp.srq != NULL) { + roce5_set_qp_srq_attr(rqp, qp_attr, pd, recv_cq); + } else { + qp_attr->chip_seg.dw15.bs.rq_size = + (rqp->rq.wqebb_cnt > 0) ? + (((u32)ROCE_ILOG2(rqp->rq.wqebb_cnt)) & 0x1f) : + 0; + qp_attr->chip_seg.dw15.bs.rq_page_size = + ((u32)((u32)rqp->buf.page_shift - PAGE_SHIFT_4K)) & + ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw15.bs.rq_pd = pd->pdn & + 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw8.bs.rq_wqebb_size = + (u32)(rqp->rq.wqe_shift - BYTES_TO_16B_SHIFT) & 0x7; + qp_attr->com_info.dw7.bs.rq_cqn = recv_cq->cqn & + 0xfffff; /* rq_cqn:20bits */ + qp_attr->chip_seg.dw15.bs.rq_rkey_en = 1; + + qp_attr->chip_seg.dw8.bs.rq_pi_on_chip = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_lth = ROCE_RQ_WQECNT_LTH; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw7.bs.rq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen0 = + ROCE_RQ_MTT_PREFETCH_MAXLEN0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxlen1 = 0; + qp_attr->chip_seg.dw9.bs.rq_mtt_prefetch_maxwqe = + ROCE_RQ_MTT_PREFETCH_MAXWQE; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_minnum = 0; + qp_attr->chip_seg.dw7.bs.rq_wqe_prefetch_maxnum = + ROCE_RQ_WQE_PREFETCH_MAXNUM; + } +} + +static void roce5_qp_rst2init_set_qpcc(roce_verbs_qp_attr_s *qp_attr, + struct roce5_qp *rqp, + const struct roce5_device *rdev) +{ + qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = rqp->mtt.mtt_paddr; + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->db.dma & (~0x3ULL); /* bits:63-2 */ + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num |= + rqp->mtt.mtt_layers & 0x3; /* bits:1-0 */ + if (rqp->qp_type == IB_QPT_XRC_TGT) { + qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = + rqp->wb_dma; + } + + qp_attr->chip_seg.dw4.bs.sq_rq_mtt_page_size = + rqp->mtt.mtt_page_shift - PAGE_SHIFT_4K; + /* rc_entry_size:2bits */ + qp_attr->chip_seg.dw10.bs.rc_entry_size = + ((u32)ROCE_ILOG2(rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .rdmarc_entry_sz >> + BYTES_TO_16B_SHIFT) & + 0x3); + qp_attr->chip_seg.dw14.bs.qp_rkey_en = 1; + qp_attr->chip_seg.dw10.bs.rc_entry_prefetch_maxnum = 0; +} + +static void roce5_qp_rst2init_set_sqc_sqac(roce_verbs_qp_attr_s *qp_attr, + const struct roce5_qp *rqp, + const struct roce5_pd *pd, + const struct roce5_cq *send_cq) +{ + qp_attr->chip_seg.dw5.bs.sq_pi_on_chip = rqp->pi_on_chip; + qp_attr->chip_seg.dw16.bs.sq_wqebb_size = + ((u32)(rqp->sq.wqe_shift - ROCE_SQ_WQEBB_SIZE_SHIFT)) & + ROCE_SQ_WQEBB_SIZE_MASK; + qp_attr->chip_seg.dw16.bs.sq_pd = ((u32)pd->pdn) & + 0x3ffff; /* pd:18bits */ + qp_attr->chip_seg.dw16.bs.sq_page_size = + ((u32)(rqp->buf.page_shift - PAGE_SHIFT_4K)) & + ROCE_SQ_PAGE_SIZE_MASK; + qp_attr->chip_seg.dw5.bs.sq_inline_en = (rqp->max_inline_data > 0) ? 1 : + 0; + + qp_attr->chip_seg.dw16.bs.sq_rkey_en = 1; + + qp_attr->chip_seg.dw5.bs.sq_wqecnt_lth = ROCE_SQ_WQECNT_LTH; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl_en = 0; + qp_attr->chip_seg.dw5.bs.sq_wqecnt_rctl = 0; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_minnum = + ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_prefetch_maxnum = + ROCE_SQ_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw5.bs.sq_wqe_cache_thd_sel = 0; + qp_attr->chip_seg.dw6.bs.sq_wqe_prefetch_mode = 0; + qp_attr->chip_seg.dw6.bs.sq_mtt_prefetch_maxlen = 0; + + if (rqp->sq.wqebb_cnt != 0) { + qp_attr->chip_seg.dw5.bs.sq_size = + (u32)(ROCE_ILOG2((u32)rqp->sq.wqebb_cnt)) & + ROCE_SQ_SIZE_MASK; + } + + qp_attr->chip_seg.dw17.bs.sqa_cqn = send_cq->cqn & ROCE_SQ_CQN_MASK; + + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_minnum = + ROCE_SQ_PREFETCH_MINNUM; + qp_attr->chip_seg.dw5.bs.sqa_wqe_prefetch_maxnum = + ROCE_WQE_PREFETCH_MAXNUM; + qp_attr->chip_seg.dw6.bs.sqa_mtt_prefetch_maxlen = 0; + qp_attr->chip_seg.dw5.bs.sqa_wqe_cache_thd_sel = 0; +} + +static void roce5_qp_rst2init_set_drv(roce_verbs_qp_attr_s *qp_attr, + struct roce5_qp *rqp, + struct roce5_device *rdev) +{ + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce5_qp_type((enum ib_qp_type)rqp->qp_type) & 0x7; + qp_attr->com_info.dw3.bs.dif_en = (u32)rqp->signature_en; + + if (!rqp->ibqp.uobject) { + qp_attr->com_info.dw0.bs.fre = 1; + } + + qp_attr->com_info.dw3.bs.xrc_vld = + (u32)(rqp->qp_type == IB_QPT_XRC_INI); + qp_attr->com_info.dw1.bs.local_qp = rqp->qpn & + 0xffffff; /* local_qp:24bits */ + qp_attr->com_info.dw5.bs.ep = rdev->hw_info.ep_id; + qp_attr->chip_seg.dw4.bs.dsgl = rqp->double_sgl_mode & 0x1; +} + +static void roce5_qp_rst2init_set_common(roce_verbs_qp_attr_s *qp_attr, + const struct roce5_pd *pd, + const struct roce5_qp *rqp, + const struct roce5_cq *recv_cq, + const struct roce5_cq *send_cq) +{ + qp_attr->chip_seg.dw18.bs.ud_pd = pd->pdn & 0x3ffff; /* pd:18bits */ + + if (rqp->shadow != 0) { + qp_attr->com_info.dw11.bs.shadow = 1; + if (recv_cq != NULL) { + qp_attr->com_info.dw11.bs.shadow_vfid = + recv_cq->shadow_vfid; + } + + if (send_cq != NULL) { + qp_attr->com_info.dw11.bs.shadow_cq_vfid = + send_cq->shadow_vfid; + } + } +} + +static void roce5_qp_rst2init_set_xrc_attr(const struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, + const struct roce5_device *rdev, + const struct roce5_cq *recv_cq) +{ + qp_attr->com_info.dw2.bs.xrc_srq_en = 1; + + if ((((u32)attr->qp_access_flags & XRC_CONTAINER_FLAG) != 0) || + (rdev->cfg_info.srq_container_en != 0)) { + qp_attr->chip_seg.dw9.bs.container_en = 1; + qp_attr->com_info.dw3.bs.srq_container = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & + 0xffff; /* xrcd:16bits */ + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.srq_rqecnt_th = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw13.bs.srq_state = ROCE_SRQ_INIT_STATE; + } else { + qp_attr->chip_seg.dw12.bs.srq_rkey_en = 1; + qp_attr->com_info.dw2.bs.srq_en = 1; + qp_attr->chip_seg.dw8.bs.xrcd = rqp->xrcdn & + 0xffff; /* xrcd:16bits */ + qp_attr->chip_seg.dw9.bs.srq_mtt_prefetch_maxlen1 = 1; + qp_attr->com_info.dw3.bs.xrc_vld = 1; + qp_attr->chip_seg.dw14.bs.srq_xrcd = rqp->xrcdn & + 0xffff; /* xrcd:16bits */ + } + if (recv_cq != NULL) { + qp_attr->chip_seg.dw13.bs.srq_cqn = recv_cq->cqn & + 0xfffff; /* rq_cqn:20bits */ + } +} + +/* **************************************************************************** + Prototype : roce5_qp_rst2init + Description : parameter set when QP is from RESET to INIT + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2015/8/6 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +void roce5_qp_rst2init(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ + struct roce5_pd *pd = NULL; + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + struct roce5_device *rdev = NULL; + + rdev = to_roce5_dev(rqp->ibqp.device); + pd = roce5_get_pd(rqp); + roce5_get_cqs(rqp, &send_cq, &recv_cq); + + /* CHIP_SEG: QPCC */ + roce5_qp_rst2init_set_qpcc(qp_attr, rqp, rdev); + + /* CHIP_SEG: SQC & SQAC */ + roce5_qp_rst2init_set_sqc_sqac(qp_attr, rqp, pd, send_cq); + + /* RRWC */ + qp_attr->chip_seg.dw4.bs.rrw_mtt_prefetch_maxlen = 0; + + /* RCC */ + qp_attr->chip_seg.dw10.bs.rc_mtt_prefetch_maxlen = + ROCE_QP_MAX_PFETCH_MTT_LAYER; + + /* TIMER_SEG */ + qp_attr->com_info.dw3.bs.tss_timer_num = ROCE_QP_MAX_TIMER_NUM; + + /* DRV_SEG */ + roce5_qp_rst2init_set_drv(qp_attr, rqp, rdev); + + /* UCODE COM_SEG */ + roce5_qp_rst2init_set_common(qp_attr, pd, rqp, recv_cq, send_cq); + + if (rqp->qp_type == IB_QPT_XRC_TGT) { + roce5_qp_rst2init_set_xrc_attr(rqp, attr, qp_attr, rdev, + recv_cq); + } else { + roce5_set_qp_rst2init_attr(rqp, qp_attr, pd, recv_cq); + } + +#ifdef ROCE_BONDING_EN + roce5_bond_rr_set_flow(rdev, rqp, qp_attr); +#endif +} + +/* **************************************************************************** + Prototype : roce5_qp_init2rtr + Description : init2rtr attributes modify + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2015/8/1 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +static void roce5_qp_init2rtr(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ + if (rqp->ibqp.srq || (rqp->qp_type == IB_QPT_XRC_TGT) || + (rqp->qp_type == IB_QPT_XRC_INI) || (rqp->qp_type == IB_QPT_UD) || + (rqp->qp_type == IB_QPT_UC) || (rqp->qp_type == IB_QPT_GSI)) { + qp_attr->com_info.dw3.bs.invalid_credit = 1; + } + + if ((rqp->qp_type == IB_QPT_GSI) || (rqp->qp_type == IB_QPT_UD)) { + qp_attr->com_info.dw3.bs.pmtu = IB_MTU_4096 & + 0x7; /* for UD, mtu = 4096 */ + qp_attr->com_info.dw5.bs.mtu_code = ROCE_MTU_CODE_4K; + } + + qp_attr->com_info.dw2.bs.ack_to = 0xf; /* ack_to:5bits */ + qp_attr->com_info.dw0.bs.service_type = + (u32)to_roce5_qp_type((enum ib_qp_type)rqp->qp_type) & + 0x7; /* service_type: 3bit */ +} + +/* **************************************************************************** + Prototype : roce5_qp_rtr2rts + Description : roce5_qp_rtr2rts + Input : struct roce5_qp *rqp + const struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + Output : None + + 1.Date : 2015/8/2 + Author : zhaoxinteng + Modification : Created function + +**************************************************************************** */ +static void roce5_qp_rtr2rts(struct roce5_qp *rqp, + const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ +} + +static void roce5_qp_2err(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr) +{ +} + +/* **************************************************************************** + Prototype : roce5_set_abs_field + Description : set abs field to qpc + Input : struct roce5_qp *rqp + struct ib_qp_attr *attr + roce_verbs_qp_attr_s *qp_attr + enum ib_qp_state cur_state + enum ib_qp_state new_state + Output : None + + 1.Date : 2016/01/30 + Modification : Created function + +**************************************************************************** */ +static void roce5_set_abs_field(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, roce_verbs_qp_attr_s *qp_attr) +{ + enum ib_qp_state cur_state, new_state; + + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) { + roce5_qp_rst2init(rqp, attr, qp_attr); + } + + if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR)) { + roce5_qp_init2rtr(rqp, attr, qp_attr); + } + + if ((cur_state == IB_QPS_RTR) && (new_state == IB_QPS_RTS)) { + roce5_qp_rtr2rts(rqp, attr, qp_attr); + } + + if (new_state == IB_QPS_ERR) { + roce5_qp_2err(rqp, attr, qp_attr); + } + return; +} + +#ifdef ROCE_LATENCY_PROF_EN +#define ROCE_SET_LATENCY_PROF_EN 5 +static int roce5_latency_prof_en_check(struct roce5_device *rdev, + u32 *latency_prof_en, + u32 *frequency_reduction_ratio) +{ + roce_dfx_set_inbuf_s buf_in = { 0 }; + roce_dfx_set_outbuf_s buf_out = { 0 }; + u32 out_size = sizeof(buf_out); + int ret = 0; + buf_in.show = 1; + buf_in.cmd_type = ROCE_CMD_ROCE_SET; + buf_in.sub_cmd_type = ROCE_SET_LATENCY_PROF_EN; + ret = roce5_adm_dfx_roce_set(rdev, &buf_in, sizeof(buf_in), &buf_out, + &out_size); + if (ret != 0) { + ROCE_ERR("latency prof query l2d err"); + return ret; + } + *latency_prof_en = buf_out.sub_param.latency_prof.latency_prof_en; + *frequency_reduction_ratio = + buf_out.sub_param.latency_prof.frequency_reduction_ratio; + return 0; +} + +static int +roce5_latency_prof_time_get(struct roce5_device *rdev, + struct roce5_modify_qp_ex_resp_data *resp) +{ + struct hinic5_hwdev *hwdev = (struct hinic5_hwdev *)rdev->hwdev; + u32 latency_prof_en = 0; + u32 frequency_reduction_ratio = 1; + u64 time_diff = 0; + int ret = 0; + if (hwdev == NULL) { + ROCE_ERR("latency prof hwdev is null"); + return (-EINVAL); + } + ret = roce5_latency_prof_en_check(rdev, &latency_prof_en, + &frequency_reduction_ratio); + if (ret != 0) { + return ret; + } + if (latency_prof_en == 0) { + resp->latency_prof_en = 0; + resp->non_ptp_time_diff = 0; + return 0; + } + //get clock time and non_ptp_chip time of difference value, convenient at later calculate timestamp + ret = hinic5_get_non_ptp_time_diff(hwdev, &time_diff); + if (ret != 0) { + ROCE_ERR("Failed to read n_ptp time, ret(%d)\n", ret); + return ret; + } + resp->latency_prof_en = 1; + resp->non_ptp_time_diff = time_diff; + resp->frequency_reduction_ratio = frequency_reduction_ratio; + return 0; +} +#endif +static int roce5_record_opt_field(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata, + roce_verbs_cmd_qp_modify_rsp_s *modify_rsp) +{ + int ret = 0; + int rsp_vld = 0; + enum ib_qp_state cur_state, new_state; + struct roce5_modify_qp_ex_resp_data resp = { 0 }; + struct roce5_device *rdev = NULL; + + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + rqp->qp_state = new_state; + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + rqp->sl = attr->ah_attr.sl & 0x7; + } + + if ((((u32)attr_mask) & IB_QP_ACCESS_FLAGS) != 0) { + rqp->atomic_rd_en = (u8)attr->qp_access_flags; + } + + if ((((u32)attr_mask) & IB_QP_PORT) != 0) { + rqp->port = attr->port_num; + } + + if (roce5_sqp_check(rqp) != 0) { + ret = store_sqp_attrs(rqp, attr, attr_mask); + if (ret != 0) { + ROCE_ERR("Failed to store sqp attribute."); + return ret; + } + } + + rdev = to_roce5_dev(rqp->ibqp.device); + + /* + * TGT QP no has update pi requirement, and db trigger microcode thread and driver no has strict timing disable system, one general in init status trigger microcode, + * but if XRC TGT QP migration 至 rts after again trigger microcode thread, SQ hardware none permission, fetch wqe will exception, because this TGT QP not next db + * but non- TGT QP i.e. make in qp rst status trigger, will go via wqe empty 终结, not will exception + */ + if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT) && + (rqp->pi_on_chip != 0) && (rqp->qp_type != IB_QPT_XRC_TGT)) { + /* avoid PI on chip when PI 残keep, primary dynamic clear to zero */ + ret = roce5_write_sq_db(rdev, rqp, 0, udata); + if (ret != 0) { + ROCE_ERR("Failed to clear PI on chip"); + return -1; + } + } + + if ((udata != NULL) && (udata->outbuf != NULL)) { + if ((cur_state == IB_QPS_INIT) && (new_state == IB_QPS_RTR) && + (((u32)attr_mask & IB_QP_AV) != 0)) { + resp.cos = rqp->db_cos; + resp.is_cos_valid = 1; + rsp_vld = 1; + } + +#ifdef ROCE_LATENCY_PROF_EN + if (ROCE_IS_DEV_1825V100(rdev) && (cur_state == IB_QPS_RTR) && + (new_state == IB_QPS_RTS)) { + ret = roce5_latency_prof_time_get(rdev, &resp); + if (ret != 0) { + ROCE_ERR("Failed to get time diff, ret(%d).", + ret); + return -1; + } + } +#endif + /* compatible old card not will return ssn value, driver not perform update */ + if (modify_rsp != NULL && + modify_rsp->magic == ROCE_VERBS_MODIFY_RSP_MAGIC) { + resp.is_ssn_valid = modify_rsp->dw1.bs.is_ssn_vld; + resp.ssn = modify_rsp->dw1.bs.ssn; + rqp->sq_wqe_ssn = resp.ssn; + rsp_vld = 1; + } + + if (rsp_vld != 0) { + ret = ib_copy_to_udata( + udata, &resp, + sizeof(struct roce5_modify_qp_ex_resp_data)); + if (ret != 0) { + ROCE_ERR("Failed to copy data to user"); + return -1; + } + } + } + return 0; +} + +/* **************************************************************************** + Prototype : roce5_qp_modify_2rst_cmd + Description : cache out qpc mtt + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2016/6/17 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** */ +int roce5_qp_modify_2rst_cmd(struct roce5_device *rdev, u32 qpn) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_qp_modify2rst_s *qp_moidfy2rst_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_modify2rst_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + qp_moidfy2rst_inbuf = + (roce_uni_cmd_qp_modify2rst_s *)cqm_cmd_inbuf->buf; + qp_moidfy2rst_inbuf->com.index = cpu_to_be32(qpn); + qp_moidfy2rst_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_2RST_QP, cqm_cmd_inbuf, + NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send 2RST_QP command, qpn(0x%x), func_id(%d), ret(0x%x)", + qpn, rdev->glb_func_id, ret); + + ret = (-EINVAL); + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + goto err_out; + } + +err_out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_qp_check_rdmarc + Description : roce5_qp_check_rdmarc + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2017/04/26 + Modification : Create function + +**************************************************************************** */ +static int roce5_qp_check_rdmarc(struct roce5_device *rdev, + struct roce5_qp *rqp, int *cost) +{ + int times = 0; + u32 rc_num = 0; + u32 rc_check_index = 0; + void *rdmarc_vaddr = NULL; + dma_addr_t rdmarc_paddr = 0; + + rc_num = + (u32)(((u32)1 << rqp->rdmarc.order) / RDMARC_NUM_PER_CACHELINE); + rdmarc_vaddr = rqp->rdmarc.vaddr; + rdmarc_paddr = rqp->rdmarc.dma_addr; + + times = rdev->try_times; + do { + if ((*((u32 *)rdmarc_vaddr) == + ROCE_RDMARC_DESTROY_CHECK_VALUE) && + (cpu_to_be32(*((u32 *)rdmarc_vaddr + 1)) == rqp->qpn) && + (cpu_to_be64(*((u64 *)rdmarc_vaddr + 1)) == + ((u64)rdmarc_paddr + + ((u64)ROCE_CACHE_LINE_SIZE * (u64)rc_check_index)))) { + rdmarc_vaddr = (void *)((u8 *)rdmarc_vaddr + + ROCE_CACHE_LINE_SIZE); + rc_check_index++; + if (rc_num == rc_check_index) { + break; + } + } else { + ROCE_DELAY(); + times--; + } + } while (times != 0); + + if (rc_num != rc_check_index) { + ROCE_DEV_ERR( + rdev, + "Failed to read rdmarc field back after try %d times, func_id(%d) rc_check_index:%u rc_num:%u", + ((rdev->try_times - times) + 1), rdev->glb_func_id, + rc_check_index, rc_num); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static int roce5_qp_check_state(struct roce5_device *rdev, + const struct roce5_qp *rqp, const u32 *wb_va, + int *cost) +{ + int times = 0; + int read_back_flag = 0; + + times = rdev->try_times; + while ((times--) != 0) { + if (*(u32 *)wb_va == ROCE_QP_DESTROY_CHECK_VALUE_HIGH_4B && + (*((u32 *)wb_va + 1) == + ROCE_QP_DESTROY_CHECK_VALUE_LOW_4B)) { + read_back_flag = 1; + break; + } + ROCE_DELAY(); + } + + if (read_back_flag == 0) { + ROCE_DEV_ERR( + rdev, + "Failed to read QP field back after try %d times,qpn(0x%x), wb_data(0x%x), wb_data+1(0x%x) func_id(%d)", + ((rdev->try_times - times) + 1), rqp->qpn, + *(u32 *)wb_va, *((u32 *)wb_va + 1), rdev->glb_func_id); + *cost = (rdev->try_times - times) + 1; + return -1; + } + *cost = (rdev->try_times - times) + 1; + return 0; +} + +static void roce5_qp_fill_cacheout_inbuf(struct roce5_device *rdev, + struct roce5_qp *rqp, + dma_addr_t wb_dma, + cqm_cmd_buf_s *cqm_cmd_inbuf) +{ + roce_uni_cmd_qp_cache_invalid_s *qp_cacheout_inbuf = NULL; + struct rdma_service_cap *rdma_cap = NULL; + u32 page_size = roce5_get_page_size(rdev, true); + + qp_cacheout_inbuf = + (roce_uni_cmd_qp_cache_invalid_s *)(cqm_cmd_inbuf)->buf; + rdma_cap = &rdev->rdma_cap; + + qp_cacheout_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + qp_cacheout_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_cacheout_inbuf->qp_cache.sq_buf_len = cpu_to_be32( + ALIGN(((u32)rqp->sq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), + page_size)); + qp_cacheout_inbuf->qp_cache.rq_buf_len = cpu_to_be32( + ALIGN(((u32)rqp->rq.wqebb_cnt) << ((u32)rqp->sq.wqe_shift), + page_size)); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_flags = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_num = 0; + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + qp_cacheout_inbuf->qp_cache.cmtt_cache.mtt_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + qp_cacheout_inbuf->qp_cache.wb_gpa = cpu_to_be64(wb_dma); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_flags = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_num = 0; + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + qp_cacheout_inbuf->qp_cache.wqe_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); +} + +static int roce5_qp_send_cache_out_cmd(struct roce5_device *rdev, + struct roce5_qp *rqp, void *wb_va, + dma_addr_t wb_dma, int *cost) +{ + unsigned long end; + int times = 0; + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_cache_invalid_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + roce5_qp_fill_cacheout_inbuf(rdev, rqp, wb_dma, cqm_cmd_inbuf); + + end = jiffies + + (unsigned long)((unsigned int)rdev->destory_flow_timeout * HZ); + while (time_before(jiffies, end)) { + times++; + ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, + ROCE_CMD_MISC_CACHE_INVLD, + cqm_cmd_inbuf, NULL, + ROCE_CMD_TIME_CLASS_C); + if (ret <= 0) { /* return success or exception happened */ + break; + } + cond_resched(); + }; + + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed MISC_CACHE_INVLD command after try %d times, ret(%d), func_id(%d) qpn(%d)", + times, ret, rdev->glb_func_id, rqp->qpn); + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = times; + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR( + rdev, + "Card not present, return ERROR, func_id(%d)", + rdev->glb_func_id); + ret = -ENODEV; + } + return ret; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + *cost = (rdev->try_times - times) + 1; + return 0; +} + +/* **************************************************************************** + Prototype : roce5_qp_cache_out_cmd + Description : cache out qpc mtt + Input : struct roce5_device *rdev + struct roce5_qp *rqp + Output : None + + 1.Date : 2016/9/24 + Modification : Created function + + 2.Date : 2017/04/26 + Modification : Modify function + +**************************************************************************** */ +/* roce5_dealloc_latch_resource() collect use construct temporary change amount rqp to reuse this function. such as newly added for rqp intomember of use, need 排 query whether need adaptation */ +int roce5_qp_cache_out_cmd(struct roce5_device *rdev, struct roce5_qp *rqp) +{ + int ret = 0; + int cost = 0; + void *wb_va = NULL; + dma_addr_t wb_dma = 0; + + wb_va = dma_alloc_coherent(rdev->dev, (size_t)8UL, &wb_dma, GFP_KERNEL); + if (wb_va == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc wb_va, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + ret = roce5_qp_send_cache_out_cmd(rdev, rqp, wb_va, wb_dma, &cost); + if (ret != 0) { + goto err; + } + + if (rqp->rsp_depth > 0) { + ret = roce5_qp_check_rdmarc(rdev, rqp, &cost); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to check qp rdmarc, func_id(%d)", + rdev->glb_func_id); + goto err; + } + } + + ret = roce5_qp_check_state(rdev, rqp, (u32 *)wb_va, &cost); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check qp state, func_id(%d)", + rdev->glb_func_id); + goto err; + } + +err: + dma_free_coherent(rdev->dev, sizeof(u32), wb_va, wb_dma); + + /* At the end of the check, if the failure is caused by a chip link down, release resources on the host side. */ + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + + return ret; +} + +static u8 roce5_get_qp_cmdq_cmd(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask) +{ + static u16 op[ROCE_QP_STATE_NUM][ROCE_QP_STATE_NUM]; + enum ib_qp_state cur_state, new_state; + + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + if ((cur_state > IB_QPS_ERR) || (new_state > IB_QPS_ERR)) { + ROCE_DEV_ERR( + rdev, + "QP state modify invalid, cur_state(%d), new_state(%d), func_id(%d)", + cur_state, new_state, rdev->glb_func_id); + return 0; + } + op[IB_QPS_RESET][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RESET][IB_QPS_INIT] = ROCE_CMD_RST2INIT_QP; + + op[IB_QPS_INIT][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_INIT][IB_QPS_INIT] = ROCE_CMD_INIT2INIT_QP; + op[IB_QPS_INIT][IB_QPS_RTR] = ROCE_CMD_INIT2RTR_QP; + + op[IB_QPS_RTR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTR][IB_QPS_RTS] = ROCE_CMD_RTR2RTS_QP; + + op[IB_QPS_RTS][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_RTS][IB_QPS_RTS] = ROCE_CMD_RTS2RTS_QP; + op[IB_QPS_RTS][IB_QPS_SQD] = ROCE_CMD_RTS2SQD_QP; + + op[IB_QPS_SQD][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQD][IB_QPS_RTS] = ROCE_CMD_SQD2RTS_QP; + op[IB_QPS_SQD][IB_QPS_SQD] = ROCE_CMD_SQD2SQD_QP; + + op[IB_QPS_SQE][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + op[IB_QPS_SQE][IB_QPS_RTS] = ROCE_CMD_SQERR2RTS_QP; + + op[IB_QPS_ERR][IB_QPS_ERR] = ROCE_CMD_2ERR_QP; + + return (u8)op[cur_state][new_state]; +} + +static int roce5_qp_rts2sqd_cmd(struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask) +{ + int ret = 0; + int sqd_event = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_qp_modify_rts2sqd_s *qp_rts2sqd_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_modify_rts2sqd_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + sqd_event = ((((u32)attr_mask) & IB_QP_EN_SQD_ASYNC_NOTIFY) != 0) && + (attr->en_sqd_async_notify != 0); + qp_rts2sqd_inbuf = + (roce_uni_cmd_qp_modify_rts2sqd_s *)cqm_cmd_inbuf->buf; + qp_rts2sqd_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_rts2sqd_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + qp_rts2sqd_inbuf->sqd_event_en = cpu_to_be32((u32)(sqd_event != 0)); + + ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, ROCE_CMD_RTS2SQD_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RTS2SQD_QP cmd, func_id(%d), ret(0x%x)", + rdev->glb_func_id, ret); + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return 0; +} + +static int roce5_qp_handle_err(struct roce5_device *rdev, + const struct roce5_qp *rqp, u8 cmd, int ret) +{ + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_QP command, command(%u), func_id(%d), ret(%d)", + cmd, rdev->glb_func_id, ret); + + if ((ret == (-ETIMEDOUT)) || (ret == (-EPERM))) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + return -1; +} + +static int roce5_qp_modify_cmd(struct roce5_device *rdev, struct roce5_qp *rqp, + u8 cmd, roce_modify_qp_param_s *modify_para) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_modify_qpc_s *qp_modify_inbuf = NULL; + roce_verbs_qp_attr_s *qp_attr = &modify_para->qp_attr; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_qpc_s), &cqm_cmd_outbuf, + ROCE_QP_MODIFY_CMD_OUT_BUF_SIZE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return (-ENOMEM); + } + + qp_modify_inbuf = (roce_uni_cmd_modify_qpc_s *)cqm_cmd_inbuf->buf; + + roce5_transmit_ci_record_addr_ext(rqp, qp_attr); + roce5_qpc_to_be(qp_attr, rqp, (u32 *)&qp_modify_inbuf->qp_attr); + qp_modify_inbuf->com.index = cpu_to_be32(rqp->qpn); + qp_modify_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + qp_modify_inbuf->com.opt = cpu_to_be32(modify_para->optpar); + ret = roce5_send_qp_lb_cmd(rqp->qpn, rdev, cmd, cqm_cmd_inbuf, + cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ret = roce5_qp_handle_err(rdev, rqp, cmd, ret); + } else { + roce5_be32_2_le32( + cqm_cmd_outbuf->buf, (u32 *)&modify_para->modify_rsp, + sizeof(roce_verbs_cmd_qp_modify_rsp_s) / sizeof(u32)); + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +void roce5_cq_clean(struct roce5_cq *cq, u32 qpn, struct roce5_srq *srq) +{ + spin_lock_irq(&cq->lock); + roce5_cq_clean_process(cq, qpn, srq); + spin_unlock_irq(&cq->lock); +} + +static int roce5_qp_clean_kernel_res(struct roce5_qp *rqp) +{ + int ret = 0; + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + + roce5_get_cqs(rqp, &send_cq, &recv_cq); + + roce5_cq_clean(recv_cq, rqp->qpn, + rqp->ibqp.srq ? to_roce5_srq(rqp->ibqp.srq) : NULL); + if (send_cq != recv_cq) { + roce5_cq_clean(send_cq, rqp->qpn, NULL); + } + + rqp->rq.head = 0; + rqp->rq.tail = 0; + + rqp->sq.head = 0; + rqp->sq.tail = 0; + + rqp->sq_next_wqe = 0; + rqp->sq_wqe_ssn = 0; + + *(rqp->db.db_record) = 0; + + if (rqp->qp_type != IB_QPT_XRC_TGT) { + if (rqp->has_rq && rqp->qp_buf_info->q_header_vaddr) { + ret = memset_s(rqp->qp_buf_info->q_header_vaddr, + sizeof(u64), 0, sizeof(u64)); + if (ret != 0) { + ROCE_ERR( + "Failed to memset QP(0x%06x) q_header_vaddr(ret:%d)", + rqp->qpn, ret); + return (-ENOMEM); + } + } + } + + return ret; +} + +static int roce5_qp_modify_2rst(struct roce5_device *rdev, struct roce5_qp *rqp) +{ + int ret = 0; + + ret = roce5_qp_modify_2rst_cmd(rdev, rqp->qpn); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) to RESET, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + ret = roce5_qp_cache_out_cmd(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "QP(0x%06x) cache invalid failed, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + rqp->qp_state = IB_QPS_RESET; + + if (rqp->ibqp.uobject == NULL) { + ret = roce5_qp_clean_kernel_res(rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Clean kernel QP(0x%06x) failed, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static void roce5_set_stub_field(struct roce5_device *rdev, + roce_verbs_qp_attr_s *qp_attr) +{ + qp_attr->com_info.dw6.bs.force_local = rdev->local_en; +} + +static int roce5_modify_qp_cmd_process(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + roce_modify_qp_param_s *modify_para) +{ + u8 cmd; + int ret; + cmd = roce5_get_qp_cmdq_cmd(rdev, rqp, attr, attr_mask); + if (cmd == 0) { + ROCE_DEV_ERR(rdev, + "Invalid cmd, func_id(%u), qpn(%u), cmd(%u).", + rdev->glb_func_id, rqp->qpn, cmd); + return -EINVAL; + } + if (cmd == ROCE_CMD_RTS2SQD_QP) { + ret = roce5_qp_rts2sqd_cmd(rdev, rqp, attr, attr_mask); + } else { + ret = roce5_qp_modify_cmd(rdev, rqp, cmd, modify_para); + } + return ret; +} + +static int roce5_qp_modify(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + int ret = 0; + roce_user_handle_t hdl; + roce_modify_qp_param_s modify_para; + roce_verbs_qp_attr_s *qp_attr = &modify_para.qp_attr; + + (void)memset_s(&modify_para, sizeof(roce_modify_qp_param_s), 0, + sizeof(roce_modify_qp_param_s)); + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + ret = roce5_alloc_opt_rdmarc(rqp, attr, attr_mask); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdmarc, func(%d).", + rdev->glb_func_id); + return ret; + } + } + + ret = roce5_set_opt_field(rqp, attr, attr_mask, &modify_para); + if (ret != 0) { + goto err_out; + } + + roce5_set_abs_field(rqp, attr, attr_mask, qp_attr); + roce5_set_stub_field(rdev, qp_attr); + + (void)memset_s(&hdl, sizeof(roce_user_handle_t), 0, + sizeof(roce_user_handle_t)); + ROCE_CONSTUCT_ULD_HDL_RQP(&hdl, rdev, rqp, qp_attr, NULL, NULL, NULL); + ret = roce5_uld_qp_modify((roce_uld_handle)&hdl, attr, attr_mask, + udata); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify uld qp, func(%d), qpn(%d), ret(%d).", + rdev->glb_func_id, rqp->qpn, ret); + goto err_out; + } + + ret = roce5_modify_qp_cmd_process(rdev, rqp, attr, attr_mask, + &modify_para); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to modify qp, func(%d), qpn(%d), ret(%d).", + rdev->glb_func_id, rqp->qpn, ret); + goto err_out; + } + + ret = roce5_record_opt_field(rqp, attr, attr_mask, udata, + &modify_para.modify_rsp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to record_opt_field, func(%d), qpn(%d).", + rdev->glb_func_id, rqp->qpn); + goto err_out; + } + + return 0; + +err_out: + if ((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) { + roce5_free_opt_rdmarc(rqp); + } + + return ret; +} + +static int roce5_qp_get_path(struct roce5_device *rdev, struct roce5_qp *rqp, + struct rdma_ah_attr *ah_attr) +{ + int ret = 0; + u16 vlan_id; + + if ((ah_attr->ah_flags & IB_AH_GRH) == 0) { + ROCE_DEV_ERR(rdev, "Not supported, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + ret = roce5_resolve_grh(rdev, ah_attr, &vlan_id, NULL); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to find dmac by grh, func_id(%u)", + (u32)rdev->glb_func_id); + return ret; + } + + return 0; +} + +static int +roce5_qp_modify_check_rd_and_mtu(const struct roce5_qp *rqp, + const struct ib_qp_attr *attr, int attr_mask, + const struct rdma_service_cap *rdma_cap) +{ + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + enum ib_qp_state cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + enum ib_qp_state new_state = (enum ib_qp_state)( + ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state); + enum roce5_mtu rmtu = (enum roce5_mtu)attr->path_mtu; + + if (((((u32)attr_mask) & IB_QP_MAX_QP_RD_ATOMIC) != 0) && + (attr->max_rd_atomic > + rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma)) { + ROCE_ERR( + "Qpn(0x%x), max_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)", + rqp->ibqp.qp_num, attr->max_rd_atomic, cur_state, + new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_MAX_DEST_RD_ATOMIC) != 0) && + (attr->max_dest_rd_atomic > + rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma)) { + ROCE_ERR( + "Qpn(0x%x), max_dest_rd_atomic (%d) too large. Transition %d to %d. qp_type(%d)", + rqp->ibqp.qp_num, attr->max_dest_rd_atomic, cur_state, + new_state, rqp->ibqp.qp_type); + return -EINVAL; + } + + if (((((u32)attr_mask) & IB_QP_PATH_MTU) != 0) && + ((rmtu < ROCE_MTU_256) || + (rmtu > ((ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? ROCE_MTU_8192 : + ROCE_MTU_4096)))) { + ROCE_ERR("Input path MTU(%u) is invalid", rmtu); + return -EINVAL; + } + + return 0; +} + +static int roce5_qp_modify_check(struct roce5_qp *rqp, struct ib_qp_attr *attr, + int attr_mask, + struct rdma_service_cap *rdma_cap) +{ + int ret; + enum ib_qp_state cur_state, new_state; + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_STATE) != 0) ? + attr->qp_state : + cur_state); + +#ifndef ROCE_VBS_EN + if (!roce5_check_qp_modify_ok( + cur_state, new_state, (enum ib_qp_type)(rqp->ibqp.qp_type), + (enum ib_qp_attr_mask)attr_mask, IB_LINK_LAYER_ETHERNET)) { + ROCE_ERR( + "Qpn(0x%x), invalid attribute mask specified for transition %d to %d, qp_type(%d), " + "attr_mask(0x%x)", + rqp->ibqp.qp_num, cur_state, new_state, + rqp->ibqp.qp_type, attr_mask); + return (-EINVAL); + } +#endif + + if (((((u32)attr_mask) & IB_QP_PORT) != 0) && + (attr->port_num != ROCE_DEFAULT_PORT_NUM)) { + ROCE_ERR( + "Qpn(0x%x), invalid port number(%d) specified for transition %d to %d. qp_type(%d)", + rqp->ibqp.qp_num, attr->port_num, cur_state, new_state, + rqp->ibqp.qp_type); + return (-EINVAL); + } + + if ((((u32)attr_mask) & IB_QP_PKEY_INDEX) != 0) { + if (attr->pkey_index > 0) { + ROCE_ERR( + "Qpn(0x%x), invalid pkey index (%d) specified for transition %d to %d. " + "qp_type(%d)", + rqp->ibqp.qp_num, attr->pkey_index, cur_state, + new_state, rqp->ibqp.qp_type); + return (-EINVAL); + } + } + + ret = roce5_qp_modify_check_rd_and_mtu(rqp, attr, attr_mask, rdma_cap); + + return ret; +} + +#define ROCE5_QP_HANDLE 0 +#define ROCE5_QP_CONTINUE 1 +static int roce5_modify_qp_pre(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + int ret; + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + enum ib_qp_state cur_state, new_state; + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + cur_state = + (enum ib_qp_state)(((((u32)attr_mask) & IB_QP_CUR_STATE) != 0) ? + attr->cur_qp_state : + rqp->qp_state); + new_state = ((((u32)attr_mask) & IB_QP_STATE) != 0) ? attr->qp_state : + cur_state; + + ret = roce5_qp_modify_check(rqp, attr, attr_mask, &rdev->rdma_cap); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed qp modify check, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (new_state == IB_QPS_RESET) { + if (cur_state != new_state) { + ret = roce5_qp_modify_2rst(rdev, rqp); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify 2rst, func_id(%d)", + rdev->glb_func_id); + } + } + return ROCE5_QP_HANDLE; + } + + ret = roce5_modify_qp_pre_ext(rdev, rqp, attr, attr_mask, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to modify qp pre ext, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return ROCE5_QP_CONTINUE; +} + +int roce5_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + mutex_lock(&rqp->mutex); + + ret = roce5_modify_qp_pre(ibqp, attr, attr_mask, udata); + if ((ret == ROCE5_QP_HANDLE) || (ret < 0)) { + goto out; + } + + if ((((u32)attr_mask) & IB_QP_AV) != 0) { + ret = roce5_qp_get_path(rdev, rqp, &attr->ah_attr); + if (ret != 0) { + goto out; + } + } + + ret = roce5_qp_modify(rdev, rqp, attr, attr_mask, udata); + if (ret != 0) { + goto out; + } + +out: + mutex_unlock(&rqp->mutex); + return ret; +} + +static void roce5_verbs_shadow_set(struct roce5_device *rdev, u32 qpn, + cqm_cmd_buf_s *buf_in) +{ + roce_verbs_cmd_header_s *cmd_hdr = + (roce_verbs_cmd_header_s *)buf_in->buf; + struct roce5_qp *rqp = NULL; + + if ((qpn == ROCE_QP_SMI_QP_NUM) || (qpn == ROCE_QP_GSI_QP_NUM)) { + return; + } + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + return; + } + + if (rqp->shadow == 0 || cmd_hdr == NULL || + cmd_hdr->dw0.bs.cmd_bitmask == VERBS_CMD_TYPE_VROCE_BITMASK) { + goto shadow_out; + } + + cmd_hdr->dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK); + cmd_hdr->dw3.bs.shadow_vfid = cpu_to_be16(rqp->shadow_vfid); + +shadow_out: + roce5_rqp_put(rqp); +} + +int roce5_send_qp_lb_cmd(u32 qpn, struct roce5_device *rdev, u8 cmd, + cqm_cmd_buf_s *buf_in, cqm_cmd_buf_s *buf_out, + u32 timeout) +{ + int ret = 0; + + roce5_verbs_shadow_set(rdev, qpn, buf_in); + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, cmd, buf_in, + buf_out, NULL, timeout, HINIC5_CHANNEL_ROCE); + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c new file mode 100644 index 000000000..fa48a2671 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_recv.c @@ -0,0 +1,238 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/* **************************************************************************** + Prototype : roce5_get_recv_wqe + Description : roce5_get_recv_wqe + Input : struct roce5_qp *rqp + int n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +static void *roce5_get_recv_wqe(struct roce5_qp *rqp, u32 n) +{ + return roce5_get_wqe(rqp, rqp->rq.offset + + (n << (unsigned)rqp->rq.wqe_shift)); +} + +static int roce5_post_recv_check_qp(const struct roce5_qp *qp, + const struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr) +{ + if (ROCE_UNLIKELY((qp->qp_type == IB_QPT_XRC_INI) || + (qp->qp_type == IB_QPT_XRC_TGT))) { + ROCE_DEV_ERR( + rdev, + "Can't post WQE when TGT XRC QP, func_id(%d), qp_type(%d)", + rdev->glb_func_id, qp->qp_type); + *bad_wr = wr; + return -EINVAL; + } + + if (ROCE_UNLIKELY(qp->qp_state == IB_QPS_RESET)) { + ROCE_DEV_ERR(rdev, + "Can't post WQE when QP is RST state, func_id(%d)", + rdev->glb_func_id); + *bad_wr = wr; + return -EINVAL; + } + + return 0; +} + +static int roce5_post_recv_check_wr(struct roce5_qp *qp, + const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, u32 wr_num) +{ + if (roce5_wq_overflow(&qp->rq, wr_num, qp->ibqp.recv_cq) != 0) { + ROCE_ERR_ONCE("SQ is full head:%x, tali:%x, wr_num:%d", + (&qp->rq)->head, (&qp->rq)->tail, wr_num); + *bad_wr = wr; + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > qp->rq.max_sge)) { + *bad_wr = wr; + return -EINVAL; + } + + return 0; +} + +static int roce5_post_recv_check_length(struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + const struct ib_recv_wr *wr, int i) +{ + u32 data_len = 0; + + /* single SGEָ signs data len should less than (2G-1)B */ + if (ROCE_UNLIKELY(wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - + 1)) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Sge data len is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)", + i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return -EINVAL; + } + + data_len += wr->sg_list[i].length; + /* all SGEָ signs data len should less than 2GB */ + if (ROCE_UNLIKELY( + data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz)) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Data_len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)", + data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz, + rdev->glb_func_id); + return -EINVAL; + } + + return 0; +} +static int roce5_post_recv_set_data_seg(const struct ib_recv_wr *wr, + struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + struct roce5_qp *qp, u32 index) +{ + int i = 0; + int ret = 0; + struct roce_wqe_data_seg *dseg = NULL; + + dseg = (struct roce_wqe_data_seg *)roce5_get_recv_wqe(qp, index); + for (i = 0; i < wr->num_sge; ++i) { + ret = roce5_post_recv_check_length(rdev, bad_wr, wr, i); + if (ret != 0) { + return ret; + } + + roce5_set_data_seg(dseg, wr->sg_list + i); + dseg++; + } + + if (wr->num_sge != 0) { + dseg--; + dseg->key = + cpu_to_be32((u32)(wr->sg_list[wr->num_sge - 1].lkey | + ROCE_WQE_NEXT_SGE_INVALID)); + } else { + dseg->key = ROCE_WQE_NEXT_SGE_INVALID; + dseg->key = cpu_to_be32(dseg->key); + dseg->dw2.bs.len = 0; + } + + return 0; +} +/* **************************************************************************** + Prototype : roce5_post_recv + Description : roce5_post_recv + Input : struct ib_qp *ibqp + struct ib_recv_wr *wr + struct ib_recv_wr **bad_wr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +int roce5_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct roce5_qp *qp = NULL; + struct roce5_device *rdev = NULL; + unsigned long flags = 0; /* type required by kernel API */ + int ret = 0; + u32 wr_num = 0; + u32 index = 0; + const struct ib_recv_wr *wr_tmp = wr; + + if (ROCE_UNLIKELY(ibqp == NULL || bad_wr == NULL)) { + ROCE_ERR("ibqp(%d) or bad_wr(%d) is NULL.", !!ibqp, !!bad_wr); + return -EINVAL; + } + + qp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + ret = roce5_post_recv_check_qp(qp, rdev, bad_wr, wr_tmp); + if (ret != 0) { + return ret; + } + + spin_lock_irqsave(&qp->rq.lock, flags); + + index = (unsigned int)qp->rq.head & + ((unsigned int)qp->rq.wqebb_cnt - 1); + + for (wr_num = 0; wr_tmp != NULL; ++wr_num, wr_tmp = wr_tmp->next) { + ret = roce5_post_recv_check_wr(qp, bad_wr, wr_tmp, wr_num); + if (ret != 0) { + goto out; + } + + ret = roce5_post_recv_set_data_seg(wr_tmp, rdev, bad_wr, qp, + index); + if (ret != 0) { + goto out; + } + + qp->rq.wrid[index] = wr_tmp->wr_id; + + index = (index + 1) & (qp->rq.wqebb_cnt - 1); + } + +out: + if (ROCE_LIKELY(wr_num != 0)) { + qp->rq.head += (u32)wr_num; + + wmb(); + + /* SQ need head 4B,RQ need tail 4B */ + *(qp->db.db_record + 1) = cpu_to_be32(qp->rq.head & 0xffff); + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return ret; +} + +int roce5_post_recv_ofa(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + return roce5_post_recv(ibqp, (const struct ib_recv_wr *)wr, + (const struct ib_recv_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c new file mode 100644 index 000000000..ad1a8043a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_post_send.c @@ -0,0 +1,1205 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <rdma/ib_verbs.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "roce_compat.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_post.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_kernel_compat_gen.h" + +#include "securec.h" + +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +#define GOTO_LOCAL 1 + +#define GOTO_OUT 2 + +#define ROCE_NEED_JUMP 1 + +#define BYTE_MASK 255 + +#define ROCE_CTRL_SEG_SL 2 +/* **************************************************************************** + Prototype : roce5_get_send_wqe + Description : roce5_get_send_wqe + Input : struct roce5_qp *rqp + u32 n + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +static void *roce5_get_send_wqe(struct roce5_qp *rqp, u32 n) +{ + u32 wqebb_index = (n & (((u32)rqp->sq.wqebb_cnt) - 1)); + u32 wqebb_pos = wqebb_index << rqp->sq.wqe_shift; + return roce5_get_wqe(rqp, rqp->sq.offset + wqebb_pos); +} + +/* **************************************************************************** + Prototype : roce5_wq_overflow + Description : roce5_wq_overflow + Input : struct roce5_wq *wq + int wr_num + struct ib_cq *ibcq + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +int roce5_wq_overflow(struct roce5_wq *wq, u32 wr_num, struct ib_cq *ibcq) +{ + unsigned int cur = 0; + struct roce5_cq *cq = NULL; + + cur = wq->head - wq->tail; + if (ROCE_LIKELY((cur + wr_num) < (unsigned int)wq->max_post)) { + return 0; + } + + cq = to_roce5_cq(ibcq); + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return ((cur + (unsigned int)wr_num) >= (unsigned int)wq->max_post); +} + +/* **************************************************************************** + Prototype : roce5_ib_opcode + Description : + Input : + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/7/30 + Author : + Modification : Created function + +**************************************************************************** */ +static const u32 roce5_ib_opcode[] = { + [IB_WR_RDMA_WRITE] = ROCE_WQE_OPCODE_RDMA_WRITE, + [IB_WR_RDMA_WRITE_WITH_IMM] = ROCE_WQE_OPCODE_RDMA_WRITE_IMM, + [IB_WR_SEND] = ROCE_WQE_OPCODE_SEND, + [IB_WR_SEND_WITH_IMM] = ROCE_WQE_OPCODE_SEND_IMM, + [IB_WR_RDMA_READ] = ROCE_WQE_OPCODE_RDMA_READ, + [IB_WR_ATOMIC_CMP_AND_SWP] = ROCE_WQE_OPCODE_ATOMIC_CMP_SWP, + [IB_WR_ATOMIC_FETCH_AND_ADD] = ROCE_WQE_OPCODE_ATOMIC_FETCH_ADD, + [IB_WR_SEND_WITH_INV] = ROCE_WQE_OPCODE_SEND_INVAL, + [IB_WR_LOCAL_INV] = ROCE_WQE_OPCODE_LOCAL_INVAL, + [IB_WR_REG_MR] = ROCE_WQE_OPCODE_FRMR, + [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP, + [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD, + [IB_WR_REG_MR_INTEGRITY] = ROCE_WQE_OPCODE_REG_SIG_MR, +#if defined(IB_WR_OPCODE_HAVE_BIND_MW) + [IB_WR_BIND_MW] = ROCE_WQE_OPCODE_BIND_MW, +#endif +}; + +static const roce_opcode_attr_s g_roce5_opcode_attr[ROCE_KERNEL_OPCODE_NUM] = { + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND = 0x0, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND_WITH_INV, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_SEND_WITH_IMM, + { 0 }, //ROCE_WQE_OPCODE_SEND_FT_TEST + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_WRITE = 0x4, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_WRITE_WITH_IMM, + { 0 }, + { 0 }, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_RDMA_READ = 0x8, + { 0 }, //ROCE_WQE_OPCODE_ATOMIC_WRITE, + { 0 }, //ROCE_WQE_OPCODE_CFLUSH, + { .remote_opcode = + 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_EXT_CMP_AND_SWP, /* support max 64B */ + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_CMP_AND_SWP = 0xc, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_ATOMIC_FETCH_AND_ADD, + { .remote_opcode = 0x1 }, //ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_AND_SWP, + { .remote_opcode = + 0x1 }, //ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_AND_ADD, + { 0 }, //ROCE_WQE_OPCODE_FAST_REG_PMR, /* 8'h10-Fast Register PMR is not used by userspace */ + { 0 }, //ROCE_WQE_OPCODE_LOCAL_INV = 0x11, + { 0 }, //ROCE_WQE_OPCODE_BIND_MW, + { 0 } //ROCE_WQE_OPCODE_REG_SIG_MR = 0x13, /* 8'h13-Local opreation(Extended for further local opreation) */ +}; + +static void roce5_set_send_seg(struct roce_wqe_send_tsk_seg *snd_tsk_seg, + const struct ib_send_wr *wr) +{ + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + snd_tsk_seg->immdata_invkey = wr->ex.imm_data; + } + + if (wr->opcode == IB_WR_SEND_WITH_INV) { + snd_tsk_seg->immdata_invkey = + cpu_to_be32(wr->ex.invalidate_rkey); + } +} + +static void roce5_set_atomic_seg(struct roce_wqe_atomic_tsk_seg *atomic_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_atomic_wr *atomic = atomic_wr(wr); + + atomic_tsk_seg->key = cpu_to_be32(atomic->rkey); + atomic_tsk_seg->dw2.va = cpu_to_be64(atomic->remote_addr); + + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + atomic_tsk_seg->dw4.swap_add_data = cpu_to_be64(atomic->swap); + atomic_tsk_seg->dw6.cmp_data = cpu_to_be64(atomic->compare_add); + } else if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { + atomic_tsk_seg->dw4.swap_add_data = + cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->dw6.cmp_data = 0; + } else { + /* IB_WR_MASKED_ATOMIC_FETCH_AND_ADD */ + atomic_tsk_seg->dw4.swap_add_data = + cpu_to_be64(atomic->compare_add); + atomic_tsk_seg->dw6.cmp_data = + cpu_to_be64(atomic->compare_add_mask); + } +} + +static __always_inline void +roce5_set_rdma_seg(struct roce_wqe_rdma_tsk_seg *rdma_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_rdma_wr *rdma = rdma_wr(wr); + + rdma_tsk_seg->va = cpu_to_be64(rdma->remote_addr); + rdma_tsk_seg->rkey = cpu_to_be32(rdma->rkey); + + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { + rdma_tsk_seg->imm_data = cpu_to_be32(wr->ex.imm_data); + } +} + +static void +roce5_set_local_inv_seg(struct roce_wqe_local_inv_tsk_seg *inv_tsk_seg, + const struct ib_send_wr *wr) +{ + inv_tsk_seg->inv_key = cpu_to_be32(wr->ex.invalidate_rkey); +} + +static void +roce5_set_ud_seg_cycle1(struct roce_wqe_ud_tsk_seg_cycle1 *ud_tsk_seg_cycle1, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ud_wr(wr); + struct roce5_ah *rah = to_roce5_ah(ud->ah); + int ret; + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg_cycle1->immdata_invkey = wr->ex.imm_data; + } + + ud_tsk_seg_cycle1->dw2.value = rah->priv_ah.dw0.value; + ud_tsk_seg_cycle1->dw3.value = rah->priv_ah.dw1.value; + ud_tsk_seg_cycle1->dw4.value = rah->priv_ah.dw2.value; + ret = memcpy_s((void *)ud_tsk_seg_cycle1->dgid, ROCE_GID_LEN, + (void *)rah->priv_ah.dgid, ROCE_GID_LEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return; + } + ud_tsk_seg_cycle1->dw9.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg_cycle1->qkey = cpu_to_be32(ud->remote_qkey); +} + +static void +roce5_set_ud_seg_cycle2(struct roce_wqe_ud_tsk_seg_cycle2 *ud_tsk_seg_cycle2, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ud_wr(wr); + struct roce5_ah *rah = to_roce5_ah(ud->ah); + + ud_tsk_seg_cycle2->dw0.value = rah->priv_ah.dw7.value; + ud_tsk_seg_cycle2->dmac_l32 = rah->priv_ah.dmac_l32; +} + +static void roce5_set_ud_seg(struct roce_wqe_ud_tsk_seg *ud_tsk_seg, + const struct ib_send_wr *wr) +{ + const struct ib_ud_wr *ud = ud_wr(wr); + struct roce5_ah *rah = to_roce5_ah(ud->ah); + int ret; + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + } + + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + ret = memcpy_s((void *)ud_tsk_seg->dgid, sizeof(ud_tsk_seg->dgid), + (void *)rah->priv_ah.dgid, sizeof(ud_tsk_seg->dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return; + } + ud_tsk_seg->dw10.value = cpu_to_be32(ud->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32(ud->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; +} + +void roce5_set_data_seg(struct roce_wqe_data_seg *dseg, struct ib_sge *sge) +{ + dseg->addr = cpu_to_be64(sge->addr); + + dseg->dw2.bs.len = sge->length & 0x7fffffff; + dseg->dw2.bs.rsvd = 0; + + dseg->dw2.value = cpu_to_be32(dseg->dw2.value); + + dseg->key = cpu_to_be32(sge->lkey); +} + +/* **************************************************************************** + Prototype : roce5_set_gsi_seg + Input : struct roce5_qp *rqp + struct ib_send_wr *wr + void *wqe + Output : None + + 1.Date : 2015/8/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_set_gsi_seg(struct roce5_qp *rqp, const struct ib_send_wr *wr, + void *wqe) +{ + struct roce_wqe_ud_tsk_seg *ud_tsk_seg = + (struct roce_wqe_ud_tsk_seg *)wqe; + struct ib_ud_wr *ud_wr = (struct ib_ud_wr *)(void *)wr; + struct roce5_ah *rah = to_roce5_ah(ud_wr->ah); + int ret; + + if (rqp->sqp_attr == NULL) { + ROCE_ERR("Error! sqp attribute is NULL, qpn(%u).", rqp->qpn); + return -EINVAL; + } + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + ud_tsk_seg->immdata_invkey = wr->ex.imm_data; + ud_tsk_seg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + ud_tsk_seg->dw2.value = cpu_to_be32(ud_tsk_seg->dw2.value); + } + + ud_tsk_seg->common.bs.c = 1; + ud_tsk_seg->dw3.value = rah->priv_ah.dw0.value; + ud_tsk_seg->dw4.value = rah->priv_ah.dw1.value; + ud_tsk_seg->dw5.value = rah->priv_ah.dw2.value; + ret = memcpy_s((void *)ud_tsk_seg->dgid, sizeof(ud_tsk_seg->dgid), + (void *)rah->priv_ah.dgid, sizeof(ud_tsk_seg->dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return ret; + } + ud_tsk_seg->dw10.value = cpu_to_be32(ud_wr->remote_qpn); + ud_tsk_seg->qkey = cpu_to_be32( + ((ud_wr->remote_qkey & 0x80000000) != 0) ? rqp->sqp_attr->qkey : + ud_wr->remote_qkey); + ud_tsk_seg->dw12.value = rah->priv_ah.dw7.value; + ud_tsk_seg->dmac_l32 = rah->priv_ah.dmac_l32; + + return 0; +} + +/* **************************************************************************** + Prototype : write_invalid_wqe + Description : set the next wqe invalidate + Input : struct roce5_qp *qp + u32 index + Output : None + + 1.Date : 2015/9/8 + Author : qianyuanpan + Modification : Created function + +**************************************************************************** */ +static void write_invalid_wqe(struct roce5_qp *qp, u32 index) +{ + u32 invalid_wqe_dw0 = 0; + struct roce5_wqe_ctrl_seg *ctrl_seg = NULL; + + ctrl_seg = (struct roce5_wqe_ctrl_seg *)roce5_get_send_wqe(qp, index); + invalid_wqe_dw0 = ((index & (u32)qp->sq.wqebb_cnt) == 0) ? 0xff000000 : + 0x7f000000; + ctrl_seg->dw0.value = cpu_to_be32(invalid_wqe_dw0); +} + +#define ROCE5_PCIE_ATOMIC_EN 0x1 +static inline bool roce5_is_pcie_atomic(struct roce5_qp *rqp) +{ + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + return (rdev->cfg_info.pcie_atomic_en == ROCE5_PCIE_ATOMIC_EN); +} + +static inline bool roce5_is_extend_atomic(enum ib_wr_opcode opcode) +{ + return ((roce5_ib_opcode[opcode] == + ROCE_WQE_OPCODE_MASKED_ATOMIC_CMP_SWP) || + (roce5_ib_opcode[opcode] == + ROCE_WQE_OPCODE_MASKED_ATOMIC_FETCH_ADD)); +} + +static inline int roce5_is_vaild_wr_opcode(struct roce5_qp *rqp, + enum ib_wr_opcode opcode) +{ + if (ROCE_UNLIKELY((opcode >= ARRAY_SIZE(roce5_ib_opcode)))) { + return -EINVAL; + } + + if ((roce5_ib_opcode[opcode] == 0) && (opcode != IB_WR_SEND)) { + return -EINVAL; + } + + if (roce5_is_pcie_atomic(rqp) && roce5_is_extend_atomic(opcode)) { + ROCE_ERR( + "The atomic mode is pcie atomic, but extended atomic is not supported."); + return -EINVAL; + } + + return 0; /* Vaild IB opcode */ +} + +/* **************************************************************************** + Prototype : roce5_validate_wr + Description : roce5_validate_wr + Input : struct roce5_qp *rqp + struct ib_send_wr *wr + int wr_num + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_validate_wr(struct roce5_qp *rqp, const struct ib_send_wr *wr, + u32 wr_num) +{ + if (roce5_wq_overflow(&rqp->sq, wr_num, rqp->ibqp.send_cq) != 0) { + ROCE_ERR("SQ is full"); + return -ENOMEM; + } + + if (ROCE_UNLIKELY(((u32)wr->num_sge) > rqp->sq.max_sge)) { + ROCE_ERR( + "Sge num is invalide, wr->num_sge(%d), rqp->sq.max_sge(%d)", + wr->num_sge, rqp->sq.max_sge); + return -EINVAL; + } + + if (ROCE_UNLIKELY(roce5_is_vaild_wr_opcode(rqp, wr->opcode) != 0)) { + ROCE_ERR("wr->opcode(%d)", wr->opcode); + return -EINVAL; + } + + return 0; +} + +static void +roce5_set_rc_wqe_for_optype_send(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_send_seg((struct roce_wqe_send_tsk_seg *)param->wqe, wr); + param->data_len_addr = + &((struct roce_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_send_tsk_seg); +} + +static void +roce5_set_rc_wqe_for_optype_atomic(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_atomic_seg((struct roce_wqe_atomic_tsk_seg *)param->wqe, wr); + param->wqe = + ((u8 *)param->wqe + sizeof(struct roce_wqe_atomic_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_atomic_tsk_seg); +} + +static void +roce5_set_rc_wqe_for_optype_write(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_rdma_seg((struct roce_wqe_rdma_tsk_seg *)param->wqe, wr); + param->data_len_addr = + &((struct roce_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + sizeof(struct roce_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)(sizeof(struct roce_wqe_rdma_tsk_seg)); +} + +static void +roce5_set_rc_wqe_for_optype_local(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce5_set_local_inv_seg((struct roce_wqe_local_inv_tsk_seg *)param->wqe, + wr); + param->wqe_size += (u32)sizeof(struct roce_wqe_local_inv_tsk_seg); + param->inline_flag = 0; + param->ctrl_seg_tmp.dw0.bs.df = 0; +} + +static void roce5_set_wqe_last_len(u8 *wqe) +{ + struct roce_wqe_send_tsk_seg *snd_task = + (struct roce_wqe_send_tsk_seg *)(void *)wqe; + snd_task->dw3.value = 0; + snd_task->dw3.bs.last_ext_len = ROCE_IMM_EXT_LEN; + snd_task->dw3.value = cpu_to_be32(snd_task->dw3.value); +} + +static inline u32 roce5_log2(u32 x) +{ + u32 shift; + + for (shift = 0; (1U << shift) < x; ++shift) { + ; + } + + return shift; +} + +/* RoCE supports FRMR, for at privileged user, can via post send WR of way for at at free status of MR perform fast register. + frmr wqe contain a pbl(page buffer list) table start address, start va, length, page size etc. information. + 预先 allocate of store space of start address (pa) need write pbl. only supports one level 寻址, maximum supports 512(count word) PA. */ +static int set_frmr_wqe_section(const struct ib_send_wr *wr, + struct roce5_post_send_normal_param *param) +{ + u64 iova; + u32 fbo; + struct ib_reg_wr *frmr_reg_wr = NULL; + struct roce5_mr *roce5_kernel_frmr = NULL; + struct roce_wqe_frmr_tsk_seg *frmr_wqe_task = NULL; + + frmr_reg_wr = (struct ib_reg_wr *)reg_wr((struct ib_send_wr *)wr); + roce5_kernel_frmr = to_roce5_mr(frmr_reg_wr->mr); + + /* base section set */ + param->wqe_size += sizeof(struct roce_wqe_frmr_tsk_seg); + + frmr_wqe_task = + (struct roce_wqe_frmr_tsk_seg *)((void *)param->tsk_com_seg); + param->dseg = + (roce_wqe_data_seg_s *)(((u8 *)param->tsk_com_seg) + + sizeof(struct roce_wqe_frmr_tsk_seg)); + param->ctrl_seg_tmp.dw0.bs.tsl = + sizeof(struct roce_wqe_frmr_tsk_seg) / ROCE_TASK_SEG_ALIGN; + /* fill wqe ctrl head section:cl bit */ + param->ctrl_seg_tmp.dw1.bs.cl = 1; + + /* local invalidate associated section set */ + param->tsk_com_seg->bs.so = 1; + + iova = (u64)(void *)(uintptr_t)(frmr_reg_wr->mr->iova); + fbo = (u32)(iova & (FRMR_PAGE_SIZE - 1)); + frmr_wqe_task->dw2.bs.fbo = fbo & 0x3FFFFF; + frmr_wqe_task->dw2.value = cpu_to_be32(frmr_wqe_task->dw2.value); + + frmr_wqe_task->dw5.va = + (((u32)frmr_reg_wr->access & IB_ZERO_BASED) != 0) ? + 0 : + cpu_to_be64(iova); + + /* fill in access permission */ + frmr_wqe_task->dw3.bs.zbva = + (u32)(((u32)frmr_reg_wr->access & IB_ZERO_BASED) != 0); + frmr_wqe_task->dw3.bs.be = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_MW_BIND) != + 0); /* bind operation enable */ + frmr_wqe_task->dw3.bs.lre = 1; /* this place read enable */ + frmr_wqe_task->dw3.bs.lwe = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_LOCAL_WRITE) != + 0); /* this place write enable */ + frmr_wqe_task->dw3.bs.rre = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_READ) != + 0); /* remote read enable */ + frmr_wqe_task->dw3.bs.rwe = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_WRITE) != + 0); /* remote write enable */ + frmr_wqe_task->dw3.bs.rae = + (u32)(((u32)frmr_reg_wr->access & IB_ACCESS_REMOTE_ATOMIC) != + 0); /* remote Atomic enable */ + frmr_wqe_task->dw3.bs.block = 1; + frmr_wqe_task->dw3.bs.rsvd = 0; + frmr_wqe_task->dw3.bs.pa_num = (roce5_kernel_frmr->npages == 0) ? + 1 : + roce5_kernel_frmr->npages; + /* MR memory page size. equals to (2^page_size)*4KB */ + frmr_wqe_task->dw3.bs.page_size = + roce5_log2(frmr_reg_wr->mr->page_size) - 12; //12 is 4K page of shift + if (frmr_reg_wr->mr->page_size != PAGE_SIZE) { + ROCE_WARN( + "Frmr page size doesn't match page frame of OS! It may lead to unexpected memory translation."); + } + frmr_wqe_task->dw3.value = cpu_to_be32(frmr_wqe_task->dw3.value); + + frmr_wqe_task->m_key = cpu_to_be32(frmr_reg_wr->key); + frmr_wqe_task->dw7.len = cpu_to_be64(frmr_reg_wr->mr->length); + frmr_wqe_task->dw9.pbl_addr = cpu_to_be64(roce5_kernel_frmr->page_map); + /* set inline and direct wqe handle section */ + frmr_wqe_task->rsvd[0] = 0; + frmr_wqe_task->rsvd[1] = 0; + + param->inline_flag = 0; + return 0; +} + +static int roce5_post_send_rc(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, + const struct ib_send_wr *wr) +{ + int ret = 0; + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND_WITH_INV: + roce5_set_wqe_last_len(param->wqe); + roce5_set_rc_wqe_for_optype_send(wr, param); + break; + case IB_WR_SEND: + roce5_set_rc_wqe_for_optype_send(wr, param); + break; + + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + roce5_set_rc_wqe_for_optype_atomic(wr, param); + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + roce5_set_wqe_last_len(param->wqe); + roce5_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + roce5_set_rc_wqe_for_optype_write(wr, param); + break; + + case IB_WR_LOCAL_INV: + roce5_set_rc_wqe_for_optype_local(wr, param); + ret = ROCE_NEED_JUMP; + break; + + case IB_WR_REG_MR: + set_frmr_wqe_section(wr, param); + ret = ROCE_NEED_JUMP; + break; + + default: + ROCE_DEV_ERR(rdev, "Invalid opcode(%d), func_id(%d)", + wr->opcode, rdev->glb_func_id); + break; + } + + return ret; +} + +static int roce5_post_send_uc(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, + const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND: + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + roce5_set_wqe_last_len(param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_send_seg((struct roce_wqe_send_tsk_seg *)param->wqe, + wr); + param->data_len_addr = + &((struct roce_wqe_send_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + + sizeof(struct roce_wqe_send_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_send_tsk_seg); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_WRITE: + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { + roce5_set_wqe_last_len(param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_RDMA_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + roce5_set_rdma_seg((struct roce_wqe_rdma_tsk_seg *)param->wqe, + wr); + param->data_len_addr = + &((struct roce_wqe_rdma_tsk_seg *)param->wqe)->data_len; + param->wqe = ((u8 *)param->wqe + + sizeof(struct roce_wqe_rdma_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_rdma_tsk_seg); + break; + + case IB_WR_LOCAL_INV: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_SEND_LOCAL_WQE_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->tsk_com_seg->bs.so = 1; + roce5_set_local_inv_seg( + (struct roce_wqe_local_inv_tsk_seg *)param->wqe, wr); + param->wqe_size += + (u32)sizeof(struct roce_wqe_local_inv_tsk_seg); + return ROCE_NEED_JUMP; + + default: + ROCE_DEV_ERR(rdev, "Invalid opcode(%d), func_id(%d)", + wr->opcode, rdev->glb_func_id); + break; + } + + return 0; +} + +static int roce5_post_send_ud(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct roce_wqe_ud_tsk_seg *tsk_sg = + (struct roce_wqe_ud_tsk_seg *)param->wqe; + + if (ROCE_UNLIKELY((wr->opcode != IB_WR_SEND) && + (wr->opcode != IB_WR_SEND_WITH_IMM))) { + ROCE_DEV_ERR(rdev, "Wr->opcode(%d), func_id(%d)", wr->opcode, + rdev->glb_func_id); + *bad_wr = wr; + return ROCE_NEED_JUMP; + } + + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + tsk_sg->dw2.bs.last_ext_len = ROCE_IMM_EXT_LEN; + tsk_sg->dw2.value = cpu_to_be32(tsk_sg->dw2.value); + } + + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + param->data_len_addr = + &((struct roce_wqe_ud_tsk_seg *)param->wqe)->data_len; + + if (param->sq_rmd_size > (1UL << (unsigned int)rqp->sq.wqe_shift)) { + roce5_set_ud_seg((struct roce_wqe_ud_tsk_seg *)(param->wqe), + wr); + param->wqe = + ((u8 *)param->wqe + sizeof(struct roce_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_ud_tsk_seg); + } else { + roce5_set_ud_seg_cycle1( + (struct roce_wqe_ud_tsk_seg_cycle1 *)param->wqe, wr); + param->wqe = rqp->sq_head_addr; + param->wqe_size += + (u32)sizeof(struct roce_wqe_ud_tsk_seg_cycle1); + roce5_set_ud_seg_cycle2( + (struct roce_wqe_ud_tsk_seg_cycle2 *)param->wqe, wr); + param->wqe = ((u8 *)param->wqe + + sizeof(struct roce_wqe_ud_tsk_seg_cycle2)); + param->wqe_size += + (u32)sizeof(struct roce_wqe_ud_tsk_seg_cycle2); + } + + return 0; +} + +static void roce5_post_send_set_normal_data_seg(const struct ib_send_wr *wr, + struct roce_wqe_data_seg **dseg, + struct roce5_qp *rqp) +{ + if (wr->num_sge != 0) { + if ((u8 *)(*dseg) == rqp->sq_head_addr) { + *dseg = (struct roce_wqe_data_seg *)(( + void *)(rqp->sq_tail_addr - + sizeof(struct roce_wqe_data_seg))); + } else { + (*dseg)--; + } + (*dseg)->key = (u32)wr->sg_list[wr->num_sge - 1].lkey | + ROCE_WQE_NEXT_SGE_INVALID; + (*dseg)->key = cpu_to_be32((*dseg)->key); + } +} + +static int +roce5_post_send_set_normal_sge(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int i; + + param->data_len = 0; + param->dseg = (struct roce_wqe_data_seg *)param->wqe; + param->wqe_size += + (u32)(wr->num_sge * (int)sizeof(struct roce_wqe_data_seg)); + for (i = 0; i < wr->num_sge; ++i) { + if (param->sq_rmd_size < sizeof(struct roce_wqe_data_seg)) { + param->dseg = (struct roce_wqe_data_seg + *)((void *)rqp->sq_head_addr); + } + + if (ROCE_UNLIKELY(wr->sg_list[i].length > + (rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz))) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Sge_data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)", + i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + param->data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(param->data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz)) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Data len is over range, data_len(%d), max_msg_sz(%d), func_id(%d)", + param->data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + roce5_set_data_seg(param->dseg, wr->sg_list + i); + + (param->dseg)++; + param->sq_rmd_size = + (u32)(rqp->sq_tail_addr - (u8 *)param->dseg); + } + + return 0; +} + +static void roce5_copy_inline_data(struct roce5_post_send_normal_param *param, + const struct ib_send_wr *wr, + struct roce5_qp *rqp, int i) +{ + int ret = 0; + struct ib_sge *sge = wr->sg_list + i; + + if (param->sq_rmd_size >= wr->sg_list[i].length) { + /* Set inline data */ + ret = memcpy_s(param->wqe, sge->length, + (void *)(uintptr_t)sge->addr, sge->length); + if (ret != 0) { + ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret); + return; + } + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length); + if ((u8 *)param->wqe == rqp->sq_tail_addr) { + param->wqe = rqp->sq_head_addr; + } + } else { + /* Set inline data cycle1 */ + u32 size = param->sq_rmd_size; + ret = memcpy_s(param->wqe, size, (void *)(uintptr_t)sge->addr, + size); + if (ret != 0) { + ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret); + return; + } + param->wqe = rqp->sq_head_addr; + + /* Set inline data cycle2 */ + size = wr->sg_list[i].length - param->sq_rmd_size; + ret = memcpy_s( + param->wqe, size, + (void *)((u8 *)(void *)(uintptr_t)sge->addr + size), + size); + if (ret != 0) { + ROCE_ERR("Failed to memcpy param->wqe.(ret:%d)", ret); + return; + } + param->wqe = ((u8 *)param->wqe + wr->sg_list[i].length - + param->sq_rmd_size); + + param->cycle = 1; + } +} + +static int +roce5_post_send_set_data_seg(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int i; + int ret; + + if ((((unsigned int)wr->send_flags & (unsigned int)IB_SEND_INLINE) != + 0) && + (wr->num_sge != 0) && (param->inline_flag != 0)) { + param->data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + param->wqe_size += wr->sg_list[i].length; + param->data_len += wr->sg_list[i].length; + if (param->data_len > rqp->max_inline_data) { + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Data len is too big, data_len(%d), max_inline_data(%d), func_id(%d)", + param->data_len, rqp->max_inline_data, + rdev->glb_func_id); + return ROCE_NEED_JUMP; + } + + if ((u8 *)param->wqe == rqp->sq_head_addr) { + param->cycle = 1; + } + + roce5_copy_inline_data(param, wr, rqp, i); + + param->sq_rmd_size = + (u32)(rqp->sq_tail_addr - (u8 *)param->wqe); + } + param->ctrl_seg_tmp.dw0.bs.bdsl = + (ALIGN((u32)(param->data_len), ROCE_TASK_SEG_ALIGN) / + ROCE_TASK_SEG_ALIGN) & + 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 1; + param->inline_flag = 1; + } else { + ret = roce5_post_send_set_normal_sge(param, rdev, rqp, wr, + bad_wr); + if (ret != 0) { + return ret; + } + + roce5_post_send_set_normal_data_seg(wr, &param->dseg, rqp); + + param->ctrl_seg_tmp.dw0.bs.bdsl = + (u32)((wr->num_sge * + (int)sizeof(struct roce_wqe_data_seg)) / + ROCE_TASK_SEG_ALIGN) & + 0x7ff; + param->ctrl_seg_tmp.dw0.bs.df = 0; + } + + return 0; +} + +static void roce5_fill_task_data_len(u32 *data_len_addr, u32 data_len) +{ + if (data_len_addr != NULL) { + *data_len_addr = cpu_to_be32(data_len); + } +} + +static int roce5_construct_wqe(struct roce5_post_send_normal_param *param, + struct roce5_device *rdev, struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + int ret; + + switch (rqp->qp_type) { + case IB_QPT_RC: + need_goto = roce5_post_send_rc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) { + return GOTO_LOCAL; + } + break; + + case IB_QPT_UC: + need_goto = roce5_post_send_uc(param, rdev, wr); + if (need_goto == ROCE_NEED_JUMP) { + return GOTO_LOCAL; + } + break; + + case IB_QPT_UD: + need_goto = roce5_post_send_ud(param, rdev, rqp, wr, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + return GOTO_OUT; + } + break; + + case IB_QPT_GSI: + param->ctrl_seg_tmp.dw0.bs.tsl = ROCE_UD_WQE_COM_TSL; + param->ctrl_seg_tmp.dw1.bs.cl = 1; + + ret = roce5_set_gsi_seg(rqp, wr, param->wqe); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to set gsi seg, ret(%d).", + ret); + return GOTO_OUT; + } + + param->data_len_addr = + &((struct roce_wqe_ud_tsk_seg *)param->wqe)->data_len; + param->wqe = + ((u8 *)param->wqe + sizeof(struct roce_wqe_ud_tsk_seg)); + param->wqe_size += (u32)sizeof(struct roce_wqe_ud_tsk_seg); + break; + + default: + ROCE_DEV_ERR(rdev, "Unknown qp type, func_id(%d)", + rdev->glb_func_id); + break; + } + + return 0; +} + +static void roce5_init_ctrl_seg(struct roce5_post_send_normal_param *param, + const struct ib_send_wr *wr, + const struct roce5_qp *rqp) +{ + u32 wqe_opcode = roce5_ib_opcode[wr->opcode] & 0x1f; + param->ctrl_seg_tmp.dw0.bs.owner = + ((param->index & (u32)rqp->sq.wqebb_cnt) != 0) ? 1 : 0; + param->ctrl_seg_tmp.dw0.bs.drvsl = 0; + param->ctrl_seg_tmp.dw0.bs.wf = 0; + param->ctrl_seg_tmp.dw0.bs.cf = 0; + param->ctrl_seg_tmp.dw0.bs.va = 1; + param->ctrl_seg_tmp.dw0.bs.df = 0; /* va:1bits, 0-sge,1-inline */ + param->ctrl_seg_tmp.dw0.bs.cr = + (u32)(!(((((u32)wr->send_flags) & IB_SEND_SIGNALED) | + rqp->sq_signal_bits) == 0)); + param->ctrl_seg_tmp.dw0.bs.difsl = 0; + param->ctrl_seg_tmp.dw0.bs.csl = 0; + param->ctrl_seg_tmp.dw0.bs.ctrlsl = ROCE_CTRL_SEG_SL; + param->ctrl_seg_tmp.dw0.bs.bdsl = 0; + + param->ctrl_seg_tmp.dw0.bs.wqe_ssn = (param->sq_wqe_ssn & 0x3); + param->sq_wqe_ssn += (u8)g_roce5_opcode_attr[wqe_opcode].remote_opcode; +} + +static void roce5_init_task_com_seg(union roce5_wqe_tsk_com_seg **tsk_com_seg, + u8 *wqe, const struct ib_send_wr *wr, + const struct roce5_qp *rqp) +{ + struct roce_wqe_send_tsk_seg *send_task = + (struct roce_wqe_send_tsk_seg *)(void *)wqe; + *tsk_com_seg = (union roce5_wqe_tsk_com_seg *)(void *)wqe; + + send_task->dw3.value = 0; + (*tsk_com_seg)->bs.c = (u32)(!( + (((unsigned)wr->send_flags & (unsigned)IB_SEND_SIGNALED) | + rqp->sq_signal_bits) == 0)); + (*tsk_com_seg)->bs.se = (u32)(!( + ((unsigned)wr->send_flags & (unsigned)IB_SEND_SOLICITED) == 0)); + (*tsk_com_seg)->bs.f = (u32)(!( + ((unsigned)wr->send_flags & (unsigned)IB_SEND_FENCE) == 0)); + (*tsk_com_seg)->bs.op_type = roce5_ib_opcode[wr->opcode] & 0x1f; +} + +static void +roce5_post_send_local_operation(struct roce5_post_send_normal_param *param, + struct roce5_qp *rqp, struct roce5_device *rdev) +{ + param->opcode = param->tsk_com_seg->bs.op_type; + param->tsk_com_seg->value = cpu_to_be32(param->tsk_com_seg->value); + param->ctrl_seg->dw1.value = cpu_to_be32(param->ctrl_seg_tmp.dw1.value); + param->index += (u32)ROCE_DIV_ROUND_UP(param->wqe_size, + 1U << (u32)rqp->sq.wqe_shift); + + write_invalid_wqe(rqp, param->index); + + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + wmb(); + + param->ctrl_seg->dw0.value = cpu_to_be32(param->ctrl_seg_tmp.dw0.value); +} +/* **************************************************************************** + Prototype : roce5_post_send_normal + Description : roce5_post_send_normal + Input : struct roce5_device *rdev + struct roce5_qp *rqp + struct ib_send_wr *wr + struct ib_send_wr **bad_wr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + + 2.Date : 2015/8/8 + Modification : Modify function + +**************************************************************************** */ +static int roce5_post_send_normal(struct roce5_device *rdev, + struct roce5_qp *rqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int need_goto = 0; + int ret = 0; + struct roce5_post_send_normal_param param = { 0 }; + const struct ib_send_wr *wr_tmp = wr; + + spin_lock_irqsave(&rqp->sq.lock, param.flags); + + param.index = rqp->sq_next_wqe; + param.sq_wqe_ssn = rqp->sq_wqe_ssn; + + for (param.wr_num = 0; wr_tmp != NULL; + ++(param.wr_num), wr_tmp = wr_tmp->next) { + ret = roce5_validate_wr(rqp, wr_tmp, param.wr_num); + if (ret != 0) { + *bad_wr = wr_tmp; + ROCE_DEV_ERR(rdev, + " Failed to validate wr, func(%d) qpn(%u)", + rdev->glb_func_id, rqp->qpn); + goto out; + } + + param.wqe = (u8 *)roce5_get_send_wqe(rqp, param.index); + param.ctrl_seg = (struct roce5_wqe_ctrl_seg *)param.wqe; + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + rqp->sq.wrid[(rqp->sq.head + param.wr_num) & + (u32)(rqp->sq.wqebb_cnt - 1)] = wr_tmp->wr_id; + + roce5_init_ctrl_seg(&param, wr_tmp, rqp); + + param.wqe = ((u8 *)param.wqe + sizeof(*(param.ctrl_seg))); + param.wqe_size = (u32)sizeof(*param.ctrl_seg); + roce5_init_task_com_seg(&param.tsk_com_seg, param.wqe, wr_tmp, + rqp); + + ret = roce5_construct_wqe(&param, rdev, rqp, wr_tmp, bad_wr); + if (ret == GOTO_LOCAL) { + ret = 0; + goto local; + } else if (ret == GOTO_OUT) { + ret = -EINVAL; + goto out; + } + + param.sq_rmd_size = (u32)(rqp->sq_tail_addr - (u8 *)param.wqe); + + need_goto = roce5_post_send_set_data_seg(&param, rdev, rqp, + wr_tmp, bad_wr); + if (need_goto == ROCE_NEED_JUMP) { + ret = -EINVAL; + goto out; + } + + roce5_fill_task_data_len(param.data_len_addr, param.data_len); + +local: + roce5_post_send_local_operation(&param, rqp, rdev); + } + +out: + if (ROCE_LIKELY(param.wr_num != 0)) { + rqp->sq.head += param.wr_num; + + wmb(); + + ret = roce5_write_sq_db(rdev, rqp, param.index, NULL); + rqp->sq_next_wqe = param.index; + rqp->sq_wqe_ssn = param.sq_wqe_ssn; + } + + spin_unlock_irqrestore(&rqp->sq.lock, param.flags); + + return ret; +} + +static int roce5_post_send_check_qp_type(const struct roce5_qp *rqp) +{ + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_TGT)) { + ROCE_ERR("Can't post WQE when TGT XRC QP"); + return -EINVAL; + } + + if (ROCE_UNLIKELY(rqp->qp_type == IB_QPT_XRC_INI)) { + ROCE_ERR("not support xrc in kernel space"); + return -EINVAL; + } + + if (ROCE_UNLIKELY((rqp->qp_state == IB_QPS_RESET) || + (rqp->qp_state == IB_QPS_INIT) || + (rqp->qp_state == IB_QPS_RTR))) { + ROCE_ERR("Can't post WQE when QP is RST/INIT/RTR state"); + return -EINVAL; + } + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_post_send + Description : roce5_post_send + Input : struct ib_qp *ibqp + struct ib_send_wr *wr + const struct ib_send_wr **bad_wr + Output : None +**************************************************************************** */ +int roce5_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + int ret = 0; + struct roce5_qp *rqp = to_roce5_qp(ibqp); + struct roce5_device *rdev = to_roce5_dev(ibqp->device); + + ret = roce5_post_send_check_qp_type(rqp); + if (ret != 0) { + *bad_wr = wr; + ROCE_DEV_ERR(rdev, "Failed to check qp.ret(%d)", ret); + return ret; + } + + ret = roce5_post_send_normal(rdev, rqp, wr, bad_wr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to post send normal wr, ret(%d)", + ret); + return ret; + } + + return 0; +} + +int roce5_post_send_ofa(struct ib_qp *ibqp, struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + return roce5_post_send(ibqp, (const struct ib_send_wr *)wr, + (const struct ib_send_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c new file mode 100644 index 000000000..260eca49e --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/qp/roce_qp_query.c @@ -0,0 +1,543 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include <rdma/ib_verbs.h> + +#include "roce_compat.h" + +#include "hinic5_crm.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_mr.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_qp.h" +#include "roce_restore.h" +#include "roce_npu_cmd.h" +#include "roce_main_extension.h" +#include "roce_com.h" +#include "securec.h" +#ifdef __ROCE_DFX__ +#include "roce_dfx.h" +#endif + +/** + * @brief Get roce qp by qpn. When querying succeeded, you have to call roce5_rqp_put() to free + the cqm_obj_qp. But there is no need to call roce5_rqp_put() if querying failed. + * @return Success: return the roce qp. Failure: return NULL. + */ +struct roce5_qp *roce5_get_rqp_by_qpn(struct roce5_device *rdev, u32 qpn) +{ + cqm_object_s *cqm_obj_qp = NULL; + struct roce5_qp *rqp = NULL; + + cqm_obj_qp = cqm5_object_get(rdev->hwdev, CQM_OBJECT_SERVICE_CTX, qpn, + false); + if (cqm_obj_qp == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find rqp according to qpn(0x%x), func_id(%d)", + qpn, rdev->glb_func_id); + return NULL; + } + + rqp = cqmobj_to_roce_qp(cqm_obj_qp); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to convert obj to rqp."); + cqm5_object_put(cqm_obj_qp); + } + + return rqp; +} + +static enum ib_mig_state to_ib_mig_state(int roce5_mig_state) +{ + switch (roce5_mig_state) { + case ROCE_QP_PM_ARMED: + return IB_MIG_ARMED; + + case ROCE_QP_PM_REARM: + return IB_MIG_REARM; + + case ROCE_QP_PM_MIGRATED: + return IB_MIG_MIGRATED; + + default: + return (enum ib_mig_state)(-1); + } +} + +static int to_ib_qp_access_flags(const roce_verbs_qp_attr_s *verbs_qp_attr) +{ + unsigned int ib_flags = 0; + u32 tmp = 0; + + /* RRE */ + if (verbs_qp_attr->com_info.dw0.bs.rre != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_READ; + ib_flags = tmp; + } + + /* RWE */ + if (verbs_qp_attr->com_info.dw0.bs.rwe != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_WRITE; + ib_flags = tmp; + } + + /* RAE */ + if (verbs_qp_attr->com_info.dw0.bs.rae != 0) { + tmp = ib_flags | IB_ACCESS_REMOTE_ATOMIC; + ib_flags = tmp; + } + + return (int)ib_flags; +} + +/* **************************************************************************** + Prototype : to_ib_ah_attr + Description : to_ib_ah_attr + Input : struct roce5_device *rdev + struct ib_ah_attr *ah_attr(struct rdma_ah_attr *ah_attr) + struct roce5_qp_context *context + Output : None + + 1.Date : 2015/5/26 + Modification : Created function + +**************************************************************************** */ +static void to_ib_ah_attr(struct roce5_device *rdev, + struct rdma_ah_attr *ah_attr, + roce_verbs_qp_attr_s *verbs_qp_attr) +{ + int ret; + + ret = memset_s(ah_attr, sizeof(*ah_attr), 0, sizeof(*ah_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset ah_attr.(ret:%d)", ret); + return; + } + + ah_attr->port_num = verbs_qp_attr->com_info.dw5.bs.port; + + ah_attr->sl = 0x7 - verbs_qp_attr->path_info.dw7.bs.sl; + + ah_attr->static_rate = 0; + + ah_attr->ah_flags = IB_AH_GRH; + + ah_attr->grh.sgid_index = verbs_qp_attr->path_info.dw7.bs.sgid_index; + ah_attr->grh.hop_limit = verbs_qp_attr->path_info.dw7.bs.hoplmt; + ah_attr->grh.traffic_class = verbs_qp_attr->path_info.dw6.bs.tclass; + ah_attr->grh.flow_label = verbs_qp_attr->path_info.dw6.bs.flow_label; + + ret = memcpy_s((void *)ah_attr->grh.dgid.raw, + sizeof(ah_attr->grh.dgid.raw), + (void *)verbs_qp_attr->path_info.dgid, + sizeof(ah_attr->grh.dgid.raw)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy dgid.(ret:%d)", ret); + } +} + +int roce5_uni_cmd_qp_query(struct roce5_device *rdev, u32 qpn, + u32 *verbs_qp_attr, int qp_attr_size) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_qp_query_s *qp_query_inbuf = NULL; + roce_uni_cmd_qp_query_outbuf_s *qp_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_qp_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_qp_query_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + qp_query_inbuf = (roce_uni_cmd_qp_query_s *)cqm_cmd_inbuf->buf; + qp_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + qp_query_inbuf->com.index = cpu_to_be32(qpn); + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_QUERY_QP, cqm_cmd_inbuf, + cqm_cmd_outbuf, ROCE_CMD_TIME_CLASS_B); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to send QUERY_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto free_cqm_buf; + } + + qp_query_outbuf = (roce_uni_cmd_qp_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = (int)memcpy_s((void *)verbs_qp_attr, (size_t)qp_attr_size, + (void *)&qp_query_outbuf->qp_info.qp_attr, + (size_t)qp_attr_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy verbs_qp_attr, ret(%d).", + ret); + ret = -EINVAL; + } +free_cqm_buf: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int roce5_qp_query(struct roce5_device *rdev, u32 qpn, u32 *context, + int qpc_size) +{ + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_qp_query(rdev, qpn, context, qpc_size); + } + return -EINVAL; +} + +void qpc_seg_to_le32(struct roce5_qp_context *be_ctx, + struct roce5_qp_context *le_ctx, u32 srq_vld) +{ + /* DRV Seg */ + int ret; + + ret = memcpy_s((void *)le_ctx->sw_seg.path_seg.dgid, + sizeof(be_ctx->sw_seg.path_seg.dgid), + (void *)be_ctx->sw_seg.path_seg.dgid, + sizeof(be_ctx->sw_seg.path_seg.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return; + } + + /* CHIP Seg */ + le_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa = + be64_to_cpu(be_ctx->chip_seg.qpcc.sq_rq_l0mtt_gpa); + le_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num = cpu_to_be64( + be_ctx->chip_seg.qpcc.sq_rq_pi_record_gpa_at_hop_num); + le_ctx->chip_seg.rcc.rc_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rcc.rc_curt_sge_va); + le_ctx->chip_seg.sqc.sq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.sqc.sq_curt_sge_va); + le_ctx->chip_seg.sqac.sqa_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.sqac.sqa_curt_sge_va); + + if (srq_vld != 0) { + le_ctx->chip_seg.srqc.srq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.srqc.srq_curt_sge_va); + } else { + le_ctx->chip_seg.rqc.rq_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rqc.rq_curt_sge_va); + } + + le_ctx->chip_seg.rrwc.rrw_curt_sge_va = + be64_to_cpu(be_ctx->chip_seg.rrwc.rrw_curt_sge_va); +} + +void roce5_be32_2_le32(void *context, u32 *le_ctx, u32 ctx_size) +{ + u32 *ctx = NULL, *ctx1 = NULL; + u32 i = 0; + + ctx = le_ctx; + ctx1 = (u32 *)context; + + for (i = 0; i < ctx_size; ++i, ++ctx1, ++ctx) { + *ctx = be32_to_cpu(*ctx1); + } + + return; +} + +static void roce5_get_ah_attr(const struct roce5_qp *rqp, + struct ib_qp_attr *qp_attr, + struct roce5_device *rdev, + roce_verbs_qp_attr_s *verbs_qp_attr) +{ + if ((rqp->qp_type == IB_QPT_RC) || (rqp->qp_type == IB_QPT_UC)) { + to_ib_ah_attr(rdev, &qp_attr->ah_attr, verbs_qp_attr); + } +} + +static void roce5_query_set_common_attr(struct ib_qp_attr *qp_attr, + const struct ib_qp *ibqp, + struct roce5_qp *rqp, + struct ib_qp_init_attr *qp_init_attr) +{ + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = (u32)rqp->rq.wqebb_cnt; + qp_attr->cap.max_recv_sge = (u32)rqp->rq.max_sge; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = (u32)rqp->sq.max_post; + qp_attr->cap.max_send_sge = (u32)rqp->sq.max_sge; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_attr->cap.max_inline_data = rqp->max_inline_data; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = (enum ib_qp_create_flags)0; + + qp_init_attr->sq_sig_type = (rqp->sq_signal_bits == 1) ? + IB_SIGNAL_ALL_WR : + IB_SIGNAL_REQ_WR; +} + +static void roce5_query_set_attr(struct ib_qp_attr *qp_attr, u8 tmp_qp_state, + struct roce5_qp *rqp, + roce_verbs_qp_attr_s *verbs_qp_attr, + struct roce5_device *rdev) +{ + qp_attr->qp_state = (enum ib_qp_state)rqp->qp_state; + qp_attr->path_mtu = (enum ib_mtu)verbs_qp_attr->com_info.dw3.bs.pmtu; + qp_attr->path_mig_state = + (enum ib_mig_state)to_ib_mig_state(ROCE_QP_PM_MIGRATED); + qp_attr->qkey = verbs_qp_attr->com_info.q_key; + qp_attr->rq_psn = verbs_qp_attr->com_info.dw9.bs.next_rcv_psn; + qp_attr->sq_psn = verbs_qp_attr->com_info.dw8.bs.next_send_psn; + qp_attr->dest_qp_num = verbs_qp_attr->com_info.dw0.bs.dest_qp; + qp_attr->qp_access_flags = to_ib_qp_access_flags(verbs_qp_attr); + + roce5_get_ah_attr(rqp, qp_attr, rdev, verbs_qp_attr); + + qp_attr->pkey_index = 0; + + if (qp_attr->qp_state == IB_QPS_INIT) { + qp_attr->port_num = rqp->port; + } else { + qp_attr->port_num = verbs_qp_attr->com_info.dw5.bs.port; + } + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + + /* SQ Draining FLAG */ + qp_attr->sq_draining = + (tmp_qp_state == + (u8)(enum roce_qp_state)ROCE_QP_STATE_SQ_DRAINING); + + qp_attr->max_rd_atomic = (1 << verbs_qp_attr->com_info.dw1.bs.sra_max); + qp_attr->max_dest_rd_atomic = + (1 << verbs_qp_attr->com_info.dw1.bs.rra_max); + + qp_attr->min_rnr_timer = verbs_qp_attr->com_info.dw2.bs.min_rnr_nak; + qp_attr->timeout = verbs_qp_attr->com_info.dw2.bs.ack_to; + qp_attr->retry_cnt = verbs_qp_attr->com_info.dw1.bs.to_retry_limit; + qp_attr->rnr_retry = verbs_qp_attr->com_info.dw1.bs.rnr_retry_limit; +} + +static int roce5_query_qp_struct_clean(struct ib_qp_attr *qp_attr, + struct ib_qp_init_attr *qp_init_attr, + roce_verbs_qp_attr_s *verbs_qp_attr) +{ + int ret; + + ret = memset_s(qp_attr, sizeof(struct ib_qp_attr), 0, + sizeof(struct ib_qp_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = memset_s(qp_init_attr, sizeof(struct ib_qp_init_attr), 0, + sizeof(struct ib_qp_init_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_init_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = memset_s(verbs_qp_attr, sizeof(roce_verbs_qp_attr_s), 0, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset verbs_qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} + +static void roce_qp_attr_com_info_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + verbs_qp_attr->com_info.dw0.value = + be32_to_cpu(verbs_qp_attr->com_info.dw0.value); + verbs_qp_attr->com_info.dw1.value = + be32_to_cpu(verbs_qp_attr->com_info.dw1.value); + verbs_qp_attr->com_info.dw2.value = + be32_to_cpu(verbs_qp_attr->com_info.dw2.value); + verbs_qp_attr->com_info.dw3.value = + be32_to_cpu(verbs_qp_attr->com_info.dw3.value); + verbs_qp_attr->com_info.q_key = + be32_to_cpu(verbs_qp_attr->com_info.q_key); + verbs_qp_attr->com_info.dw5.value = + be32_to_cpu(verbs_qp_attr->com_info.dw5.value); + verbs_qp_attr->com_info.dw6.value = + be32_to_cpu(verbs_qp_attr->com_info.dw6.value); + verbs_qp_attr->com_info.dw7.value = + be32_to_cpu(verbs_qp_attr->com_info.dw7.value); + verbs_qp_attr->com_info.dw8.value = + be32_to_cpu(verbs_qp_attr->com_info.dw8.value); + verbs_qp_attr->com_info.dw9.value = + be32_to_cpu(verbs_qp_attr->com_info.dw9.value); + verbs_qp_attr->com_info.lsn = be32_to_cpu(verbs_qp_attr->com_info.lsn); + verbs_qp_attr->com_info.dw11.value = + be32_to_cpu(verbs_qp_attr->com_info.dw11.value); + verbs_qp_attr->com_info.rsvd = + be32_to_cpu(verbs_qp_attr->com_info.rsvd); +} + +static void roce_qp_attr_path_info_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + verbs_qp_attr->path_info.dw0.value = + be32_to_cpu(verbs_qp_attr->path_info.dw0.value); + verbs_qp_attr->path_info.dmac_l32 = + be32_to_cpu(verbs_qp_attr->path_info.dmac_l32); + verbs_qp_attr->path_info.dw6.value = + be32_to_cpu(verbs_qp_attr->path_info.dw6.value); + verbs_qp_attr->path_info.dw7.value = + be32_to_cpu(verbs_qp_attr->path_info.dw7.value); +} + +static void roce_qp_attr_chip_seg_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + verbs_qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa = + be64_to_cpu(verbs_qp_attr->chip_seg.dw0.sq_rq_l0mtt_gpa); + verbs_qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num = be64_to_cpu( + verbs_qp_attr->chip_seg.dw2.sq_rq_pi_record_gpa_at_hop_num); + verbs_qp_attr->chip_seg.dw4.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw4.value); + verbs_qp_attr->chip_seg.dw5.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw5.value); + verbs_qp_attr->chip_seg.dw6.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw6.value); + verbs_qp_attr->chip_seg.dw7.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw7.value); + verbs_qp_attr->chip_seg.dw8.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw8.value); + verbs_qp_attr->chip_seg.dw9.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw9.value); + verbs_qp_attr->chip_seg.dw10.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw10.value); + verbs_qp_attr->chip_seg.rc_page_gpa_l = + be32_to_cpu(verbs_qp_attr->chip_seg.rc_page_gpa_l); + verbs_qp_attr->chip_seg.dw12.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw12.value); + verbs_qp_attr->chip_seg.dw13.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw13.value); + verbs_qp_attr->chip_seg.dw14.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw14.value); + verbs_qp_attr->chip_seg.dw15.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw15.value); + verbs_qp_attr->chip_seg.dw16.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw16.value); + verbs_qp_attr->chip_seg.dw17.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw17.value); + verbs_qp_attr->chip_seg.dw18.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw18.value); + verbs_qp_attr->chip_seg.dw19.value = + be32_to_cpu(verbs_qp_attr->chip_seg.dw19.value); +} + +static void roce5_qp_attr_to_le32(roce_verbs_qp_attr_s *verbs_qp_attr) +{ + roce_qp_attr_com_info_to_le32(verbs_qp_attr); + roce_qp_attr_path_info_to_le32(verbs_qp_attr); + roce_qp_attr_chip_seg_to_le32(verbs_qp_attr); +} + +/* **************************************************************************** + Prototype : roce5_query_qp + Description : roce5_query_qp + Input : struct ib_qp *ibqp + struct ib_qp_attr *qp_attr + int qp_attr_mask + struct ib_qp_init_attr *qp_init_attr + Output : None + + 1.Date : 2015/4/29 + Modification : Created function + +**************************************************************************** */ +int roce5_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) +{ + int ret = 0; + struct roce5_qp *rqp = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_qp_attr_s verbs_qp_attr; + u32 srq_vld; + u8 tmp_qp_state = 0; + + if ((ibqp == NULL) || (qp_attr == NULL) || (qp_init_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + ret = roce5_query_qp_struct_clean(qp_attr, qp_init_attr, + &verbs_qp_attr); + if (ret != 0) { + return ret; + } + + rqp = to_roce5_qp(ibqp); + rdev = to_roce5_dev(ibqp->device); + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state == (u8)(enum ib_qp_state)IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + ret = roce5_qp_query(rdev, rqp->qpn, + (u32 *)(void *)(&verbs_qp_attr), + sizeof(verbs_qp_attr)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to query qp with 25V100 or 23V200, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + mutex_unlock(&rqp->mutex); + return ret; + } + } + + srq_vld = (!(rqp->ibqp.srq)) ? ROCE_QP_NO_SRQ : ROCE_QP_HAS_SRQ; + + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + roce5_qp_attr_to_le32(&verbs_qp_attr); + rqp->qp_state = verbs_qp_attr.com_info.dw6.bs + .state; /* 25V100 or 23V200 qp state */ + } + + tmp_qp_state = rqp->qp_state; + if (rqp->qp_state == + (u8)(enum roce_qp_state)ROCE_QP_STATE_SQ_DRAINING) { + rqp->qp_state = (u8)(enum ib_qp_state)IB_QPS_SQD; + } + + if (ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) != 0) { + /* 25V100 or 23V200 set attr */ + roce5_query_set_attr(qp_attr, tmp_qp_state, rqp, &verbs_qp_attr, + rdev); + } + +done: + roce5_query_set_common_attr(qp_attr, ibqp, rqp, qp_init_attr); + + mutex_unlock(&rqp->mutex); + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c new file mode 100644 index 000000000..257e9a8aa --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.c @@ -0,0 +1,22 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" + +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev) +{ + struct rdma_comp_priv *comp_private = NULL; + comp_private = (struct rdma_comp_priv *)hinic5_get_service_adapter( + hwdev, SERVICE_T_ROCE); + return comp_private; +} + +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->pd_bitmap); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h new file mode 100644 index 000000000..8a1a8d665 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp.h @@ -0,0 +1,166 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef RDMA_COMP_H +#define RDMA_COMP_H + +/* x86 */ +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#include <linux/delay.h> +#include <linux/types.h> +#ifdef ROCE_KNL_DT +#include <linux/compiler_types.h> +#endif + +#include "hinic5_crm.h" +#include "hinic5_cqm.h" +#include "hinic5_rdma.h" + +#include "roce_gid.h" +#include "roce_npu_cmd_defs.h" +#include "roce.h" +#include "roce_compat.h" + +#include "rdma_bitmap.h" +#include "hmm_em_inner.h" +#include "hmm_common.h" + +#ifndef pr_fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": [RDMA]" fmt +#endif + +#define RDMA_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define RDMA_INVALID_GUID 0 + +#define RDMA_SERVICE_TYPE_ROCE SERVICE_T_ROCE + +#define RDMA_BIT_SHIFT_1 1 +#define RDMA_BIT_SHIFT_2 2 +#define RDMA_BIT_SHIFT_4 4 +#define RDMA_BIT_SHIFT_8 8 +#define RDMA_BIT_SHIFT_16 16 + +#define ROCE5_RDMARC_MIN_DEPTH 1 +#define ROCE5_RDMARC_MAX_DEPTH 512 +#define ROCE5_RDMARC_MIN_ENTRY 8 /* min entry aligned required by engine */ +#define ROCE5_RDMARC_MAX_ENTRY 128 /* max entry supported by engine */ +#define ROCE5_RDMARC_EXT_ENTRY 384 //384 /* ext tbl 12K. 12k/32 = 384 */ + +#define ROCE5_RDMARC_GPA_H_SHIFT 40 /* high:24bit, 64-24= 40 */ +#define ROCE5_RDMARC_GPA_L_SHIFT 8 /* bit[39:8] */ +#define ROCE5_RDMARC_RRA_MAX_MASK 0x7 /* rra_max:3bit */ +#define ROCE5_RDMARC_EXT_ORDER_SHIFT 3 + +extern u32 g_mtt_page_size; + +#define RDMA_EM_MIN_ORDER 2 /* rdma rc extend table at least 4 page */ + +#define ROCE_MTT_PAGE_SIZE_4K 4096 +#define ROCE_MTT_PAGE_SIZE_64K (64 * 1024) +#define ROCE_MTT_PAGE_SIZE_2M (2 * 1024 * 1024) +#define ROCE_MTT_PAGE_SIZE_4K_SHIFT 12 +#define ROCE_MTT_PAGE_SIZE_64K_SHIFT 16 +#define ROCE_MTT_PAGE_SIZE_2M_SHIFT 21 + +enum { + ROCE5_RDMA_MTT_PAGE_SIZE_4K = 0, + ROCE5_RDMA_MTT_PAGE_SIZE_64K = 1, + ROCE5_RDMA_MTT_PAGE_SIZE_2M = 2 +}; + +enum { + ROCE5_RDMA_CMD_TIME_OUT_A = 30000, + ROCE5_RDMA_CMD_TIME_OUT_B = 40000, + ROCE5_RDMA_CMD_TIME_OUT_C = 50000 +}; + +/** + * @brief struct rdma_comp_resource + * @details RDMA component resource structure, used to store RDMA component of related resources + */ +struct rdma_comp_resource { + struct mutex mutex; /**< gid_entry mutex used by */ + __be64 node_guid; /**< and ibdev in of node_guid same as */ + struct rdma_gid_entry * + *gid_table; /**< gid_entry in rdma component initialize when allocate memory */ +}; + +/** + * @brief enum rdma_mw_type + * @details define an enum type, used to represent RDMA memory window of type + */ +enum rdma_mw_type { RDMA_MW_TYPE_1 = 1, RDMA_MW_TYPE_2 = 2 }; + +/** + * @brief struct rdma_mw + * @details define a rdma_mw structure, used to represent a RDMA memory window + */ +struct rdma_mw { + struct mpt mpt; + u32 key; /**< mw corresponding to key */ + u32 pdn; /**< mw bind of pdn */ + enum rdma_mw_type type; /**< mw of type,type1,type2 */ + int enabled; /**< mw of status */ +}; + +struct rdma_comp_priv { + struct hmm_comp_priv hmm_priv; + struct rdma_comp_resource rdma_comp_res; /* gid & guid */ + struct hmm_rdma rsvd_lkey; + struct hmm_rdma fixed_mr; + struct rdma_service_cap rdma_cap; + + struct rdma_bitmap pd_bitmap; + struct rdma_bitmap xrcd_bitmap; + + struct hmm_buddy rdmarc_buddy; + struct hmm_em_table rdmarc_em_table; +}; + +/* Only V100 use */ +struct rdma_gid_update_inbuf { + roce_verbs_cmd_com_s com; + __be32 port; + __be32 rsvd; + struct rdma_gid_entry gid_entry; +}; + +struct rdma_gid_clear_inbuf { + __be32 port; + __be32 gid_num; +}; + +typedef struct tag_roce_clear_gid { + roce_verbs_cmd_com_s com; + + u32 port; + u32 gid_num; +} roce_cmd_clear_gid_s; + +/* for llt to set stub */ +void rdma_cleanup_pd_table(struct rdma_comp_priv *comp_priv); +struct rdma_comp_priv *get_rdma_comp_priv(void *hwdev); +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, + struct rdma_gid_entry *gid_entry); +int roce5_update_gid_mac(struct roce5_device *rdev, u32 port, int *index, + struct rdma_gid_entry *gid_entry); +int roce5_update_gid(struct roce5_device *rdev, u32 port, u32 update_index, + struct rdma_gid_entry *gid_entry); +int roce5_reset_gid_table(struct roce5_device *rdev, u32 port); +int roce5_get_gid(struct roce5_device *rdev, u32 port, u32 gid_index, + struct rdma_gid_entry *gid); + +#endif //RDMA_COMP_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c new file mode 100644 index 000000000..b7919b409 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_gid.c @@ -0,0 +1,489 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "roce.h" +#include "roce_cqm_cmd.h" +#include "roce_npu_cmd.h" +#include "roce_mix.h" +#include "roce_gid.h" +#include "securec.h" +#include "roce_npu_cmd_gid_defs.h" + +#define ROCE_GID_IFID_DW_INDEX \ + 2 /* Interface ID starts at dword index 2 in u32 gid[4] (indices 0-1: subnet_prefix, 2-3: interface_id) */ + +static int roce5_uni_cmd_update_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, + struct rdma_gid_entry *new_gid_entry, + int port, int gid_index) +{ + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_update_gid_s *gid_update_inbuf = NULL; + int ret; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_update_gid_s), NULL, 0); + if (ret != 0) { + ROCE_ERR("Failed to alloc cqm_cmd_inoutbuf, ret(%d)", ret); + return -ENOMEM; + } + + gid_update_inbuf = (roce_uni_cmd_update_gid_s *)cqm_cmd_inbuf->buf; + *(u64 *)(&gid_update_inbuf->gid_attr.gid[0]) = + new_gid_entry->global.subnet_prefix; + *(u64 *)(&gid_update_inbuf->gid_attr.gid[ROCE_GID_IFID_DW_INDEX]) = + new_gid_entry->global.interface_id; + gid_update_inbuf->gid_attr.dw4.value = + cpu_to_be32(new_gid_entry->dw4.value); + gid_update_inbuf->gid_attr.dw5.value = + cpu_to_be32(new_gid_entry->hdr_len_value); + gid_update_inbuf->gid_attr.dw6.bs1.ctrl_val = + new_gid_entry->dw6_h.value; + gid_update_inbuf->gid_attr.dw6.bs1.smac_hi16 = + be16_to_cpu(*((u16 *)(&new_gid_entry->smac[0]))); + gid_update_inbuf->gid_attr.dw6.value = + cpu_to_be32(gid_update_inbuf->gid_attr.dw6.value); + gid_update_inbuf->gid_attr.smac_lo32 = + *((u32 *)(&new_gid_entry->smac[2])); //4=lo32 + + gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index); + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + gid_update_inbuf->com.opt = cpu_to_be32((u32)port); + if (rdev->shadow != 0) { + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK); + gid_update_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rdev->shadow_vfid); + gid_update_inbuf->gid_attr.dw10.shadow.shadow_vf_num = + rdev->shadow_vf_num; + gid_update_inbuf->gid_attr.dw10.value = + cpu_to_be32(gid_update_inbuf->gid_attr.dw10.value); + } + + ROCE_PRINT_GID(rdev, new_gid_entry, "gid_index(0x%x)", + gid_update_inbuf->com.index); + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Send cmd update_gid failed, ret(%d)", ret); + } + + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +static int roce5_cmd_update_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, + struct rdma_gid_entry *new_gid_entry, int port, + int gid_index) +{ + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + struct rdma_gid_update_inbuf *gid_update_inbuf = NULL; + int ret; + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + return roce5_uni_cmd_update_gid(rdev, comp_priv, new_gid_entry, + port, gid_index); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct rdma_gid_update_inbuf), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret(%d)", + ret); + return -ENOMEM; + } + + gid_update_inbuf = (struct rdma_gid_update_inbuf *)cqm_cmd_inbuf->buf; + gid_update_inbuf->port = cpu_to_be32((u32)port); + gid_update_inbuf->com.index = cpu_to_be32((u32)gid_index); + gid_update_inbuf->gid_entry.global.subnet_prefix = + new_gid_entry->global.subnet_prefix; + gid_update_inbuf->gid_entry.global.interface_id = + new_gid_entry->global.interface_id; + gid_update_inbuf->gid_entry.dw4.value = + cpu_to_be32(new_gid_entry->dw4.value); + gid_update_inbuf->gid_entry.dw6_h.value = + cpu_to_be16(new_gid_entry->dw6_h.value); + gid_update_inbuf->gid_entry.hdr_len_value = + cpu_to_be32(new_gid_entry->hdr_len_value); + gid_update_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + + ret = memcpy_s(&gid_update_inbuf->gid_entry.smac[0], ETH_ALEN, + &new_gid_entry->smac[0], ETH_ALEN); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy_s smac.(ret:%d)", ret); + ret = -ENOMEM; + goto err; + } + + ROCE_PRINT_GID(rdev, new_gid_entry, "gid_index(0x%x) ", + gid_update_inbuf->com.index); + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_UPDATE_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Send cmd update_gid failed, ret(%d)", ret); + ret = -1; + } + +err: + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +int rdma_gid_entry_cmp(struct rdma_gid_entry *gid_tbl_entry, + struct rdma_gid_entry *gid_entry) +{ + struct rdma_gid_entry entry1; + struct rdma_gid_entry entry2; + int ret; + + ret = memcpy_s(&entry1, sizeof(struct rdma_gid_entry), gid_tbl_entry, + sizeof(struct rdma_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy entry1.(ret:%d)", ret); + return -ENOMEM; + } + ret = memcpy_s(&entry2, sizeof(struct rdma_gid_entry), gid_entry, + sizeof(struct rdma_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy entry2.(ret:%d)", ret); + return -ENOMEM; + } + + /* compare ip+vlan, compare after clear smac */ + ret = memset_s((void *)entry1.smac, sizeof(entry1.smac), 0, + sizeof(entry1.smac)); + if (ret != 0) { + ROCE_ERR("Failed to memset entry1 smac.(ret:%d)", ret); + return -ENOMEM; + } + ret = memset_s((void *)entry2.smac, sizeof(entry2.smac), 0, + sizeof(entry2.smac)); + if (ret != 0) { + ROCE_ERR("Failed to memset entry2.smac.(ret:%d)", ret); + return -ENOMEM; + } + return memcmp((void *)&entry1, (void *)&entry2, + sizeof(struct rdma_gid_entry)); +} + +static int roce5_uni_cmd_reset_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, u32 port, + u32 gid_num) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_clear_gid_s *gid_clear_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + comp_priv->hmm_priv.hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_clear_gid_s), NULL, 0); + if (ret != 0) { + ROCE_ERR("Failed to alloc cqm_cmd_inoutbuf, ret(%d)", ret); + return -ENOMEM; + } + + gid_clear_inbuf = (roce_uni_cmd_clear_gid_s *)cqm_cmd_inbuf->buf; + gid_clear_inbuf->gid_clear.gid_num = cpu_to_be32(gid_num); + gid_clear_inbuf->com.index = 0; + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + gid_clear_inbuf->com.opt = cpu_to_be32((u32)port); + + if (rdev->shadow != 0) { + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SHADOW_BITMASK); + gid_clear_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rdev->shadow_vfid); + gid_clear_inbuf->gid_clear.dw1.shadow.shadow_vf_num = + rdev->shadow_vf_num; + gid_clear_inbuf->gid_clear.dw1.value = + cpu_to_be32(gid_clear_inbuf->gid_clear.dw1.value); + } + + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Send cmd clear_gid failed, ret(%d)", ret); + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +static int roce5_cmd_reset_gid(struct roce5_device *rdev, + struct rdma_comp_priv *comp_priv, u32 port, + u32 gid_num) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_cmd_clear_gid_s *gid_clear_inbuf = NULL; + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + return roce5_uni_cmd_reset_gid(rdev, comp_priv, port, gid_num); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(comp_priv->hmm_priv.hwdev, + &cqm_cmd_inbuf, + (u16)sizeof(roce_cmd_clear_gid_s), + NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc cqm_cmd_inoutbuf, ret(%d)", + ret); + return -ENOMEM; + } + + gid_clear_inbuf = (roce_cmd_clear_gid_s *)cqm_cmd_inbuf->buf; + gid_clear_inbuf->port = cpu_to_be32(port); + gid_clear_inbuf->gid_num = cpu_to_be32(gid_num); + gid_clear_inbuf->com.index = 0; + gid_clear_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_GID_BITMASK); + + ret = cqm5_send_cmd_box(comp_priv->hmm_priv.hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_CLEAR_GID, cqm_cmd_inbuf, NULL, NULL, + ROCE5_RDMA_CMD_TIME_OUT_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Send cmd clear_gid failed, ret(%d)", ret); + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(comp_priv->hmm_priv.hwdev, cqm_cmd_inbuf, + NULL); + + return ret; +} + +int roce5_update_gid_mac(struct roce5_device *rdev, u32 port, int *index, + struct rdma_gid_entry *gid_entry) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + u32 gid_index = 0; + int update_index = + -1; /* -1 is initvalue, indicating no need to update */ + int ret = 0; + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_ERR("Hwdev or gid_tbl is null"); + return -EINVAL; + } + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + /* port num index from zero */ + if (port >= comp_priv->rdma_cap.num_ports) { + ROCE_ERR("Input port(%d) is invalid", port); + return -EINVAL; + } + /* invalid gid cause of with the head fe80:: of IPV6 */ + if (cpu_to_be64(gid_entry->global.subnet_prefix) == + RDMA_DEFAULT_GID_SUBNET_PREFIX) { + return 0; + } + gid_table = comp_priv->rdma_comp_res.gid_table; + mutex_lock(&comp_priv->rdma_comp_res.mutex); + /* loop all gid, ensure all gid cleared not exist */ + for (gid_index = 1; gid_index < comp_priv->rdma_cap.max_gid_per_port; + gid_index++) { + if (rdma_gid_entry_cmp(&gid_table[port][gid_index], + gid_entry) == 0) { + update_index = (int)gid_index; + break; + } + } + + if (update_index > 0) { + ret = roce5_cmd_update_gid(rdev, comp_priv, gid_entry, + (int)port, update_index); + if (ret != 0) { + ROCE_ERR("Update gid tbl failed, ret(%d)", ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + /* update gid table after CMDQ */ + ret = memcpy_s((void *)&gid_table[port][update_index], + sizeof(*gid_entry), (void *)gid_entry, + sizeof(*gid_entry)); + if (ret != 0) { + ROCE_ERR( + "Failed to memcpy gid_entry to gid_table.(ret:%d)", + ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return (-ENOMEM); + } + + *index = update_index; + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; + } + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return 0; +} + +int roce5_update_gid(struct roce5_device *rdev, u32 port, u32 update_index, + struct rdma_gid_entry *gid_entry) +{ + int ret = 0; + u32 port_num = 0; + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_ERR("Hwdev or gid_tbl is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + + /* port start from zero */ + port_num = comp_priv->rdma_cap.num_ports; + if (port >= port_num) { + ROCE_DEV_ERR(rdev, "Input port(%d) is invalid", port); + return -EINVAL; + } + + mutex_lock(&comp_priv->rdma_comp_res.mutex); + ret = roce5_cmd_update_gid(rdev, comp_priv, gid_entry, (int)port, + (int)update_index); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Rdma_update_gid failed"); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return ret; + } + + ret = memcpy_s((void *)&gid_table[port][update_index], + sizeof(*gid_entry), (void *)gid_entry, + sizeof(*gid_entry)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy gid_entry.(ret:%d)", ret); + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + return -ENOMEM; + } + mutex_unlock(&comp_priv->rdma_comp_res.mutex); + + return 0; +} + +int roce5_reset_gid_table(struct roce5_device *rdev, u32 port) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_gid_entry **gid_table = NULL; + int ret = 0; + u32 gids_per_port = 0; + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + ROCE_ERR("Gid_table is null"); + return -EINVAL; + } + + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + + if (gids_per_port == 0) { + ROCE_ERR("Gids_per_port(%d) is invalid", gids_per_port); + return -EINVAL; + } + + ret = memset_s(&gid_table[port][0], + gids_per_port * sizeof(struct rdma_gid_entry), 0, + gids_per_port * sizeof(struct rdma_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_table.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_cmd_reset_gid(rdev, comp_priv, port, gids_per_port); + if (ret != 0) { + ROCE_ERR("Reset gid table failed, gid_num(%d), ret(%d)", + gids_per_port, ret); + return ret; + } + + return 0; +} + +int roce5_get_gid(struct roce5_device *rdev, u32 port, u32 gid_index, + struct rdma_gid_entry *gid) +{ + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret; + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + if (gid == NULL) { + ROCE_ERR("Gid is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + if (port < 1 || port > rdma_cap->num_ports) { + ROCE_ERR("Port(%d) invalid", port); + return -EINVAL; + } + + if (gid_index >= rdma_cap->max_gid_per_port) { + ROCE_ERR("gid_index(%d) invalid", gid_index); + return -EINVAL; + } + + ret = memcpy_s( + (void *)gid, sizeof(*gid), + (void *)&comp_priv->rdma_comp_res.gid_table[port - 1][gid_index], + sizeof(*gid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c new file mode 100644 index 000000000..ca5e88aff --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_init.c @@ -0,0 +1,443 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "roce_infra.h" +#include "hinic5_hmm.h" +#include "securec.h" + +#define ROCE_MAX_RDMA_RC_EXTEND 384 /* 12K */ + +static int rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.num_pds; + reserved_bot = comp_priv->rdma_cap.reserved_pds; + + ret = rdma_bitmap_init(&comp_priv->pd_bitmap, num, num - 1, + reserved_bot, 0); + if (ret != 0) { + ROCE_ERR("Can't initialize pd's bitmap, ret(%d)", ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + int i = 0; + int port_num = 0; + + gid_table = comp_priv->rdma_comp_res.gid_table; + if (gid_table == NULL) { + ROCE_ERR("Gid_table is null"); + return; + } + + port_num = (int)comp_priv->rdma_cap.num_ports; + for (i = 0; i < port_num; i++) { + if (gid_table[i]) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + } + + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; + + return; +} + +static int roce5_get_gid_nums(struct rdma_comp_priv *comp_priv) +{ + struct hinic5_hwdev *dev = comp_priv->hmm_priv.hwdev; + struct service_cap *cap = &(dev->cfg_mgmt->svc_cap); + struct dev_roce_svc_own_cap *roce_cap = + &(cap->rdma_cap.dev_rdma_cap.roce_own_cap); + + /* 23v100 and earlier generations: The configuration file does not contain max gid. */ + return (roce_cap->max_gid != 0 ? roce_cap->max_gid : + comp_priv->rdma_cap.max_gid_per_port); +} + +static int rdma_init_gid_table(struct rdma_comp_priv *comp_priv) +{ + struct rdma_gid_entry **gid_table = NULL; + u32 i = 0; + u32 port_num = 0; + u32 gids_per_port = 0; + + port_num = comp_priv->rdma_cap.num_ports; + gids_per_port = comp_priv->rdma_cap.max_gid_per_port; + + if ((port_num == 0) || (gids_per_port == 0)) { + ROCE_ERR( + "Alloc memory for gid_tbl failed, port_num(%d), gids_per_ports(%d)", + port_num, gids_per_port); + return -EINVAL; + } + + gid_table = (struct rdma_gid_entry **)kzalloc( + port_num * sizeof(struct rdma_gid_entry *), GFP_KERNEL); + if (gid_table == NULL) { + ROCE_ERR( + "Alloc memory for gid_tbl failed, ret(%d), port_num(%d)", + -ENOMEM, port_num); + return -ENOMEM; + } + + comp_priv->rdma_comp_res.gid_table = gid_table; + + for (i = 0; i < port_num; i++) { + gid_table[i] = (struct rdma_gid_entry *)kzalloc( + gids_per_port * sizeof(struct rdma_gid_entry), + GFP_KERNEL); + if (gid_table[i] == NULL) { + ROCE_ERR( + "Alloc memory for gid_tbl[%d] failed, ret(%d), gids_per_ports(%d)", + i, -ENOMEM, gids_per_port); + goto err_out; + } + } + return 0; +err_out: + for (i = 0; i < port_num; i++) { + if (gid_table[i]) { + kfree(gid_table[i]); + gid_table[i] = NULL; + } + } + kfree(gid_table); + comp_priv->rdma_comp_res.gid_table = NULL; + + return -ENOMEM; +} + +static int rdma_init_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 num = 0; + u32 reserved_bot = 0; + + num = comp_priv->rdma_cap.max_xrcds; + reserved_bot = comp_priv->rdma_cap.reserved_xrcds; + + ret = rdma_bitmap_init(&comp_priv->xrcd_bitmap, num, num - 1, + reserved_bot, 0); + if (ret != 0) { + ROCE_ERR("Can't initialize xrcd's bitmap, ret(%d)", ret); + return ret; + } + + return 0; +} + +static void rdma_cleanup_xrcd_table(struct rdma_comp_priv *comp_priv) +{ + rdma_bitmap_cleanup(&comp_priv->xrcd_bitmap); +} + +static int rdma_init_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + u32 i = 0; + u32 max_order = 0; /* rdmarc buddy max priority num */ + u32 qp_num = 0; + u32 rdmarc_per_qp = 0; /* rdmarc num per qp */ + u32 rdmarc_size = 0; + u32 log_rdmarc_per_seg = 0; /* min num of entry, 2^log_rdmarc_per_seg */ + u32 rdmarc_pow_of_two = 0; + + qp_num = comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qps; + rdmarc_per_qp = + comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.max_qp_dest_rdma + + ROCE_MAX_RDMA_RC_EXTEND; + rdmarc_size = + comp_priv->rdma_cap.dev_rdma_cap.roce_own_cap.rdmarc_entry_sz; + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + for (i = 1; i < qp_num * rdmarc_per_qp; i <<= 1) { + max_order++; + } + + max_order = (max_order > log_rdmarc_per_seg) ? + (max_order - log_rdmarc_per_seg) : + 0; + + ret = hmm_buddy_init(&comp_priv->rdmarc_buddy, max_order); + if (ret != 0) { + ROCE_ERR("Initialize rdmarc's buddy failed, ret(%d)", -ENOMEM); + return -ENOMEM; + } + rdmarc_pow_of_two = + roce_roundup_pow_of_two((u32)(qp_num * rdmarc_per_qp)); + ret = hmm_em_init_table(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table, rdmarc_size, + rdmarc_pow_of_two, 0, RDMA_EM_MIN_ORDER); + if (ret != 0) { + ROCE_ERR("Initialize rdmarc's em_table failed, ret(%d)", ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); + + return -ENOMEM; +} + +static int roce5_rdma_init_pd_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + ret = rdma_init_pd_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize pd's table failed, ret(%d)", ret); + return ret; + } + + return 0; +} + +static int roce5_rdma_init_mtt_table(struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + ret = hmm_mtt_init((struct hmm_comp_priv *)(void *)comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize mtt's table failed, ret(%d)", ret); + return ret; + } + + return 0; +} + +static int roce5_rdma_init_bitmap(struct rdma_comp_priv *comp_priv) +{ + int ret; + ret = roce5_rdma_init_pd_table(comp_priv); + if (ret != 0) { + return ret; + } + + ret = roce5_rdma_init_mtt_table(comp_priv); + if (ret != 0) { + rdma_cleanup_pd_table(comp_priv); + return ret; + } + + return 0; +} + +static void roce5_rdma_unit_bitmap(struct rdma_comp_priv *comp_priv) +{ + hmm_mtt_cleanup((struct hmm_comp_priv *)(void *)comp_priv); + rdma_cleanup_pd_table(comp_priv); +} + +static int roce5_rdma_init_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + int ret = 0; + + ret = roce5_rdma_init_bitmap(comp_priv); + if (ret != 0) { + return ret; + } + + ret = rdma_init_gid_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize gid table failed, ret(%d)", ret); + goto err_init_gid_table; + } + + ret = rdma_init_xrcd_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize xrcd's table failed, ret(%d)", ret); + goto err_init_xrcd; + } + + ret = rdma_init_rdmarc_table(comp_priv); + if (ret != 0) { + ROCE_ERR("Initialize rdmarc's table failed, ret(%d)", ret); + goto err_init_rdmarc_table; + } + + ROCE_INFO("Rdma init resource successful."); + return 0; + +err_init_rdmarc_table: + rdma_cleanup_xrcd_table(comp_priv); +err_init_xrcd: + rdma_cleanup_gid_table(comp_priv); +err_init_gid_table: + roce5_rdma_unit_bitmap(comp_priv); + return ret; +} + +static void rdma_cleanup_rdmarc_table(struct rdma_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table); + + hmm_buddy_cleanup(&comp_priv->rdmarc_buddy); + + return; +} + +static void roce5_rdma_unit_table(void *hwdev, struct rdma_comp_priv *comp_priv) +{ + if (hinic5_support_roce(hwdev, NULL)) { + rdma_cleanup_rdmarc_table(comp_priv); + rdma_cleanup_xrcd_table(comp_priv); + rdma_cleanup_gid_table(comp_priv); + } + + roce5_rdma_unit_bitmap(comp_priv); +} + +void roce5_rdma_cleanup_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return; + } + + if (!hinic5_support_rdma(hwdev, NULL)) { + ROCE_ERR("Not support rdma service"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + roce5_rdma_unit_table(hwdev, comp_priv); + + kfree(comp_priv); + + hinic5_unregister_service_adapter((void *)hwdev, SERVICE_T_ROCE); + + ROCE_INFO("Rdma cleanup resource successful."); + + return; +} + +static int roce5_rdma_init_comp_priv(struct rdma_comp_priv *comp_priv, + void *hwdev, + struct rdma_service_cap *rdma_cap) +{ + int ret; + + mutex_init(&comp_priv->rdma_comp_res.mutex); + comp_priv->hmm_priv.hwdev = hwdev; + + comp_priv->hmm_priv.dev = + (struct device *)(((struct hinic5_hwdev *)hwdev)->dev_hdl); + if (comp_priv->hmm_priv.dev == NULL) { + ROCE_ERR("Failed to init, device is NULL."); + return -EINVAL; + } + + ret = memcpy_s((void *)&comp_priv->rdma_cap, + sizeof(struct rdma_service_cap), (void *)rdma_cap, + sizeof(struct rdma_service_cap)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy rdma_cap.(ret:%d)", ret); + return -ENOMEM; + } + //to adapt hmm struct + comp_priv->hmm_priv.dev_cap.log_mtt = rdma_cap->log_mtt; + comp_priv->hmm_priv.dev_cap.num_mtts = rdma_cap->num_mtts; + comp_priv->hmm_priv.dev_cap.log_mtt_seg = rdma_cap->log_mtt_seg; + comp_priv->hmm_priv.dev_cap.mtt_entry_sz = rdma_cap->mtt_entry_sz; + comp_priv->hmm_priv.dev_cap.mpt_entry_sz = rdma_cap->mpt_entry_sz; + comp_priv->hmm_priv.dev_cap.dmtt_cl_start = rdma_cap->dmtt_cl_start; + comp_priv->hmm_priv.dev_cap.dmtt_cl_end = rdma_cap->dmtt_cl_end; + comp_priv->hmm_priv.dev_cap.dmtt_cl_sz = rdma_cap->dmtt_cl_sz; + + switch (g_mtt_page_size) { + case ROCE5_RDMA_MTT_PAGE_SIZE_4K: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + case ROCE5_RDMA_MTT_PAGE_SIZE_64K: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_64K; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_64K_SHIFT; + break; + case ROCE5_RDMA_MTT_PAGE_SIZE_2M: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_2M; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_2M_SHIFT; + break; + default: + comp_priv->hmm_priv.mtt_page_size = ROCE_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = + ROCE_MTT_PAGE_SIZE_4K_SHIFT; + break; + } + + /* Recalculate the gid num. */ + comp_priv->rdma_cap.max_gid_per_port = + (u32)roce5_get_gid_nums(comp_priv); + + ret = roce5_rdma_init_table(hwdev, comp_priv); + if (ret != 0) { + return ret; + } + + ret = hinic5_register_service_adapter((void *)hwdev, (void *)comp_priv, + SERVICE_T_ROCE); + if (ret != 0) { + roce5_rdma_unit_table(hwdev, comp_priv); + ROCE_ERR("put rdma_comp_res failed, ret(%d)", ret); + return ret; + } + + return 0; +} + +int roce5_rdma_init_resource(void *hwdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + if (hinic5_support_rdma(hwdev, &rdma_cap) == false) { + ROCE_INFO("Neither ROCE nor IWARP"); + return 0; + } + + comp_priv = (struct rdma_comp_priv *)kzalloc( + sizeof(struct rdma_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) { + ROCE_ERR("Alloc memory for comp_priv failed, ret(%d)", -ENOMEM); + return -ENOMEM; + } + + ret = roce5_rdma_init_comp_priv(comp_priv, hwdev, &rdma_cap); + if (ret != 0) { + kfree(comp_priv); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c new file mode 100644 index 000000000..aa252e2db --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "roce_cqm_cmd.h" +#include "hmm_common.h" +#include "securec.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_main_extension.h" +#include "roce_compat.h" +#include "roce_mr.h" +#include "rdma_comp_mw_mr.h" + +static int rdma_mpt_alloc_rsvd_lkey(struct rdma_comp_priv *comp_priv, + struct mpt *mpt) +{ + u32 mpt_entry_size = 0; + + mpt_entry_size = comp_priv->hmm_priv.dev_cap.mpt_entry_sz; + + /* call cqm interface allocate mpt, and mpt_index and vaddr save in rdma_mpt structure */ + mpt->mpt_object = (void *)cqm5_object_qpc_mpt_create( + comp_priv->hmm_priv.hwdev, RDMA_SERVICE_TYPE_ROCE, + CQM_OBJECT_MPT, mpt_entry_size, mpt, mpt->mpt_index, 0, 0); + if (mpt->mpt_object == NULL) { + ROCE_ERR("Alloc mpt_object failed, err(%d)", -ENOMEM); + return -ENOMEM; + } + + mpt->vaddr = (void *)((cqm_qpc_mpt_s *)mpt->mpt_object)->vaddr; + + return 0; +} + +int roce5_rdma_init_rsvd_lkey(struct roce5_device *rdev) +{ + struct hmm_rdma *mr = NULL; + struct rdma_service_cap *rdma_cap = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (rdev == NULL) { + ROCE_ERR("rdev is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + rdma_cap = &comp_priv->rdma_cap; + mr = &comp_priv->rsvd_lkey; + mr->mpt.mpt_index = rdma_cap->reserved_lkey >> MR_KEY_LEFT_SHIFT_OFS; + if (mr->mpt.mpt_index >= rdma_cap->reserved_mrws) { + ROCE_ERR("Cfg err, reserved_lkey(0x%x), reserved_mrws(0x%x)", + rdma_cap->reserved_lkey, rdma_cap->reserved_mrws); + return -EINVAL; + } + /* Alloc MPT */ + ret = rdma_mpt_alloc_rsvd_lkey(comp_priv, &mr->mpt); + if (ret != 0) { + ROCE_ERR("Rdma_mpt_alloc_rsvd_lkey failed, ret(%d)", ret); + return ret; + } + mr->mtt.mtt_layers = 0; + mr->iova = 0; + mr->size = ~0ULL; + mr->key = comp_priv->rdma_cap.reserved_lkey; + mr->type = roce5_mr_type2hmm(RDMA_RSVD_LKEY); + mr->access = RDMA_IB_ACCESS_LOCAL_WRITE; + /* Enable MPT */ + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, mr, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Rdma_enable_mr_mpt failed, ret(%d)", ret); + goto err_out; + } + return 0; +err_out: + hmm_mpt_free(rdev->hwdev, &mr->mpt); + return ret; +} + +void roce5_rdma_free_rsvd_lkey(struct roce5_device *rdev) +{ + struct hmm_rdma *mr = NULL; + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (rdev == NULL) { + ROCE_ERR("rdev is null"); + return; + } + + comp_priv = get_rdma_comp_priv(rdev->hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + mr = &comp_priv->rsvd_lkey; + ret = hmm_rdma_disable_mpt(rdev->lld_dev, rdev->hwdev, mr, + RDMA_SERVICE_TYPE_ROCE, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Disable mpt of mr failed, ret(%d)", ret); + return; + } + + hmm_mpt_free(rdev->hwdev, &mr->mpt); + + return; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h new file mode 100644 index 000000000..1e9c1132d --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_mw_mr.h @@ -0,0 +1,12 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef RDMA_COMP_MW_MR_H +#define RDMA_COMP_MW_MR_H + +#include "roce.h" + +int roce5_rdma_init_rsvd_lkey(struct roce5_device *rdev); +void roce5_rdma_free_rsvd_lkey(struct roce5_device *rdev); + +#endif /* RDMA_COMP_MW_MR_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c new file mode 100644 index 000000000..ee3486e97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_pd.c @@ -0,0 +1,51 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" + +int roce5_rdma_pd_alloc(void *hwdev, u32 *pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (pdn == NULL)) { + ROCE_ERR("Hwdev or pdn is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + *pdn = rdma_bitmap_alloc(&comp_priv->pd_bitmap); + if (*pdn == RDMA_INVALID_INDEX) { + ROCE_ERR("Can't get valid pdn, err(%d)", -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +void roce5_rdma_pd_free(void *hwdev, u32 pdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + rdma_bitmap_free(&comp_priv->pd_bitmap, pdn); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c new file mode 100644 index 000000000..d1265e54e --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/rdma_comp_res.c @@ -0,0 +1,140 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "rdma_comp.h" +#include "hmm_em_inner.h" +#include "hmm_buddy.h" + +static int roce5_rdma_rdmarc_alloc_check(struct rdma_comp_priv **comp_priv, + void *hwdev, + const struct rdma_rdmarc *rdmarc, + u32 *log_rdmarc_per_seg, u32 num) +{ + if ((hwdev == NULL) || (rdmarc == NULL)) { + ROCE_ERR("Hwdev or rdmarc is null"); + return -EINVAL; + } + + *comp_priv = get_rdma_comp_priv(hwdev); + if (*comp_priv == NULL || (*comp_priv)->hmm_priv.dev == NULL) { + ROCE_ERR("Comp_priv(%d) or PCI device is NULL.", + !!(*comp_priv)); + return -EINVAL; + } + + *log_rdmarc_per_seg = (*comp_priv)->rdma_cap.log_rdmarc_seg; + + if ((num < ROCE5_RDMARC_MIN_DEPTH) || (num > ROCE5_RDMARC_MAX_DEPTH)) { + ROCE_ERR("Num is invalid, ret(%d)", -EINVAL); + return -EINVAL; + } + + return 0; +} + +static void roce5_rdma_rdmarc_order_set(struct rdma_rdmarc *rdmarc, u32 order, + u32 ext_order, u32 log_rdmarc_per_seg) +{ + rdmarc->order = order; + rdmarc->ext_order = ext_order + log_rdmarc_per_seg; + return; +} + +int roce5_rdma_rdmarc_alloc(void *hwdev, u32 num, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + int ret = 0; + u32 i = 0; + u32 order = 0; + u32 ext_order = 0; + u32 ext_num = num + ROCE5_RDMARC_EXT_ENTRY; + u32 offset = 0; + void *vaddr = NULL; + u32 log_rdmarc_per_seg = 0; + u32 chip_num = (num < ROCE5_RDMARC_MIN_ENTRY) ? ROCE5_RDMARC_MIN_ENTRY : + num; + ret = roce5_rdma_rdmarc_alloc_check(&comp_priv, hwdev, rdmarc, + &log_rdmarc_per_seg, num); + if (ret != 0) { + return ret; + } + for (i = 1; i < chip_num; i <<= 1) { + order++; + } + for (i = 1; i < ext_num; i <<= 1) { + ext_order++; + } + ext_order = (ext_order > log_rdmarc_per_seg) ? + (ext_order - log_rdmarc_per_seg) : + 0; + roce5_rdma_rdmarc_order_set(rdmarc, order, ext_order, + log_rdmarc_per_seg); + offset = hmm_buddy_alloc(&comp_priv->rdmarc_buddy, ext_order); + if (offset == RDMA_INVALID_INDEX) { + ROCE_ERR("Alloc rdmarc index failed, ret(%d)", -ENOMEM); + return -ENOMEM; + } + rdmarc->offset = offset << log_rdmarc_per_seg; + + ret = hmm_em_table_get_range( + comp_priv->hmm_priv.dev, &comp_priv->rdmarc_em_table, + rdmarc->offset, rdmarc->offset + (1U << rdmarc->ext_order) - 1); + if (ret != 0) { + ROCE_ERR("Alloc rdmarc entry failed, ret(%d)", -ENOMEM); + goto err_table_get; + } + vaddr = hmm_em_table_find(&comp_priv->rdmarc_em_table, rdmarc->offset, + &rdmarc->dma_addr); + if (vaddr == NULL) { + ret = -ENOMEM; + ROCE_ERR("Can't find va and pa of rdmarc entry, ret(%d)", + -ENOMEM); + goto err_rdmarc_find; + } + rdmarc->vaddr = vaddr; + return 0; + +err_rdmarc_find: + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); +err_table_get: + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, ext_order); + return ret; +} + +void roce5_rdma_rdmarc_free(void *hwdev, struct rdma_rdmarc *rdmarc) +{ + struct rdma_comp_priv *comp_priv = NULL; + u32 order = 0; + u32 offset = 0; + u32 log_rdmarc_per_seg = 0; + + if ((hwdev == NULL) || (rdmarc == NULL)) { + ROCE_ERR("Hwdev or rdmarc is null"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL || comp_priv->hmm_priv.dev == NULL) { + ROCE_ERR("Comp_priv(%d) or comp_priv->hmm_priv.dev is NULL", + !!comp_priv); + return; + } + + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->rdmarc_em_table, rdmarc->offset, + rdmarc->offset + (1U << rdmarc->ext_order) - 1); + + log_rdmarc_per_seg = comp_priv->rdma_cap.log_rdmarc_seg; + + order = rdmarc->ext_order - log_rdmarc_per_seg; + offset = rdmarc->offset >> log_rdmarc_per_seg; + + hmm_buddy_free(&comp_priv->rdmarc_buddy, offset, order); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h new file mode 100644 index 000000000..f268757a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_com.h @@ -0,0 +1,78 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_COM_H +#define ROCE_COM_H + +#include "roce_cfg_format.h" + +/* kernel mode basic capability negotiation support macro */ +#define ROCE5_SUPPORT_IB_FEATURE(rdev, feature) \ + ((rdev)->cfg_info.feature_cap.roce_kernel_ability & \ + (ROCE_SUPPORT_IB_##feature)) + +#define ROCE5_SUPPORT_IB_CMD_RESIZE_CQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_RESIZE_CQ_MSG_V2) +#define ROCE5_SUPPORT_IB_CMD_QUERY_QP_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_QUERY_QP_MSG_V2) +#define ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_MODIFY_SRQ_MSG_V2) +#define ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_IB_FEATURE(rdev, CMD_DESTROY_SRQ_MSG_V2) + +/* kernel mode extended capability negotiation support macro */ +#define ROCE5_SUPPORT_FEATURE(rdev, feature) \ + ((rdev)->cfg_info.feature_cap.roce_kernel_ext_ability & \ + (ROCE_SUPPORT_##feature)) + +#define ROCE5_SUPPORT_CQ_MODIFY_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CQ_MODIFY_MSG_V2) +#define ROCE5_SUPPORT_CQ_CHECK_DATA_STATE_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CQ_CHECK_DATA_STATE_MSG_V2) +#define ROCE5_SUPPORT_CQ_CACHE_OUT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CQ_CACHE_OUT_MSG_V2) +#define ROCE5_SUPPORT_QP_SRQN_LB_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, QP_SRQN_LB_MSG_V2) +#define ROCE5_SUPPORT_SET_PI_ON_CHIP_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SET_PI_ON_CHIP_MSG_V2) +#define ROCE5_SUPPORT_SQ_DB_TYPE_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SQ_DB_TYPE_MSG_V2) +#define ROCE5_SUPPORT_GET_QP_RX_PORT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, GET_QP_RX_PORT_MSG_V2) +#define ROCE5_SUPPORT_MODIFY_UDP_SRC_PORT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, MODIFY_UDP_SRC_PORT_MSG_V2) +#define ROCE5_SUPPORT_GET_QP_UDP_SRC_PORT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, GET_QP_UDP_SRC_PORT_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_MPT_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_MPT_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_GID_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_GID_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_PI_CI_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_PI_CI_MSG_V2) +#define ROCE5_SUPPORT_DFX_CMD_QUERY_CQ_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, DFX_CMD_QUERY_CQ_MSG_V2) +#define ROCE5_SUPPORT_ALLOC_QPC_MSG_FEATURE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, ALLOC_QPC_MSG_FEATURE) +#define ROCE5_SUPPORT_GET_OQID_MSG_FEATURE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, GET_OQID_MSG_FEATURE) +#define ROCE5_SUPPORT_RETRY_LOAD_CQC_MSG_FEATURE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, RETRY_LOAD_CQC_MSG_FEATURE) +#define ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SET_HASH_VALUE_MSG_V2) +#define ROCE5_SUPPORT_CC_GET_DCQCN_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_DCQCN_ENABLE) +#define ROCE5_SUPPORT_CC_GET_LDCP_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_LDCP_ENABLE) +#define ROCE5_SUPPORT_CC_GET_IPQCN_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_IPQCN_ENABLE) +#define ROCE5_SUPPORT_CC_GET_USER_ALGO_ENABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, CC_GET_USER_ALGO_ENABLE) +#define ROCE5_SUPPORT_CC_PARAM_V1(rdev) ROCE5_SUPPORT_FEATURE(rdev, CC_PARAM_V1) +#define ROCE5_SUPPORT_CC_PARAM_V2(rdev) ROCE5_SUPPORT_FEATURE(rdev, CC_PARAM_V2) +#define ROCE5_SUPPORT_8K_MTU(rdev) ROCE5_SUPPORT_FEATURE(rdev, KERNEL_8K_MTU) +#define ROCE5_SUPPORT_SML_MAC_TABLE(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SML_MAC_TABLE) +#define ROCE5_SUPPORT_SQ_DB_TYPE_21(rdev) \ + ROCE5_SUPPORT_FEATURE(rdev, SQ_DB_TYPE_21) +#endif /* ROCE_COM_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c new file mode 100644 index 000000000..a15e865ec --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.c @@ -0,0 +1,1594 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <rdma/ib_mad.h> +#include <rdma/ib_verbs.h> +#include <linux/pci.h> +#include <linux/types.h> + +#include "securec.h" /* Huawei secure functions; */ + +#include "hinic5_crm.h" +#include "hinic5_srv_nic.h" +#include "hinic5_lld.h" + +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_pd.h" +#include "roce_qp.h" +#include "roce_cmd.h" +#include "roce_gid.h" +#include "roce_netdev.h" +#include "roce_main_extension.h" +#include "roce_qp_extension.h" +#include "roce_restore.h" +#include "roce_infra.h" +#include "roce_ctx_format.h" +#include "roce_kernel_compat_gen.h" +#include "roce5_gid_type.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif +#include "roce_mix.h" + +uint8_t roce5_get_db_cos_from_vlan_pri(struct roce5_device *rdev, + uint8_t vlan_pri) +{ + uint8_t db_cos; + int ret; + + ret = hinic5_get_cos_by_pri(rdev->hwdev, vlan_pri, &db_cos); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "ret:%d vlan_pri:%u", ret, vlan_pri); + db_cos = 0; + } + + return db_cos; +} + +struct net_device *roce5_ib_get_netdev(struct ib_device *ibdev, u32 port_num) +{ + struct roce5_device *rdev = NULL; + struct net_device *netdev = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return NULL; + } + + rdev = to_roce5_dev(ibdev); + +#ifdef ROCE_BONDING_EN + netdev = roce5_bond_get_netdev(rdev); + if (netdev != NULL) { + return netdev; + } +#endif + + netdev = rdev->ndev; + + dev_hold(netdev); + + return netdev; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +struct net_device *roce5_ib_get_netdev_compat(struct ib_device *ibdev, + u8 port_num) +{ + return roce5_ib_get_netdev(ibdev, (u32)port_num); +} + +static void roce5_parse_fw_version(struct roce5_device *rdev, u64 *fw_ver) +{ + int ret; + int i = 0; + struct hinic5_fw_version fw_version; + char *fw_str = (char *)fw_version.microcode_ver; + char *fw_temp = NULL; + u64 fw_verion[ROCE_FW_VERSION_LEN] = { 0 }; + + ret = hinic5_get_fw_version(rdev->hwdev, &fw_version, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_WARN("get fw version failed"); + *fw_ver = ROCE_FW_VER; + return; + } + ROCE_INFO("fw ver:%s - %s - %s", fw_version.boot_ver, + fw_version.mgmt_ver, fw_version.microcode_ver); + + while (((fw_temp = strsep(&fw_str, ".")) != NULL) && + (i < ROCE_FW_VERSION_LEN)) { + ret = sscanf_s(fw_temp, "%llu", &fw_verion[i]); + if (ret != 1) { + ROCE_WARN("parse fw version failed ret(%d)", ret); + *fw_ver = ROCE_FW_VER; + return; + } + + i++; + } + + *fw_ver = (((fw_verion[0] & 0xffffffff) + << 32) | // 0 is fw_version array idx, 32 is offset + ((fw_verion[1] & 0xffff) + << 16) | // 1 is fw_version array idx, 16 is offset + ((fw_verion[2] & 0xff) + << 8) | // 2 is fw_version array idx, 8 is offset + (fw_verion[3] & 0xff)); // 3 is fw_version array idx +} + +static void roce5_set_local_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if (((rdma_cap->flags & RDMA_BMME_FLAG_LOCAL_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_REMOTE_INV) != 0) && + ((rdma_cap->flags & RDMA_BMME_FLAG_FAST_REG_WR) != 0)) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_MGT_EXTENSIONS; + } +} + +static void roce5_set_bmme_cap_flag(const struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if ((rdma_cap->flags & RDMA_BMME_FLAG_TYPE_2_WIN) != 0) { + if ((rdma_cap->flags & RDMA_BMME_FLAG_WIN_TYPE_2B) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2B; + } else { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW_TYPE_2A; + } + } +} + +static void roce5_query_device_props_set(struct roce5_device *rdev, + struct rdma_service_cap *rdma_cap, + struct ib_device_attr *props) +{ + if (rdev->lld_dev->dev_type == HINIC5_DEVICE_T_PCI) { + struct pci_dev *pdev = + container_of(rdev->dev, struct pci_dev, dev); + props->vendor_id = pdev->vendor; + props->vendor_part_id = pdev->device; + } + roce5_parse_fw_version(rdev, &props->fw_ver); + props->hw_ver = ROCE_HW_VER; + + /* sys_image_guid equal GID */ + props->sys_image_guid = rdev->ib_dev.node_guid; + + props->max_mr_size = ~0ULL; + props->page_size_cap = rdma_cap->page_size_cap; + props->max_qp = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_qps - + rdma_cap->reserved_qps); + props->max_qp_wr = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_wqes; + /* We should ensure sq's max size(max_sq_sg) as the max_sge */ + props->max_send_sge = (int)rdma_cap->max_sq_sg; + props->max_recv_sge = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg; + props->max_cq = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs - + rdma_cap->reserved_cqs); + props->max_cqe = (int)rdma_cap->max_cqes; + props->max_mr = (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_mpts - + rdma_cap->reserved_mrws); + props->max_mw = props->max_mr; + props->max_pd = (int)(rdma_cap->num_pds - rdma_cap->reserved_pds); + props->max_qp_rd_atom = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_dest_rdma; + props->max_qp_init_rd_atom = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_qp_init_rdma; + props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; + props->max_srq = + (int)(rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs - + rdma_cap->dev_rdma_cap.roce_own_cap.reserved_srqs); + props->max_srq_wr = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_wqes; + props->max_srq_sge = + (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_srq_sge; + props->max_fast_reg_page_list_len = rdma_cap->max_frpl_len; + props->local_ca_ack_delay = (u8)rdma_cap->local_ca_ack_delay; + props->atomic_cap = + ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_ATOMIC) != 0) ? + IB_ATOMIC_HCA : + IB_ATOMIC_NONE; + props->masked_atomic_cap = props->atomic_cap; + props->max_pkeys = (u16)rdma_cap->max_pkeys; + props->max_sge_rd = (int)rdma_cap->dev_rdma_cap.roce_own_cap.max_rq_sg; +#if defined(IB_DEV_ATTR_HAVE_MAX_FMR) + props->max_fmr = props->max_map_per_fmr; +#endif +#if defined(IB_DEV_ATTR_HAVE_MAX_MAP_PER_FMR) + props->max_map_per_fmr = (int)rdma_cap->max_fmr_maps; +#endif + props->max_ah = INT_MAX; + return; +} +/* **************************************************************************** + Prototype : roce5_query_device + Description : query device attribute + Input : struct ib_device *ibdev + struct ib_device_attr *props + struct ib_udata *uhw + Output : struct ib_device_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_query_device(struct ib_device *ibdev, struct ib_device_attr *props, + struct ib_udata *uhw) +{ + struct roce5_device *rdev = NULL; + struct rdma_service_cap *rdma_cap = NULL; + int ret; + + if ((ibdev == NULL) || (props == NULL)) { + ROCE_ERR("Ibdev or props is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + ret = memset_s(props, sizeof(*props), 0, sizeof(*props)); + if (ret != 0) { + ROCE_ERR("Failed to memset props.(ret:%d)", ret); + return -ENOMEM; + } + + rdma_cap = &rdev->rdma_cap; + + props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | + IB_DEVICE_RC_RNR_NAK_GEN; + + /* APM */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_APM) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_AUTO_PATH_MIG; + } + + /* rsvd_lKey */ + if ((rdma_cap->flags & RDMA_BMME_FLAG_RESERVED_LKEY) != 0) { +#if defined(HAVE_IB_DEVICE_LOCAL_DMA_LKEY) + props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; +#elif defined(HAVE_IB_KERNEL_CAP_FLAGS) + props->kernel_cap_flags |= IBK_LOCAL_DMA_LKEY; +#endif + } + + roce5_set_local_cap_flag(rdma_cap, props); + + /* support XRC */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_XRC) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_XRC; + } + + /* support MW */ + if ((rdma_cap->flags & RDMA_DEV_CAP_FLAG_MEM_WINDOW) != 0) { + props->device_cap_flags = props->device_cap_flags | + IB_DEVICE_MEM_WINDOW; + } + + roce5_set_bmme_cap_flag(rdma_cap, props); + + roce5_query_device_props_set(rdev, rdma_cap, props); + return 0; +} + +static void roce5_eth_link_get_speed(struct ib_port_attr *props, + enum mag_cmd_port_speed speed) +{ + switch (speed) { + /* 10G <==> 1X x 10G */ + case PORT_SPEED_10GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 25G <==> 1X x 25G */ + case PORT_SPEED_25GB: + props->active_width = IB_WIDTH_1X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 40G <==> 4X x 10G */ + case PORT_SPEED_40GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_QDR; + break; + + /* 100G <==> 4X x 25G */ + case PORT_SPEED_100GB: + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 200G <==> 8X x 25G */ + case PORT_SPEED_200GB: + props->active_width = IB_WIDTH_8X; + props->active_speed = IB_SPEED_EDR; + break; + + /* 400G <==> 8X x 50G */ + case PORT_SPEED_400GB: + props->active_width = IB_WIDTH_8X; + props->active_speed = IB_SPEED_HDR; + break; + + default: + props->active_width = 0; + props->active_speed = 0; + break; + } +} + +static void roce5_set_ib_port_attr(struct ib_port_attr *props, + struct roce5_device *rdev) +{ + struct rdma_comp_priv *comp_priv = get_rdma_comp_priv(rdev->hwdev); + + if (comp_priv == NULL) { + ROCE_ERR("comp_priv is NULL"); + return; + } + + props->port_cap_flags = IB_PORT_CM_SUP; + props->gid_tbl_len = (int)comp_priv->rdma_cap.max_gid_per_port; + props->max_msg_sz = rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz; + props->pkey_tbl_len = (u16)rdev->rdma_cap.max_pkeys; + props->max_mtu = (enum ib_mtu)((ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? + ROCE_MTU_8192 : + ROCE_MTU_4096); + props->state = IB_PORT_DOWN; + props->phys_state = ROCE_PORT_PHYS_STATE_DISABLED; + props->active_mtu = IB_MTU_256; +} + +static inline int roce5_get_port_speed_by_rdev(struct roce5_device *rdev, + enum mag_cmd_port_speed *speed) +{ + enum mag_cmd_port_speed speed_table[] = { + [ROCE5_4_25G_HCA] = PORT_SPEED_25GB, + [ROCE5_2_25G_HCA] = PORT_SPEED_25GB, + [ROCE5_1_100G_HCA] = PORT_SPEED_100GB, + [ROCE5_2_100G_HCA] = PORT_SPEED_100GB, + [ROCE5_2_200G_HCA] = PORT_SPEED_200GB, + [ROCE5_2_400G_HCA] = PORT_SPEED_400GB, + [ROCE5_4_200G_HCA] = PORT_SPEED_200GB, + }; + int hca_type = rdev->hw_info.hca_type; + + if (hca_type >= ARRAY_SIZE(speed_table) || + speed_table[hca_type] == PORT_SPEED_NOT_SET) { + return -EINVAL; + } + + *speed = speed_table[hca_type]; + return 0; +} + +static void roce5_eth_link_query_port(struct ib_device *ibdev, + struct ib_port_attr *props) +{ + struct roce5_device *rdev = NULL; + struct net_device *netdev = NULL; +#ifdef ROCE_BONDING_EN + struct net_device *upper = NULL; +#endif + enum roce5_mtu rmtu; + enum mag_cmd_port_speed speed = PORT_SPEED_10GB; + int ret = 0; + + rdev = to_roce5_dev(ibdev); + + roce5_set_ib_port_attr(props, rdev); + + ret = roce5_get_port_speed_by_rdev(rdev, &speed); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to get speed, func_id(%d), hca_type(%d)", + rdev->glb_func_id, rdev->hw_info.hca_type); + props->active_width = 0; + props->active_speed = 0; + return; + } + + roce5_eth_link_get_speed(props, speed); + + netdev = roce5_ib_get_netdev(ibdev, ROCE_DEFAULT_PORT_NUM); + if (netdev == NULL) { + return; + } + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + rcu_read_lock(); + upper = netdev_master_upper_dev_get_rcu(netdev); + if (upper != NULL) { + dev_put(netdev); + netdev = upper; + dev_hold(netdev); + } + rcu_read_unlock(); + } +#endif + + if (netif_running(netdev) && netif_carrier_ok(netdev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = ROCE_PORT_PHYS_STATE_LINKUP; + } + + rmtu = (ROCE5_SUPPORT_8K_MTU(rdev) != 0) ? + roce5_get_mtu((int)netdev->mtu) : + (enum roce5_mtu)iboe_get_mtu((int)netdev->mtu); + + dev_put(netdev); + + props->active_mtu = + (enum ib_mtu)ROCE_MIN((enum roce5_mtu)props->max_mtu, rmtu); +} + +/* **************************************************************************** + Prototype : roce5_query_port + Description : query port attribute + Input : struct ib_device *ibdev + u8 port + struct ib_port_attr *props + Output : struct ib_port_attr *props + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props) +{ + int ret; + if ((ibdev == NULL) || (props == NULL)) { + ROCE_ERR("Ibdev or props is null"); + return -EINVAL; + } + + ret = memset_s(props, sizeof(*props), 0, sizeof(*props)); + if (ret != 0) { + ROCE_ERR("Failed to memset props.(ret:%d)", ret); + return -ENOMEM; + } + + roce5_eth_link_query_port(ibdev, props); + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_query_port_compat(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) +{ + return roce5_query_port(ibdev, (u32)port, props); +} + +/* **************************************************************************** + Prototype : roce5_query_gid + Description : query gid + Input : struct ib_device *ibdev + u8 port + int index + union ib_gid *gid + Output : union ib_gid *gid + + 1.Date : 2015/5/8 + Modification : Created function + + 2.Date : 2015/6/8 + Modification : Modify function + +**************************************************************************** */ +int roce5_query_gid(struct ib_device *ibdev, u32 port, int index, + union ib_gid *gid) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + struct rdma_gid_entry gid_entry; + + if ((ibdev == NULL) || (gid == NULL)) { + ROCE_ERR("Ibdev or gid is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + ret = roce5_get_gid(rdev, port, (u32)index, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = memcpy_s((void *)gid->raw, sizeof(*gid), (void *)gid_entry.raw, + sizeof(*gid)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy gid.(ret:%d)", ret); + return -ENOMEM; + } + + // per per OFED of gid generation method convert GID, only IPv4 scenario need convert + if (gid_entry.dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + // not add of gid returns directly, no need to convert + if ((gid->global.subnet_prefix == 0) && + (gid->global.interface_id == 0)) { + return 0; + } + ipv6_addr_set_v4mapped(*((u32 *)(void *)gid + ROCE_GID_IP_IDX), + (struct in6_addr *)gid); + } + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_query_gid_compat(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid) +{ + return roce5_query_gid(ibdev, (u32)port, index, gid); +} + +/* **************************************************************************** + Prototype : roce5_query_pkey + Description : query pkey + Input : struct ib_device *ibdev + u8 port + u16 index + u16 *pkey + Output : u16 *pkey + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) +{ + struct roce5_device *rdev = NULL; + if ((ibdev == NULL) || (pkey == NULL)) { + ROCE_ERR("Ibdev or pkey is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + if (pkey == NULL) { + ROCE_ERR("Pkey is null"); + return -EINVAL; + } + + *pkey = 0xffff; + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_query_pkey_compat(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey) +{ + return roce5_query_pkey(ibdev, (u32)port, index, pkey); +} + +/* **************************************************************************** + Prototype : roce5_modify_device + Description : modify device attribute + Input : struct ib_device *ibdev + int mask + struct ib_device_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_device(struct ib_device *ibdev, int mask, + struct ib_device_modify *props) +{ + unsigned long flags = 0; /* type required by kernel API */ + struct roce5_device *rdev = NULL; + int ret; + + if ((ibdev == NULL) || (props == NULL)) { + ROCE_ERR("Ibdev or props is null"); + return -EINVAL; + } + + if (((unsigned int)mask & ~IB_DEVICE_MODIFY_NODE_DESC) != 0) { + ROCE_ERR("Not supported to modify node description"); + return -EOPNOTSUPP; + } + + if ((((u32)mask) & IB_DEVICE_MODIFY_NODE_DESC) == 0) { + ROCE_INFO("No need to modify node description"); + return 0; + } + + rdev = to_roce5_dev(ibdev); + + spin_lock_irqsave(&rdev->node_desc_lock, flags); + ret = memcpy_s((void *)ibdev->node_desc, IB_DEVICE_NODE_DESC_MAX, + (void *)props->node_desc, IB_DEVICE_NODE_DESC_MAX); + if (ret != 0) { + ROCE_ERR("Failed to memcpy node_desc.(ret:%d)", ret); + spin_unlock_irqrestore(&rdev->node_desc_lock, flags); + return -ENOMEM; + } + spin_unlock_irqrestore(&rdev->node_desc_lock, flags); + + return 0; +} + +/* **************************************************************************** + Prototype : roce5_modify_port + Description : modify port attribute + Input : struct ib_device *ibdev + u8 port + int mask + struct ib_port_modify *props + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_port(struct ib_device *ibdev, u32 port, int mask, + struct ib_port_modify *props) +{ + int ret = 0; + struct ib_port_attr attr; + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + + ret = memset_s(&attr, sizeof(struct ib_port_attr), 0, + sizeof(struct ib_port_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset attr.(ret:%d)", ret); + return -ENOMEM; + } + + mutex_lock(&rdev->cap_mask_mutex); + + ret = roce5_query_port(ibdev, port, &attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query port, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + mutex_unlock(&rdev->cap_mask_mutex); + + return ret; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_modify_port_compat(struct ib_device *ibdev, u8 port, int mask, + struct ib_port_modify *props) +{ + return roce5_modify_port(ibdev, (u32)port, mask, props); +} + +static int roce5_alloc_ucontext_set(struct roce5_device *rdev, + struct roce5_alloc_ucontext_resp *resp) +{ + struct rdma_service_cap *rdma_cap = NULL; + + rdma_cap = &rdev->rdma_cap; + + resp->num_qps = rdma_cap->dev_rdma_cap.roce_own_cap.max_qps; + resp->num_xsrqs = rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs; + resp->cqe_size = rdma_cap->cqe_size; + resp->wqebb_size = rdma_cap->wqebb_size; + resp->dwqe_size = rdma_cap->direct_wqe_size; + resp->max_msg_size = rdma_cap->dev_rdma_cap.roce_own_cap.max_msg_sz; + resp->max_comp_vector = rdma_cap->num_comp_vectors; + resp->max_inline_size = + rdma_cap->dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz; + + resp->storage_aa_en = 0; + resp->phy_port = rdev->hw_info.phy_port; + resp->srq_container_en = rdev->cfg_info.srq_container_en; + resp->srq_container_mode = rdev->cfg_info.srq_container_mode; + resp->xrc_srq_container_mode = rdev->cfg_info.xrc_srq_container_mode; + resp->warn_th = rdev->cfg_info.warn_th; + resp->func_id = rdev->glb_func_id; + resp->pcie_atomic_en = rdev->cfg_info.pcie_atomic_en; + resp->roce_user_ext_ability = + rdev->cfg_info.feature_cap.roce_user_ext_ability; + resp->roce_uld_ext_ability = + rdev->cfg_info.feature_cap.uld_feature.roce_uld_user_ext_ability; + resp->max_send_sge = (uint8_t)rdev->rdma_cap.max_sq_sg; + resp->max_recv_sge = + (uint8_t)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg; + resp->max_sq_wqe_len = rdev->rdma_cap.max_sq_desc_sz; + resp->huge_page_en = rdev->huge_page_en; + return hinic5_get_chip_node_id(rdev->lld_dev, &(resp->card_id)); +} + +static int roce5_alloc_ucontext_pre_check(struct ib_device *ibdev, + const struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + + if ((ibdev == NULL) || (udata == NULL)) { + ROCE_ERR("Ibdev or udata is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibdev); + if (!rdev->ib_active) { + ROCE_DEV_ERR(rdev, "Device is abnormal, func_id(%d)", + rdev->glb_func_id); + return -EAGAIN; + } + + return 0; +} + +static int roce5_alloc_ucontext_return(struct roce5_device *rdev, + struct ib_udata *udata, + struct roce5_ucontext *context, + struct roce5_alloc_ucontext_resp *resp) +{ + int ret; + + resp->db_offset = context->db_dma_addr & ((1 << PAGE_SHIFT) - 1); +#ifdef ROCE_DWQE_EN + resp->dwqe_offset = context->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1); + + if (context->dwqe_dma_addr == 0) { + ROCE_DEV_ERR(rdev, "dwqe_dma_addr = 0"); + resp->dwqe_size = 0; + } +#else + resp->dwqe_size = 0; +#endif + + INIT_LIST_HEAD(&context->db_page_list); + mutex_init(&context->db_page_mutex); + + /* Copy data to user space */ + ret = ib_copy_to_udata(udata, resp, + sizeof(struct roce5_alloc_ucontext_resp)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to copy data to user space, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + return 0; +} + +int roce5_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) +{ + int ret; + + struct roce5_ucontext *context = to_roce5_ucontext(ibucontext); + struct roce5_device *rdev = to_roce5_dev(ibucontext->device); + struct roce5_alloc_ucontext_resp *resp = NULL; + + ret = roce5_alloc_ucontext_pre_check(ibucontext->device, udata); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "pre check err, ret(%d), func_id(%d)", ret, + rdev->glb_func_id); + goto err; + } + + resp = kzalloc(sizeof(struct roce5_alloc_ucontext_resp), GFP_KERNEL); + if (resp == NULL) { + ret = (-ENOMEM); + ROCE_DEV_ERR(rdev, "Failed alloc ucontext, func_id(%d)", + rdev->glb_func_id); + goto err; + } + + ret = roce5_alloc_ucontext_set(rdev, resp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get chip node id, func_id(%d)", + rdev->glb_func_id); + goto err_db; + } + /* Alloc user space context Doorbell and DWQE */ + ret = hinic5_alloc_db_phy_addr(rdev->hwdev, &context->db_dma_addr, + &context->dwqe_dma_addr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc DB pa, ret(%d), func_id(%d)", ret, + rdev->glb_func_id); + goto err_db; + } + + /* Copy data to user space */ + ret = roce5_alloc_ucontext_return(rdev, udata, context, resp); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to return to user, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto err_return; + } + + kfree(resp); +#ifdef __ROCE_DFX__ + rdev->ucontext = context; +#endif + return 0; + +err_return: + hinic5_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, + context->dwqe_dma_addr); +err_db: + kfree(resp); +err: + return ret; +} + +void roce5_dealloc_ucontext(struct ib_ucontext *ibcontext) +{ + struct roce5_ucontext *context = NULL; + struct roce5_device *rdev = NULL; + + if (ibcontext == NULL) { + ROCE_ERR("Ibcontext is null"); + return; + } + + context = to_roce5_ucontext(ibcontext); + rdev = to_roce5_dev(ibcontext->device); + + hinic5_free_db_phy_addr(rdev->hwdev, context->db_dma_addr, + context->dwqe_dma_addr); + + return; +} + +struct ib_ucontext *roce5_alloc_ucontext_ofa(struct ib_device *ibdev, + struct ib_udata *udata) +{ + int ret; + struct roce5_ucontext *context = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return ERR_PTR(-EINVAL); + } + + context = kzalloc(sizeof(struct roce5_ucontext), GFP_KERNEL); + if (context == NULL) { + ROCE_ERR("Failed to alloc ucontext"); + return ERR_PTR(-ENOMEM); + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + context->ibucontext.device = ibdev; + ret = roce5_alloc_ucontext(&context->ibucontext, udata); + if (ret != 0) { + kfree(context); + return ERR_PTR(ret); + } + + return (struct ib_ucontext *)&context->ibucontext; +} + +/* **************************************************************************** + Prototype : roce5_dealloc_ucontext_ofa + Description : dealloc ucontext + Input : struct ib_ucontext *ibcontext + Output : None + + 1.Date : 2023/11/9 + Modification : Created function + +**************************************************************************** */ +int roce5_dealloc_ucontext_ofa(struct ib_ucontext *ibcontext) +{ + struct roce5_ucontext *context = NULL; + + if (ibcontext == NULL) { + ROCE_ERR("Ibcontext is null"); + return -EINVAL; + } + + roce5_dealloc_ucontext(ibcontext); + context = to_roce5_ucontext(ibcontext); + + kfree(context); + return 0; +} + +/* **************************************************************************** + Prototype : roce5_mmap + Description : memory map + Input : struct ib_ucontext *ibcontext + struct vm_area_struct *vma + Output : None + + 1.Date : 2015/5/8 + Modification : Created function + +**************************************************************************** */ +int roce5_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) +{ + struct roce5_device *rdev = NULL; + struct roce5_ucontext *ucontext = NULL; + unsigned long db_pfn = 0; + unsigned long dwqe_pfn = 0; + + if ((ibcontext == NULL) || (vma == NULL)) { + ROCE_ERR("Ibcontext or vma is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibcontext->device); + ucontext = to_roce5_ucontext(ibcontext); + db_pfn = ucontext->db_dma_addr >> PAGE_SHIFT; + dwqe_pfn = ucontext->dwqe_dma_addr >> PAGE_SHIFT; + + if ((vma->vm_end - vma->vm_start) != PAGE_SIZE) { + ROCE_DEV_ERR( + rdev, + "(Vm_end(%lu) - vm_start(%lu)) is not equal to PAGE_SIZE, func_id(%d)", + vma->vm_end, vma->vm_start, rdev->glb_func_id); + return -EINVAL; + } + + /* map hw DB to physical page from user */ + if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* construct vm_start~vm_start+PAGE_SIZE page table + * db_pfn is page number + * vm_page_prot means attr + */ + if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to do db io remap, func_id(%d)", + rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } + +#ifdef ROCE_DWQE_EN + if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && + (rdev->rdma_cap.direct_wqe_size != 0)) { // DWQE mmap +#ifdef __aarch64__ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + + if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to do dwqe io remap, func_id(%d)", + rdev->glb_func_id); + return -EAGAIN; + } + return 0; + } +#endif + + ROCE_DEV_ERR(rdev, "Not support, func_id(%d)", rdev->glb_func_id); + return -EINVAL; +} + +enum rdma_link_layer roce5_port_link_layer(struct ib_device *ibdev, + u32 port_num) +{ + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + return IB_LINK_LAYER_UNSPECIFIED; + } + + rdev = to_roce5_dev(ibdev); + + if (port_num != ROCE_DEFAULT_PORT_NUM) { + return IB_LINK_LAYER_UNSPECIFIED; + } + + return IB_LINK_LAYER_ETHERNET; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +enum rdma_link_layer roce5_port_link_layer_compat(struct ib_device *ibdev, + u8 port_num) +{ + return roce5_port_link_layer(ibdev, (u32)port_num); +} + +static void roce5_resolve_cb(int status, struct sockaddr *src_addr, + struct rdma_dev_addr *addr, void *context) +{ + ((struct roce5_resolve_cb_context *)context)->status = status; + complete(&((struct roce5_resolve_cb_context *)context)->comp); +} + +static int roce5_rdma_addr_find_l2_eth_by_grh( + const union ib_gid *sgid, const union ib_gid *dgid, u8 *dmac, + const struct net_device *ndev, int *hoplimit, struct roce5_device *rdev, + const struct ib_gid_attr *sgid_attr) +{ + struct rdma_dev_addr dev_addr; + struct roce5_resolve_cb_context ctx; + union { + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } sgid_addr, dgid_addr; + int ret; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid); + rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid); + + ret = memset_s(&dev_addr, sizeof(dev_addr), 0, sizeof(dev_addr)); + if (ret != 0) { + ROCE_ERR("Failed to memset dev_addr.(ret:%d)", ret); + return -ENOMEM; + } + + if (ndev) { + dev_addr.bound_dev_if = ndev->ifindex; + dev_addr.net = dev_net(ndev); + } else { + dev_addr.net = &init_net; + } + + init_completion(&ctx.comp); +#if defined(RDMA_RESOLVE_IP_HAVE_GID_ATTR) + dev_addr.sgid_attr = sgid_attr; + ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr, + (struct sockaddr *)&dgid_addr, &dev_addr, + RESOLVE_IP_TIME_OUT, roce5_resolve_cb, true, + &ctx); +#else + ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr, + (struct sockaddr *)&dgid_addr, &dev_addr, + RESOLVE_IP_TIME_OUT, roce5_resolve_cb, &ctx); +#endif + if (ret != 0) { + ROCE_ERR("rdma_resolve_ip failed. Igonore the err(%d).", ret); + roce5_resolve_cb(0, (struct sockaddr *)&sgid_addr, &dev_addr, + &ctx); + } + + wait_for_completion(&ctx.comp); + + ret = memcpy_s(dmac, ETH_ALEN, dev_addr.dst_dev_addr, ETH_ALEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac.(ret:%d)", ret); + return -ENOMEM; + } + if (hoplimit) { + *hoplimit = dev_addr.hoplimit; + } + + return 0; +} + +static int roce5_ah_valid_check(struct ib_global_route *grh, u16 *vlan_id, + u8 *dmac) +{ + u8 unicast_gid0[ROCE_GID_LEN] = { 0 }; + u8 unicast_gid1[ROCE_GID_LEN] = { 0 }; + + /* check gid(unicast gid can not be 0 or 1) */ + unicast_gid0[ROCE_GID_HIGHEST_BYTE] = 0; + unicast_gid1[ROCE_GID_HIGHEST_BYTE] = 1; + + if ((ROCE_MEMCMP(grh->dgid.raw, unicast_gid0, sizeof(union ib_gid)) == + 0) || + (ROCE_MEMCMP(grh->dgid.raw, unicast_gid1, sizeof(union ib_gid)) == + 0)) { + ROCE_ERR( + "Invalid unicast dgid, The ROCE_GID_HIGHEST_BYTE of unicast_dgid is: 0x%02x", + *(grh->dgid.raw + ROCE_GID_HIGHEST_BYTE)); + return (-EINVAL); + } + + if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) != 0) { + rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw, dmac); + *vlan_id = ROCE_DEFAULT_VLAN_ID; + } + + return 0; +} + +static int roce5_fill_gid_attr(struct roce5_device *rdev, + struct rdma_ah_attr *ah_attr, union ib_gid *sgid, + const struct ib_gid_attr **sgid_attr) +{ + int ret = 0; + + ret = rdma_query_gid(&rdev->ib_dev, ah_attr->port_num, + ah_attr->grh.sgid_index, sgid); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to query gid func_id(%hu),port_num(%d),gid_index(%d),ret(%d)", + rdev->glb_func_id, ah_attr->port_num, + ah_attr->grh.sgid_index, ret); + return ret; + } + + *sgid_attr = rdma_get_gid_attr(&rdev->ib_dev, ah_attr->port_num, + ah_attr->grh.sgid_index); + if (IS_ERR_OR_NULL(*sgid_attr)) { + ret = (int)PTR_ERR(*sgid_attr); + ROCE_PRINT_GID(rdev, sgid->raw, + "Failed to get sgid_attr, func_id(%hu), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + return ret; +} + +static void roce5_release_gid_ref_cnt(const struct ib_gid_attr *sgid_attr) +{ + rdma_put_gid_attr(sgid_attr); +} + +static struct net_device *roce5_fill_netdev(struct roce5_device *rdev, + union ib_gid *sgid) +{ + struct net_device *netdev = NULL; + union { + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } socket_addr; + + rdma_gid2ip((struct sockaddr *)&socket_addr, sgid); + + /* find netdev,rdev->ndevis not valid in vlan scenario */ + netdev = ip_dev_find(&init_net, + socket_addr._sockaddr_in.sin_addr.s_addr); + if (netdev) { + dev_put(netdev); + } + + return netdev; +} + +int roce5_resolve_grh(struct roce5_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata) +{ + int ret = 0; + u8 zero_mac[ETH_ALEN] = { 0 }; + u8 *dmac = NULL; + union ib_gid sgid = { 0 }; + const struct ib_gid_attr *sgid_attr = NULL; + struct net_device *netdev = NULL; + + if ((rdev == NULL) || (ah_attr == NULL) || (vlan_id == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + dmac = ah_attr->roce.dmac; + ret = roce5_ah_valid_check(&ah_attr->grh, vlan_id, dmac); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to check grh input, func_id(%d), ret(%d)," + "The grh inputs are: vlan_id(%d), dmac:%02x-%02x-%02x-%02x-%02x-%02x", + rdev->glb_func_id, ret, *vlan_id, + *((u8 *)dmac + MAC_INDEX_0), + *((u8 *)dmac + MAC_INDEX_1), + *((u8 *)dmac + MAC_INDEX_2), + *((u8 *)dmac + MAC_INDEX_3), + *((u8 *)dmac + MAC_INDEX_4), + *((u8 *)dmac + MAC_INDEX_5)); + return ret; + } + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) != 0) { + return 0; + } + + ret = roce5_fill_gid_attr(rdev, ah_attr, &sgid, &sgid_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to fill gid attr, func_id(%hu), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + netdev = roce5_fill_netdev(rdev, &sgid); + + /* reparse dmac avoiding invalid damc from OFED */ + ret = roce5_rdma_addr_find_l2_eth_by_grh( + &sgid, &ah_attr->grh.dgid, dmac, netdev, NULL, rdev, sgid_attr); + if (ret != 0) { + ROCE_PRINT_GID( + rdev, sgid.raw, + "Failed to find dmac by grh, func_id(%hu), ret(%d)", + rdev->glb_func_id, ret); + goto resolve_grh_end; + } + + if (ROCE_MEMCMP(dmac, zero_mac, ETH_ALEN) == 0) { + ROCE_DEV_ERR(rdev, "Failed to find valid dmac, func_id(%hu)", + rdev->glb_func_id); + ret = (-EINVAL); + goto resolve_grh_end; + } + + if (sgid_attr != NULL && sgid_attr->ndev != NULL) { + *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev); + } + +resolve_grh_end: + roce5_release_gid_ref_cnt(sgid_attr); + + return ret; +} + +static int roce5_ah_get_vlan_id(struct roce5_device *rdev, struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, u32 *vlan_id) +{ + u32 ret = 0; + struct net_device *ndev = NULL; + rcu_read_lock(); + ndev = rcu_dereference(ah_attr->grh.sgid_attr->ndev); + if (ndev == NULL) { + rcu_read_unlock(); + ROCE_DEV_ERR(rdev, "Net device is NULL, func_id(%hu)", + rdev->glb_func_id); + return -EINVAL; + } + ret = rdma_vlan_dev_vlan_id(ndev); + if (ret != 0xffff) { + *vlan_id = ret; + } + rcu_read_unlock(); + return 0; +} + +static int roce5_create_ib_ah_kernel(struct roce5_device *rdev, + struct ib_pd *pd, struct roce5_ah *rah, + struct rdma_ah_attr *ah_attr) +{ + int ret; + u8 *dmac = ah_attr->roce.dmac; + u32 vlan_id = 0; + + ret = roce5_ah_get_vlan_id(rdev, pd, ah_attr, &vlan_id); + if (ret != 0) { + ROCE_ERR("Failed to get vlan_id (ret:%d)", ret); + return -EFAULT; + } + + if (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) != 0) { + ret = memcpy_s((void *)rah->priv_ah.dgid, + sizeof(rah->priv_ah.dgid), + (void *)ah_attr->grh.dgid.raw, + sizeof(rah->priv_ah.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return -ENOMEM; + } + rah->priv_ah.dw2.bs.flow_label = ah_attr->grh.flow_label & + 0xfffff; + rah->priv_ah.dw1.bs.sgid_index = ah_attr->grh.sgid_index & 0x7f; + rah->priv_ah.dw1.bs.hoplimit = ah_attr->grh.hop_limit; + rah->priv_ah.dw1.bs.tclass = + (u8)(ah_attr->grh.traffic_class | IP_ECN_PERMIT); + } + rah->priv_ah.dw0.bs.pd = to_roce5_pd(pd)->pdn & 0x3ffff; + rah->priv_ah.dw0.bs.wqe_cos = + roce5_get_db_cos_from_vlan_pri(rdev, ah_attr->sl); + rah->priv_ah.dw0.value = cpu_to_be32(rah->priv_ah.dw0.value); + + rah->priv_ah.dw1.bs.port = ah_attr->port_num & 0xf; + rah->priv_ah.dw2.bs.smac_index = + rdev->glb_func_id; /* set global Function ID */ + rah->priv_ah.dw2.value = cpu_to_be32(rah->priv_ah.dw2.value); + + rah->priv_ah.dw1.bs.resv = 0; + rah->priv_ah.dw7.bs.vlan_id = vlan_id & 0xfff; + rah->priv_ah.dw7.bs.vlan_pri = ah_attr->sl & 0x7; + + rah->priv_ah.dw1.value = cpu_to_be32(rah->priv_ah.dw1.value); + + rah->priv_ah.dw7.bs.dmac_h16 = (dmac[0] << ROCE_RAH_DMAC_H16_SHIFT) | + dmac[1]; + rah->priv_ah.dw7.value = cpu_to_be32(rah->priv_ah.dw7.value); + + ret = memcpy_s((void *)&rah->priv_ah.dmac_l32, + sizeof(rah->priv_ah.dmac_l32), + (void *)&dmac[ROCE_RAH_DMAC_L32_START], + sizeof(rah->priv_ah.dmac_l32)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dmac_l32.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} + +static int roce5_create_ib_ah_user(struct roce5_device *rdev, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata) +{ + struct roce5_create_ah_resp resp; + roce5_get_cos_inbuf_s dcb_inbuf; + int ret; + + ret = memset_s(ah_attr->roce.dmac, sizeof(ah_attr->roce.dmac), 0, + sizeof(ah_attr->roce.dmac)); + if (ret != 0) { + ROCE_ERR("Failed to memset dmac."); + return (-ENOMEM); + } + + ret = roce5_resolve_grh(rdev, ah_attr, &resp.vlan_id, udata); + if (ret != 0) { + ROCE_ERR("Failed to resolve grh"); + return -EINVAL; + } + + ret = memcpy_s(resp.dmac, ETH_ALEN, ah_attr->roce.dmac, ETH_ALEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy_s ah dmac."); + return (-ENOMEM); + } + + dcb_inbuf.sl = ah_attr->sl; + dcb_inbuf.sgid_index = ah_attr->grh.sgid_index; + dcb_inbuf.port_num = ah_attr->port_num; + dcb_inbuf.traffic_class = ah_attr->grh.traffic_class; + ret = roce5_get_dcb_cfg_cos(rdev, &dcb_inbuf, &resp.cos, IB_QPT_UD); + if (ret != 0) { + ROCE_ERR("Failed to get cos from dcb info, ret:%d", ret); + return ret; + } + + ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (ret != 0) { + ROCE_ERR("Failed to copy data to user"); + return ret; + } + + return 0; +} + +int roce5_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_ah *rah = NULL; + struct roce5_device *rdev = NULL; + struct rdma_ah_attr *ah_attr = NULL; + enum rdma_ah_attr_type ah_type = 0; + + if ((ibah == NULL) || (init_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rah = to_roce5_ah(ibah); + rdev = to_roce5_dev(ibah->device); + ah_attr = init_attr->ah_attr; + ah_type = ah_attr->type; + + if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && + (((u32)rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) == 0)) { + ROCE_DEV_ERR(rdev, "Invalid ah_type(%d)", ah_type); + return -EINVAL; + } + + if (udata != NULL) { + ret = roce5_create_ib_ah_user(rdev, ah_attr, udata); + } else { + ret = roce5_create_ib_ah_kernel(rdev, ibah->pd, rah, ah_attr); + } + return ret; +} + +struct ib_ah *roce5_create_ah_ofa(struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_ah *rah = NULL; + struct roce5_device *rdev = NULL; + struct rdma_ah_init_attr init_attr; + + if ((pd == NULL) || (ah_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + ret = -EINVAL; + goto out; + } + + rdev = to_roce5_dev(pd->device); + init_attr.ah_attr = ah_attr; + + rah = kzalloc(sizeof(*rah), GFP_ATOMIC); + if (rah == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, "Failed to alloc ah, func_id(%d)", + rdev->glb_func_id); + goto out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + rah->ibah.device = pd->device; + rah->ibah.pd = pd; + ret = roce5_create_ah(&rah->ibah, &init_attr, udata); + if (ret != 0) { + kfree(rah); + } + +out: + return (ret != 0) ? ERR_PTR(ret) : &rah->ibah; +} + +int roce5_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ + struct roce5_ah *ah = NULL; + struct roce5_priv_ah priv_ah; + int ret; + + if ((ibah == NULL) || (ah_attr == NULL)) { + ROCE_ERR("Ibah or ah_attr is null"); + return -EINVAL; + } + + ah = to_roce5_ah(ibah); + ret = memset_s(ah_attr, sizeof *ah_attr, 0, sizeof *ah_attr); + if (ret != 0) { + ROCE_ERR("Failed to memset ah_attr.(ret:%d)", ret); + return -ENOMEM; + } + + priv_ah.dw1.value = be32_to_cpu(ah->priv_ah.dw1.value); + priv_ah.dw2.value = be32_to_cpu(ah->priv_ah.dw2.value); + priv_ah.dw7.value = be32_to_cpu(ah->priv_ah.dw7.value); + + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->sl = priv_ah.dw7.bs.vlan_pri; + ah_attr->port_num = priv_ah.dw1.bs.port; + ah_attr->grh.traffic_class = priv_ah.dw1.bs.tclass; + ah_attr->grh.hop_limit = priv_ah.dw1.bs.hoplimit; + ah_attr->grh.sgid_index = priv_ah.dw1.bs.sgid_index; + ah_attr->grh.flow_label = priv_ah.dw2.bs.flow_label; + + ret = memcpy_s((void *)ah_attr->grh.dgid.raw, sizeof(ah->priv_ah.dgid), + (void *)ah->priv_ah.dgid, sizeof(ah->priv_ah.dgid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy dgid.(ret:%d)", ret); + return -ENOMEM; + } + return 0; +} + +int roce5_destroy_ah(struct ib_ah *ibah, u32 flags) +{ + return 0; +} + +int roce5_destroy_ah_ofa(struct ib_ah *ibah) +{ + struct roce5_ah *rah = NULL; + + if (ibah == NULL) { + ROCE_ERR("Ibah is null"); + return -EINVAL; + } + + rah = to_roce5_ah(ibah); + + kfree(rah); + + return 0; +} + +int roce5_port_immutable(struct ib_device *ibdev, u32 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + + if ((ibdev == NULL) || (immutable == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + immutable->core_cap_flags = + RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; // only rocev2 + +#if defined(IB_QUERY_PORT_HAVE_U8_PORT) + err = ib_query_port(ibdev, (u8)port_num, &attr); +#elif defined(IB_QUERY_PORT_HAVE_U32_PORT) + err = ib_query_port(ibdev, port_num, &attr); +#endif + if (err != 0) { + ROCE_ERR("dev(%s) query ib port(%d) failed, err(%d)", + ibdev->name, port_num, err); + return err; + } + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + + return 0; +} + +/* compatible version for kernel 5.10 and kernel 4.19 */ +int roce5_port_immutable_compat(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + return roce5_port_immutable(ibdev, (u32)port_num, immutable); +} + +#define ROCE_DEFAULT_COS 3 + +int roce5_get_dcb_cfg_cos(struct roce5_device *rdev, + roce5_get_cos_inbuf_s *inbuf, u8 *cos, + enum ib_qp_type qp_type) +{ + int ret; + u8 pri; + struct rdma_gid_entry gid; + struct hinic5_dcb_state dcb = { 0 }; + +#if !defined(UVERBS_IOCTL_HAVE_DECLARE_UVERBS_WRITE_EX) + /* Note: low version of ofed in modify qp when unused udate->outbuf, cause cannot from kernel mode get db cos, therefore stub uses a fixed value. */ + *cos = ROCE_DEFAULT_COS; + goto cos_success; +#endif + + ret = roce5_get_gid(rdev, inbuf->port_num, inbuf->sgid_index, &gid); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init gid info, ret(%d).", ret); + return -EINVAL; + } + + ret = hinic5_get_dcb_state(rdev->hwdev, &dcb); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "hinic5_get_dcb_state failed, ret(%d).", + ret); + return -EINVAL; + } + + *cos = dcb.default_cos; + gid.dw6_h.value = cpu_to_le16(gid.dw6_h.value); + if ((dcb.trust == ROCE5_DCB_PCP) && + (gid.dw6_h.bs.tag == ROCE_GID_VLAN_INVALID)) { + // pcp cfg & no vlan should use default cos + goto cos_success; + } + + pri = (dcb.trust == ROCE5_DCB_PCP) ? + inbuf->sl : + (inbuf->traffic_class >> ROCE5_DSCP_IDX); + ret = hinic5_get_cos_by_pri(rdev->hwdev, pri, cos); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "get cos failed, ret(%d), pri(%u), dcb_on(%u), trust(%u).", + ret, pri, dcb.dcb_on, dcb.trust); + return -EINVAL; + } + +cos_success: + roce5_cos_update_ext(rdev, cos, qp_type); + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h new file mode 100644 index 000000000..2fb0eb2b9 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_mix.h @@ -0,0 +1,269 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MIX_H +#define ROCE_MIX_H + +#include <net/arp.h> +#include <net/neighbour.h> +#include <net/route.h> +#include <net/addrconf.h> +#include <net/ip6_route.h> +#include <linux/mutex.h> +#include <linux/inetdevice.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <linux/module.h> + +#include <rdma/ib_addr.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_pack.h> + +#include "roce.h" + +#define ROCE_HW_VER 0ULL +#define ROCE_FW_VER_0 1ULL +#define ROCE_FW_VER_1 4ULL +#define ROCE_FW_VER_2 0ULL +#define ROCE_FW_VER_3 0ULL +#define ROCE_FW_VER \ + ((ROCE_FW_VER_0 << 32) | (ROCE_FW_VER_1 << 16) | \ + (ROCE_FW_VER_2 << 8) | ROCE_FW_VER_3) + +#define RESOLVE_IP_TIME_OUT 1000 +#define ROCE_GID_HIGHEST_BYTE 15 +#define ROCE_FW_VERSION_LEN 4 +#define ROCE_DEFAULT_VLAN_ID 0xFFFF +#define ROCE_RAH_TC_SHIFT 2 +#define ROCE_RAH_DMAC_H16_SHIFT 8 +#define ROCE_RAH_DMAC_L32_START 2 +#define ROCE_UDP_SRC_PORT_H8_SHIFT 8 + +#define ROCE5_DCB_PCP 0 +#define ROCE5_DCB_DSCP 1 + +#define ROCE5_DSCP_IDX 2 //dscp begin from bit 2 in traffic_class + +#define ROCE_GID_IP_IDX 3 + +#define MAC_INDEX_0 0 +#define MAC_INDEX_1 1 +#define MAC_INDEX_2 2 +#define MAC_INDEX_3 3 +#define MAC_INDEX_4 4 +#define MAC_INDEX_5 5 + +#define IP_INDEX_0 0 +#define IP_INDEX_1 1 +#define IP_INDEX_2 2 +#define IP_INDEX_3 3 +#define IP_INDEX_4 4 +#define IP_INDEX_5 5 +#define IP_INDEX_6 6 +#define IP_INDEX_7 7 + +#define ROCE_MTU_256_INT 256 +#define ROCE_MTU_512_INT 512 +#define ROCE_MTU_1024_INT 1024 +#define ROCE_MTU_2048_INT 2048 +#define ROCE_MTU_4096_INT 4096 +#define ROCE_MTU_8192_INT 8192 + +enum { + USR_MMAP_DB_OFFSET = 0, + USR_MMAP_DWQE_OFFSET, + USR_MMAP_DFX_OFFSET, + USR_MMAP_DBAR3_OFFSET +}; + +enum roce_port_state_e { + ROCE_PORT_PHYS_STATE_NO_CHANGE = 0, + ROCE_PORT_PHYS_STATE_SLEEP, + ROCE_PORT_PHYS_STATE_POLLING, + ROCE_PORT_PHYS_STATE_DISABLED = 3, + ROCE_PORT_PHYS_STATE_PORTCONFTRAIN, + ROCE_PORT_PHYS_STATE_LINKUP = 5, + ROCE_PORT_PHYS_STATE_LINKERRRECOVER, + ROCE_PORT_PHYS_STATE_PHYTEST +}; + +enum ROCE_MBOX_CMD { ROCE_MBOX_CMD_SEND_MAIL_BOX, ROCE_MBOX_CMD_MAX }; + +enum roce5_mtu { + ROCE_MTU_256 = IB_MTU_256, + ROCE_MTU_512 = IB_MTU_512, + ROCE_MTU_1024 = IB_MTU_1024, + ROCE_MTU_2048 = IB_MTU_2048, + ROCE_MTU_4096 = IB_MTU_4096, + ROCE_MTU_8192, +}; + +static inline int roce5_mtu_enum_to_int(enum roce5_mtu mtu) +{ + switch (mtu) { + case ROCE_MTU_256: + return ROCE_MTU_256_INT; + case ROCE_MTU_512: + return ROCE_MTU_512_INT; + case ROCE_MTU_1024: + return ROCE_MTU_1024_INT; + case ROCE_MTU_2048: + return ROCE_MTU_2048_INT; + case ROCE_MTU_4096: + return ROCE_MTU_4096_INT; + case ROCE_MTU_8192: + return ROCE_MTU_8192_INT; + default: + return -1; + } +} + +static inline enum roce5_mtu roce5_get_mtu(int mtu) +{ + int local_mtu = mtu - (IB_GRH_BYTES + IB_UDP_BYTES + IB_BTH_BYTES + + IB_EXT_XRC_BYTES + IB_EXT_ATOMICETH_BYTES + + IB_ICRC_BYTES); + + if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_8192)) + return ROCE_MTU_8192; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_4096)) + return ROCE_MTU_4096; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_2048)) + return ROCE_MTU_2048; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_1024)) + return ROCE_MTU_1024; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_512)) + return ROCE_MTU_512; + else if (local_mtu >= roce5_mtu_enum_to_int(ROCE_MTU_256)) + return ROCE_MTU_256; + else + return 0; +} + +struct roce5_priv_ah { + /* DW0 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 pd : 18; + u32 rsvd0 : 6; + u32 stat_rate : 4; + u32 wqe_cos : 3; + u32 fl : 1; /* forcelock flag, it is used in multicast communication */ +#else + u32 f1 : 1; + u32 wqe_cos : 3; + u32 stat_rate : 4; + u32 rsvd0 : 6; + u32 pd : 18; +#endif + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 hoplimit : 8; + u32 sgid_index : 7; + u32 resv : 1; + u32 port : 4; + u32 rsvd1 : 4; + u32 tclass : 8; +#else + u32 tclass : 8; + u32 rsvd1 : 4; + u32 port : 4; + u32 resv : 1; + u32 sgid_index : 7; + u32 hoplimit : 8; +#endif + } bs; + u32 value; + } dw1; + + /* DW2 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 flow_label : 20; + u32 smac_index : 10; + u32 rsvd : 2; +#else + u32 rsvd : 2; + u32 smac_index : 10; + u32 flow_label : 20; +#endif + } bs; + u32 value; + } dw2; + + /* DW3~6 */ + u8 dgid[16]; + + /* DW7 */ + union { + struct { +#if (BYTE_ORDER != BIG_ENDIAN) + u32 dmac_h16 : 16; + u32 vlan_id : 12; + u32 rsvd : 1; + u32 vlan_pri : 3; +#else + u32 vlan_pri : 3; + u32 rsvd : 1; + u32 vlan_id : 12; + u32 dmac_h16 : 16; +#endif + } bs; + u32 value; + } dw7; + + /* DW8 */ + u32 dmac_l32; +}; + +struct roce5_ah { + struct ib_ah ibah; + struct roce5_priv_ah priv_ah; +}; + +union gid_addr { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; +}; + +struct mailbox_header { + u32 dest_host_id; + u32 dest_global_function_id; +}; + +struct roce5_mail_box_buffer { + struct mailbox_header header; + u32 data[0]; +}; + +struct roce5_resolve_cb_context { + struct completion comp; + int status; +}; + +typedef struct roce5_get_cos_inbuf { + u8 sl; + u8 sgid_index; + u8 port_num; + u8 traffic_class; +} roce5_get_cos_inbuf_s; + +int roce5_resolve_grh(struct roce5_device *rdev, struct rdma_ah_attr *ah_attr, + u16 *vlan_id, struct ib_udata *udata); +int roce5_get_dcb_cfg_cos(struct roce5_device *rdev, + roce5_get_cos_inbuf_s *inbuf, u8 *cos, + enum ib_qp_type qp_type); +uint8_t roce5_get_db_cos_from_vlan_pri(struct roce5_device *rdev, + uint8_t vlan_pri); + +#endif /* ROCE_MIX_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c new file mode 100644 index 000000000..92a3ada97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.c @@ -0,0 +1,138 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_pd.h" + +int roce5_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + int ret; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if (CHECK_2LAYER_IS_NULL(ibpd, device)) { + ROCE_ERR( + "Invalid NULL parameter, ibpd or ibpd->device is NULL."); + ret = -EINVAL; + goto err_out; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + ret = roce5_rdma_pd_alloc(rdev->hwdev, &pd->pdn); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc pdn, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto err_out; + } + + if (udata != NULL) { + if (udata->outbuf == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid NULL parameter, udata->outbuf is NULL."); + ret = -EINVAL; + goto err_copy_to_user; + } + + if (ib_copy_to_udata(udata, &pd->pdn, sizeof(u32)) != 0) { + ret = -EFAULT; + ROCE_DEV_ERR( + rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + goto err_copy_to_user; + } + } + + return 0; + +err_copy_to_user: + roce5_rdma_pd_free(rdev->hwdev, pd->pdn); +err_out: + return ret; +} + +struct ib_pd *roce5_alloc_pd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret = 0; + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if (ibdev == NULL) { + ret = -EINVAL; + ROCE_ERR("Ibdev is null"); + goto err_out; + } + + rdev = to_roce5_dev(ibdev); + + pd = (struct roce5_pd *)kmalloc(sizeof *pd, GFP_KERNEL); + if (pd == NULL) { + ret = -ENOMEM; + ROCE_DEV_ERR(rdev, + "Failed to alloc memory for roce pd, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + pd->ibpd.device = ibdev; + ret = roce5_alloc_pd(&pd->ibpd, udata); + if (ret != 0) { + goto err_alloc_pdn; + } + + return &pd->ibpd; + +err_alloc_pdn: + kfree(pd); + +err_out: + return (struct ib_pd *)ERR_PTR((long)ret); +} + +int roce5_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return -EINVAL; + } + + pd = to_roce5_pd(ibpd); + rdev = to_roce5_dev(ibpd->device); + + roce5_rdma_pd_free(rdev->hwdev, pd->pdn); + + return 0; +} + +int roce5_dealloc_pd_ofa(struct ib_pd *ibpd) +{ + int ret; + struct roce5_pd *pd = NULL; + + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return -EINVAL; + } + + pd = to_roce5_pd(ibpd); + ret = roce5_dealloc_pd(ibpd, NULL); + if (ret == 0) { + kfree(pd); + } + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h new file mode 100644 index 000000000..e40c5a476 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/rdma/roce_pd.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_PD_H +#define ROCE_PD_H + +#include <rdma/ib_verbs.h> + +struct roce5_pd { + struct ib_pd ibpd; + u32 pdn; +}; + +#endif //ROCE_PD_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h new file mode 100644 index 000000000..1886af73f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq.h @@ -0,0 +1,191 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SRQ_H +#define ROCE_SRQ_H + +#include <linux/types.h> + +#include <rdma/ib_verbs.h> + +#include "hinic5_rdma.h" +#include "hinic5_cqm.h" + +#include "roce.h" +#include "roce_pd.h" +#include "roce_db.h" +#include "roce_compat.h" +#include "roce_wqe_format.h" + +/* definition SRQ SGE maximum max and minimum value */ +#define ROCE_SRQ_CONT_MAX_SGE 16 +#define ROCE_SRQ_MAX_SGE 15 +#define ROCE_SRQ_MID_SGE 7 +#define ROCE_SRQ_MIN_SGE 3 + +#define ROCE_SRQN_INVLD CQM_INDEX_INVALID +#define ROCE_SRQ_CONTAINER_LWM_MASK 0xFFFF +#define ROCE_SRQ_CONTAINER_WARTH_MASK 0xF + +#define MAX_SUPPORT_CONTAINER_MODE 3 +#define DWORD_LEN 32 +#define XRC_CQN_FIRST_LEN 10 +#define XRC_CQN_SECOND_LEN 3 +#define SRQ_GPA_SIG_LEN 3 +#define SRQ_WQEBB_SIZE_CAL_SECTTOR \ + 4 /* (2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 */ +#define RDMA_PREFETCH_WQE_MAX 7 +#define RDMA_PREFETCH_MTT_LEN_MAX 3 +#define ROCE_WQE_BB_SIZE_MIN 64 + +#define LAST_SGE_NO_PRESENT \ + 0x80000000UL /* Lbit:0 - The next SGE is present in the list \ + * Lbit:1 - The last SGE, no SGE is present in the list */ +#define NORMAL_FMT_AND_NEXT_SGE_PRESENT \ + 0x3FFFFFFFUL /* Ebit:b0 - Normal format, without extension. \ + * Ebit:b1 - The pointer inside of SGE points to the next SGL. \ + * "Length" and "Key" fields are */ +#define NORMAL_FMT_AND_LAST_SGE_NO_PRESENT 0xBFFFFFFFUL + +/** + * CHIP container_mode: + * chip mode: 0 -> Not container + * chip mode: 1 -> container_size: 2 + * chip mode: 2 -> container_size: 4 + * chip mode: 3 -> container_size: 8 + * chip mode: 4 -> container_size: 16 + */ +enum roce5_chip_srq_mode { + ROCE_CHIP_SRQ_MODE_N = 0, + ROCE_CHIP_SRQ_MODE_1, + ROCE_CHIP_SRQ_MODE_2, + ROCE_CHIP_SRQ_MODE_3, + ROCE_CHIP_SRQ_MODE_4 +}; + +enum srq_state { + ROCE_SRQ_STATE_INVALID = 0x0, + ROCE_SRQ_STATE_ERR = 0x1, + ROCE_SRQ_STATE_VALID = 0xf, + ROCE_SRQ_STATE_MEM_INIT = 0xa +}; + +#define ROCE_SRQ_STATE_CHECK_VALUE 0x0 + +struct roce5_wqe_srq_next_seg { + u16 reserved1; + __be16 pcnt; /* indicate the pi */ + u8 signature; + u8 reserved2; + __be16 next_wqe_index; + u32 reserved3[2]; +}; + +struct roce5_wqe_container_srq_next_seg { + u32 next_gpa_h; + + struct { + u32 rsvd : 11; + u32 next_gpa_vd : 1; + u32 next_gpa_h : 20; /* indicate the pi */ + } dw1; + + struct { + u32 next_idx : 16; + u32 rsvd : 16; + } dw2; + + struct { + u32 rsvd2 : 30; + u32 link_flag : 1; + u32 rsvd : 1; + } dw3; + + struct { + u32 osd_next_idx : 16; + u32 osd_cur_idx : 16; + } dw4; +}; + +#define ROCE_SRQ_SGE_LAST 1 +#define ROCE_SRQ_SGE_NLAST 0 + +#define ROCE_SRQ_SGE_LKEY_NOEXT 0 +#define ROCE_SRQ_SGE_LKEY_EXT 1 + +struct roce5_srq { + struct ib_srq ibsrq; /* ibsrq */ + cqm_queue_s *cqm_srq; + + u32 srqn; + int max_depth; + int max_gs; + int wqe_shift; + + cqm_buf_s *buf; + struct roce5_db db; + u64 *wrid; + spinlock_t lock; + int head; + int tail; + u16 wqe_ctr; + u8 xrc_en; + u8 rsvd; + struct ib_umem *umem; + struct mtt mtt; + struct mutex mutex; + u32 rqe_cnt_th; + u8 container_flag; + u8 container_size; + u8 container_mode; + u8 container_warn_th; + + int buf_sz; +}; + +static inline u8 roce5_srq_get_max_sge(struct roce5_srq *rsrq) +{ + if (rsrq->container_flag != 0) { + return ROCE_SRQ_CONT_MAX_SGE; + } + return ROCE_SRQ_MAX_SGE; +} + +static inline int roce5_srq_init_wqe_size(struct roce5_srq *rsrq) +{ + int desc_size = 0; + + if (rsrq->container_flag != 0) { + desc_size = rsrq->max_gs * sizeof(struct roce_wqe_srq_data_seg); + } else { + desc_size = sizeof(struct roce5_wqe_srq_next_seg) + + rsrq->max_gs * sizeof(struct roce_wqe_srq_data_seg); + } + desc_size = (int)(ROCE_ROUNDUP_POW_OF_TWO((u32)desc_size)); + desc_size = ROCE_MAX(ROCE_WQE_BB_SIZE_MIN, desc_size); + + rsrq->wqe_shift = (int)ROCE_ILOG2((unsigned)desc_size); + return desc_size; +} + +void roce5_free_srq_wqe(struct roce5_srq *srq, int wqe_index); +void roce5_srq_async_event(struct roce5_device *rdev, struct roce5_srq *srq, + int type); +void *roce5_srq_get_wqe(struct roce5_srq *srq, int n); +u8 roce5_get_container_sz(u32 container_mode); +u32 roce5_srq_max_avail_wr_set(struct roce5_srq *rsrq); +int roce5_create_srq_common(struct roce5_device *rdev, struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, u32 index); +struct roce5_srq *roce5_get_rsrq_by_srqn(struct roce5_device *rdev, u32 srqn); +static inline void roce5_rsrq_put(struct roce5_srq *rsrq) +{ + if (rsrq == NULL) { + return; + } + cqm5_object_put(&rsrq->cqm_srq->object); +} + +#endif //ROCE_SRQ_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c new file mode 100644 index 000000000..0a3beb338 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_comm.c @@ -0,0 +1,116 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_aeq_format.h" +#include "roce_srq_extension.h" +#include "roce_restore.h" + +static void *roce5_srq_buf_offset(struct tag_cqm_buf *buf, int offset) +{ + return (void *)((char *)buf->direct.va + offset); +} + +void *roce5_srq_get_wqe(struct roce5_srq *srq, int n) +{ + return roce5_srq_buf_offset(srq->buf, + (int)((u32)n << (unsigned)srq->wqe_shift)); +} + +void roce5_srq_async_event(struct roce5_device *rdev, struct roce5_srq *srq, + int type) +{ + struct ib_srq *ibsrq = &srq->ibsrq; + struct ib_event event; + int ret; + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR_LIMIT(rdev, "Failed to memset event.(ret:%d)", + ret); + } + if (ibsrq->event_handler) { + event.device = ibsrq->device; + event.element.srq = ibsrq; + switch (type) { + case ROCE_EVENT_TYPE_SRQ_LIMIT: + event.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + + case ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + event.event = IB_EVENT_SRQ_ERR; + break; + + default: + ROCE_DEV_WARN_LIMIT( + rdev, + "unexpected event type(%d) on SRQ(%06x), func_id(%d)", + type, srq->srqn, rdev->glb_func_id); + return; + } + + ibsrq->event_handler(&event, ibsrq->srq_context); + } +} + +/* **************************************************************************** + Prototype : roce5_free_srq_wqe + Description : roce5_free_srq_wqe + Input : struct roce5_srq *srq + int wqe_index + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +void roce5_free_srq_wqe(struct roce5_srq *srq, int wqe_index) +{ + struct roce5_wqe_srq_next_seg *next = NULL; + + spin_lock(&srq->lock); + + next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(srq, + srq->tail); + next->next_wqe_index = (u16)wqe_index; + srq->tail = wqe_index; + + spin_unlock(&srq->lock); +} + +/** + * @brief Get roce srq by srqn. When querying succeeded, you have to call roce5_rsrq_put() to free + the cqm_obj_srq. But there is no need to call roce5_rsrq_put() if querying failed. + * @return Success: return the roce srq. Failure: return NULL. + */ +struct roce5_srq *roce5_get_rsrq_by_srqn(struct roce5_device *rdev, u32 srqn) +{ + cqm_object_s *cqm_obj_srq = NULL; + struct roce5_srq *rsrq = NULL; + + cqm_obj_srq = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + ROCE_DEV_ERR( + rdev, + "Can't find srq according to srqn(%u), func_id(%d).", + srqn, rdev->glb_func_id); + return NULL; + } + + rsrq = cqmobj_to_roce5_srq(cqm_obj_srq); + if (rsrq == NULL) { + ROCE_DEV_ERR(rdev, "Failed to convert obj to rsrq."); + cqm5_object_put(cqm_obj_srq); + } + + return rsrq; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c new file mode 100644 index 000000000..47c86b2fa --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_create.c @@ -0,0 +1,712 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/kernel.h> + +#include "comm_defs.h" +#include "roce.h" +#include "roce_cq.h" +#include "roce_cqm_cmd.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_pd.h" +#include "roce_srq.h" +#include "roce_restore.h" +#include "roce_npu_cmd_srq_defs.h" +#include "roce_npu_cmd_defs.h" + +#include "roce_srq_extension.h" +#include "roce_main_extension.h" +#include "hinic5_hmm.h" +#include "roce_npu_cmd.h" +#include "securec.h" +#include "roce_os_compat.h" + +/* **************************************************************************** + Prototype : roce5_srq_sw2hw + Description : roce5_srq_sw2hw + Input : struct roce5_device *rdev + struct roce5_srq *rsrq + u32 pdn + u32 cqn + u16 xrcdn + int page_shift + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + + 1.Date : 2015/7/27 + Modification : modify function +**************************************************************************** */ +static void roce5_srq_sw2hw_set(roce_verbs_srq_attr_s *srq_attr, + struct roce5_srq *rsrq, u32 pdn, u32 cqn, + u16 xrcdn, int page_shift) +{ + srq_attr->dw0.bs.size = (u32)(ROCE_ILOG2((unsigned)rsrq->max_depth)); + srq_attr->dw0.bs.page_size = (u32)page_shift; + srq_attr->dw0.bs.wqebb_size = + (u32)rsrq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->dw0.bs.mtt_page_size = + rsrq->mtt.mtt_page_shift - PAGE_SHIFT_4K; + srq_attr->dw0.bs.xrcd = xrcdn; + + srq_attr->dw1.bs.dma_attr_idx = 0; + srq_attr->dw1.bs.so_ro = 1; + srq_attr->dw1.bs.state = ROCE_SRQ_STATE_VALID; + + srq_attr->dw3.bs.container = rsrq->container_flag; + srq_attr->dw3.bs.lth_pre_en = + (rsrq->container_flag == 0); //rdma engine prefetch + srq_attr->dw3.bs.pcnt_on_chip = 0; + srq_attr->dw3.bs.rkey_en = 1; + srq_attr->dw3.bs.pd = pdn; + + srq_attr->srqn = cpu_to_be32(rsrq->srqn); + srq_attr->xrc_cqn = cpu_to_be32(cqn); + + if (rsrq->container_flag == 0) { //normal srq + srq_attr->dw1.bs.ep = 0; + srq_attr->dw1.bs.cos = 0; + + srq_attr->dw2.value = 0; + srq_attr->dw2.bs.wqe_prefetch_max_num = + RDMA_PREFETCH_WQE_MAX; //rdma engine wqe prefetch max num + srq_attr->dw2.bs.wqecnt_lth = 0xe; + srq_attr->dw2.bs.mtt_prefetch_maxlen = + 0; //rdma engine mtt prefetch max len + + srq_attr->dw3.bs.lth_pre_en = 1; //rdma engine prefetch + srq_attr->dw3.bs.lth_gap = 0; + } else { + srq_attr->cont.dw2.bs.head_idx = 0; + srq_attr->cont.dw2.bs.warn_th = + 0; //Do NOT init warn_th until user arm srq + srq_attr->cont.dw2.bs.cont_size = + (MAX_SUPPORT_CONTAINER_MODE - rsrq->container_mode) & + 0x3; + srq_attr->cont.dw2.value = + cpu_to_be32(srq_attr->cont.dw2.value); + srq_attr->record_gpa_at_hop_num = + cpu_to_be64((rsrq->mtt.mtt_layers & 0x3)); + } + + srq_attr->dw0.value = cpu_to_be32(srq_attr->dw0.value); + srq_attr->dw1.value = cpu_to_be32(srq_attr->dw1.value); + srq_attr->dw2.value = cpu_to_be32(srq_attr->dw2.value); + srq_attr->dw3.value = cpu_to_be32(srq_attr->dw3.value); + + srq_attr->l0mtt_gpa = rsrq->mtt.mtt_paddr; + + srq_attr->l0mtt_gpa = cpu_to_be64(srq_attr->l0mtt_gpa); + + srq_attr->record_gpa_at_hop_num = + cpu_to_be64(rsrq->db.dma | (rsrq->mtt.mtt_layers & 0x3)); + + return; +} + +static int roce5_srq_sw2hw(struct roce5_device *rdev, struct roce5_srq *rsrq, + u32 pdn, u32 cqn, u16 xrcdn, int page_shift) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_verbs_srq_attr_s *srq_attr = NULL; + roce_uni_cmd_create_srq_s *srq_sw2hw_inbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_create_srq_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_sw2hw_inbuf = (roce_uni_cmd_create_srq_s *)cqm_cmd_inbuf->buf; + srq_sw2hw_inbuf->com.index = cpu_to_be32((u32)rsrq->srqn); + srq_sw2hw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + srq_attr = &srq_sw2hw_inbuf->srq_attr; + + roce5_srq_sw2hw_set(srq_attr, rsrq, pdn, cqn, xrcdn, page_shift); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_SW2HW_SRQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send SW2HW_SRQ command, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + + /* update dev status after cmdq timeout or exception, notify PCIe reset */ + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static int roce5_create_srq_check(const struct ib_pd *ibpd, + const struct ib_srq_init_attr *init_attr, + const struct roce5_srq *rsrq) +{ + struct roce5_device *rdev = NULL; + + if ((ibpd == NULL) || (init_attr == NULL)) { + ROCE_ERR("Ibpd or init_attr is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibpd->device); + if ((init_attr->attr.max_wr > + (u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes) || + (init_attr->attr.max_sge > + ((u32)rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge + + rsrq->container_flag))) { + ROCE_DEV_ERR( + rdev, + "Invalid input parms, max_wr(%d), max_srq_wqes(%d), max_sge(%d), max_srq_sge(%d)", + init_attr->attr.max_wr, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes, + init_attr->attr.max_sge, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge); + return -EINVAL; + } + + return 0; +} + +static int roce5_create_srq_write_mtt_and_db(struct roce5_device *rdev, + struct roce5_srq *rsrq, + struct roce5_create_srq_cmd *ucmd, + struct roce5_pd *pd) +{ + int ret = 0; + + ret = roce5_umem_write_mtt(rdev, &rsrq->mtt, rsrq->umem, 1); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = roce5_db_map_user(to_roce5_ucontext(pd->ibpd.uobject->context), + ucmd->db_addr, &rsrq->db, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to map db to user space, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return ret; +} + +static int roce5_create_srq_get_umem(struct roce5_create_srq_cmd *ucmd, + struct ib_udata *udata, + struct roce5_srq *rsrq, + struct roce5_device *rdev, + struct roce5_pd *pd) +{ + int ret = 0; + + if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd)) != 0) { + ret = -EFAULT; + ROCE_DEV_ERR(rdev, + "Failed to copy from user space, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + rsrq->umem = roce5_umem_get(&rdev->ib_dev, pd->ibpd.uobject->context, + ucmd->buf_addr, (size_t)rsrq->buf_sz, 0, 0); + if (IS_ERR(rsrq->umem)) { + ROCE_DEV_ERR( + rdev, + "Failed to get ib_umem, func_id(%d) rsrq->buf_sz:%u", + rdev->glb_func_id, rsrq->buf_sz); + ret = (int)PTR_ERR(rsrq->umem); + return ret; + } + + return ret; +} + +static int roce5_create_user_srq_update(struct roce5_device *rdev, + struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + int ret = 0; + + cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce5_cq(init_attr->ext.cq)->cqn : + 0); + xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce5_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; + + /* set SRQC */ + ret = roce5_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, + (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to handle srq_sw2hw, func_id(%d)", + rdev->glb_func_id); + goto err_unmap; + } + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + + if (ib_copy_to_udata(udata, &rsrq->srqn, sizeof(u32)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + goto err_unmap; + } + + return 0; + +err_unmap: + roce5_db_unmap_user(to_roce5_ucontext(pd->ibpd.uobject->context), + &rsrq->db, 0); + return ret; +} + +u32 roce5_srq_max_avail_wr_set(struct roce5_srq *rsrq) +{ + u32 link_wqe_count = 0; + + if (rsrq->container_flag != 0) { + link_wqe_count = (u32)rsrq->max_depth / rsrq->container_size; + return ((u32)rsrq->max_depth - link_wqe_count) - + (rsrq->container_size - 1); + } else { + return (u32)rsrq->max_depth - 1; + } +} + +/* **************************************************************************** + Prototype : roce5_create_user_srq + Description : roce5_create_user_srq + Input : struct roce5_device *rdev + struct roce5_srq *rsrq + struct roce5_pd *pd + struct ib_srq_init_attr *init_attr + struct ib_udata *udata + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_user_srq(struct roce5_device *rdev, + struct roce5_srq *rsrq, struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, u32 index) +{ + int ret = 0; + u32 npages = 0; + int page_shift = 0; + struct roce5_create_srq_cmd ucmd = { 0 }; + + /* dispatch SRQN and SRQC */ + rsrq->cqm_srq = cqm5_object_rdma_queue_create(rdev->hwdev, + SERVICE_T_ROCE, + CQM_OBJECT_RDMA_SRQ, 0, + rsrq, false, index, 0, 0); + if (rsrq->cqm_srq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue by cqm_object, func_id(%d) index(%d)", + rdev->glb_func_id, index); + return (-ENOMEM); + } + + rsrq->srqn = rsrq->cqm_srq->index; + + ret = roce5_create_srq_get_umem(&ucmd, udata, rsrq, rdev, pd); + if (ret != 0) { + goto err_free_cqm_srq; + } + + page_shift = + roce5_umem_get_buf_page_shift(rdev, rsrq->umem, ucmd.buf_addr); + npages = roce5_umem_get_mtt_npages(rdev, rsrq->umem, (u32)page_shift); + + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, npages, (u32)page_shift, &rsrq->mtt, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, " Failed to alloc rdma_mtt, func_id(%d)", + rdev->glb_func_id); + goto err_release_umem; + } + + ret = roce5_create_srq_write_mtt_and_db(rdev, rsrq, &ucmd, pd); + if (ret != 0) { + goto err_free_mtt; + } + + ret = roce5_create_user_srq_update(rdev, rsrq, pd, init_attr, udata, + page_shift); + if (ret != 0) { + goto err_free_mtt; + } + + init_attr->attr.max_wr = roce5_srq_max_avail_wr_set(rsrq); + + return 0; + +err_free_mtt: + hmm_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + +err_release_umem: + ib_umem_release(rsrq->umem); + +err_free_cqm_srq: + cqm5_object_delete(&rsrq->cqm_srq->object); + + return ret; +} + +static void roce5_get_cqn_xrcdn(u32 *cqn, u16 *xrcdn, + struct ib_srq_init_attr *init_attr, + struct roce5_device *rdev) +{ + *cqn = (u32)((init_attr->srq_type == IB_SRQT_XRC) ? + to_roce5_cq(init_attr->ext.cq)->cqn : + 0); + *xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? + (u16)(to_roce5_xrcd(init_attr->ext.xrc.xrcd)->xrcdn) : + (u16)rdev->rdma_cap.reserved_xrcds; +} + +static int roce5_create_kernel_srq_update(struct roce5_device *rdev, + struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + int page_shift) +{ + u32 cqn = 0; + u16 xrcdn = 0; + int ret; + ret = roce5_buf_write_mtt(rdev, &rsrq->mtt, rsrq->buf); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_mtt; + } + + rsrq->wrid = (u64 *)kmalloc( + (size_t)((u32)rsrq->max_depth * sizeof(u64)), GFP_KERNEL); + if (rsrq->wrid == NULL) { + rsrq->wrid = (u64 *)vzalloc( + (size_t)((u32)rsrq->max_depth * sizeof(u64))); + if (rsrq->wrid == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to alloc srq wrid, func_id(%d)", + rdev->glb_func_id); + ret = -ENOMEM; + goto err_free_mtt; + } + } + + roce5_get_cqn_xrcdn(&cqn, &xrcdn, init_attr, rdev); + + ret = roce5_srq_sw2hw(rdev, rsrq, pd->pdn, cqn, xrcdn, + (page_shift - PAGE_SHIFT_4K)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to handle srq sw2hw, func_id(%d)", + rdev->glb_func_id); + goto err_free_wrid; + } + + return 0; + +err_free_wrid: + kvfree(rsrq->wrid); + rsrq->wrid = NULL; + +err_free_mtt: + hmm_mtt_free(rdev->hwdev, &rsrq->mtt, SERVICE_T_ROCE); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_create_kernel_srq + Description : roce5_create_kernel_srq + Input : struct roce5_device *rdev + struct roce5_srq *rsrq + struct roce5_pd *pd + struct ib_srq_init_attr *init_attr + Output : None + + 1.Date : 2017/5/4 + Modification : Created function + +**************************************************************************** */ +static int roce5_create_kernel_srq(struct roce5_device *rdev, + struct roce5_srq *rsrq, struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + u32 index) +{ + int i = 0; + int ret = 0; + int page_shift = 0; + struct roce5_wqe_srq_next_seg *next = NULL; + struct roce_wqe_srq_data_seg *scatter = NULL; + + /* alloc queue Buf/ Soft DB/SRQN/SRQC */ + rsrq->cqm_srq = cqm5_object_rdma_queue_create( + rdev->hwdev, SERVICE_T_ROCE, CQM_OBJECT_RDMA_SRQ, + (u32)rsrq->buf_sz, rsrq, true, index, 0, 0); + if (rsrq->cqm_srq == NULL) { + ROCE_DEV_ERR( + rdev, + "Failed to create rdma queue by cqm_object, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + rsrq->srqn = rsrq->cqm_srq->index; + rsrq->buf = &rsrq->cqm_srq->q_room_buf_1; + + /* set Soft DB */ + rsrq->db.db_record = (__be32 *)(void *)(&rsrq->cqm_srq->q_header_vaddr + ->doorbell_record); + rsrq->db.dma = rsrq->cqm_srq->q_header_paddr; + *rsrq->db.db_record = 0; + + rsrq->head = 0; + rsrq->tail = rsrq->max_depth - 1; + rsrq->wqe_ctr = 0; + + for (i = 0; i < rsrq->max_depth; ++i) { + next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe(rsrq, + i); + next->next_wqe_index = cpu_to_be16( + (u16)((u32)(i + 1) & (u32)(rsrq->max_depth - 1))); + + scatter = (struct roce_wqe_srq_data_seg *)(next + 1); + scatter->dw2.length = 0; + + /* first SGE = last SGE */ + scatter->dw3.lkey = LAST_SGE_NO_PRESENT; + scatter->dw3.lkey = cpu_to_be32(scatter->dw3.lkey); + } + + page_shift = (int)ROCE_ILOG2(rsrq->buf->buf_size); + rsrq->mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(rdev->hwdev, rsrq->buf->buf_number, (u32)page_shift, + &rsrq->mtt, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdma_mtt, func_id(%d)", + rdev->glb_func_id); + goto err_free_cqm_srq; + } + + ret = roce5_create_kernel_srq_update(rdev, rsrq, pd, init_attr, + page_shift); + if (ret != 0) { + goto err_free_cqm_srq; + } + + rsrq->ibsrq.ext.xrc.srq_num = (u32)rsrq->srqn; + init_attr->attr.max_wr = (u32)rsrq->max_depth - 1; + return 0; + +err_free_cqm_srq: + cqm5_object_delete(&rsrq->cqm_srq->object); + + return ret; +} + +static void roce5_set_srq_depth(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq) +{ + u32 link_wqe_count = 0; + u32 remain_wqe_count = 0; + + /* + * +---+---+---+---+---+---+---+---+ + * |wqe|wqe|wqe|wqe|wqe|wqe|wqe|nil| + * +---+---+---+---+---+---+---+---+ + * ^ + * | + * full condition:head==tail==nil + */ + if (rsrq->container_flag != 0) { + link_wqe_count = + init_attr->attr.max_wr / (rsrq->container_size - 1u); + remain_wqe_count = + init_attr->attr.max_wr % (rsrq->container_size - 1u); + link_wqe_count += + ((remain_wqe_count == 0) ? + 0 : + (rsrq->container_size - remain_wqe_count)); + rsrq->max_depth = (int)(init_attr->attr.max_wr + + link_wqe_count + rsrq->container_size); + } else { + rsrq->max_depth = (int)(init_attr->attr.max_wr + 1); + } + rsrq->max_depth = + (int)((ROCE_ROUNDUP_POW_OF_TWO((u32)rsrq->max_depth)) & + 0xffffffff); +} + +static void roce5_rsrq_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, struct roce5_device *rdev) +{ + int buf_size = 0; + int desc_size = 0; + + roce5_srq_container_init(init_attr, rsrq, rdev); + + mutex_init(&rsrq->mutex); + spin_lock_init(&rsrq->lock); + + roce5_set_srq_depth(init_attr, rsrq); + + rsrq->max_gs = (init_attr->attr.max_sge <= ROCE_SRQ_MIN_SGE) ? + ROCE_SRQ_MIN_SGE : + ((init_attr->attr.max_sge <= ROCE_SRQ_MID_SGE) ? + ROCE_SRQ_MID_SGE : + roce5_srq_get_max_sge(rsrq)); + init_attr->attr.max_sge = (u32)rsrq->max_gs; + + /* 0 <= max_gs <= 3, desc_size = 64 + * 4 <= max_gs <= 7, desc_size = 128 + */ + desc_size = roce5_srq_init_wqe_size(rsrq); + buf_size = rsrq->max_depth * desc_size; + + /* WQEBB align 1 PAGE */ + if (buf_size < (int)PAGE_SIZE) { + rsrq->max_depth = + (int)((u64)PAGE_SIZE >> (unsigned int)rsrq->wqe_shift); + } + + /* align with PAGE SIZE */ + buf_size = (int)ROCE_ALIGN((u32)buf_size, PAGE_SIZE); + rsrq->buf_sz = buf_size; + + ROCE_DEV_INFO( + rdev, + "RoCE srq mode: container_EN(%d), container_mode(%d), XRC_EN(%d), MAX_DEPTH(%d), MAX_SGE(%d), desc_size(%d)", + rsrq->container_flag, rsrq->container_mode, rsrq->xrc_en, + rsrq->max_depth, rsrq->max_gs, desc_size); +} + +int roce5_create_srq_common(struct roce5_device *rdev, struct roce5_srq *rsrq, + struct roce5_pd *pd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata, u32 index) +{ + int ret; + roce5_rsrq_init(init_attr, rsrq, rdev); + + ret = roce5_create_srq_check(&pd->ibpd, init_attr, rsrq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check srq info"); + return ret; + } + + if (rsrq->ibsrq.uobject != NULL) { + ret = roce5_create_user_srq(rdev, rsrq, pd, init_attr, udata, + index); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to create user srq, func_id(%d), index(%d)", + rdev->glb_func_id, index); + } + } else { + ret = roce5_create_kernel_srq(rdev, rsrq, pd, init_attr, index); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to create kernel srq, func_id(%d)", + rdev->glb_func_id); + } + } + + ROCE_DEV_INFO( + rdev, + "SRQ create:srqn:%d,container_en_mode:%d_%d,xrc_en:%d,max_depth:%d", + rsrq->srqn, rsrq->container_flag, rsrq->container_mode, + rsrq->xrc_en, rsrq->max_depth); + + return ret; +} + +int roce5_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct roce5_pd *pd = NULL; + struct roce5_device *rdev = NULL; + struct roce5_srq *rsrq = NULL; + + if ((ibsrq == NULL) || (init_attr == NULL)) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + pd = to_roce5_pd(ibsrq->pd); + rdev = to_roce5_dev(ibsrq->device); + rsrq = to_roce5_srq(ibsrq); + + return roce5_create_srq_common(rdev, rsrq, pd, init_attr, udata, + ROCE_SRQN_INVLD); +} + +struct ib_srq *roce5_create_srq_ofa(struct ib_pd *ibpd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + int ret; + struct roce5_srq *rsrq = NULL; + + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return ERR_PTR(-EINVAL); + } + + rsrq = (struct roce5_srq *)kzalloc(sizeof(struct roce5_srq), + GFP_KERNEL); + if (rsrq == NULL) { + ROCE_ERR("Failed to alloc srq"); + return ERR_PTR(-ENOMEM); + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + rsrq->ibsrq.pd = ibpd; + rsrq->ibsrq.device = ibpd->device; +#if defined(IB_SRQ_HAVE_UOBJECT) + rsrq->ibsrq.uobject = + (udata ? ibpd->uobject : + NULL); /* udata not NULL means user srq create */ +#endif + ret = roce5_create_srq(&rsrq->ibsrq, init_attr, udata); + if (ret != 0) { + kfree(rsrq); + return ERR_PTR(ret); + } + + return &rsrq->ibsrq; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c new file mode 100644 index 000000000..887a857ad --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_ctrl.c @@ -0,0 +1,635 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> +#include <linux/vmalloc.h> + +#include "comm_defs.h" +#include "roce.h" +#include "roce_compat.h" +#include "roce_user.h" +#include "roce_xrc.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_mix.h" +#include "roce_cqm_cmd.h" +#include "roce_restore.h" +#include "hinic5_hmm.h" +#include "roce_main_extension.h" +#include "roce_srq_extension.h" +#include "roce_npu_cmd.h" +#include "roce_com.h" +#include "securec.h" +#include "roce_os_compat.h" + +/* and above next is V100 compatible data structure */ +typedef struct tag_roce_cmd_srq_arm { + roce_verbs_cmd_com_s com; + union { + u32 limitwater; + struct { +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && ((BYTE_ORDER == BIG_ENDIAN)) + u32 lwm : 16; + u32 warth : 4; + u32 th_up_en : 1; + u32 cont_en : 1; + u32 rsvd : 10; +#else + u32 rsvd : 10; + u32 cont_en : 1; + u32 th_up_en : 1; + u32 warth : 4; + u32 lwm : 16; +#endif + } bs; + }; +} roce_cmd_srq_arm_s; + +typedef struct tag_roce_cmd_srq_hw2sw { + roce_verbs_cmd_com_s com; + roce_xq_mtt_info_s mtt_info; + u32 srq_buf_len; + u32 wqe_cache_line_start; + u32 wqe_cache_line_end; + u32 wqe_cache_line_size; +} roce_cmd_srq_hw2sw_s; + +static u8 roce5_cal_srq_container_num(u32 avail_wr, struct roce5_srq *srq) +{ + u32 container_num = 0; + u32 srqe_num = 0; + + /* cal container_num by avail_wr from user */ + srqe_num = srq->container_size - 1; + + /* explain the formula: round up by srqe_num */ + container_num = (avail_wr + srqe_num - 1) / srqe_num; + return roce5_calculate_cont_th(container_num); +} + +static void roce5_srq_arm_config_cqm_cmd(cqm_cmd_buf_s *cqm_cmd_inbuf, + struct roce5_srq *srq, u32 srq_limit, + u8 *warth) +{ + roce_uni_cmd_srq_arm_s *srq_arm_inbuf = NULL; + + srq_arm_inbuf = (roce_uni_cmd_srq_arm_s *)cqm_cmd_inbuf->buf; + srq_arm_inbuf->com.index = cpu_to_be32((u32)(srq->srqn)); + srq_arm_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_SRQ_BITMASK); + if (srq->container_flag != 0) { + srq_arm_inbuf->srq_arm.bs.cont_en = srq->container_flag; + srq_arm_inbuf->srq_arm.bs.th_up_en = + 1; //Valid for setting up container warn_threshold + *warth = roce5_cal_srq_container_num(srq_limit, srq); + srq_arm_inbuf->srq_arm.bs.warth = *warth & + ROCE_SRQ_CONTAINER_WARTH_MASK; + srq_arm_inbuf->srq_arm.limitwater = + cpu_to_be32(srq_arm_inbuf->srq_arm.limitwater); + } else { + srq_arm_inbuf->srq_arm.bs.lwm = srq_limit; + srq_arm_inbuf->srq_arm.limitwater = + cpu_to_be32(srq_arm_inbuf->srq_arm.limitwater); + } +} + +/* **************************************************************************** + Prototype : roce5_srq_arm + Description : roce5_srq_arm + Input : struct roce5_device *rdev + struct roce5_srq *srq + u32 srq_limit + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_srq_arm(struct roce5_device *rdev, struct roce5_srq *srq, + u32 srq_limit) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + uint8_t warth = 0; + uint16_t inbuf_size = 0; + + if (ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) != 0) { + inbuf_size = sizeof(roce_uni_cmd_srq_arm_s); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + inbuf_size, NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + if (ROCE5_SUPPORT_IB_CMD_MODIFY_SRQ_MSG_V2(rdev) != 0) { + roce5_srq_arm_config_cqm_cmd(cqm_cmd_inbuf, srq, srq_limit, + &warth); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_ARM_SRQ, + cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "send ARM_SRQ command Failed, ret(%d), func_id(%d), warth(%d)", + ret, rdev->glb_func_id, warth); + + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_modify_srq + Description : roce5_modify_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *attr + enum ib_srq_attr_mask attr_mask + struct ib_udata *udata + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + int ret = 0; + struct roce5_device *rdev = NULL; + struct roce5_srq *srq = NULL; + struct roce5_modify_srq_cmd ucmd = { 0 }; + u32 max_avail_wr = 0; + + if ((ibsrq == NULL) || (attr == NULL)) { + ROCE_ERR("Ibsrq or attr is null"); + return (-EINVAL); + } + + if (((unsigned int)attr_mask & IB_SRQ_MAX_WR) != 0) { + ROCE_ERR("Not support resizing SRQs"); + return (-EINVAL); + } + + rdev = to_roce5_dev(ibsrq->device); + srq = to_roce5_srq(ibsrq); + + if ((udata != NULL) && + (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)) != 0)) { + ROCE_DEV_ERR( + rdev, + "SRQN:(%d), Failed to copy from user space, func_id(%d)", + srq->srqn, rdev->glb_func_id); + return (-EFAULT); + } + + if (((unsigned int)attr_mask & IB_SRQ_LIMIT) != 0) { + /* calculate max_wr */ + max_avail_wr = roce5_srq_max_avail_wr_set(srq); + if (attr->srq_limit > max_avail_wr) { + ROCE_DEV_ERR( + rdev, + "srq_limit > max_wr, func_id(%d), max_avail_wr(%d)", + rdev->glb_func_id, max_avail_wr); + return (-EINVAL); + } + + mutex_lock(&srq->mutex); + ret = roce5_srq_arm(rdev, srq, attr->srq_limit); + mutex_unlock(&srq->mutex); + + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "[Failed to handle srq arm, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + } + + return 0; +} + +static int roce5_cmd_srq_query(struct roce5_device *rdev, struct roce5_srq *srq, + u32 *srq_limit) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_srq_query_s *srq_query_inbuf = NULL; + roce_uni_cmd_srq_query_outbuf_s *srq_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_srq_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_srq_query_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + srq_query_inbuf = (roce_uni_cmd_srq_query_s *)cqm_cmd_inbuf->buf; + srq_query_outbuf = + (roce_uni_cmd_srq_query_outbuf_s *)cqm_cmd_outbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_SRQ, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_A, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send QUERY_SRQ command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + if ((ret == -ETIMEDOUT) || (ret == -EPERM)) { + rdev->dev_status_to_ofed = ROCE_DEV_STATUS_CMDQ_TIMEOUT; + } + + ret = -1; + goto err_cmd; + } + + *srq_limit = be32_to_cpu(srq_query_outbuf->com.srq_limit); + + if (srq->container_flag != 0) { + /* parse srqc to get warn_th */ + u32 warn_th = be32_to_cpu(srq_query_outbuf->com.warn_th); + *srq_limit = (warn_th == 0) ? 0 : + ((u32)((srq->container_size - 1) * + (1U << warn_th))); + } + +err_cmd: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +/* **************************************************************************** + Prototype : roce5_query_srq + Description : roce5_query_srq + Input : struct ib_srq *ibsrq + struct ib_srq_attr *srq_attr + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +int roce5_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + int ret = 0; + u32 limit_water = 0; + struct roce5_device *rdev = NULL; + struct roce5_srq *rsrq = NULL; + + if ((ibsrq == NULL) || (srq_attr == NULL)) { + ROCE_ERR("Ibsrq or srq_attr is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibsrq->device); + rsrq = to_roce5_srq(ibsrq); + + ret = roce5_cmd_srq_query(rdev, rsrq, &limit_water); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query srq, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + srq_attr->srq_limit = limit_water; + srq_attr->max_wr = roce5_srq_max_avail_wr_set(rsrq); + srq_attr->max_sge = (u32)rsrq->max_gs; + + return 0; +} + +static void roce5_srq_hw2sw_config_cqm_cmd(struct roce5_device *rdev, + cqm_cmd_buf_s *cqm_cmd_inbuf, + struct rdma_service_cap *rdma_cap, + struct roce5_srq *srq) +{ + roce_uni_cmd_srq_hw2sw_s *srq_hw2sw_inbuf = NULL; + + srq_hw2sw_inbuf = (roce_uni_cmd_srq_hw2sw_s *)cqm_cmd_inbuf->buf; + srq_hw2sw_inbuf->com.index = cpu_to_be32((u32)srq->srqn); + srq_hw2sw_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + srq_hw2sw_inbuf->srq_cache.srq_buf_len = cpu_to_be32((u32)srq->buf_sz); + + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_flags = 0; + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_num = 0; + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_start); + srq_hw2sw_inbuf->srq_cache.cmtt_cache.mtt_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_end); + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.cmtt_cl_sz); + + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_start = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_start); + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_end = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_end); + srq_hw2sw_inbuf->srq_cache.wqe_cache_line_size = + cpu_to_be32(rdma_cap->dev_rdma_cap.roce_own_cap.wqe_cl_sz); +} + +/* **************************************************************************** + Prototype : roce5_srq_hw2sw + Description : roce5_srq_hw2sw + Input : struct roce5_device *rdev + struct roce5_srq *srq + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_srq_hw2sw(struct roce5_device *rdev, struct roce5_srq *srq) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + uint16_t inbuf_size = 0; + + if (ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) != 0) { + inbuf_size = sizeof(roce_uni_cmd_srq_hw2sw_s); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + inbuf_size, NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + if (ROCE5_SUPPORT_IB_CMD_DESTROY_SRQ_MSG_V2(rdev) != 0) { + roce5_srq_hw2sw_config_cqm_cmd(rdev, cqm_cmd_inbuf, + &rdev->rdma_cap, srq); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_HW2SW_SRQ, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to send HW2SW_SRQ command, func_id(%d)", + rdev->glb_func_id); + + ret = -1; + + if (hinic5_get_chip_present_flag(rdev->hwdev) == 0) { + ROCE_DEV_ERR(rdev, + "Card not present, return OK, func_id(%d)", + rdev->glb_func_id); + ret = 0; + } + + goto err_out; + } + +err_out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void roce5_free_srq(struct ib_srq *ibsrq, struct roce5_device *rdev, + struct roce5_srq *srq, struct ib_udata *udata) +{ + hmm_mtt_free(rdev->hwdev, &srq->mtt, SERVICE_T_ROCE); + + if (ibsrq->uobject) { + struct roce5_ucontext *ucontext = roce5_queue_to_ucontext( + (void *)ibsrq, udata, ROCE_Q_TYPE_SRQ); + roce5_db_unmap_user(ucontext, &srq->db, 0); + ib_umem_release(srq->umem); + } else { + kfree(srq->wrid); + } + + cqm5_object_delete(&srq->cqm_srq->object); +} + +int roce5_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + int ret = 0; + int times = 0; + struct roce5_device *rdev = NULL; + struct roce5_srq *rsrq = NULL; + + if (ibsrq == NULL) { + ROCE_ERR("Ibsrq is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibsrq->device); + rsrq = to_roce5_srq(ibsrq); + times = rdev->try_times; + + ret = roce5_srq_hw2sw(rdev, rsrq); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to handle srq hw2sw, func_id(%d), ret:%d", + rdev->glb_func_id, ret); + return ret; + } + + roce5_free_srq(ibsrq, rdev, rsrq, udata); + return 0; +} + +int roce5_destroy_srq_ofa(struct ib_srq *ibsrq) +{ + int ret = roce5_destroy_srq(ibsrq, NULL); + if (ret == 0) { + kfree(ibsrq); + } + + return ret; +} + +static int roce5_check_sge_length(const struct ib_recv_wr *wr, + struct roce5_device *rdev, + const struct ib_recv_wr **bad_wr, + struct roce_wqe_srq_data_seg *scat) +{ + int i = 0; + int ret = 0; + u32 data_len = 0; + + for (i = 0; i < wr->num_sge; ++i) { + if (ROCE_UNLIKELY( + wr->sg_list[i].length > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_msg_sz - + 1)) { + ret = -EINVAL; + *bad_wr = wr; + ROCE_DEV_ERR( + rdev, + "Sge data length is over range, sg_list(%d), length(0x%x), max_msg_sz(0x%x), func_id(%d)", + i, wr->sg_list[i].length, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + return ret; + } + + data_len += wr->sg_list[i].length; + if (ROCE_UNLIKELY(data_len > + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz)) { + ret = -EINVAL; + *bad_wr = wr; + scat[0].dw2.length = 0; + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + + ROCE_DEV_ERR( + rdev, + "Data length is over range, data_len(0x%x), max_msg_sz(0x%x), func_id(%d)", + data_len, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap + .max_msg_sz, + rdev->glb_func_id); + return ret; + } + + scat[i].dw2.length = cpu_to_be32(wr->sg_list[i].length); + scat[i].dw3.lkey = + cpu_to_be32((u32)(wr->sg_list[i].lkey & + NORMAL_FMT_AND_NEXT_SGE_PRESENT)); + scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); + } + + if (ROCE_UNLIKELY(i == 0)) { + scat[0].dw3.lkey = LAST_SGE_NO_PRESENT; + scat[0].dw3.lkey = cpu_to_be32(scat[0].dw3.lkey); + scat[0].dw2.length = 0; + } else { + scat[i - 1].dw3.lkey = cpu_to_be32( + (u32)((wr->sg_list[i - 1].lkey | LAST_SGE_NO_PRESENT) & + NORMAL_FMT_AND_LAST_SGE_NO_PRESENT)); + } + + return ret; +} + +static int roce5_post_srq_recv_precheck(struct ib_srq *ibsrq, + const struct ib_recv_wr **bad_wr) +{ + if (ibsrq == NULL) { + ROCE_ERR("Ibsrq is null"); + return -EINVAL; + } + + if (bad_wr == NULL) { + ROCE_ERR("Bad_wr is null"); + return -EINVAL; + } + + return 0; +} + +int roce5_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + int ret = 0; + int nreq = 0; + unsigned long flags = 0; /* type required by kernel API */ + struct roce5_srq *srq = NULL; + struct roce5_device *rdev = NULL; + struct roce5_wqe_srq_next_seg *next = NULL; + struct roce_wqe_srq_data_seg *scat = NULL; + const struct ib_recv_wr *wr_tmp = wr; + + ret = roce5_post_srq_recv_precheck(ibsrq, bad_wr); + if (ret != 0) { + return -EINVAL; + } + + srq = to_roce5_srq(ibsrq); + rdev = to_roce5_dev(ibsrq->device); + + spin_lock_irqsave(&srq->lock, flags); + + for (nreq = 0; wr_tmp; ++nreq, wr_tmp = wr_tmp->next) { + if (ROCE_UNLIKELY((wr_tmp->num_sge > srq->max_gs) || + (srq->head == srq->tail))) { + ret = (wr_tmp->num_sge > srq->max_gs) ? (-EINVAL) : + (-ENOMEM); + *bad_wr = wr_tmp; + ROCE_DEV_ERR( + rdev, + "Num_sge(%d) > max_gs(%d) or srq is empty(head(%d) == tail(%d)), func_id(%d)", + wr_tmp->num_sge, srq->max_gs, rdev->glb_func_id, + srq->head, srq->tail); + break; + } + + next = (struct roce5_wqe_srq_next_seg *)roce5_srq_get_wqe( + srq, srq->head); + scat = (struct roce_wqe_srq_data_seg *)(next + 1); + + ret = roce5_check_sge_length(wr_tmp, rdev, bad_wr, scat); + if (ret != 0) { + goto sge_len_err; + } + + next->pcnt = (u16)cpu_to_be16(srq->wqe_ctr + (u16)nreq); + srq->wrid[srq->head] = wr_tmp->wr_id; + srq->head = be16_to_cpu(next->next_wqe_index); + } + +sge_len_err: + if (ROCE_UNLIKELY(nreq != 0)) { + srq->wqe_ctr += (u16)nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + *srq->db.db_record = cpu_to_be32(srq->wqe_ctr); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return ret; +} + +int roce5_post_srq_recv_ofa(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + return roce5_post_srq_recv(ibsrq, (const struct ib_recv_wr *)wr, + (const struct ib_recv_wr **)bad_wr); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h new file mode 100644 index 000000000..12d501b4c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/srq/roce_srq_format.h @@ -0,0 +1,36 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SRQ_FORMAT_H +#define ROCE_SRQ_FORMAT_H + +#include <linux/types.h> + +typedef struct roce5_srq_context { + /* DW0 */ + union { + struct { + u32 size : 5; + u32 page_size : 4; + u32 wqebb_size : 3; + u32 rsvd : 2; + u32 pdn : 18; + } bs; + u32 value; + } dw0; + + /* DW1 */ + union { + struct { + u32 rsvd : 24; + u32 dma_attr_idx : 6; + u32 so_ro : 2; + } bs_c; + u32 value; + } dw1; + + u32 rsvd[14]; +} roce5_srq_context_s; + +#endif /* ROCE_SRQ_FORMAT_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c new file mode 100644 index 000000000..e66a99f39 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_api.c @@ -0,0 +1,729 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "comm_defs.h" +#include "roce_uld_kernel_api.h" +#include "roce_bond.h" +#include "roce_uld_inner.h" +#include "roce_mpu_cmd.h" +#include "roce_cqm_cmd.h" + +#define PLUGIN __attribute__((__used__, __weak__, optimize("O0"))) +#define ROCE_ULD_PRINT_BUF_SIZE 256 + +PLUGIN int roce5_uld_reg_user_mr(roce_uld_handle handle) +{ + return -1; +} + +PLUGIN int roce5_uld_dereg_user_mr(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_adm_dfx_uld_extend(roce_uld_handle handle, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + return 0; +} + +PLUGIN int roce5_uld_qp_modify(roce_uld_handle handle, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + return 0; +} + +PLUGIN void roce5_uld_rdev_remove(roce_uld_handle handle) +{ + return; +} + +PLUGIN int roce5_uld_rdev_add(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_uld_qp_destroy(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_uld_qp_create(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN int roce5_uld_cq_create(roce_uld_handle handle) +{ + return 0; +} + +PLUGIN void roce5_uld_cq_destroy(roce_uld_handle handle) +{ + return; +} + +PLUGIN int32_t roce5_uld_user2kernel_msg_handle(roce_uld_handle handle, + void *buf_in, u16 buf_in_len, + void *buf_out, u16 buf_out_len, + u16 *buf_out_size) +{ + return 0; +} + +PLUGIN int roce5_uld_service_init(void) +{ + return 0; +} + +PLUGIN void roce5_uld_service_exit(void) +{ + return; +} + +u64 roce5_mr_virt_addr_get(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return 0; + } + + return hdl->virt_addr; +} + +int roce5_mr_virt_addr_set(roce_uld_handle handle, u64 virt_addr) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + hdl->virt_addr = virt_addr; + + return 0; +} + +int roce5_mr_user_data_get(roce_uld_handle handle, u8 *user_data, + u32 user_data_len) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + u32 cpy_len; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + if (user_data == NULL) { + ROCE_ERR("user_data is NULL"); + return -EINVAL; + } + + cpy_len = (user_data_len > hdl->mr->user_data_len) ? + hdl->mr->user_data_len : + user_data_len; + (void)memcpy_s(user_data, user_data_len, hdl->mr->rdmamr.user_data, + cpy_len); + + return cpy_len; +} + +int roce5_mpt_alloc_and_init(roce_uld_handle handle, int mode, u8 *user_data, + u32 user_data_len, u32 mpt_xid_mode) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_mr *mr = NULL; + struct roce5_device *rdev = NULL; + int ret = 0; + u32 npages = 0; + u32 page_shift = PAGE_SHIFT; + u32 mpt_xid_create_mode; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + mr = hdl->mr; + rdev = hdl->rdev; + if (rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return -EINVAL; + } + + if (user_data_len > HMM_USER_DATA_MAX_BYTE) { + ROCE_ERR( + "user_data_len(%u) is invalid, max user_data_len is %u", + user_data_len, HMM_USER_DATA_MAX_BYTE); + return -EINVAL; + } + + if (mode == ROCE_ULD_MPT_NORMAL) { + mr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + npages = (u32)ib_umem_num_pages(mr->umem); +#if defined(UMEM_HAVE_PAGE_SHIFT) + page_shift = (u32)(mr->umem->page_shift); +#endif + } + + mpt_xid_create_mode = CQM_DYNAMIC_XID_MOD( + CQM_XID_SEARCH_ALL, CQM_XID_LOW_BIT_0_0_1, mpt_xid_mode); + ret = roce5_alloc_tpt(rdev, &mr->rdmamr, npages, page_shift, + mpt_xid_create_mode); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc mpt and mtt, func_id(%d), npages(%u), page_shift(%u)", + hinic5_global_func_id(rdev->hwdev), npages, page_shift); + goto err_alloc_tpt; + } + + roce5_set_rdma_mr(&mr->rdmamr, RDMA_USER_MR, hdl->pdn, hdl->virt_addr, + hdl->length, hdl->access); + + if (mode == ROCE_ULD_MPT_NORMAL) { + ret = roce5_umem_write_mtt(rdev, &mr->rdmamr.mtt, mr->umem, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to write mtt, func_id(%d)", + hinic5_global_func_id(rdev->hwdev)); + goto err_write_mtt; + } + } + + if (user_data != NULL) { + (void)memcpy_s(hdl->mr->rdmamr.user_data, user_data_len, + user_data, user_data_len); + hdl->mr->user_data_len = user_data_len; + } else { + hdl->mr->user_data_len = 0; + } + + return 0; + +err_write_mtt: + roce5_free_tpt(rdev, &mr->rdmamr); + +err_alloc_tpt: + return ret; +} + +void roce5_mpt_destroy(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return; + } + + if (hdl->mr == NULL) { + ROCE_ERR("mr is NULL"); + return; + } + + rdev = hdl->rdev; + if (rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return; + } + + if (hdl->mr->rdmamr.enabled != RDMA_MPT_DISABLED) { + roce5_free_tpt(rdev, &hdl->mr->rdmamr); + } + + return; +} + +int roce5_mpt_enable(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + int ret; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -EINVAL; + } + + rdev = hdl->rdev; + if (rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return -EINVAL; + } + + /* here hmm will userdata send to npu */ + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &hdl->mr->rdmamr, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to enable mpt of user mr, func_id(%d)", + hinic5_global_func_id(rdev->hwdev)); + } + + return ret; +} + +int roce5_uld_get_func_id(roce_uld_handle handle) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) { + ROCE_ERR("handle is invalid"); + return -1; + } + + if (hdl->rdev == NULL) { + ROCE_ERR("rdev is NULL"); + return -1; + } + + return (int)hdl->rdev->glb_func_id; +} + +int roce5_uld_send_msg_to_mgmt(roce_uld_handle handle, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((cmd < ROCE_MPU_CMD_ULP_START) || (cmd >= ROCE_MPU_CMD_MAX)) { + ROCE_ERR("cmd is invalid"); + return -1; + } + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rdev == NULL) || (buf_in == NULL) || (buf_out == NULL) || + (out_size == NULL)) { + ROCE_ERR( + "handle is invalid, or buf_in(%d), buf_out(%d), out_size(%d) is invalid", + (buf_in == NULL), (buf_out == NULL), + (out_size == NULL)); + return -1; + } + + return hinic5_msg_to_mgmt_sync(hdl->rdev->hwdev, HINIC5_MOD_ROCE, cmd, + buf_in, in_size, buf_out, out_size, 0, + HINIC5_CHANNEL_ROCE); +} + +int roce5_uld_get_qp_info(roce_uld_handle handle, roce_uld_qp_info *qp_info) +{ + struct roce5_qp *rqp = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rqp == NULL) || (qp_info == NULL)) { + ROCE_ERR("param invalid"); + return -1; + } + + rqp = (struct roce5_qp *)hdl->rqp; + qp_info->qpn = rqp->qpn; + qp_info->qp_type = rqp->qp_type; + if (hdl->qp_attr != NULL) { + qp_info->dgid = (u8 *)&hdl->qp_attr->path_info.dgid[0]; + } + return 0; +} + +int roce5_uld_modify_qp_tx_bond_hash(roce_uld_handle handle, u32 tx_bond_hash) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->qp_attr == NULL)) { + ROCE_ERR("param invalid"); + return -1; + } + + hdl->qp_attr->path_info.dw0.bs.bond_tx_hash_value = tx_bond_hash & 0xf; + return 0; +} + +u8 *roce5_get_rdev_uld_def(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + return (u8 *)&rdev->uld_def.self_def[0]; +} + +void *roce5_uld_get_rdev_hwdev(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + void *hwdev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + hwdev = hdl->rdev->hwdev; + if (hwdev == NULL) { + ROCE_ERR("handle is invalid, hwdev is NULL"); + return NULL; + } + return hwdev; +} + +struct ib_device *roce5_uld_get_rdev_ibdev(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + struct ib_device *ib_dev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + ib_dev = &(hdl->rdev->ib_dev); + return ib_dev; +} + +struct hinic5_lld_dev *roce5_uld_get_rdev_lld_dev(roce_uld_handle handle) +{ + struct roce5_device *rdev = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return NULL; + } + lld_dev = hdl->rdev->lld_dev; + if (lld_dev == NULL) { + ROCE_ERR("handle is invalid, lld_dev is NULL"); + return NULL; + } + return lld_dev; +} + +#ifdef ROCE_BONDING_EN +int roce5_uld_get_rdev_bond_dev_slave_cnt(roce_uld_handle handle, + u32 *slave_cnt, uint8_t *bond_en) +{ + struct roce5_device *rdev = NULL; + struct roce5_bond_device *bond_dev = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (slave_cnt == NULL) || (bond_en == NULL)) { + return -EINVAL; + } + rdev = hdl->rdev; + if ((rdev == NULL) || (rdev->rdev_magic != ROCE_DEVICE_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rdev == NULL) ? ROCE_MAGIC_NULL : rdev->rdev_magic); + return -EINVAL; + } + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + *slave_cnt = 0; + *bond_en = 0; + return 0; + } + *slave_cnt = hdl->rdev->bond_dev->slave_cnt; + *bond_en = 1; + return 0; +} +#endif + +u8 *roce5_get_rqp_uld_def(roce_uld_handle handle) +{ + struct roce5_qp *rqp = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rqp = (struct roce5_qp *)hdl->rqp; + if ((rqp == NULL) || (rqp->rqp_magic != ROCE_RQP_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rqp == NULL) ? ROCE_MAGIC_NULL : rqp->rqp_magic); + return NULL; + } + return (u8 *)&rqp->uld_def.uld_def[0]; +} + +u8 *roce5_get_rcq_uld_def(roce_uld_handle handle) +{ + struct roce5_cq *rcq = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + return NULL; + } + + rcq = (struct roce5_cq *)hdl->rcq; + if ((rcq == NULL) || (rcq->rcq_magic != ROCE_RCQ_MAGIC)) { + ROCE_ERR("handle is invalid, magic:0x%x", + (rcq == NULL) ? ROCE_MAGIC_NULL : rcq->rcq_magic); + return NULL; + } + return (u8 *)&rcq->uld_def.uld_def[0]; +} + +int roce5_uld_cos_set(roce_uld_handle handle, struct ib_qp_attr *attr, u8 type, + u8 value) +{ + u8 cos; + roce5_get_cos_inbuf_s inbuf = { 0 }; + struct roce5_qp *rqp = NULL; + roce_verbs_qp_attr_s *qp_attr = NULL; + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + int ret; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rqp == NULL) || (hdl->qp_attr == NULL) || + (hdl->rdev == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + rqp = hdl->rqp; + qp_attr = hdl->qp_attr; + + if (type == ROCE5_COS_SET_TYPE_CLASS) { + inbuf.sl = attr->ah_attr.sl; + inbuf.sgid_index = attr->ah_attr.grh.sgid_index; + inbuf.port_num = attr->ah_attr.port_num; + inbuf.traffic_class = value; + + ret = roce5_get_dcb_cfg_cos(hdl->rdev, &inbuf, &cos, + rqp->qp_type); + if (ret != 0) { + ROCE_ERR("get dcb cfg cos failed"); + return ret; + } + + /* update tclass and cos */ + qp_attr->path_info.dw6.bs.tclass = value; + qp_attr->com_info.dw5.bs.cos = cos; + rqp->db_cos = cos; + } else if (type == ROCE5_COS_SET_TYPE_COS) { + if (value > 7) { /* cos range is 0-7 */ + ROCE_ERR("invalid cos(%u)", value); + return -EINVAL; + } + qp_attr->com_info.dw5.bs.cos = value; + rqp->db_cos = value; + } else { + ROCE_ERR("set type(%u) invalid", type); + return -EINVAL; + } + + return 0; +} + +int roce5_support_uld_kernel_feature(roce_uld_handle handle, u64 feature) +{ + roce_user_handle_t *user_handle = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + + if ((!user_handle) || + (user_handle->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + ROCE_ERR("Invalid RoCE handle\n"); + return -EINVAL; + } + + rdev = user_handle->rdev; + + if (rdev == NULL) { + ROCE_ERR("rdev is NULL in handle\n"); + return -EINVAL; + } + + //return capability bit AND result + return (rdev)->cfg_info.feature_cap.uld_feature + .roce_uld_kernel_ext_ability & + feature; +} + +int roce5_uld_set_mpt_xid_mode(roce_uld_handle handle, u8 mode) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + struct roce5_device *rdev = NULL; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->rdev == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + rdev = hdl->rdev; + rdev->mpt_xid_mode = mode; + + return 0; +} + +int roce5_uld_get_dgid(roce_uld_handle handle, u8 **dgid) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->qp_attr == NULL) || (dgid == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + *dgid = hdl->qp_attr->path_info.dgid; + return 0; +} + +int roce5_uld_set_bond_tx_hash_value(roce_uld_handle handle, + u32 bond_tx_hash_value) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->qp_attr == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + hdl->qp_attr->path_info.dw0.bs.bond_tx_hash_value = bond_tx_hash_value; + return 0; +} + +int roce5_uld_get_mr_uld_self_def(roce_uld_handle handle, u8 **self_def, + u32 *self_def_len) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM) || + (hdl->mr == NULL) || (self_def == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *self_def = &hdl->mr->uld_def.self_def[0]; + if (self_def_len != NULL) { + *self_def_len = ROCE_MR_ULD_DEF_LEN; + } + return 0; +} + +int roce5_uld_get_rdmamr(roce_uld_handle handle, struct hmm_rdma **rdmamr_ptr) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->mr == NULL) || (rdmamr_ptr == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *rdmamr_ptr = &(hdl->mr->rdmamr); + return 0; +} + +int roce5_uld_set_qp_attr_ack_to(roce_uld_handle handle, u32 ack_to) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->qp_attr == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + hdl->qp_attr->com_info.dw2.bs.ack_to = ack_to; + return 0; +} + +int roce5_uld_get_qp_state(roce_uld_handle handle, u8 *out_qp_state) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->rqp == NULL) || (out_qp_state == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *out_qp_state = hdl->rqp->qp_state; + return 0; +} + +int roce5_uld_get_config_num_ports(roce_uld_handle handle, + int *out_cfg_num_ports) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (hdl->rdev == NULL) || + (out_cfg_num_ports == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *out_cfg_num_ports = hdl->rdev->hw_info.config_num_ports; + return 0; +} + +int roce5_uld_get_handle_length(roce_uld_handle handle, u64 *length) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + + if ((hdl == NULL) || (length == NULL)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *length = hdl->length; + return 0; +} + +void roce5_uld_print_kernel_err_impl(roce_uld_handle handle, const char *func, + int line, const char *fmt, ...) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + va_list args; + char buf[ROCE_ULD_PRINT_BUF_SIZE]; + + va_start(args, fmt); + if (vsnprintf(buf, sizeof(buf), fmt, args) < 0) { + va_end(args); + return; + } + va_end(args); + + if ((hdl != NULL) && (hdl->magic == ROCE_ULD_HANDLE_MAGIC_NUM) && + (hdl->rdev != NULL)) { + ROCE_DEV_ERR(hdl->rdev, "[ULP][%s,%d]%s", func, line, buf); + } else { + ROCE_ERR("[ULP][%s,%d]%s", func, line, buf); + } +} + +int roce5_uld_qp_get_port(roce_uld_handle handle, u8 *port) +{ + roce_user_handle_t *hdl = (roce_user_handle_t *)handle; + if ((hdl == NULL) || (hdl->rqp == NULL) || (port == NULL) || + (hdl->magic != ROCE_ULD_HANDLE_MAGIC_NUM)) { + ROCE_ERR("param invalid"); + return -EINVAL; + } + + *port = hdl->rqp->port; + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h new file mode 100644 index 000000000..e61160c2f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/uld/roce_uld_inner.h @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef ROCE_ULD_INNER_H +#define ROCE_ULD_INNER_H + +#include "roce.h" +#include "roce_mr.h" +#include "roce_qp.h" +#include "roce_cq.h" +#include "hmm_common.h" +#include "roce_npu_cmd_mr_defs.h" +#include "roce_npu_cmd_qp_defs.h" + +#define ROCE_ULD_HANDLE_MAGIC_NUM 0x524F4345 /* "ROCE" string of ascii code */ +#define ROCE_MAGIC_NULL 0x12345678 /* used to distinguish null pointer and unassigned */ + +typedef struct { + u32 magic; /* magic number, determine handle whether valid */ + struct roce5_device *rdev; + struct roce5_mr *mr; + roce_verbs_mr_attr_s *mr_attr; + u64 virt_addr; + u64 length; + u32 pdn; + int access; + + struct roce5_qp *rqp; + roce_verbs_qp_attr_s *qp_attr; + + struct roce5_cq *rcq; + int cqe; + + struct ib_udata *udata; + struct mtt * + mtt; /* hmm module of mtt management structure, shadow scenario page_shift and paddr mandatory, mtt_layers constraint is 0 */ + dma_addr_t *db_dma_handle; /* software db of dma address */ + + u8 *shadow; + u32 *shadow_vfid; + u16 *shadow_vf_num; +} roce_user_handle_t; + +#define ROCE_CONSTUCT_ULD_HDL_RQP(hdl, roce_dev, roce_qp, roce_qp_attr, \ + roce_udata, roce_mtt, roce_db_dma_handle) \ + do { \ + (hdl)->magic = ROCE_ULD_HANDLE_MAGIC_NUM; \ + (hdl)->rdev = (roce_dev); \ + (hdl)->rqp = (roce_qp); \ + (hdl)->qp_attr = (roce_qp_attr); \ + (hdl)->udata = (roce_udata); \ + (hdl)->mtt = (roce_mtt); \ + (hdl)->db_dma_handle = (roce_db_dma_handle); \ + } while (0) + +#define ROCE_CONSTUCT_ULD_HDL_RCQ(hdl, roce_dev, roce_cq, roce_udata, \ + roce_mtt, roce_db_dma_handle, roce_cqe) \ + do { \ + (hdl)->magic = ROCE_ULD_HANDLE_MAGIC_NUM; \ + (hdl)->rdev = (roce_dev); \ + (hdl)->rcq = (roce_cq); \ + (hdl)->udata = (roce_udata); \ + (hdl)->mtt = (roce_mtt); \ + (hdl)->db_dma_handle = (roce_db_dma_handle); \ + (hdl)->cqe = (roce_cqe); \ + } while (0) + +#define ROCE_CONSTUCT_ULD_HDL_SHADOW(hdl, roce_shadow, roce_shadow_vfid, \ + roce_shadow_vf_num) \ + do { \ + (hdl)->shadow = (roce_shadow); \ + (hdl)->shadow_vfid = (roce_shadow_vfid); \ + (hdl)->shadow_vf_num = (roce_shadow_vf_num); \ + } while (0) + +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c new file mode 100644 index 000000000..7530a655b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.c @@ -0,0 +1,169 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/slab.h> + +#include "roce.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "roce_xrc.h" + +static int roce5_rdma_xrcd_alloc(void *hwdev, u32 *xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (xrcdn == NULL)) { + ROCE_ERR("Hwdev or xrcdn is null"); + return -EINVAL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -EINVAL; + } + + *xrcdn = rdma_bitmap_alloc(&comp_priv->xrcd_bitmap); + if (*xrcdn == RDMA_INVALID_INDEX) { + ROCE_ERR("Can't get valid xrcdn, err(%d)", -ENOMEM); + return -ENOMEM; + } + + return 0; +} + +static void roce5_rdma_xrcd_free(void *hwdev, u32 xrcdn) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return; + } + + rdma_bitmap_free(&comp_priv->xrcd_bitmap, xrcdn); +} + +static int roce5_init_xrcd(struct ib_device *ibdev, struct roce5_device *rdev, + struct roce5_xrcd *xrcd) +{ + int ret = 0; + struct ib_cq_init_attr cq_attr = { 0 }; + + ret = roce5_rdma_xrcd_alloc(rdev->hwdev, &xrcd->xrcdn); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma xrcd, ret(%d)", ret); + goto err_alloc_xrcdn; + } + + xrcd->pd = ib_alloc_pd(ibdev, 0); + + if (IS_ERR(xrcd->pd)) { + ret = (int)PTR_ERR(xrcd->pd); + ROCE_DEV_ERR(rdev, "Failed to alloc pd, ret(%d)", ret); + goto err_alloc_pd; + } + + cq_attr.cqe = 1; + xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr); + if (IS_ERR(xrcd->cq)) { + ret = (int)PTR_ERR(xrcd->cq); + ROCE_DEV_ERR(rdev, "Failed to create cq, ret(%d)", ret); + goto err_create_cq; + } + return 0; + +err_create_cq: + ib_dealloc_pd(xrcd->pd); + +err_alloc_pd: + roce5_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + +err_alloc_xrcdn: + return ret; +} + +int roce5_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + struct roce5_xrcd *xrcd = NULL; + + if (ibxrcd == NULL) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rdev = to_roce5_dev(ibxrcd->device); + xrcd = to_roce5_xrcd(ibxrcd); + + return roce5_init_xrcd(ibxrcd->device, rdev, xrcd); +} + +struct ib_xrcd *roce5_alloc_xrcd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata) +{ + int ret; + struct roce5_device *rdev = NULL; + struct roce5_xrcd *xrcd = NULL; + + if (ibdev == NULL) { + ROCE_ERR("Ibdev is null"); + return ERR_PTR(-EINVAL); + } + + rdev = to_roce5_dev(ibdev); + + xrcd = (struct roce5_xrcd *)kzalloc(sizeof(*xrcd), GFP_KERNEL); + if (xrcd == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc memory for xrcd"); + return ERR_PTR(-ENOMEM); + } + + /* Unreasonable, but for code reuse, driver associate the OFED resource in advance. */ + xrcd->ibxrcd.device = ibdev; + + ret = roce5_alloc_xrcd(&xrcd->ibxrcd, NULL); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init xrcd, ret(%d)", ret); + kfree(xrcd); + return ERR_PTR(ret); + } + return &xrcd->ibxrcd; +} + +int roce5_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) +{ + struct roce5_device *rdev = NULL; + struct roce5_xrcd *xrcd = NULL; + if (ibxrcd == NULL) { + ROCE_ERR("Ibxrcd is null"); + return -EINVAL; + } + + rdev = to_roce5_dev(ibxrcd->device); + xrcd = to_roce5_xrcd(ibxrcd); + ib_destroy_cq(xrcd->cq); + + ib_dealloc_pd(xrcd->pd); + roce5_rdma_xrcd_free(rdev->hwdev, xrcd->xrcdn); + return 0; +} + +int roce5_dealloc_xrcd_ofa(struct ib_xrcd *ibxrcd) +{ + struct roce5_xrcd *xrcd = to_roce5_xrcd(ibxrcd); + int ret = roce5_dealloc_xrcd(ibxrcd, NULL); + if (ret == 0) { + kfree(xrcd); + } + + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h new file mode 100644 index 000000000..1a4e603ab --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/base/xrc/roce_xrc.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_XRC_H +#define ROCE_XRC_H + +#include <rdma/ib_verbs.h> + +struct roce5_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; + struct ib_pd *pd; + struct ib_cq *cq; +}; + +#endif //ROCE_XRC_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c new file mode 100644 index 000000000..d67dd0844 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.c @@ -0,0 +1,162 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +#include <rdma/ib_user_verbs.h> +#include <linux/errno.h> +#include <linux/init.h> + +#include "roce.h" +#include "hinic5_lld.h" +#include "roce_infra.h" +#include "roce_card.h" + +static struct roce5_cards g_roce5_cards[MAX_DPU_CARD_NUM]; + +void roce5_card_init(u16 idx, u64 card_id) +{ + g_roce5_cards[idx].card_status = RUNNING; + g_roce5_cards[idx].card_id = card_id; + INIT_LIST_HEAD(&g_roce5_cards[idx].rdev_list); + mutex_init(&g_roce5_cards[idx].card_mutex); +} + +void roce5_card_deinit(u16 idx) +{ + g_roce5_cards[idx].card_status = INVALID; + g_roce5_cards[idx].card_id = 0; + mutex_destroy(&g_roce5_cards[idx].card_mutex); +} + +int roce5_g_rdev_add(struct roce5_device *rdev) +{ + struct roce5_rdev_list_node *rdev_node = NULL; + u64 card_id; + u16 i; + + if (hinic5_get_chip_node_id(rdev->lld_dev, &card_id) != 0) { + ROCE_ERR("Failed to get chip node id, func_id(%d)", + rdev->glb_func_id); + return -EINVAL; + } + + rdev_node = kzalloc(sizeof(struct roce5_rdev_list_node), GFP_KERNEL); + if (rdev_node == NULL) { + ROCE_ERR("Failed to alloc memory."); + return -ENOMEM; + } + rdev_node->rdev = rdev; + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == RUNNING && + g_roce5_cards[i].card_id == card_id) { + break; + } + } + + if (i >= MAX_DPU_CARD_NUM) { + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == INVALID) { + roce5_card_init(i, card_id); + break; + } + } + + if (i >= MAX_DPU_CARD_NUM) { + ROCE_ERR("Exceeded the MAX_DPU_CARD_NUM:0x%x", + MAX_DPU_CARD_NUM); + kfree(rdev_node); + return -ENOMEM; + } + } + + mutex_lock(&g_roce5_cards[i].card_mutex); + list_add_tail(&rdev_node->node, &g_roce5_cards[i].rdev_list); + mutex_unlock(&g_roce5_cards[i].card_mutex); + return 0; +} + +void roce5_g_rdev_remove(struct roce5_device *rdev) +{ + struct roce5_rdev_list_node *rdev_node = NULL, *rdev_next = NULL; + u64 card_id; + int i; + + if (hinic5_get_chip_node_id(rdev->lld_dev, &card_id) != 0) { + ROCE_ERR("Failed to get chip node id, func_id(%d)", + rdev->glb_func_id); + return; + } + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == RUNNING && + g_roce5_cards[i].card_id == card_id) { + break; + } + } + + if (i < MAX_DPU_CARD_NUM) { + mutex_lock(&g_roce5_cards[i].card_mutex); + list_for_each_entry_safe(rdev_node, rdev_next, + &g_roce5_cards[i].rdev_list, node) { + if (rdev_node->rdev == rdev) { + list_del(&rdev_node->node); + kfree(rdev_node); + break; + } + } + mutex_unlock(&g_roce5_cards[i].card_mutex); + /* deinit card if rdev list is empty */ + if (list_empty(&g_roce5_cards[i].rdev_list)) { + roce5_card_deinit(i); + } + } +} + +struct roce5_device *roce5_get_rdev_by_card_func_id(uint64_t card_id, + uint16_t func_id) +{ + struct roce5_rdev_list_node *rdev_node = NULL; + int i; + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == RUNNING && + g_roce5_cards[i].card_id == card_id) { + break; + } + } + + if (i < MAX_DPU_CARD_NUM) { + list_for_each_entry(rdev_node, &g_roce5_cards[i].rdev_list, + node) { + if (rdev_node->rdev->glb_func_id == func_id) { + return rdev_node->rdev; + } + } + } + return NULL; +} + +struct roce5_device *roce5_get_rdev_by_name(const char *dev_name) +{ + struct roce5_rdev_list_node *rdev_node = NULL; + int i; + + if (dev_name == NULL) { + return NULL; + } + + for (i = 0; i < MAX_DPU_CARD_NUM; i++) { + if (g_roce5_cards[i].card_status == INVALID) { + continue; + } + + list_for_each_entry(rdev_node, &g_roce5_cards[i].rdev_list, + node) { + if (strncmp(rdev_node->rdev->ib_dev.name, dev_name, + IB_DEVICE_NAME_MAX) == 0) { + return rdev_node->rdev; + } + } + } + return NULL; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h new file mode 100644 index 000000000..5ceabdd1f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/card/roce_card.h @@ -0,0 +1,27 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +#ifndef ROCE_CARD_H +#define ROCE_CARD_H + +#include <linux/slab.h> +#include <linux/list.h> + +#include "roce.h" + +#define MAX_DPU_CARD_NUM \ + 1024 //current SDK PCIe on current cannot identifies VM on and bare metal on is same one 张 card, temporary add specification supports start multi VM +struct roce5_rdev_list_node { + struct list_head node; + struct roce5_device *rdev; +}; + +typedef enum roce5_card_status { INVALID = 0, RUNNING } roce5_card_status_e; + +struct roce5_cards { + u64 card_id; + roce5_card_status_e card_status; + struct mutex card_mutex; + struct list_head rdev_list; +}; +#endif /* ROCE_CARD_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c new file mode 100644 index 000000000..2ad39f97f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.c @@ -0,0 +1,1270 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/errno.h> +#include <linux/if_ether.h> +#include <linux/moduleparam.h> +#include <linux/types.h> + +#include "rdma_comp.h" +#include "roce5_gid_type.h" +#include "roce_compat.h" +#include "roce_com.h" +#include "roce_cqm_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_ctx_format.h" +#include "roce_infra.h" +#include "roce_main_extension.h" +#include "roce_mix.h" +#include "roce_npu_cmd.h" +#include "roce_npu_cmd_ext_defs.h" +#include "roce_srq_extension.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "securec.h" +#include "roce_cmd.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#define OVS_MPU_CMD_QOS_SET_VM_POLICING 30 +#define OVS_MPU_CMD_VPORT_SET 13 +#define OVS_MPU_VPORT_SET_CMD_ATTR 2 + +uint8_t roce5_srq_mode_chip_adapt(uint8_t cfg_mode) +{ + switch (cfg_mode) { + case ROCE_CHIP_SRQ_MODE_1: + return ROCE_SRQ_MODE_3; + case ROCE_CHIP_SRQ_MODE_2: + return ROCE_SRQ_MODE_2; + case ROCE_CHIP_SRQ_MODE_3: + return ROCE_SRQ_MODE_1; + case ROCE_CHIP_SRQ_MODE_4: + return ROCE_SRQ_MODE_0; + default: + return ROCE_SRQ_MODE_3; + } +} + +int roce5_add_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id) +{ + void *hwdev = rdev->hwdev; + u16 er_fwd_id = rdev->glb_func_id; + roce_cfg_mac_cmd_s add_mac_entry_info; + u16 out_size = (u16)sizeof(add_mac_entry_info); + int ret; + + if (ROCE5_SUPPORT_SML_MAC_TABLE(rdev) == 0) { + return 0; + } + + if (rdev->hca_vf_mode != 0) { + ROCE_INFO("No need to add mac table entry."); + return 0; + } + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_ERR("Hwdev or mac_addr is null"); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_ERR("Invalid vlan_id(%u)", vlan_id); + return -EINVAL; + } + + ret = memset_s(&add_mac_entry_info, sizeof(add_mac_entry_info), 0, + sizeof(add_mac_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset add_mac_entry_info.(ret:%d)", ret); + return -ENOMEM; + } + add_mac_entry_info.func_id = func_id; + add_mac_entry_info.vlan_id = vlan_id; + add_mac_entry_info.er_fwd_id = er_fwd_id; + ret = memcpy_s(add_mac_entry_info.mac, ETH_ALEN, mac_addr, ETH_ALEN); + if (ret != 0) { + ROCE_ERR("memory copy failed. ret%d", ret); + return -ENOMEM; + } + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_MAC, + &add_mac_entry_info, + sizeof(add_mac_entry_info), + &add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (add_mac_entry_info.head.status != 0)) { + ROCE_ERR( + "Failed to add mac entry, err(%d), status(0x%x), out size(0x%x)", + ret, add_mac_entry_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +int roce5_del_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id) +{ + void *hwdev = rdev->hwdev; + u16 er_fwd_id = rdev->glb_func_id; + roce_cfg_mac_cmd_s del_mac_tbl_info; + u16 out_size = (u16)sizeof(del_mac_tbl_info); + int ret; + + if (ROCE5_SUPPORT_SML_MAC_TABLE(rdev) == 0) { + return 0; + } + + if (rdev->hca_vf_mode != 0) { + ROCE_DEV_INFO(rdev, "No need to delete mac table entry."); + return 0; + } + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_ERR("Hwdev or mac_addr is null"); + return -EINVAL; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_ERR("Invalid vlan_id(%u)", vlan_id); + return -EINVAL; + } + + ret = memset_s(&del_mac_tbl_info, sizeof(del_mac_tbl_info), 0, + sizeof(del_mac_tbl_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset del_mac_tbl_info.(ret:%d)", ret); + return -ENOMEM; + } + del_mac_tbl_info.func_id = func_id; + del_mac_tbl_info.vlan_id = vlan_id; + del_mac_tbl_info.er_fwd_id = er_fwd_id; + ret = memcpy_s(del_mac_tbl_info.mac, mac_addr_len, mac_addr, + mac_addr_len); + if (ret != 0) { + ROCE_ERR("Failed to memcpy_s dmac.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_MAC, + &del_mac_tbl_info, + sizeof(del_mac_tbl_info), + &del_mac_tbl_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (del_mac_tbl_info.head.status != 0)) { + ROCE_ERR( + "Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)", + ret, del_mac_tbl_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +void roce5_add_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], u32 vlan_id, + u16 func_id, u8 er_id, u16 src_func_id) +{ + void *hwdev = rdev->hwdev; + roce_cfg_ipsu_mac_cmd_s ipsu_add_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_add_mac_entry_info); + int ret; + + if (rdev->hca_vf_mode != 0) { + ROCE_INFO("No need to add ipsu tbl mac entry"); + return; + } + +#ifdef ROCE_BONDING_EN + if (!roce5_get_bond_ipsurx_en()) { + ROCE_ERR("No need to add ipsu tbl mac entry"); + return; + } +#endif + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_ERR("Hwdev or mac_addr is null"); + return; + } + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_ERR("Invalid vlan_id(%u)", vlan_id); + } + + ret = memset_s(&ipsu_add_mac_entry_info, + sizeof(ipsu_add_mac_entry_info), 0, + sizeof(ipsu_add_mac_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_add_mac_entry_info.(ret:%d)", + ret); + return; + } + ipsu_add_mac_entry_info.func_id = func_id; + ipsu_add_mac_entry_info.er_id = er_id; + ipsu_add_mac_entry_info.src_func_id = src_func_id; + ipsu_add_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_add_mac_entry_info.vni_en = 0; + ret = memcpy_s(ipsu_add_mac_entry_info.mac, ETH_ALEN, mac_addr, + ETH_ALEN); + if (ret != 0) { + ROCE_ERR( + "Failed to memcpy ipsu_add_mac_entry_info.mac (ret:%d)", + ret); + return; + } + + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_ADD_IPSU_MAC, + &ipsu_add_mac_entry_info, + sizeof(ipsu_add_mac_entry_info), + &ipsu_add_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_add_mac_entry_info.head.status != 0) && + (ipsu_add_mac_entry_info.head.status != ERR_EXIST))) { + ROCE_ERR( + "Failed to add mac entry to IPSU, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_add_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_add_mac_entry_info.head.status == ERR_EXIST) { + ROCE_INFO("This mac entry has been added to IPSU"); + } + + return; +} + +static int32_t roce5_del_mac_entry_precheck(struct roce5_device *rdev, + void *hwdev, uint8_t *mac_addr, + uint32_t vlan_id) +{ + if (rdev->hca_vf_mode != 0) { + ROCE_DEV_INFO(rdev, "No need to del ipsu tbl mac entry."); + return -EINVAL; + } + + if ((hwdev == NULL) || (mac_addr == NULL)) { + ROCE_DEV_ERR(rdev, "hwdev or mac_addr is null."); + return -EINVAL; + } + +#ifdef ROCE_BONDING_EN + if (!roce5_get_bond_ipsurx_en()) { + ROCE_DEV_INFO(rdev, "No need to del ipsu tbl mac entry."); + return -EINVAL; + } +#endif + + if ((vlan_id & ROCE_VLAN_ID_MASK) >= ROCE_VLAN_N_VID) { + ROCE_DEV_ERR(rdev, "Invalid vlan_id(%d).", vlan_id); + return -EINVAL; + } + + return 0; +} + +void roce5_del_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], size_t eth_alen, + u32 vlan_id, u8 er_id, u16 src_func_id) +{ + void *hwdev = rdev->hwdev; + u16 func_id = rdev->glb_func_id; + roce_cfg_ipsu_mac_cmd_s ipsu_del_mac_entry_info; + u16 out_size = (u16)sizeof(ipsu_del_mac_entry_info); + int ret; + + ret = roce5_del_mac_entry_precheck(rdev, hwdev, mac_addr, vlan_id); + if (ret != 0) { + return; + } + + ret = memset_s(&ipsu_del_mac_entry_info, + sizeof(ipsu_del_mac_entry_info), 0, + sizeof(ipsu_del_mac_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_del_mac_entry_info.(ret:%d)", + ret); + return; + } + ipsu_del_mac_entry_info.func_id = func_id; + ipsu_del_mac_entry_info.er_id = er_id; + ipsu_del_mac_entry_info.src_func_id = src_func_id; + ipsu_del_mac_entry_info.vlanid_vni = vlan_id & ROCE_VLAN_ID_MASK; + ipsu_del_mac_entry_info.vni_en = 0; + ret = memcpy_s(ipsu_del_mac_entry_info.mac, eth_alen, mac_addr, + eth_alen); + if (ret != 0) { + ROCE_ERR("Failed to memcpy_s dmac.(ret:%d)", ret); + return; + } + + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_DEL_IPSU_MAC, + &ipsu_del_mac_entry_info, + sizeof(ipsu_del_mac_entry_info), + &ipsu_del_mac_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_mac_entry_info.head.status != 0) && + (ipsu_del_mac_entry_info.head.status != ERR_NOT_FOUND))) { + ROCE_ERR( + "Failed to del mac entry, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_del_mac_entry_info.head.status, out_size); + return; + } + + if (ipsu_del_mac_entry_info.head.status == ERR_NOT_FOUND) { + ROCE_INFO("This mac entry of ipsu has been deleted"); + } + + return; +} + +int roce5_add_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info) +{ + roce_cfg_ipsu_ip_cmd_s ipsu_add_ip_entry_info; + u16 out_size = sizeof(ipsu_add_ip_entry_info); + union roce_addr_info *ip = NULL; + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL) || (addr_info == NULL)) { + ROCE_ERR("rdev, hwdev or addr_info is null"); + return -EINVAL; + } + + ip = &addr_info->ip_addr; + + ret = memset_s(&ipsu_add_ip_entry_info, sizeof(ipsu_add_ip_entry_info), + 0, sizeof(ipsu_add_ip_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_add_ip_entry_info.(ret:%d)", + ret); + return -ENOMEM; + } + + ipsu_add_ip_entry_info.func_id = rdev->glb_func_id; + ipsu_add_ip_entry_info.ip_ver = addr_info->ip_ver; + ipsu_add_ip_entry_info.port_id = hinic5_physical_port_id(rdev->hwdev); + ret = memcpy_s(&ipsu_add_ip_entry_info.ip, sizeof(union roce_addr_info), + ip, sizeof(union roce_addr_info)); + if (ret) { + ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret); + return -ENOMEM; + } + + ROCE_INFO("funcid=%u,ipver=%u,portid=%u,ip=0x%x-%x-%x-%x", + ipsu_add_ip_entry_info.func_id, ipsu_add_ip_entry_info.ip_ver, + ipsu_add_ip_entry_info.port_id, ip->dw[0x0], ip->dw[0x1], + ip->dw[0x2], ip->dw[0x3]); + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_ADD_IPSU_IP, + &ipsu_add_ip_entry_info, + sizeof(ipsu_add_ip_entry_info), + &ipsu_add_ip_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_add_ip_entry_info.head.status != 0) && + (ipsu_add_ip_entry_info.head.status != ERR_EXIST))) { + ROCE_ERR( + "Failed to add ip tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_add_ip_entry_info.head.status, out_size); + return -EINVAL; + } + + if (ipsu_add_ip_entry_info.head.status == ERR_EXIST) { + ROCE_INFO("This ip tbl entry has been added to IPSU"); + } + + return 0; +} + +int roce5_del_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info) +{ + roce_cfg_ipsu_ip_cmd_s ipsu_del_ip_entry_info; + u16 out_size = sizeof(ipsu_del_ip_entry_info); + union roce_addr_info *ip = NULL; + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL) || (addr_info == NULL)) { + ROCE_ERR("rdev, hwdev or addr_info is null"); + return -EINVAL; + } + + ip = &addr_info->ip_addr; + + ret = memset_s(&ipsu_del_ip_entry_info, sizeof(ipsu_del_ip_entry_info), + 0, sizeof(ipsu_del_ip_entry_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset ipsu_add_ip_entry_info.(ret:%d)", + ret); + return -ENOMEM; + } + + ipsu_del_ip_entry_info.func_id = rdev->glb_func_id; + ipsu_del_ip_entry_info.ip_ver = addr_info->ip_ver; + ipsu_del_ip_entry_info.port_id = hinic5_physical_port_id(rdev->hwdev); + ret = memcpy_s(&ipsu_del_ip_entry_info.ip, sizeof(union roce_addr_info), + ip, sizeof(union roce_addr_info)); + if (ret) { + ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret); + return -ENOMEM; + } + + ROCE_INFO("funcid=%u,ipver=%u,portid=%u,ip=0x%x-%x-%x-%x", + ipsu_del_ip_entry_info.func_id, ipsu_del_ip_entry_info.ip_ver, + ipsu_del_ip_entry_info.port_id, ip->dw[0x0], ip->dw[0x1], + ip->dw[0x2], ip->dw[0x3]); + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_IPSU_IP, + &ipsu_del_ip_entry_info, + sizeof(ipsu_del_ip_entry_info), + &ipsu_del_ip_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_ip_entry_info.head.status != 0) && + (ipsu_del_ip_entry_info.head.status != ERR_NOT_FOUND))) { + ROCE_ERR( + "Failed to add ip tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x)", + ret, ipsu_del_ip_entry_info.head.status, out_size); + return -EINVAL; + } + + if (ipsu_del_ip_entry_info.head.status == ERR_NOT_FOUND) { + ROCE_INFO("This ip tbl entry of ipsu has been deleted"); + } + + return 0; +} + +int roce5_modify_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, u8 type) +{ + struct roce_cmd_mpu_addr_info addr_info = { 0 }; + union roce_addr_info *ip = NULL; + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_ERR("rdev, hwdev or gid_entry is null"); + return -EINVAL; + } + + ip = (union roce_addr_info *)&gid_entry->raw; + + if (rdev->hca_vf_mode == 0) { + ROCE_INFO("No need to modify ipsu tbl ip entry"); + return 0; + } + + if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) { + ROCE_ERR("gid_type:%d is unsupported", + gid_entry->dw6_h.bs.gid_type); + return -EINVAL; + } + + ret = memcpy_s(&addr_info.ip_addr, sizeof(union roce_addr_info), ip, + sizeof(union roce_addr_info)); + if (ret) { + ROCE_ERR("Failed to memcpy ip info.(ret:%d)", ret); + return -ENOMEM; + } + + if (gid_entry->dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + addr_info.ip_addr.dw[0x0] = 0x0; + addr_info.ip_addr.dw[0x1] = 0x0; + addr_info.ip_addr.dw[0x2] = 0x0; + addr_info.ip_ver = 0; + } else { + addr_info.ip_ver = 1; + } + + if (type == ROCE_MPU_CMD_ADD_IPSU_IP) { + return roce5_add_ipsu_tbl_ip_entry(rdev, &addr_info); + } + + return roce5_del_ipsu_tbl_ip_entry(rdev, &addr_info); +} + +static int32_t +roce5_add_ipsu_tcam_tbl_entry_precheck(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_DEV_ERR(rdev, "hwdev or gid_entry is null."); + return -EINVAL; + } + + if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) { + ROCE_DEV_ERR(rdev, "gid_type:%d is unsupported.", + gid_entry->dw6_h.bs.gid_type); + return -EINVAL; + } + + return 0; +} + +static int32_t roce5_ipsu_tcam_tbl_set_entry( + struct roce5_device *rdev, struct rdma_gid_entry *gid_entry, + uint32_t gid_index, roce_cfg_ipsu_tcam_cmd_s *ipsu_add_tcam_entry_info) +{ + union roce_addr_info *ip = (union roce_addr_info *)&gid_entry->raw; + int32_t i; + int32_t ret; + + ipsu_add_tcam_entry_info->func_id = rdev->glb_func_id; + ipsu_add_tcam_entry_info->ip_ver = gid_entry->dw6_h.bs.gid_type; + ipsu_add_tcam_entry_info->gid_idx = gid_index; + ret = memcpy_s(&ipsu_add_tcam_entry_info->ip, + sizeof(union roce_addr_info), ip, + sizeof(union roce_addr_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy ip info.(ret:%d).", ret); + return -EINVAL; + } + + for (i = 0; i < 0x4; i++) { + ipsu_add_tcam_entry_info->ip.dw[i] = + htonl(ipsu_add_tcam_entry_info->ip.dw[i]); + } + + if (gid_entry->dw6_h.bs.gid_type == ROCE_IPv4_ROCEv2_GID) { + ipsu_add_tcam_entry_info->ip.dw[0x0] = 0x0; + ipsu_add_tcam_entry_info->ip.dw[0x1] = 0x0; + ipsu_add_tcam_entry_info->ip.dw[0x2] = 0x0; + } + + return 0; +} + +void roce5_add_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, + u32 gid_index) +{ + roce_cfg_ipsu_tcam_cmd_s ipsu_add_tcam_entry_info; + u16 out_size = sizeof(roce_cfg_ipsu_tcam_cmd_s); + int ret; + + ret = roce5_add_ipsu_tcam_tbl_entry_precheck(rdev, gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to precheck params(ret:%d).", ret); + return; + } + + ret = memset_s(&ipsu_add_tcam_entry_info, + sizeof(ipsu_add_tcam_entry_info), 0, + sizeof(ipsu_add_tcam_entry_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to memset ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ret = roce5_ipsu_tcam_tbl_set_entry(rdev, gid_entry, gid_index, + &ipsu_add_tcam_entry_info); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ROCE_DEV_INFO(rdev, "funcid=%u,ipver=%u,gid_idx=%u,ip=0x%x-%x-%x-%x.", + ipsu_add_tcam_entry_info.func_id, + ipsu_add_tcam_entry_info.ip_ver, + ipsu_add_tcam_entry_info.gid_idx, + ipsu_add_tcam_entry_info.ip.dw[0x0], + ipsu_add_tcam_entry_info.ip.dw[0x1], + ipsu_add_tcam_entry_info.ip.dw[0x2], + ipsu_add_tcam_entry_info.ip.dw[0x3]); + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_ADD_IPSU_TCAM, + &ipsu_add_tcam_entry_info, + sizeof(ipsu_add_tcam_entry_info), + &ipsu_add_tcam_entry_info, &out_size); + if (ipsu_add_tcam_entry_info.head.status == ERR_FULL) { + ROCE_DEV_INFO(rdev, "TCAM Table is full."); + } else if ((ret == 0) && (out_size != 0) && + (ipsu_add_tcam_entry_info.head.status == ERR_EXIST)) { + ROCE_DEV_INFO(rdev, "This tcam entry has been added to IPSU."); + } else if ((ret != 0) || (out_size == 0) || + ((ipsu_add_tcam_entry_info.head.status != 0))) { + ROCE_DEV_ERR( + rdev, + "Failed to add tcam tbl entry to IPSU, err(%d), status(0x%x), out size(0x%x).", + ret, ipsu_add_tcam_entry_info.head.status, out_size); + return; + } + + return; +} + +static int32_t +roce5_del_ipsu_tcam_tbl_entry_precheck(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + if ((rdev->hwdev == NULL) || (gid_entry == NULL)) { + ROCE_DEV_ERR(rdev, "hwdev or gid_entry is null."); + return -EINVAL; + } + + if (gid_entry->dw6_h.bs.gid_type >= ROCE_ROCEv1_GID) { + ROCE_DEV_ERR(rdev, "gid_type:%d is unsupported", + gid_entry->dw6_h.bs.gid_type); + return -EINVAL; + } + + return 0; +} + +void roce5_del_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry) +{ + roce_cfg_ipsu_tcam_cmd_s ipsu_del_tcam_entry_info; + u16 out_size = sizeof(roce_cfg_ipsu_tcam_cmd_s); + int ret; + + ret = roce5_del_ipsu_tcam_tbl_entry_precheck(rdev, gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to precheck params(ret:%d).", ret); + return; + } + + ret = memset_s(&ipsu_del_tcam_entry_info, + sizeof(ipsu_del_tcam_entry_info), 0, + sizeof(ipsu_del_tcam_entry_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to memset ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ret = roce5_ipsu_tcam_tbl_set_entry(rdev, gid_entry, 0, + &ipsu_del_tcam_entry_info); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ipsu_tcam_entry_info(ret:%d).", + ret); + return; + } + + ROCE_DEV_INFO(rdev, "funcid=%u,ipver=%u,gid_idx=%u,ip=0x%x-%x-%x-%x", + ipsu_del_tcam_entry_info.func_id, + ipsu_del_tcam_entry_info.ip_ver, + ipsu_del_tcam_entry_info.gid_idx, + ipsu_del_tcam_entry_info.ip.dw[0x0], + ipsu_del_tcam_entry_info.ip.dw[0x1], + ipsu_del_tcam_entry_info.ip.dw[0x2], + ipsu_del_tcam_entry_info.ip.dw[0x3]); + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_IPSU_TCAM, + &ipsu_del_tcam_entry_info, + sizeof(ipsu_del_tcam_entry_info), + &ipsu_del_tcam_entry_info, &out_size); + if ((ret != 0) || (out_size == 0) || + ((ipsu_del_tcam_entry_info.head.status != 0) && + (ipsu_del_tcam_entry_info.head.status != ERR_NOT_FOUND))) { + ROCE_DEV_ERR( + rdev, + "Failed to del tcam entry, err(%d), status(0x%x), out size(0x%x).", + ret, ipsu_del_tcam_entry_info.head.status, out_size); + return; + } + + if (ipsu_del_tcam_entry_info.head.status == ERR_NOT_FOUND) { + ROCE_DEV_INFO(rdev, + "This tcam entry of ipsu has been deleted."); + } + return; +} + +int roce5_set_sysfs_cfg_param(void *hwdev, u16 func_id, u32 *param, + u32 param_len, u16 type) +{ + roce_cc_cfg_param_cmd_s cfg_param; + u16 out_size = (u16)sizeof(cfg_param); + int ret; + int param_idx = 0; + int param_dw_num; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + param_dw_num = (int)(param_len / sizeof(u32)); + if (param_dw_num > ROCE_CC_CFG_PARAM_MAX_DW_NUM) { + ROCE_ERR("cfg_param len :%d dw_num:%d exceed max, type:0x%x", + param_len, param_dw_num, type); + return -EINVAL; + } + + ret = memset_s(&cfg_param, sizeof(cfg_param), 0, sizeof(cfg_param)); + if (ret != 0) { + ROCE_ERR("Failed to memset cfg_param.(ret:%d)", ret); + return -ENOMEM; + } + + cfg_param.func_id = func_id; + for (; param_idx < param_dw_num; param_idx++) { + cfg_param.param[param_idx] = param[param_idx]; + } + + ret = roce5_msg_to_mgmt_sync(hwdev, type, &cfg_param, sizeof(cfg_param), + &cfg_param, &out_size); + if ((ret != 0) || (out_size == 0) || (cfg_param.head.status != 0)) { + ROCE_ERR( + "Failed to cfg param, err(%d), status(0x%x), out size(0x%x), type(0x%x)", + ret, cfg_param.head.status, out_size, type); + + return -EINVAL; + } + return 0; +} + +int roce5_set_func_tbl_func_state(void *hwdev, u8 func_state, u16 func_id) +{ + roce_set_func_state_cmd_s set_func_state_info; + u16 out_size = (u16)sizeof(set_func_state_info); + int ret; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&set_func_state_info, sizeof(set_func_state_info), 0, + sizeof(set_func_state_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset set_func_state_info.(ret:%d)", ret); + return (-ENOMEM); + } + set_func_state_info.func_id = func_id; + set_func_state_info.func_en = func_state; + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_SET_FUNC_STATE, + &set_func_state_info, + sizeof(set_func_state_info), + &set_func_state_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (set_func_state_info.head.status != 0)) { + ROCE_ERR( + "Failed to set func state(%d), err(%d), status(0x%x), out size(0x%x), func_id(%d)", + func_state, ret, set_func_state_info.head.status, + out_size, func_id); + return (-EINVAL); + } + + return 0; +} + +int roce5_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value) +{ + roce_get_func_table_cmd_s get_func_table; + roce_get_func_table_rsp_s get_func_table_rsp; + u16 out_size = (u16)sizeof(get_func_table_rsp); + int ret; + + if (hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&get_func_table, sizeof(get_func_table), 0, + sizeof(get_func_table)); + if (ret != 0) { + ROCE_ERR("Failed to memset get_func_table.(ret:%d)", ret); + return (-ENOMEM); + } + ret = memset_s(&get_func_table_rsp, out_size, 0, out_size); + if (ret != 0) { + ROCE_ERR("Failed to memset get_func_table_rsp.(ret:%d)", ret); + return (-ENOMEM); + } + get_func_table.func_id = func_id; + ret = roce5_msg_to_mgmt_sync(hwdev, ROCE_MPU_CMD_GET_FUNC_TABLE, + &get_func_table, sizeof(get_func_table), + &get_func_table_rsp, &out_size); + if ((ret != 0) || (out_size == 0) || + (get_func_table_rsp.head.status != 0)) { + ROCE_ERR( + "Failed to get func table, err(%d), status(0x%x), out size(0x%x), func_id(%d)", + ret, get_func_table_rsp.head.status, out_size, func_id); + return (-EINVAL); + } + + *func_tbl_value = get_func_table_rsp.func_tbl_value; + + return 0; +} + +int roce5_set_func_tbl_cpu_endian(struct roce5_device *rdev) +{ + roce_set_cpu_endian_cmd_s set_cpu_endian_info; + u16 out_size = (u16)sizeof(set_cpu_endian_info); + int ret; + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return -EINVAL; + } + + ret = memset_s(&set_cpu_endian_info, sizeof(set_cpu_endian_info), 0, + sizeof(set_cpu_endian_info)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to memset set_cpu_endian_info.(ret:%d).", + ret); + return -ENOMEM; + } + set_cpu_endian_info.func_id = rdev->glb_func_id; + set_cpu_endian_info.cpu_endian = rdev->hw_info.cpu_endian; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_SET_CPU_ENDIAN, + &set_cpu_endian_info, + sizeof(set_cpu_endian_info), + &set_cpu_endian_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (set_cpu_endian_info.head.status != 0)) { + ROCE_DEV_ERR( + rdev, + "Failed to set func state, err(%d), status(0x%x), out size(0x%x).", + ret, set_cpu_endian_info.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +static void roce5_init_cfg_info_parse_resp(struct roce5_device *rdev, + roce_get_cfg_info_resp_s *resp) +{ + rdev->cfg_info.scence_id = resp->scence_id; + rdev->cfg_info.lb_en = resp->lb_en; + rdev->cfg_info.lb_mode = resp->lb_mode; + rdev->cfg_info.srq_container_en = + (resp->container_mode != ROCE_CHIP_SRQ_MODE_N); + rdev->cfg_info.srq_container_mode = + roce5_srq_mode_chip_adapt(resp->container_mode); + rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3; + rdev->cfg_info.warn_th = 0; + + rdev->cfg_info.fake_en = resp->fake_en; + rdev->cfg_info.pf_start_bit = resp->pf_start_bit; + rdev->cfg_info.pf_end_bit = resp->pf_end_bit; + rdev->cfg_info.page_bit = resp->page_bit; + + rdev->cfg_info.port_num = resp->port_num; + rdev->cfg_info.host_num = resp->host_num; + rdev->cfg_info.pcie_atomic_en = resp->pcie_atomic_en; + + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes = resp->max_wqe; + rdev->rdma_cap.max_cqes = resp->max_cqe; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes = + resp->max_srq_wqe; + rdev->rdma_cap.max_sq_sg = resp->max_sq_sge; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg = resp->max_rq_sge; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge = + resp->max_srq_sge; + rdev->rdma_cap.max_sq_desc_sz = resp->max_sq_wqe_len; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_desc_sz = + resp->max_rq_wqe_len; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz = + resp->max_sq_inline; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_pd = resp->max_pd; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_xrcd = resp->max_xrcd; + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_gid = resp->max_gid; + + rdev->hca_vf_mode = resp->ip2func; + + roce5_cfg_info_update_ext(rdev, resp); +} + +int roce5_get_feature(struct roce5_device *rdev, + roce_get_chip_ability_s *roce_feature) +{ + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_verbs_cmd_header_s *cmd_hdr = NULL; + roce_get_chip_ability_s *resp_buf = NULL; + int ret = 0; + + ret = memset_s(roce_feature, sizeof(*roce_feature), 0, + sizeof(*roce_feature)); + if (ret != 0) { + ROCE_ERR("Failed to memset roce_feature, ret(%d).", ret); + return (-ENOMEM); + } + + // allocate input and output buffer + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + sizeof(roce_verbs_cmd_header_s), + &cqm_cmd_outbuf, + sizeof(roce_get_chip_ability_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to allocate cqm buffers, ret = %d.", + ret); + return ret; + } + + // fill input command + cmd_hdr = (roce_verbs_cmd_header_s *)cqm_cmd_inbuf->buf; + cmd_hdr->dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_COMM_BITMASK); + + // send command and wait response + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_GET_CHIP_ABILITY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_C, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send feature cmd, ret = %d.", + ret); + goto free_buf; + } + + // get response data + resp_buf = (roce_get_chip_ability_s *)cqm_cmd_outbuf->buf; + roce_feature->roce_kernel_ability = + be64_to_cpu(resp_buf->roce_kernel_ability); + roce_feature->roce_kernel_ext_ability = + be64_to_cpu(resp_buf->roce_kernel_ext_ability); + roce_feature->roce_user_ext_ability = + be64_to_cpu(resp_buf->roce_user_ext_ability); + + // get on layer service supported attribute + roce_feature->uld_feature.roce_uld_kernel_ext_ability = + be64_to_cpu(resp_buf->uld_feature.roce_uld_kernel_ext_ability); + roce_feature->uld_feature.roce_uld_user_ext_ability = + be64_to_cpu(resp_buf->uld_feature.roce_uld_user_ext_ability); +free_buf: + // free buffer + if (cqm_cmd_inbuf || cqm_cmd_outbuf) { + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + } + return ret; +} + +int roce5_uni_cmd_init_cfg_info(struct roce5_device *rdev) +{ + roce_get_cfg_info_cmd_s get_cfg_info; + roce_get_cfg_info_resp_s resp; + u16 out_size = (u16)sizeof(resp); + int ret = 0; + + ret = memset_s(&get_cfg_info, sizeof(get_cfg_info), 0, + sizeof(get_cfg_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset get_cfg_info, ret(%d).", ret); + return (-ENOMEM); + } + ret = memset_s(&resp, sizeof(resp), 0, sizeof(resp)); + if (ret != 0) { + ROCE_ERR("Failed to memset resp, ret(%d).", ret); + return (-ENOMEM); + } + + get_cfg_info.func_id = rdev->glb_func_id; + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO, + &get_cfg_info, sizeof(get_cfg_info), &resp, + &out_size); + if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) { + ROCE_DEV_ERR(rdev, + "Failed, err(%d), status(0x%x), out size(0x%x)", + ret, get_cfg_info.head.status, out_size); + return (-EINVAL); + } + + roce5_init_cfg_info_parse_resp(rdev, &resp); + + ROCE_DEV_INFO( + rdev, + "init, func_id(%d), lb_en(%d), lb_mode(%d), srq_mode(%d), scence_id(%d), pcie_atomic_en(%d).", + rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode, + rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id, + rdev->cfg_info.pcie_atomic_en); + ROCE_DEV_INFO( + rdev, + "max_wqes(0x%x), max_cqes(0x%x), max_srq_wqes(0x%x), max_sq_sg(0x%x), max_rq_sg(0x%x), " + "max_srq_sge(0x%x), max_sq_desc_sz(0x%x), max_rq_desc_sz(0x%x), max_sq_inline_data_sz(0x%x).", + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_wqes, + rdev->rdma_cap.max_cqes, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_wqes, + rdev->rdma_cap.max_sq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_sg, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_srq_sge, + rdev->rdma_cap.max_sq_desc_sz, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_rq_desc_sz, + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.max_sq_inline_data_sz); + + return 0; +} + +int roce5_init_cfg_info(struct roce5_device *rdev) +{ + roce_get_cfg_info_cmd_s get_cfg_info; + roce_get_cfg_info_comp_resp_s resp; + u16 out_size = (u16)sizeof(resp); + int ret = 0; + + if (rdev == NULL || rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + return roce5_uni_cmd_init_cfg_info(rdev); + } + + (void)memset_s(&get_cfg_info, sizeof(get_cfg_info), 0, + sizeof(get_cfg_info)); + (void)memset_s(&resp, sizeof(resp), 0, sizeof(resp)); + + get_cfg_info.func_id = rdev->glb_func_id; + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_GET_CFG_INFO, + &get_cfg_info, sizeof(get_cfg_info), &resp, + &out_size); + if ((ret != 0) || (out_size == 0) || (get_cfg_info.head.status != 0)) { + ROCE_DEV_ERR(rdev, + "Failed, err(%d), status(0x%x), out size(0x%x)", + ret, get_cfg_info.head.status, out_size); + return (-EINVAL); + } + + rdev->cfg_info.scence_id = resp.scence_id; + rdev->cfg_info.lb_en = resp.lb_en; + rdev->cfg_info.lb_mode = resp.lb_mode; + rdev->cfg_info.srq_container_en = + (resp.container_mode != ROCE_CHIP_SRQ_MODE_N); + rdev->cfg_info.srq_container_mode = + roce5_srq_mode_chip_adapt(resp.container_mode); + rdev->cfg_info.xrc_srq_container_mode = ROCE_SRQ_MODE_3; + rdev->cfg_info.warn_th = 0; + + rdev->cfg_info.fake_en = resp.fake_en; + rdev->cfg_info.pf_start_bit = resp.pf_start_bit; + rdev->cfg_info.pf_end_bit = resp.pf_end_bit; + rdev->cfg_info.page_bit = resp.page_bit; + + rdev->cfg_info.port_num = resp.port_num; + rdev->cfg_info.host_num = resp.host_num; + rdev->cfg_info.pcie_atomic_en = resp.pcie_atomic_en; + + rdev->hca_vf_mode = 0; // V100 fixed to mac mode + + ROCE_DEV_INFO( + rdev, + "init, func_id(%d), lb_en(%d), lb_mode(%d), srq_mode(%d), scence_id(%d), pcie_atomic_en(%d).", + rdev->glb_func_id, rdev->cfg_info.lb_en, rdev->cfg_info.lb_mode, + rdev->cfg_info.srq_container_mode, rdev->cfg_info.scence_id, + rdev->cfg_info.pcie_atomic_en); + return 0; +} + +int roce5_del_func_res(struct roce5_device *rdev) +{ + roce_set_func_state_cmd_s del_func_res_info; + u16 out_size = (u16)sizeof(del_func_res_info); + int ret; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + ROCE_ERR("rdev/Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&del_func_res_info, sizeof(del_func_res_info), 0, + sizeof(del_func_res_info)); + if (ret != 0) { + ROCE_ERR("Failed to memset del_func_res_info.(ret:%d)", ret); + return (-ENOMEM); + } + del_func_res_info.func_id = rdev->glb_func_id; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DEL_FUNC_RES, + &del_func_res_info, + sizeof(del_func_res_info), + &del_func_res_info, &out_size); + if ((ret != 0) || (out_size == 0) || + (del_func_res_info.head.status != 0)) { + ROCE_ERR( + "Failed to set func state, err(%d), status(0x%x), out size(0x%x)", + ret, del_func_res_info.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce5_set_bw_ctrl_state(struct roce5_device *rdev, u8 cmd, + struct roce5_bw_ctrl_inbuf *inbuf) +{ + int ret = 0; + roce_cc_cfg_bw_ctrl_cmd_s set_bw_ctrl; + u16 out_size = (u16)sizeof(set_bw_ctrl); + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&set_bw_ctrl, sizeof(set_bw_ctrl), 0, + sizeof(set_bw_ctrl)); + if (ret != 0) { + ROCE_ERR("Memset failed when set_bw_ctrl cmd"); + return (-ENOMEM); + } + + set_bw_ctrl.func_id = rdev->glb_func_id; + set_bw_ctrl.cmd = cmd; + set_bw_ctrl.cir = inbuf->ctrl_param.cir; + set_bw_ctrl.pir = inbuf->ctrl_param.pir; + set_bw_ctrl.cnp = inbuf->ctrl_param.cnp; + ROCE_INFO("func_id:%u, cmd:%u, cir:%u, pir:%u, cnp:%u", + rdev->glb_func_id, set_bw_ctrl.cmd, set_bw_ctrl.cir, + set_bw_ctrl.pir, set_bw_ctrl.cnp); + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_SET_BW_CTRL, + &set_bw_ctrl, sizeof(set_bw_ctrl), + &set_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (set_bw_ctrl.head.status != 0)) { + ROCE_ERR( + "Failed to set set_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)", + ret, set_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce5_query_bw_ctrl_state(struct roce5_device *rdev, + struct roce5_bw_ctrl_param *bw_ctrl_param) +{ + int ret = 0; + roce_cc_cfg_bw_ctrl_cmd_s query_bw_ctrl; + u16 out_size = (u16)sizeof(query_bw_ctrl); + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&query_bw_ctrl, sizeof(query_bw_ctrl), 0, + sizeof(query_bw_ctrl)); + if (ret != 0) { + ROCE_ERR("Memset failed when set_bw_ctrl cmd"); + return (-ENOMEM); + } + + query_bw_ctrl.func_id = rdev->glb_func_id; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_CC_QUERY_BW_CTRL, + &query_bw_ctrl, sizeof(query_bw_ctrl), + &query_bw_ctrl, &out_size); + if ((ret != 0) || (out_size == 0) || (query_bw_ctrl.head.status != 0)) { + ROCE_ERR( + "Failed to set query_bw_ctrl cmd, err(%d), status(0x%x), out size(0x%x)", + ret, query_bw_ctrl.head.status, out_size); + return (-EINVAL); + } + + ret = memcpy_s((void *)bw_ctrl_param, + sizeof(struct roce5_bw_ctrl_param), + (void *)((u8 *)(&query_bw_ctrl) + ROCE_CMD_DEFAULT_SIZE), + sizeof(struct roce5_bw_ctrl_param)); + if (ret != 0) { + ROCE_ERR("memory copy failed. ret%d", ret); + return -ENOMEM; + } + + ROCE_INFO("query_bw_ctrl_state:func_id:%u, cir:%u, pir:%u, cnp:%u", + rdev->glb_func_id, bw_ctrl_param->cir, bw_ctrl_param->pir, + bw_ctrl_param->cnp); + + return 0; +} + +#ifdef ROCE_CHIP_TEST +int roce5_set_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf) +{ + int ret = 0; + roce_cc_cfg_vnic_waterline_cmd_s set_vnic_waterline; + u16 out_size = (u16)sizeof(set_vnic_waterline); + + (void)outbuf; + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&set_vnic_waterline, sizeof(set_vnic_waterline), 0, + sizeof(set_vnic_waterline)); + if (ret != 0) { + ROCE_ERR("Memset failed when set_vnic_waterline cmd"); + return (-ENOMEM); + } + + set_vnic_waterline.func_id = rdev->glb_func_id; + set_vnic_waterline.ip_ver = 0; + set_vnic_waterline.param.red_waterline = + inbuf->waterline_param.red_waterline; + set_vnic_waterline.param.yellow_waterline = + inbuf->waterline_param.yellow_waterline; + set_vnic_waterline.param.ip_info.ipv4.val = + inbuf->waterline_param.ip_val; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, + ROCE_MPU_CMD_CC_SET_VNIC_WATERLINE, + &set_vnic_waterline, + sizeof(set_vnic_waterline), + &set_vnic_waterline, &out_size); + if ((ret != 0) || (out_size == 0) || + (set_vnic_waterline.head.status != 0)) { + ROCE_ERR( + "Failed to set set_vnic_waterline cmd, err(%d), status(0x%x), out size(0x%x)", + ret, set_vnic_waterline.head.status, out_size); + return (-EINVAL); + } + + return 0; +} + +int roce5_get_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf) +{ + int ret = 0; + roce_cc_cfg_vnic_waterline_cmd_s get_vnic_waterline; + u16 out_size = (u16)sizeof(get_vnic_waterline); + + if (rdev->hwdev == NULL) { + ROCE_ERR("Hwdev is null"); + return (-EINVAL); + } + + ret = memset_s(&get_vnic_waterline, sizeof(get_vnic_waterline), 0, + sizeof(get_vnic_waterline)); + if (ret != 0) { + ROCE_ERR("Memset failed when get_vnic_waterline cmd"); + return (-ENOMEM); + } + + get_vnic_waterline.func_id = rdev->glb_func_id; + get_vnic_waterline.ip_ver = 0; + get_vnic_waterline.param.ip_info.ipv4.val = + inbuf->waterline_param.ip_val; + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, + ROCE_MPU_CMD_CC_GET_VNIC_WATERLINE, + &get_vnic_waterline, + sizeof(get_vnic_waterline), + &get_vnic_waterline, &out_size); + if ((ret != 0) || (out_size == 0) || + (get_vnic_waterline.head.status != 0)) { + ROCE_ERR( + "Failed to set get_vnic_waterline cmd, err(%d), status(0x%x), out size(0x%x)", + ret, get_vnic_waterline.head.status, out_size); + return (-EINVAL); + } + + outbuf->waterline_param.red_waterline = + get_vnic_waterline.param.red_waterline; + outbuf->waterline_param.yellow_waterline = + get_vnic_waterline.param.yellow_waterline; + + return 0; +} +#endif /* ROCE_CHIP_TEST */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h new file mode 100644 index 000000000..5fd787a67 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/cmd/roce_cmd.h @@ -0,0 +1,82 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CMD_H +#define ROCE_CMD_H + +#include "comm_defs.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" + +#include "roce_mpu_cmd_defs.h" +#include "roce_cfg_format.h" + +#define ROCE_VLAN_ID_MASK 0x7FFF +#define ROCE_VLAN_N_VID 4096 + +#define ROCE_CMD_DEFAULT_SIZE 12 + +#define ERR_EXIST 6 +#define ERR_FULL 12 +#define ERR_NOT_FOUND 13 + +#define roce5_msg_to_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, out_size) \ + hinic5_msg_to_mgmt_sync(hwdev, HINIC5_MOD_ROCE, cmd, buf_in, in_size, \ + buf_out, out_size, 0, HINIC5_CHANNEL_ROCE) +#define roce5_msg_to_ovs_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out, \ + out_size) \ + hinic5_msg_to_mgmt_sync(hwdev, HINIC5_MOD_OVS, cmd, buf_in, in_size, \ + buf_out, out_size, 90000, HINIC5_CHANNEL_OVS) + +int roce5_add_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id); +int roce5_del_mac_tbl_mac_entry(struct roce5_device *rdev, u8 *mac_addr, + size_t mac_addr_len, u32 vlan_id, u16 func_id); +typedef int (*roce5_modify_mac_tbl)(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], size_t eth_alen, + u32 vlan_id, u16 func_id); +void roce5_add_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], u32 vlan_id, + u16 func_id, u8 er_id, u16 src_func_id); +void roce5_del_ipsu_tbl_mac_entry(struct roce5_device *rdev, + u8 mac_addr[ETH_ALEN], size_t eth_alen, + u32 vlan_id, u8 er_id, u16 src_func_id); +int roce5_add_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info); +int roce5_del_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct roce_cmd_mpu_addr_info *addr_info); +int roce5_modify_ipsu_tbl_ip_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, u8 type); +void roce5_add_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, + u32 gid_index); +void roce5_del_ipsu_tcam_tbl_entry(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry); +typedef void (*roce5_modify_ipsu_tbl_mac)(void *hwdev, u8 mac_addr[ETH_ALEN], + u32 vlan_id, u16 func_id, u8 er_id); + +int roce5_set_sysfs_cfg_param(void *hwdev, u16 func_id, u32 *param, + u32 param_len, u16 type); +int roce5_set_bw_ctrl_state(struct roce5_device *rdev, u8 cmd, + struct roce5_bw_ctrl_inbuf *inbuf); +int roce5_query_bw_ctrl_state(struct roce5_device *rdev, + struct roce5_bw_ctrl_param *bw_ctrl_param); +#ifdef ROCE_CHIP_TEST +int roce5_set_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf); +int roce5_get_vnic_waterline_state(struct roce5_device *rdev, + struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf); +#endif /* ROCE_CHIP_TEST */ +int roce5_set_func_tbl_func_state(void *hwdev, u8 func_state, u16 func_id); +int roce5_get_func_table(void *hwdev, u16 func_id, u32 *func_tbl_value); +int roce5_set_func_tbl_cpu_endian(struct roce5_device *rdev); +int roce5_get_feature(struct roce5_device *rdev, + roce_get_chip_ability_s *roce_feature); +int roce5_init_cfg_info(struct roce5_device *rdev); +int roce5_del_func_res(struct roce5_device *rdev); + +#endif /* ROCE_CMD_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c new file mode 100644 index 000000000..11bd63d04 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.c @@ -0,0 +1,86 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/kernel.h> + +#include "roce.h" +#include "roce_os_compat.h" + +int roce5_db_map_user(struct roce5_ucontext *context, unsigned long virt, + struct roce5_db *db, u8 shadow_db) +{ + int ret = 0; + struct roce5_db_page *db_page = NULL; + + if ((context == NULL) || (db == NULL)) { + return -EINVAL; + } + if (shadow_db != 0) { + return ret; + } + mutex_lock(&context->db_page_mutex); + + list_for_each_entry(db_page, &context->db_page_list, list) { + if (db_page->user_virt == (virt & PAGE_MASK)) { + goto found; + } + } + + db_page = (struct roce5_db_page *)kmalloc(sizeof(*db_page), GFP_KERNEL); + if (db_page == NULL) { + ret = -ENOMEM; + ROCE_ERR("Failed to alloc DB page"); + goto out; + } + + db_page->user_virt = (virt & PAGE_MASK); + db_page->refcnt = 0; + db_page->umem = roce5_umem_get(context->ibucontext.device, + &context->ibucontext, virt & PAGE_MASK, + PAGE_SIZE, 0, 0); + if (IS_ERR(db_page->umem)) { + ret = (int)PTR_ERR(db_page->umem); + ROCE_ERR("Failed to get ib_umem ret:%d", ret); + kfree(db_page); + goto out; + } + + list_add(&db_page->list, &context->db_page_list); + +found: +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + db->dma = sg_dma_address(db_page->umem->sgt_append.sgt.sgl) + + (virt & ~PAGE_MASK); +#else + db->dma = sg_dma_address(db_page->umem->sg_head.sgl) + + (virt & ~PAGE_MASK); +#endif + db->user_page = db_page; + ++db_page->refcnt; + +out: + mutex_unlock(&context->db_page_mutex); + + return ret; +} + +void roce5_db_unmap_user(struct roce5_ucontext *context, struct roce5_db *db, + u8 shadow_db) +{ + if ((context == NULL) || (db == NULL)) { + return; + } + if (shadow_db != 0) { + return; + } + mutex_lock(&context->db_page_mutex); + + if ((--db->user_page->refcnt) == 0) { + list_del(&db->user_page->list); + ib_umem_release(db->user_page->umem); + kfree(db->user_page); + } + + mutex_unlock(&context->db_page_mutex); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h new file mode 100644 index 000000000..7232e9edc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/db/roce_db.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_DB_H +#define ROCE_DB_H + +#include <rdma/ib_umem.h> +#include <linux/types.h> +#include <linux/list.h> + +struct roce5_db_page { + struct list_head list; + struct ib_umem *umem; + unsigned long user_virt; + int refcnt; +}; + +struct roce5_db { + __be32 *db_record; + dma_addr_t dma; + struct roce5_db_page *user_page; +}; + +static inline void roce5_write64(u32 val[2], void __iomem *dest) +{ + __raw_writeq(*(u64 *)(void *)val, dest); +} + +#endif //ROCE_DB_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c b/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c new file mode 100644 index 000000000..7d589afe1 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/init/roce_main.c @@ -0,0 +1,1701 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <net/ipv6.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <rdma/ib_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_user_verbs.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> +#include <linux/security.h> +#include <linux/kref.h> +#include <linux/list.h> +#include <linux/netdevice.h> +#include <linux/sizes.h> + +#include "hinic5_crm.h" +#include "hinic5_hw.h" +#include "hinic5_mt.h" +#include "hinic5_srv_nic.h" +#include "hinic5_rdma.h" +#include "hinic5_id_tbl.h" + +#include "roce_dfx.h" +#include "roce_mr.h" +#include "roce_main_extension.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_netlink.h" +#include "roce_cmd.h" +#include "roce_qp.h" +#include "roce_infra.h" +#include "rdma_comp_mw_mr.h" +#include "roce_xrc.h" +#include "roce_com.h" +#include "roce_kernel_compat_gen.h" +#include "roce_netdev.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +#ifdef ENABLE_HYPER_ROCE +#include "hyper_roce_tool.h" +#endif + +#include "securec.h" + +MODULE_AUTHOR(HIROCE5_DRV_AUTHOR); +MODULE_DESCRIPTION(HIROCE5_DRV_DESC); +MODULE_VERSION(HIROCE5_DRV_VERSION); +MODULE_LICENSE("GPL"); + +static int g_loop_times = 50000; +module_param(g_loop_times, int, S_IRUGO); +MODULE_PARM_DESC(g_loop_times, + "default: 50000. The number must be greater than 0."); + +static u8 g_destory_flow_timeout = 10; +module_param(g_destory_flow_timeout, byte, S_IRUGO); +MODULE_PARM_DESC(g_destory_flow_timeout, "default: 10"); + +static bool g_local_en = false; // IP mode because IP in this place need switch control whether advancein this place switch +module_param(g_local_en, bool, S_IRUGO); +MODULE_PARM_DESC(g_local_en, "default false"); + +static bool g_full_qp_spec = true; // calculate standard card scenario, supports single cq create full specification qp +module_param(g_full_qp_spec, bool, S_IRUGO); +MODULE_PARM_DESC( + g_full_qp_spec, + "Supports creating a maximum number of QPs through a single CQ. default true."); + +static bool g_huge_page_en = 0; +module_param(g_huge_page_en, bool, S_IRUGO); +MODULE_PARM_DESC( + g_huge_page_en, + "default: 0. value: 0 or 1, set huge page en, and users need to reserve huge pages in advance."); + +#define ROCE_BOND_SLAVE_BIT_MASK 0xf + +/* **************************************************************************** + Prototype : get_cpu_endian + Description : Acquire CPU's enianness + Return Value : 0: little endian; 1: big endian + + 1.Date : 2016/6/23 + Modification : Created function + +**************************************************************************** */ +static int get_cpu_endian(void) +{ + int cpu_mode = 0; + union { + unsigned int i; + unsigned char s[4]; + } c; + c.i = 0x12345678; + + if (c.s[0] == 0x12) { + ROCE_INFO("CPU is be"); + cpu_mode = 1; + } else { + ROCE_INFO("CPU is le"); + cpu_mode = 0; + } + + return cpu_mode; +} + +static int roce5_alloc_hw_resource(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_rdma_init_rsvd_lkey(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to init rsvd lkey, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + + /* Clear gid table before register for a new IB device */ + ret = roce5_reset_gid_table(rdev, 0); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to reset gid_table, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_gid_reset; + } + + return 0; + +err_gid_reset: + roce5_rdma_free_rsvd_lkey(rdev); + + return ret; +} + +static void roce5_dealloc_hw_resource(struct roce5_device *rdev) +{ + roce5_rdma_free_rsvd_lkey(rdev); +} + +/* 1825 v100 esl sub did */ +#define HI_1825_V100_SUB_DEV_ID_2X100 0x1825 +#define HI_1825_V100_SUB_DEV_ID_COMPUTE_2X200 0x40B1 +#define HI_1825_V100_SUB_DEV_ID_COMPUTE_DY_YH 0x30C4 +#define HI_1825_V100_SUB_DEV_ID_COMPUTE_UBX 0x30C8 + +static int roce5_init_ver_attr_pci(struct hinic5_lld_dev *lld_dev, + struct roce5_device *rdev) +{ + struct pci_dev *pdev = container_of(lld_dev->dev, struct pci_dev, dev); + + if ((pdev->device == HINIC5_DEV_ID_25V1_PF) || + (pdev->device == HINIC5_DEV_ID_25V1_VF)) { + rdev->device_type = ROCE_DEV_TYPE_HI1825_V100; + return 0; + } + + switch (pdev->subsystem_device) { + case HI_1825_V100_SUB_DEV_ID_2X100: + case HI_1825_V100_SUB_DEV_ID_COMPUTE_2X200: + case HI_1825_V100_SUB_DEV_ID_COMPUTE_DY_YH: + case HI_1825_V100_SUB_DEV_ID_COMPUTE_UBX: + rdev->device_type = ROCE_DEV_TYPE_HI1825_V100; + break; + default: + ROCE_DEV_ERR(rdev, "Unknown subsystem id(%u).", + pdev->subsystem_device); + rdev->device_type = ROCE_DEV_TYPE_UNKNOWN; + return -ENXIO; + } + + return 0; +} + +static int roce5_init_ver_attr(struct hinic5_lld_dev *lld_dev, + struct roce5_device *rdev) +{ + int ret; + + switch (lld_dev->dev_type) { + case HINIC5_DEVICE_T_PCI: + ret = roce5_init_ver_attr_pci(lld_dev, rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to init PCI version attributes, ret(%d).", + ret); + return ret; + } + break; + + case HINIC5_DEVICE_T_UB: + /* + * The 'struct ub_dev' contains a 'module' member that can be used to distinguish the version of the card + * under the UBC. However, this member is only available in the most recent version of 'struct ub_dev', and + * the current development environment does not include the updated structure definition, making it + * inaccessible at present. Here, the fixed version(Hi1825V100) is returned. + */ + rdev->device_type = ROCE_DEV_TYPE_HI1825_V100; + break; + + default: + ROCE_DEV_ERR(rdev, "Unknown low level device type(%u).", + lld_dev->dev_type); + rdev->device_type = ROCE_DEV_TYPE_UNKNOWN; + return -ENXIO; + } + + return 0; +} + +static int roce5_get_perf_cq_step(struct roce5_device *rdev) +{ + int perf_step = 0; + + if (ROCE_IS_DEV_1825V100(rdev)) { + perf_step = ROCE_1825V100_STEP; + } else { + ROCE_DEV_INFO( + rdev, + "not support high performance cq, device type(%u), func_id(%u).", + rdev->device_type, rdev->glb_func_id); + } + return perf_step; +} + +static int roce5_get_perf_cq_cnt(struct roce5_device *rdev) +{ + if (rdev->cqn_ctrl.rsvd_cq_step == 0) { + return 0; + } + /* When you insmod the driver, CQ0 is consumed by default. */ + return (int)((rdev->rdma_cap.dev_rdma_cap.roce_own_cap.reserved_cqs - + 1) / + rdev->cqn_ctrl.rsvd_cq_step) + + 1; +} + +void roce5_init_perf_cq_attr(struct roce5_device *rdev) +{ + /* Calculate the step size and number of high-performance CQs */ + rdev->cqn_ctrl.rsvd_cq_step = roce5_get_perf_cq_step(rdev); + rdev->cqn_ctrl.high_perf_cq_cnt = roce5_get_perf_cq_cnt(rdev); +#ifdef ROCE_VBS_EN + // VBS RoCE scenario disable high performance cq + rdev->cqn_ctrl.high_perf_cq_cnt = 0; +#endif + rdev->cqn_ctrl.alloc_cq_cnt = 0; +} + +static struct roce5_device * +roce5_rdev_alloc(struct hinic5_lld_dev *lld_dev, void **uld_dev, + const struct rdma_service_cap *rdma_cap) +{ + struct roce5_device *rdev = NULL; + int ret; + + rdev = (struct roce5_device *)roce5_rdev_alloc_ext(); + if (rdev == NULL) { + ROCE_ERR("Failed to alloc rdev"); + return NULL; + } + + *uld_dev = (void *)rdev; + rdev->lld_dev = lld_dev; + rdev->hwdev = lld_dev->hwdev; + rdev->dev = lld_dev->dev; + dma_set_max_seg_size(rdev->dev, SZ_2G); + ret = roce5_init_ver_attr(lld_dev, rdev); + if (ret != 0) { + ROCE_ERR( + "Failed to init version attribute of the device, ret(%d)", + ret); + goto alloc_failed; + } + + mutex_init(&rdev->qp_cnt.cur_qps_mutex); + mutex_init(&rdev->cqn_ctrl.cq_ctrl_mutex); + + rdev->hwdev_hdl = ((struct hinic5_hwdev *)(rdev->hwdev))->dev_hdl; + if (memcpy_s((void *)&rdev->rdma_cap, sizeof(*rdma_cap), + (void *)rdma_cap, sizeof(*rdma_cap)) != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy rdma_cap."); + goto alloc_failed; + } + + rdev->ndev = hinic5_get_netdev_by_lld(rdev->lld_dev); + if (rdev->ndev == NULL) { + ROCE_DEV_ERR(rdev, "roce add failed, netdev is null."); + goto alloc_failed; + } + + roce5_init_perf_cq_attr(rdev); + rdev->rdev_magic = ROCE_DEVICE_MAGIC; + return rdev; + +alloc_failed: + ib_dealloc_device(&rdev->ib_dev); + return NULL; +} + +static int roce5_board_info_get(struct roce5_device *rdev) +{ + int ret = 0; + + ret = hinic5_get_board_info(rdev->hwdev, &rdev->board_info, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to get board info"); + return ret; + } + + ROCE_DEV_INFO( + rdev, + "Get board info success, board_type:0x%x, port_num:0x%x, pf_num:0x%x, vf_total_num:0x%x, " + "work_mode:0x%x, service_mode:0x%x, port_speed:0x%x.", + rdev->board_info.board_type, rdev->board_info.port_num, + rdev->board_info.pf_num, rdev->board_info.vf_total_num, + rdev->board_info.work_mode, rdev->board_info.service_mode, + rdev->board_info.port_speed); + + ret = roce5_board_cfg_check(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to check board cfg info"); + return ret; + } + + return 0; +} + +static void roce5_fix_ibdev_name(struct roce5_device *rdev) +{ + int ret; + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + ret = strcpy_s(rdev->ib_dev.name, IB_DEVICE_NAME_MAX, + "hrn5_bond_%d"); + if (ret != EOK) { + ROCE_DEV_ERR(rdev, "Strcpy_s error"); + } + return; + } +#endif + ret = strcpy_s(rdev->ib_dev.name, IB_DEVICE_NAME_MAX, "hrn5_%d"); + if (ret != EOK) { + ROCE_DEV_ERR(rdev, "Strcpy_s error"); + } +} + +static int roce5_register_template(struct roce5_device *rdev) +{ + service_register_template_s svc_template; + int ret; + + ret = memset_s(&svc_template, sizeof(svc_template), 0, + sizeof(svc_template)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memset svc_template.(ret:%d)", + ret); + return -ENOMEM; + } + + svc_template.service_type = SERVICE_T_ROCE; + svc_template.scq_ctx_size = rdev->rdma_cap.cqc_entry_sz; + svc_template.srq_ctx_size = + rdev->rdma_cap.dev_rdma_cap.roce_own_cap.srqc_entry_sz; + svc_template.service_handle = rdev; + svc_template.embedded_cq_ceq_callback = NULL; + svc_template.no_cq_ceq_callback = NULL; + + svc_template.shared_cq_ceq_callback = roce5_cq_completion; + svc_template.aeq_level_callback = roce5_async_event_level; + svc_template.aeq_callback = roce5_async_event; + + ret = cqm5_service_register(rdev->hwdev, &svc_template); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to register cqm_service, func(%d).", + rdev->glb_func_id); + return ret; + } + + return 0; +} + +/* Alloc CEQs and alloc CEQN */ +static int roce5_alloc_ceq(struct roce5_device *rdev) +{ + int ret; + + ret = hinic5_alloc_ceqs(rdev->hwdev, SERVICE_T_ROCE, + rdev->ib_dev.num_comp_vectors, rdev->ceqn, + &rdev->ceq_num); + if (ret < 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqes, num_comp_vectors(%d), func_id(%d), ret(%d)", + rdev->ib_dev.num_comp_vectors, rdev->glb_func_id, ret); + return ret; + } + + return 0; +} + +static void roce5_free_ceq(struct roce5_device *rdev) +{ + int i; + + for (i = 0; i < rdev->ceq_num; ++i) { + hinic5_free_ceq(rdev->hwdev, SERVICE_T_ROCE, rdev->ceqn[i]); + } +} + +static void roce5_init_hw_info(struct roce5_device *rdev) +{ + rdev->hw_info.config_num_ports = (int)rdev->rdma_cap.num_ports; + rdev->hw_info.ep_id = hinic5_ep_id(rdev->hwdev); + rdev->hw_info.phy_port = 1; + rdev->hw_info.is_vf = roce5_func_is_vf(rdev->hwdev); + rdev->hw_info.cpu_endian = (u8)get_cpu_endian(); +} + +static int roce5_init_dev_info(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_set_func_tbl_cpu_endian(rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set func_tbl cpu_endian, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_init; + } + + /* Set function table to ENABLE */ + ret = roce5_set_func_tbl_func_state(rdev->hwdev, ROCE_FUNC_ENABLE, + rdev->glb_func_id); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set func_tbl, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_init; + } + + return 0; + +err_init: + return ret; +} +#ifdef __ROCE_DFX__ +static void device_list_lock_init(struct roce5_device_res *device_res) +{ + mutex_init(&device_res->mr_list.node_lock); + INIT_LIST_HEAD(&device_res->mr_list.node); +} + +static void device_list_lock_deinit(struct roce5_device_res *device_res) +{ + struct roce5_mr *mr = NULL; + struct roce5_mr *tmp_mr = NULL; + + mutex_lock(&device_res->mr_list.node_lock); + list_for_each_entry_safe(mr, tmp_mr, &device_res->mr_list.node, node) { + list_del(&(mr->node)); + } + mutex_unlock(&device_res->mr_list.node_lock); + + mutex_destroy(&(device_res->mr_list.node_lock)); +} + +static int roce5_dfx_device_init(struct roce5_device *rdev) +{ + rdev->device_res = kzalloc(sizeof(struct roce5_device_res), GFP_KERNEL); + if (rdev->device_res == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc device resource."); + return -ENOMEM; + } + + device_list_lock_init(rdev->device_res); + + return 0; +} + +static void roce5_dfx_device_deinit(struct roce5_device *rdev) +{ + if (rdev->device_res != NULL) { + device_list_lock_deinit(rdev->device_res); + kfree(rdev->device_res); + rdev->device_res = NULL; + } +} +#endif // __ROCE_DFX__ + +static void roce5_deinit_dev_info(struct roce5_device *rdev) +{ + (void)roce5_set_func_tbl_func_state(rdev->hwdev, ROCE_FUNC_DISABLE, + rdev->glb_func_id); +} + +static int roce5_ib_register_device(struct roce5_device *rdev) +{ +#if defined(IB_REG_DEV_HAVE_PORT_CALLBACK) + roce5_fix_ibdev_name(rdev); + return ib_register_device(&rdev->ib_dev, NULL); +#elif defined(IB_REG_DEV_HAVE_NAME) && defined(IB_REG_DEV_HAVE_DEVICE) +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + return ib_register_device(&rdev->ib_dev, "hrn5_bond_%d", + rdev->dev); + } +#endif + + return ib_register_device(&rdev->ib_dev, "hrn5_%d", rdev->dev); +#endif +} + +static int roce5_dev_register(struct roce5_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce5_ib_register_device(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to reg ibdev, func_id(%d)", + rdev->glb_func_id); + goto err_reg_dev; + } + + ret = memcpy_s(uld_dev_name, ROCE_ULD_DEV_NAME_LEN, rdev->ib_dev.name, + ROCE_ULD_DEV_NAME_LEN); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy uld_dev_name.(ret:%d)", + ret); + goto err_memcpy_uld; + } + + return 0; + +err_memcpy_uld: + ib_unregister_device(&rdev->ib_dev); +err_reg_dev: + return ret; +} + +static void roce5_dev_unregister(struct roce5_device *rdev) +{ + ib_unregister_device(&rdev->ib_dev); +} + +static int roce5_handle_register_event(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_add_real_device_mac(rdev, rdev->ndev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to add real device mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (netif_running(rdev->ndev) && + test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) { + roce5_ifconfig_up_down_event_report(rdev, IB_EVENT_PORT_ACTIVE); + } + + return 0; +} + +static int roce5_init_dev(struct roce5_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce5_init_dev_info(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init dev info, func_id(%d)", + rdev->glb_func_id); + goto err_init_dev_info; + } + + ret = roce5_alloc_hw_resource(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc hw res, func_id(%d)", + rdev->glb_func_id); + goto err_alloc_hw_res; + } + + ret = roce5_dev_register(rdev, uld_dev_name); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init register, func_id(%d)", + rdev->glb_func_id); + goto err_init_register; + } + + ret = roce5_g_rdev_add(rdev); + if (ret != 0) { + ROCE_ERR("Failed to add rdev to g_rdev"); + goto err_add_rdev; + } + + ret = roce5_get_feature(rdev, &rdev->cfg_info.feature_cap); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get feature, ret=%d", ret); + goto err_get_feature_or_init_sysfs; + } + + ret = roce5_init_sysfs(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init sysfs, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + goto err_get_feature_or_init_sysfs; + } + rdev->ib_active = true; + + ret = roce5_handle_register_event(rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to handle register event, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + goto err_register_event; + } + + ROCE_DEV_INFO(rdev, "Add init all ok, func_id(%d), dev_name(%s)", + rdev->glb_func_id, rdev->ib_dev.name); + + return 0; + +err_register_event: + roce5_remove_sysfs(rdev); +err_get_feature_or_init_sysfs: + roce5_g_rdev_remove(rdev); +err_add_rdev: + roce5_dev_unregister(rdev); +err_init_register: + roce5_dealloc_hw_resource(rdev); +err_alloc_hw_res: + roce5_deinit_dev_info(rdev); +err_init_dev_info: + return ret; +} + +static void roce5_rdev_qp_lb_init(struct roce5_device *rdev) +{ + int i = 0; + mutex_init(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); + for (; i < ROCE_LB_QPN_MAX; i++) { + rdev->qp_lb_scqn_ctrl.qp[i] = 0; + } +} + +static int roce5_add_rdev_init(struct roce5_device *rdev) +{ + int ret = 0; + + ret = roce5_board_info_get(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get board info"); + return ret; + } + +#ifdef ROCE_BONDING_EN + ret = roce5_bond_attach(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to attach bond device."); + return ret; + } + + atomic_set(&rdev->next_port, 0); +#endif + + rdev->gid_dev = (struct net_device **)kzalloc( + rdev->rdma_cap.max_gid_per_port * sizeof(struct net_device *), + GFP_KERNEL); + if (rdev->gid_dev == NULL) { + ROCE_DEV_ERR(rdev, "Failed to alloc gid dev memory"); + return -ENOMEM; + } + + ret = roce5_rdma_init_resource(rdev->hwdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init rdma resources."); + goto err_init_resource; + } + + rdev->glb_func_id = hinic5_global_func_id(rdev->hwdev); + roce5_rdev_qp_lb_init(rdev); + + roce5_init_hw_info(rdev); + + ret = roce5_init_cfg_info(rdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get roce cfg, func_id(%d)", + rdev->glb_func_id); + goto err_init_cfg; + } + + ROCE_DEV_INFO(rdev, "Initializing."); + return 0; + +err_init_cfg: + roce5_rdma_cleanup_resource(rdev->hwdev); + +err_init_resource: + if (rdev->gid_dev != NULL) { + kfree(rdev->gid_dev); + } + return ret; +} + +static void roce5_add_rdev_unit(struct roce5_device *rdev) +{ + roce5_rdma_cleanup_resource(rdev->hwdev); + if (rdev->gid_dev != NULL) { + kfree(rdev->gid_dev); + } +} + +static __be64 rdma_gen_node_guid(u8 *dev_mac) +{ + u8 guid[8]; + u8 mac_addr[6]; + __be64 node_guid = 0; + int ret; + + if (dev_mac == NULL) { + ROCE_ERR("Dev_mac is null"); + return RDMA_INVALID_GUID; + } + + ret = memcpy_s((void *)&mac_addr[0], sizeof(mac_addr), (void *)dev_mac, + sizeof(mac_addr)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mac_addr.(ret:%d)", ret); + return ENOMEM; + } + + guid[0] = mac_addr[0] ^ + DEV_ADDR_FIRST_BYTE_VAL_MASK; // 0 is mac & guid array idx + guid[1] = mac_addr[1]; // 1 is mac & guid array idx + guid[2] = mac_addr[2]; // 2 is mac & guid array idx + guid[3] = 0xff; // 3 is guid array idx + guid[4] = 0xfe; // 4 is guid array idx + guid[5] = mac_addr[3]; // 5 is guid array idx, 3 is mac array idx + guid[6] = mac_addr[4]; // 6 is guid array idx, 4 is mac array idx + guid[7] = mac_addr[5]; // 7 is guid array idx, 5 is mac array idx + + /* node_guid is calculated by guid. */ + node_guid = + ((u64)guid[0] << 56) | // 0 is guid array idx, 56 is guid offset + ((u64)guid[1] << 48) | // 1 is guid array idx, 48 is guid offset + ((u64)guid[2] << 40) | // 2 is guid array idx, 40 is guid offset + ((u64)guid[3] << 32) | // 3 is guid array idx, 32 is guid offset + ((u64)guid[4] << 24) | // 4 is guid array idx, 24 is guid offset + ((u64)guid[5] << 16) | // 5 is guid array idx, 16 is guid offset + ((u64)guid[6] << 8) | // 6 is guid array idx, 8 is guid offset + (u64)guid[7]; // 7 is guid array idx + + return (__be64)cpu_to_be64(node_guid); +} + +static __be64 roce5_rdma_init_guid(void *hwdev, struct net_device *netdev) +{ + struct rdma_comp_priv *comp_priv = NULL; + + if ((hwdev == NULL) || (netdev == NULL)) { + ROCE_ERR("Hwdev or netdev is null"); + return ~0ULL; + } + + comp_priv = get_rdma_comp_priv(hwdev); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return ~0ULL; + } + + comp_priv->rdma_comp_res.node_guid = + rdma_gen_node_guid((u8 *)netdev->dev_addr); + + return comp_priv->rdma_comp_res.node_guid; +} + +#if defined(HAVE_IB_DEVICE_OPS) +static const struct ib_device_ops dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HRN, + .uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION, + + .modify_qp = roce5_modify_qp, + .query_qp = roce5_query_qp, + .destroy_qp = roce5_destroy_qp, + .post_send = roce5_post_send, + .post_recv = roce5_post_recv, + .create_cq = roce5_create_cq, + .modify_cq = roce5_modify_cq, + .resize_cq = roce5_resize_cq, + .destroy_cq = roce5_destroy_cq, + .poll_cq = roce5_poll_cq, + .req_notify_cq = roce5_arm_cq, + .create_srq = roce5_create_srq, + .modify_srq = roce5_modify_srq, + .query_srq = roce5_query_srq, + .destroy_srq = roce5_destroy_srq, + .post_srq_recv = roce5_post_srq_recv, + .map_mr_sg = roce5_map_kernel_frmr_sg, + .get_dma_mr = roce5_get_dma_mr, + .reg_user_mr = roce5_reg_user_mr, + .dereg_mr = roce5_dereg_mr, + .alloc_mr = roce5_alloc_mr, + .alloc_mw = roce5_alloc_mw, + .dealloc_mw = roce5_dealloc_mw, + .query_device = roce5_query_device, + .add_gid = roce5_ib_add_gid, + .del_gid = roce5_ib_del_gid, + .modify_device = roce5_modify_device, + .alloc_ucontext = roce5_alloc_ucontext, + .dealloc_ucontext = roce5_dealloc_ucontext, + .mmap = roce5_mmap, + .alloc_pd = roce5_alloc_pd, + .dealloc_pd = roce5_dealloc_pd, + .create_ah = roce5_create_ah, + .query_ah = roce5_query_ah, + .destroy_ah = roce5_destroy_ah, + .alloc_xrcd = roce5_alloc_xrcd, + .dealloc_xrcd = roce5_dealloc_xrcd, + +#if defined(ROCE_CREATE_QP_INT) + .create_qp = roce5_create_qp, +#elif defined(ROCE_CREATE_QP_IB_QP) + .create_qp = roce5_create_qp_compat, +#endif + +#if defined(HAVE_ALL_ROCE_FUNCS_U32_PORT) + .query_port = roce5_query_port, +#elif defined(HAVE_ALL_ROCE_FUNCS_U8_PORT) + .query_port = roce5_query_port_compat, +#endif + +#if defined(GET_LINK_LAYER_HAVE_U32_PORT) + .get_link_layer = roce5_port_link_layer, +#elif defined(GET_LINK_LAYER_HAVE_U8_PORT) + .get_link_layer = roce5_port_link_layer_compat, +#endif + +#if defined(QUERY_GID_HAVE_U32_PORT) + .query_gid = roce5_query_gid, +#elif defined(QUERY_GID_HAVE_U8_PORT) + .query_gid = roce5_query_gid_compat, +#endif + +#if defined(QUERY_PKEY_HAVE_U32_PORT) + .query_pkey = roce5_query_pkey, +#elif defined(QUERY_PKEY_HAVE_U8_PORT) + .query_pkey = roce5_query_pkey_compat, +#endif + +#if defined(MODIFY_PORT_HAVE_U32_PORT) + .modify_port = roce5_modify_port, +#elif defined(MODIFY_PORT_HAVE_U8_PORT) + .modify_port = roce5_modify_port_compat, +#endif + +#if defined(GET_PORT_IMMUTABLE_HAVE_U32_PORT) + .get_port_immutable = roce5_port_immutable, +#elif defined(GET_PORT_IMMUTABLE_HAVE_U8_PORT) + .get_port_immutable = roce5_port_immutable_compat, +#endif + +#if defined(GET_NETDEV_HAVE_U32_PORT) + .get_netdev = roce5_ib_get_netdev, +#elif defined(GET_NETDEV_HAVE_U8_PORT) + .get_netdev = roce5_ib_get_netdev_compat, +#endif + +#if defined(IB_DEVICE_OPS_HAVE_CREATE_USER_AH) + .create_user_ah = roce5_create_ah, +#endif + + INIT_RDMA_OBJ_SIZE(ib_ah, roce5_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, roce5_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_pd, roce5_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_srq, roce5_srq, ibsrq), + INIT_RDMA_OBJ_SIZE(ib_ucontext, roce5_ucontext, ibucontext), + INIT_RDMA_OBJ_SIZE(ib_xrcd, roce5_xrcd, ibxrcd), + INIT_RDMA_OBJ_SIZE(ib_mw, roce5_mw, ibmw), +#if defined(IB_DEVICE_OPS_HAVE_IB_QP_OBJ) + INIT_RDMA_OBJ_SIZE(ib_qp, roce5_qp, ibqp), +#endif +}; + +static void roce5_add_init(struct roce5_device *rdev) +{ + struct ib_device *ib_dev = &rdev->ib_dev; + + ib_dev->local_dma_lkey = rdev->rdma_cap.reserved_lkey; + ib_dev->phys_port_cnt = (u8)rdev->rdma_cap.num_ports; + ib_dev->num_comp_vectors = + (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ? + (int)rdev->rdma_cap.num_comp_vectors : + MAX_CEQ_NEED; + ib_dev->node_type = RDMA_NODE_IB_CA; + ib_dev->node_guid = roce5_rdma_init_guid(rdev->hwdev, rdev->ndev); + ib_dev->dma_device = rdev->dev; + ib_dev->dev.parent = ib_dev->dma_device; + (void)strcpy_s(ib_dev->node_desc, IB_DEVICE_NODE_DESC_MAX, "hrn5"); + + rdev->ib_dev.uverbs_cmd_mask = ROCE_UVERBS_CMD_BASE_MASK; +#if defined(IB_DEV_HAVE_UVERBS_EX_CMD_MASK) + rdev->ib_dev.uverbs_ex_cmd_mask = ROCE_UVERBS_EXT_CMD_MASK; +#endif + + ib_set_device_ops(ib_dev, &dev_ops); + roce5_init_dev_ext_handlers(rdev); +} + +#else +static void roce5_init_dev_qp_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.create_qp = roce5_create_qp_compat; + rdev->ib_dev.modify_qp = roce5_modify_qp; + rdev->ib_dev.query_qp = roce5_query_qp; + rdev->ib_dev.destroy_qp = roce5_destroy_qp_ofa; + rdev->ib_dev.post_send = roce5_post_send; + rdev->ib_dev.post_recv = roce5_post_recv; +} + +static void roce5_init_dev_cq_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.create_cq = roce5_create_cq_ofa; + rdev->ib_dev.modify_cq = roce5_modify_cq; + rdev->ib_dev.resize_cq = roce5_resize_cq; + rdev->ib_dev.destroy_cq = roce5_destroy_cq_ofa; + rdev->ib_dev.poll_cq = roce5_poll_cq; + rdev->ib_dev.req_notify_cq = roce5_arm_cq; +} + +static void roce5_init_dev_srq_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.create_srq = roce5_create_srq_ofa; + rdev->ib_dev.modify_srq = roce5_modify_srq; + rdev->ib_dev.query_srq = roce5_query_srq; + rdev->ib_dev.destroy_srq = roce5_destroy_srq_ofa; + rdev->ib_dev.post_srq_recv = roce5_post_srq_recv; +} + +static void roce5_init_dev_mr_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.map_mr_sg = roce5_map_kernel_frmr_sg; + + rdev->ib_dev.get_dma_mr = roce5_get_dma_mr; + rdev->ib_dev.reg_user_mr = roce5_reg_user_mr; + rdev->ib_dev.dereg_mr = roce5_dereg_mr_ofa; + rdev->ib_dev.alloc_mr = roce5_alloc_mr; + + rdev->ib_dev.alloc_mw = roce5_alloc_mw_ofa; + rdev->ib_dev.dealloc_mw = roce5_dealloc_mw_ofa; +} + +static void roce5_init_dev_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.query_device = roce5_query_device; + rdev->ib_dev.query_port = roce5_query_port_compat; + rdev->ib_dev.get_link_layer = roce5_port_link_layer_compat; + rdev->ib_dev.query_gid = roce5_query_gid_compat; + rdev->ib_dev.add_gid = roce5_ib_add_gid; + rdev->ib_dev.del_gid = roce5_ib_del_gid; + rdev->ib_dev.query_pkey = roce5_query_pkey_compat; + rdev->ib_dev.modify_device = roce5_modify_device; + rdev->ib_dev.modify_port = roce5_modify_port_compat; + + rdev->ib_dev.alloc_ucontext = roce5_alloc_ucontext_ofa; + rdev->ib_dev.dealloc_ucontext = roce5_dealloc_ucontext_ofa; + rdev->ib_dev.mmap = roce5_mmap; + + rdev->ib_dev.alloc_pd = roce5_alloc_pd_ofa; + rdev->ib_dev.dealloc_pd = roce5_dealloc_pd_ofa; + + rdev->ib_dev.create_ah = roce5_create_ah_ofa; + rdev->ib_dev.query_ah = roce5_query_ah; + rdev->ib_dev.destroy_ah = roce5_destroy_ah_ofa; + + roce5_init_dev_qp_handlers(rdev); + roce5_init_dev_cq_handlers(rdev); + roce5_init_dev_srq_handlers(rdev); + roce5_init_dev_mr_handlers(rdev); + + rdev->ib_dev.alloc_xrcd = roce5_alloc_xrcd_ofa; + rdev->ib_dev.dealloc_xrcd = roce5_dealloc_xrcd_ofa; + + rdev->ib_dev.get_port_immutable = roce5_port_immutable_compat; + rdev->ib_dev.get_netdev = roce5_ib_get_netdev_compat; +} + +static void roce5_add_init(struct roce5_device *rdev) +{ + int ret; + + rdev->ib_dev.owner = THIS_MODULE; + rdev->ib_dev.node_type = RDMA_NODE_IB_CA; + rdev->ib_dev.local_dma_lkey = rdev->rdma_cap.reserved_lkey; + rdev->ib_dev.phys_port_cnt = (u8)rdev->rdma_cap.num_ports; + rdev->ib_dev.num_comp_vectors = + (rdev->rdma_cap.num_comp_vectors <= MAX_CEQ_NEED) ? + (int)rdev->rdma_cap.num_comp_vectors : + MAX_CEQ_NEED; + rdev->ib_dev.dma_device = rdev->dev; + rdev->ib_dev.dev.parent = rdev->ib_dev.dma_device; + rdev->ib_dev.uverbs_abi_ver = ROCE_IB_UVERBS_ABI_VERSION; + rdev->ib_dev.node_guid = roce5_rdma_init_guid(rdev->hwdev, rdev->ndev); + ret = memcpy_s((void *)rdev->ib_dev.node_desc, ROCE_NODE_DESC_LEN, + "hrn5", ROCE_NODE_DESC_LEN); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy node_desc.(ret:%d)", ret); + return; + } + + rdev->ib_dev.uverbs_cmd_mask = roce5_get_uverbs_cmd_mask(); + rdev->ib_dev.uverbs_ex_cmd_mask = ROCE_UVERBS_EXT_CMD_MASK; + + roce5_init_dev_handlers(rdev); + roce5_init_dev_ext_handlers(rdev); + return; +} +#endif + +static void roce5_mod_param_parse(struct roce5_device *rdev) +{ + if (g_loop_times <= 0) { + ROCE_DEV_WARN( + rdev, + "g_loop_times has been amended to 1, because original value(%d) is smaller than " + "or equal to 0.", + g_loop_times); + g_loop_times = 1; + } + rdev->try_times = g_loop_times; + rdev->destory_flow_timeout = g_destory_flow_timeout; + rdev->local_en = g_local_en; + rdev->full_qp_spec = g_full_qp_spec; + rdev->huge_page_en = g_huge_page_en; +} + +static void roce5_init_mac_vlan(struct roce5_device *rdev) +{ + INIT_LIST_HEAD(&rdev->mac_vlan_list_head); + mutex_init(&rdev->mac_vlan_mutex); +} + +static int roce5_rdev_init(struct roce5_device *rdev) +{ + int ret; + + ret = hinic5_stateful_init(rdev->hwdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init stateful resource"); + return ret; + } + + roce5_init_mac_vlan(rdev); + + ret = roce5_add_rdev_init(rdev); + if (ret != 0) { + goto err; + } + + spin_lock_init(&rdev->node_desc_lock); + rwlock_init(&rdev->rwlock_func_tbl_bond); + mutex_init(&rdev->cap_mask_mutex); + spin_lock_init(&rdev->reset_flow_resource_lock); + INIT_LIST_HEAD(&rdev->qp_list); + roce5_mod_param_parse(rdev); + + roce5_fix_ibdev_name(rdev); + + return 0; +err: + roce5_clean_vlan_device_mac(rdev); + hinic5_stateful_deinit(rdev->hwdev); + return ret; +} + +static void roce5_rdev_unit(struct roce5_device *rdev) +{ + /* FLR by MPU when hotplug, don't need deinit anymore */ + hinic5_stateful_deinit(rdev->hwdev); + + roce5_add_rdev_unit(rdev); + + mutex_destroy(&rdev->qp_lb_scqn_ctrl.qp_lb_scqn_ctrl_mutex); +} + +static int roce5_add_do_init(struct roce5_device *rdev, char *uld_dev_name) +{ + int ret; + + ret = roce5_rdev_init(rdev); + if (ret != 0) { + goto err_init_rdev; + } + + ret = roce5_register_template(rdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to register cqm_service, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + goto err_cqm_register; + } + + ret = hinic5_alloc_db_addr(rdev->hwdev, &rdev->kernel_db_map, NULL); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc db or dwqe, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_alloc_db; + } +#ifdef __ROCE_DFX__ + ret = roce5_dfx_device_init(rdev); + if (ret != 0) { + ROCE_ERR("Failed to init dfx device"); + goto err_init_dfx; + } +#endif + roce5_add_init(rdev); + + ret = roce5_alloc_ceq(rdev); + if (ret < 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc ceqs, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + goto err_alloc_ceq; + } + + ret = roce5_init_dev(rdev, uld_dev_name); + if (ret != 0) { + goto err_init_dev; + } + + return 0; + +err_init_dev: + roce5_free_ceq(rdev); + +err_alloc_ceq: +#ifdef __ROCE_DFX__ + roce5_dfx_device_deinit(rdev); + +err_init_dfx: +#endif + hinic5_free_db_addr(rdev->hwdev, rdev->kernel_db_map, NULL); + +err_alloc_db: + cqm5_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + +err_cqm_register: + roce5_rdev_unit(rdev); + +err_init_rdev: + return ret; +} + +static int roce5_add_check(const struct hinic5_lld_dev *lld_dev) +{ + int ret = 0; + + if (lld_dev == NULL || lld_dev->hwdev == NULL) { + ROCE_ERR("Lld_dev is null"); + return (-EINVAL); + } + + ret = roce5_add_check_ext(lld_dev); + return ret; +} + +static int roce5_stateful_init(struct hinic5_lld_dev *lld_dev) +{ + int ret = 0; + struct hinic5_lld_dev *ppf_lld_dev = NULL; + + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + return 0; + } + + ppf_lld_dev = hinic5_get_ppf_lld_dev(lld_dev); + if (ppf_lld_dev == NULL) { + ROCE_ERR("Failed to get ppf lld dev, func id(%u).", + hinic5_global_func_id(lld_dev->hwdev)); + return -EINVAL; + } + + ret = hinic5_stateful_init(ppf_lld_dev->hwdev); + hinic5_lld_dev_put(ppf_lld_dev); + + return ret; +} + +static void roce5_stateful_deinit(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_lld_dev *ppf_lld_dev = NULL; + + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + return; + } + + ppf_lld_dev = hinic5_get_ppf_lld_dev(lld_dev); + if (ppf_lld_dev == NULL) { + ROCE_ERR("Failed to get ppf lld dev, func id(%u).", + hinic5_global_func_id(lld_dev->hwdev)); + return; + } + + hinic5_stateful_deinit(ppf_lld_dev->hwdev); + + hinic5_lld_dev_put(ppf_lld_dev); +} + +static void roce5_do_remove(struct roce5_device *rdev, u16 glb_func_id, + const char *dev_name) +{ +#ifdef __ROCE_DFX__ + roce5_dfx_device_deinit(rdev); +#endif + + roce5_g_rdev_remove(rdev); + + roce5_remove_sysfs(rdev); + + ib_unregister_device(&rdev->ib_dev); + + roce5_dealloc_hw_resource(rdev); + + roce5_deinit_dev_info(rdev); + +#ifdef ROCE_BONDING_EN + roce5_set_bond_ipsurx_en(true); +#endif + + roce5_clean_vlan_device_mac(rdev); + roce5_clean_real_device_mac(rdev); + + roce5_free_ceq(rdev); + + hinic5_free_db_addr(rdev->hwdev, rdev->kernel_db_map, NULL); + + cqm5_service_unregister(rdev->hwdev, SERVICE_T_ROCE); + + roce5_del_func_res(rdev); + ROCE_INFO("Function level resource clear ok, func_id(%d), name(%s)", + glb_func_id, dev_name); + + roce5_rdev_unit(rdev); +} + +static void roce5_remove(struct hinic5_lld_dev *lld_dev, void *uld_dev) +{ + struct roce5_device *rdev = NULL; + char *device_name = NULL; + u16 glb_func_id; + roce_user_handle_t hdl = { 0 }; + + if ((uld_dev == NULL) || (lld_dev == NULL)) { + ROCE_ERR("input param is null"); + return; + } + + if (hinic5_support_roce(lld_dev->hwdev, NULL) == false) { + ROCE_WARN("Not support RoCE."); + return; + } + + rdev = (struct roce5_device *)uld_dev; + rdev->ib_active = false; + device_name = rdev->ib_dev.name; + glb_func_id = rdev->glb_func_id; + + ROCE_DEV_INFO(rdev, "Remove start, func_id(%d), name(%s)", + rdev->glb_func_id, device_name); + + hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + hdl.rdev = rdev; + roce5_uld_rdev_remove((roce_uld_handle)&hdl); + + roce5_do_remove(rdev, glb_func_id, device_name); + +#ifdef ROCE_BONDING_EN + roce5_bond_detach(rdev); +#endif + + ib_dealloc_device(&rdev->ib_dev); + + roce5_stateful_deinit(lld_dev); + + ROCE_INFO("Remove end, func_id(%d), name(%s)", glb_func_id, + device_name); +} + +/* **************************************************************************** + Prototype : roce5_add + Description : roce5_add + Input : struct hinic_lld_dev *lld_dev + void **uld_dev + char *uld_dev_name + Output : None + + 1.Date : 2015/5/27 + Modification : Created function + +**************************************************************************** */ +static int roce5_add(struct hinic5_lld_dev *lld_dev, void **uld_dev, + char *uld_dev_name) +{ + struct roce5_device *rdev = NULL; + struct rdma_service_cap rdma_cap; + int ret = 0; + roce_user_handle_t hdl = { 0 }; + + ret = roce5_add_check(lld_dev); + if (ret != 0) { + return ret; + } + + /* return 0(false) if the hwdev don't support ROCE, make sure it probe success */ + if (hinic5_support_roce(lld_dev->hwdev, &rdma_cap) == false) { + ROCE_WARN("Not support RoCE, func: %d", + hinic5_global_func_id(lld_dev->hwdev)); + // this pf no has enable RoCE, but is not affect RoCE driver load, no need return exception value + ret = 0; + goto err_check; + } + + ret = roce5_stateful_init(lld_dev); + if (ret != 0) { + ROCE_ERR("Failed to add ref to ppf."); + goto err_check; + } + + rdev = roce5_rdev_alloc(lld_dev, uld_dev, &rdma_cap); + if (rdev == NULL) { + ROCE_ERR("Failed to alloc rdev, func: %d", + hinic5_global_func_id(lld_dev->hwdev)); + ret = -EINVAL; + goto err_alloc; + } + + ret = roce5_add_do_init(rdev, uld_dev_name); + if (ret != 0) { + ROCE_ERR("Failed to init rdev."); + goto err_do_init; + } + + hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + hdl.rdev = rdev; + ROCE_CONSTUCT_ULD_HDL_SHADOW(&hdl, &rdev->shadow, &rdev->shadow_vfid, + &rdev->shadow_vf_num); + ret = roce5_uld_rdev_add((roce_uld_handle)&hdl); + if (ret != 0) { + ROCE_ERR("Failed to add uld rdev."); + roce5_do_remove(rdev, rdev->glb_func_id, uld_dev_name); + goto err_do_init; + } + + ROCE_DEV_INFO(rdev, "Device(0x%p) init success.", rdev); + return 0; + +err_do_init: + ib_dealloc_device(&rdev->ib_dev); +err_alloc: + roce5_stateful_deinit(lld_dev); +err_check: + *uld_dev = NULL; + return ret; +} + +static int roce5_set_nic_event(struct roce5_device *rdev, + const struct hinic5_event_info *event) +{ + switch (event->type) { + case EVENT_NIC_LINK_UP: + if (test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) != 0) { + return -1; + } + ROCE_DEV_INFO(rdev, "Report NIC LINK_UP, func_id(%d)", + rdev->glb_func_id); + return IB_EVENT_PORT_ACTIVE; + + case EVENT_NIC_LINK_DOWN: + if (test_and_clear_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) { + return -1; + } + ROCE_DEV_INFO(rdev, "RoCE report NIC LINK_DOWN, func_id(%d)", + rdev->glb_func_id); + return IB_EVENT_PORT_ERR; + + case EVENT_NIC_DCB_STATE_CHANGE: + ROCE_DEV_INFO( + rdev, + "DCB state change no longer requires RoCE involvement, func_id(%d)", + rdev->glb_func_id); + return -1; + + default: + ROCE_DEV_ERR(rdev, + "Nic event unsupported, func_id(%d), event(%d)", + rdev->glb_func_id, event->type); + return -1; + } +} + +static int roce5_set_ib_event(struct roce5_device *rdev, + const struct hinic5_event_info *event) +{ + switch (event->service) { + case EVENT_SRV_NIC: + return roce5_set_nic_event(rdev, event); + + case EVENT_SRV_COMM: + return roce5_set_comm_event(rdev, event); + + default: + ROCE_DEV_ERR( + rdev, + "func_id(%d) event from svc(%d) is not supported, event(%d)", + rdev->glb_func_id, event->service, event->type); + return -1; + } +} + +void roce5_event(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct hinic5_event_info *event) +{ + struct ib_event ibevent; + struct roce5_device *rdev = NULL; + int type, ret = 0; + + if ((lld_dev == NULL) || (uld_dev == NULL)) { + ROCE_ERR("Input params is null"); + goto out; + } + roce5_lock_rdev(); + + ret = roce5_get_rdev_by_uld(lld_dev, uld_dev, &rdev, event); + if ((ret != 0) || (rdev == NULL)) { + ROCE_ERR("find rdev failed"); + goto err_unlock; + } +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + roce5_handle_bonded_port_state_event(rdev); + goto err_unlock; + } +#endif + type = roce5_set_ib_event(rdev, event); + if (type == -1) { + goto err_unlock; + } + + ibevent.event = (enum ib_event_type)type; + ibevent.device = &rdev->ib_dev; + ibevent.element.port_num = ROCE_DEFAULT_PORT_NUM; + + if (rdev->ib_active) { + ROCE_DEV_INFO(rdev, "report event: %d.", type); + ib_dispatch_event(&ibevent); + } + +err_unlock: + roce5_unlock_rdev(); + +out: + return; +} + +int roce5_adm_dfx_get_dev_type(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct roce5_dfx_common_inbuf *inbuf = + (struct roce5_dfx_common_inbuf *)buf_in; + union roce5_dfx_common_outbuf *outbuf = + (union roce5_dfx_common_outbuf *)buf_out; + + struct roce5_device *find_rdev = + roce5_get_rdev_by_name(inbuf->dev_name); + + if (in_size < sizeof(struct roce5_dfx_common_inbuf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, + (int)sizeof(struct roce5_dfx_common_inbuf)); + return -ENOMEM; + } + if (find_rdev == NULL) { + ROCE_ERR("Failed to find rdev by device name(%s).", + inbuf->dev_name); + return -EINVAL; + } + + outbuf->dev_type = find_rdev->device_type; + *out_size = (u32)sizeof(union roce5_dfx_common_outbuf); + return 0; +} + +static roce5_adm_cmd_proc_t g_roce5_adm_cmd_proc[] = { +#ifdef __ROCE_DFX__ + { ROCE_CMD_GET_QPC_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_QPC_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_CQC_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_CQC_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_SRQC_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_SRQC_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_MPT_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_MPT_FROM_HOST, roce5_adm_dfx_query }, + { ROCE_CMD_GET_GID_FROM_CACHE, roce5_adm_dfx_query }, + { ROCE_CMD_GET_QPC_CQC_PI_CI, roce5_adm_dfx_query }, + { ROCE_CMD_GET_QP_COUNT, roce5_adm_dfx_query }, + { ROCE_CMD_GET_DEV_ALGO, roce5_adm_dfx_query }, + { ROCE_CMD_GET_DEV_TYPE, roce5_adm_dfx_get_dev_type }, +#ifdef ENABLE_DEBUG + { ROCE_CMD_DFX_ATTACK, roce5_dfx_attack_func }, +#endif +#ifdef ROCE_CHIP_TEST + { ROCE_CMD_SET_VNIC_WATERLINE, roce5_adm_dfx_vnic_wl_ctrl }, + { ROCE_CMD_GET_VNIC_WATERLINE, roce5_adm_dfx_vnic_wl_ctrl }, +#endif /* ROCE_CHIP_TEST */ + { ROCE_CMD_ROCE_SET, roce5_adm_dfx_roce_set }, + { ROCE_CMD_ROCE_SET_DEBUG, roce5_adm_dfx_roce_set_debug }, +#endif /* __ROCE_DFX__ */ + { ROCE_CMD_ENABLE_BW_CTRL, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_DISABLE_BW_CTRL, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_CHANGE_BW_CTRL_PARAM, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_QUERY_BW_CTRL_PARAM, roce5_adm_dfx_bw_ctrl }, + { ROCE_CMD_SET_LDCP_PARAM, roce5_adm_dfx_set_ldcp_param }, + { ROCE_CMD_PORT_TRAFFIC, roce5_adm_dfx_port_traffic }, + { ROCE_CMD_ULD_IOCTL_EXTEND, + (roce5_adm_func_t)roce5_adm_dfx_uld_extend }, +#ifdef ENABLE_HYPER_ROCE + { ROCE_CMD_HYPER, roce5_hyper_tool_dispatch }, +#endif +}; + +static int roce5_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct roce5_device *rdev = (struct roce5_device *)uld_dev; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *)g_roce5_adm_cmd_proc; + int elem_count = + sizeof(g_roce5_adm_cmd_proc) / sizeof(g_roce5_adm_cmd_proc[0]); + roce5_adm_func_t roce5_adm_func = NULL; + + /* check whether is root user */ + if (!capable(CAP_SYS_ADMIN)) { + ROCE_ERR("Invalid permission"); + return -EPERM; /* if not is root, return error */ + } + + if (cmd == GET_DRV_VERSION) { + return roce5_get_drv_version(buf_out, out_size); + } + + if (uld_dev == NULL || buf_in == NULL || buf_out == NULL || + out_size == NULL) { + ROCE_ERR("Input params is null"); + return -EINVAL; + } + + if (hinic5_support_roce(rdev->hwdev, NULL) == false) { + ROCE_ERR("Not support RoCE."); + return -EINVAL; + } + + if (cmd == GET_DRV_VERSION) { + return roce5_get_drv_version(buf_out, out_size); + } + + roce5_adm_func = (roce5_adm_func_t)roce5_adm_generic_find_cmd_func( + generic_array, elem_count, cmd); + if (roce5_adm_func == NULL) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", cmd); + return -EINVAL; + } + if (cmd == ROCE_CMD_ULD_IOCTL_EXTEND) { + roce_user_handle_t hdl = { 0 }; + hdl.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + hdl.rdev = rdev; + return roce5_adm_dfx_uld_extend(&hdl, buf_in, in_size, buf_out, + out_size); + } else { + return roce5_adm_func(rdev, buf_in, in_size, buf_out, out_size); + } +} + +struct hinic5_uld_info roce5_info = { + .probe = roce5_add, + .remove = roce5_remove, + .suspend = NULL, + .resume = NULL, + .event = roce5_event, + .ioctl = roce5_ioctl, +}; + +struct hinic5_uld_info *roce5_info_get(void) +{ + return &roce5_info; +} + +/* **************************************************************************** + Prototype : roce5_service_init + Description : + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** */ +static int __init roce5_service_init(void) +{ + int ret; + + roce5_service_init_pre(); + + ret = hinic5_register_uld(SERVICE_T_ROCE, roce5_info_get()); + if (ret != 0) { + ROCE_ERR("Failed to register uld. ret(%d)", ret); + return ret; + } + + ret = roce5_netlink_init(); + if (ret != 0) { + ROCE_ERR("Failed to init netlink, ret(%d)", ret); + hinic5_unregister_uld(SERVICE_T_ROCE); + return ret; + } + +#ifdef ROCE_BONDING_EN + ret = roce5_bond_init(); + if (ret != 0) { + ROCE_ERR("Failed to init RoCE Bond. ret(%d).", ret); + return ret; + } +#endif + + roce5_uld_service_init(); + + ret = roce5_register_netdev_notifier(); + if (ret != 0) { + ROCE_ERR("Failed to register netdev notifier. ret(%d)", ret); + return ret; + } + + ROCE_INFO("Register roce service successful"); + return 0; +} + +/* **************************************************************************** + Prototype : roce5_service_exit + Description : roce service disable + Input : void + Output : None + Return Value : + Calls : + Called By : + + History : + 1.Date : 2015/5/27 + Author : + Modification : Created function + +**************************************************************************** */ +static void __exit roce5_service_exit(void) +{ + roce5_unregister_netdev_notifier(); + + roce5_uld_service_exit(); + +#ifdef ROCE_BONDING_EN + roce5_bond_exit(); +#endif + + roce5_netlink_deinit(); + + hinic5_unregister_uld(SERVICE_T_ROCE); + + ROCE_INFO("Unregister roce service successful"); +} + +module_init(roce5_service_init); +module_exit(roce5_service_exit); diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c new file mode 100644 index 000000000..99d3b3afa --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.c @@ -0,0 +1,1082 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_netdev.h" +#include "roce_main_extension.h" +#include "roce_ctx_format.h" +#include "roce_gid.h" +#include "roce_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_restore.h" +#include "rdma_comp.h" +#include "roce_os_compat.h" +#include "roce5_gid_type.h" + +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif +#include "hinic5_srv_nic.h" + +static struct notifier_block g_roce5_netdev_notifier; + +/* **************************************************************************** + Prototype : roce5_fill_gid_type + Description : fill gid type + Input : struct rdma_gid_entry *gid_entry + Output : None + + 1.Date : 2016/5/27 + Modification : Created function + +**************************************************************************** */ +static void roce5_fill_gid_type(struct rdma_gid_entry *gid_entry, + enum roce5_gid_type new_gid_type) +{ + static u32 index_roce5_gid_mapping[32][4] = { + { 0x120E1436, 3452, 224529784, 0x180E252A }, + { 0x120E004A, 3472, 224660876, 0x6C0E403E }, + { 0x120E0042, 3464, 224660836, 0x840E4036 }, + { 0x120E0042, 3464, 224398688, 0xC00E0036 }, + { 0x1212143A, 3456, 224791932, 0x1812252E }, + { 0x1212004E, 3476, 224923024, 0x6C124042 }, + { 0x12120046, 3468, 224922984, 0x8412403A }, + { 0x12120046, 3468, 224660836, 0xC012003A }, + { 0x1216143E, 3460, 225054080, 0x18162532 }, + { 0x12160052, 3480, 225185172, 0x6C164046 }, + { 0x1216004A, 3472, 225185132, 0x8416403E }, + { 0x1216004A, 3472, 224922984, 0xC016003E }, + { 0x10301458, 3484, 226626968, 0x1830254C }, + { 0x1030006C, 3504, 226758060, 0x6C304060 }, + { 0x10300064, 3496, 226758020, 0x84304058 }, + { 0x10300064, 3496, 226495872, 0xC0300058 }, + { 0x10401468, 838864300, 227675560, 0x1840255C }, + { 0x1040007C, 1174408640, 227806652, 0x6C404070 }, + { 0x10400074, 1040190904, 227806612, 0x84404068 }, + { 0x10400074, 1040190904, 227544464, 0xC0400068 }, + { 0x1044146C, 905973168, 227937708, 0x18442560 }, + { 0x10440080, 1241517508, 228068800, 0x6C444074 }, + { 0x10440078, 1107299772, 228068760, 0x8444406C }, + { 0x10440078, 1107299772, 227806612, 0xC044006C }, + { 0x10481470, 973082036, 228199856, 0x18482564 }, + { 0x10480084, 1308626376, 228330948, 0x6C484078 }, + { 0x1048007C, 1174408640, 228330908, 0x84484070 }, + { 0x1048007C, 1174408640, 228068760, 0xC0480070 }, + { 0x1262148A, 1040190928, 230034892, 0x1862257E }, + { 0x1262009E, 1375735268, 230165984, 0x6C624092 }, + { 0x12620096, 1241517532, 230165944, 0x8462408A }, + { 0x12620096, 1241517532, 229903796, 0xC062008A }, + }; + + u8 i = 0; + gid_entry->dw6_h.bs.gid_type = new_gid_type; + gid_entry->dw6_h.bs.gid_update = (new_gid_type == ROCE_IPv4_ROCEv2_GID); + + i = ROCE_GID_MAP_TBL_IDX_GET(gid_entry->dw6_h.bs.tag, (u8)new_gid_type); + gid_entry->hdr_len_value = index_roce5_gid_mapping[i][0]; + if (new_gid_type == ROCE_IPv4_ROCEv2_GID) { + *((u32 *)(void *)gid_entry + 1) = + cpu_to_be32(index_roce5_gid_mapping[i][1]); + *((u32 *)(void *)gid_entry + ROCE_GID_MAP_TBL_IDX2) = + cpu_to_be32( + index_roce5_gid_mapping[i] + [ROCE_GID_MAP_TBL_IDX2]); + } +} + +static void roce5_fill_gid_smac(struct net_device *net_dev, + struct rdma_gid_entry *gid_entry) +{ + int ret; + ret = memcpy_s((void *)gid_entry->smac, sizeof(gid_entry->smac), + (void *)net_dev->dev_addr, sizeof(gid_entry->smac)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy smac.(ret:%d)", ret); + } +} + +/* + 2024-04-05 add rdev parameters, because is in roce driver load, via roce5_ib_add_gid to next call, roce5_fill_gid_entry_ext + via net_dev get to rdev is empty, function in advance return, cannot execute expected of tunnel assignment logical +*/ +static void roce5_fill_gid_vlan(const struct roce5_device *rdev, + struct net_device *net_dev, + struct rdma_gid_entry *gid_entry) +{ + if (rdma_vlan_dev_vlan_id(net_dev) != 0xffff) { + gid_entry->dw4.bs.cvlan = rdma_vlan_dev_vlan_id(net_dev) & + 0xfff; + gid_entry->dw6_h.bs.tag = ROCE_GID_VLAN_VALID; + } + + roce5_fill_gid_entry_ext(rdev, gid_entry); +} + +/* **************************************************************************** + Prototype : roce5_dispatch_event + Description : roce5_dispatch_event + Input : struct roce5_device *rdev + int port_num + enum ib_event_type type + Output : None + + 1.Date : 2016/3/29 + Modification : Created function + +**************************************************************************** */ +static void roce5_dispatch_event(struct roce5_device *rdev, int port_num, + enum ib_event_type type) +{ + struct ib_event event; + int ret; + + if (rdev == NULL) { + ROCE_ERR("Rdev is null"); + return; + } + + ret = memset_s(&event, sizeof(event), 0, sizeof(event)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memset event.(ret:%d)", ret); + } + + event.device = &rdev->ib_dev; + event.element.port_num = (u8)port_num; + event.event = type; + + ROCE_DEV_INFO(rdev, "driver dispatch event: %d.", type); + ib_dispatch_event(&event); +} + +static int roce5_del_old_gid_entry(struct roce5_device *rdev) +{ + struct rdma_gid_entry old_gid_entry; + int ret = 0; + int index = 0; + + ret = memset_s(&old_gid_entry, sizeof(old_gid_entry), 0, + sizeof(old_gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + ret = roce5_get_gid(rdev, 1, index, &old_gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &old_gid_entry, + ROCE_MPU_CMD_DEL_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to del ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_del_ipsu_tcam_tbl_entry(rdev, &old_gid_entry); + } + return ret; +} + +static int roce5_netdev_event_ipv4_dev_scan(struct roce5_device *rdev, + struct net_device *event_netdev, + struct rdma_gid_entry *gid_entry) +{ + struct in_device *in_dev = NULL; + int ret = 0; + int index = 0; + + ret = roce5_del_old_gid_entry(rdev); + if (ret != 0) { + ROCE_ERR("Failed to del gid_entry.(ret:%d)", ret); + return ret; + } + + in_dev = in_dev_get(event_netdev); + if (in_dev) { + for_ifa(in_dev) + { + ipv6_addr_set_v4mapped( + ifa->ifa_address, + (struct in6_addr *)gid_entry->raw); + + roce5_fill_gid_vlan(rdev, event_netdev, gid_entry); + roce5_fill_gid_smac(event_netdev, gid_entry); + + /* RoCE V2 */ + if (rdma_protocol_roce_udp_encap( + &rdev->ib_dev, ROCE_DEFAULT_PORT_NUM)) { + roce5_fill_gid_type(gid_entry, + ROCE_IPv4_ROCEv2_GID); + ret = roce5_update_gid_mac(rdev, 0, &index, + gid_entry); + } + + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update type(%u) gid(IPv4) when process netdev's event, func_id(%d)", + gid_entry->dw6_h.bs.gid_type, + rdev->glb_func_id); + in_dev_put(in_dev); + return ret; + } + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry( + rdev, gid_entry, + ROCE_MPU_CMD_ADD_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to add ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tcam_tbl_entry(rdev, gid_entry, + index); + } + roce5_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, + IB_EVENT_GID_CHANGE); + } + endfor_ifa(in_dev); + in_dev_put(in_dev); + } + + return 0; +} + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) +struct sin6_list { + struct list_head list; + struct sockaddr_in6 sin6; +}; + +static int roce5_gid_fill(struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry, + struct net_device *event_netdev, + struct sin6_list *sin6_iter) +{ + int ret = 0; + int index; + + ret = roce5_del_old_gid_entry(rdev); + if (ret != 0) { + ROCE_ERR("Failed to del gid_entry.(ret:%d)", ret); + return ret; + } + + if (cpu_to_be64(gid_entry->global.subnet_prefix) == + ROCE_DEFAULT_GID_SUBNET_PREFIX) { + return ret; + } + + roce5_fill_gid_vlan(rdev, event_netdev, gid_entry); + roce5_fill_gid_smac(event_netdev, gid_entry); + + /* RoCE V2 */ + if (rdma_protocol_roce_udp_encap(&rdev->ib_dev, + ROCE_DEFAULT_PORT_NUM)) { + roce5_fill_gid_type(gid_entry, ROCE_IPv6_ROCEv2_GID); + ret = roce5_update_gid_mac(rdev, 0, &index, gid_entry); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update V2 gid(IPv6), func_id(%d)", + rdev->glb_func_id); + list_del(&sin6_iter->list); + kfree(sin6_iter); + return ret; + } + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry( + rdev, gid_entry, ROCE_MPU_CMD_ADD_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to add ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tcam_tbl_entry(rdev, gid_entry, index); + } + } + + roce5_dispatch_event(rdev, ROCE_DEFAULT_PORT_NUM, IB_EVENT_GID_CHANGE); + + return ret; +} + +static int roce5_netdev_event_ipv6_dev_scan(struct roce5_device *rdev, + struct net_device *event_netdev, + struct rdma_gid_entry *gid_entry) +{ + struct sin6_list *sin6_temp = NULL; + struct sin6_list *sin6_iter = NULL; + struct inet6_dev *in6_dev = NULL; + struct inet6_ifaddr *ifp = NULL; + struct sin6_list *entry = NULL; + int ret; + + LIST_HEAD(sin6_init_list); + + in6_dev = in6_dev_get(event_netdev); + if (in6_dev == NULL) { + return 0; + } + + read_lock_bh(&in6_dev->lock); + list_for_each_entry(ifp, &in6_dev->addr_list, if_list) { + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (entry == NULL) { + continue; + } + + entry->sin6.sin6_family = AF_INET6; + entry->sin6.sin6_addr = ifp->addr; + list_add_tail(&entry->list, &sin6_init_list); + } + + read_unlock_bh(&in6_dev->lock); + in6_dev_put(in6_dev); + + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + if (memcpy_s((void *)gid_entry->raw, sizeof(union ib_gid), + (void *)&sin6_iter->sin6.sin6_addr, + sizeof(union ib_gid)) != 0) { + ROCE_WARN("Failed to do memcpy_s"); + } + + ret = roce5_gid_fill(rdev, gid_entry, event_netdev, sin6_iter); + if (ret != 0) { + goto err_free; + } + + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + + return 0; + +err_free: + list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_init_list, list) { + list_del(&sin6_iter->list); + kfree(sin6_iter); + } + return ret; +} +#endif + +static int roce5_netdev_event_ip_dev_scan(struct roce5_device *rdev, + struct net_device *event_netdev) +{ + struct rdma_gid_entry gid_entry; + int ret; + + ret = memset_s(&gid_entry, sizeof(gid_entry), 0, sizeof(gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + ret = roce5_netdev_event_ipv4_dev_scan(rdev, event_netdev, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to scan IPv4, func_id(%d)", + rdev->glb_func_id); + return ret; + } + +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) + ret = roce5_netdev_event_ipv6_dev_scan(rdev, event_netdev, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to scan IPv6, func_id(%d)", + rdev->glb_func_id); + return ret; + } +#endif + + return 0; +} + +static struct roce5_vlan_dev_list * +roce5_find_vlan_device_list(const struct list_head *mac_list, + const struct net_device *netdev) +{ + struct roce5_vlan_dev_list *vlan_dev_list = NULL; + + list_for_each_entry(vlan_dev_list, mac_list, list) { + if (vlan_dev_list->net_dev == netdev) { + return vlan_dev_list; + } + } + + return NULL; +} + +static int roce5_add_vlan_device(struct roce5_device *rdev, + struct net_device *netdev) +{ + struct roce5_vlan_dev_list *new_list = NULL; + int ret; + + new_list = (struct roce5_vlan_dev_list *)kzalloc(sizeof(*new_list), + GFP_KERNEL); + if (new_list == NULL) { + ROCE_DEV_ERR(rdev, + "Malloc mac_vlan list_head is NULL, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + + new_list->ref_cnt = 1; + new_list->net_dev = netdev; + new_list->vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID( + ((u32)rdma_vlan_dev_vlan_id(netdev))); + ret = memcpy_s(new_list->mac, ROCE_MAC_ADDR_LEN, netdev->dev_addr, + ROCE_MAC_ADDR_LEN); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mac.(ret:%d)", ret); + return -ENOMEM; + } + INIT_LIST_HEAD(&new_list->list); + list_add_tail(&new_list->list, &rdev->mac_vlan_list_head); + + return 0; +} + +static void roce5_del_vlan_device(struct roce5_device *rdev, + struct roce5_vlan_dev_list *old_list) +{ + list_del(&old_list->list); + kfree(old_list); +} + +int roce5_add_vlan_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + int ret = 0; + u32 vlan_id = 0; + + vlan_id = ROCE_GID_SET_VLAN_32BIT_VLAID( + ((u32)rdma_vlan_dev_vlan_id(netdev))); + ROCE_DEV_INFO( + rdev, + "enter roce5_add_vlan_device_mac, vlan_id(0x%x), func_id(%d).", + vlan_id, rdev->glb_func_id); + + ret = roce5_add_mac_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, + ETH_ALEN, vlan_id, rdev->glb_func_id); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set vlan device mac, vlan_id(0x%x), func_id(%d)", + vlan_id, rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, vlan_id, + rdev->glb_func_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + ret = roce5_add_bond_vlan_slave_mac( + rdev, (u8 *)netdev->dev_addr, (u16)vlan_id); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set bond vlan slave mac, vlan_id(0x%x), func_id(%d)", + vlan_id, rdev->glb_func_id); + goto err_add_bond_vlan_slave_mac; + } + } +#endif + + return ret; + +#ifdef ROCE_BONDING_EN +err_add_bond_vlan_slave_mac: + roce5_del_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, ETH_ALEN, + vlan_id, hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + + (void)hinic5_del_mac(rdev->hwdev, (u8 *)netdev->dev_addr, (u16)vlan_id, + rdev->glb_func_id, HINIC5_CHANNEL_ROCE); + + return ret; +#endif +} + +void roce5_del_vlan_device_mac(struct roce5_device *rdev, + struct roce5_vlan_dev_list *old_list) +{ +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + roce5_del_bond_vlan_slave_mac(rdev, old_list->mac, + (u16)old_list->vlan_id); + } +#endif + + roce5_del_ipsu_tbl_mac_entry(rdev, old_list->mac, ETH_ALEN, + old_list->vlan_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + + (void)roce5_del_mac_tbl_mac_entry(rdev, old_list->mac, ETH_ALEN, + old_list->vlan_id, rdev->glb_func_id); +} + +static int roce5_update_vlan_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + struct roce5_vlan_dev_list *old_list = NULL; + int ret; + + mutex_lock(&rdev->mac_vlan_mutex); + old_list = + roce5_find_vlan_device_list(&rdev->mac_vlan_list_head, netdev); + if (old_list == NULL) { + mutex_unlock(&rdev->mac_vlan_mutex); + return 0; + } + + if (ROCE_MEMCMP(old_list->mac, netdev->dev_addr, ROCE_MAC_ADDR_LEN) != + 0) { + roce5_del_vlan_device_mac(rdev, old_list); + + ret = roce5_add_vlan_device_mac(rdev, netdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to add mac_vlan, func_id(%d)", + rdev->glb_func_id); + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + + ret = memcpy_s(old_list->mac, ROCE_MAC_ADDR_LEN, + netdev->dev_addr, ROCE_MAC_ADDR_LEN); + if (ret != 0) { + roce5_del_vlan_device_mac(rdev, old_list); + mutex_unlock(&rdev->mac_vlan_mutex); + return ret; + } + } + + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +int roce5_add_real_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + int ret; + + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)netdev->dev_addr, 0, + rdev->glb_func_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + ret = roce5_add_bond_real_slave_mac(rdev, + (u8 *)netdev->dev_addr); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to add bond read device ipsu mac, func_id(%d)", + rdev->glb_func_id); + roce5_del_ipsu_tbl_mac_entry( + rdev, (u8 *)netdev->dev_addr, ETH_ALEN, 0, + hinic5_er_id(rdev->hwdev), rdev->glb_func_id); + return ret; + } + } +#endif + + ret = memcpy_s(rdev->mac, sizeof(rdev->mac), netdev->dev_addr, + sizeof(rdev->mac)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mac.(ret:%d)", ret); + return -ENOMEM; + } + + return 0; +} + +void roce5_del_real_device_mac(struct roce5_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + roce5_del_bond_real_slave_mac(rdev); + } +#endif + + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); +} + +static int roce5_update_real_device_mac(struct roce5_device *rdev, + struct net_device *netdev) +{ + int ret; + + roce5_del_real_device_mac(rdev); + + ret = roce5_add_real_device_mac(rdev, netdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to add real device ipsu mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + return 0; +} + +void roce5_clean_vlan_device_mac(struct roce5_device *rdev) +{ + struct roce5_vlan_dev_list *pos = NULL; + struct roce5_vlan_dev_list *tmp = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + list_for_each_entry_safe(pos, tmp, &rdev->mac_vlan_list_head, list) { +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + (void)roce5_del_bond_vlan_slave_mac(rdev, pos->mac, + (u16)pos->vlan_id); + } +#endif + roce5_del_ipsu_tbl_mac_entry(rdev, pos->mac, ETH_ALEN, + pos->vlan_id, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); + + (void)roce5_del_mac_tbl_mac_entry(rdev, pos->mac, ETH_ALEN, + pos->vlan_id, + rdev->glb_func_id); + + list_del(&pos->list); + kfree(pos); + } + mutex_unlock(&rdev->mac_vlan_mutex); +} + +void roce5_clean_real_device_mac(struct roce5_device *rdev) +{ +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + (void)roce5_del_bond_real_slave_mac(rdev); + } +#endif + + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, + hinic5_er_id(rdev->hwdev), + rdev->glb_func_id); +} + +void roce5_ifconfig_up_down_event_report(struct roce5_device *rdev, + u8 net_event) +{ + struct ib_event event = {}; + + event.device = &rdev->ib_dev; + event.event = net_event; + event.element.port_num = ROCE_DEFAULT_PORT_NUM; + + ib_dispatch_event(&event); +} + +static int roce5_netdev_event_raw_dev(unsigned long event, + struct roce5_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_CHANGEADDR) { + ret = roce5_update_real_device_mac(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update real device mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = roce5_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to scan ip_dev, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + +#ifdef ROCE_BONDING_EN + if (roce5_bond_is_active(rdev)) { + if ((event == NETDEV_DOWN) || (event == NETDEV_UP)) { + roce5_handle_bonded_port_state_event(rdev); + } + return 0; + } +#endif + if (event == NETDEV_DOWN) { + if (test_and_clear_bit(ROCE5_PORT_EVENT, &rdev->status) != 0) { + roce5_ifconfig_up_down_event_report(rdev, + IB_EVENT_PORT_ERR); + } + } + + if (event == NETDEV_UP) { + if (test_and_set_bit(ROCE5_PORT_EVENT, &rdev->status) == 0) { + roce5_ifconfig_up_down_event_report( + rdev, IB_EVENT_PORT_ACTIVE); + } + } + + return 0; +} + +static int roce5_netdev_event_vlan_dev(unsigned long event, + struct roce5_device *rdev, + struct net_device *event_netdev) +{ + int ret; + + if (event == NETDEV_CHANGEADDR) { + ret = roce5_update_vlan_device_mac(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to update vlan device mac, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + ret = roce5_netdev_event_ip_dev_scan(rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to scan vlan device ip list, func_id(%d)", + rdev->glb_func_id); + return ret; + } + } + + return 0; +} + +static int roce5_netdev_event(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct net_device *event_netdev = NULL; + struct roce5_device *rdev = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + struct net_device *real_dev = NULL; + int ret; + + if (nb == NULL || ptr == NULL) { + goto err_out; + } + + event_netdev = netdev_notifier_info_to_dev(ptr); + if (event_netdev->type != ARPHRD_ETHER) { + goto err_out; + } + + real_dev = (rdma_vlan_dev_real_dev(event_netdev) != 0) ? + rdma_vlan_dev_real_dev(event_netdev) : + event_netdev; + + lld_dev = hinic5_get_lld_dev_by_netdev(real_dev); + if (lld_dev == NULL) { + goto err_out; + } + + /* + * hinic5_get_uld_dev internal will call uld5_dev_hold add device reference count + * via uld5_dev_put free reference count + */ + rdev = (struct roce5_device *)hinic5_get_uld_dev(lld_dev, + SERVICE_T_ROCE); + + /* + * passed in of rdev is empty value when,roce5_bond_event_cfg_rdev will found lld corresponding to bond device rdev, call uld5_dev_hold add device reference count + * passed in of rdev non-null value when, reserved rdev original value, not will call uld5_dev_hold add device reference count + * via uld5_dev_put free reference count + */ + ret = roce5_bond_event_cfg_rdev(lld_dev, (void *)rdev, &rdev, true); + if (ret != 0 || rdev == NULL) { + goto err_out; + } + + /* not need extra of roce5_is_port_of_net_dev check */ + if (real_dev == event_netdev) { + ret = roce5_netdev_event_raw_dev(event, rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to deal with raw device, func_id(%d)", + rdev->glb_func_id); + goto err_put; + } + } else { + ret = roce5_netdev_event_vlan_dev(event, rdev, event_netdev); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to deal with vlan device, func_id(%d)", + rdev->glb_func_id); + goto err_put; + } + } + +err_put: + uld5_dev_put(rdev->lld_dev, SERVICE_T_ROCE); +err_out: + return NOTIFY_DONE; +} + +int roce5_register_netdev_notifier(void) +{ + int ret; + + g_roce5_netdev_notifier.notifier_call = roce5_netdev_event; + ret = register_netdevice_notifier(&g_roce5_netdev_notifier); + if (ret != 0) { + ROCE_ERR("Failed to register netdev notifier, ret(%d)", ret); + g_roce5_netdev_notifier.notifier_call = NULL; + } + + return ret; +} + +void roce5_unregister_netdev_notifier(void) +{ + if (g_roce5_netdev_notifier.notifier_call != NULL) { + unregister_netdevice_notifier(&g_roce5_netdev_notifier); + g_roce5_netdev_notifier.notifier_call = NULL; + } +} + +static int roce5_set_gid_entry(const struct ib_gid_attr *attr, + struct rdma_gid_entry *gid_entry) +{ + int ret = 0; + enum rdma_network_type network_type; + static enum roce5_gid_type gid_type_map[] = { + // map rdma_network_type to roce5_gid_type + [RDMA_NETWORK_IB] = ROCE_ROCEv1_GID, + [RDMA_NETWORK_ROCE_V1] = ROCE_ROCEv1_GID, + [RDMA_NETWORK_IPV4] = ROCE_IPv4_ROCEv2_GID, + [RDMA_NETWORK_IPV6] = ROCE_IPv6_ROCEv2_GID, + }; + + ret = memset_s(gid_entry, sizeof(*gid_entry), 0, sizeof(*gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + network_type = rdma_gid_attr_network_type(attr); + ret = memcpy_s((void *)gid_entry->raw, sizeof(union ib_gid), + (void *)(&attr->gid), sizeof(union ib_gid)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + if (network_type == RDMA_NETWORK_ROCE_V1 || + network_type == RDMA_NETWORK_IB) { + ROCE_ERR("network type(%d) is unsupported.", network_type); + return -EINVAL; + } + + roce5_fill_gid_vlan(to_roce5_dev(attr->device), attr->ndev, gid_entry); + roce5_fill_gid_type(gid_entry, gid_type_map[network_type]); + roce5_fill_gid_smac(attr->ndev, gid_entry); + + return 0; +} + +/* add vlan_mac list or ref_cnt + 1 + * add mac_vlan when new list add + */ +static int roce5_notify_vlan(struct roce5_device *rdev, + const struct ib_gid_attr *attr) +{ + int ret; + struct roce5_vlan_dev_list *old_list = NULL; + + mutex_lock(&rdev->mac_vlan_mutex); + ROCE_DEV_INFO(rdev, "Add vlan: func_id(%d), netdev(%s).", + rdev->glb_func_id, attr->ndev->name); + old_list = roce5_find_vlan_device_list(&rdev->mac_vlan_list_head, + attr->ndev); + if (old_list != NULL) { + old_list->ref_cnt++; + ret = 0; + goto out; + } + + ret = roce5_add_vlan_device(rdev, attr->ndev); + if (ret != 0) { + goto out; + } + + ret = roce5_add_vlan_device_mac(rdev, attr->ndev); + if (ret != 0) { + old_list = roce5_find_vlan_device_list( + &rdev->mac_vlan_list_head, attr->ndev); + if (old_list) { + roce5_del_vlan_device(rdev, old_list); + } + } + +out: + mutex_unlock(&rdev->mac_vlan_mutex); + + return ret; +} + +int roce5_ib_add_gid_common(const struct ib_gid_attr *attr) +{ + int ret; + struct rdma_gid_entry gid_entry; + struct roce5_device *rdev = NULL; + unsigned int index; + + // the input gid pointer is not needed to check on Euler OS + if (attr == NULL) { + ROCE_ERR("Input pointer is NULL"); + return (-EINVAL); + } + + rdev = to_roce5_dev(attr->device); + index = attr->index; + + ret = roce5_set_gid_entry(attr, &gid_entry); + if (ret != 0) { + ROCE_ERR("Failed to set gid entry."); + return ret; + } + + ret = roce5_update_gid(rdev, 0, index, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to update gid. ret(%d),func_id(%d)", + ret, rdev->glb_func_id); + return ret; + } + + rdev->gid_dev[index] = attr->ndev; + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &gid_entry, + ROCE_MPU_CMD_ADD_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to add ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_add_ipsu_tcam_tbl_entry(rdev, &gid_entry, index); + } + + if (rdma_vlan_dev_real_dev(attr->ndev)) { + ret = roce5_notify_vlan(rdev, attr); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to nofify vlan, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return ret; + } + } + + return 0; +} + +int roce5_ib_add_gid(const struct ib_gid_attr *attr, + __always_unused void **context) +{ + return roce5_ib_add_gid_common(attr); +} + +int roce5_ib_del_gid(const struct ib_gid_attr *attr, + __always_unused void **context) +{ + struct roce5_vlan_dev_list *old_list = NULL; + struct roce5_device *rdev = NULL; + struct rdma_gid_entry gid_entry; + u32 index = 0; + int ret; + + if ((attr == NULL) || (attr->device == NULL)) { + ROCE_ERR("Attr or attr->device is null"); + return (-EINVAL); + } + + rdev = to_roce5_dev(attr->device); + index = attr->index; + if (index >= rdev->rdma_cap.max_gid_per_port) { + ROCE_DEV_ERR(rdev, "Invalid gid index(%u), func_id(%d)", index, + rdev->glb_func_id); + return (-EINVAL); + } + + /* del vlan_mac list or ref_cnt - 1 */ + /* del mac_vlan when ref_cnt = 0 */ + if (rdev->ndev != rdev->gid_dev[index]) { + mutex_lock(&rdev->mac_vlan_mutex); + old_list = roce5_find_vlan_device_list( + &rdev->mac_vlan_list_head, rdev->gid_dev[index]); + if (old_list) { + old_list->ref_cnt--; + if (old_list->ref_cnt == 0) { + roce5_del_vlan_device_mac(rdev, old_list); + + roce5_del_vlan_device(rdev, old_list); + } + } + mutex_unlock(&rdev->mac_vlan_mutex); + } + + ret = memset_s(&gid_entry, sizeof(gid_entry), 0, sizeof(gid_entry)); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + + ret = roce5_get_gid(rdev, 1, index, &gid_entry); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get gid, func_id(%d), index(%u)", + rdev->glb_func_id, index); + return ret; + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ret = roce5_modify_ipsu_tbl_ip_entry(rdev, &gid_entry, + ROCE_MPU_CMD_DEL_IPSU_IP); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to del ipsu ip entry, func_id(%d)", + rdev->glb_func_id); + return ret; + } + + roce5_del_ipsu_tcam_tbl_entry(rdev, &gid_entry); + } + + /* + * delete gid no longer send cmd to ucode which write 0s to target entry, + * preventing PPE from building malformed packets. */ + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h new file mode 100644 index 000000000..f77f75dbc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netdev/roce_netdev.h @@ -0,0 +1,70 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_NETDEV_H +#define ROCE_NETDEV_H + +#include <net/ipv6.h> +#include <net/bonding.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/inetdevice.h> +#include <linux/if_vlan.h> + +#include <rdma/ib_addr.h> + +#include "roce.h" +#include "roce_compat.h" +#include "roce_mix.h" +#include "roce_cmd.h" + +#ifdef ROCE_BONDING_EN +#include "hinic5_srv_nic.h" +#endif + +#include "securec.h" + +#define ROCE_GID_SET_VLAN_32BIT_VLAID(vlan_id) \ + ((vlan_id) | (((u32)1) << 31)) /* the 31 bit is vlan eable bit */ +#define ROCE_MAC_ADDR_LEN 6 + +/* ipv4 header len. unit:4B */ +#define IPV4_HDR_LEN 5 + +#define IP_ECN_PERMIT 0x2 //ip header tclass bit[0:2] + +#define ROCE_DEFAULT_GID_SUBNET_PREFIX 0xFE80000000000000ULL + +#define ROCE_GID_MAP_TBL_IDX_GET(tag, new_gid_type) \ + ((u8)(((tag) << 2) | (new_gid_type))) + +#define ROCE_GID_MAP_TBL_IDX2 2 + +enum roce5_gid_vlan_type { ROCE_GID_VLAN_INVALID = 0, ROCE_GID_VLAN_VALID }; + +enum roce5_gid_tunnel_type { + ROCE_GID_TUNNEL_INVALID = 0, + ROCE_GID_TUNNEL_VALID +}; + +struct roce5_vlan_dev_list { + u8 mac[ETH_ALEN]; + u32 vlan_id; + u32 ref_cnt; + struct net_device *net_dev; + struct list_head list; +}; + +#if !defined(INETDEV_HAVE_FOR_IFA) +#define for_ifa(in_dev) \ + { \ + struct in_ifaddr *ifa; \ + for (ifa = (in_dev)->ifa_list; ifa; ifa = ifa->ifa_next) +#define endfor_ifa(in_dev) } +#endif + +int roce5_add_real_device_mac(struct roce5_device *rdev, + struct net_device *netdev); + +#endif /* ROCE_NETDEV_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c new file mode 100644 index 000000000..6a16c7f29 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.c @@ -0,0 +1,413 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. + + +#include <linux/genetlink.h> +#include <linux/netlink.h> +#include <linux/rtnetlink.h> +#include <linux/skbuff.h> +#include <linux/socket.h> +#include <net/genetlink.h> +#include <net/net_namespace.h> +#include <net/netlink.h> + +#include "roce.h" +#include "hinic5_crm.h" +#include "hinic5_lld.h" +#include "roce_drv_msg_intf.h" +#include "roce_infra.h" +#include "roce_kernel_compat_gen.h" +#include "roce_netlink_process.h" +#include "securec.h" +#include "udk_usrnl.h" +#ifdef ENABLE_HYPER_ROCE +#include "hyper_roce_netlink.h" +#endif +#include "roce_netlink.h" + +static int roce5_netlink_cmd_dispatch(struct sk_buff *skb, + struct genl_info *nl_info); + +static DEFINE_RWLOCK(g_cmd_process_rwlock); /* lock for cmd_cbs */ +static roce5_netlink_cmd_excute_callback cmd_cbs[USRNL_ROCE_CMD_NUM]; + +static int roce5_netlink_register_cb(enum roce_usrnl_cmd cmd, + roce5_netlink_cmd_excute_callback cb) +{ + int ret = 0; + + if (cmd >= USRNL_ROCE_CMD_NUM) { + ROCE_ERR("Invalid command(%u)", cmd); + return -EINVAL; + } + + write_lock(&g_cmd_process_rwlock); + + if (cmd_cbs[cmd] != NULL) { + ROCE_ERR( + "Repeated command setting, the callback of the command(%u) already exists.", + cmd); + ret = -EINVAL; + goto out; + } + + cmd_cbs[cmd] = cb; + +out: + write_unlock(&g_cmd_process_rwlock); + return ret; +} + +static void roce5_netlink_unregister_cb(enum roce_usrnl_cmd cmd) +{ + if (cmd >= USRNL_ROCE_CMD_NUM) { + ROCE_ERR("Invalid command(%u)", cmd); + return; + } + + write_lock(&g_cmd_process_rwlock); + cmd_cbs[cmd] = NULL; + write_unlock(&g_cmd_process_rwlock); +} + +int roce5_netlink_register_cbs(roce5_netlink_cmd_process cmd_procs[], + u8 cmd_num) +{ + int ret, i; + + for (i = 0; i < cmd_num; ++i) { + ret = roce5_netlink_register_cb(cmd_procs[i].cmd, + cmd_procs[i].callback); + if (ret != 0) { + ROCE_ERR("Failed to set callback in netlink, ret(%d)", + ret); + goto err_out; + } + } + + return 0; + +err_out: + for (--i; i >= 0; --i) { + roce5_netlink_unregister_cb(cmd_procs[i].cmd); + } + return ret; +} + +static roce5_netlink_cmd_process g_cmd_process[] = { +#ifdef ROCE_BONDING_EN + { USRNL_ROCE_CMD_QUERY_QP_TX_PORT, roce5_genl_query_qp_tx_port }, + { USRNL_ROCE_CMD_SET_QP_TX_PORT, roce5_genl_set_qp_tx_port }, + { USRNL_ROCE_CMD_QUERY_QP_RX_PORT, roce5_genl_query_qp_rx_port }, + { USRNL_ROCE_CMD_QUERY_BOND_PORT_INFO, + roce5_genl_query_bond_port_info }, + { USRNL_ROCE_CMD_SET_QP_UDP_SRC_PORT, roce5_genl_set_qp_udp_src_port }, + { USRNL_ROCE_CMD_QUERY_QP_UDP_SRC_PORT, + roce5_genl_query_qp_udp_src_port }, +#endif + { USRNL_ROCE_CMD_QUERY_NEXT_WQE_IDX, roce5_genl_query_next_wqe_idx }, + { USRNL_ROCE_CMD_QUERY_QPC, roce5_genl_query_qpc }, + { USRNL_ROCE_CMD_QUERY_CQC, roce5_genl_query_cqc }, + { USRNL_ROCE_CMD_QP_HASH_VALUE_PROCESS, + roce5_genl_qp_hash_value_process }, + { USRNL_ROCE_CMD_ULD, + roce5_netlink_uld_process }, /* usr kernel of uld interface */ +}; + +static int roce5_netlink_init_cbs(void) +{ + int ret; + + ret = roce5_netlink_register_cbs( + g_cmd_process, + sizeof(g_cmd_process) / sizeof(roce5_netlink_cmd_process)); + if (ret != 0) { + ROCE_ERR("Failed to register netlink callbacks, ret(%d).", ret); + return ret; + } + +#ifdef ENABLE_HYPER_ROCE + ret = roce5_hyper_netlink_init(); + if (ret != 0) { + ROCE_ERR("Failed to init hyper roce in netlink, ret(%d).", ret); + return ret; + } +#endif + + return 0; +} + +#if defined(NLA_POLICY_HAVE_MIN_MAX) +static const struct nla_policy roce5_policy[USRNL_GENL_ATTR_MAX + 1] = { + [USRNL_GENL_ATTR] = { .type = NLA_BINARY, + .len = CMD_BUF_LEN_MAX, + .min = 0, + .max = CMD_BUF_LEN_MAX }, +}; +#else +static const struct nla_policy roce5_policy[USRNL_GENL_ATTR_MAX + 1] = { + [USRNL_GENL_ATTR] = { .type = NLA_BINARY, .len = CMD_BUF_LEN_MAX }, +}; +#endif + +static const struct genl_ops roce5_genl_ops[] = { { + .cmd = USRNL_DEV_DRV_CMD_EXECUTE, +#ifdef HAVE_GENL_OPS_FIELD_VALIDATE + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, +#endif + .flags = GENL_ADMIN_PERM, + .policy = roce5_policy, + .doit = roce5_netlink_cmd_dispatch, +} }; + +static struct genl_family roce5_genl_family = { + .hdrsize = 0, + .name = ROCE_GENL_FAMILY_NAME, + .version = 1, + .maxattr = USRNL_GENL_ATTR_MAX, + .netnsok = true, + .parallel_ops = true, + .ops = roce5_genl_ops, + .n_ops = ARRAY_SIZE(roce5_genl_ops), +}; + +static void roce5_drv_msg_hdr_push(struct sk_buff *skb, + usrnl_drv_msg_hdr_s *in_msg_hdr, + uint16_t out_msg_data_len) +{ + usrnl_drv_msg_hdr_s *out_msg_hdr = NULL; + + out_msg_hdr = skb_push(skb, ROCE_DRV_MSG_HDRLEN); + out_msg_hdr->cmd = in_msg_hdr->cmd; + out_msg_hdr->msg_len = ROCE_DRV_MSG_HDRLEN + out_msg_data_len; + out_msg_hdr->module = in_msg_hdr->module; + out_msg_hdr->src_func_idx = in_msg_hdr->src_func_idx; + out_msg_hdr->err_code = 0; +} + +static void roce5_nlahdr_push(struct sk_buff *skb, uint16_t type) +{ + struct nlattr *nlahdr = NULL; + uint16_t attrlen = (uint16_t)skb->len; + uint32_t padlen = (uint32_t)nla_padlen(attrlen); + + nlahdr = (struct nlattr *)skb_push(skb, NLA_HDRLEN); + nlahdr->nla_len = (uint16_t)nla_attr_size(attrlen); + nlahdr->nla_type = type; + + if ((skb->end - skb->tail) >= padlen) { + (void)skb_put(skb, padlen); + } +} + +static struct nlmsghdr *nlmsg_push(struct sk_buff *skb, uint32_t portid, + uint32_t seq, uint16_t type, uint32_t len, + uint16_t flags) +{ + struct nlmsghdr *nlh = NULL; + uint32_t size = (uint32_t)nlmsg_msg_size(len); + + nlh = (struct nlmsghdr *)skb_push(skb, NLMSG_ALIGN(size)); + if (nlh != NULL) { + nlh->nlmsg_type = type; + nlh->nlmsg_len = skb->len; + nlh->nlmsg_flags = flags; + nlh->nlmsg_pid = portid; + nlh->nlmsg_seq = seq; + if (!__builtin_constant_p(size) || + NLMSG_ALIGN(size) - size != 0) { + (void)memset_s((char *)nlmsg_data(nlh) + len, + NLMSG_ALIGN(size) - size, 0, + NLMSG_ALIGN(size) - size); + } + } + + return nlh; +} + +static inline struct nlmsghdr *roce5_nlmsg_push(struct sk_buff *skb, + uint32_t portid, uint32_t seq, + uint16_t type, uint32_t payload, + uint16_t flags) +{ + if (unlikely(skb_headroom(skb) < + (unsigned int)nlmsg_total_size(payload))) { + return NULL; + } + + return nlmsg_push(skb, portid, seq, type, payload, flags); +} + +static void *roce5_genlmsg_push(struct sk_buff *skb, uint32_t portid, + uint32_t seq, const struct genl_family *family, + uint16_t flags, uint8_t cmd) +{ + struct nlmsghdr *nlh = NULL; + struct genlmsghdr *hdr = NULL; + + nlh = roce5_nlmsg_push(skb, portid, seq, (int)family->id, + (int)(GENL_HDRLEN + family->hdrsize), flags); + if (nlh == NULL) { + return NULL; + } + + hdr = nlmsg_data(nlh); + hdr->cmd = cmd; + hdr->version = (uint8_t)family->version; + hdr->reserved = 0; + + return (void *)nlh; +} + +static inline roce5_netlink_cmd_excute_callback +roce5_netlink_get_callback(uint16_t cmd) +{ + roce5_netlink_cmd_excute_callback cb = NULL; + + if (cmd >= USRNL_ROCE_CMD_NUM) { + ROCE_ERR("Invalid command(%u).", cmd); + return NULL; + } + + read_lock(&g_cmd_process_rwlock); + cb = cmd_cbs[cmd]; + read_unlock(&g_cmd_process_rwlock); + + return cb; +} + +static int32_t +roce5_netlink_buf_handler(struct nlattr *cmd_attr, + roce5_netlink_cmd_excute_callback callback, + struct roce5_device *rdev, struct genl_info *nl_info, + struct sk_buff *reply) +{ + int32_t ret = 0; + uint16_t buf_in_len = 0; + uint16_t max_buf_out_len = 0; + uint16_t buf_out_len = 0; + usrnl_drv_msg_hdr_s *in_msg_hdr = NULL; + + in_msg_hdr = (usrnl_drv_msg_hdr_s *)nla_data(cmd_attr); + if (in_msg_hdr->msg_len < ROCE_DRV_MSG_HDRLEN) { + ROCE_DEV_ERR(rdev, "Invalid parameter: msg_len(%d).", + in_msg_hdr->msg_len); + return -EINVAL; + } + + buf_in_len = in_msg_hdr->msg_len - ROCE_DRV_MSG_HDRLEN; + max_buf_out_len = + NLMSG_DEFAULT_SIZE - + (NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN + ROCE_DRV_MSG_HDRLEN); + + if (buf_in_len == 0) { + ret = callback(rdev, NULL, 0, reply->data, max_buf_out_len, + &buf_out_len); + } else { + ret = callback(rdev, nla_data(cmd_attr) + ROCE_DRV_MSG_HDRLEN, + buf_in_len, reply->data, max_buf_out_len, + &buf_out_len); + } + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Do netlink %u cmd failed, errcode:%d", + nl_info->genlhdr->cmd, ret); + nlmsg_free(reply); + return -EIO; + } + + if (buf_out_len != 0) { + skb_put(reply, buf_out_len); + } + + roce5_drv_msg_hdr_push(reply, in_msg_hdr, buf_out_len); + + return 0; +} + +static int roce5_netlink_cmd_dispatch(struct sk_buff *skb, + struct genl_info *nl_info) +{ + int32_t ret = 0; + struct sk_buff *reply = NULL; + struct nlattr *cmd_attr = NULL; + roce5_drv_msg_hdr_s *in_msg_hdr = NULL; + roce5_netlink_cmd_excute_callback callback = NULL; + struct roce5_device *rdev = NULL; + + cmd_attr = nl_info->attrs[USRNL_GENL_ATTR]; + if (cmd_attr == NULL) { + ROCE_ERR("Do netlink cmd from portid:%u failed for NULL attr", + nl_info->snd_portid); + return -EINVAL; + } else if (nla_len(cmd_attr) < ROCE_DRV_MSG_HDRLEN) { + ROCE_ERR("Invalid parameter: nla_len(%d).", nla_len(cmd_attr)); + return -EINVAL; + } + + in_msg_hdr = (roce5_drv_msg_hdr_s *)nla_data(cmd_attr); + + rdev = roce5_get_rdev_by_card_func_id( + in_msg_hdr->card_id, in_msg_hdr->common_hdr.src_func_idx); + if (rdev == NULL) { + ROCE_ERR("Invalid param: src_func_idx: %u, card_id: %llu", + in_msg_hdr->common_hdr.src_func_idx, + in_msg_hdr->card_id); + return -EINVAL; + } + + callback = roce5_netlink_get_callback(in_msg_hdr->common_hdr.cmd); + if (callback == NULL) { + ROCE_DEV_ERR(rdev, "Unable to handle cmd: %u", + in_msg_hdr->common_hdr.cmd); + return -EINVAL; + } + + reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (reply == NULL) { + ROCE_DEV_ERR(rdev, "Alloc reply buf for netlink cmd failed"); + return -ENOMEM; + } + + skb_reserve(reply, NLMSG_HDRLEN + GENL_HDRLEN + NLA_HDRLEN + + ROCE_DRV_MSG_HDRLEN); + + ret = roce5_netlink_buf_handler(cmd_attr, callback, rdev, nl_info, + reply); + if (ret != 0) { + return ret; + } + + roce5_nlahdr_push(reply, USRNL_GENL_ATTR); + roce5_genlmsg_push(reply, nl_info->snd_portid, nl_info->snd_seq, + &roce5_genl_family, 0, nl_info->genlhdr->cmd); + + genlmsg_reply(reply, nl_info); + + return 0; +} + +int32_t roce5_netlink_init(void) +{ + int ret; + + ret = roce5_netlink_init_cbs(); + if (ret != 0) { + ROCE_ERR("Failed to init netlink callbacks, ret(%d).", ret); + return ret; + } + + ret = genl_register_family(&roce5_genl_family); + if (ret != 0) { + ROCE_ERR("Failed to register netlink, ret(%d).", ret); + return ret; + } + + ROCE_INFO("netlink init successfully."); + return 0; +} + +void roce5_netlink_deinit(void) +{ + genl_unregister_family(&roce5_genl_family); + ROCE_INFO("netlink deinit successfully."); +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h new file mode 100644 index 000000000..9c4cfff02 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink.h @@ -0,0 +1,53 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. + + +#ifndef ROCE5_NETLINK_H +#define ROCE5_NETLINK_H + +#include <linux/types.h> +#include "roce_drv_msg_intf.h" +#include "roce.h" + +#define ROCE_GENL_FAMILY_NAME "roce5_genl" +#define ROCE_DRV_MSG_HDRLEN (sizeof(roce5_drv_msg_hdr_s)) +#define CMD_BUF_LEN_MAX 2048 + +typedef int32_t (*roce5_netlink_cmd_excute_callback)(struct roce5_device *rdev, + void *in, uint16_t in_len, + void *out, + uint16_t out_len, + uint16_t *out_size); + +enum roce_usrnl_cmd { + USRNL_ROCE_CMD_QUERY_QP_TX_PORT, + USRNL_ROCE_CMD_SET_QP_TX_PORT, + USRNL_ROCE_CMD_QUERY_QP_RX_PORT, + USRNL_ROCE_CMD_QUERY_BOND_PORT_INFO, + USRNL_ROCE_CMD_SET_QP_UDP_SRC_PORT, + USRNL_ROCE_CMD_QUERY_QP_UDP_SRC_PORT, + USRNL_ROCE_CMD_QUERY_NEXT_WQE_IDX, + USRNL_ROCE_CMD_QUERY_QPC, + USRNL_ROCE_CMD_QUERY_CQC, + USRNL_ROCE_CMD_QP_HASH_VALUE_PROCESS, + USRNL_ROCE_CMD_ULD, /* uld use */ + + /* Command number for HyperRoCE */ + USRNL_ROCE_HYPER_ROCE_START = 16, + USRNL_ROCE_CMD_HYPER_QP_SET = USRNL_ROCE_HYPER_ROCE_START, + USRNL_ROCE_CMD_MODIFY_QP_EXTEND, + USRNL_ROCE_HYPER_ROCE_END = 31, + USRNL_ROCE_CMD_NUM +}; + +typedef struct { + enum roce_usrnl_cmd cmd; + roce5_netlink_cmd_excute_callback callback; +} roce5_netlink_cmd_process; + +int32_t roce5_netlink_init(void); +void roce5_netlink_deinit(void); +int roce5_netlink_register_cbs(roce5_netlink_cmd_process cmd_procs[], + u8 cmd_num); + +#endif /* ROCE5_NETLINK_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c new file mode 100644 index 000000000..8b7445ab0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.c @@ -0,0 +1,832 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/types.h> +#include <linux/semaphore.h> + +#include "hinic5_cqm.h" +#include "hinic5_mt.h" +#include "roce_cqm_cmd.h" +#include "roce_cmd.h" +#include "roce_com.h" +#include "roce_dfx.h" +#include "roce_main_extension.h" +#include "roce_mix.h" +#include "roce_npu_cmd.h" +#include "roce_npu_cmd_defs.h" +#include "roce_qp.h" +#include "roce_restore.h" +#include "roce_srq.h" +#include "roce_uld_inner.h" +#include "roce_uld_kernel_api.h" +#include "roce_netlink_process.h" +#ifdef ROCE_BONDING_EN +#include "roce_bond.h" +#endif + +static int32_t roce5_set_hash_value(struct roce5_device *rdev, uint32_t qpn, + uint32_t hash_type, uint32_t new_hash_value) +{ + int32_t ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + void *modify_hash_value = NULL; + size_t buf_size = 0; + + if (ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) != 0) { + buf_size = sizeof(roce_uni_cmd_modify_hash_s); + } + + ret = roce5_cqm_cmd_zalloc_inoutbuf(rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)buf_size, NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + modify_hash_value = cqm_cmd_inbuf->buf; + + if (ROCE5_SUPPORT_SET_HASH_VALUE_MSG_V2(rdev) != 0) { + roce_uni_cmd_modify_hash_s *cmd = + (roce_uni_cmd_modify_hash_s *)modify_hash_value; + cmd->com.index = cpu_to_be32(qpn); + cmd->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + cmd->info.type = cpu_to_be32(hash_type); + cmd->info.hash_value = cpu_to_be32(new_hash_value); + } + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_HASH_VALUE_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d), qpn(%u), type(%u), hash_value(%u), ret=%d", + rdev->glb_func_id, qpn, hash_type, new_hash_value, ret); + ret = -1; + } + + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int32_t roce5_get_hash_value(struct roce5_device *rdev, uint32_t qpn, + uint32_t hash_type, uint32_t *hash_value) +{ + int32_t ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_get_hash_s *get_hash_value = NULL; + roce_verbs_qp_hash_info_s *get_hash_value_out = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_get_hash_s), &cqm_cmd_outbuf, + (uint16_t)sizeof(roce_verbs_qp_hash_info_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_hash_value = (roce_uni_cmd_get_hash_s *)cqm_cmd_inbuf->buf; + get_hash_value->com.index = cpu_to_be32(qpn); + get_hash_value->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + get_hash_value->info.type = cpu_to_be32(hash_type); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_HASH_VALUE_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, + ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send ROCE_CMD_GET_HASH_VALUE_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto out; + } + + get_hash_value_out = (roce_verbs_qp_hash_info_s *)cqm_cmd_outbuf->buf; + *hash_value = be32_to_cpu(get_hash_value_out->hash_value); + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +#ifdef ROCE_BONDING_EN +int32_t roce5_genl_query_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + uint32_t slave_cnt = 0; + struct roce5_qp *rqp = NULL; + uint32_t func_tbl_value = 0; + struct roce5_qp_in *msg_in = NULL; + struct roce5_port_out *msg_out = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_port_out) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + *out_size = sizeof(struct roce5_port_out); + msg_out = (struct roce5_port_out *)out; + + if (!roce5_bond_is_active(rdev)) { + msg_out->port = 0; + return 0; + } + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + read_lock(&rdev->rwlock_func_tbl_bond); + func_tbl_value = rdev->func_tbl_bond; + slave_cnt = func_tbl_value & 0xfU; + if (slave_cnt != 0) { + msg_out->port = (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - + ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & + ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + } + read_unlock(&rdev->rwlock_func_tbl_bond); + + if (slave_cnt == 0) { + ROCE_ERR("slave_cnt is 0"); + ret = -EINVAL; + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int32_t roce5_set_new_tx_hash_value(struct roce5_device *rdev, + struct roce5_qp *rqp, + struct roce5_qp_port_in *msg_in) +{ + int32_t ret = 0; + uint32_t func_tbl_value = 0; + uint32_t slave_cnt = 0; + uint32_t target_hash_port = 0; + uint32_t old_hash_port = 0; + uint32_t new_tx_hash_value; + + read_lock(&rdev->rwlock_func_tbl_bond); + func_tbl_value = rdev->func_tbl_bond; + read_unlock(&rdev->rwlock_func_tbl_bond); + + slave_cnt = func_tbl_value & 0xfU; + if (slave_cnt == 0) { + ROCE_DEV_ERR(rdev, "slave_cnt is 0."); + return -EINVAL; + } + target_hash_port = msg_in->port & ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + old_hash_port = + (func_tbl_value >> (ROCE_BOND_FWD_ID_TBL_ALL_BITS - + ((rqp->tx_hash_value % slave_cnt + 1) * + ROCE_BOND_FWD_ID_TBL_PER_BITS))) & + ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + + if (target_hash_port == old_hash_port) { + return 0; + } + + new_tx_hash_value = + rqp->tx_hash_value + + (((slave_cnt - old_hash_port) + target_hash_port) % slave_cnt); + ret = roce5_set_hash_value(rdev, rqp->qpn, ROCE_VERBS_QP_HASH_TYPE_BOND, + new_tx_hash_value); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) Bond hash value, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + return ret; + } + + rqp->tx_hash_value = new_tx_hash_value; + + return 0; +} + +int32_t roce5_genl_set_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_qp_port_in *msg_in = NULL; + struct roce5_qp *rqp = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_qp_port_in) || out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_port_in *)in; + + if (!roce5_bond_is_active(rdev)) { + return 0; + } + + /* check qp exist */ + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + ret = roce5_set_new_tx_hash_value(rdev, rqp, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set new tx hash value, func_id(%d)", + rdev->glb_func_id); + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_uni_cmd_get_qp_rx_port(struct roce5_device *rdev, uint32_t qpn, + uint32_t *rx_port) +{ + int32_t ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_get_port_info_s *get_rx_port_inbuf = NULL; + roce_uni_cmd_get_port_info_outbuf_s *get_rx_port_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_s), &cqm_cmd_outbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_rx_port_inbuf = (roce_uni_cmd_get_port_info_s *)cqm_cmd_inbuf->buf; + get_rx_port_inbuf->com.index = cpu_to_be32(qpn); + get_rx_port_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_RX_PORT_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, + ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send ROCE_CMD_GET_RX_PORT_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto out; + } + get_rx_port_outbuf = + (roce_uni_cmd_get_port_info_outbuf_s *)cqm_cmd_outbuf->buf; + *rx_port = be32_to_cpu(get_rx_port_outbuf->value.rx_port); +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static int32_t roce5_get_qp_rx_port(struct roce5_device *rdev, uint32_t qpn, + uint32_t *rx_port) +{ + if (ROCE5_SUPPORT_GET_QP_RX_PORT_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_get_qp_rx_port(rdev, qpn, rx_port); + } + return -EINVAL; +} + +int32_t roce5_genl_query_qp_rx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + uint32_t rx_port = 0; + struct roce5_qp *rqp = NULL; + struct roce5_qp_in *msg_in = NULL; + struct roce5_port_out *msg_out = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_port_out) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + if (rqp->qp_type != IB_QPT_RC) { + ROCE_DEV_ERR(rdev, "not support qp type(%d), only support RC.", + rqp->qp_type); + ret = -EINVAL; + goto err_out; + } + + *out_size = sizeof(struct roce5_port_out); + msg_out = (struct roce5_port_out *)out; + + if (!roce5_bond_is_active(rdev)) { + msg_out->port = 0; + ret = 0; + goto err_out; + } + + ret = roce5_get_qp_rx_port(rdev, msg_in->qpn, &rx_port); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get QP(0x%06x) rx port", + msg_in->qpn); + goto err_out; + } + + msg_out->port = rx_port; + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +int32_t roce5_genl_query_bond_port_info(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + uint32_t func_tbl_value = 0; + struct roce5_bond_device *bond_dev = NULL; + struct roce5_bond_port_info_out *msg_out = NULL; + uint32_t alive_port_num; + uint32_t i; + + if (out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_bond_port_info_out) || + out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, out:%d, out_len:%u, out_size:%d", + !!out, out_len, !!out_size); + return -EINVAL; + } + + *out_size = sizeof(struct roce5_bond_port_info_out); + msg_out = (struct roce5_bond_port_info_out *)out; + + if (!roce5_bond_is_active(rdev)) { + return 0; + } + + bond_dev = rdev->bond_dev; + + /* Read func_tbl_bond and derive alive port info under lock to prevent data race */ + read_lock(&rdev->rwlock_func_tbl_bond); + func_tbl_value = rdev->func_tbl_bond; + alive_port_num = func_tbl_value & 0xfU; + for (i = 0; i < alive_port_num; i++) { + msg_out->alive_port[i] = + (func_tbl_value >> + (ROCE_BOND_FWD_ID_TBL_ALL_BITS - + ((i + 1) * ROCE_BOND_FWD_ID_TBL_PER_BITS))) & + ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK; + } + read_unlock(&rdev->rwlock_func_tbl_bond); + msg_out->alive_port_num = + (int32_t)alive_port_num; /* netlink protocol field */ + + /* Read slave info under slave_lock (separate from rwlock to avoid sleeping in atomic context) */ + msg_out->original_port_num = + (int32_t)bond_dev->slave_cnt; /* netlink protocol field */ + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + msg_out->original_port[i] = bond_dev->slaves[i].func_id; + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +static int32_t roce5_uni_cmd_modify_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, + uint32_t new_udp_src_port) +{ + int32_t ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_set_udp_src_port_s *modify_udp_src_port = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_set_udp_src_port_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + modify_udp_src_port = + (roce_uni_cmd_set_udp_src_port_s *)cqm_cmd_inbuf->buf; + modify_udp_src_port->com.index = cpu_to_be32(qpn); + modify_udp_src_port->com.dw0.bs.cmd_bitmask = + cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + modify_udp_src_port->info.udp_src_port = cpu_to_be32(new_udp_src_port); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_MODIFY_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, NULL, ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send MODIFY_HASH_VALUE_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + } + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int32_t roce5_modify_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, + uint32_t new_udp_src_port) +{ + if (ROCE5_SUPPORT_MODIFY_UDP_SRC_PORT_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_modify_udp_src_port(rdev, qpn, + new_udp_src_port); + } + return -EINVAL; +} + +int32_t roce5_genl_set_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_qp *rqp = NULL; + struct roce5_qp_udp_src_port_in *msg_in = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_qp_udp_src_port_in) || + out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_udp_src_port_in *)in; + + /* check qp exist */ + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp"); + return -EINVAL; + } + + ret = roce5_modify_udp_src_port(rdev, rqp->qpn, msg_in->udp_src_port); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to modify QP(0x%06x) hash value, func_id(%d)", + rqp->qpn, rdev->glb_func_id); + ret = -EIO; + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int32_t roce5_uni_cmd_get_qp_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, + uint32_t *udp_src_port) +{ + int32_t ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_get_port_info_s *get_udp_src_port = NULL; + roce_uni_cmd_get_port_info_outbuf_s *get_udp_src_port_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s), + &cqm_cmd_outbuf, + (uint16_t)sizeof(roce_uni_cmd_get_port_info_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + get_udp_src_port = (roce_uni_cmd_get_port_info_s *)cqm_cmd_inbuf->buf; + get_udp_src_port->com.index = cpu_to_be32(qpn); + get_udp_src_port->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_QP_BITMASK)); + + ret = roce5_send_qp_lb_cmd(qpn, rdev, ROCE_CMD_GET_UDP_SRC_PORT_QP, + cqm_cmd_inbuf, cqm_cmd_outbuf, + ROCE_CMD_TIME_CLASS_A); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send ROCE_CMD_GET_UDP_SRC_PORT_QP command, func_id(%d)", + rdev->glb_func_id); + ret = -1; + goto out; + } + + get_udp_src_port_outbuf = + (roce_uni_cmd_get_port_info_outbuf_s *)cqm_cmd_outbuf->buf; + *udp_src_port = + be32_to_cpu(get_udp_src_port_outbuf->value.udp_src_port); +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int32_t roce5_get_qp_udp_src_port(struct roce5_device *rdev, + uint32_t qpn, uint32_t *udp_src_port) +{ + if (ROCE5_SUPPORT_GET_QP_UDP_SRC_PORT_MSG_V2(rdev) != 0) { + return roce5_uni_cmd_get_qp_udp_src_port(rdev, qpn, + udp_src_port); + } + return -EINVAL; +} + +int32_t roce5_genl_query_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + uint32_t udp_src_port = 0; + struct roce5_qp_in *msg_in = NULL; + struct roce5_port_out *msg_out = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_port_out) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + + ret = roce5_get_qp_udp_src_port(rdev, msg_in->qpn, &udp_src_port); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to get QP(0x%06x) udp src port", + msg_in->qpn); + return ret; + } + + *out_size = sizeof(struct roce5_port_out); + msg_out = (struct roce5_port_out *)out; + + msg_out->port = udp_src_port; + + return 0; +} +#endif + +int32_t roce5_genl_query_next_wqe_idx(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size) +{ + int32_t ret = 0; + int32_t next_wqe_idx = 0; + uint32_t container_mode = 0; + uint32_t container_size = 0; + struct roce5_next_wqe_idx_in *msg_in = NULL; + struct roce5_next_wqe_idx_out *msg_out = NULL; + roce_verbs_srq_attr_s *srq_attr = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_next_wqe_idx_in) || out == NULL || + out_len == 0 || out_len < sizeof(struct roce5_next_wqe_idx_out) || + out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + srq_attr = kmalloc(sizeof(roce_verbs_srq_attr_s), GFP_KERNEL); + if (srq_attr == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to allocate memory for dfx_query_outbuf"); + return -ENOMEM; + } + + msg_in = (struct roce5_next_wqe_idx_in *)in; + msg_out = (struct roce5_next_wqe_idx_out *)out; + ret = roce5_dfx_cmd_query_srq(rdev, msg_in->srqn, srq_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to do roce5_dfx_cmd_query_srq, ret(%d)", + ret); + ret = -EINVAL; + goto free_outbuf; + } + + srq_attr->dw2.value = be32_to_cpu(srq_attr->dw2.value); + srq_attr->dw3.value = be32_to_cpu(srq_attr->dw3.value); + srq_attr->cont.dw2.value = be32_to_cpu(srq_attr->cont.dw2.value); + if (srq_attr->dw3.bs.container == 0) { + next_wqe_idx = (int32_t)(srq_attr->dw2.bs.next_wqe_idx); + } else { + container_mode = MAX_SUPPORT_CONTAINER_MODE - + srq_attr->cont.dw2.bs.cont_size; + container_size = roce5_get_container_sz(container_mode); + next_wqe_idx = (int32_t)(container_size * + srq_attr->cont.dw2.bs.head_idx); + } + + msg_out->next_wqe_idx = next_wqe_idx; + *out_size = sizeof(struct roce5_next_wqe_idx_out); + ret = 0; + +free_outbuf: + kfree(srq_attr); + return ret; +} + +int32_t roce5_genl_query_qpc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_qp_in *msg_in = NULL; + roce_verbs_qp_attr_s *verbs_qp_attr = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_qp_in) || + out == NULL || out_len == 0 || + out_len < sizeof(roce_verbs_qp_attr_s) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_qp_in *)in; + verbs_qp_attr = (roce_verbs_qp_attr_s *)out; + + ret = roce5_qp_query(rdev, msg_in->qpn, (u32 *)(void *)verbs_qp_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query qpc, ret(%0x)", ret); + return ret; + } + + *out_size = sizeof(roce_verbs_qp_attr_s); + return 0; +} + +int32_t roce5_genl_query_cqc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_cq_in *msg_in = NULL; + roce_verbs_cq_attr_s *verbs_cq_attr = NULL; + + if (in == NULL || in_len == 0 || in_len < sizeof(struct roce5_cq_in) || + out == NULL || out_len == 0 || + out_len < sizeof(roce_verbs_cq_attr_s) || out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid param, in:%d, in_len:%u, out:%d, out_len:%u, out_size:%d", + !!in, in_len, !!out, out_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_cq_in *)in; + verbs_cq_attr = (roce_verbs_cq_attr_s *)out; + + ret = roce5_dfx_query_cq(rdev, msg_in->cqn, + (u32 *)(void *)verbs_cq_attr, + sizeof(roce_verbs_cq_attr_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query cqc, ret(%0x)", ret); + return ret; + } + + *out_size = sizeof(roce_verbs_cq_attr_s); + return 0; +} + +int32_t roce5_genl_qp_hash_value_process(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size) +{ + uint32_t hash_value = 0; + struct roce5_qp_hash_value_in *msg_in = NULL; + struct roce5_qp_hash_value_out *msg_out = NULL; + int32_t ret; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_qp_hash_value_in)) { + ROCE_DEV_ERR(rdev, "Invalid input param, in:%d, in_len:%u", + !!in, in_len); + return -EINVAL; + } + + msg_in = (struct roce5_qp_hash_value_in *)in; + + if (msg_in->type == ROCE_QP_HASH_VALUE_GET) { + if (out == NULL || out_len == 0 || + out_len < sizeof(struct roce5_qp_hash_value_out) || + out_size == NULL) { + ROCE_DEV_ERR( + rdev, + "Invalid output param, out:%d, out_len:%u, out_size:%d", + !!out, out_len, !!out_size); + return -EINVAL; + } + ret = roce5_get_hash_value(rdev, msg_in->qpn, + ROCE_VERBS_QP_HASH_TYPE_UBC, + &hash_value); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to get QP(0x%06x) hash value, func_id(%d)", + msg_in->qpn, rdev->glb_func_id); + return ret; + } + + *out_size = sizeof(struct roce5_qp_hash_value_out); + msg_out = (struct roce5_qp_hash_value_out *)out; + msg_out->hash_value = hash_value; + } else { + ret = roce5_set_hash_value(rdev, msg_in->qpn, + ROCE_VERBS_QP_HASH_TYPE_UBC, + msg_in->hash_value); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set QP(0x%06x) hash value, func_id(%d)", + msg_in->qpn, rdev->glb_func_id); + } + } + + return ret; +} + +int32_t roce5_netlink_uld_process(struct roce5_device *rdev, void *buf_in, + u16 buf_in_len, void *buf_out, + u16 buf_out_len, u16 *buf_out_size) +{ + int32_t ret = 0; + roce_user_handle_t handle; + + handle.rdev = rdev; + handle.magic = ROCE_ULD_HANDLE_MAGIC_NUM; + + ret = roce5_uld_user2kernel_msg_handle((void *)&handle, buf_in, + buf_in_len, buf_out, buf_out_len, + buf_out_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "ULD process err, func_id(%d)", + rdev->glb_func_id); + } + return ret; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h new file mode 100644 index 000000000..d09bcf733 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/netlink/roce_netlink_process.h @@ -0,0 +1,133 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_NETLINK_PROCESS_H +#define ROCE_NETLINK_PROCESS_H + +#include "roce.h" +#include "roce_compat.h" + +typedef enum { + ROCE_QP_HASH_VALUE_SET, + ROCE_QP_HASH_VALUE_GET, +} ROCE_QP_HASH_VALUE_PROC_TYPE; + +#ifdef ROCE_BONDING_EN +/* and above next data structure is adaptation V100 use */ +struct roce5_get_rx_port { + __be32 rx_port; +}; + +struct roce5_get_udp_src_port { + __be32 udp_src_port; +}; + +typedef struct tag_roce_get_qp_rx_port { + roce_verbs_cmd_com_s com; +} roce_get_qp_rx_port_s; + +typedef struct tag_roce_modify_hash_value { + roce_verbs_cmd_com_s com; + u32 hash_value; +} roce_modify_hash_value_s; + +typedef struct tag_roce_modify_udp_src_port { + roce_verbs_cmd_com_s com; + u32 udp_src_port; +} roce_modify_udp_src_port_s; + +typedef struct roce_get_qp_udp_src_port { + roce_verbs_cmd_com_s com; +} roce_get_qp_udp_src_port_s; + +struct roce5_qp_port_in { + uint32_t qpn; + uint32_t port; +}; + +struct roce5_port_out { + uint32_t port; +}; + +struct roce5_bond_port_info_out { + int32_t original_port_num; + uint8_t original_port[8]; + int32_t rsvd1; + + int32_t alive_port_num; + uint8_t alive_port[8]; + int32_t rsvd2; +}; + +struct roce5_qp_udp_src_port_in { + uint32_t qpn; + uint32_t udp_src_port; +}; + +#endif + +struct roce5_qp_in { + uint32_t qpn; +}; + +struct roce5_cq_in { + uint32_t cqn; +}; + +struct roce5_next_wqe_idx_in { + uint32_t srqn; +}; + +struct roce5_next_wqe_idx_out { + int32_t next_wqe_idx; +}; + +struct roce5_qp_hash_value_in { + uint32_t type; + uint32_t qpn; + uint32_t hash_value; +}; + +struct roce5_qp_hash_value_out { + uint32_t hash_value; +}; + +#ifdef ROCE_BONDING_EN + +int32_t roce5_genl_query_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_set_qp_tx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size); +int32_t roce5_genl_query_qp_rx_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_query_bond_port_info(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_set_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); +int32_t roce5_genl_query_qp_udp_src_port(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size); + +#endif + +int32_t roce5_genl_query_next_wqe_idx(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size); +int32_t roce5_genl_query_qpc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size); +int32_t roce5_genl_query_cqc(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, uint16_t out_len, + uint16_t *out_size); +int32_t roce5_genl_qp_hash_value_process(struct roce5_device *rdev, void *in, + u16 in_len, void *out, u16 out_len, + u16 *out_size); +int32_t roce5_netlink_uld_process(struct roce5_device *rdev, void *buf_in, + u16 buf_in_len, void *buf_out, + u16 buf_out_len, u16 *buf_out_size); +#endif /* ROCE_NETLINK_PROCESS_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c new file mode 100644 index 000000000..69549fbe3 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.c @@ -0,0 +1,74 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "securec.h" + +#include "roce_cqm_cmd.h" + +void roce5_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf) +{ + if (cqm_cmd_outbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_outbuf); + } + + if (cqm_cmd_inbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_inbuf); + } +} + +int roce5_cqm_cmd_zalloc_inoutbuf(void *ex_handle, + cqm_cmd_buf_s **cqm_cmd_inbuf, u16 inbuf_size, + cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size) +{ + int ret; + + if (cqm_cmd_inbuf != NULL) { + *cqm_cmd_inbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_inbuf == NULL) { + ret = -ENOMEM; + goto inbuf_err; + } + (*cqm_cmd_inbuf)->size = inbuf_size; + + ret = memset_s((*cqm_cmd_inbuf)->buf, inbuf_size, 0, + inbuf_size); + if (ret != 0) { + ret = -EINVAL; + goto inbuf_err; + } + } + + if (cqm_cmd_outbuf != NULL) { + *cqm_cmd_outbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_outbuf == NULL) { + ret = -ENOMEM; + goto outbuf_err; + } + (*cqm_cmd_outbuf)->size = outbuf_size; + + ret = memset_s((*cqm_cmd_outbuf)->buf, outbuf_size, 0, + outbuf_size); + if (ret != 0) { + ret = -EINVAL; + goto outbuf_err; + } + } + + return 0; +outbuf_err: + if (cqm_cmd_outbuf != NULL) { + roce5_cqm5_cmd_free_inoutbuf(ex_handle, NULL, *cqm_cmd_outbuf); + *cqm_cmd_outbuf = NULL; + } +inbuf_err: + if (cqm_cmd_inbuf != NULL) { + roce5_cqm5_cmd_free_inoutbuf(ex_handle, *cqm_cmd_inbuf, NULL); + *cqm_cmd_inbuf = NULL; + } + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h new file mode 100644 index 000000000..0130fa320 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sdk_api/roce_cqm_cmd.h @@ -0,0 +1,21 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_CQM_CMD_H +#define ROCE_CQM_CMD_H +#ifdef ROCE_KNL_DT +#include <linux/compiler_types.h> +#endif +#include "hinic5_cqm.h" + +#include "roce.h" + +void roce5_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf); +int roce5_cqm_cmd_zalloc_inoutbuf(void *ex_handle, + cqm_cmd_buf_s **cqm_cmd_inbuf, u16 inbuf_size, + cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size); + +#endif //ROCE_CQM_CMD_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c new file mode 100644 index 000000000..3000c6b97 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.c @@ -0,0 +1,240 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + +#include "roce.h" +#include "roce_sysfs.h" +#include "roce_cc_format.h" + +#include "roce_cmd.h" +#include "roce_multipath_sysfs.h" + +#define MAX_STRING_FORMAT_LEN 64 +#define to_roce5_ecn_multipath_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_multipath_ctx, ecn_multipath_root) + +static void roce_ecn_multipath_sysfs_release(struct kobject *kobj) +{ +} + +static ssize_t roce5_show_slow_path_psn_threshold(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + (u32)ecn_multipath_ctx->slow_path_psn_threshold); +} + +static ssize_t roce5_store_slow_path_psn_threshold(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, + size_t count) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + u32 old_slow_path_psn_threshold = 0; + int slow_path_psn_threshold = 0; + int ret; + char remaining; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + ret = sscanf_s(buf, "%d %c", &slow_path_psn_threshold, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, slow_path_psn_threshold should be unsigned int\n"); + return -EIO; + } + + if (slow_path_psn_threshold > + ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX) { + ROCE_ERR( + "Invalid input, slow_path_psn_threshold(%d) over max slow_path_psn_threshold(%u), slow_path_psn_threshold", + slow_path_psn_threshold, + (u32)ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX); + return -EIO; + } + + if (ecn_multipath_ctx->slow_path_psn_threshold != + (u32)slow_path_psn_threshold) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_slow_path_psn_threshold = + ecn_multipath_ctx->slow_path_psn_threshold; + ecn_multipath_ctx->slow_path_psn_threshold = + (u32)slow_path_psn_threshold; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update slow_path_psn_threshold(%d), ret(%d).", + slow_path_psn_threshold, ret); + ecn_multipath_ctx->slow_path_psn_threshold = + old_slow_path_psn_threshold; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce5_show_rtt_req_event_mode(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_multipath_ctx->rtt_req_event_mode); +} + +static ssize_t roce5_store_rtt_req_event_mode(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 old_rtt_req_event_mode = 0; + u32 rtt_req_event_mode = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx = + to_roce5_ecn_multipath_ctx(kobj); + char remaining; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + ret = sscanf_s(buf, "%u %c", &rtt_req_event_mode, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_req_event_mode should be unsigned int ret(%d), rtt_req_event_mode(%u)\n", + ret, rtt_req_event_mode); + return -EIO; + } + + if ((rtt_req_event_mode & ROCE_MULTIPATH_RTT_EN_MASK) != 0) { + ROCE_ERR( + "Invalid input, rtt_req_event_mode (%u)only 0 and 1 supported\n", + rtt_req_event_mode); + return -EIO; + } + + if (ecn_multipath_ctx->rtt_req_event_mode != (u32)rtt_req_event_mode) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rtt_req_event_mode = ecn_multipath_ctx->rtt_req_event_mode; + ecn_multipath_ctx->rtt_req_event_mode = (u32)rtt_req_event_mode; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update rtt_req_event_mode(%d), ret(%d).", + rtt_req_event_mode, ret); + ecn_multipath_ctx->rtt_req_event_mode = + old_rtt_req_event_mode; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_ATTR_RW(slow_path_psn_threshold, roce5_show_slow_path_psn_threshold, + roce5_store_slow_path_psn_threshold); +ROCE_ATTR_RW(rtt_req_event_mode, roce5_show_rtt_req_event_mode, + roce5_store_rtt_req_event_mode); + +static struct attribute *ecn_multipath_ctx_attrs[] = { + ROCE_ATTR_PTR(rtt_req_event_mode), + ROCE_ATTR_PTR(slow_path_psn_threshold), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_multipath_ctx_attr_grp = { + .attrs = ecn_multipath_ctx_attrs, +}; + +static const struct attribute_group *ecn_multipath_ctx_groups[] = { + &ecn_multipath_ctx_attr_grp, + NULL, +}; +#endif + +static struct kobj_type roce_ecn_multipath_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_multipath_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_multipath_ctx_groups, +#else + .default_attrs = ecn_multipath_ctx_attrs, +#endif +}; + +int roce5_init_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx) +{ + int ret = 0; + + if (rdev->device_type != ROCE_DEV_TYPE_HI1825_V100) { + return 0; + } + if (ecn_multipath_ctx == NULL) { + pr_err_ratelimited( + "[ROCE, ERR] %s: ecn_multipath_ctx is null\n", + __func__); + return -EINVAL; + } + + ecn_multipath_ctx->rtt_req_event_mode = + ROCE_MULTIPATH_DEFAULT_RTT_EVENT_MODE; + ecn_multipath_ctx->slow_path_psn_threshold = + ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD; + + ret = kobject_init_and_add(&ecn_multipath_ctx->ecn_multipath_root, + &roce_ecn_multipath_ktype, kobj, + "roce5_multipath"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_ultipath.(ret:%d)\n", + ret); + kobject_put(&ecn_multipath_ctx->ecn_multipath_root); + return ret; + } + + return 0; +} + +void roce5_remove_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx) +{ + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + if (ecn_multipath_ctx == NULL) { + ROCE_ERR("ecn_multipath_ctx is null\n"); + return; + } + + kobject_put(&ecn_multipath_ctx->ecn_multipath_root); + } + + return; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h new file mode 100644 index 000000000..2a26de8e2 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_multipath_sysfs.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifndef ROCE_MULTIPATH_SYSFS_H +#define ROCE_MULTIPATH_SYSFS_H + +#include <linux/types.h> +#include <linux/stat.h> +#include <linux/kobject.h> + +struct roce5_device; + +#define ROCE_MULTIPATH_DEFAULT_RTT_EVENT_MODE 0 +#define ROCE_MULTIPATH_RTT_EN_MASK 0xfffffffe +#define ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD 1 +#define ROCE_MULTIPATH_DEFAULT_SLOW_PATH_PSN_THRESHOLD_MAX 255 + +struct roce5_ecn_multipath_ctx { + struct kobject ecn_multipath_root; + u32 slow_path_psn_threshold; + u32 rtt_req_event_mode; +}; +int roce5_init_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx); +void roce5_remove_ecn_multipath_sysfs( + struct roce5_device *rdev, struct kobject *kobj, + struct roce5_ecn_multipath_ctx *ecn_multipath_ctx); +#endif /* ROCE_MULTIPATH_SYSFS_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c new file mode 100644 index 000000000..869a97571 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.c @@ -0,0 +1,2157 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/stat.h> + +#include "roce.h" +#include "ccp_algo_format.h" +#include "hinic5_hw.h" +#include "roce_cc_format.h" +#include "roce_cmd.h" +#include "roce_com.h" +#include "roce_dfx.h" +#include "roce_infra.h" +#include "roce_kernel_compat_gen.h" +#include "roce_mix.h" +#include "roce_mpu_cmd.h" +#include "roce_multipath_sysfs.h" +#include "roce_npu_cmd_srq_defs.h" +#include "roce_qp.h" +#include "roce_ucc_sysfs.h" +#include "securec.h" +#include "roce_sysfs.h" + +#define MAX_STRING_FORMAT_LEN 64 + +/* generic macro template: generate show/store function, used for rp_ctx parameters (with range check) */ +#define ROCE5_PARAM_SHOW_STORE_RP(_name, _member, _min_macro, _max_macro) \ + static ssize_t roce5_show_##_name( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct roce5_ecn_rp_ctx *ecn_rp_ctx = \ + to_roce5_ecn_rp_ctx(kobj); \ + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \ + (int)ecn_rp_ctx->_member); \ + } \ + static ssize_t roce5_store_##_name(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + int ret, _val; \ + u32 _old; \ + struct roce5_ecn_ctx *ecn_ctx = \ + to_roce5_ecn_ctx(kobj->parent); \ + struct roce5_ecn_rp_ctx *ecn_rp_ctx = \ + to_roce5_ecn_rp_ctx(kobj); \ + struct roce5_device *rdev = \ + container_of(ecn_ctx, struct roce5_device, ecn_ctx); \ + ret = sscanf_s(buf, "%d", &_val); \ + if (ret != 1) \ + return -EIO; \ + if (ROCE5_PARAM_CHECK_BOUND( \ + _val, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._min_macro, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._max_macro)) \ + return -EIO; \ + if (ecn_rp_ctx->_member != (u32)_val) { \ + mutex_lock(&ecn_ctx->ecn_mutex); \ + _old = ecn_rp_ctx->_member; \ + ecn_rp_ctx->_member = (u32)_val; \ + ret = roce5_update_ecn_param(ecn_ctx); \ + if (ret != 0) { \ + ecn_rp_ctx->_member = _old; \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + return -EIO; \ + } \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + } \ + return (ssize_t)count; \ + } + +/* generic macro template: generate show/store function, used for np_ctx parameters (with range check) */ +#define ROCE5_PARAM_SHOW_STORE_NP(_name, _member, _min_macro, _max_macro) \ + static ssize_t roce5_show_##_name( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct roce5_ecn_np_ctx *ecn_np_ctx = \ + to_roce5_ecn_np_ctx(kobj); \ + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \ + (int)ecn_np_ctx->_member); \ + } \ + static ssize_t roce5_store_##_name(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + int ret, _val; \ + u32 _old; \ + struct roce5_ecn_ctx *ecn_ctx = \ + to_roce5_ecn_ctx(kobj->parent); \ + struct roce5_ecn_np_ctx *ecn_np_ctx = \ + to_roce5_ecn_np_ctx(kobj); \ + struct roce5_device *rdev = \ + container_of(ecn_ctx, struct roce5_device, ecn_ctx); \ + ret = sscanf_s(buf, "%d", &_val); \ + if (ret != 1) \ + return -EIO; \ + if (ROCE5_PARAM_CHECK_BOUND( \ + _val, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._min_macro, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._max_macro)) \ + return -EIO; \ + if (ecn_np_ctx->_member != (u32)_val) { \ + mutex_lock(&ecn_ctx->ecn_mutex); \ + _old = ecn_np_ctx->_member; \ + ecn_np_ctx->_member = (u32)_val; \ + ret = roce5_update_ecn_param(ecn_ctx); \ + if (ret != 0) { \ + ecn_np_ctx->_member = _old; \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + return -EIO; \ + } \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + } \ + return (ssize_t)count; \ + } + +/* generic macro template: generate show/store function, used for ldcp_ctx parameters (with range check) */ +#define ROCE5_PARAM_SHOW_STORE_LDCP(_name, _member, _min_macro, _max_macro) \ + static ssize_t roce5_show_##_name( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct roce5_ecn_ldcp_ctx *ecn_ldcp_ctx = \ + to_roce5_ecn_ldcp_ctx(kobj); \ + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", \ + (int)ecn_ldcp_ctx->_member); \ + } \ + static ssize_t roce5_store_##_name(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + int ret, _val; \ + u32 _old; \ + struct roce5_ecn_ctx *ecn_ctx = \ + to_roce5_ecn_ctx(kobj->parent); \ + struct roce5_ecn_ldcp_ctx *ecn_ldcp_ctx = \ + to_roce5_ecn_ldcp_ctx(kobj); \ + struct roce5_device *rdev = \ + container_of(ecn_ctx, struct roce5_device, ecn_ctx); \ + ret = sscanf_s(buf, "%d", &_val); \ + if (ret != 1) { \ + ROCE_ERR("Failed to get " #_member ", ret(%d).", ret); \ + return -EIO; \ + } \ + if (ROCE5_PARAM_CHECK_BOUND( \ + _val, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._min_macro, \ + g_roce_cc_ecn_param[rdev->cc_param_version] \ + ._max_macro)) { \ + ROCE_ERR(#_member "(%u) param is out bound.", _val); \ + return -EIO; \ + } \ + if (ecn_ldcp_ctx->_member != (u32)_val) { \ + mutex_lock(&ecn_ctx->ecn_mutex); \ + _old = ecn_ldcp_ctx->_member; \ + ecn_ldcp_ctx->_member = (u32)_val; \ + ret = roce5_update_ecn_param(ecn_ctx); \ + if (ret != 0) { \ + ecn_ldcp_ctx->_member = _old; \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + return -EIO; \ + } \ + mutex_unlock(&ecn_ctx->ecn_mutex); \ + } \ + return (ssize_t)count; \ + } + +const struct roce_cc_ecn_param g_roce_cc_ecn_param[] = { + { 0 }, // invalid + { .default_ip_enable_en = 1, + .default_bypass = 0, + .default_min_cnp_period = 16, + .default_alpha_dec_period = 160, + .default_rate_dec_period = 32, + .default_rate_inc_period = 480, + .default_factor_gita = 7, + .default_initial_alpha = 1023, + .default_rate_inc_ai = 2, + .default_rate_inc_hai = 8, + .default_rate_first_set = 1024, + .default_rate_target_clamp = 1, + .default_cnp_prio_enable = 0, + .default_cnp_prio = 0, + .default_token_period = 16, + .default_min_rate = 1, + .default_ecn_cnp_enable = 1, + .default_flow_ctrl_unit = 0, + .default_rtt_rsp_prio = 0, + .default_rtt_rsp_prio_enable = 0, + .default_rtt_version = 0, + .max_rtt_version = 0, + .min_bypass = 0, + .max_bypass = 0, + .min_rate_inc_hai = 1, + .max_rate_inc_hai = 255, + .min_rate_inc_ai = 1, + .max_rate_inc_ai = 63, + .min_initial_alpha = 1, + .max_initial_alpha = 1023, + .min_rate_inc_period = 1, + .max_rate_inc_period = 1023, + .min_alpha_dec_period = 1, + .max_alpha_dec_period = 1023, + .min_token_period = 4, + .max_token_period = 255, + .min_flow_min_rate = 1, + .max_flow_min_rate = 63, + .min_rate_dec_period = 1, + .max_rate_dec_period = 255, + .min_min_cnp_period = 1, + .max_min_cnp_period = 255, + .min_factor_gita = 1, + .max_factor_gita = 15, + .min_rate_first_set = 128, + .max_rate_first_set = 8191, + .default_ldcp_wnd_min = 5, + .default_ldcp_wnd_default = 1024, + .default_ldcp_alpha = 2, + .default_ldcp_beta = 12, + .default_ldcp_gamma = 9, + .default_ldcp_eta = 3, + .min_ldcp_wnd_min = 1, + .max_ldcp_wnd_min = 0x8, + .min_ldcp_wnd_default = 1024, + .max_ldcp_wnd_default = 32768, + .min_ldcp_alpha = 0, + .max_ldcp_alpha = 0xf, + .min_ldcp_beta = 0, + .max_ldcp_beta = 0xf, + .min_ldcp_gamma = 0, + .max_ldcp_gamma = 0xf, + .min_ldcp_eta = 0, + .max_ldcp_eta = 0xf }, // v1 + { .default_ip_enable_en = 1, + .default_bypass = 0, + .default_min_cnp_period = 16, + .default_alpha_dec_period = 160, + .default_rate_dec_period = 48, + .default_rate_inc_period = 480, + .default_factor_gita = 7, + .default_initial_alpha = 1023, + .default_rate_inc_ai = 2, + .default_rate_inc_hai = 12, + .default_rate_first_set = 1024, + .default_rate_target_clamp = 1, + .default_cnp_prio_enable = 0, + .default_cnp_prio = 0, + .default_token_period = 16, + .default_min_rate = 1, + .default_ecn_cnp_enable = 1, + .default_flow_ctrl_unit = 0, + .default_rtt_rsp_prio = 0, + .default_rtt_rsp_prio_enable = 0, + .default_rtt_version = 0, + .max_rtt_version = 0, + .min_bypass = 0, + .max_bypass = 2, + .min_rate_inc_hai = 1, + .max_rate_inc_hai = 255, + .min_rate_inc_ai = 1, + .max_rate_inc_ai = 255, + .min_initial_alpha = 1, + .max_initial_alpha = 1023, + .min_rate_inc_period = 1, + .max_rate_inc_period = 1023, + .min_alpha_dec_period = 1, + .max_alpha_dec_period = 1023, + .min_token_period = 4, + .max_token_period = 255, + .min_flow_min_rate = 1, + .max_flow_min_rate = 4095, + .min_rate_dec_period = 1, + .max_rate_dec_period = 255, + .min_min_cnp_period = 1, + .max_min_cnp_period = 255, + .min_factor_gita = 1, + .max_factor_gita = 15, + .min_rate_first_set = 128, + .max_rate_first_set = 50000, + .default_ldcp_wnd_min = 5, + .default_ldcp_wnd_default = 1024, + .default_ldcp_alpha = 2, + .default_ldcp_beta = 9, + .default_ldcp_gamma = 6, + .default_ldcp_eta = 6, + .min_ldcp_wnd_min = 1, + .max_ldcp_wnd_min = 0x8, + .min_ldcp_wnd_default = 1024, + .max_ldcp_wnd_default = 32768, + .min_ldcp_alpha = 0, + .max_ldcp_alpha = 0xf, + .min_ldcp_beta = 0, + .max_ldcp_beta = 0xf, + .min_ldcp_gamma = 0, + .max_ldcp_gamma = 0xf, + .min_ldcp_eta = 0, + .max_ldcp_eta = 0xf }, // v2 +}; + +static int roce5_update_ver_check(struct roce5_device *rdev, u32 cc_algo) +{ + u32 algo_bitmap = 0; + /* driver intercept not support algorithm. 1823V200 only supports LDCP algorithm, 1825V100 current only supports DCQCN, IPQCN, user custom A & B algorithm */ + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + algo_bitmap = BIT(ROCE_CC_DISABLE); + if (ROCE5_SUPPORT_CC_GET_DCQCN_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_DCQCN_ALGO); + } + if (ROCE5_SUPPORT_CC_GET_LDCP_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_LDCP_ALGO); + } + if (ROCE5_SUPPORT_CC_GET_IPQCN_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_IPQCN_ALGO); + } + if (ROCE5_SUPPORT_CC_GET_USER_ALGO_ENABLE(rdev) != 0) { + algo_bitmap |= BIT(ROCE_CC_USER_A_ALGO); + algo_bitmap |= BIT(ROCE_CC_USER_B_ALGO); + } + } + + if (ROCE_CC_ALGO_TEST_BIT(algo_bitmap, cc_algo) == 0) { + ROCE_DEV_ERR(rdev, "func_id(0x%x), cc algo(0x%x) not support!", + rdev->glb_func_id, cc_algo); + return -EINVAL; + } + return 0; +} + +static int roce5_update_cfg_cc_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_cc_param_s cc_param; + int ret; + + ret = roce5_update_ver_check(rdev, ecn_ctx->cc_algo); + if (ret != 0) { + return ret; + } + + (void)memset_s(&cc_param, sizeof(cc_param), 0, sizeof(cc_param)); + + cc_param.dw0.bs.bypass = ecn_ctx->rp_ctx.bypass; + cc_param.dw0.bs.rtt_rsp_prio = ecn_ctx->np_ctx.rtt_rsp_prio; + cc_param.dw0.bs.rtt_rsp_prio_enable = + ecn_ctx->np_ctx.rtt_rsp_prio_enable; + cc_param.dw0.bs.rtt_version = ecn_ctx->np_ctx.rtt_version; + cc_param.dw0.bs.ecn_cnp_en = ecn_ctx->np_ctx.ecn_cnp_en; + cc_param.dw0.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + cc_param.dw0.bs.cnp_prio_enable = ecn_ctx->np_ctx.cnp_prio_enable; + cc_param.dw0.bs.slow_path_psn_threshold = + ecn_ctx->multipath_ctx.slow_path_psn_threshold; + cc_param.dw0.bs.rtt_req_event_mode = + ecn_ctx->multipath_ctx.rtt_req_event_mode; + cc_param.dw1.bs.cnp_prio = ecn_ctx->np_ctx.cnp_prio; + cc_param.dw1.bs.cnp_cos = + roce5_get_db_cos_from_vlan_pri(rdev, cc_param.dw1.bs.cnp_prio); + cc_param.dw1.bs.cc_appid = ecn_ctx->cc_algo; + + if (cc_param.dw1.bs.cc_appid == ROCE_CC_LDCP_ALGO) { + cc_param.dw0.bs.algo_mode = CCP_ALGO_MODE_CWND; + } else if (cc_param.dw1.bs.cc_appid == ROCE_CC_DCQCN_ALGO || + cc_param.dw1.bs.cc_appid == ROCE_CC_IPQCN_ALGO) { + cc_param.dw0.bs.algo_mode = CCP_ALGO_MODE_RATE; + } else { + cc_param.dw0.bs.algo_mode = ecn_ctx->ucc_ctx.algo_mode; + } + + if (cc_param.dw0.bs.algo_mode == CCP_ALGO_MODE_CWND) { + cc_param.dw0.bs.flow_ctrl_unit = FLOW_CTRL_UNIT_QP; + } else { + cc_param.dw0.bs.flow_ctrl_unit = ecn_ctx->np_ctx.flow_ctrl_unit; + } + + ROCE_DEV_INFO(rdev, "set cc_appid(%u), func_id(%u).", + cc_param.dw1.bs.cc_appid, rdev->glb_func_id); + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)&cc_param, + sizeof(roce_cc_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_CC_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set cc param, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce5_update_dcqcn_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + ccp_dcqcn_para_s dcqcn_param; + int ret; + + if (ecn_ctx->cc_algo != ROCE_CC_DCQCN_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not dcqcn, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + + (void)memset_s(&dcqcn_param, sizeof(dcqcn_param), 0, + sizeof(dcqcn_param)); + + dcqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + dcqcn_param.dw0.bs.min_rate = ecn_ctx->rp_ctx.min_rate; + dcqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + dcqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + dcqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + dcqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + dcqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + dcqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + dcqcn_param.dw3.bs.gita_shift = ecn_ctx->rp_ctx.factor_gita; + dcqcn_param.dw3.bs.rate_target_clamp = + ecn_ctx->rp_ctx.rate_target_clamp; + dcqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + // adjust_en + dcqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&dcqcn_param, + sizeof(ccp_dcqcn_para_s), + (u16)ROCE_MPU_CMD_CC_CFG_DCQCN_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set dcqcn param, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce5_update_ldcp_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_ldcp_param_s ldcp_param; + int ret; + + if (ecn_ctx->cc_algo != ROCE_CC_LDCP_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not ldcp, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + + ROCE_DEV_INFO( + rdev, + "update ldcp param, func_id(%d), alpha(%u), beta(%u), gamma(%u), eta(%u) wnd_min(%u) wnd_default(%u)", + rdev->glb_func_id, ecn_ctx->ldcp_ctx.alpha, + ecn_ctx->ldcp_ctx.beta, ecn_ctx->ldcp_ctx.gamma, + ecn_ctx->ldcp_ctx.eta, ecn_ctx->ldcp_ctx.wnd_min, + ecn_ctx->ldcp_ctx.wnd_default); + + (void)memset_s(&ldcp_param, sizeof(ldcp_param), 0, sizeof(ldcp_param)); + + ldcp_param.dw0.bs.alpha = ecn_ctx->ldcp_ctx.alpha; + ldcp_param.dw0.bs.beta = ecn_ctx->ldcp_ctx.beta; + ldcp_param.dw0.bs.gamma = ecn_ctx->ldcp_ctx.gamma; + ldcp_param.dw0.bs.eta = ecn_ctx->ldcp_ctx.eta; + ldcp_param.dw0.bs.wnd_min = ecn_ctx->ldcp_ctx.wnd_min; + ldcp_param.dw1.bs.wnd_default = ecn_ctx->ldcp_ctx.wnd_default; + ldcp_param.dw0.bs.set_flag = 1; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ldcp_param, + sizeof(roce_ldcp_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_LDCP_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ldcp param, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +static int roce5_update_ipqcn_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_ipqcn_param_s ipqcn_param; + int ret; + + if (ecn_ctx->cc_algo != ROCE_CC_IPQCN_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not ipqcn, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + + (void)memset_s(&ipqcn_param, sizeof(ipqcn_param), 0, + sizeof(ipqcn_param)); + + ipqcn_param.dw0.bs.token_period = ecn_ctx->rp_ctx.token_period; + ipqcn_param.dw0.bs.flow_min_rate = ecn_ctx->rp_ctx.min_rate; + ipqcn_param.dw1.bs.rate_inc_period = ecn_ctx->rp_ctx.rate_inc_period; + ipqcn_param.dw1.bs.alpha_dec_period = ecn_ctx->rp_ctx.alpha_dec_period; + ipqcn_param.dw2.bs.rate_inc_ai = ecn_ctx->rp_ctx.rate_inc_ai; + ipqcn_param.dw2.bs.rate_inc_hai = ecn_ctx->rp_ctx.rate_inc_hai; + ipqcn_param.dw2.bs.rate_dec_period = ecn_ctx->rp_ctx.rate_dec_period; + ipqcn_param.dw2.bs.min_cnp_period = ecn_ctx->np_ctx.min_cnp_period; + ipqcn_param.dw3.bs.factor_gita = ecn_ctx->rp_ctx.factor_gita; + ipqcn_param.dw3.bs.rt_clamp = ecn_ctx->rp_ctx.rate_target_clamp; + ipqcn_param.dw3.bs.initial_alpha = ecn_ctx->rp_ctx.initial_alpha; + ipqcn_param.dw3.bs.rate_first_set = ecn_ctx->rp_ctx.rate_first_set; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ipqcn_param, + sizeof(roce_ipqcn_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_IPQCN_PARAM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to set ipqcn param, func_id(%d) ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +typedef int (*roce5_update_param_t)(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx); + +static roce5_update_param_t roce5_update_param_funcs[] = { + roce5_update_cfg_cc_param, roce5_update_dcqcn_param, + roce5_update_ipqcn_param, roce5_update_ldcp_param, + roce5_update_ucc_param, +}; + +int roce5_update_ecn_param(const struct roce5_ecn_ctx *ecn_ctx) +{ + unsigned int i; + int ret = 0; + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + /* current only supports LDCP algorithm, its remaining algorithm can only modify its algorithm parameters, cannot switch */ + for (i = 0; i < (sizeof(roce5_update_param_funcs) / + sizeof(roce5_update_param_t)); + i++) { + ret = roce5_update_param_funcs[i](rdev, ecn_ctx); + if (ret != 0) { + ret = -EINVAL; + break; + } + } + + return ret; +} + +static ssize_t roce5_show_cc_algo(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + switch ((int)ecn_ctx->cc_algo) { + case ROCE_CC_DISABLE: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "disable"); + case ROCE_CC_DCQCN_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "dcqcn"); + case ROCE_CC_LDCP_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "ldcp"); + case ROCE_CC_IPQCN_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "hc3_1/ipqcn"); + case ROCE_CC_USER_A_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "user_cc_6"); + case ROCE_CC_USER_B_ALGO: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "user_cc_7"); + default: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", + "error_type"); + } +} + +static ssize_t roce5_store_cc_algo(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + u32 old_cc_algo = 0; + int cc_algo = 0; + const char *ptr = NULL; + int ret; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + /* dcqcn -> 0x0, ldcp -> 0x2, hc3_1 -> 0x80 */ + if (strncmp(buf, "disable", strlen("disable")) == 0) { + cc_algo = ROCE_CC_DISABLE; + } else if (strncmp(buf, "dcqcn", strlen("dcqcn")) == 0) { + cc_algo = ROCE_CC_DCQCN_ALGO; + } else if ((strncmp(buf, "ipqcn", strlen("ipqcn")) == 0) || + (strncmp(buf, "hc3_1", strlen("hc3_1")) == 0)) { + cc_algo = ROCE_CC_IPQCN_ALGO; + } else if (strncmp(buf, "ldcp", strlen("ldcp")) == 0) { + cc_algo = ROCE_CC_LDCP_ALGO; + } else if (strncmp(buf, "user_cc_", strlen("user_cc_")) == 0) { + if (strlen(buf) > ROCE_USER_CC_ID_MAX_LEN) { + ROCE_ERR( + "Invalid buffer length, expected <= %d, got %zu, Buffer: '%s'", + ROCE_USER_CC_ID_MAX_LEN, strlen(buf), buf); + return -EINVAL; + } + ptr = buf + strlen("user_cc_"); + cc_algo = (unsigned char)(*ptr) - CHAR_0; + if (cc_algo != ROCE_CC_USER_A_ALGO && + cc_algo != ROCE_CC_USER_B_ALGO) { + ROCE_ERR("Invalid cc_algo(%d),buf(%s)", cc_algo, buf); + return -EINVAL; + } + } else { + ROCE_ERR("Invalid cc_algo(%d),buf(%s)", cc_algo, buf); + return -EIO; + } + + if (ecn_ctx->cc_algo != (u32)cc_algo) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cc_algo = ecn_ctx->cc_algo; + ecn_ctx->cc_algo = (u32)cc_algo; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Failed to update cc_algo(%d), ret(%d).", + cc_algo, ret); + ecn_ctx->cc_algo = old_cc_algo; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce5_show_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce5_prio_enable_ctx, enable); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)prio_enable_ctx->prio_en); +} + +static ssize_t roce5_store_ecn_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce5_ecn_rp_ctx *rp_ctx = NULL; + struct roce5_ecn_np_ctx *np_ctx = NULL; + struct roce5_ecn_ctx *ecn_ctx = NULL; + struct roce5_prio_enable_ctx *prio_enable_ctx = + container_of(attr, struct roce5_prio_enable_ctx, enable); + struct roce5_ecn_enable_ctx *ecn_enable_ctx = + (struct roce5_ecn_enable_ctx *)prio_enable_ctx->ecn_enable_ctx; + + if (ecn_enable_ctx->np_rp == ROCE_DCQCN_NP) { + np_ctx = container_of(ecn_enable_ctx, struct roce5_ecn_np_ctx, + enable_ctx); + ecn_ctx = container_of(np_ctx, struct roce5_ecn_ctx, np_ctx); + } else { + rp_ctx = container_of(ecn_enable_ctx, struct roce5_ecn_rp_ctx, + enable_ctx); + ecn_ctx = container_of(rp_ctx, struct roce5_ecn_ctx, rp_ctx); + } + + ret = sscanf_s(buf, "%d", &prio_en); + if (ret != 1) { + return -EIO; + } + + if (((u32)prio_en & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = prio_enable_ctx->prio_en; + prio_enable_ctx->prio_en = (u32)prio_en; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update prio_en(%d), ret(%d).", + prio_en, ret); + prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static ssize_t roce5_show_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce5_ip_prio_enable_ctx, ip_enable); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ip_prio_enable_ctx->prio_en); +} + +static ssize_t roce5_store_ecn_ip_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int prio_en = 0; + u32 old_prio_en = 0; + struct roce5_ip_prio_enable_ctx *ip_prio_enable_ctx = + container_of(attr, struct roce5_ip_prio_enable_ctx, ip_enable); + struct roce5_ecn_ip_enable_ctx *ip_enable_ctx = + (struct roce5_ecn_ip_enable_ctx *) + ip_prio_enable_ctx->ecn_ip_enable_ctx; + struct roce5_ecn_ctx *ecn_ctx = container_of( + ip_enable_ctx, struct roce5_ecn_ctx, ip_enable_ctx); + + ret = sscanf_s(buf, "%d", &prio_en); + if (ret != 1) { + return -EIO; + } + + if (((u32)prio_en & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ip_prio_enable_ctx->prio_en != (u32)prio_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_prio_en = ip_prio_enable_ctx->prio_en; + ip_prio_enable_ctx->prio_en = (u32)prio_en; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update prio_en(%d), ret(%d).", + prio_en, ret); + ip_prio_enable_ctx->prio_en = old_prio_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* rp_ctx parameters use macro generate show/store function */ +ROCE5_PARAM_SHOW_STORE_RP(bypass, bypass, min_bypass, max_bypass) +ROCE5_PARAM_SHOW_STORE_RP(rate_inc_hai, rate_inc_hai, min_rate_inc_hai, + max_rate_inc_hai) +ROCE5_PARAM_SHOW_STORE_RP(rate_inc_ai, rate_inc_ai, min_rate_inc_ai, + max_rate_inc_ai) +ROCE5_PARAM_SHOW_STORE_RP(initial_alpha, initial_alpha, min_initial_alpha, + max_initial_alpha) +ROCE5_PARAM_SHOW_STORE_RP(factor_gita, factor_gita, min_factor_gita, + max_factor_gita) +ROCE5_PARAM_SHOW_STORE_RP(rate_inc_period, rate_inc_period, min_rate_inc_period, + max_rate_inc_period) +ROCE5_PARAM_SHOW_STORE_RP(rate_dec_period, rate_dec_period, min_rate_dec_period, + max_rate_dec_period) +ROCE5_PARAM_SHOW_STORE_RP(token_period, token_period, min_token_period, + max_token_period) +ROCE5_PARAM_SHOW_STORE_RP(min_rate, min_rate, min_flow_min_rate, + max_flow_min_rate) +ROCE5_PARAM_SHOW_STORE_RP(alpha_dec_period, alpha_dec_period, + min_alpha_dec_period, max_alpha_dec_period) +ROCE5_PARAM_SHOW_STORE_RP(rate_first_set, rate_first_set, min_rate_first_set, + max_rate_first_set) + +/* rate_target_clamp special handle: use ROCE_ENABLE_CHECK_MASK but non- range check */ +static ssize_t roce5_show_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_rp_ctx *ecn_rp_ctx = to_roce5_ecn_rp_ctx(kobj); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_rp_ctx->rate_target_clamp); +} + +static ssize_t roce5_store_rate_target_clamp(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int rate_target_clamp = 0; + u32 old_rate_target_clamp = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_rp_ctx *ecn_rp_ctx = to_roce5_ecn_rp_ctx(kobj); + + ret = sscanf_s(buf, "%d", &rate_target_clamp); + if (ret != 1) { + return -EIO; + } + + if (((u32)rate_target_clamp & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_rp_ctx->rate_target_clamp != (u32)rate_target_clamp) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rate_target_clamp = ecn_rp_ctx->rate_target_clamp; + ecn_rp_ctx->rate_target_clamp = (u32)rate_target_clamp; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update rate_target_clamp(%d), ret(%d).", + rate_target_clamp, ret); + ecn_rp_ctx->rate_target_clamp = old_rate_target_clamp; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* np_ctx parameters use macro generate show/store function */ +ROCE5_PARAM_SHOW_STORE_NP(min_cnp_period, min_cnp_period, min_min_cnp_period, + max_min_cnp_period) + +/* flow_ctrl_unit special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_flow_ctrl_unit(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->flow_ctrl_unit); +} +static ssize_t roce5_store_flow_ctrl_unit(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret, _val; + u32 _old; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + char remaining; + + ret = sscanf_s(buf, "%d %c", &_val, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, flow_ctrl_unit should be unsigned int, ret(%d), flow_ctrl_unit(%d)", + ret, _val); + return -EIO; + } + + if (((u32)_val & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->flow_ctrl_unit != (u32)_val) { + mutex_lock(&ecn_ctx->ecn_mutex); + _old = ecn_np_ctx->flow_ctrl_unit; + ecn_np_ctx->flow_ctrl_unit = (u32)_val; + ret = roce5_update_ecn_param(ecn_ctx); + + if (ret != 0) { + ecn_np_ctx->flow_ctrl_unit = _old; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + return (ssize_t)count; +} + +/* rtt_version special handle: only check maximum value */ +static ssize_t roce5_show_rtt_version(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->rtt_version); +} +static ssize_t roce5_store_rtt_version(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 _val, _old; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + char remaining; + + ret = sscanf_s(buf, "%u %c", &_val, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_version should be unsigned int, ret(%d), rtt_version(%u)", + ret, _val); + return -EIO; + } + + if (_val > + g_roce_cc_ecn_param[rdev->cc_param_version].max_rtt_version) { + ROCE_ERR("Invalid input, rtt_version(%u) over max rtt version", + _val); + return -EIO; + } + + if (ecn_np_ctx->rtt_version != _val) { + mutex_lock(&ecn_ctx->ecn_mutex); + _old = ecn_np_ctx->rtt_version; + ecn_np_ctx->rtt_version = _val; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_np_ctx->rtt_version = _old; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + return (ssize_t)count; +} + +/* rtt_rsp_prio special handle: use PRI_ARRAY_LEN - 1 check */ +static ssize_t roce5_show_rtt_rsp_prio(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + ecn_np_ctx->rtt_rsp_prio); +} + +static ssize_t roce5_store_rtt_rsp_prio(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + u32 rtt_rsp_prio = 0; + u32 old_rtt_rsp_prio = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + char remaining; + ret = sscanf_s(buf, "%u %c", &rtt_rsp_prio, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_rsp_prio should be unsigned int, ret(%d), rtt_rsp_prio(%u)\n", + ret, rtt_rsp_prio); + return -EIO; + } + + if (rtt_rsp_prio > (PRI_ARRAY_LEN - 1)) { + return -EIO; + } + + if (ecn_np_ctx->rtt_rsp_prio != (u32)rtt_rsp_prio) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rtt_rsp_prio = ecn_np_ctx->rtt_rsp_prio; + ecn_np_ctx->rtt_rsp_prio = (u32)rtt_rsp_prio; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update rtt_rsp_prio(%d), ret(%d).", + rtt_rsp_prio, ret); + ecn_np_ctx->rtt_rsp_prio = old_rtt_rsp_prio; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* rtt_rsp_prio_enable special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_rtt_rsp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + ecn_np_ctx->rtt_rsp_prio_enable); +} + +static ssize_t roce5_store_rtt_rsp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + u32 rtt_rsp_prio_enable = 0; + u32 old_rtt_rsp_prio_enable = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + char remaining; + ret = sscanf_s(buf, "%u %c", &rtt_rsp_prio_enable, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, rtt_rsp_prio_enable should be unsigned int, ret(%d), rtt_rsp_prio_enable(%u)", + ret, rtt_rsp_prio_enable); + return -EIO; + } + + if ((rtt_rsp_prio_enable & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->rtt_rsp_prio_enable != (u32)rtt_rsp_prio_enable) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_rtt_rsp_prio_enable = ecn_np_ctx->rtt_rsp_prio_enable; + ecn_np_ctx->rtt_rsp_prio_enable = (u32)rtt_rsp_prio_enable; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR( + "Fail to update rtt_rsp_prio_enable(%d), ret(%d).", + rtt_rsp_prio_enable, ret); + ecn_np_ctx->rtt_rsp_prio_enable = + old_rtt_rsp_prio_enable; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* ecn_cnp_en special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_ecn_cnp_en(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->ecn_cnp_en); +} + +static ssize_t roce5_store_ecn_cnp_en(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + u32 ecn_cnp_en = 0; + u32 old_ecn_cnp_en = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + char remaining; + ret = sscanf_s(buf, "%u %c", &ecn_cnp_en, &remaining, 1); + if (ret != 1) { + ROCE_ERR( + "Invalid input, ecn_cnp_en should be unsigned int, ret(%d), ecn_cnp_en(%u)", + ret, ecn_cnp_en); + return -EIO; + } + + if ((ecn_cnp_en & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->ecn_cnp_en != (u32)ecn_cnp_en) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ecn_cnp_en = ecn_np_ctx->ecn_cnp_en; + ecn_np_ctx->ecn_cnp_en = (u32)ecn_cnp_en; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update ecn_cnp_en(%d), ret(%d).", + ecn_cnp_en, ret); + ecn_np_ctx->ecn_cnp_en = old_ecn_cnp_en; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* cnp_prio_enable special handle: use ROCE_ENABLE_CHECK_MASK check */ +static ssize_t roce5_show_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->cnp_prio_enable); +} + +static ssize_t roce5_store_cnp_prio_enable(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int cnp_prio_enable = 0; + u32 old_cnp_prio_enable = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + ret = sscanf_s(buf, "%d", &cnp_prio_enable); + if (ret != 1) { + return -EIO; + } + + if (((u32)cnp_prio_enable & ROCE_ENABLE_CHECK_MASK) != 0) { + return -EIO; + } + + if (ecn_np_ctx->cnp_prio_enable != (u32)cnp_prio_enable) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio_enable = ecn_np_ctx->cnp_prio_enable; + ecn_np_ctx->cnp_prio_enable = (u32)cnp_prio_enable; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update cnp_prio_enable(%d), ret(%d).", + cnp_prio_enable, ret); + ecn_np_ctx->cnp_prio_enable = old_cnp_prio_enable; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +/* cnp_prio special handle: use PRI_ARRAY_LEN - 1 check */ +static ssize_t roce5_show_cnp_prio(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_np_ctx->cnp_prio); +} + +static ssize_t roce5_store_cnp_prio(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret = 0; + int cnp_prio = 0; + u32 old_cnp_prio = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_np_ctx *ecn_np_ctx = to_roce5_ecn_np_ctx(kobj); + + ret = sscanf_s(buf, "%d", &cnp_prio); + if (ret != 1) { + return -EIO; + } + + if (((u32)cnp_prio) > (PRI_ARRAY_LEN - 1)) { + return -EIO; + } + + if (ecn_np_ctx->cnp_prio != (u32)cnp_prio) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_cnp_prio = ecn_np_ctx->cnp_prio; + ecn_np_ctx->cnp_prio = (u32)cnp_prio; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update cnp_prio(%d), ret(%d).", + cnp_prio, ret); + ecn_np_ctx->cnp_prio = old_cnp_prio; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +static int +roce5_init_ecn_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_enable_ctx *ecn_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = { "0", "1", "2", "3", "4", "5", "6", "7" }; + struct roce5_prio_enable_ctx *prio_enable = NULL; + + ecn_enable_ctx->enable_root = kobject_create_and_add("enable", kobj); + if (ecn_enable_ctx->enable_root == NULL) { + ROCE_ERR("Failed to create kobject for ecn enable.(ret:%d)", + ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + prio_enable = &ecn_enable_ctx->prio_enable[i]; + prio_enable->ecn_enable_ctx = (void *)ecn_enable_ctx; + prio_enable->prio = (u32)i; + prio_enable->prio_en = 1; + sysfs_attr_init(&prio_enable->enable.attr); + prio_enable->enable.attr.name = prio_arr[i]; + prio_enable->enable.attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + prio_enable->enable.show = roce5_show_ecn_prio_enable; + prio_enable->enable.store = roce5_store_ecn_prio_enable; + ret = sysfs_create_file(ecn_enable_ctx->enable_root, + &prio_enable->enable.attr); + if (ret != 0) { + ROCE_ERR( + "Failed to create file for ecn enable.(ret:%d)", + ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); + + return ret; +} + +static void +roce5_remove_ecn_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_enable_ctx *ecn_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file(ecn_enable_ctx->enable_root, + &ecn_enable_ctx->prio_enable[i].enable.attr); + } + + kobject_put(ecn_enable_ctx->enable_root); +} + +static int +roce5_init_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int ret = 0; + int i = 0; + int j = 0; + const char *prio_arr[8] = { "0", "1", "2", "3", "4", "5", "6", "7" }; + struct roce5_ip_prio_enable_ctx *ip_prio_enable = NULL; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + ip_enable_ctx->ip_enable_root = + kobject_create_and_add("ip_enable", kobj); + if (ip_enable_ctx->ip_enable_root == NULL) { + ROCE_ERR("Failed to create kobject for ecn ip enable.(ret:%d)", + ret); + return -ENOMEM; + } + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + ip_prio_enable = &ip_enable_ctx->ip_prio_enable[i]; + ip_prio_enable->ecn_ip_enable_ctx = (void *)ip_enable_ctx; + ip_prio_enable->prio = (u32)i; + ip_prio_enable->prio_en = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_ip_enable_en; + sysfs_attr_init(&ip_prio_enable->ip_enable.attr); + ip_prio_enable->ip_enable.attr.name = prio_arr[i]; + ip_prio_enable->ip_enable.attr.mode = + (S_IWUSR | S_IRUSR | S_IRGRP); + ip_prio_enable->ip_enable.show = roce5_show_ecn_ip_prio_enable; + ip_prio_enable->ip_enable.store = + roce5_store_ecn_ip_prio_enable; + ret = sysfs_create_file(ip_enable_ctx->ip_enable_root, + &ip_prio_enable->ip_enable.attr); + if (ret != 0) { + ROCE_ERR( + "Failed to create file for ecn ip enable.(ret:%d)", + ret); + goto err_create_file; + } + } + + return 0; + +err_create_file: + for (j = i - 1; j >= 0; j--) { + sysfs_remove_file( + ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); + + return ret; +} + +static void +roce5_remove_ecn_ip_enable_sysfs(struct kobject *kobj, + struct roce5_ecn_ip_enable_ctx *ip_enable_ctx) +{ + int i; + + for (i = 0; i < PRI_ARRAY_LEN; i++) { + sysfs_remove_file( + ip_enable_ctx->ip_enable_root, + &ip_enable_ctx->ip_prio_enable[i].ip_enable.attr); + } + + kobject_put(ip_enable_ctx->ip_enable_root); +} + +ROCE_ATTR_RW(bypass, roce5_show_bypass, roce5_store_bypass); +ROCE_ATTR_RW(alpha_dec_period, roce5_show_alpha_dec_period, + roce5_store_alpha_dec_period); +ROCE_ATTR_RW(rate_dec_period, roce5_show_rate_dec_period, + roce5_store_rate_dec_period); +ROCE_ATTR_RW(rate_inc_period, roce5_show_rate_inc_period, + roce5_store_rate_inc_period); +ROCE_ATTR_RW(factor_gita, roce5_show_factor_gita, roce5_store_factor_gita); +ROCE_ATTR_RW(initial_alpha, roce5_show_initial_alpha, + roce5_store_initial_alpha); +ROCE_ATTR_RW(rate_inc_ai, roce5_show_rate_inc_ai, roce5_store_rate_inc_ai); +ROCE_ATTR_RW(rate_inc_hai, roce5_show_rate_inc_hai, roce5_store_rate_inc_hai); +ROCE_ATTR_RW(rate_first_set, roce5_show_rate_first_set, + roce5_store_rate_first_set); +ROCE_ATTR_RW(rate_target_clamp, roce5_show_rate_target_clamp, + roce5_store_rate_target_clamp); +ROCE_ATTR_RW(token_period, roce5_show_token_period, roce5_store_token_period); +ROCE_ATTR_RW(min_rate, roce5_show_min_rate, roce5_store_min_rate); + +static struct attribute *ecn_rp_ctx_attrs[] = { + ROCE_ATTR_PTR(bypass), + ROCE_ATTR_PTR(alpha_dec_period), + ROCE_ATTR_PTR(rate_dec_period), + ROCE_ATTR_PTR(rate_inc_period), + ROCE_ATTR_PTR(factor_gita), + ROCE_ATTR_PTR(initial_alpha), + ROCE_ATTR_PTR(rate_inc_ai), + ROCE_ATTR_PTR(rate_inc_hai), + ROCE_ATTR_PTR(rate_first_set), + ROCE_ATTR_PTR(rate_target_clamp), + ROCE_ATTR_PTR(token_period), + ROCE_ATTR_PTR(min_rate), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_rp_ctx_attr_grp = { + .attrs = ecn_rp_ctx_attrs, +}; + +static const struct attribute_group *ecn_rp_ctx_attr_grps[] = { + &ecn_rp_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_rp_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_rp_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_rp_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_rp_ctx_attr_grps, +#else + .default_attrs = ecn_rp_ctx_attrs, +#endif +}; + +static int roce5_init_ecn_rp_sysfs(struct kobject *kobj, + struct roce5_ecn_rp_ctx *ecn_rp_ctx) +{ + int ret = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + ecn_rp_ctx->bypass = + g_roce_cc_ecn_param[rdev->cc_param_version].default_bypass; + ecn_rp_ctx->alpha_dec_period = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_alpha_dec_period; + ecn_rp_ctx->rate_dec_period = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_dec_period; + ecn_rp_ctx->rate_inc_period = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_inc_period; + ecn_rp_ctx->factor_gita = + g_roce_cc_ecn_param[rdev->cc_param_version].default_factor_gita; + ecn_rp_ctx->initial_alpha = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_initial_alpha; + ecn_rp_ctx->rate_inc_ai = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rate_inc_ai; + ecn_rp_ctx->rate_inc_hai = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rate_inc_hai; + ecn_rp_ctx->rate_first_set = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_first_set; + ecn_rp_ctx->rate_target_clamp = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rate_target_clamp; + ecn_rp_ctx->token_period = + g_roce_cc_ecn_param[rdev->cc_param_version].default_token_period; + ecn_rp_ctx->min_rate = + g_roce_cc_ecn_param[rdev->cc_param_version].default_min_rate; + ret = kobject_init_and_add(&ecn_rp_ctx->ecn_rp_root, &roce_ecn_rp_ktype, + kobj, "roce5_rp"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_rp.(ret:%d)", ret); + kobject_put(&ecn_rp_ctx->ecn_rp_root); + return ret; + } + + ecn_rp_ctx->enable_ctx.np_rp = ROCE_DCQCN_RP; + ret = roce5_init_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, + &ecn_rp_ctx->enable_ctx); + if (ret != 0) { + goto err_init_prio_enable_sysfs; + } + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_rp_ctx->ecn_rp_root); + + return ret; +} + +static void roce5_remove_ecn_rp_sysfs(struct kobject *kobj, + struct roce5_ecn_rp_ctx *ecn_rp_ctx) +{ + roce5_remove_ecn_enable_sysfs(&ecn_rp_ctx->ecn_rp_root, + &ecn_rp_ctx->enable_ctx); + + kobject_put(&ecn_rp_ctx->ecn_rp_root); +} + +ROCE_ATTR_RW(rtt_rsp_prio, roce5_show_rtt_rsp_prio, roce5_store_rtt_rsp_prio); +ROCE_ATTR_RW(rtt_rsp_prio_enable, roce5_show_rtt_rsp_prio_enable, + roce5_store_rtt_rsp_prio_enable); +ROCE_ATTR_RW(rtt_version, roce5_show_rtt_version, roce5_store_rtt_version); +ROCE_ATTR_RW(ecn_cnp_en, roce5_show_ecn_cnp_en, roce5_store_ecn_cnp_en); +ROCE_ATTR_RW(flow_ctrl_unit, roce5_show_flow_ctrl_unit, + roce5_store_flow_ctrl_unit); +ROCE_ATTR_RW(min_cnp_period, roce5_show_min_cnp_period, + roce5_store_min_cnp_period); +ROCE_ATTR_RW(cnp_prio_enable, roce5_show_cnp_prio_enable, + roce5_store_cnp_prio_enable); +ROCE_ATTR_RW(cnp_prio, roce5_show_cnp_prio, roce5_store_cnp_prio); + +static struct attribute *ecn_np_ctx_attrs[] = { + ROCE_ATTR_PTR(min_cnp_period), + ROCE_ATTR_PTR(cnp_prio_enable), + ROCE_ATTR_PTR(cnp_prio), + ROCE_ATTR_PTR(rtt_rsp_prio), + ROCE_ATTR_PTR(rtt_rsp_prio_enable), + ROCE_ATTR_PTR(rtt_version), + ROCE_ATTR_PTR(ecn_cnp_en), + ROCE_ATTR_PTR(flow_ctrl_unit), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_np_ctx_attr_grp = { + .attrs = ecn_np_ctx_attrs, +}; + +static const struct attribute_group *ecn_np_ctx_attr_grps[] = { + &ecn_np_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_np_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_np_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_np_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_np_ctx_attr_grps, +#else + .default_attrs = ecn_np_ctx_attrs, +#endif +}; + +static int roce5_init_ecn_np_sysfs(struct kobject *kobj, + struct roce5_ecn_np_ctx *ecn_np_ctx) +{ + int ret = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + ecn_np_ctx->min_cnp_period = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_min_cnp_period; + ecn_np_ctx->cnp_prio_enable = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_cnp_prio_enable; + ecn_np_ctx->cnp_prio = + g_roce_cc_ecn_param[rdev->cc_param_version].default_cnp_prio; + ecn_np_ctx->ecn_cnp_en = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_ecn_cnp_enable; + ecn_np_ctx->flow_ctrl_unit = g_roce_cc_ecn_param[rdev->cc_param_version] + .default_flow_ctrl_unit; + ecn_np_ctx->rtt_rsp_prio = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rtt_rsp_prio; + ecn_np_ctx->rtt_rsp_prio_enable = + g_roce_cc_ecn_param[rdev->cc_param_version] + .default_rtt_rsp_prio_enable; + ecn_np_ctx->rtt_version = + g_roce_cc_ecn_param[rdev->cc_param_version].default_rtt_version; + ret = kobject_init_and_add(&ecn_np_ctx->ecn_np_root, &roce_ecn_np_ktype, + kobj, "roce5_np"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_np.(ret:%d)", ret); + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; + } + + ecn_np_ctx->enable_ctx.np_rp = ROCE_DCQCN_NP; + ret = roce5_init_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, + &ecn_np_ctx->enable_ctx); + if (ret != 0) { + goto err_init_prio_enable_sysfs; + } + + return 0; + +err_init_prio_enable_sysfs: + kobject_put(&ecn_np_ctx->ecn_np_root); + return ret; +} + +/* ldcp_ctx parameters use macro generate show/store function */ +ROCE5_PARAM_SHOW_STORE_LDCP(wnd_min, wnd_min, min_ldcp_wnd_min, + max_ldcp_wnd_min); +ROCE5_PARAM_SHOW_STORE_LDCP(wnd_default, wnd_default, min_ldcp_wnd_default, + max_ldcp_wnd_default); +ROCE5_PARAM_SHOW_STORE_LDCP(alpha, alpha, min_ldcp_alpha, max_ldcp_alpha); +ROCE5_PARAM_SHOW_STORE_LDCP(beta, beta, min_ldcp_beta, max_ldcp_beta); +ROCE5_PARAM_SHOW_STORE_LDCP(gamma, gamma, min_ldcp_gamma, max_ldcp_gamma); +ROCE5_PARAM_SHOW_STORE_LDCP(eta, eta, min_ldcp_eta, max_ldcp_eta); + +ROCE_ATTR_RW(wnd_min, roce5_show_wnd_min, roce5_store_wnd_min); +ROCE_ATTR_RW(wnd_default, roce5_show_wnd_default, roce5_store_wnd_default); +ROCE_ATTR_RW(alpha, roce5_show_alpha, roce5_store_alpha); +ROCE_ATTR_RW(beta, roce5_show_beta, roce5_store_beta); +ROCE_ATTR_RW(gamma, roce5_show_gamma, roce5_store_gamma); +ROCE_ATTR_RW(eta, roce5_show_eta, roce5_store_eta); + +static struct attribute *ecn_ldcp_ctx_attrs[] = { + ROCE_ATTR_PTR(wnd_min), + ROCE_ATTR_PTR(wnd_default), + ROCE_ATTR_PTR(alpha), + ROCE_ATTR_PTR(beta), + ROCE_ATTR_PTR(gamma), + ROCE_ATTR_PTR(eta), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_ldcp_ctx_attr_grp = { + .attrs = ecn_ldcp_ctx_attrs, +}; + +static const struct attribute_group *ecn_ldcp_ctx_attr_grps[] = { + &ecn_ldcp_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_ldcp_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_ldcp_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_ldcp_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_ldcp_ctx_attr_grps, +#else + .default_attrs = ecn_ldcp_ctx_attrs, +#endif +}; + +static int roce5_init_ecn_ldcp_sysfs(struct kobject *kobj, + roce5_ecn_ldcp_ctx_t *ecn_ldcp_ctx) +{ + int ret; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + struct roce5_device *rdev = + container_of(ecn_ctx, struct roce5_device, ecn_ctx); + + ecn_ldcp_ctx->wnd_min = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_wnd_min; + ecn_ldcp_ctx->wnd_default = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_wnd_default; + ecn_ldcp_ctx->alpha = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_alpha; + ecn_ldcp_ctx->beta = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_beta; + ecn_ldcp_ctx->gamma = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_gamma; + ecn_ldcp_ctx->eta = + g_roce_cc_ecn_param[rdev->cc_param_version].min_ldcp_eta; + ret = kobject_init_and_add(&ecn_ldcp_ctx->ecn_ldcp_root, + &roce_ecn_ldcp_ktype, kobj, "roce5_ldcp"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_ldcp.(ret:%d)", ret); + kobject_put(&ecn_ldcp_ctx->ecn_ldcp_root); + return ret; + } + + return 0; +} + +static void roce5_remove_ecn_ldcp_sysfs(roce5_ecn_ldcp_ctx_t *ecn_ldcp_ctx) +{ + kobject_put(&ecn_ldcp_ctx->ecn_ldcp_root); +} + +static void roce5_remove_ecn_np_sysfs(struct kobject *kobj, + struct roce5_ecn_np_ctx *ecn_np_ctx) +{ + roce5_remove_ecn_enable_sysfs(&ecn_np_ctx->ecn_np_root, + &ecn_np_ctx->enable_ctx); + + kobject_put(&ecn_np_ctx->ecn_np_root); +} + +static ssize_t roce5_show_ecn_ver(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%d\n", + (int)ecn_ctx->ecn_ver); +} + +static ssize_t roce5_store_ecn_ver(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret = 0; + int ecn_ver = 0; + u32 old_ecn_ver = 0; + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj); + + ret = sscanf_s(buf, "%d", &ecn_ver); + if (ret != 1) { + return -EIO; + } + + if (ROCE5_PARAM_CHECK_BOUND(ecn_ver, ECN_VER_DCQCN, ECN_VER_PATHQCN)) { + return -EIO; + } + + if (ecn_ctx->ecn_ver != (u32)ecn_ver) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ecn_ver = ecn_ctx->ecn_ver; + ecn_ctx->ecn_ver = (u32)ecn_ver; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Fail to update ecn_ver(%d), ret(%d).", + ecn_ver, ret); + ecn_ctx->ecn_ver = old_ecn_ver; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_ATTR_RW(ecn_ver, roce5_show_ecn_ver, roce5_store_ecn_ver); +ROCE_ATTR_RW(cc_algo, roce5_show_cc_algo, roce5_store_cc_algo); + +static struct attribute *ecn_ctx_attrs[] = { + ROCE_ATTR_PTR(ecn_ver), + ROCE_ATTR_PTR(cc_algo), + NULL, +}; + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_ctx_attr_grp = { + .attrs = ecn_ctx_attrs, +}; + +static const struct attribute_group *ecn_ctx_attr_grps[] = { + &ecn_ctx_attr_grp, + NULL, +}; +#endif + +static void roce_ecn_sysfs_release(struct kobject *kobj) +{ +} + +static struct kobj_type roce_ecn_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_ctx_attr_grps, +#else + .default_attrs = ecn_ctx_attrs, +#endif +}; + +static void roce5_init_cc_algo(struct roce5_device *rdev, + struct roce5_ecn_ctx *ecn_ctx) +{ + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + ecn_ctx->cc_algo = ROCE_CC_DCQCN_ALGO; + } else { + ecn_ctx->cc_algo = ROCE_CC_DISABLE; + } +} + +static void roce5_init_cc_param_version(struct roce5_device *rdev) +{ + rdev->cc_param_version = + ROCE5_SUPPORT_CC_PARAM_V1(rdev) != 0 ? ROCE_CC_PARAM_V1 : + ROCE5_SUPPORT_CC_PARAM_V2(rdev) != 0 ? ROCE_CC_PARAM_V2 : + ROCE_CC_PARAM_INVALID; +} + +static int roce5_init_ecn_sysfs(struct roce5_device *rdev, + struct roce5_ecn_ctx *ecn_ctx) +{ + int ret = 0; + struct net_device *ndev = rdev->ndev; + + ret = memset_s(ecn_ctx, sizeof(*ecn_ctx), 0, sizeof(*ecn_ctx)); + if (ret != 0) { + ROCE_ERR("Failed to memset ecn_ctx.(ret:%d)", ret); + return -ENOMEM; + } + + mutex_init(&ecn_ctx->ecn_mutex); + + ecn_ctx->ecn_ver = ECN_VER_DCQCN; + roce5_init_cc_algo(rdev, ecn_ctx); + roce5_init_cc_param_version(rdev); + + ret = kobject_init_and_add(&ecn_ctx->ecn_root, &roce_ecn_ktype, + &ndev->dev.kobj, "ecn"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject ecn.(ret:%d)", ret); + kobject_put(&ecn_ctx->ecn_root); + return ret; + } + + ret = roce5_init_ecn_ldcp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ldcp_ctx); + if (ret != 0) { + goto err_init_ldcp_sysfs; + } + + ret = roce5_init_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + if (ret != 0) { + goto err_init_np_sysfs; + } + + ret = roce5_init_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + if (ret != 0) { + goto err_init_rp_sysfs; + } + + ret = roce5_init_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, + &ecn_ctx->ip_enable_ctx); + if (ret != 0) { + goto err_init_ip_enable_sysfs; + } + + ret = roce5_init_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx); + if (ret != 0) { + goto err_init_ecn_ucc_sysfs; + } + + ret = roce5_init_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root, + &ecn_ctx->multipath_ctx); + if (ret != 0) { + goto err_init_multipath_sysfs; + } + + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + goto err_update; + } + + return 0; + +err_update: + roce5_remove_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root, + &ecn_ctx->multipath_ctx); +err_init_multipath_sysfs: + roce5_remove_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx); +err_init_ecn_ucc_sysfs: + roce5_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, + &ecn_ctx->ip_enable_ctx); +err_init_ip_enable_sysfs: + roce5_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); +err_init_rp_sysfs: + roce5_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); +err_init_np_sysfs: + roce5_remove_ecn_ldcp_sysfs(&ecn_ctx->ldcp_ctx); +err_init_ldcp_sysfs: + kobject_put(&ecn_ctx->ecn_root); + return ret; +} + +static void roce5_prase_qpc(roce_verbs_qp_attr_s *qp_attr) +{ + qp_attr->com_info.rsvd = be32_to_cpu(qp_attr->com_info.rsvd); + qp_attr->ext_seg.dfx_info.dw0.value = + be32_to_cpu(qp_attr->ext_seg.dfx_info.dw0.value); + qp_attr->ext_seg.dfx_info.dw2.value = + be32_to_cpu(qp_attr->ext_seg.dfx_info.dw2.value); + qp_attr->ext_seg.dfx_info.dw4.value = + be32_to_cpu(qp_attr->ext_seg.dfx_info.dw4.value); +} + +static ssize_t roce5_store_dfx_qpc(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret; + u32 qpn; + struct roce5_dfx_qpc_ctx *qpc_ctx = NULL; + struct roce5_dfx_ctx *dfx_ctx = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_qp_attr_s qp_attr; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + ROCE_ERR("Invalid input para."); + return -EINVAL; + } + + qpc_ctx = container_of(attr, struct roce5_dfx_qpc_ctx, kattr); + dfx_ctx = container_of(qpc_ctx, struct roce5_dfx_ctx, qpc_ctx); + rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx); + + (void)memset_s(&qp_attr, sizeof(roce_verbs_qp_attr_s), 0, + sizeof(roce_verbs_qp_attr_s)); + + ret = (int)sscanf_s(buf, "%u", &qpn); + if (ret != 1) { + ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if ((qpn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_qps) || + (qpn < ROCE_MIN_QPN)) { + ROCE_DEV_ERR(rdev, "Invalid qpn(%u), max_qps(%u)", qpn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_qps); + return -EIO; + } + + ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)&qp_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query qp_attr, ret(%d)", ret); + return (ssize_t)ret; + } + + roce5_prase_qpc(&qp_attr); + + roce5_dfx_print("qpn(0x%x): sqc pi(0x%x), sqc ci(0x%x).", qpn, + qp_attr.ext_seg.dfx_info.dw0.bs.sq_pi, + qp_attr.ext_seg.dfx_info.dw0.bs.sq_ci); + roce5_dfx_print( + "sqac ci(0x%x), sqac wqe_prefetch_ci(0x%x), rqc pi(0x%x), rqc ci(0x%x).", + qp_attr.ext_seg.dfx_info.dw2.bs.sqa_ci, + qp_attr.ext_seg.dfx_info.dw2.bs.sqa_wqe_prefetch_ci, + qp_attr.ext_seg.dfx_info.dw4.bs.rq_pi, + qp_attr.ext_seg.dfx_info.dw4.bs.rq_ci); + + return (ssize_t)count; +} + +static ssize_t roce5_store_dfx_cqc(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret; + u32 cqn; + struct roce5_dfx_cqc_ctx *cqc_ctx = NULL; + struct roce5_dfx_ctx *dfx_ctx = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_cq_attr_s cq_attr; + roce5_cq_query_outbuf_s cq_query; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + ROCE_ERR("Invalid input para."); + return -EINVAL; + } + + cqc_ctx = container_of(attr, struct roce5_dfx_cqc_ctx, kattr); + dfx_ctx = container_of(cqc_ctx, struct roce5_dfx_ctx, cqc_ctx); + rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx); + + (void)memset_s(&cq_query, sizeof(roce5_cq_query_outbuf_s), 0, + sizeof(roce5_cq_query_outbuf_s)); + + ret = (int)sscanf_s(buf, "%u", &cqn); + if (ret != 1) { + ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (cqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs) { + ROCE_DEV_ERR(rdev, "Invalid cqn(%u), max_cqs(%u)", cqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_cqs); + return -EIO; + } + + ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)&cq_query, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query cqc, ret(%d)", ret); + return (ssize_t)ret; + } + + if (rdev->device_type == ROCE_DEV_TYPE_HI1825_V100) { + (void)memcpy_s(&cq_attr, sizeof(roce_verbs_cq_attr_s), + &cq_query, sizeof(roce_verbs_cq_attr_s)); + cq_attr.dfx_info.dw0.value = + be32_to_cpu(cq_attr.dfx_info.dw0.value); + cq_attr.dfx_info.dw1.value = + be32_to_cpu(cq_attr.dfx_info.dw1.value); + roce5_dfx_print("cqn(0x%x): pi(0x%x), ci(0x%x).", cqn, + cq_attr.dfx_info.dw1.bs.pi, + cq_attr.dfx_info.dw0.bs.ci); + } else { + ROCE_DEV_ERR(rdev, + "Unsupport rdev device_type, device_type(%d)", + rdev->device_type); + } + + return (ssize_t)count; +} + +static ssize_t roce5_store_dfx_srqc(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int ret; + u32 srqn; + struct roce5_dfx_srqc_ctx *srqc_ctx = NULL; + struct roce5_dfx_ctx *dfx_ctx = NULL; + struct roce5_device *rdev = NULL; + roce_verbs_srq_attr_s srq_attr; + struct rdma_service_cap *rdma_cap = NULL; + + if (kobj == NULL || attr == NULL || buf == NULL || count == 0) { + ROCE_ERR("Invalid input para."); + return -EINVAL; + } + + srqc_ctx = container_of(attr, struct roce5_dfx_srqc_ctx, kattr); + dfx_ctx = container_of(srqc_ctx, struct roce5_dfx_ctx, srqc_ctx); + rdev = container_of(dfx_ctx, struct roce5_device, dfx_ctx); + + (void)memset_s(&srq_attr, sizeof(roce_verbs_srq_attr_s), 0, + sizeof(roce_verbs_srq_attr_s)); + + ret = (int)sscanf_s(buf, "%u", &srqn); + if (ret != 1) { + ROCE_DEV_ERR(rdev, "Failed to do sscanf_s, ret(%d)", ret); + return -EIO; + } + + rdma_cap = &rdev->rdma_cap; + if (srqn >= rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs) { + ROCE_DEV_ERR(rdev, "Invalid srqn(%u), max_srqs(%u)", srqn, + rdma_cap->dev_rdma_cap.roce_own_cap.max_srqs); + return -EIO; + } + + ret = roce5_dfx_cmd_query_srq(rdev, srqn, &srq_attr); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to query srq_attr, ret(%d)", ret); + return (ssize_t)ret; + } + + srq_attr.dfx_info.dw0.value = be32_to_cpu(srq_attr.dfx_info.dw0.value); + + roce5_dfx_print("srqn(0x%x): pcnt(0x%x), ccnt(0x%x).", srqn, + srq_attr.dfx_info.dw0.bs.pcnt, + srq_attr.dfx_info.dw0.bs.ccnt); + + return (ssize_t)count; +} + +static int roce5_init_dfx_sub_ctx_sysfs(struct kobject *kobj, + struct roce5_dfx_ctx *dfx_ctx) +{ + int ret; + struct kobj_attribute *qpc_kattr = NULL; + struct kobj_attribute *cqc_kattr = NULL; + struct kobj_attribute *srqc_kattr = NULL; + + if (kobj == NULL || dfx_ctx == NULL) { + ROCE_ERR("Invalid kobj or dfx_ctx."); + return -EINVAL; + } + + qpc_kattr = &dfx_ctx->qpc_ctx.kattr; + cqc_kattr = &dfx_ctx->cqc_ctx.kattr; + srqc_kattr = &dfx_ctx->srqc_ctx.kattr; + + sysfs_attr_init(&qpc_kattr->attr); + qpc_kattr->attr.name = "qpc"; + qpc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + qpc_kattr->store = roce5_store_dfx_qpc; + ret = sysfs_create_file(kobj, &qpc_kattr->attr); + if (ret != 0) { + ROCE_ERR("QPC failed to do sysfs_create_file, ret(%d)", ret); + return ret; + } + + sysfs_attr_init(&cqc_kattr->attr); + cqc_kattr->attr.name = "cqc"; + cqc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + cqc_kattr->store = roce5_store_dfx_cqc; + ret = sysfs_create_file(kobj, &cqc_kattr->attr); + if (ret != 0) { + ROCE_ERR("CQC failed to do sysfs_create_file, ret(%d)", ret); + goto err_create_cqc_sysfs; + } + + sysfs_attr_init(&srqc_kattr->attr); + srqc_kattr->attr.name = "srqc"; + srqc_kattr->attr.mode = (S_IWUSR | S_IRUSR | S_IRGRP); + srqc_kattr->store = roce5_store_dfx_srqc; + ret = sysfs_create_file(kobj, &srqc_kattr->attr); + if (ret != 0) { + ROCE_ERR("SRQC failed to do sysfs_create_file, ret(%d)", ret); + goto err_create_srqc_sysfs; + } + + return 0; + +err_create_srqc_sysfs: + sysfs_remove_file(kobj, &cqc_kattr->attr); + +err_create_cqc_sysfs: + sysfs_remove_file(kobj, &qpc_kattr->attr); + + return ret; +} + +static int roce5_init_dfx_sysfs(struct net_device *ndev, + struct roce5_dfx_ctx *dfx_ctx) +{ + int ret; + + (void)memset_s(dfx_ctx, sizeof(*dfx_ctx), 0, sizeof(*dfx_ctx)); + + dfx_ctx->dfx_root = + kobject_create_and_add("roce5_dfx", &ndev->dev.kobj); + if (dfx_ctx->dfx_root == NULL) { + ROCE_ERR("Failed to do kobject_create_and_add."); + return -ENOMEM; + } + + ret = roce5_init_dfx_sub_ctx_sysfs(dfx_ctx->dfx_root, dfx_ctx); + if (ret != 0) { + ROCE_ERR("Failed to init dfx sub ctx sysfs, ret(%d)", ret); + kobject_put(dfx_ctx->dfx_root); + return ret; + } + + return 0; +} + +static void roce5_remove_ecn_sysfs(struct roce5_device *rdev) +{ + struct roce5_ecn_ctx *ecn_ctx = &rdev->ecn_ctx; + roce5_remove_ecn_multipath_sysfs(rdev, &ecn_ctx->ecn_root, + &ecn_ctx->multipath_ctx); + + roce5_remove_ecn_ucc_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->ucc_ctx); + + roce5_remove_ecn_ip_enable_sysfs(&ecn_ctx->ecn_root, + &ecn_ctx->ip_enable_ctx); + + roce5_remove_ecn_rp_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->rp_ctx); + + roce5_remove_ecn_np_sysfs(&ecn_ctx->ecn_root, &ecn_ctx->np_ctx); + + roce5_remove_ecn_ldcp_sysfs(&ecn_ctx->ldcp_ctx); + + kobject_put(&ecn_ctx->ecn_root); +} + +int roce5_init_sysfs(struct roce5_device *rdev) +{ + int ret; + + ret = roce5_init_ecn_sysfs(rdev, &rdev->ecn_ctx); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init ecn sysfs, ret(%d)", ret); + return ret; + } + + ret = roce5_init_dfx_sysfs(rdev->ndev, &rdev->dfx_ctx); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to init dfx sysfs, ret(%d)", ret); + return ret; + } + + return 0; +} + +static void roce5_remove_dfx_sub_sysfs(struct roce5_dfx_ctx *dfx_ctx) +{ + if (dfx_ctx->dfx_root != NULL) { + sysfs_remove_file(dfx_ctx->dfx_root, + &dfx_ctx->srqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, + &dfx_ctx->cqc_ctx.kattr.attr); + + sysfs_remove_file(dfx_ctx->dfx_root, + &dfx_ctx->qpc_ctx.kattr.attr); + + kobject_put(dfx_ctx->dfx_root); + + dfx_ctx->dfx_root = NULL; + } +} + +void roce5_remove_sysfs(struct roce5_device *rdev) +{ + roce5_remove_dfx_sub_sysfs(&rdev->dfx_ctx); + roce5_remove_ecn_sysfs(rdev); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h new file mode 100644 index 000000000..814f29ddc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_sysfs.h @@ -0,0 +1,241 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SYSFS_H +#define ROCE_SYSFS_H + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/kobject.h> +#include "roce_ucc_sysfs.h" +#include "roce_multipath_sysfs.h" + +#define ROCE_MIN_QPN 2 +#define BYTE_TO_BIT 8 + +#define PRI_ARRAY_LEN 8 + +#define ROCE_MAX_PORT_CNT 8 +#define ROCE_MAX_PF_NUM 32 + +#define ROCE_ENABLE_CHECK_MASK 0xfffffffe + +/* ECN VER */ +#define ECN_VER_DCQCN 0 +#define ECN_VER_PATHQCN 1 + +#define FLOW_CTRL_UNIT_QP 0 +#define FLOW_CTRL_UNIT_IP 1 + +enum { ROCE_DCQCN_NP = 0, ROCE_DCQCN_RP = 1 }; + +enum { + ROCE_CC_PARAM_INVALID = 0, + ROCE_CC_PARAM_V1 = 1, + ROCE_CC_PARAM_V2 = 2, +}; + +#define HRN0_K_MAGIC_NUM_3_SYSFS 3 +#define HRN0_K_MAGIC_NUM_7_SYSFS 7 +#define HRN0_K_MAGIC_NUM_8_SYSFS 8 +#define HRN0_K_MAGIC_NUM_M_SYSFS 1000000 + +#define to_roce5_ecn_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_ctx, ecn_root) +#define to_roce5_ecn_np_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_np_ctx, ecn_np_root) +#define to_roce5_ecn_rp_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_rp_ctx, ecn_rp_root) +#define to_roce5_ecn_ldcp_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_ldcp_ctx, ecn_ldcp_root) + +#define ROCE5_PARAM_CHECK_BOUND(val, min, max) \ + ((val) < (min)) || ((val) > (max)) + +#define INIT_ROCE_KOBJ_ATTR(_name, _mode, _show, _store) \ + { \ + .attr = { .name = __stringify(_name), .mode = _mode }, \ + .show = (_show), .store = (_store), \ + } + +#define ROCE_ATTR_RW(_name, _show, _store) \ + static struct kobj_attribute roce_attr_##_name = INIT_ROCE_KOBJ_ATTR( \ + _name, (S_IWUSR | S_IRUSR | S_IRGRP), (_show), (_store)) + +#define ROCE_ATTR_PTR(_name) (&roce_attr_##_name.attr) + +#define ROCE_CC_ALGO_TEST_BIT(bitmap, cc_algo) (!!(bitmap & (1U << cc_algo))) + +struct roce5_prio_enable_ctx { + struct kobj_attribute enable; + void *ecn_enable_ctx; + u32 prio_en; + u32 prio; +}; + +struct roce5_ip_prio_enable_ctx { + struct kobj_attribute ip_enable; + void *ecn_ip_enable_ctx; + u32 prio_en; + u32 prio; +}; + +struct roce5_ecn_ip_enable_ctx { + struct kobject *ip_enable_root; + struct roce5_ip_prio_enable_ctx ip_prio_enable[PRI_ARRAY_LEN]; +}; + +struct roce5_ecn_enable_ctx { + struct kobject *enable_root; + struct roce5_prio_enable_ctx prio_enable[PRI_ARRAY_LEN]; + u32 np_rp; +}; + +struct roce5_dfx_qpc_ctx { + struct kobj_attribute kattr; +}; + +struct roce5_dfx_cqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce5_dfx_srqc_ctx { + struct kobj_attribute kattr; +}; + +struct roce5_dfx_ctx { + struct kobject *dfx_root; + struct roce5_dfx_qpc_ctx qpc_ctx; + struct roce5_dfx_cqc_ctx cqc_ctx; + struct roce5_dfx_srqc_ctx srqc_ctx; +}; + +struct roce5_ecn_rp_ctx { + struct kobject ecn_rp_root; + u32 alpha_dec_period; + u32 rate_dec_period; + u32 rate_inc_period; + u32 factor_gita; + u32 initial_alpha; + u32 rate_inc_ai; + u32 rate_inc_hai; + u32 rate_first_set; + u32 rate_target_clamp; + u32 token_period; + u32 min_rate; + u32 bypass; + struct roce5_ecn_enable_ctx enable_ctx; +}; + +struct roce5_ecn_np_ctx { + struct kobject ecn_np_root; + u32 min_cnp_period; + u32 cnp_prio_enable; + u32 cnp_prio; + u32 rtt_rsp_prio; + u32 rtt_rsp_prio_enable; + u32 rtt_version; + u32 ecn_cnp_en; + u32 flow_ctrl_unit; + struct roce5_ecn_enable_ctx enable_ctx; +}; + +typedef struct roce5_ecn_ldcp_ctx { + struct kobject ecn_ldcp_root; + u32 wnd_min; + u32 wnd_default; /* initial default window size, unit: Byte */ + u32 alpha; + u32 beta; + u32 gamma; + u32 eta; +} roce5_ecn_ldcp_ctx_t; + +struct roce5_ecn_ctx { + struct kobject ecn_root; + struct mutex ecn_mutex; + u32 ecn_ver; + u32 cc_algo; + roce5_ecn_ldcp_ctx_t ldcp_ctx; + struct roce5_ecn_np_ctx np_ctx; + struct roce5_ecn_rp_ctx rp_ctx; + struct roce5_ecn_ip_enable_ctx ip_enable_ctx; + struct roce5_ecn_ucc_ctx ucc_ctx; + struct roce5_ecn_multipath_ctx multipath_ctx; +}; + +struct roce_cc_ecn_param { + u32 default_ip_enable_en; + u32 default_bypass; + u32 default_min_cnp_period; + u32 default_alpha_dec_period; + u32 default_rate_dec_period; + u32 default_rate_inc_period; + u32 default_factor_gita; + u32 default_initial_alpha; + u32 default_rate_inc_ai; + u32 default_rate_inc_hai; + u32 default_rate_first_set; + u32 default_rate_target_clamp; + u32 default_cnp_prio_enable; + u32 default_cnp_prio; + u32 default_token_period; + u32 default_min_rate; + u32 default_ecn_cnp_enable; + u32 default_flow_ctrl_unit; + u32 default_rtt_rsp_prio; + u32 default_rtt_rsp_prio_enable; + u32 default_rtt_version; + + u32 max_rtt_version; + u32 min_bypass; + u32 max_bypass; + u32 min_rate_inc_hai; + u32 max_rate_inc_hai; + u32 min_rate_inc_ai; + u32 max_rate_inc_ai; + u32 min_initial_alpha; + u32 max_initial_alpha; + u32 min_rate_inc_period; + u32 max_rate_inc_period; + u32 min_alpha_dec_period; + u32 max_alpha_dec_period; + u32 min_token_period; + u32 max_token_period; + u32 min_flow_min_rate; + u32 max_flow_min_rate; + u32 min_rate_dec_period; + u32 max_rate_dec_period; + u32 min_min_cnp_period; + u32 max_min_cnp_period; + u32 min_factor_gita; + u32 max_factor_gita; + u32 min_rate_first_set; + u32 max_rate_first_set; + + u32 default_ldcp_wnd_min; + u32 default_ldcp_wnd_default; + u32 default_ldcp_alpha; + u32 default_ldcp_beta; + u32 default_ldcp_gamma; + u32 default_ldcp_eta; + + u32 min_ldcp_wnd_min; + u32 max_ldcp_wnd_min; + u32 min_ldcp_wnd_default; + u32 max_ldcp_wnd_default; + u32 min_ldcp_alpha; + u32 max_ldcp_alpha; + u32 min_ldcp_beta; + u32 max_ldcp_beta; + u32 min_ldcp_gamma; + u32 max_ldcp_gamma; + u32 min_ldcp_eta; + u32 max_ldcp_eta; +}; + +int roce5_update_ecn_param(const struct roce5_ecn_ctx *ecn_ctx); + +#endif //ROCE_SYSFS_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c new file mode 100644 index 000000000..e163f895b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "roce_sysfs.h" +#include "roce_cc_format.h" +#include "roce_mpu_cmd.h" +#include "roce_cmd.h" +#include "ccp_algo_format.h" +#include "roce_kernel_compat_gen.h" + +#define MAX_STRING_FORMAT_LEN 64 +#define to_roce5_ecn_ucc_ctx(_kobj) \ + container_of(_kobj, struct roce5_ecn_ucc_ctx, ecn_ucc_root) + +static char g_param_buffer[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN]; +static char g_param_show_tmp[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN]; +static char g_param_show_format[ROCE_UCC_MAX_PARAM_NUM][ROCE_UCC_PARAM_BUFF_LEN]; + +static int roce5_ucc_store_input_check(struct kobject *kobj, const char *buf, + u8 param_index) +{ + if (kobj == NULL || buf == NULL) { + ROCE_ERR("kobj or buf is null\n"); + return -EINVAL; + } + + if (param_index >= ROCE_UCC_MAX_PARAM_NUM) { + ROCE_ERR("invalid index:%hhu.\n", param_index); + return -EINVAL; + } + + if (strlen(buf) > (ROCE_UCC_PARAM_BUFF_LEN - 1)) { + ROCE_ERR("input buffer is beyond 1022.\n"); + return -EIO; + } + + return 0; +} + +static int roce5_init_ucc_global_buffer(u8 param_index, const char *buf) +{ + int ret; + ret = memset_s(g_param_show_tmp[param_index], + sizeof(g_param_show_tmp[param_index]), 0, + sizeof(g_param_show_tmp[param_index])); + if (ret != 0) { + ROCE_ERR("Failed to memset buffer.(ret:%d)\n", ret); + return -EIO; + } + + ret = strncpy_s(g_param_buffer[param_index], + sizeof(g_param_buffer[param_index]), buf, + sizeof(g_param_buffer[param_index]) - 1); + if (ret != 0) { + ROCE_ERR("Failed to strncpy buffer.(ret:%d)\n", ret); + return -EIO; + } + + g_param_buffer[param_index][ROCE_UCC_PARAM_BUFF_LEN - 1] = '\0'; + + return 0; +} + +static int check_key_unique(char (*pkey)[ROCE_UCC_MAX_KEY_SIZE], u8 key_num) +{ + u8 key_index = 0; + + if (key_num >= ROCE_UCC_MAX_KEY_SIZE) { + ROCE_ERR("invalid key value:%hhu.\n", key_num); + return -EINVAL; + } + + for (key_index = 0; key_index < key_num; key_index++) { + if (strcmp(pkey[key_index], pkey[key_num]) == 0) { + ROCE_ERR("key:%s is not unique.\n", pkey[key_num]); + return -EINVAL; + } + } + return 0; +} + +static int parse_each_line(char *param_line, + char (*pkey)[ROCE_UCC_MAX_KEY_SIZE], u8 key_num, + u32 *param_value, u8 *param_bits, u16 *sum_bits) +{ + u64 temp_value = 0; + u64 temp_bits = 0; + u64 temp_bit_sum = *sum_bits; + char extra = 0; + char value_str[ROCE_UCC_PARAM_VALUE_LEN] = { 0 }; + char bits_str[ROCE_UCC_PARAM_BITS_LEN] = { 0 }; + + if (key_num >= ROCE_UCC_MAX_KEY_SIZE) { + ROCE_ERR("invalid key value:%hhu.\n", key_num); + return -EINVAL; + } + + /* input of each line per per key:value:length perform parse, obtain value and length size, and check key whether repeat */ + if (sscanf_s(param_line, " %31[^: ] : %10[0-9] : %2[0-9] %c", + pkey[key_num], (unsigned)sizeof(pkey[key_num]), value_str, + (unsigned)sizeof(value_str), bits_str, + (unsigned)sizeof(bits_str), &extra, + 1) == ROCE_UCC_LINE_PARAM_TH) { + if (kstrtoull(value_str, NUM_10, &temp_value) < 0) { + ROCE_ERR("invalid input value:%s.\n", value_str); + return -EINVAL; + } + + if (kstrtoull(bits_str, NUM_10, &temp_bits) < 0) { + ROCE_ERR("invalid input bits:%s.\n", bits_str); + return -EINVAL; + } + + if ((temp_bits == 0) || (temp_bits > TYPE_BITS(u32)) || + (temp_value > BITS_VALUE(temp_bits))) { + ROCE_ERR("bits:%llu or value:%llu is invalid.\n", + temp_bits, temp_value); + return -EINVAL; + } + + temp_bit_sum += temp_bits; + if (temp_bit_sum > TYPE_BITS(u32)) { + ROCE_ERR("the length of bits:%llu beyond 32.\n", + temp_bit_sum); + return -EINVAL; + } + + if (check_key_unique(pkey, key_num) != 0) { + return -EINVAL; + } + + *param_value = (u32)temp_value; + *param_bits = (u8)temp_bits; + *sum_bits = (u16)temp_bit_sum; + return 0; + } + + ROCE_ERR("param format is wrong, please enter key:value:length param, " + "value less than 4294967296, length less than 33.\n"); + return -EINVAL; +} + +static int roce5_parse_ucc_param(u32 *ucc_param, char *param_buffer, + char *show_tmpkey) +{ + u8 key_num = 0; + u32 temp_param = 0; + /* param_value\param_bits used to store each line parse of values size and so occupy bit width result */ + u32 param_value = 0; + u8 param_bits = 0; + u16 sum_bits = 0; + /* param_line\ppbuffer used for input parameters partinterval parse */ + char *param_line = NULL; + char **ppbuffer = &param_buffer; + + char(*pkey)[ROCE_UCC_MAX_KEY_SIZE] = NULL; + char arr_tempkey[ROCE_UCC_MAX_KEY_NUM][ROCE_UCC_MAX_KEY_SIZE] = { 0 }; + char append_buff[ROCE_UCC_LINE_MAX_LEN] = { 0 }; + pkey = arr_tempkey; + + while ((param_line = strsep(ppbuffer, "\n")) != NULL) { + if (*param_line == '\0') { + break; + } + /* per per key:value:length parse input of each one line */ + if (parse_each_line(param_line, pkey, key_num, &param_value, + &param_bits, &sum_bits) != 0) { + return -EIO; + } + + if (snprintf_s(append_buff, ROCE_UCC_LINE_MAX_LEN, + ROCE_UCC_LINE_MAX_LEN - 1, "%s:%u:%hhu\n", + pkey[key_num], param_value, param_bits) < 0) { + ROCE_ERR("snprintf_s err!\n"); + return -EIO; + } + + if (strncat_s(show_tmpkey, ROCE_UCC_PARAM_BUFF_LEN, append_buff, + ROCE_UCC_PARAM_BUFF_LEN - strlen(show_tmpkey) - + 1) != 0) { + ROCE_ERR("strncat_s err!\n"); + return -EIO; + } + key_num++; + temp_param = key_num > 1 ? ((temp_param << param_bits) + + param_value) : + param_value; + } + + if (sum_bits < TYPE_BITS(u32)) { + temp_param = temp_param << (TYPE_BITS(u32) - sum_bits); + } + + if (ucc_param != NULL) { + *ucc_param = temp_param; + } + return 0; +} + +static int roce5_store_ucc_param_inner(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, u8 param_index) +{ + int ret = 0; + u32 ucc_param = 0; + u32 old_ucc_param = 0; + struct roce5_ecn_ctx *ecn_ctx = NULL; + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = NULL; + + if (roce5_ucc_store_input_check(kobj, buf, param_index) != 0) { + return -EINVAL; + } + + ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj); + + if (roce5_init_ucc_global_buffer(param_index, buf) != 0) { + return -EIO; + } + + if (roce5_parse_ucc_param(&ucc_param, g_param_buffer[param_index], + g_param_show_tmp[param_index]) != 0) { + return -EIO; + } + + mutex_lock(&ecn_ctx->ecn_mutex); + if (strncpy_s(g_param_show_format[param_index], + sizeof(g_param_show_format[param_index]), + g_param_show_tmp[param_index], + sizeof(g_param_show_format[param_index]) - 1) != 0) { + ROCE_ERR("Failed to strncpy buffer.\n"); + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + + mutex_unlock(&ecn_ctx->ecn_mutex); + + if ((ecn_ucc_ctx != NULL) && + (ecn_ucc_ctx->param[param_index] != ucc_param)) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_ucc_param = ecn_ucc_ctx->param[param_index]; + ecn_ucc_ctx->param[param_index] = ucc_param; + ret = (ssize_t)roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ecn_ucc_ctx->param[param_index] = old_ucc_param; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return 0; +} + +ssize_t roce5_show_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct roce5_ucc_param_ctx *ucc_param = + (struct roce5_ucc_param_ctx *)to_roce5_ucc_param_ctx(attr); + return sprintf_s(buf, sizeof(g_param_show_format[ucc_param->param_id]), + "%s", g_param_show_format[ucc_param->param_id]); +} + +ssize_t roce5_store_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct roce5_ucc_param_ctx *ucc_param = + (struct roce5_ucc_param_ctx *)to_roce5_ucc_param_ctx(attr); + return (roce5_store_ucc_param_inner(kobj, attr, buf, + ucc_param->param_id) == 0) ? + (ssize_t)count : + -EIO; +} + +static ssize_t roce5_show_algo_mode(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj); + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + switch ((int)ecn_ucc_ctx->algo_mode) { + case CCP_ALGO_MODE_CWND: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + (u32)CCP_ALGO_MODE_CWND); + case CCP_ALGO_MODE_RATE: + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%u\n", + (u32)CCP_ALGO_MODE_RATE); + default: + break; + } + return sprintf_s(buf, MAX_STRING_FORMAT_LEN, "%s\n", "error_type"); +} + +static ssize_t roce5_store_algo_mode(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct roce5_ecn_ctx *ecn_ctx = to_roce5_ecn_ctx(kobj->parent); + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx = to_roce5_ecn_ucc_ctx(kobj); + u32 old_algo_mode = 0; + int algo_mode = 0; + int ret; + char remaining; + + if ((attr == NULL) || (buf == NULL)) { + ROCE_ERR("attr or buf is null"); + return -EINVAL; + } + + ret = sscanf_s(buf, "%d %c", &algo_mode, &remaining, 1); + if (ret != 1) { + ROCE_ERR("Invalid input, algo_mode should be unsigned int\n"); + return -EIO; + } + + if (algo_mode != CCP_ALGO_MODE_CWND && + algo_mode != CCP_ALGO_MODE_RATE) { + ROCE_ERR("Invalid algo_mode(%d),buf(%s)\n", algo_mode, buf); + return -EIO; + } + + if (ecn_ucc_ctx->algo_mode != (u32)algo_mode) { + mutex_lock(&ecn_ctx->ecn_mutex); + old_algo_mode = ecn_ucc_ctx->algo_mode; + ecn_ucc_ctx->algo_mode = (u32)algo_mode; + ret = roce5_update_ecn_param(ecn_ctx); + if (ret != 0) { + ROCE_ERR("Failed to update algo_mode(%d), ret(%d).", + algo_mode, ret); + ecn_ucc_ctx->algo_mode = old_algo_mode; + mutex_unlock(&ecn_ctx->ecn_mutex); + return -EIO; + } + mutex_unlock(&ecn_ctx->ecn_mutex); + } + + return (ssize_t)count; +} + +ROCE_UCC_PARAM_RW(param0, 0); +ROCE_UCC_PARAM_RW(param1, 1); +ROCE_UCC_PARAM_RW(param2, 2); +ROCE_UCC_PARAM_RW(param3, 3); +ROCE_UCC_PARAM_RW(param4, 4); +ROCE_UCC_PARAM_RW(param5, 5); +ROCE_UCC_PARAM_RW(param6, 6); +ROCE_UCC_PARAM_RW(param7, 7); +ROCE_UCC_PARAM_RW(param8, 8); +ROCE_UCC_PARAM_RW(param9, 9); +ROCE_UCC_PARAM_RW(param10, 10); +ROCE_UCC_PARAM_RW(param11, 11); +ROCE_UCC_PARAM_RW(param12, 12); +ROCE_UCC_PARAM_RW(param13, 13); +ROCE_UCC_PARAM_RW(param14, 14); +ROCE_UCC_PARAM_RW(param15, 15); +ROCE_ATTR_RW(algo_mode, roce5_show_algo_mode, roce5_store_algo_mode); + +static struct attribute *ecn_ucc_ctx_attrs[] = { + ROCE_UCC_PARAM_ATTR_PTR(param0), ROCE_UCC_PARAM_ATTR_PTR(param1), + ROCE_UCC_PARAM_ATTR_PTR(param2), ROCE_UCC_PARAM_ATTR_PTR(param3), + ROCE_UCC_PARAM_ATTR_PTR(param4), ROCE_UCC_PARAM_ATTR_PTR(param5), + ROCE_UCC_PARAM_ATTR_PTR(param6), ROCE_UCC_PARAM_ATTR_PTR(param7), + ROCE_UCC_PARAM_ATTR_PTR(param8), ROCE_UCC_PARAM_ATTR_PTR(param9), + ROCE_UCC_PARAM_ATTR_PTR(param10), ROCE_UCC_PARAM_ATTR_PTR(param11), + ROCE_UCC_PARAM_ATTR_PTR(param12), ROCE_UCC_PARAM_ATTR_PTR(param13), + ROCE_UCC_PARAM_ATTR_PTR(param14), ROCE_UCC_PARAM_ATTR_PTR(param15), + ROCE_ATTR_PTR(algo_mode), NULL, +}; + +static void roce_ecn_ucc_sysfs_release(struct kobject *kobj) +{ +} + +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) +static const struct attribute_group ecn_ucc_ctx_attr_grp = { + .attrs = ecn_ucc_ctx_attrs, +}; + +static const struct attribute_group *ecn_ucc_ctx_groups[] = { + &ecn_ucc_ctx_attr_grp, + NULL, +}; +#endif + +static struct kobj_type roce_ecn_ucc_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = roce_ecn_ucc_sysfs_release, +#if defined(KBOJ_TYPE_HAVE_ATTRS_GROUP) + .default_groups = ecn_ucc_ctx_groups, +#else + .default_attrs = ecn_ucc_ctx_attrs, +#endif +}; + +int roce5_update_ucc_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx) +{ + roce_ucc_param_s ucc_param; + int ret; + int param_idx = 0; + + if (ecn_ctx->cc_algo != ROCE_CC_USER_A_ALGO && + ecn_ctx->cc_algo != ROCE_CC_USER_B_ALGO) { + ROCE_DEV_INFO( + rdev, + "func_id(%d) cc_algo(%u) is not user algo, ignore update.", + rdev->glb_func_id, ecn_ctx->cc_algo); + return 0; + } + (void)memset_s(&ucc_param, sizeof(ucc_param), 0, sizeof(ucc_param)); + + for (; param_idx < ROCE_UCC_MAX_PARAM_NUM; param_idx++) { + ucc_param.param[param_idx] = ecn_ctx->ucc_ctx.param[param_idx]; + } + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ucc_param, + sizeof(roce_ucc_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_UCC_PARAM); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set user algo param, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + } + + return ret; +} + +int roce5_init_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx) +{ + int ret = 0; + int param_idx = 0; + + if (ecn_ucc_ctx == NULL) { + pr_err_ratelimited("[ROCE, ERR] %s: ecn_ucc_ctx is null\n", + __func__); + return -EINVAL; + } + + ecn_ucc_ctx->algo_mode = CCP_ALGO_MODE_CWND; + for (; param_idx < ROCE_UCC_MAX_PARAM_NUM; param_idx++) { + ecn_ucc_ctx->param[param_idx] = ROCE_DEFAULT_UCC_PARAM; + } + + ret = kobject_init_and_add(&ecn_ucc_ctx->ecn_ucc_root, + &roce_ecn_ucc_ktype, kobj, "roce5_ucc"); + if (ret != 0) { + ROCE_ERR("Failed to add kobject roce5_ucc.(ret:%d)\n", ret); + kobject_put(&ecn_ucc_ctx->ecn_ucc_root); + return ret; + } + + return 0; +} + +void roce5_remove_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx) +{ + if (ecn_ucc_ctx == NULL) { + ROCE_ERR("ecn_ucc_ctx is null\n"); + return; + } + + kobject_put(&ecn_ucc_ctx->ecn_ucc_root); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h new file mode 100644 index 000000000..bbbe92b30 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/dev/sysfs/roce_ucc_sysfs.h @@ -0,0 +1,73 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef ROCE_UCC_SYSFS_H +#define ROCE_UCC_SYSFS_H + +#include <linux/types.h> +#include <linux/stat.h> +#include <linux/kobject.h> + +#define ROCE_UCC_MAX_PARAM_NUM 16 +#define ROCE_UCC_PARAM_BUFF_LEN 1024 +#define ROCE_DEFAULT_UCC_PARAM 0 +#define ROCE_UCC_MAX_KEY_SIZE 32 +#define ROCE_UCC_MAX_KEY_NUM 32 +#define ROCE_UCC_LINE_MAX_LEN 64 +#define ROCE_UCC_PARAM_VALUE_LEN 11 +#define ROCE_UCC_PARAM_BITS_LEN 3 +#define ROCE_UCC_LINE_PARAM_TH 3 + +#define CHAR_0 '0' +#define CHAR_9 '9' +#define NUM_10 10 +#define ROCE_USER_CC_ID_LABLE_LEN 8 +#define ROCE_USER_CC_ID_MAX_LEN (ROCE_USER_CC_ID_LABLE_LEN + 2) + +#define TYPE_BITS(type) (sizeof(type) * 8) +#define BITS_VALUE(bits) (((u64)1 << (bits)) - 1) + +/* forward-declaration */ +struct roce5_device; +struct roce5_ecn_ctx; + +typedef struct roce_ucc_param { + u32 param[ROCE_UCC_MAX_PARAM_NUM]; +} roce_ucc_param_s; + +struct roce5_ecn_ucc_ctx { + struct kobject ecn_ucc_root; + u32 algo_mode; + u32 param[ROCE_UCC_MAX_PARAM_NUM]; +}; + +struct roce5_ucc_param_ctx { + struct kobj_attribute param; + u32 param_id; +}; + +ssize_t roce5_show_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + char *buf); +ssize_t roce5_store_ucc_param(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count); +#define ROCE_UCC_PARAM_RW(_name, _idx) \ + static struct roce5_ucc_param_ctx roce_ucc_attr_##_name = { \ + .param = INIT_ROCE_KOBJ_ATTR( \ + _name, (S_IWUSR | S_IRUSR | S_IRGRP), \ + roce5_show_ucc_param, roce5_store_ucc_param), \ + .param_id = (_idx), \ + } + +#define ROCE_UCC_PARAM_ATTR_PTR(_name) (&roce_ucc_attr_##_name.param.attr) +#define to_roce5_ucc_param_ctx(_kattr) \ + container_of(_kattr, struct roce5_ucc_param_ctx, param) + +int roce5_update_ucc_param(struct roce5_device *rdev, + const struct roce5_ecn_ctx *ecn_ctx); +int roce5_init_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx); +void roce5_remove_ecn_ucc_sysfs(struct kobject *kobj, + struct roce5_ecn_ucc_ctx *ecn_ucc_ctx); + +#endif /* ROCE_UCC_SYSFS_H */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c new file mode 100644 index 000000000..145904486 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.c @@ -0,0 +1,13 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + + +#include "roce_main_extension.h" +#include "roce_mr_chip_ext.h" + +void roce5_init_dev_ext_handlers(struct roce5_device *rdev) +{ + rdev->ib_dev.check_mr_status = roce5_check_mr_status; + rdev->ib_dev.reg_user_fast_mr = roce5_reg_user_fast_mr; + rdev->ib_dev.map_frmr_sg = roce5_map_user_frmr_sg; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h new file mode 100644 index 000000000..a5f0847f8 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_main_chip_ext.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_MAIN_CHIP_EXT_C +#define ROCE_MAIN_CHIP_EXT_C + +#define ROCE_UVERBS_CMD_EXT_MASK \ + ((1ULL << IB_USER_VERBS_CMD_CREATE_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_ARM_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_DCT) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_MR) | \ + (1ULL << IB_USER_VERBS_CMD_REG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_FAST_MR) | \ + (1ULL << IB_USER_VERBS_CMD_MAP_FRMR_SG)) + +#endif /* ROCE_MAIN_CHIP_EXT_C */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c new file mode 100644 index 000000000..c460ca8a5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.c @@ -0,0 +1,348 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "roce_mr.h" +#include "roce_infra.h" +#include "roce_mr_extension.h" +#include "roce_main_extension.h" +#include "roce_restore.h" +#include "hmm_common.h" +#include "roce_user.h" + +int roce5_check_alloc_mr_type(enum ib_mr_type mr_type) +{ + int ret = 0; + + if (mr_type != IB_MR_TYPE_MEM_REG && mr_type != IB_MR_INDIRECT_REG && + mr_type != IB_MR_TYPE_FAST_MR) { + ret = -EINVAL; + ROCE_ERR("mr_type is invalid. mr_type:%u", mr_type); + } + + return ret; +} + +enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type) +{ + switch (ib_mr_type) { + case IB_MR_TYPE_MEM_REG: + return RDMA_DMA_MR; + + case IB_MR_INDIRECT_REG: + return RDMA_INDIRECT_MR; + + case IB_MR_TYPE_FAST_MR: + return RDMA_FRMR; + + default: + return RDMA_DMA_MR; + } +} + +static int roce5_alloc_usr_frmr(struct ib_pd *ibpd, struct ib_udata *udata, + struct roce5_reg_frmr_cmd *ucmd, + struct roce5_device **rdev, + struct roce5_mr **rmr) +{ + struct ib_umem **fumem = NULL; + if (ibpd == NULL) { + ROCE_ERR("Ibpd is null"); + return -EINVAL; + } + + if (udata == NULL) { + ROCE_ERR("Udata is null"); + return -EINVAL; + } + + *rdev = to_roce5_dev(ibpd->device); + + if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd))) { + ROCE_DEV_ERR((*rdev), + "Failed to copy_from_udata while creating frmr"); + return -EFAULT; + } + + *rmr = (struct roce5_mr *)kzalloc(sizeof(struct roce5_mr), GFP_KERNEL); + if (*rmr == NULL) { + ROCE_DEV_ERR((*rdev), "Failed to alloc memory for roce mr"); + return -ENOMEM; + } + + fumem = (struct ib_umem **)kzalloc( + sizeof(struct ib_umem *) * ROCE5_FRMR_UMEM_NUM, GFP_KERNEL); + if (fumem == NULL) { + ROCE_DEV_ERR((*rdev), "Failed to alloc memory for frmr umem"); + kfree(*rmr); + return -ENOMEM; + } + (*rmr)->fumem = fumem; + return 0; +} + +static int roce5_get_frmr_umem(struct roce5_mr *rmr, struct ib_pd *ibpd, + struct roce5_reg_frmr_cmd *ucmd, + struct roce5_device *rdev) +{ + rmr->umem = ib_umem_get(ibpd->uobject->context, ucmd->pbl_vaddr, + PAGE_SIZE_4K, 0, 0); + if (rmr->umem == NULL) { + ROCE_DEV_ERR(rdev, "Rmr->umem is null"); + return -ENOMEM; + } + + if (IS_ERR(rmr->umem)) { + ROCE_DEV_ERR(rdev, "Failed to get ib umem, func_id(%d)", + rdev->glb_func_id); + return -ENOMEM; + } + return 0; +} + +static int roce5_init_frmr_attr(u64 *pbl_paddr, struct roce5_mr *rmr, + struct roce5_device *rdev, struct ib_pd *ibpd) +{ + int ret; + u32 mpt_xid_mode; + struct roce5_pd *rpd = NULL; + + *pbl_paddr = sg_dma_address(rmr->umem->sg_head.sgl); + if (*pbl_paddr == 0ULL) { + ROCE_DEV_ERR(rdev, "Invalid pbl_paddr"); + return -EINVAL; + } + + rmr->rdmamr.mtt.mtt_type = MTT_DMTT_TYPE; + mpt_xid_mode = roce5_get_mpt_xid_mode(rdev); + ret = roce5_alloc_tpt(rdev, &rmr->rdmamr, ROCE_FRMR_MAX_PAGES, + PAGE_SHIFT, mpt_xid_mode); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc mpt and mtt, ret(%d)", ret); + return ret; + } + + rpd = to_roce5_pd(ibpd); + roce5_set_rdma_mr(&rmr->rdmamr, RDMA_FRMR, rpd->pdn, 0ULL, 0ULL, + ROCE5_MR_DEFAULT_ACCESS); + + return 0; +} + +struct ib_mr *roce5_reg_user_fast_mr(struct ib_pd *ibpd, struct ib_udata *udata) +{ + int ret; + struct roce5_device *rdev = NULL; + struct roce5_mr *rmr = NULL; + struct roce5_reg_frmr_cmd ucmd; + u64 pbl_paddr; + + ret = roce5_alloc_usr_frmr(ibpd, udata, &ucmd, &rdev, &rmr); + if (ret != 0) { + ROCE_ERR("Failed to pre check, ret(%d)", ret); + goto err_out; + } + + ret = roce5_get_frmr_umem(rmr, ibpd, &ucmd, rdev); + if (ret != 0) { + goto err_ib_umem_get; + } + + /* Use valloc to allocate a page of memory, just need to get the first physical address */ + ret = roce5_init_frmr_attr(&pbl_paddr, rmr, rdev, ibpd); + if (ret != 0) { + goto err_pbl_addr; + } + + ret = hmm_rdma_enable_mpt(rdev->lld_dev, rdev->hwdev, &(rmr->rdmamr), + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to enable mpt of frmr, ret(%d)", + ret); + goto err_enable_mpt; + } + + /* User mode also needs to send out pbl_paddr */ + if (ib_copy_to_udata(udata, &pbl_paddr, sizeof(u64)) != 0) { + ROCE_DEV_ERR(rdev, + "Failed to copy data to user space, func_id(%d)", + rdev->glb_func_id); + ret = -EFAULT; + goto err_enable_mpt; + } + + rmr->ibmr.lkey = rmr->rdmamr.key; + rmr->ibmr.rkey = rmr->rdmamr.key; + + return &rmr->ibmr; + +err_enable_mpt: + roce5_free_tpt(rdev, &rmr->rdmamr); + +err_pbl_addr: + ib_umem_release(rmr->umem); + +err_ib_umem_get: + kfree(rmr->fumem, sizeof(struct ib_umem *) * ROCE5_FRMR_UMEM_NUM); + kfree(rmr); + +err_out: + return (struct ib_mr *)ERR_PTR((long)ret); +} + +int roce5_frmr_check_user_param(struct ib_mr *ibmr, unsigned int page_num, + struct ib_udata *udata) +{ + if (ibmr == NULL) { + ROCE_ERR("Ibmr is null"); + return -EINVAL; + } + + if (page_num == 0 || page_num > ROCE5_FRMR_UMEM_NUM) { + ROCE_ERR("page_num(%d) is invalid", page_num); + return -EINVAL; + } + + if (udata == NULL) { + ROCE_ERR("udata is NULL"); + return -EINVAL; + } + return 0; +} + +void roce5_frmr_free_addr_list(unsigned long long *va_list, + unsigned long long *pa_list) +{ + vfree(va_list); + vfree(pa_list); +} + +int roce5_frmr_alloc_addr_list(struct roce5_device *rdev, + unsigned long long **va_list, + unsigned long long **pa_list, u32 len) +{ + int ret; + unsigned long long *val = NULL; + unsigned long long *pal = NULL; + + val = (unsigned long long *)vmalloc(len); + if (val == NULL) { + ROCE_DEV_ERR(rdev, "Failed to vmalloc va_list"); + return -EAGAIN; + } + pal = (unsigned long long *)vmalloc(len); + if (pal == NULL) { + ROCE_DEV_ERR(rdev, "Failed to vmalloc pa_list"); + vfree(val); + return -EAGAIN; + } + ret = memset_s(val, len, 0, len); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to memset va_list.(ret:%d)", ret); + ret = -ENOMEM; + goto err_free; + } + ret = memset_s(pal, len, 0, len); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to memset pa_list.(ret:%d)", ret); + ret = -ENOMEM; + goto err_free; + } + + *va_list = val; + *pa_list = pal; + return 0; + +err_free: + roce5_frmr_free_addr_list(val, pal); + return ret; +} + +static int process_va_list_and_allocate_umem( + struct roce5_mr *rmr, struct roce5_device *rdev, struct ib_mr *ibmr, + unsigned long long *va_list, unsigned long long *pa_list, u32 page_num) +{ + u32 i; + int ret; + for (i = 0; i < page_num; i++) { + ROCE_DEV_INFO(rdev, "i:%d, va(%llu)", i, va_list[i]); + rmr->fumem[i] = ib_umem_get(ibmr->pd->uobject->context, + va_list[i], FRMR_PAGE_SIZE, 0, 0); + if (IS_ERR(rmr->fumem[i])) { + ret = (int)PTR_ERR(rmr->fumem[i]); + roce5_free_frmr_umem(rmr->fumem, i - 1); + ROCE_DEV_ERR(rdev, + "Alloc umem failed, func_id(%d) err:%d", + rdev->glb_func_id, ret); + return ret; + } + + /* use valloc allocate a page of memory, only needs need get the a physical address */ + pa_list[i] = sg_dma_address(rmr->fumem[i]->sg_head.sgl); + ROCE_DEV_INFO(rdev, "PA_LIST:%llu", pa_list[i]); + if (pa_list[i] == 0ULL) { + ret = -EINVAL; + roce5_free_frmr_umem(rmr->fumem, i); + ROCE_DEV_ERR(rdev, "Invalid physicaL addr index(%d)", + i); + return ret; + } + } + return 0; +} + +int roce5_map_user_frmr_sg(struct ib_mr *ibmr, unsigned int page_num, + struct ib_udata *udata) +{ + struct roce5_mr *rmr = NULL; + struct roce5_device *rdev = NULL; + int ret; + unsigned long long *va_list = NULL; + unsigned long long *pa_list = NULL; + u32 len; + + ret = roce5_frmr_check_user_param(ibmr, page_num, udata); + if (ret) { + return ret; + } + + len = page_num * sizeof(unsigned long long); + rmr = to_roce5_mr(ibmr); + rdev = to_roce5_dev(ibmr->device); + ret = roce5_frmr_alloc_addr_list(rdev, &va_list, &pa_list, len); + if (ret) { + return ret; + } + + ROCE_DEV_INFO(rdev, "func_id(%d), page_num(%u)", rdev->glb_func_id, + page_num); + + ret = ib_copy_from_udata((void *)va_list, udata, len); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to copy udata, ret(%d)", ret); + goto err_free; + } + + ret = process_va_list_and_allocate_umem(rmr, rdev, ibmr, va_list, + pa_list, page_num); + if (ret != 0) { + goto err_free; + } + + /* 讲 pa information pass to user mode */ + if (ib_copy_to_udata(udata, pa_list, len)) { + ret = -EFAULT; + ROCE_DEV_ERR(rdev, "Failed to copy udata, func_id(%d)", + rdev->glb_func_id); + goto err_free; + } + + rmr->fumem_nr = page_num; + roce5_frmr_free_addr_list(va_list, pa_list); + return 0; + +err_free: + rmr->fumem_nr = 0; + roce5_frmr_free_addr_list(va_list, pa_list); + return ret; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h new file mode 100644 index 000000000..c206ca7f6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_mr_chip_ext.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MR_CHIP_EXT_H +#define ROCE_MR_CHIP_EXT_H + +#include <linux/types.h> +#include <linux/io-mapping.h> +#include <linux/list.h> +#include <linux/cdev.h> + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> + +int roce5_check_mr_status(struct ib_mr *ibmr, u32 check_mask, + struct ib_mr_status *mr_status); +struct ib_mr *roce5_reg_user_fast_mr(struct ib_pd *ibpd, + struct ib_udata *udata); +int roce5_map_user_frmr_sg(struct ib_mr *ibmr, unsigned int page_num, + struct ib_udata *udata); + +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c new file mode 100644 index 000000000..903da9898 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_qp_chip_ext.c @@ -0,0 +1,44 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#include "roce_qp_extension.h" + +void roce5_set_qp_dif_attr(struct roce5_qp *rqp, + const struct ib_qp_init_attr *init_attr, + const struct roce5_device *rdev) +{ + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EN) != 0) { + rqp->signature_en = true; + ROCE_DEV_INFO(rdev, + "func(%d) qp(%u) roce5_create_qp signature_en.", + rdev->glb_func_id, rqp->qpn); + } + + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_DOUBLE_SGL) != 0) { + rqp->double_sgl_mode = 1; + ROCE_DEV_INFO( + rdev, + "func(%d) qp(%u) roce5_create_qp double_sgl_mode.", + rdev->glb_func_id, rqp->qpn); + } + + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EXT_MTU) != 0) { + rqp->ext_mtu = 1; + ROCE_DEV_INFO(rdev, "func(%d) qp(%u) roce5_create_qp ext_mtu.", + rdev->glb_func_id, rqp->qpn); + } + + if (((unsigned int)init_attr->create_flags & + IB_QP_CREATE_SIGNATURE_EXT_MTU_MODE) != 0) { + rqp->ext_mtu_mode = 1; + ROCE_DEV_INFO(rdev, + "func(%d) qp(%u) roce5_create_qp ext_mtu_mode.", + rdev->glb_func_id, rqp->qpn); + } + + return; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c new file mode 100644 index 000000000..a6323afd0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/chip_ext/roce_srq_chip_ext.c @@ -0,0 +1,26 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + + +#include "roce_srq_extension.h" + +void roce5_srq_container_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, struct roce5_device *rdev) +{ + u8 srq_cont_en = 0; + srq_cont_en = init_attr->container_flag; + + rsrq->xrc_en = (init_attr->srq_type == IB_SRQT_XRC); + rsrq->container_flag = srq_cont_en ? init_attr->container_flag : + rdev->cfg_info.srq_container_en; + rsrq->container_mode = + srq_cont_en ? + init_attr->container_mode : + (rsrq->xrc_en ? rdev->cfg_info.xrc_srq_container_mode : + rdev->cfg_info.srq_container_mode); + rsrq->container_warn_th = init_attr->attr.srq_limit; + rsrq->rqe_cnt_th = srq_cont_en ? init_attr->container_warn_th : + rdev->cfg_info.warn_th; + + rsrq->container_size = roce5_get_container_sz(rsrq->container_mode); +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h new file mode 100644 index 000000000..b956ce82c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_common.h @@ -0,0 +1,13 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_COMMON +#define HYPER_ROCE_COMMON + +enum roce5_hyper_mode { + ROCE_HYPER_MODE_DISABLE, + ROCE_HYPER_MODE_UDP, + ROCE_HYPER_MODE_AR, +}; + +#endif /* HYPER_ROCE_COMMON */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c new file mode 100644 index 000000000..ca38186cc --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.c @@ -0,0 +1,511 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/types.h> +#include "hinic5_rdma.h" +#include "rdma_comp.h" +#include "roce_hyper_npu_cmd.h" +#include "roce_npu_cmd.h" +#include "roce_hyper_npu_cmd.h" +#include "comm_defs.h" + +#include "roce.h" +#include "comm_defs.h" +#include "roce_qp.h" +#include "roce_qp_ext.h" +#include "roce_netlink.h" +#include "roce_cqm_cmd.h" +#include "hyper_roce_common.h" +#include "hyper_roce_qp.h" +#include "hyper_roce_netlink.h" + +#define HYPER_ROCE_RDMARC_ENTRY_SIZE_BYTES 32 + +static int roce5_send_cmd_set_qp_hyper_mode(struct roce5_device *rdev, + struct roce5_hyper_set_in *msg_in) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_enable_hyper_roce_s *input = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_enable_hyper_roce_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_enable_hyper_roce_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(msg_in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_ENABLE_HYPER_ROCE_QP); + + input->cmd_info.dw0.bs.hyper_mode = msg_in->mode; + input->cmd_info.dw0.bs.srp_en = msg_in->sack_en; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int +roce5_send_cmd_modify_qp_extend(struct roce5_device *rdev, + struct roce5_hyper_extend_config_in *msg_in) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_enable_hyper_roce_s *input = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_enable_hyper_roce_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_enable_hyper_roce_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(msg_in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_ENABLE_HYPER_ROCE_QP); + + switch (msg_in->attr_mask) { + case IBV_QP_ATTR_EXTEND_UDP_SRC_PORT: + input->cmd_info.dw0.bs.base_src_port = msg_in->udp_src_port; + break; + case IBV_QP_ATTR_EXTEND_HYROCE_FEATURE: + input->cmd_info.dw0.bs.hyper_mode = msg_in->feature.type; + input->cmd_info.dw0.bs.srp_en = msg_in->feature.sack_enable; + break; + case IBV_QP_ATTR_EXTEND_LB_MODE: + input->cmd_info.dw0.bs.lb_mode = msg_in->lb_mode; + break; + case IBV_QP_ATTR_EXTEND_MP_CONFIG: + input->cmd_info.dw1.bs.sport_repeat_num = + msg_in->mp.flowlet_pkg_num; + input->cmd_info.dw1.bs.path_num = msg_in->mp.path_num; + input->cmd_info.dw1.bs.sport_interval = msg_in->mp.interval; + input->cmd_info.dw1.bs.path_rr_enable = + msg_in->mp.path_rr_enable; + input->cmd_info.dw1.bs.cc_mode = msg_in->mp.cc_mode; + break; + case IBV_QP_ATTR_EXTEND_AR_CONFIG: + input->cmd_info.dw1.bs.port_rr_enable = + msg_in->ar.port_rr_enable; + break; + case IBV_QP_ATTR_EXTEND_SACK_CONFIG: + input->cmd_info.dw2.bs.oor_range_m = msg_in->sack.oor_range; + input->cmd_info.dw2.bs.srp_range_n = msg_in->sack.srp_range; + break; + default: + ROCE_DEV_ERR(rdev, "Unsupport attr mask!\n"); + break; + } + + input->cmd_info.dw0.bs.attr_ext_mask = msg_in->attr_mask; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + input->cmd_info.dw1.value = cpu_to_be32(input->cmd_info.dw1.value); + input->cmd_info.dw2.value = cpu_to_be32(input->cmd_info.dw2.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int +roce5_send_cmd_set_qp_msnc(struct roce5_device *rdev, u32 qpn, + struct roce5_qp_hyper_extend *rqp_hyper_ext) +{ + int ret = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_init_msnc_s *input = NULL; + u64 bitmap_addr = 0; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_init_msnc_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_init_msnc_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_INIT_MSNC_QP); + + input->cmd_info.dw0.bs.sack_bitmap_size = + HYPER_ROCE_TX_SACK_BITMAP_SIZE; + input->cmd_info.dw0.bs.txmsn_entry_size = HYPER_ROCE_MSN_ENTRY_SIZE; + input->cmd_info.dw0.bs.txmsn_table_size = HYPER_ROCE_MSN_TABLE_SIZE; + input->cmd_info.dw0.bs.txmsn_table_gpa_h = + (rqp_hyper_ext->tx_msnc.dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & + 0xffffff; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + input->cmd_info.txmsn_table_gpa_l = + cpu_to_be32((u32)(rqp_hyper_ext->tx_msnc.dma_addr >> + ROCE5_RDMARC_GPA_L_SHIFT)); + + input->cmd_info.dw2.bs.rxmsn_entry_size0 = HYPER_ROCE_MSN_ENTRY_SIZE; + input->cmd_info.dw2.bs.rxmsn_table_size0 = HYPER_ROCE_MSN_TABLE_SIZE; + input->cmd_info.dw2.bs.rxmsn_table_gpa0_h = + (rqp_hyper_ext->rx_msnc.dma_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & + 0xffffff; + input->cmd_info.dw2.value = cpu_to_be32(input->cmd_info.dw2.value); + + input->cmd_info.rxmsn_table_gpa0_l = + cpu_to_be32((u32)(rqp_hyper_ext->rx_msnc.dma_addr >> + ROCE5_RDMARC_GPA_L_SHIFT)); + + bitmap_addr = (u64)rqp_hyper_ext->rx_msnc.dma_addr + + HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE * + HYPER_ROCE_RDMARC_ENTRY_SIZE_BYTES; + + input->cmd_info.dw4.bs.sack_bitmap_size = + HYPER_ROCE_RX_SACK_BITMAP_SIZE; + input->cmd_info.dw4.bs.rxmsn_entry_size1 = HYPER_ROCE_MSN_ENTRY_SIZE; + input->cmd_info.dw4.bs.rxmsn_table_size1 = HYPER_ROCE_MSN_TABLE_SIZE; + input->cmd_info.dw4.bs.rxmsn_table_gpa1_h = + (bitmap_addr >> ROCE5_RDMARC_GPA_H_SHIFT) & 0xffffff; + input->cmd_info.dw4.value = cpu_to_be32(input->cmd_info.dw4.value); + + input->cmd_info.rxmsn_table_gpa1_l = + cpu_to_be32((u32)(bitmap_addr >> ROCE5_RDMARC_GPA_L_SHIFT)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_INIT_MSNC_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int roce5_alloc_msnc(struct roce5_device *rdev, + struct roce5_qp_hyper_extend *rqp_hyper_ext) +{ + int ret; + ret = roce5_rdma_rdmarc_alloc(rdev->hwdev, + HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE, + &rqp_hyper_ext->tx_msnc); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdmarc, func_id(%u).", + rdev->glb_func_id); + return ret; + } + + ret = roce5_rdma_rdmarc_alloc( + rdev->hwdev, + HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE + + HYPER_ROCE_RXMSN_BITMAP_RDMARC_ALLOC_SIZE, + &rqp_hyper_ext->rx_msnc); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc rdma rdmarc, func_id(%u).", + rdev->glb_func_id); + roce5_rdma_rdmarc_free(rdev->hwdev, &rqp_hyper_ext->tx_msnc); + return ret; + } + + rqp_hyper_ext->msnc_en = true; + + return 0; +} + +static int32_t roce5_genl_set_qp_hyper_mode(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, + uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_hyper_set_in *msg_in = NULL; + struct roce5_qp *rqp = NULL; + struct roce5_qp_hyper_extend *rqp_hyper_ext = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_hyper_set_in) || out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_hyper_set_in *)in; + + if (msg_in->qpn == ROCE_QP_SMI_QP_NUM || + msg_in->qpn == ROCE_QP_GSI_QP_NUM) { + ROCE_DEV_ERR(rdev, "Invalid qpn(%u).", msg_in->qpn); + return -EINVAL; + } + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp."); + return -EINVAL; + } + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state >= IB_QPS_RTR) { + ROCE_DEV_ERR(rdev, "Invalid qp state, current state(%u).", + rqp->qp_state); + ret = -EINVAL; + goto out; + } + + ret = roce5_send_cmd_set_qp_hyper_mode(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + goto out; + } + + rqp_hyper_ext = (struct roce5_qp_hyper_extend *)roce5_qp_get_ext_buf( + rqp, ROCE_EXT_HYPER_ROCE); + if (rqp_hyper_ext == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to the buf of qp hyper extend, qpn(%u).", + rqp->qpn); + ret = -EINVAL; + goto out; + } + + rqp_hyper_ext->mode = msg_in->mode; + rqp_hyper_ext->sack_en = msg_in->sack_en; + + /* ROCE_HYPER_MODE_DISABLE indicates not enable out-of-order receive, not need allocate rdmarc resource; enable sack_en selective retransmission, must enable out-of-order receive */ + if (rqp_hyper_ext->mode != ROCE_HYPER_MODE_DISABLE) { + ret = roce5_alloc_msnc(rdev, rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to alloc msnc, ret(%d).", + ret); + goto out; + } + + ret = roce5_send_cmd_set_qp_msnc(rdev, msg_in->qpn, + rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + goto out; + } + } +out: + mutex_unlock(&rqp->mutex); + roce5_rqp_put(rqp); + return ret; +} + +static int32_t roce5_genl_set_default_multipath_config( + struct roce5_device *rdev, struct roce5_hyper_extend_config_in *msg_in) +{ + int ret = 0; + + msg_in->lb_mode = HYPER_ROCE_DEFAULT_LB_MODE; + msg_in->mp.flowlet_pkg_num = HYPER_ROCE_DEFAULT_MULTI_PKG_NUM; + msg_in->mp.path_num = HYPER_ROCE_DEFAULT_MULTI_PATH_NUM; + msg_in->mp.interval = HYPER_ROCE_DEFAULT_MULTI_INTERVAL; + msg_in->mp.path_rr_enable = HYPER_ROCE_DEFAULT_MULTI_PATHRR_ENABLE; + msg_in->mp.cc_mode = HYPER_ROCE_DEFAULT_MULTI_CC_MODE; + + msg_in->attr_mask = IBV_QP_ATTR_EXTEND_LB_MODE; + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + + msg_in->attr_mask = IBV_QP_ATTR_EXTEND_MP_CONFIG; + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + + return ret; +} + +static int32_t roce5_modify_hyper_config( + struct roce5_device *rdev, struct roce5_hyper_extend_config_in *msg_in, + struct roce5_qp_hyper_extend *rqp_hyper_ext, struct roce5_qp *rqp) +{ + int ret = 0; + + if (msg_in->attr_mask == IBV_QP_ATTR_EXTEND_HYROCE_FEATURE) { + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + + rqp_hyper_ext->mode = (u8)msg_in->feature.type; + rqp_hyper_ext->version = (u8)msg_in->feature.version; + rqp_hyper_ext->sack_en = (u8)msg_in->feature.sack_enable; + + /* ROCE_HYPER_MODE_DISABLE indicates not enable out-of-order receive, not need allocate rdmarc resource; enable sack_en selective retransmission, must enable out-of-order receive */ + if (rqp_hyper_ext->mode != ROCE_HYPER_MODE_DISABLE) { + ret = roce5_alloc_msnc(rdev, rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to alloc msnc, ret(%d).", + ret); + return ret; + } + + ret = roce5_send_cmd_set_qp_msnc(rdev, msg_in->qpn, + rqp_hyper_ext); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to send CMDQ, ret(%d)", + ret); + return ret; + } + } + + ret = roce5_genl_set_default_multipath_config(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to set default mp config, ret(%d).", + ret); + return ret; + } + } else { + if (rqp_hyper_ext->mode == ROCE_HYPER_MODE_DISABLE) { + ROCE_DEV_ERR(rdev, + "Hyper_roce is disable, current mode(%u).", + rqp_hyper_ext->mode); + ret = -EINVAL; + return ret; + } + + ret = roce5_send_cmd_modify_qp_extend(rdev, msg_in); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send CMDQ, ret(%d)", ret); + return ret; + } + } + + return ret; +} + +static int32_t roce5_genl_modify_qp_extend(struct roce5_device *rdev, void *in, + uint16_t in_len, void *out, + uint16_t out_len, uint16_t *out_size) +{ + int32_t ret = 0; + struct roce5_hyper_extend_config_in *msg_in = NULL; + struct roce5_qp *rqp = NULL; + struct roce5_qp_hyper_extend *rqp_hyper_ext = NULL; + + if (in == NULL || in_len == 0 || + in_len < sizeof(struct roce5_hyper_extend_config_in) || + out_size == NULL) { + ROCE_DEV_ERR(rdev, + "Invalid param, in:%d, in_len:%u, out_size:%d", + !!in, in_len, !!out_size); + return -EINVAL; + } + + msg_in = (struct roce5_hyper_extend_config_in *)in; + + if (msg_in->qpn == ROCE_QP_SMI_QP_NUM || + msg_in->qpn == ROCE_QP_GSI_QP_NUM) { + ROCE_DEV_ERR(rdev, "Invalid qpn(%u).", msg_in->qpn); + return -EINVAL; + } + + rqp = roce5_get_rqp_by_qpn(rdev, msg_in->qpn); + if (rqp == NULL) { + ROCE_DEV_ERR(rdev, "Failed to look up rqp."); + return -EINVAL; + } + + mutex_lock(&rqp->mutex); + + if (rqp->qp_state >= IB_QPS_RTR) { + ROCE_DEV_ERR(rdev, "Invalid qp state, current state(%u).", + rqp->qp_state); + ret = -EINVAL; + goto out; + } + + rqp_hyper_ext = (struct roce5_qp_hyper_extend *)roce5_qp_get_ext_buf( + rqp, ROCE_EXT_HYPER_ROCE); + if (rqp_hyper_ext == NULL) { + ROCE_DEV_ERR(rdev, + "Failed to the buf of qp hyper extend, qpn(%u).", + rqp->qpn); + ret = -EINVAL; + goto out; + } + + ret = roce5_modify_hyper_config(rdev, msg_in, rqp_hyper_ext, rqp); + if (ret) { + ROCE_DEV_ERR(rdev, "Failed to modify hyper roce config."); + } +out: + mutex_unlock(&rqp->mutex); + roce5_rqp_put(rqp); + return ret; +} + +static roce5_netlink_cmd_process g_cmd_process[] = { + { USRNL_ROCE_CMD_HYPER_QP_SET, roce5_genl_set_qp_hyper_mode }, + { USRNL_ROCE_CMD_MODIFY_QP_EXTEND, roce5_genl_modify_qp_extend }, +}; + +int roce5_hyper_netlink_init(void) +{ + int ret; + + ret = roce5_netlink_register_cbs( + g_cmd_process, + sizeof(g_cmd_process) / sizeof(roce5_netlink_cmd_process)); + if (ret != 0) { + ROCE_ERR( + "Failed to register hyper roce netlink callbacks, ret(%d).", + ret); + return ret; + } + + return 0; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h new file mode 100644 index 000000000..27de1b070 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_netlink.h @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_NETLINK_H +#define HYPER_ROCE_NETLINK_H + +#include <linux/types.h> + +#define HYPER_ROCE_DEFAULT_LB_MODE 1 +#define HYPER_ROCE_DEFAULT_MULTI_PKG_NUM 4 +#define HYPER_ROCE_DEFAULT_MULTI_PATH_NUM 4 +#define HYPER_ROCE_DEFAULT_MULTI_INTERVAL 1 +#define HYPER_ROCE_DEFAULT_MULTI_PATHRR_ENABLE 0 +#define HYPER_ROCE_DEFAULT_MULTI_CC_MODE 2 + +struct roce5_hyper_set_in { + uint32_t qpn; + uint8_t mode; /* @see enum roce_hyper_mode */ + uint8_t sack_en; + uint8_t rsvd[2]; +}; + +int roce5_hyper_netlink_init(void); + +enum ibv_qp_attr_extend_mask { + IBV_QP_ATTR_EXTEND_UDP_SRC_PORT = 1 << 0, /* source UDP port number */ + IBV_QP_ATTR_EXTEND_HYROCE_FEATURE = 1 << 1, /* high-order RoCE of feature */ + IBV_QP_ATTR_EXTEND_LB_MODE = 1 << 2, /* load balance mode */ + IBV_QP_ATTR_EXTEND_MP_CONFIG = 1 << 3, /* Multi-Path multipath mode configuration */ + IBV_QP_ATTR_EXTEND_AR_CONFIG = + 1 << 4, /* Adaptive-Routing multipath mode configuration */ + IBV_QP_ATTR_EXTEND_SACK_CONFIG = + 1 << 5, /* Selective Ack selective retransmission parameters configuration */ +}; + +/* high-order RoCE feature configuration structure */ +struct ibv_hyroce_feature { + uint32_t type; /* high-order RoCE type, value range: enum ibv_hyroce_feature_type */ + uint32_t version; /* high-order RoCE version, value range: enum ibv_hyroce_feature_version */ + uint32_t sack_enable; /* selective retransmission switch, 0- disable, 1- enable */ + + uint32_t resv[32]; /* reserved field, used for not to extension feature */ +}; + +/* multipath (Multi-Path) configuration parameters structure */ +struct ibv_mp_config { + uint32_t flowlet_pkg_num; /* each sub-flow (flowlet) of packet count, used for flow splitting */ + uint32_t path_num; /* multipath of path count */ + uint32_t interval; /* UDP port number increment interval, used for distinguish different path */ + uint32_t path_rr_enable; /* path whether supports poll (RR) select network port, 0- not support, 1- supports */ + uint32_t cc_mode; + + uint32_t resv[32]; /* reserved field, used for not to extension */ +}; + +/* adaptive path by (Adaptive Routing) configuration structure */ +struct ibv_ar_config { + uint32_t port_rr_enable; /* whether enable network port side port poll (RR) per-packet function, 0- disable, 1- enable */ + + uint32_t resv[32]; /* reserved field, used for not to extension */ +}; + +/* selective retransmission (SACK - Selective Ack) configuration structure */ +struct ibv_sack_config { + uint32_t srp_range; /* TX(send) direction maximum repass packet count */ + uint32_t oor_range; /* RX(receive) direction out-of-order repass window of packet count */ + + uint32_t resv[32]; /* reserved field, used for not to extension */ +}; + +struct roce5_hyper_extend_config_in { + uint32_t qpn; + uint32_t udp_src_port; /* source UDP port number */ + struct ibv_hyroce_feature feature; /* high-order RoCE feature */ + uint32_t lb_mode; /* load balance mode, value range: enum ibv_lb_mode type */ + struct ibv_mp_config mp; /* multipath (Multi-Path) configuration */ + struct ibv_ar_config ar; /* adaptive path by (Adaptive Routing) configuration */ + struct ibv_sack_config sack; /* selective retransmission (SACK) configuration */ + enum ibv_qp_attr_extend_mask attr_mask; +}; + +#endif /* HYPER_ROCE_NETLINK_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c new file mode 100644 index 000000000..1cc831f16 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.c @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "roce_qp.h" +#include "roce_restore.h" +#include "hyper_roce_common.h" +#include "hyper_roce_qp.h" + +int roce5_hyper_qp_init(void *parent, void *ext_data) +{ + struct roce5_qp_hyper_extend *hyper_qp_ext = + (struct roce5_qp_hyper_extend *)ext_data; + hyper_qp_ext->mode = ROCE_HYPER_MODE_DISABLE; + hyper_qp_ext->sack_en = false; + return 0; +} + +void roce5_hyper_qp_cleanup(void *parent, void *ext_data) +{ + struct roce5_qp *rqp = (struct roce5_qp *)parent; + struct roce5_device *rdev = to_roce5_dev(rqp->ibqp.device); + struct roce5_qp_hyper_extend *hyper_qp_ext = + (struct roce5_qp_hyper_extend *)ext_data; + + if (hyper_qp_ext->msnc_en) { + roce5_rdma_rdmarc_free(rdev->hwdev, &(hyper_qp_ext->tx_msnc)); + roce5_rdma_rdmarc_free(rdev->hwdev, &(hyper_qp_ext->rx_msnc)); + } +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h new file mode 100644 index 000000000..52c38f0a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/hyper_roce_qp.h @@ -0,0 +1,35 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_QP_H +#define HYPER_ROCE_QP_H + +#include <linux/types.h> +#include "base_type.h" +#include "hinic5_rdma.h" + +/* Entry_size for TXMSN/RXMSN Table 00: 32B 01: 64B Others: reserved. */ +#define HYPER_ROCE_MSN_ENTRY_SIZE 0u +/* Entry number for TXMSN/RXMSN Table 00: 256 01: 512 10: 1024 11: 2048 Others: reserved. */ +#define HYPER_ROCE_MSN_TABLE_SIZE 0u +/* 000: 128bit 001: 256bit 010: 512bit 011: 1024bit 100: 2048bit Others: reserved. */ +#define HYPER_ROCE_TX_SACK_BITMAP_SIZE 0u +/* 011: 1024bit 100: 2048bit Others: reserved. */ +#define HYPER_ROCE_RX_SACK_BITMAP_SIZE 3u +#define HYPER_ROCE_MSN_RDMARC_ALLOC_SIZE \ + (256u << (HYPER_ROCE_MSN_ENTRY_SIZE & 0x3)) +#define HYPER_ROCE_RXMSN_BITMAP_RDMARC_ALLOC_SIZE (4) + +struct roce5_qp_hyper_extend { + u8 mode; /* @see enum roce_hyper_mode */ + u8 version; + u8 sack_en; + struct rdma_rdmarc tx_msnc; + struct rdma_rdmarc rx_msnc; + bool msnc_en; +}; + +int roce5_hyper_qp_init(void *parent, void *ext_data); +void roce5_hyper_qp_cleanup(void *parent, void *ext_data); + +#endif /* HYPER_ROCE_QP_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c new file mode 100644 index 000000000..fe4e5cf64 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.c @@ -0,0 +1,60 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "roce.h" +#include "roce_dfx.h" +#include "hyper_roce_tool_pktdrop.h" +#include "hyper_roce_tool.h" + +enum roce5_hyper_cmds { + ROCE_HYPER_CMD_START = 1, + ROCE_HYPER_CMD_START_DROP_QP, + ROCE_HYPER_CMD_STOP_DROP_QP, + ROCE_HYPER_CMD_MAX +}; + +const roce5_adm_func_t hyper_roce5_adm_func[] = { + [ROCE_HYPER_CMD_START_DROP_QP] = roce5_hyper_tool_start_drop_qp, + [ROCE_HYPER_CMD_STOP_DROP_QP] = roce5_hyper_tool_stop_drop_qp, +}; + +int roce5_hyper_tool_dispatch(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct roce5_hyper_cmd_head *head = NULL; + const void *data_ptr = NULL; + u32 data_len; + int ret; + + if (in_size < ROCE_HYPER_CMD_HEAD_SIZE) { + ROCE_DEV_ERR(rdev, "Invalid command length(%d).", in_size); + return -EINVAL; + } + + data_ptr = (void *)((char *)buf_in + ROCE_HYPER_CMD_HEAD_SIZE); + data_len = in_size - ROCE_HYPER_CMD_HEAD_SIZE; + head = data_ptr; + + if (head->sub_cmd < ROCE_HYPER_CMD_START || + head->sub_cmd >= ROCE_HYPER_CMD_MAX) { + ROCE_DEV_ERR(rdev, "Unknown hyper roce command(%d).", + head->sub_cmd); + return -EINVAL; + } + + if (hyper_roce5_adm_func[head->sub_cmd] == NULL) { + ROCE_DEV_ERR(rdev, "The callback of cmd(%d) is NULL.", + head->sub_cmd); + return -EINVAL; + } + + ret = hyper_roce5_adm_func[head->sub_cmd](rdev, buf_in, in_size, + buf_out, out_size); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to execute the callback of cmd(%d)", + head->sub_cmd); + return ret; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h new file mode 100644 index 000000000..2438bbb78 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool.h @@ -0,0 +1,21 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_TOOL_H +#define HYPER_ROCE_TOOL_H + +#include "base_type.h" +#include "roce.h" + +#define ROCE_CMD_HYPER 0x220 + +struct roce5_hyper_cmd_head { + u32 sub_cmd; +}; + +#define ROCE_HYPER_CMD_HEAD_SIZE sizeof(struct roce5_hyper_cmd_head) + +int roce5_hyper_tool_dispatch(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* HYPER_ROCE_TOOL_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c new file mode 100644 index 000000000..7d305e069 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.c @@ -0,0 +1,142 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/types.h> +#include "hinic5_rdma.h" +#include "rdma_comp.h" +#include "roce_npu_cmd.h" +#include "roce_hyper_npu_cmd.h" +#include "comm_defs.h" + +#include "roce.h" +#include "roce_qp.h" +#include "roce_qp_ext.h" +#include "roce_netlink.h" +#include "roce_cqm_cmd.h" +#include "hyper_roce_tool_pktdrop.h" + +int roce5_hyper_tool_start_drop_qp(struct roce5_device *rdev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret = 0; + u32 drop_pkt_ratio = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_modify_qp_drop_percent_s *input = NULL; + struct pktdrop_cmd_inbuf *in = NULL; + + if (in_size < sizeof(struct pktdrop_cmd_inbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct pktdrop_cmd_inbuf)); + return -EINVAL; + } + + if (*out_size < sizeof(union pktdrop_outbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, (int)sizeof(union pktdrop_outbuf)); + return -ENOMEM; + } + + in = (struct pktdrop_cmd_inbuf *)buf_in; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_qp_drop_percent_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + drop_pkt_ratio = in->data.attr.dw15.bs.drop_pkt_ratio; + input = (roce_uni_cmd_modify_qp_drop_percent_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_MODIFY_QP_DROP_PERCENT); + + input->cmd_info.dw0.bs.drop_precent = drop_pkt_ratio; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int roce5_hyper_tool_stop_drop_qp(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret = 0; + u32 drop_pkt_ratio = 0; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + roce_uni_cmd_modify_qp_drop_percent_s *input = NULL; + struct pktdrop_cmd_inbuf *in = NULL; + + if (in_size < sizeof(struct pktdrop_cmd_inbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct pktdrop_cmd_inbuf)); + return -EINVAL; + } + + if (*out_size < sizeof(union pktdrop_outbuf)) { + ROCE_DEV_ERR( + rdev, + "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, (int)sizeof(union pktdrop_outbuf)); + return -ENOMEM; + } + + in = (struct pktdrop_cmd_inbuf *)buf_in; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_modify_qp_drop_percent_s), NULL, 0); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to alloc cqm_cmd_inoutbuf, func_id(%d), ret(%d)", + rdev->glb_func_id, ret); + return -ENOMEM; + } + + input = (roce_uni_cmd_modify_qp_drop_percent_s *)cqm_cmd_inbuf->buf; + input->com.index = cpu_to_be32(in->qpn); + input->com.dw0.bs.cmd_bitmask = cpu_to_be16(VERBS_CMD_TYPE_QP_BITMASK); + input->com.opt = cpu_to_be32(RoCE_CMD_MODIFY_QP_DROP_PERCENT); + + input->cmd_info.dw0.bs.drop_precent = drop_pkt_ratio; + input->cmd_info.dw0.value = cpu_to_be32(input->cmd_info.dw0.value); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + RoCE_CMD_HYPER_ROCE, cqm_cmd_inbuf, NULL, NULL, + ROCE_CMD_TIME_CLASS_A, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR( + rdev, + "Failed to send RoCE_CMD_ENABLE_HYPER_ROCE_QP command, ret(%d), func_id(%d)", + ret, rdev->glb_func_id); + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h new file mode 100644 index 000000000..4d3dbd1d5 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/extern/hyper_roce/tool/hyper_roce_tool_pktdrop.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef HYPER_ROCE_TOOL_PKTDROP_H +#define HYPER_ROCE_TOOL_PKTDROP_H +#include "hyper_roce_pub_cmd.h" +#include "roce.h" + +struct pktdrop_cmd_inbuf { + u32 cmd_type; + u32 hyper_roce_sub_cmd; + u32 mode; + u32 qpn; + u32 drop_level; + struct pktdrop_l2d_tbl_dfx data; +}; + +int roce5_hyper_tool_start_drop_qp(struct roce5_device *rdev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int roce5_hyper_tool_stop_drop_qp(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce.h new file mode 100644 index 000000000..3fbc3343b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce.h @@ -0,0 +1,665 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_H +#define ROCE_H + +#include <linux/types.h> +#include <linux/io-mapping.h> +#include <linux/list.h> + +#include <rdma/ib_verbs.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> +#include "roce_verbs_compat.h" + +#include "hinic5_hw.h" +#include "hinic5_crm.h" +#include "hinic5_hw_cfg.h" +#include "hinic5_lld.h" +#include "hinic5_cqm.h" +#include "hinic5_rdma.h" +#include "nic_cfg_comm.h" +#include "hmm_common.h" + +#include "roce_infra.h" +#include "roce_db.h" +#include "roce_sysfs.h" + +#include "roce_uld_kernel_api.h" +#include "roce_npu_cmd_defs.h" +#include "roce_npu_cmd_ext_defs.h" + +#define HIROCE5_DRV_NAME "roce5_drv" +#define HIROCE5_DRV_AUTHOR "Huawei Technologies CO., Ltd" +#define HIROCE5_DRV_DESC \ + "Huawei(R) Intelligent Network Interface Card, RoCE Driver" +#define HIROCE5_DRV_VERSION GLOBAL_VERSION_STR + +#define RDMA_DRIVER_HRN 8395 +#define ROCE_IB_UVERBS_ABI_VERSION 1 +#define ROCE_ULD_DEV_NAME_LEN 16 + +#define MAX_CEQ_NEED 256 + +#define DEV_ADDR_FIRST_BYTE_VAL_MASK 2 + +#define ROCE_NODE_DESC_LEN 5 + +#define ROCE_SQ_WQEBB_SIZE 64 + +#define ROCE_GID_LEN 16 + +#define ROCE_PCI_CFG_REGS_BAR0 0 +#define ROCE_PCI_CFG_REGS_BAR3 3 + +#define DEFAULT_ROCE_DEV_NODE_PRI 0666 + +#define ROCE_MAX_PORT_NUM 8 + +#ifndef BIG_ENDIAN +#define BIG_ENDIAN 0x4321 +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN 0x1234 +#endif + +#ifndef BYTE_ORDER +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#define MAX_ROCE_DEV (28 * 4) + +#define CHECK_1LAYER_IS_NULL(ptr) ((ptr) == NULL) +#define CHECK_2LAYER_IS_NULL(ptr, l2_member) \ + (CHECK_1LAYER_IS_NULL(ptr) || (((ptr)->l2_member) == NULL)) + +static inline bool roce5_func_is_vf(void *hwdev) +{ + return hinic5_func_type(hwdev) == TYPE_VF; +} + +static inline bool roce5_func_is_pf(void *hwdev) +{ + return hinic5_func_type(hwdev) == TYPE_PF; +} + +enum { + ROCE5_1_PORT_NUM = 1, //EVB single die 形mode + ROCE5_2_PORT_NUM = 2, + ROCE5_4_PORT_NUM = 4 +}; + +enum { + ROCE5_25G_PORT_SPEED = 25, + ROCE5_100G_PORT_SPEED = 100, + ROCE5_200G_PORT_SPEED = 200, + ROCE5_400G_PORT_SPEED = + 201, //because MPU bit width not 够衍generate of custom enum, refer to port_speed_mode_e + ROCE5_800G_PORT_SPEED = 202 +}; + +enum { + ROCE5_INVALID_HCA = -1, + ROCE5_2_100G_HCA = 0, + ROCE5_4_25G_HCA = 1, + ROCE5_2_25G_HCA = 2, + ROCE5_2_200G_HCA = 3, + ROCE5_1_100G_HCA = 4, + ROCE5_2_400G_HCA = 5, + ROCE5_4_200G_HCA = 6 +}; + +enum ROCE5_100G_BW_PARAM_E { + ROCE5_100G_CIR = 46500000, + ROCE5_100G_PIR = 52500000, + ROCE5_100G_CNP = 100 +}; + +enum ROCE5_25G_BW_PARAM_E { + ROCE5_25G_CIR = 23200000, + ROCE5_25G_PIR = 25500000, + ROCE5_25G_CNP = 3 +}; + +enum roce_bitshift_e { + BYTES_TO_2B_SHIFT = 1, + BYTES_TO_4B_SHIFT, + BYTES_TO_8B_SHIFT, + BYTES_TO_16B_SHIFT, + BYTES_TO_32B_SHIFT +}; + +/* BIG/LITTLE ENGIAN switch */ +#ifdef HW_CONVERT_ENDIAN +#define roce5_convert_be32(val) (val) +#define roce5_convert_cpu32(val) (val) +#else +#define roce5_convert_be32(val) cpu_to_be32(val) +#define roce5_convert_cpu32(val) be32_to_cpu(val) +#endif + +enum roce5_aeq_type { + /* ofed err */ + OFED_ET_PATH_MIG = 0, + OFED_ET_COMM_EST, + OFED_ET_SQ_DRAINED, + OFED_ET_SRQ_QP_LAST_WQE, + OFED_ET_WQ_CATAS_ERR, + + OFED_ET_PATH_MIG_FAILED = 5, + OFED_ET_WQ_INVAL_REQ_ERR, + OFED_ET_WQ_ACCESS_ERR, + OFED_ET_CQ_ERR, + OFED_ET_SRQ_LIMIT, + OFED_ET_SRQ_CATAS_ERR, + + /* non ofed err */ + NON_OFED_ET_QPC_LOOKUP_ERR = 11, + NON_OFED_ET_OTHER_TYPE_ERR, + + /* non ofed extend err */ + NON_OFED_ET_DEBUG_ERR = 32, + + /* NOF AA err */ + OFED_NOF_AA_QP_DISCONNECT = 64, + OFED_NOF_AA_MASTER_CHANGE, + + INVAL_ET_ERR +}; + +enum { + ROCE_CMD_TIME_CLASS_A = 30000, //30s, TODO: use ASIC card after need modify is 3s + ROCE_CMD_TIME_CLASS_B = 40000, //40s, TODO: use ASIC card after need modify is 4s + ROCE_CMD_TIME_CLASS_C = 50000 //50s, TODO: use ASIC card after need modify is 5s +}; + +enum roce5_ctrl_status { ROCE5_PORT_EVENT = BIT(0) }; + +#define ROCE_DEFAULT_PORT_NUM 1 + +#define ROCE_UVERBS_CMD_BASE_MASK \ + ((1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_AH) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_AH) | \ + (1ULL << IB_USER_VERBS_CMD_REG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_RESIZE_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | \ + (1ULL << IB_USER_VERBS_CMD_CREATE_XSRQ) | \ + (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_CLOSE_XRCD) | \ + (1ULL << IB_USER_VERBS_CMD_OPEN_QP)) + +#define ROCE_UVERBS_EXT_CMD_MASK \ + ((1ULL << IB_USER_VERBS_EX_CMD_MODIFY_CQ) | \ + (1ULL << IB_USER_VERBS_EX_CMD_MODIFY_QP)) + +enum roce5_load_balance_mode_e { + ROCE_LB_MODE_0 = 0, + ROCE_LB_MODE_1, + ROCE_LB_MODE_2, + ROCE_LB_MODE_N, +}; + +enum push_ofed_device_status { + ROCE_DEV_STATUS_NORMAL = 0, + ROCE_DEV_STATUS_CMDQ_TIMEOUT +}; + +enum roce5_func_state { ROCE_FUNC_DISABLE = 0, ROCE_FUNC_ENABLE }; + +enum { + ROCE_Cl_TYPE_QPC = 0x0, /* cl_start: 0x0, cl_end: 0x7f, cl_size: 0 */ + ROCE_CL_TYPE_MPT = 0x1, /* cl_start: 0x150, cl_end: 0x15f, cl_size: 0 */ + ROCE_CL_TYPE_SQ_WQE = + 0x2, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_RQ_WQE = + 0x3, /* cl_start: 0x80, cl_end: 0xff, cl_size: 0 */ + ROCE_CL_TYPE_CQC_SRQC = + 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_RDMARC = + 0x5, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ + ROCE_CL_TYPE_CMTT = + 0x6, /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ + ROCE_CL_TYPE_DMTT = 0x7 /* cl_start: 0x100, cl_end: 0x11f, cl_size: 0 */ +}; + +#define XRC_CONTAINER_FLAG ((u32)(1L << 20)) + +struct roce5_buf_list { + void *buf; + dma_addr_t map; +}; + +struct roce5_buf { + struct roce5_buf_list direct; + struct roce5_buf_list *page_list; + int nbufs; + + int npages; + int page_shift; +}; + +struct shadow_hw_db { + u32 page_idx; + u32 refcnt; +}; + +struct roce5_ucontext { + struct ib_ucontext ibucontext; + void __iomem *db_map; + void __iomem *dwqe_map; + u64 db_dma_addr; + u64 dwqe_dma_addr; + struct list_head db_page_list; + struct mutex db_page_mutex; + + struct shadow_hw_db shadow_db_ctx; +}; + +struct roce5_qp_cnt { + struct mutex cur_qps_mutex; + u32 alloc_qp_cnt; + u32 del_qp_cnt; +}; + +struct roce5_dev_hw_info { + int config_num_ports; /* Number of ports from configuration file */ + int hca_type; /* HCA version: 4x25G or 2x100G */ + u8 phy_port; + u8 ep_id; /* EP ID */ + u8 cpu_endian; + u8 rsvd; + + bool is_vf; +}; + +struct roce5_dev_cfg_info { + u8 scence_id; /* load scenes ID as aa_en */ + u8 lb_en; /* load balance enable */ + u8 lb_mode; /* load balance mode */ + struct { + u8 pcie_atomic_en : 1; + u8 rsvd : 7; + }; + + u8 srq_container_en; + u8 srq_container_mode; + u8 xrc_srq_container_mode; + u8 warn_th; + + u8 fake_en; + u8 page_bit; + u8 pf_start_bit; + u8 pf_end_bit; + + u8 port_num; /* cfg data port num */ + u8 host_num; /* cfg data host num */ + u8 rsvd1[2]; + roce_get_chip_ability_s feature_cap; +}; +#define ROCE_LB_QPN_MAX \ + 1 //less qp, same a smf on of qp will distribution to same a icq queue, because this decrease low lb of qp count +struct roce5_qp_lb_scqn_ctrl { + struct mutex qp_lb_scqn_ctrl_mutex; + //record device in lb of qpn + u32 qp[ROCE_LB_QPN_MAX]; +}; + +struct roce5_cq_ctrl { + struct mutex cq_ctrl_mutex; + u8 alloc_cq_cnt; + u8 high_perf_cq_cnt; + u8 rsvd_cq_step; + unsigned long cq_bitmap; +}; + +enum roce5_device_type { ROCE_DEV_TYPE_UNKNOWN, ROCE_DEV_TYPE_HI1825_V100 }; + +enum roce5_perf_cq_step { ROCE_1825V100_STEP = 8 }; + +#define ROCE_IS_DEV_1825V100(rdev) \ + (((rdev)->device_type) == ROCE_DEV_TYPE_HI1825_V100) + +#ifdef __ROCE_DFX__ +struct roce5_list { + struct list_head node; + struct mutex node_lock; +}; +struct roce5_device_res { + struct roce5_list mr_list; +}; +#endif //__ROCE_DFX__ + +//uld service can extension structure space +#define ROCE_DEVICE_ULD_DEF_LEN 64 +#define ROCE_DEVICE_MAGIC 0xaabacada +struct roce5_device_uld_def { + u8 self_def[ROCE_DEVICE_ULD_DEF_LEN]; +}; + +struct roce5_device { + struct ib_device ib_dev; + struct hinic5_lld_dev *lld_dev; + struct net_device *ndev; + struct device *dev; + void *hwdev; + void *hwdev_hdl; + //struct dev_version_info dev_ver; /*version info */ + struct hinic5_board_info board_info; + + void __iomem *kernel_db_map; + void __iomem *kernel_dwqe_map; + + spinlock_t node_desc_lock; + struct mutex cap_mask_mutex; + struct rdma_service_cap rdma_cap; + + u8 mac[6]; /* Mac addr. */ + u16 glb_func_id; + unsigned long status; + int ceq_num; + int ceqn[MAX_CEQ_NEED]; + int try_times; + int destory_flow_timeout; + bool ib_active; + enum roce5_device_type device_type; + + u8 group_rc_cos; + u8 group_ud_cos; + u8 group_xrc_cos; + u8 mpt_xid_mode; + struct roce5_qp_lb_scqn_ctrl qp_lb_scqn_ctrl; + struct roce5_dev_hw_info hw_info; /* Hw info read from nic/pcie */ + struct roce5_dev_cfg_info cfg_info; /* Cfg data info read from MPU */ + +#ifdef ROCE_BONDING_EN + enum ib_port_state port_state; + struct roce5_bond_device *bond_dev; + atomic_t next_port; +#endif + + struct mutex mac_vlan_mutex; + struct list_head mac_vlan_list_head; + struct net_device **gid_dev; + + struct roce5_ecn_ctx ecn_ctx; + struct roce5_dfx_ctx dfx_ctx; + + struct roce5_qp_cnt qp_cnt; + struct roce5_cq_ctrl cqn_ctrl; + + struct srcu_struct mr_srcu; + atomic_t num_prefetch; + struct completion comp_prefetch; + enum push_ofed_device_status dev_status_to_ofed; + + spinlock_t reset_flow_resource_lock; + struct list_head qp_list; + + void *fake_data_buf; + dma_addr_t fake_data_page_addr; + void *pri_roce_dev + [ROCE_MAX_PORT_NUM]; //mailbox, ppf store all pf's roce device, to improve the effiency. + int hca_vf_mode; //HCA uses mode to init MAC/IP vf table + bool local_en; //STUB local + bool full_qp_spec; + bool huge_page_en; + bool is_vroce; + u8 cc_param_version; + u8 shadow; + u8 rsvd[2]; + u32 shadow_vfid; + u16 shadow_vf_num; + + rwlock_t rwlock_func_tbl_bond; + u32 func_tbl_bond; + +#ifdef __ROCE_DFX__ + struct roce5_device_res *device_res; + struct roce5_ucontext *ucontext; +#endif //__ROCE_DFX__ + + struct roce5_device_uld_def uld_def; + u32 rdev_magic; /* used for uld check, preventstop踩踏 */ +}; + +int roce5_db_map_user(struct roce5_ucontext *context, unsigned long virt, + struct roce5_db *db, u8 shadow_db); +void roce5_db_unmap_user(struct roce5_ucontext *context, struct roce5_db *db, + u8 shadow_db); + +int roce5_buf_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct tag_cqm_buf *buf); +int roce5_umem_write_mtt(struct roce5_device *rdev, struct mtt *mtt, + struct ib_umem *umem, bool use_huge); + +int roce5_query_device(struct ib_device *ibdev, struct ib_device_attr *props, + struct ib_udata *uhw); + +int roce5_query_port_compat(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props); + +int roce5_query_port(struct ib_device *ibdev, u32 port, + struct ib_port_attr *props); + +enum rdma_link_layer roce5_port_link_layer_compat(struct ib_device *ibdev, + u8 port_num); + +enum rdma_link_layer roce5_port_link_layer(struct ib_device *ibdev, + u32 port_num); + +int roce5_query_gid_compat(struct ib_device *ibdev, u8 port, int index, + union ib_gid *gid); + +int roce5_query_gid(struct ib_device *ibdev, u32 port, int index, + union ib_gid *gid); + +int roce5_query_pkey_compat(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey); + +int roce5_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); + +int roce5_modify_device(struct ib_device *ibdev, int mask, + struct ib_device_modify *props); + +int roce5_modify_port_compat(struct ib_device *ibdev, u8 port, int mask, + struct ib_port_modify *props); + +int roce5_modify_port(struct ib_device *ibdev, u32 port, int mask, + struct ib_port_modify *props); + +int roce5_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma); + +int roce5_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); + +int roce5_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); + +int roce5_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); + +int roce5_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int roce5_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +void roce5_dealloc_ucontext(struct ib_ucontext *ibcontext); + +int roce5_alloc_ucontext(struct ib_ucontext *ibucontext, + struct ib_udata *udata); + +int roce5_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); + +int roce5_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); +int roce5_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); + +int roce5_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); + +int roce5_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int roce5_modify_cq(struct ib_cq *ibcq, u16 cq_count, u16 cq_period); + +int roce5_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); + +int roce5_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); + +int roce5_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); + +struct ib_qp *roce5_create_qp_compat(struct ib_pd *ibpd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata); + +int roce5_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); + +int roce5_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); + +int roce5_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata); + +int roce5_destroy_ah(struct ib_ah *ibah, u32 flags); + +int roce5_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + +int roce5_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + +int roce5_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); + +int roce5_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); + +struct ib_mr *roce5_get_dma_mr(struct ib_pd *ibpd, int access); +struct ib_mr *roce5_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, + u32 max_num_sg); +struct ib_mr *roce5_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, + u64 virt_addr, int access, + struct ib_udata *udata); + +int roce5_map_kernel_frmr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); + +int roce5_dealloc_mw(struct ib_mw *ibmw); + +int roce5_port_immutable_compat(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); +int roce5_port_immutable(struct ib_device *ibdev, u32 port_num, + struct ib_port_immutable *immutable); +int roce5_query_device_status(struct ib_device *ibdev, int *dev_status); + +int roce5_init_sysfs(struct roce5_device *rdev); +void roce5_remove_sysfs(struct roce5_device *rdev); + +struct net_device *roce5_ib_get_netdev_compat(struct ib_device *ibdev, + u8 port_num); +struct net_device *roce5_ib_get_netdev(struct ib_device *ibdev, u32 port_num); + +void roce5_async_event(void *svc_hd, u8 event_type, u8 *val); +u8 roce5_async_event_level(void *svc_hd, u8 event_type, u8 *val); +void roce5_ifconfig_up_down_event_report(struct roce5_device *rdev, + u8 net_event); +int roce5_register_netdev_notifier(void); +void roce5_unregister_netdev_notifier(void); +void roce5_clean_vlan_device_mac(struct roce5_device *rdev); +void roce5_clean_real_device_mac(struct roce5_device *rdev); + +int roce5_ib_add_gid(const struct ib_gid_attr *attr, + __always_unused void **context); + +int roce5_ib_del_gid(const struct ib_gid_attr *attr, + __always_unused void **context); + +/* OFA - OpenFabrics Alliance */ +struct ib_pd *roce5_alloc_pd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata); + +int roce5_dealloc_pd_ofa(struct ib_pd *ibpd); + +struct ib_srq *roce5_create_srq_ofa(struct ib_pd *ibpd, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata); + +struct ib_ucontext *roce5_alloc_ucontext_ofa(struct ib_device *ibdev, + struct ib_udata *udata); + +int roce5_dealloc_ucontext_ofa(struct ib_ucontext *ibcontext); + +struct ib_xrcd *roce5_alloc_xrcd_ofa(struct ib_device *ibdev, + struct ib_ucontext *ibcontext, + struct ib_udata *udata); + +int roce5_dealloc_xrcd_ofa(struct ib_xrcd *ibxrcd); + +int roce5_destroy_srq_ofa(struct ib_srq *ibsrq); + +int roce5_post_srq_recv_ofa(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +int roce5_post_recv_ofa(struct ib_qp *ibqp, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +struct ib_cq *roce5_create_cq_ofa(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *ibcontext, + struct ib_udata *udata); + +int roce5_destroy_cq_ofa(struct ib_cq *ibcq); + +struct ib_ah *roce5_create_ah_ofa(struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata); + +int roce5_destroy_ah_ofa(struct ib_ah *ibah); + +struct ib_mw *roce5_alloc_mw_ofa(struct ib_pd *ibpd, enum ib_mw_type type, + struct ib_udata *udata); + +int roce5_dealloc_mw_ofa(struct ib_mw *ibmw); + +int roce5_dereg_mr_ofa(struct ib_mr *ibmr); + +int roce5_destroy_qp_ofa(struct ib_qp *ibqp); + +int roce5_ib_add_gid_ofa(const union ib_gid *gid, + const struct ib_gid_attr *attr, + __always_unused void **context); + +int roce5_g_rdev_add(struct roce5_device *rdev); + +void roce5_g_rdev_remove(struct roce5_device *rdev); + +struct roce5_device *roce5_get_rdev_by_card_func_id(uint64_t card_id, + uint16_t func_id); + +struct roce5_device *roce5_get_rdev_by_name(const char *dev_name); + +#endif //ROCE_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h new file mode 100644 index 000000000..da2b4f65a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_compat.h @@ -0,0 +1,87 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_COMPAT_H +#define ROCE_COMPAT_H + +#include <linux/delay.h> +#include <linux/types.h> + +#define ROCE_LIKELY(x) likely(x) + +#define ROCE_UNLIKELY(x) unlikely(x) + +static inline u32 roce_ilog2(u32 n) +{ + return ilog2(n); +} +#define ROCE_ILOG2(n) roce_ilog2(n) + +static inline u32 roce_roundup_pow_of_two(u32 n) +{ + return roundup_pow_of_two(n); +} +#define ROCE_ROUNDUP_POW_OF_TWO(n) roce_roundup_pow_of_two(n) + +#ifndef _lint +#define ROCE_MDELAY(n) mdelay(n) +#define ROCE_UDELAY(n) udelay(n) +#else +#define ROCE_MDELAY(n) ((void)n) +#define ROCE_UDELAY(n) ((void)n) +#endif + +#define US_PERF_DELAY 100 +#define ROCE_DELAY() ROCE_UDELAY(US_PERF_DELAY) + +#define ROCE_ALIGN(a, b) ALIGN(a, b) +#define ROCE_MAX(a, b) max(a, b) +#define ROCE_MIN(a, b) min(a, b) +#define ROCE_DIV_ROUND_UP(a, b) DIV_ROUND_UP(a, b) +#define ROCE_FLS(n) fls(n) + +#define ROCE_MEMCMP(a, b, count) memcmp(a, b, count) +#define ROCE_IO_MAPPING_MAP_WC(mapping, offset) \ + io_mapping_map_wc(mapping, offset) +#define ROCE_IOREMAP(phys_addr, size) ioremap(phys_addr, size) +#define ROCE_IOUNMAP(addr) iounmap(addr) +#define ROCE_IO_MAPPING_UNMAP(vaddr) io_mapping_unmap(vaddr) + +#ifndef wc_wmb + +#if defined(__i386__) +#define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") +#elif defined(__x86_64__) +#define wc_wmb() asm volatile("sfence" ::: "memory") +#elif defined(__ia64__) +#define wc_wmb() asm volatile("fwb" ::: "memory") +#else +#define wc_wmb() wmb() +#endif + +#endif + +typedef struct tag_roce_verbs_cmd_com { + union { + u32 value; + + struct { + u32 version : 8; + u32 rsvd : 8; + u32 cmd_bitmask : 16; //CMD_TYPE_BITMASK_E + } bs; + } dw0; + + u32 index; //qpn/cqn/srqn/mpt_index/gid idx +} roce_verbs_cmd_com_s; + +typedef struct tag_roce_xq_mtt_info { + u32 mtt_flags; /* Indicates whether to kick out cache. by queue (0) or VF(1). */ + u32 mtt_num; /* Number of cmtt, which needs to be assigned by the driver when the is kicked out by queue. */ + u32 mtt_cache_line_start; /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_end; /* The driver needs to read the driver from the configuration file. */ + u32 mtt_cache_line_size; /* 0:256B,1:512B */ +} roce_xq_mtt_info_s; + +#endif //ROCE_COMPAT_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h new file mode 100644 index 000000000..649a4275b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_event_extension.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#ifndef ROCE_EVENT_EXTENSION_H +#define ROCE_EVENT_EXTENSION_H + +#include "roce.h" +#include "roce_event.h" + +int to_unkown_event_str_index(uint8_t event_type, const uint8_t *val); +void roce5_event_report_extend(const struct roce5_device *rdev, u8 *val, + int event_str_index); +int roce5_async_event_handle_extend(u8 event_type, u8 *val, + struct roce5_device *rdev); + +#endif /* ROCE_EVENT_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h new file mode 100644 index 000000000..54cfc7bf6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_gid.h @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + + +#ifndef ROCE_GID_H +#define ROCE_GID_H + +#include <linux/types.h> + +struct rdma_gid_entry { + union { + u8 raw[16]; + struct { + __be64 subnet_prefix; + __be64 interface_id; + } global; + }; + union { + struct { + u32 rsvd : 8; + u32 cvlan : 12; /* in layer vlan customer vlan */ + u32 svlan : 12; /* outside layer vlan */ + } bs; + u32 value; + } dw4; + + union { + u32 hdr_len_value; + }; + + union { + struct { + u16 tag : 2; /* 0: no has vlan; 1: one layer vlan; 2: 2 layer vlan; 3: stag */ + u16 tunnel : 1; // rsvd for ppe, don't use. 'tunnel' + u16 gid_type : 2; + u16 ppe_rsvd1 : 1; + u16 outer_tag : 2; // rsvd for ppe, don't use. 'outer_tag' + u16 ppe_rsvd3 : 1; // rsvd for ppe, don't use. 'stag' + u16 gid_update : 1; + u16 rsvd : 6; + } bs; + u16 value; + } dw6_h; + + u8 smac[6]; +}; + +#endif // ROCE_GID_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h new file mode 100644 index 000000000..b055d5c92 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_infra.h @@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + +#ifndef ROCE_INFRA_H +#define ROCE_INFRA_H + +#include <linux/kern_levels.h> +#include <linux/sched.h> + +#define ROCE_INFO(format, ...) \ + printk(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_WARN(format, ...) \ + printk(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_ERR(format, ...) \ + printk(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_INFO_LIMIT(format, ...) \ + printk_ratelimited(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format \ + "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_WARN_LIMIT(format, ...) \ + printk_ratelimited(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format \ + "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_ERR_LIMIT(format, ...) \ + printk_ratelimited(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_INFO_ONCE(format, ...) \ + printk_once(KERN_INFO "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_WARN_ONCE(format, ...) \ + printk_once(KERN_WARNING "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_ERR_ONCE(format, ...) \ + printk_once(KERN_ERR "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +/* Importent, ensure dev is not null pointer when using ROCE_DEV_XXX() !!! */ +#define ROCE_DEV_INFO(dev, format, ...) \ + dev_info((dev)->hwdev_hdl, "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_WARN(dev, format, ...) \ + dev_warn((dev)->hwdev_hdl, "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_ERR(dev, format, ...) \ + dev_err((dev)->hwdev_hdl, "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_DEV_INFO_LIMIT(dev, format, ...) \ + dev_info_ratelimited((dev)->hwdev_hdl, \ + "[ROCE, INFO][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_WARN_LIMIT(dev, format, ...) \ + dev_warn_ratelimited((dev)->hwdev_hdl, \ + "[ROCE, WARN][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) +#define ROCE_DEV_ERR_LIMIT(dev, format, ...) \ + dev_err_ratelimited((dev)->hwdev_hdl, \ + "[ROCE, ERR][tid:%x][%s,%d]:" format "\n", \ + current->pid, __func__, __LINE__, ##__VA_ARGS__) + +#define ROCE_PRINT_GID(dev, raw, format, ...) \ + ROCE_DEV_WARN( \ + (dev), \ + "gid:%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x==>" format, \ + *((u16 *)(raw) + IP_INDEX_0), *((u16 *)(raw) + IP_INDEX_1), \ + *((u16 *)(raw) + IP_INDEX_2), *((u16 *)(raw) + IP_INDEX_3), \ + *((u16 *)(raw) + IP_INDEX_4), *((u16 *)(raw) + IP_INDEX_5), \ + *((u16 *)(raw) + IP_INDEX_6), *((u16 *)(raw) + IP_INDEX_7), \ + ##__VA_ARGS__) + +#endif //ROCE_INFRA_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h new file mode 100644 index 000000000..b1bca23b0 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_kernel_compat_gen.h @@ -0,0 +1,6 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_KERNEL_COMPAT_GEN_H +#define ROCE_KERNEL_COMPAT_GEN_H +#endif /* ROCE_KERNEL_COMPAT_GEN_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h new file mode 100644 index 000000000..2244bb236 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_main_extension.h @@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MAIN_EXTENSION_H +#define ROCE_MAIN_EXTENSION_H + +#include <linux/pci.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "hinic5_crm.h" +#include "hinic5_lld.h" +#include "hinic5_srv_nic.h" + +#include "roce.h" +#include "roce_mix.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_qp.h" +#include "roce_dfx.h" +#include "roce_cmd.h" +#include "roce_user.h" +#include "rdma_comp.h" +#ifdef ROCE_CHIP_TEST +#include "roce_main_chip_ext.h" +#endif + +#define ROCE_MAX_WQES_COMPUTE (4 * K_UNIT - 1) +#define RDMA_MAX_SQ_DESC_SZ_COMPUTE 512 +#define ROCE_MAX_SQ_INLINE_DATA_SZ_COMPUTE 448 + +#define ROCE5_ULP_CMD(rdev, bitmask) \ + (((rdev)->is_vroce) ? (VERBS_CMD_TYPE_VROCE_BITMASK) : (bitmask)) + +static inline u64 roce5_get_uverbs_cmd_mask(void) +{ + u64 mask = 0; + mask |= ROCE_UVERBS_CMD_BASE_MASK; +#ifdef ROCE_CHIP_TEST + mask |= ROCE_UVERBS_CMD_EXT_MASK; +#endif + return mask; +} + +struct hinic5_uld_info *roce5_info_get(void); + +void roce5_service_init_pre(void); + +void roce5_lock_rdev(void); + +void roce5_unlock_rdev(void); + +int roce5_get_rdev_by_uld(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, + struct hinic5_event_info *event); + +void roce5_init_dev_ext_handlers(struct roce5_device *rdev); + +int roce5_board_cfg_check(struct roce5_device *rdev); + +void roce5_remove_clean_res_ext(struct roce5_device *rdev); + +int roce5_set_comm_event(const struct roce5_device *rdev, + const struct hinic5_event_info *event); + +void *roce5_rdev_alloc_ext(void); + +void roce5_fill_gid_entry_ext(const struct roce5_device *rdev, + struct rdma_gid_entry *gid_entry); + +void roce5_cfg_info_update_ext(struct roce5_device *rdev, + roce_get_cfg_info_resp_s *resp); + +int roce5_add_check_ext(const struct hinic5_lld_dev *lld_dev); + +#endif /* ROCE_MAIN_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h new file mode 100644 index 000000000..ff0464a31 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_mr_extension.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_MR_EXTEND_H +#define ROCE_MR_EXTEND_H + +#include <rdma/ib_verbs.h> +#include "roce.h" + +int roce5_check_alloc_mr_type(enum ib_mr_type mr_type); +enum rdma_mr_type roce5_get_mrtype(enum ib_mr_type ib_mr_type); + +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h new file mode 100644 index 000000000..6b8a3113a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_os_compat.h @@ -0,0 +1,237 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_OS_COMPAT_H +#define ROCE_OS_COMPAT_H +#include <rdma/ib_verbs.h> +#include "roce.h" +#include "roce_restore.h" +#include "roce_verbs_compat.h" +#include "roce_kernel_compat_gen.h" +#if defined(UVERBS_IOCTL_HAVE_RDMA_UDATA_TO_DRV_CONTEXT) +#include <rdma/uverbs_ioctl.h> +#endif + +#define PAGE_SIZE_4KB (1ULL << PAGE_SHIFT_4KB) +#define PAGE_SIZE_64KB (1ULL << PAGE_SHIFT_64KB) +#define PAGE_SIZE_2MB (1ULL << PAGE_SHIFT_2MB) + +#define BIT_NUM_PER_BYTE (8) +#define PAGE_SHIFT_4KB (12) +#define PAGE_SHIFT_64KB (16) +#define PAGE_SHIFT_2MB (21) + +#if !defined(HAVE_IB_UMEM_NUM_DMA_BLOCKS) +static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem, + unsigned long pgsz) +{ + return (size_t)((ALIGN(umem->address + umem->length, pgsz) - + ALIGN_DOWN(umem->address, pgsz))) / + pgsz; +} +#endif + +#if !defined(HAVE_RDMA_UMEM_FOR_EACH_DMA_BLOCK) +#define rdma_umem_for_each_dma_block(umem, biter, pgsz) \ + for (__rdma_umem_block_iter_start(biter, umem, pgsz); \ + __rdma_block_iter_next(biter);) + +static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter, + struct ib_umem *umem, + unsigned long pgsz) +{ + __rdma_block_iter_start(biter, umem->sg_head.sgl, umem->nmap, pgsz); +} +static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter) +{ + return __rdma_block_iter_next(biter) && biter->__sg_numblocks--; +} +#endif + +#if defined(IB_UMEM_NEED_IB_UMEM_FIND_BEST_PGSZ) +static inline unsigned int rdma_find_pg_bit(unsigned long addr, + unsigned long pgsz_bitmap) +{ + unsigned long align; + unsigned long pgsz; + + align = addr & -addr; + + pgsz = pgsz_bitmap & ~(-align << 1); + if (!pgsz) + return __ffs(pgsz_bitmap); + + return __fls(pgsz); +} + +static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, + unsigned long pgsz_bitmap, + unsigned long virt) +{ + struct scatterlist *sg; + unsigned int best_pg_bit; + unsigned long va, pgoff; + dma_addr_t mask; + int i; + + if (WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0)))) + return 0; + + va = virt; + mask = roundup_pow_of_two(umem->length); + pgoff = umem->address & ~PAGE_MASK; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + mask |= (sg_dma_address(sg) + pgoff) ^ va; + if (i && i != (umem->nmap - 1)) + mask |= sg_dma_len(sg); + va += sg_dma_len(sg) - pgoff; + pgoff = 0; + } + best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap); + + return BIT_ULL(best_pg_bit); +} +#endif + +static inline u32 roce5_umem_get_page_count(struct ib_umem *umem) +{ +#if defined(HAVE_IB_UMEM_PAGE_COUNT) + return (u32)ib_umem_page_count(umem); +#elif defined(HAVE_IB_UMEM_NUM_PAGES) + return (u32)ib_umem_num_pages(umem); +#endif +} + +static inline u32 roce5_umem_get_page_shift(struct ib_umem *umem) +{ +#if defined(UMEM_HAVE_PAGE_SHIFT) + return umem->page_shift; +#else + return PAGE_SHIFT; +#endif +} + +static inline u32 roce5_umem_get_page_size(struct ib_umem *umem) +{ +#if defined(UMEM_HAVE_PAGE_SHIFT) + return BIT((unsigned int)umem->page_shift); +#else + return PAGE_SIZE; +#endif +} + +static inline u32 roce5_get_page_size(struct roce5_device *rdev, bool use_huge) +{ + if (use_huge && rdev->huge_page_en) { + return PAGE_SIZE_2MB; + } + return PAGE_SIZE; +} + +static inline size_t roce5_get_page_size_cap(size_t page_size_cap, + size_t length) +{ + size_t max_shift; + + if (length == 0) { + return 0; + } + + max_shift = fls_long(length) - 1; + if (max_shift == sizeof(size_t) * BIT_NUM_PER_BYTE - 1) + return page_size_cap; + else + return page_size_cap & (((size_t)1 << (max_shift + 1)) - 1); +} + +static inline u32 roce5_umem_get_buf_page_shift(struct roce5_device *rdev, + struct ib_umem *umem, + u64 buf_addr) +{ + u32 real_page_size_cap, page_shift; + + real_page_size_cap = (u32)roce5_get_page_size_cap( + (size_t)rdev->rdma_cap.page_size_cap, umem->length); + + if (rdev->huge_page_en) { + page_shift = (u32)order_base_2(ib_umem_find_best_pgsz( + umem, real_page_size_cap, buf_addr)); + } else { + page_shift = roce5_umem_get_page_shift(umem); + } + + return page_shift; +} + +static inline u32 roce5_umem_get_mtt_npages(struct roce5_device *rdev, + struct ib_umem *umem, + u32 page_shift) +{ + u32 npages; + + if (rdev->huge_page_en) { + npages = ib_umem_num_dma_blocks(umem, BIT(page_shift)); + } else { + npages = roce5_umem_get_page_count(umem); + } + + return npages; +} + +/* These APIs can be reused by different OSs */ +static inline struct ib_umem *roce5_umem_get(struct ib_device *ibdev, + struct ib_ucontext *context, + unsigned long addr, size_t size, + int access, int dmasync) +{ +#if (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(MMU_INTERVAL_NOTIFIER)) || \ + (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(IB_UMEM_GET_HAVE_IB_DEVICE)) + struct uverbs_attr_bundle *attrs; +#endif + +#if (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(MMU_INTERVAL_NOTIFIER)) || \ + (!defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) && \ + !defined(IB_UMEM_GET_HAVE_IB_DEVICE)) + attrs = container_of(&context, struct uverbs_attr_bundle, context); +#endif + +#if defined(IB_UMEM_GET_HAVE_IB_UCONTEXT) + return ib_umem_get(context, addr, size, access, dmasync); +#elif defined(IB_UMEM_GET_HAVE_IB_DEVICE) && defined(MMU_INTERVAL_NOTIFIER) + return ib_umem_get(ibdev, addr, size, access); +#elif defined(IB_UMEM_GET_HAVE_DMASYNC) + return ib_umem_get(&attrs->driver_udata, addr, size, access, dmasync); +#else + return ib_umem_get(&attrs->driver_udata, addr, size, access); +#endif +} + +#if defined(IB_UMEM_GET_PEER) +static inline struct ib_umem * +roce5_mr_umem_get_peer(struct roce5_device *rdev, size_t length, int access, + u64 addr, struct ib_udata *udata, bool use_huge) +{ + u64 page_size = PAGE_SIZE; + size_t buf_size = length; + + page_size = roce5_get_page_size(rdev, use_huge); + buf_size = (size_t)ROCE_ALIGN(buf_size, page_size); + +#if defined(IB_UMEM_GET_PEER_HAVE_IBDEVICE) && defined(MMU_INTERVAL_NOTIFIER) + return ib_umem_get_peer(&rdev->ib_dev, addr, buf_size, access, + (IB_PEER_MEM_ALLOW | IB_PEER_MEM_INVAL_SUPP)); +#else + return ib_umem_get_peer(udata, addr, buf_size, access, + (IB_PEER_MEM_ALLOW | IB_PEER_MEM_INVAL_SUPP)); +#endif +} +#endif + +struct roce5_ucontext * +roce5_queue_to_ucontext(void *ibq, struct ib_udata *udata, int q_type); + +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h new file mode 100644 index 000000000..2cbcd5d71 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_qp_extension.h @@ -0,0 +1,61 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_QP_EXTENSION_H +#define ROCE_QP_EXTENSION_H + +#include <rdma/ib_verbs.h> + +#include "roce_qp.h" +#include "roce_qp_exp.h" + +int to_roce5_qp_type(enum ib_qp_type qp_type); + +bool roce5_check_qp_modify_ok(enum ib_qp_state cur_state, + enum ib_qp_state next_state, enum ib_qp_type type, + enum ib_qp_attr_mask mask, + enum rdma_link_layer ll); + +int roce5_is_qp_normal(struct roce5_qp *rqp, struct ib_qp_init_attr *init_attr); + +void roce5_set_qp_dif_attr(struct roce5_qp *rqp, + const struct ib_qp_init_attr *init_attr, + const struct roce5_device *rdev); + +#ifdef ROCE_EXTEND + +#define ROCE_QP_VBS_FLAG (1U << 22) +#define ROCE_QP_OSD_FLAG (1U << 23) +#define QPC_ROCE_VBS_QPC_OFFSET_FOR_SQPC 204800 //200K + +struct roce5_set_qp_ext_attr_cmd { + u32 qpn; + u32 attr_mask; +}; + +struct roce5_modify_qp_vbs_cmd { + u32 qpn; + u64 ci_record_addr; +}; + +struct roce5_vbs_qp { + cqm_qpc_mpt_s *vbs_sqpc_info; /* vbs private data */ + struct roce5_db db; /* to record ci_record_pa */ +}; + +long roce5_vbs_create_sqpc(struct roce5_device *rdev, void *buf); +#endif + +void roce5_destroy_pcqc_ext(struct roce5_qp *rqp); +int roce5_modify_qp_pre_ext(struct roce5_device *rdev, struct roce5_qp *rqp, + struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata); +void roce5_set_attr_ext(struct roce5_qp *rqp, const struct ib_qp_attr *attr, + roce_verbs_qp_attr_s *qp_attr, int attr_mask); +void roce5_transmit_ci_record_addr_ext(struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr); +void roce5_cos_update_ext(struct roce5_device *rdev, u8 *cos, + enum ib_qp_type qp_type); + +#endif /* ROCE_QP_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h new file mode 100644 index 000000000..c772472a6 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_restore.h @@ -0,0 +1,88 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_RESTORE_H +#define ROCE_RESTORE_H +#include <rdma/ib_verbs.h> + +#include "roce.h" +#include "roce_qp.h" +#include "roce_srq.h" +#include "roce_cq.h" +#include "roce_mr.h" +#include "roce_mix.h" +#include "roce_pd.h" +#include "roce_xrc.h" +#include "hinic5_cqm.h" + +static inline struct roce5_device *to_roce5_dev(const struct ib_device *ibdev) +{ + return container_of(ibdev, struct roce5_device, ib_dev); +} + +static inline struct roce5_ucontext * +to_roce5_ucontext(const struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct roce5_ucontext, ibucontext); +} + +static inline struct roce5_qp *to_roce5_qp(const struct ib_qp *ibqp) +{ + return container_of(ibqp, struct roce5_qp, ibqp); +} + +static inline struct roce5_qp *cqmobj_to_roce_qp(const cqm_object_s *object) +{ + cqm_qpc_mpt_s *qpc_info = NULL; + qpc_info = container_of(object, cqm_qpc_mpt_s, object); + return (struct roce5_qp *)qpc_info->priv; +} + +static inline struct roce5_srq *to_roce5_srq(const struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct roce5_srq, ibsrq); +} + +static inline struct roce5_srq *cqmobj_to_roce5_srq(const cqm_object_s *object) +{ + cqm_queue_s *cqm_srq = NULL; + cqm_srq = container_of(object, cqm_queue_s, object); + return (struct roce5_srq *)cqm_srq->priv; +} + +static inline struct roce5_cq *to_roce5_cq(const struct ib_cq *ibcq) +{ + return container_of(ibcq, struct roce5_cq, ibcq); +} + +static inline struct roce5_mr *to_roce5_mr(const struct ib_mr *ibmr) +{ + return container_of(ibmr, struct roce5_mr, ibmr); +} + +static inline struct roce5_mw *to_roce5_mw(const struct ib_mw *ibmw) +{ + return container_of(ibmw, struct roce5_mw, ibmw); +} + +static inline struct roce5_mr * +rdma_mr_to_roce5_mr(const struct hmm_rdma *rdmamr) +{ + return container_of(rdmamr, struct roce5_mr, rdmamr); +} + +static inline struct roce5_ah *to_roce5_ah(const struct ib_ah *ibah) +{ + return container_of(ibah, struct roce5_ah, ibah); +} + +static inline struct roce5_pd *to_roce5_pd(const struct ib_pd *ibpd) +{ + return container_of(ibpd, struct roce5_pd, ibpd); +} + +static inline struct roce5_xrcd *to_roce5_xrcd(const struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct roce5_xrcd, ibxrcd); +} +#endif \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h new file mode 100644 index 000000000..b931f9426 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_srq_extension.h @@ -0,0 +1,38 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_SRQ_EXTENSION_H +#define ROCE_SRQ_EXTENSION_H + +#include <linux/types.h> +#include "roce_srq.h" + +/** + * container_mode: + * mode: 0 -> container_size: 16 + * mode: 1 -> container_size: 8 + * mode: 2 -> container_size: 4 + * mode: 3 -> container_size: 2 + */ +enum roce5_srq_mode { + ROCE_SRQ_MODE_0 = 0, + ROCE_SRQ_MODE_1, + ROCE_SRQ_MODE_2, + ROCE_SRQ_MODE_3 +}; + +enum roce5_srq_cont_num_mode { + ROCE_SRQ_CONT_NUM_MODE3 = 2, + ROCE_SRQ_CONT_NUM_MODE2 = 4, + ROCE_SRQ_CONT_NUM_MODE1 = 8, + ROCE_SRQ_CONT_NUM_MODE0 = 16 +}; + +u8 roce5_get_container_sz(u32 container_mode); +u8 roce5_calculate_cont_th(u32 srq_limit); +void roce5_srq_container_init(struct ib_srq_init_attr *init_attr, + struct roce5_srq *rsrq, + struct roce5_device *rdev); + +#endif /* ROCE_SRQ_EXTENSION_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h new file mode 100644 index 000000000..81effef16 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_user.h @@ -0,0 +1,100 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_USER_H +#define ROCE_USER_H + +#include <linux/types.h> + +struct roce5_alloc_ucontext_resp { + uint32_t num_qps; /* the maximum of qps */ + uint32_t num_xsrqs; /* the maximum of xsrqs */ + uint32_t cqe_size; /* the cqe entry size that device support, the valid value is 32B or 64B; */ + uint32_t wqebb_size; /* the wqebb entry size that device support, the valid value is 64B or 128B */ + uint32_t dwqe_size; /* the direct maximum wqe size that device support, 256B */ + uint32_t max_msg_size; /* the maximum message size that device support, (2G -1)B */ + uint32_t max_comp_vector; /* the maximum completion vector */ + uint32_t max_inline_size; /* the maximum of qp inline size that device support */ + uint32_t dev_caps; + uint8_t phy_port; + uint8_t storage_aa_en; + uint16_t func_id; + uint64_t card_id; + uint16_t db_offset; + uint16_t dwqe_offset; + struct { + uint16_t pcie_atomic_en : 1; + uint16_t rsvd1 : 15; + }; + uint8_t srq_container_en; + uint8_t srq_container_mode; + uint8_t xrc_srq_container_mode; + uint8_t warn_th; + uint8_t huge_page_en; + uint8_t rsvd; + uint8_t max_send_sge; /* the maximum of sq sge */ + uint8_t max_recv_sge; /* the maximum of rq sge */ + uint32_t max_sq_wqe_len; /* the maximum sq wqe length that device support */ + uint64_t roce_user_ext_ability; + uint64_t roce_uld_ext_ability; +}; + +struct roce5_create_cq_cmd { + u64 buf_addr; + u64 db_addr; +}; + +struct roce5_resize_cq_cmd { + u64 buf_addr; /* resize cq's 'buf va */ + u64 stage; /* describe the resize stage,0 or 1; */ +}; + +struct roce5_create_srq_cmd { + u64 buf_addr; + u64 db_addr; + u32 rsvd; +}; + +struct create_qp_cmd { + u64 buf_addr; /* describe the qp buf address that used to store cqe; */ + u64 db_addr; /* describe the qp sw db address */ + u32 comp_mask; /* kernel can judge whether handle receive inline through comp_mask */ + u8 log_sq_bb_count; /* wqebb number = 2 << log_sq_bb_count */ + u8 log_sq_stride; /* wqebb size = 2 << log_sq_stride */ + u16 reserved; /* reserved is used to aligned cmd */ +}; + +struct roce5_modify_srq_cmd { + u8 container_flag; + u8 container_warn_th; + u16 rsvd; +}; + +struct roce5_reg_frmr_cmd { + u64 pbl_vaddr; +}; + +struct roce5_modify_qp_ex_resp_data { + u8 is_cos_valid; + u8 cos; + u8 is_ssn_valid; + u8 rsvd0; + u32 ssn; + u32 rsvd[6]; +#ifdef ROCE_LATENCY_PROF_EN + uint8_t latency_prof_en; + uint8_t rsvd1[3]; + uint32_t frequency_reduction_ratio; + uint64_t non_ptp_time_diff; +#endif +}; + +struct roce5_create_ah_resp { + u8 dmac[ETH_ALEN]; + u16 vlan_id; + u8 cos; + u8 rsvd[3]; +}; + +#endif //ROCE_USER_H diff --git a/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h b/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h new file mode 100644 index 000000000..cd87856ea --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/include/roce_verbs_compat.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_VERBS_COMPAT_H +#define ROCE_VERBS_COMPAT_H + +#include <rdma/ib_verbs.h> +#include "roce_kernel_compat_gen.h" + +enum { ROCE_Q_TYPE_QP, ROCE_Q_TYPE_SRQ, ROCE_Q_TYPE_CQ }; + +#if !defined(HAVE_RDMA_AH_INIT_ATTR) +/* kernel 4.19 not defined but kernel 5.10 defined, reused for ah creating */ +struct rdma_ah_init_attr { + struct rdma_ah_attr *ah_attr; +}; + +enum ib_wr_opcode_compat { + IB_WR_REG_MR_INTEGRITY = IB_WR_REG_SIG_MR, +}; + +#endif + +#endif /* ROCE_VERBS_COMPAT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h new file mode 100644 index 000000000..099b76a1a --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond.h @@ -0,0 +1,79 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef ROCE_BOND_H +#define ROCE_BOND_H + +#include <net/bonding.h> +#include <linux/netdevice.h> + +#include "hinic5_lld.h" +#include "drv_bond_api.h" + +#include "roce.h" +#include "roce_qp.h" +#include "roce_cmd.h" + +#define ROCE_BOND_MIN_SLAVE 1 +#define ROCE_BOND_MAX_SLAVE BOND_PORT_MAX_NUM + +#define ROCE_BOND_RSVD_VLAN_ID 4095 + +#define ROCE_BOND_FWD_ID_TBL_ALL_BITS 32U +#define ROCE_BOND_FWD_ID_TBL_PER_BITS 3U +#define ROCE_BOND_FWD_ID_TBL_PER_BITS_MASK 0x7U + +struct roce5_bond_slave { + struct net_device *netdev; + struct hinic5_lld_dev *lld_dev; + struct netdev_lag_lower_state_info netdev_state; + u32 update_cnt; + u16 func_id; + u8 er_id; +}; + +struct roce5_bond_transition_res { + struct hinic5_lld_dev *ppf_lld; +}; + +struct roce5_bond_device { + char name[IFNAMSIZ]; /* the name of nic bond device */ + struct list_head entry; + struct hinic5_lld_dev *bond_lld_dev; + struct roce5_bond_slave slaves[ROCE_BOND_MAX_SLAVE]; + struct mutex slave_lock; /* lock for slaves */ + u32 slave_cnt; + atomic_t next_port; + struct work_struct detach_work; + struct roce5_device *attached_rdev; + struct bond_attr attr; + struct roce5_bond_transition_res transition_res; + u8 need_clean_on_remove; +}; + +bool roce5_bond_is_active(struct roce5_device *rdev); +void roce5_bond_detach(struct roce5_device *rdev); +struct net_device *roce5_bond_get_netdev(struct roce5_device *rdev); + +int roce5_add_bond_real_slave_mac(struct roce5_device *rdev, u8 *mac); +int roce5_add_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id); +void roce5_del_bond_real_slave_mac(struct roce5_device *rdev); +void roce5_del_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id); + +void roce5_bond_rr_set_flow(struct roce5_device *rdev, struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr); + +int roce5_bond_event_cfg_rdev(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, bool hold_rdev); +void roce5_handle_bonded_port_state_event(struct roce5_device *rdev); + +bool roce5_get_bond_ipsurx_en(void); +void roce5_set_bond_ipsurx_en(bool ipsurx_en); + +int roce5_bond_attach(struct roce5_device *rdev); +int roce5_bond_init(void); +void roce5_bond_exit(void); +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c new file mode 100644 index 000000000..f295f241d --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_cfm.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "bond_common_defs.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +static int roce5_bond_check_slave_port(struct hinic5_lld_dev *lld_dev, + u8 slave_ports[], u8 slave_ports_num, + bool *result) +{ + int ret, i; + roce_bond_check_slave_port_req_s req = { 0 }; + roce_bond_check_slave_port_rsp_s rsp = { 0 }; + u16 out_size = (u16)sizeof(rsp); + + for (i = 0; i < slave_ports_num; ++i) { + req.slave_ports[i] = slave_ports[i]; + } + req.slave_ports_num = slave_ports_num; + + ret = roce5_msg_to_mgmt_sync(lld_dev->hwdev, + ROCE_MPU_CMD_BOND_CHECK_SLAVE_PORT, &req, + sizeof(req), &rsp, &out_size); + if ((ret != 0) || (out_size == 0) || (rsp.head.status != 0)) { + ROCE_ERR( + "Failed to check slave port, ret(%d), out size(%u), status(%u)", + ret, out_size, rsp.head.status); + return -EINVAL; + } + + *result = false; + for (i = 0; i < slave_ports_num; ++i) { + *result |= (rsp.slave_ports_result[i] != false) ? true : false; + } + + return 0; +} + +/* + * roce5_can_do_bond - Check if RoCE can attach to bond device + * @bond: bond device to check + * Return: true if can attach, false otherwise + */ +bool roce5_can_do_bond(struct bonding *bond) +{ + bool ret = false; + int slave_cnt = 0; + int err; + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + char chip_name[IFNAMSIZ] = { 0 }; + u8 slave_ports[ROCE_BOND_PORT_MAX_NUM]; + bool result; + + if (bond == NULL) { + ROCE_ERR("Invalid Parameter, ptr is NULL."); + return false; + } + + if (roce5_bond_mode_is_supported(bond->params.mode) == false) { + ROCE_ERR("Invalid bond mode(%u).", bond->params.mode); + return false; + } + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + lld_dev = hinic5_get_lld_dev_by_netdev(slave->dev); + if (lld_dev == NULL) { + goto out; + } + + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + ROCE_WARN("VF device %s skipped", slave->dev->name); + continue; + } + + err = roce5_bond_slave_check(lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond slave check err %d.", err); + goto out; + } + + if (slave_cnt < ROCE_BOND_PORT_MAX_NUM) { + slave_ports[slave_cnt] = + hinic5_physical_port_id(lld_dev->hwdev); + } + + slave_cnt++; + ROCE_INFO("can do bond? slave_cnt(%d), slave_name(%s).", + slave_cnt, slave->dev->name); + } + + if (((slave_cnt >= ROCE_BOND_MIN_SLAVE) && + (slave_cnt <= ROCE_BOND_MAX_SLAVE)) == false) { + goto out; + } + + if (lld_dev == NULL) { + ROCE_INFO("lld dev is NULL."); + goto out; + } + + err = roce5_bond_check_slave_port(lld_dev, slave_ports, slave_cnt, + &result); + if (err != 0) { + ROCE_ERR("Failed to check slave ports, err(%d).", err); + goto out; + } + + if (result == false) { + ROCE_INFO("Ports don't meet requirements."); + goto out; + } + + ret = true; + +out: + rcu_read_unlock(); + return ret; +} + +static int +roce5_bond_create_transition_store(struct roce5_bond_device *bond_dev, + const struct bond_tracker *tracker) +{ + int ret; + struct hinic5_lld_dev *ppf_lld = NULL; + + ppf_lld = roce5_bond_get_ppf_lld(tracker); + if (ppf_lld == NULL) { + ROCE_ERR("Failed to get lld of the PPF."); + return -EINVAL; + } + + ret = roce5_bond_transition_store(bond_dev, ppf_lld); + if (ret != 0) { + ROCE_ERR("Failed to store resources of transition, ret(%d).", + ret); + hinic5_lld_dev_put(ppf_lld); + return ret; + } + + return 0; +} + +static void +roce5_bond_create_transition_free(struct roce5_bond_device *bond_dev) +{ + roce5_bond_transition_free(bond_dev); +} + +/* + * roce5_before_bond_active - Pre-active callback for bond creation + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_before_bond_active(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + u16 bond_id; + int i, slave_cnt; + int ret; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR( + "Invalid parameter, roce bond_name(%d) or attr(%d) is NULL.", + !!bond_name, !!attr); + return; + } + + bond_id = attr->bond_id; + ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to get bond tracker, bond name(%s), ret(%d).", + bond_name, ret); + goto err; + } + + if (roce5_bond_before_active_check(&tracker, attr) == false) { + ROCE_ERR("Failed to check before active bond."); + goto err; + } + + bond_dev = roce5_bond_create_bond_device(bond_name); + if (IS_ERR(bond_dev)) { + ROCE_ERR("Failed to create roce bond device."); + goto err; + } + + /* Store transition resources during the bond creating period to prevent abnormal resources using. */ + ret = roce5_bond_create_transition_store(bond_dev, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to store resources of transition."); + goto err; + } + + /* Remove the functions of slaves. */ + for (i = 0, slave_cnt = 0; + i < BOND_PORT_MAX_NUM && slave_cnt < tracker.cnt; ++i) { + if (tracker.ndev[i] == NULL) { + continue; + } + + ret = roce5_bond_add_slave(bond_dev, tracker.ndev[i], + &tracker.netdev_state[i], false); + if (ret != 0) { + ROCE_ERR( + "Failed to add slave, name(%s), slave name(%s).", + bond_dev->name, tracker.ndev[i]->name); + goto err_with_transition; + } + + ++slave_cnt; + } + + /* Free transition resources during the bond creating period. */ + roce5_bond_create_transition_free(bond_dev); + bond_dev->attr = *attr; + + return; + +err_with_transition: + roce5_bond_create_transition_free(bond_dev); +err: + roce5_bond_destroy(bond_name, bond_id); +} + +/* + * roce5_after_bond_active - Post-active callback for bond creation + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_after_bond_active(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR( + "Invalid parameter, bond name(%d) or attr(%d) is NULL.", + !!bond_name, !!attr); + return; + } + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("Can't find roce bond device by name(%s).", bond_name); + return; + } + + roce5_update_rdev_func_val(bond_dev); +} + +/* + * roce5_before_bond_modify - Pre-modify callback for bond modification + * @bond_name: name of bond device + * @attr: new bond attributes + * @err: error code from previous operation + */ +void roce5_before_bond_modify(const char *bond_name, struct bond_attr *attr, + int err) +{ + int ret; + struct roce5_bond_device *bond_dev = NULL; + struct bond_tracker tracker; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR("roce bond_name or attr is NULL."); + return; + } + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("Can't find roce bond device by name(%s).", bond_name); + return; + } + + ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to get bond tracker, bond name(%s), ret(%d).", + bond_name, ret); + return; + } + + ret = roce5_handle_diff_slave(bond_dev, &tracker); + if (ret != 0) { + ROCE_ERR("Failed to handle different slave."); + return; + } + + return; +} + +/* + * roce5_after_bond_modify - Post-modify callback for bond modification + * @bond_name: name of bond device + * @attr: new bond attributes + * @err: error code from previous operation + */ +void roce5_after_bond_modify(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + + if (bond_name == NULL || attr == NULL) { + ROCE_ERR("roce bond_name or attr is NULL."); + return; + } + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("can't find bond device by name(%s).", bond_name); + return; + } + + bond_dev->attr = *attr; + + roce5_update_rdev_func_val(bond_dev); +} + +/* + * roce5_before_bond_deactive - Pre-deactive callback for bond deletion + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_before_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err) +{ + struct roce5_bond_device *bond_dev = NULL; + + bond_dev = roce5_get_bond_dev_by_name(bond_name); + if (bond_dev == NULL) { + ROCE_ERR("can't find bond device by name(%s).", bond_name); + return; + } + + roce5_bond_destroy(bond_dev->name, attr->bond_id); +} + +/* + * roce5_after_bond_deactive - Post-deactive callback for bond deletion + * @bond_name: name of bond device + * @attr: bond attributes + * @err: error code from previous operation + */ +void roce5_after_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err) +{ + /* destroy switch thread handle, bond device may already be delete, current thread not disable care handle result */ +} + +/* + * roce5_bond_mode_is_supported - Check if bond mode is supported + * @bond_mode: bond mode to check + * Return: true if supported, false otherwise + */ +bool roce5_bond_mode_is_supported(u16 bond_mode) +{ + if (bond_mode != BOND_MODE_8023AD && bond_mode != BOND_MODE_XOR && + bond_mode != BOND_MODE_ACTIVEBACKUP) { + ROCE_ERR( + "Bond mode only support 802.3ad/xor/active-backup, current bond mode(%u).", + bond_mode); + return false; + } + return true; +} + +/* + * roce5_bond_slave_check - Check if slave device is valid for RoCE bond + * @lld_dev: LLD device to check + * @chip_name: buffer to store chip name + * @chip_name_size: size of chip_name buffer + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_slave_check(struct hinic5_lld_dev *lld_dev, char *chip_name, + size_t chip_name_size) +{ + int err; + char tmp_name[IFNAMSIZ] = { 0 }; + + if (lld_dev == NULL) { + ROCE_ERR("get lld dev err."); + return -ENODEV; + } + + if (roce5_func_is_vf(lld_dev->hwdev)) { + ROCE_ERR("Bond Slave device is VF."); + return -EINVAL; + } + + err = hinic5_get_chip_name(lld_dev, tmp_name, sizeof(tmp_name)); + if (err != 0) { + ROCE_ERR("Bond Slave get chip name err %d.", err); + return err; + } + + if (strlen(chip_name) == 0) { + err = memcpy_s(chip_name, chip_name_size, tmp_name, + sizeof(tmp_name)); + if (err != 0) { + ROCE_ERR("Failed to memcpy chip_name.(ret:%d)", err); + return -ENOMEM; + } + } else if (strcmp(tmp_name, chip_name) != 0) { + ROCE_ERR( + "Creating bond cross cards is not supported, bond dev chip_name %s, slave chip name %s.", + chip_name, tmp_name); + return -EINVAL; + } + + return 0; +} + +/* + * roce5_bond_before_active_check - Pre-active check for bond creation + * @tracker: bond tracker from CFM framework + * @attr: bond attributes + * Return: true if check passes, false otherwise + */ +bool roce5_bond_before_active_check(struct bond_tracker *tracker, + struct bond_attr *attr) +{ + int i, slave_cnt; + int err; + struct hinic5_lld_dev *lld_dev = NULL; + char chip_name[IFNAMSIZ] = { 0 }; + + if (roce5_bond_mode_is_supported(attr->bond_mode) == false) { + ROCE_ERR("Invalid bond mode(%u).", attr->bond_mode); + return false; + } + + if (tracker->cnt < ROCE_BOND_MIN_SLAVE || + tracker->cnt > ROCE_BOND_MAX_SLAVE) { + ROCE_ERR("Not support the slave number(%u) of the bond.", + tracker->cnt); + return false; + } + + for (i = 0, slave_cnt = 0; + i < BOND_PORT_MAX_NUM && slave_cnt < tracker->cnt; ++i) { + if (tracker->ndev[i] == NULL) { + continue; + } + + lld_dev = hinic5_get_lld_dev_by_netdev(tracker->ndev[i]); + if (lld_dev == NULL) { + ROCE_ERR("get lld dev err."); + return false; + } + + err = roce5_bond_slave_check(lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond slave check err %d.", err); + return false; + } + + ++slave_cnt; + } + return true; +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c new file mode 100644 index 000000000..cb3612778 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_device.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "bond_common_defs.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* Global variables: g_roce5_bond_mutex protects g_roce5_bond_list */ +LIST_HEAD(g_roce5_bond_list); +DEFINE_MUTEX(g_roce5_bond_mutex); + +static bool g_roce5_bond_ipsurx_en = true; + +int roce5_bond_transition_store(struct roce5_bond_device *bond_dev, + struct hinic5_lld_dev *ppf_lld) +{ + int ret = 0; + + ret = hinic5_stateful_init(ppf_lld->hwdev); + if (ret != 0) { + ROCE_ERR( + "Failed to init stateful resource of the PPF, ret(%d).", + ret); + goto out; + } + + bond_dev->transition_res.ppf_lld = ppf_lld; +out: + return ret; +} + +void roce5_bond_transition_free(struct roce5_bond_device *bond_dev) +{ + if (bond_dev->transition_res.ppf_lld == NULL) { + return; + } + + hinic5_stateful_deinit(bond_dev->transition_res.ppf_lld->hwdev); + hinic5_lld_dev_put(bond_dev->transition_res.ppf_lld); + bond_dev->transition_res.ppf_lld = NULL; +} + +/* + * Caller has to call hinic5_lld_dev_put() to free resource when finish using ppf lld. + * There's no need to call hinic5_lld_dev_put() to free resource when this function return NULL. + */ +struct hinic5_lld_dev * +roce5_bond_get_ppf_lld(const struct bond_tracker *tracker) +{ + int i; + struct net_device *first_slave = NULL; + struct hinic5_lld_dev *first_slave_lld = NULL, *ppf_lld = NULL; + + for (i = 0; i < BOND_PORT_MAX_NUM; ++i) { + if (tracker->ndev[i] != NULL) { + first_slave = tracker->ndev[i]; + break; + } + } + + if (first_slave == NULL) { + ROCE_ERR("Failed to get net device of the first slave."); + return NULL; + } + + first_slave_lld = hinic5_get_lld_dev_by_netdev(first_slave); + if (first_slave_lld == NULL) { + ROCE_ERR("Failed to get lld of the first slave."); + return NULL; + } + + ppf_lld = hinic5_get_ppf_lld_dev(first_slave_lld); + if (ppf_lld == NULL) { + ROCE_ERR("Failed to get lld of the PPF."); + return NULL; + } + + return ppf_lld; +} + +/* + * roce5_get_bond_dev - Get bond device by name (internal) + * @bond_name: name of bond device + * Return: pointer to roce5_bond_device if found, NULL otherwise + * Lock: caller must hold g_roce5_bond_mutex + */ +struct roce5_bond_device *roce5_get_bond_dev(const char *bond_name) +{ + struct roce5_bond_device *bdev = NULL; + + list_for_each_entry(bdev, &g_roce5_bond_list, entry) { + if (strcmp(bdev->name, bond_name) == 0) { + return bdev; + } + } + + return NULL; +} + +/* + * roce5_get_bond_dev_by_name - Get bond device by name (public wrapper) + * @bond_name: name of bond device + * Return: pointer to roce5_bond_device if found, NULL otherwise + */ +struct roce5_bond_device *roce5_get_bond_dev_by_name(const char *bond_name) +{ + struct roce5_bond_device *bdev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + bdev = roce5_get_bond_dev(bond_name); + mutex_unlock(&g_roce5_bond_mutex); + return bdev; +} + +struct roce5_bond_device *roce5_bond_create_bond_device(const char *bond_name) +{ + struct roce5_bond_device *rbdev = NULL; + errno_t ret; + + if (bond_name == NULL) { + ROCE_ERR("Invalid parameter, ptr is NULL."); + return ERR_PTR(-EINVAL); + } + + rbdev = roce5_get_bond_dev_by_name(bond_name); + if (rbdev != NULL) { + ROCE_ERR("Bond device(%s) already exists.", bond_name); + return ERR_PTR(-EINVAL); + } + + rbdev = kzalloc(sizeof *rbdev, GFP_KERNEL); + if (rbdev == NULL) { + ROCE_ERR("Failed to alloc memory."); + return ERR_PTR(-ENOMEM); + } + + ret = strncpy_s(rbdev->name, IFNAMSIZ, bond_name, strlen(bond_name)); + if (ret != EOK) { + ROCE_ERR("Failed to copy string, ret(%d).", ret); + kfree(rbdev); + return ERR_PTR(-EFAULT); + } + + rbdev->need_clean_on_remove = true; + mutex_init(&rbdev->slave_lock); + + mutex_lock(&g_roce5_bond_mutex); + list_add_tail(&rbdev->entry, &g_roce5_bond_list); + mutex_unlock(&g_roce5_bond_mutex); + + return rbdev; +} + +static int +roce5_bond_destroy_transition_store(struct roce5_bond_device *bond_dev) +{ + int ret; + struct hinic5_lld_dev *ppf_lld = NULL; + + if (bond_dev->slave_cnt == 0) { + ROCE_WARN("There's no slave in roce bond."); + return 0; + } + + ppf_lld = hinic5_get_ppf_lld_dev(bond_dev->slaves[0].lld_dev); + if (ppf_lld == NULL) { + ROCE_ERR("Failed to get lld of the PPF."); + return -EINVAL; + } + + ret = roce5_bond_transition_store(bond_dev, ppf_lld); + if (ret != 0) { + ROCE_ERR("Failed to store resources of transition, ret(%d).", + ret); + hinic5_lld_dev_put(ppf_lld); + return ret; + } + + return 0; +} + +static void +roce5_bond_destroy_transition_free(struct roce5_bond_device *bond_dev) +{ + roce5_bond_transition_free(bond_dev); +} +/* + * roce5_bond_destroy - Destroy bond device + * @bond_name: name of bond device + * @bond_id: bond ID + */ +void roce5_bond_destroy(const char *bond_name, u16 bond_id) +{ + int ret; + u32 i; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + bond_dev = roce5_get_bond_dev(bond_name); + if (bond_dev == NULL) { + /* destroy switch thread handle, bond device may already be delete, current thread not disable care handle result */ + ROCE_INFO("bond name(%s) has been detached", bond_name); + mutex_unlock(&g_roce5_bond_mutex); + return; + } + + list_del(&bond_dev->entry); + mutex_unlock(&g_roce5_bond_mutex); + + if (bond_dev->bond_lld_dev != NULL) { + ret = roce5_bond_destroy_transition_store(bond_dev); + if (ret != 0) { + ROCE_ERR( + "Failed to store transition resources on destroy."); + } + + bond_dev->need_clean_on_remove = false; + hinic5_detach_service(bond_dev->bond_lld_dev, SERVICE_T_ROCE); + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + ret = hinic5_attach_service(bond_dev->slaves[i].lld_dev, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR( + "Failed to attach roce device, ret(%d).", + ret); + } + } + bond_dev->slave_cnt = 0; + roce5_bond_destroy_transition_free(bond_dev); + mutex_unlock(&bond_dev->slave_lock); + } + + kfree(bond_dev); + return; +} + +/* + * roce5_bond_attach - Attach bond device to RoCE + * @rdev: RoCE device to attach + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_attach(struct roce5_device *rdev) +{ + u32 i; + int ret = 0; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) { + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (rdev->ndev != bond_dev->slaves[i].netdev) { + continue; + } + + if (bond_dev->attached_rdev == NULL) { + bond_dev->attached_rdev = rdev; + rdev->bond_dev = bond_dev; + } else { + ret = -EEXIST; + } + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + mutex_unlock(&bond_dev->slave_lock); + } +out: + mutex_unlock(&g_roce5_bond_mutex); + return ret; +} + +/* + * roce5_bond_is_active - Check if bond is active for RoCE device + * @rdev: RoCE device to check + * Return: true if bond is active, false otherwise + */ +bool roce5_bond_is_active(struct roce5_device *rdev) +{ + return (rdev->bond_dev != NULL); +} + +void roce5_bond_detach(struct roce5_device *rdev) +{ + int ret; + u32 i; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + bond_dev = rdev->bond_dev; + if (bond_dev == NULL) { + mutex_unlock(&g_roce5_bond_mutex); + ROCE_INFO("bond_dev is NULL."); + return; + } + + if (bond_dev->need_clean_on_remove == false) { + mutex_unlock(&g_roce5_bond_mutex); + ROCE_INFO("bond_dev no need to detached on roce remove."); + return; + } + + if (roce5_get_bond_dev(bond_dev->name) == bond_dev) { + list_del(&bond_dev->entry); + } + + mutex_unlock(&g_roce5_bond_mutex); + ret = roce5_bond_destroy_transition_store(bond_dev); + if (ret != 0) { + ROCE_ERR("Failed to store transition resources on destroy."); + } + hinic5_bond_detach(bond_dev->attr.bond_id, HINIC5_BOND_USER_ROCE); + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->bond_lld_dev == bond_dev->slaves[i].lld_dev) { + continue; + } + ret = hinic5_attach_service(bond_dev->slaves[i].lld_dev, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to attach roce device, ret(%d).", ret); + } + } + bond_dev->slave_cnt = 0; + mutex_unlock(&bond_dev->slave_lock); + roce5_bond_destroy_transition_free(bond_dev); + mutex_lock(&g_roce5_bond_mutex); + kfree(bond_dev); + rdev->bond_dev = NULL; + mutex_unlock(&g_roce5_bond_mutex); +} +/* + * roce5_bond_event_cfg_rdev - Configure RoCE device for bond events + * @lld_dev: LLD device + * @uld_dev: ULD device + * @rdev: pointer to RoCE device pointer (output) + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_event_cfg_rdev(struct hinic5_lld_dev *lld_dev, void *uld_dev, + struct roce5_device **rdev, bool hold_rdev) +{ + u32 i = 0; + struct roce5_bond_device *bond_dev = NULL; + + if (lld_dev == NULL) { + ROCE_ERR("lld_dev is null."); + return -EINVAL; + } + + if (uld_dev != NULL) { + *rdev = (struct roce5_device *)uld_dev; + return 0; + } + + mutex_lock(&g_roce5_bond_mutex); + list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) { + if (bond_dev == NULL) { + ROCE_ERR("bond_dev is null"); + continue; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + if (bond_dev->slaves[i].lld_dev == lld_dev) { + *rdev = bond_dev->attached_rdev; + if (hold_rdev == true && *rdev != NULL) { + uld5_dev_hold((*rdev)->lld_dev, + SERVICE_T_ROCE); + } + mutex_unlock(&bond_dev->slave_lock); + goto out; + } + } + mutex_unlock(&bond_dev->slave_lock); + } + +out: + mutex_unlock(&g_roce5_bond_mutex); + return *rdev ? 0 : -EINVAL; +} + +/* + * roce5_update_rdev_func_val - Update function table values for RoCE device + * @bond_dev: bond device + */ +void roce5_update_rdev_func_val(struct roce5_bond_device *bond_dev) +{ + struct roce5_device *rdev = NULL; + u32 func_tbl_bond; + int ret; + + if (bond_dev->bond_lld_dev == NULL) { + return; + } + + rdev = hinic5_get_uld_dev(bond_dev->bond_lld_dev, SERVICE_T_ROCE); + if (rdev == NULL) { + ROCE_ERR("Failed to find roce device."); + return; + } + + ret = roce5_get_func_table(rdev->hwdev, rdev->glb_func_id, + &func_tbl_bond); + if (ret != 0) { + ROCE_DEV_ERR(rdev, + "Failed to get func table value, func_id(%d).", + rdev->glb_func_id); + uld5_dev_put(bond_dev->bond_lld_dev, SERVICE_T_ROCE); + return; + } + + write_lock(&rdev->rwlock_func_tbl_bond); + rdev->func_tbl_bond = func_tbl_bond; + write_unlock(&rdev->rwlock_func_tbl_bond); + + uld5_dev_put(bond_dev->bond_lld_dev, SERVICE_T_ROCE); +} + +bool roce5_get_bond_ipsurx_en(void) +{ + return g_roce5_bond_ipsurx_en; +} + +void roce5_set_bond_ipsurx_en(bool ipsurx_en) +{ + g_roce5_bond_ipsurx_en = ipsurx_en; +} + +/* + * roce5_bond_init - Initialize RoCE bond module + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_init(void) +{ + int ret; + struct bond_srv_func roce_func = { + .before_active = roce5_before_bond_active, + .after_active = roce5_after_bond_active, + .before_modify = roce5_before_bond_modify, + .after_modify = roce5_after_bond_modify, + .before_deactive = roce5_before_bond_deactive, + .after_deactive = roce5_after_bond_deactive, + .can_attach = roce5_can_do_bond, + }; + + ret = hinic5_bond_register(HINIC5_BOND_USER_ROCE, &roce_func); + if (ret != 0) { + ROCE_ERR("Failed to register bond, ret(%d).", ret); + return ret; + } + + return 0; +} + +void roce5_bond_exit(void) +{ + int ret; + struct roce5_bond_device *bond_dev = NULL; + + mutex_lock(&g_roce5_bond_mutex); + list_for_each_entry(bond_dev, &g_roce5_bond_list, entry) { + bond_dev->need_clean_on_remove = false; + } + mutex_unlock(&g_roce5_bond_mutex); + + ret = hinic5_bond_unregister(HINIC5_BOND_USER_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to unregister service func, ret(%d).", ret); + } +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h new file mode 100644 index 000000000..fd9853215 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_internal.h @@ -0,0 +1,97 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifndef ROCE_BOND_INTERNAL_H +#define ROCE_BOND_INTERNAL_H + +#ifdef ROCE_BONDING_EN + +#include <net/bonding.h> +#include <linux/types.h> +#include <linux/netdevice.h> +#include <linux/mutex.h> +#include <linux/workqueue.h> + +#include "hinic5_lld.h" +#include "drv_bond_api.h" +#include "bond_common_defs.h" +#include "roce_bond.h" + +/* Slave synchronization structure for bond modify */ +struct roce5_bond_slave_sync { + bool exist_arr + [ROCE_BOND_MAX_SLAVE]; /* Indicate which slave need to be deleted */ + struct net_device * + new_ndev[ROCE_BOND_MAX_SLAVE]; /* New slaves need to be added */ + struct netdev_lower_state_info new_ndev_state[BOND_PORT_MAX_NUM]; + u32 new_ndev_num; +}; + +/* Global variables declaration */ +extern struct list_head g_roce5_bond_list; //TODO: later via interface access +extern struct mutex g_roce5_bond_mutex; //TODO: later via interface access +extern struct workqueue_struct *g_bond_wq; //TODO: later via interface access + +/* CFM Interface Layer - Internal Functions */ +bool roce5_bond_mode_is_supported(u16 bond_mode); +int roce5_bond_slave_check(struct hinic5_lld_dev *lld_dev, char *chip_name, + size_t chip_name_size); +bool roce5_bond_before_active_check(struct bond_tracker *tracker, + struct bond_attr *attr); +bool roce5_can_do_bond(struct bonding *bond); + +/* CFM Callback Functions (called by CFM framework) */ +void roce5_before_bond_active(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_after_bond_active(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_before_bond_modify(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_after_bond_modify(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_before_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err); +void roce5_after_bond_deactive(const char *bond_name, struct bond_attr *attr, + int err); + +/* Device Management Layer - Internal Functions */ +struct roce5_bond_device *roce5_get_bond_dev(const char *bond_name); +struct roce5_bond_device *roce5_get_bond_dev_by_name(const char *bond_name); +void roce5_bond_destroy(const char *bond_name, u16 bond_id); +void roce5_update_rdev_func_val(struct roce5_bond_device *bond_dev); +struct hinic5_lld_dev * +roce5_bond_get_ppf_lld(const struct bond_tracker *tracker); +struct roce5_bond_device *roce5_bond_create_bond_device(const char *bond_name); + +/* Slave Management Layer - Internal Functions */ +struct roce5_bond_slave * +roce5_find_next_slave_slot(struct roce5_bond_device *rbdev); +int roce5_get_bond_slave_index_by_ndev(struct roce5_bond_device *rbdev, + struct net_device *ndev); +int roce5_bond_add_slave(struct roce5_bond_device *rbdev, + struct net_device *ndev, + struct netdev_lower_state_info *ndev_info, + bool is_update_table); +int roce5_bond_del_slave(struct roce5_bond_device *rbdev, u32 slave_index); +int roce5_find_same_slave_and_update(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker, + struct roce5_bond_slave_sync *slave_sync); +int roce5_handle_diff_slave(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker); + +/* MAC Table Operation Layer - Internal Functions */ +int roce5_bond_add_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id); +int roce5_bond_del_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id); + +/* TX Port Selection Layer - Internal Functions (declared in roce_bond.h as public) */ + +int roce5_bond_transition_store(struct roce5_bond_device *bond_dev, + struct hinic5_lld_dev *ppf_lld); +void roce5_bond_transition_free(struct roce5_bond_device *bond_dev); + +#endif /* ROCE_BONDING_EN */ + +#endif /* ROCE_BOND_INTERNAL_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c new file mode 100644 index 000000000..75264917f --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_mac.c @@ -0,0 +1,260 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* + * roce5_add_bond_real_slave_mac - Add real slave MAC to table + * @rdev: RoCE device + * @mac: MAC address to add + * Return: 0 on success, negative error code on failure + */ +int roce5_add_bond_real_slave_mac(struct roce5_device *rdev, u8 *mac) +{ + struct roce5_bond_device *bond_dev = rdev->bond_dev; + struct roce5_bond_slave *slave = NULL; + int ret; + u32 i; + + if (bond_dev == NULL) { + ROCE_DEV_ERR(rdev, "Failed to find bond_dev, func_id(%u).", + rdev->glb_func_id); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + + ret = roce5_add_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, + ROCE_BOND_RSVD_VLAN_ID, + slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + ROCE_DEV_ERR(rdev, + "Failed to add mac_vlan entry, ret(%d).", + ret); + return ret; + } + + if (slave->func_id != rdev->glb_func_id) { + /* The IPSU MAC table is used for fast forwarding. Even if the addition fails, the forwarding information + can still be obtained by checking the MAC table later, without judging the execution result. */ + roce5_add_ipsu_tbl_mac_entry(rdev, mac, 0, + rdev->glb_func_id, + slave->er_id, + slave->func_id); + } + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +/* + * roce5_add_bond_vlan_slave_mac - Add VLAN slave Mac to table + * @rdev: RoCE device + * @mac: MAC address to add + * @vlan_id: VLAN ID + * Return: 0 on success, negative error code on failure + */ +int roce5_add_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id) +{ + struct roce5_bond_device *bond_dev = rdev->bond_dev; + struct roce5_bond_slave *slave = NULL; + int ret; + u32 i; + + if (bond_dev == NULL) { + ROCE_DEV_ERR(rdev, "Failed to find bond_dev"); + return -EINVAL; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) { + continue; + } + + ret = roce5_add_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id, + slave->func_id); + if (ret != 0) { + mutex_unlock(&bond_dev->slave_lock); + ROCE_DEV_ERR(rdev, + "Failed to add mac_vlan entry, ret(%d).", + ret); + return ret; + } + + /* The IPSU MAC table is used for fast forwarding. Even if the addition fails, the forwarding information + can still be obtained by checking the MAC table later, without judging the execution result. */ + roce5_add_ipsu_tbl_mac_entry(rdev, mac, vlan_id, + rdev->glb_func_id, slave->er_id, + slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); + + return 0; +} + +/* + * roce5_del_bond_real_slave_mac - Delete real slave Mac from table + * @rdev: RoCE device + */ +void roce5_del_bond_real_slave_mac(struct roce5_device *rdev) +{ + u32 i; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_device *bond_dev = rdev->bond_dev; + + if (bond_dev == NULL) { + return; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id != rdev->glb_func_id) { + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, + 0, slave->er_id, + slave->func_id); + } + + (void)roce5_del_mac_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, + ROCE_BOND_RSVD_VLAN_ID, + slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); +} + +/* + * roce5_del_bond_vlan_slave_mac - Delete VLAN slave Mac from table + * @rdev: RoCE device + * @mac: MAC address to delete + * @vlan_id: VLAN ID + */ +void roce5_del_bond_vlan_slave_mac(struct roce5_device *rdev, u8 *mac, + u16 vlan_id) +{ + u32 i; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_device *bond_dev = rdev->bond_dev; + + if (bond_dev == NULL) { + return; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + slave = &bond_dev->slaves[i]; + if (slave->func_id == rdev->glb_func_id) { + continue; + } + + roce5_del_ipsu_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id, + slave->er_id, slave->func_id); + + (void)roce5_del_mac_tbl_mac_entry(rdev, mac, ETH_ALEN, vlan_id, + slave->func_id); + } + mutex_unlock(&bond_dev->slave_lock); +} + +/* + * roce5_bond_add_slave_table - Add slave table entry + * @rbdev: bond device + * @slave_er_id: slave ER ID + * @slave_func_id: slave function ID + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_add_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id) +{ + struct roce5_device *rdev = rbdev->attached_rdev; + struct roce5_vlan_dev_list *vlan_dev_list = NULL; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + ROCE_ERR("Missing attached roce device."); + return -EINVAL; + } + + /* Add entry of non-vlan device */ + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)rdev->mac, 0, + rdev->glb_func_id, slave_er_id, + slave_func_id); + /* Add entries of vlan device */ + mutex_lock(&rdev->mac_vlan_mutex); + if (rdev->mac_vlan_list_head.next == NULL) { + ROCE_ERR("mac vlan list not init."); + mutex_unlock(&rdev->mac_vlan_mutex); + return -EINVAL; + } + list_for_each_entry(vlan_dev_list, &rdev->mac_vlan_list_head, list) { + roce5_add_ipsu_tbl_mac_entry(rdev, (u8 *)rdev->mac, + vlan_dev_list->vlan_id, + rdev->glb_func_id, slave_er_id, + slave_func_id); + } + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +/* + * roce5_bond_del_slave_table - Delete slave table entry + * @rbdev: bond device + * @slave_er_id: slave ER ID + * @slave_func_id: slave function ID + * Return: 0 on success, negative error code on failure + */ +int roce5_bond_del_slave_table(struct roce5_bond_device *rbdev, u8 slave_er_id, + u16 slave_func_id) +{ + struct roce5_device *rdev = rbdev->attached_rdev; + struct roce5_vlan_dev_list *vlan_dev_list = NULL; + + if ((rdev == NULL) || (rdev->hwdev == NULL)) { + ROCE_ERR("Missing attached roce device."); + return -EINVAL; + } + + /* Delete entry of non-vlan device */ + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, 0, slave_er_id, + slave_func_id); + /* Delete entries of vlan device */ + mutex_lock(&rdev->mac_vlan_mutex); + if (rdev->mac_vlan_list_head.next == NULL) { + ROCE_ERR("mac vlan list not init."); + mutex_unlock(&rdev->mac_vlan_mutex); + return -EINVAL; + } + list_for_each_entry(vlan_dev_list, &rdev->mac_vlan_list_head, list) { + roce5_del_ipsu_tbl_mac_entry(rdev, rdev->mac, ETH_ALEN, + vlan_dev_list->vlan_id, + slave_er_id, slave_func_id); + } + mutex_unlock(&rdev->mac_vlan_mutex); + + return 0; +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c new file mode 100644 index 000000000..a0fee96a4 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_slave.c @@ -0,0 +1,459 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> + +#include "securec.h" + +#include "bond_common_defs.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* + * roce5_find_next_slave_slot - Find next available slave slot + * @rbdev: bond device + * Return: pointer to available slave slot, NULL if no slot available + * Lock: caller must hold rbdev->slave_lock + */ +struct roce5_bond_slave * +roce5_find_next_slave_slot(struct roce5_bond_device *rbdev) +{ + if (rbdev->slave_cnt >= ROCE_BOND_MAX_SLAVE) { + ROCE_ERR( + "RoCE bond device slave slots are full, max number of slaves is %d.", + ROCE_BOND_MAX_SLAVE); + return NULL; + } + + if (rbdev->slaves[rbdev->slave_cnt].netdev != NULL) { + ROCE_ERR("Error occur! Slave slot is broken, slot index(%d).", + rbdev->slave_cnt); + return NULL; + } + + return &rbdev->slaves[rbdev->slave_cnt]; +} + +/* + * roce5_get_bond_slave_index_by_ndev - Get slave index by netdev + * @rbdev: bond device + * @ndev: netdev to search for + * Return: slave index if found, negative error code otherwise + * Lock: caller must hold rbdev->slave_lock + */ +int roce5_get_bond_slave_index_by_ndev(struct roce5_bond_device *rbdev, + struct net_device *ndev) +{ + u32 index; + + for (index = 0; index < rbdev->slave_cnt; ++index) { + if (rbdev->slaves[index].netdev == ndev) { + return (int)index; + } + } + + return -ENODEV; +} + +static void roce5_bond_slave_init(struct roce5_bond_slave *slave, + struct net_device *ndev, + struct hinic5_lld_dev *ndev_lld, + struct netdev_lower_state_info *ndev_info, + bool is_update_table, + struct roce5_bond_device *rbdev) +{ + int ret; + + slave->netdev = ndev; + slave->netdev_state.link_up = ndev_info->link_up; + slave->netdev_state.tx_enabled = ndev_info->tx_enabled; + + slave->lld_dev = ndev_lld; + slave->func_id = hinic5_global_func_id(slave->lld_dev->hwdev); + slave->er_id = hinic5_er_id(slave->lld_dev->hwdev); + + hinic5_detach_service(slave->lld_dev, SERVICE_T_ROCE); + + if (is_update_table) { + ret = roce5_bond_add_slave_table(rbdev, slave->er_id, + slave->func_id); + if (ret != 0) { + ROCE_WARN( + "Failed to add table entries of slave(%u), ret(%d).", + slave->func_id, ret); + } + } +} + +static void roce5_bond_slave_deinit(struct roce5_bond_slave *slave, + bool is_update_table, + struct roce5_bond_device *rbdev) +{ + int ret; + + if (is_update_table) { + ret = roce5_bond_del_slave_table(rbdev, slave->er_id, + slave->func_id); + if (ret != 0) { + ROCE_ERR( + "Failed to delete slave table entries, ret(%d).", + ret); + } + } + + ret = hinic5_attach_service(slave->lld_dev, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR( + "Failed to attach roce function, func_id(%u), ret(%d).", + slave->func_id, ret); + } +} + +/* + * roce5_bond_add_slave - Add slave to bond device + * @rbdev: bond device + * @ndev: slave netdev + * @ndev_info: slave state info + * @is_update_table: whether to update MAC table + * Return: 0 on success, negative error code on failure + * Lock: caller must hold rbdev->slave_lock + */ +int roce5_bond_add_slave(struct roce5_bond_device *rbdev, + struct net_device *ndev, + struct netdev_lower_state_info *ndev_info, + bool is_update_table) +{ + int ret, err_ret; + struct hinic5_lld_dev *ndev_lld = NULL; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_slave slave_temp; + + ndev_lld = hinic5_get_lld_dev_by_netdev(ndev); + if (ndev_lld == NULL) { + ROCE_ERR("Failed to get lld dev by netdev %s.", ndev->name); + return -ENODEV; + } + + if (hinic5_support_roce(ndev_lld->hwdev, NULL) == false) { + return 0; + } + + roce5_bond_slave_init(&slave_temp, ndev, ndev_lld, ndev_info, + is_update_table, rbdev); + + /* Add slave to roce bond device. */ + mutex_lock(&rbdev->slave_lock); + + slave = roce5_find_next_slave_slot(rbdev); + if (slave == NULL) { + ROCE_ERR("Failed to find next slave slot."); + ret = -ENXIO; + mutex_unlock(&rbdev->slave_lock); + goto err_without_slot; + } + + *slave = slave_temp; + ++rbdev->slave_cnt; + + mutex_unlock(&rbdev->slave_lock); + + /* Try to attach roce device to roce bond device. */ + if (rbdev->attached_rdev == NULL) { + if (rbdev->bond_lld_dev != NULL) { + ROCE_ERR("Impossible situation! bond lld isn't NULL."); + ret = -EINVAL; + goto err_check; + } + + rbdev->bond_lld_dev = ndev_lld; + + ret = hinic5_attach_service(rbdev->bond_lld_dev, + SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR( + "Failed to attach roce service, ret(%d), bond name(%s).", + ret, rbdev->name); + ret = -EINVAL; + goto err_attach; + } + + if (rbdev->attached_rdev == NULL) { + ROCE_ERR( + "Failed to attach roce bond device with roce device, bond name(%s).", + rbdev->name); + ret = -EINVAL; + goto err_attach_bond; + } + } + + return 0; + +err_attach_bond: + hinic5_detach_service(rbdev->bond_lld_dev, SERVICE_T_ROCE); +err_attach: + rbdev->bond_lld_dev = NULL; +err_check: + mutex_lock(&rbdev->slave_lock); + err_ret = memset_s(slave, sizeof(struct roce5_bond_slave), 0, + sizeof(struct roce5_bond_slave)); + if (err_ret != 0) { + ROCE_ERR("Failed to clean memory of slave, ret(%d).", err_ret); + } + --rbdev->slave_cnt; + mutex_unlock(&rbdev->slave_lock); +err_without_slot: + roce5_bond_slave_deinit(&slave_temp, is_update_table, rbdev); + return ret; +} + +/* + * roce5_bond_del_slave - Delete slave from bond device + * @rbdev: bond device + * @slave_index: index of slave to delete + * Return: 0 on success, negative error code on failure + * Lock: caller must hold rbdev->slave_lock + */ +int roce5_bond_del_slave(struct roce5_bond_device *rbdev, u32 slave_index) +{ + int ret; + struct roce5_bond_slave *slave = NULL; + struct roce5_bond_slave slave_temp; + size_t mem_copy_len; + + mutex_lock(&rbdev->slave_lock); + + if (slave_index >= rbdev->slave_cnt) { + ROCE_ERR("Invalid parameter, slave index(%u), slave count(%u).", + slave_index, rbdev->slave_cnt); + ret = -EINVAL; + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + + slave = &rbdev->slaves[slave_index]; + if (rbdev->bond_lld_dev != NULL && + slave->lld_dev == rbdev->bond_lld_dev) { + ROCE_WARN( + "Slave(%s) lld dev is the same as the roce bond device(%s), skip removing.", + slave->netdev->name, + (rbdev->attached_rdev != NULL) ? + rbdev->attached_rdev->ib_dev.name : + "NULL"); + ret = 0; + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + + slave_temp = *slave; + + ret = memset_s(slave, sizeof(struct roce5_bond_slave), 0, + sizeof(struct roce5_bond_slave)); + if (ret != 0) { + ROCE_WARN("Failed to clear slave information."); + } + + if (slave_index != ROCE_BOND_MAX_SLAVE - 1) { + mem_copy_len = sizeof(struct roce5_bond_slave) * + (ROCE_BOND_MAX_SLAVE - slave_index - 1); + ret = memmove_s((void *)&rbdev->slaves[slave_index], + mem_copy_len, + (void *)&rbdev->slaves[slave_index + 1], + mem_copy_len); + if (ret != EOK) { + ROCE_ERR("Failed to execute memmove, ret(%d).", ret); + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + } + + ret = memset_s((void *)&rbdev->slaves[ROCE_BOND_MAX_SLAVE - 1], + sizeof(struct roce5_bond_slave), 0, + sizeof(struct roce5_bond_slave)); + if (ret != 0) { + ROCE_ERR("Failed to execute memmove, ret(%d).", ret); + mutex_unlock(&rbdev->slave_lock); + goto err_out; + } + + --rbdev->slave_cnt; + + mutex_unlock(&rbdev->slave_lock); + + if (rbdev->slave_cnt == 0 && rbdev->attached_rdev != NULL) { + if (rbdev->bond_lld_dev == NULL) { + ROCE_ERR("Impossible situation! bond lld is NULL."); + ret = -EINVAL; + goto err_out; + } + + hinic5_detach_service(rbdev->bond_lld_dev, SERVICE_T_ROCE); + if (rbdev->attached_rdev != NULL) { + ROCE_ERR( + "Failed to detach roce service, bond name(%s).", + rbdev->name); + ret = -EINVAL; + goto err_out; + } + + rbdev->bond_lld_dev = NULL; + } + + ret = roce5_bond_del_slave_table(rbdev, slave_temp.er_id, + slave_temp.func_id); + if (ret != 0) { + ROCE_WARN( + "Failed to delete table entries of slave(%u), ret(%d).", + slave_temp.func_id, ret); + ret = 0; + } + + ret = hinic5_attach_service(slave_temp.lld_dev, SERVICE_T_ROCE); + if (ret != 0) { + ROCE_ERR("Failed to attach roce function, func_id(%u).", + slave_temp.func_id); + goto err_out; + } + + return 0; + +err_out: + return ret; +} + +/* + * roce5_find_same_slave_and_update - Find and update existing slave + * @rbdev: bond device + * @tracker: bond tracker from CFM framework + * @slave_sync: slave synchronization info + * Return: 0 on success, negative error code on failure + */ +int roce5_find_same_slave_and_update(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker, + struct roce5_bond_slave_sync *slave_sync) +{ + int ret = 0; + int i; + int roce_slave_index; + int tracker_slave_cnt = 0; + struct hinic5_lld_dev *lld_dev = NULL; + char chip_name[IFNAMSIZ] = { 0 }; + int err; + + ret = memset_s(slave_sync, sizeof(struct roce5_bond_slave_sync), 0, + sizeof(struct roce5_bond_slave_sync)); + if (ret != 0) { + ROCE_ERR("Failed to init slave sync info, ret(%d).", ret); + return ret; + } + + mutex_lock(&rbdev->slave_lock); + + if (rbdev->bond_lld_dev != NULL) { + err = hinic5_get_chip_name(rbdev->bond_lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond dev get chip name err %d.", err); + mutex_unlock(&rbdev->slave_lock); + return err; + } + } + + for (i = 0; i < BOND_PORT_MAX_NUM && tracker_slave_cnt < tracker->cnt; + ++i) { + if (tracker->ndev[i] == NULL) { + continue; + } + + roce_slave_index = roce5_get_bond_slave_index_by_ndev( + rbdev, tracker->ndev[i]); + if (roce_slave_index == -ENODEV) { + lld_dev = + hinic5_get_lld_dev_by_netdev(tracker->ndev[i]); + err = roce5_bond_slave_check(lld_dev, chip_name, + sizeof(chip_name)); + if (err != 0) { + ROCE_ERR("Bond slave check err %d.", err); + mutex_unlock(&rbdev->slave_lock); + return err; + } + + slave_sync->new_ndev[slave_sync->new_ndev_num] = + tracker->ndev[i]; + slave_sync->new_ndev_state[slave_sync->new_ndev_num] = + tracker->netdev_state[i]; + ++slave_sync->new_ndev_num; + } else { + slave_sync->exist_arr[roce_slave_index] = true; + + /* update state of the slave */ + rbdev->slaves[roce_slave_index].netdev_state.link_up = + tracker->netdev_state[i].link_up; + rbdev->slaves[roce_slave_index].netdev_state.tx_enabled = + tracker->netdev_state[i].tx_enabled; + } + + ++tracker_slave_cnt; + } + + mutex_unlock(&rbdev->slave_lock); + + return ret; +} + +/* + * roce5_handle_diff_slave - Handle different slaves during bond modify + * @rbdev: bond device + * @tracker: bond tracker from CFM framework + * Return: 0 on success, negative error code on failure + */ +int roce5_handle_diff_slave(struct roce5_bond_device *rbdev, + struct bond_tracker *tracker) +{ + u32 i; + int ret = 0; + struct roce5_bond_slave_sync slave_sync; + + ret = roce5_find_same_slave_and_update(rbdev, tracker, &slave_sync); + if (ret != 0) { + ROCE_ERR("Failed to find same netdev."); + return ret; + } + + /* delete slaves */ + for (i = rbdev->slave_cnt; i > 0; --i) { + if (slave_sync.exist_arr[i - 1] == false) { + ret = roce5_bond_del_slave(rbdev, i - 1); + if (ret != 0) { + ROCE_ERR("Failed to delete slave, ret(%d).", + ret); + return ret; + } + } + } + + /* add new slaves */ + for (i = 0; i < slave_sync.new_ndev_num; ++i) { + ret = roce5_bond_add_slave(rbdev, slave_sync.new_ndev[i], + &slave_sync.new_ndev_state[i], true); + if (ret != 0) { + ROCE_ERR("Failed to add new slave, ret(%d).", ret); + return ret; + } + } + + return ret; +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c new file mode 100644 index 000000000..dd34d319e --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/bond/roce_bond_tx.c @@ -0,0 +1,94 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2026-2026. All rights reserved. + + +#ifdef ROCE_BONDING_EN + +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <net/bonding.h> +#include <linux/atomic.h> + +#include "securec.h" + +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "roce_bond.h" +#include "roce_bond_internal.h" +#include "roce_cmd.h" +#include "roce_netdev.h" +#include "roce_infra.h" + +/* + * roce5_bond_get_netdev - Get bond netdev for TX + * @rdev: RoCE device + * Return: netdev with increased reference count, NULL if no bond + * Refcount: caller must call dev_put() when done + */ +struct net_device *roce5_bond_get_netdev(struct roce5_device *rdev) +{ + u32 i; + struct roce5_bond_device *bond_dev = rdev->bond_dev; + struct net_device *ret_dev = NULL; + struct slave *slave = NULL; + + mutex_lock(&g_roce5_bond_mutex); + if (bond_dev == NULL) { + mutex_unlock(&g_roce5_bond_mutex); + return ret_dev; + } + + mutex_lock(&bond_dev->slave_lock); + for (i = 0; i < bond_dev->slave_cnt; i++) { + rcu_read_lock(); + slave = bond_slave_get_rcu(bond_dev->slaves[i].netdev); + rcu_read_unlock(); + if (slave == NULL) { + continue; + } + if (bond_is_active_slave(slave)) { + if (netif_running(bond_dev->slaves[i].netdev) && + netif_carrier_ok(bond_dev->slaves[i].netdev)) { + ret_dev = bond_dev->slaves[i].netdev; + } else if (netif_running( + bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt] + .netdev) && + netif_carrier_ok( + bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt] + .netdev)) { + ret_dev = bond_dev->slaves[(i + 1) % + bond_dev->slave_cnt] + .netdev; + } else { + ret_dev = bond_dev->slaves[i].netdev; + } + dev_hold(ret_dev); + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce5_bond_mutex); + return ret_dev; + } + } + mutex_unlock(&bond_dev->slave_lock); + mutex_unlock(&g_roce5_bond_mutex); + return ret_dev; +} + +/* + * roce5_bond_rr_set_flow - Set flow hash for round-robin TX selection + * @rdev: RoCE device + * @rqp: RoCE QP + * @qp_attr: QP attributes + */ +void roce5_bond_rr_set_flow(struct roce5_device *rdev, struct roce5_qp *rqp, + roce_verbs_qp_attr_s *qp_attr) +{ + rqp->tx_hash_value = (u32)atomic_add_return(1, &rdev->next_port); + qp_attr->path_info.dw0.bs.bond_tx_hash_value = + (rqp->tx_hash_value & 0xf); +} + +#endif /* ROCE_BONDING_EN */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h b/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h new file mode 100644 index 000000000..6b919d753 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/debug/roce_debug.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_DEBUG_H +#define ROCE_DEBUG_H + +/* + * DEBUG_ONLY: statement is only valid in debug mode. + * + * RELEASE_ONLY: statement is only valid in release mode. + * + * DEBUG_REPLACE: used in functions of release to indicate the function may be replaced by another function with the + * same name defined in the debug mode. + */ +#ifdef ENABLE_DEBUG +#define DEBUG_ONLY(statement) statement +#define RELEASE_ONLY(statement) +#define DEBUG_REPLACE __attribute__((weak)) +#else +#define DEBUG_ONLY(statement) +#define RELEASE_ONLY(statement) statement +#define DEBUG_REPLACE +#endif + +#endif /* ROCE_DEBUG_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c new file mode 100644 index 000000000..787a86791 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.c @@ -0,0 +1,375 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. + + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> + +#include "roce.h" +#include "hinic5_mt.h" +#include "roce_cc_format.h" +#include "roce_cmd.h" +#include "roce_mpu_cmd.h" +#include "roce_mpu_cmd_defs.h" +#include "securec.h" +#include "roce_dfx.h" + +int roce5_get_drv_version(void *buf_out, u32 *out_size) +{ + struct drv_version_info *ver_info = buf_out; + int rc; + + if (!buf_out || !out_size) { + ROCE_ERR("Input params is NULL."); + return -EINVAL; + } + + if (*out_size != sizeof(*ver_info)) { + ROCE_ERR("Unexpect out buf size from user :%u, expect: %lu", + *out_size, sizeof(*ver_info)); + return -EINVAL; + } + + rc = snprintf_s(ver_info->ver, sizeof(ver_info->ver), + sizeof(ver_info->ver) - 1, "%s %s", + HIROCE5_DRV_VERSION, __TIME_STR__); + if (rc == -1) { + ROCE_ERR("Snprintf roce version err"); + return -EFAULT; + } + + return 0; +} + +void * +roce5_adm_generic_find_cmd_func(const roce5_adm_generic_cmd_proc_t *cmd_array, + int elem_count, u32 target_cmd) +{ + int i; + + if (cmd_array == NULL || elem_count <= 0) { + return NULL; + } + + for (i = 0; i < elem_count; i++) { + if (cmd_array[i].cmd_type == target_cmd && + cmd_array[i].proc_func != NULL) { + return cmd_array[i].proc_func; + } + } + return NULL; +} + +static int roce5_dfx_enable_bw_ctrl(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + if (rdev->hw_info.hca_type == ROCE5_2_100G_HCA) { + inbuf->ctrl_param.cir = ROCE5_100G_CIR; + inbuf->ctrl_param.pir = ROCE5_100G_PIR; + inbuf->ctrl_param.cnp = ROCE5_100G_CNP; + } else { + inbuf->ctrl_param.cir = ROCE5_25G_CIR; + inbuf->ctrl_param.pir = ROCE5_25G_PIR; + inbuf->ctrl_param.cnp = ROCE5_25G_CNP; + } + return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_EN, inbuf); +} + +static int roce5_dfx_disable_bw_ctrl(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + inbuf->ctrl_param.cir = 0; + inbuf->ctrl_param.pir = 0; + inbuf->ctrl_param.cnp = 0; + return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_DIS, inbuf); +} + +static int roce5_dfx_change_bw_ctrl_param(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + return roce5_set_bw_ctrl_state(rdev, ROCE_BW_CTRL_RESET, inbuf); +} + +static int roce5_dfx_query_bw_ctrl_param(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf) +{ + return roce5_query_bw_ctrl_state(rdev, &outbuf->bw_ctrl_param); +} + +static const roce5_adm_dfx_bw_ctrl_cmd_proc_t + g_roce5_adm_dfx_bw_ctrl_cmd_proc[] = { + { ROCE_CMD_ENABLE_BW_CTRL, roce5_dfx_enable_bw_ctrl }, + { ROCE_CMD_DISABLE_BW_CTRL, roce5_dfx_disable_bw_ctrl }, + { ROCE_CMD_CHANGE_BW_CTRL_PARAM, + roce5_dfx_change_bw_ctrl_param }, + { ROCE_CMD_QUERY_BW_CTRL_PARAM, roce5_dfx_query_bw_ctrl_param }, + }; + +int roce5_adm_dfx_bw_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + struct roce5_bw_ctrl_inbuf *inbuf = + (struct roce5_bw_ctrl_inbuf *)buf_in; + struct roce5_bw_ctrl_outbuf *outbuf = + (struct roce5_bw_ctrl_outbuf *)buf_out; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *) + g_roce5_adm_dfx_bw_ctrl_cmd_proc; + int elem_count = sizeof(g_roce5_adm_dfx_bw_ctrl_cmd_proc) / + sizeof(g_roce5_adm_dfx_bw_ctrl_cmd_proc[0]); + roce5_adm_dfx_bw_ctrl_func_t roce5_adm_dfx_bw_ctrl_func = NULL; + + if (in_size < sizeof(struct roce5_bw_ctrl_inbuf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct roce5_bw_ctrl_inbuf)); + return -ENOMEM; + } + if (*out_size < sizeof(struct roce5_bw_ctrl_outbuf)) { + ROCE_ERR( + "Input parameter(out_zise: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, + (int)sizeof(struct roce5_bw_ctrl_outbuf)); + return -ENOMEM; + } + ret = memset_s(buf_out, sizeof(struct roce5_bw_ctrl_outbuf), 0, + sizeof(struct roce5_bw_ctrl_outbuf)); + if (ret != 0) { + ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret); + return -ENOMEM; + } + *out_size = sizeof(struct roce5_bw_ctrl_outbuf); + + if (inbuf->cmd_type >= VM_COMPAT_TEST) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + roce5_adm_dfx_bw_ctrl_func = + (roce5_adm_dfx_bw_ctrl_func_t)roce5_adm_generic_find_cmd_func( + generic_array, elem_count, inbuf->cmd_type); + if (roce5_adm_dfx_bw_ctrl_func == NULL) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + return roce5_adm_dfx_bw_ctrl_func(rdev, inbuf, outbuf); +} + +int roce5_adm_dfx_set_ldcp_param(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + roce_ldcp_param_s ldcp_param; + const struct roce5_ldcp_param *inbuf = + (struct roce5_ldcp_param *)buf_in; + + if (in_size < sizeof(struct roce5_ldcp_param)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(struct roce5_ldcp_param)); + return -ENOMEM; + } + if (inbuf->cmd_type != ROCE_CMD_SET_LDCP_PARAM) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + ldcp_param.dw0.bs.alpha = inbuf->ldcp_param.alpha & 0xf; + ldcp_param.dw0.bs.beta = inbuf->ldcp_param.beta & 0xf; + ldcp_param.dw0.bs.gamma = inbuf->ldcp_param.gamma & 0xf; + ldcp_param.dw0.bs.eta = inbuf->ldcp_param.eta & 0xf; + ldcp_param.dw0.bs.set_flag = inbuf->ldcp_param.set_flag & 0x1; + + ret = roce5_set_sysfs_cfg_param(rdev->hwdev, rdev->glb_func_id, + (u32 *)(void *)&ldcp_param, + sizeof(roce_ldcp_param_s), + (u16)ROCE_MPU_CMD_CC_CFG_LDCP_PARAM); + if (ret != 0) { + ROCE_ERR("Failed to set ldcp param.(ret:%d)", ret); + } else { + ROCE_WARN("Set ldcp param 0x%x", ldcp_param.dw0.value); + } + return ret; +} + +int roce5_adm_dfx_port_traffic(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + roce_mpu_port_statics_cmd_inbuf_t inbuf; + roce_port_sta_outbuf_t *outbuf = (roce_port_sta_outbuf_t *)buf_out; + int ret; + + memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf)); + inbuf.cmd = *(roce_port_sta_inbuf_t *)buf_in; + outbuf->head.status = 0xFF; + + if (in_size < sizeof(roce_port_sta_inbuf_t)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(roce_port_sta_inbuf_t)); + return -ENOMEM; + } + if (inbuf.cmd.cmd_type != ROCE_CMD_PORT_TRAFFIC) { + ROCE_DEV_ERR(rdev, "Not support this type(%d).", + inbuf.cmd.cmd_type); + return -EINVAL; + } + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DFX_PORT_TRAFFIC, + (void *)&inbuf, (u16)sizeof(inbuf), + buf_out, (u16 *)(void *)out_size); + if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) { + ROCE_ERR( + "Failed to query port traffic, err(%d), out size(0x%x) status(0x%x)", + ret, *out_size, outbuf->head.status); + return -EINVAL; + } + + return 0; +} + +#ifdef ROCE_CHIP_TEST +static const roce5_adm_dfx_vnic_waterline_cmd_proc_t + g_roce5_adm_dfx_vnic_wl_cmd_proc[] = { + { ROCE_CMD_SET_VNIC_WATERLINE, roce5_set_vnic_waterline_state }, + { ROCE_CMD_GET_VNIC_WATERLINE, roce5_get_vnic_waterline_state }, + }; + +int roce5_adm_dfx_vnic_wl_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + struct roce5_vnic_waterline_ctrl_buf *inbuf = + (struct roce5_vnic_waterline_ctrl_buf *)buf_in; + struct roce5_vnic_waterline_ctrl_buf *outbuf = + (struct roce5_vnic_waterline_ctrl_buf *)buf_out; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *) + g_roce5_adm_dfx_vnic_wl_cmd_proc; + int elem_count = sizeof(g_roce5_adm_dfx_vnic_wl_cmd_proc) / + sizeof(g_roce5_adm_dfx_vnic_wl_cmd_proc[0]); + roce5_adm_dfx_vnic_waterline_ctrl_func_t roce5_adm_dfx_vnic_wl_ctrl_func; + + if (in_size < sizeof(struct roce5_vnic_waterline_ctrl_buf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, + (int)sizeof(struct roce5_vnic_waterline_ctrl_buf)); + return -EINVAL; + } + + if (inbuf->cmd_type >= VM_COMPAT_TEST) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + ret = memset_s(buf_out, sizeof(struct roce5_vnic_waterline_ctrl_buf), 0, + sizeof(struct roce5_vnic_waterline_ctrl_buf)); + if (ret != 0) { + ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret); + return -ENOMEM; + } + + roce5_adm_dfx_vnic_wl_ctrl_func = + (roce5_adm_dfx_vnic_waterline_ctrl_func_t) + roce5_adm_generic_find_cmd_func( + generic_array, elem_count, inbuf->cmd_type); + if (roce5_adm_dfx_vnic_wl_ctrl_func == NULL) { + ROCE_DEV_ERR(rdev, "Not support this type(%d)", + inbuf->cmd_type); + return -EINVAL; + } + + return roce5_adm_dfx_vnic_wl_ctrl_func(rdev, inbuf, outbuf); +} + +#endif /* ROCE_CHIP_TEST */ + +int roce5_adm_dfx_roce_set(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) + +{ + roce_dfx_set_cmd_s inbuf; + roce_dfx_set_outbuf_s *outbuf = (roce_dfx_set_outbuf_s *)buf_out; + int ret; + + if (in_size < sizeof(roce_dfx_set_inbuf_s)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(roce_dfx_set_inbuf_s)); + return -ENOMEM; + } + + memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf)); + inbuf.cmd = *(roce_dfx_set_inbuf_s *)buf_in; + inbuf.cmd.func_id = rdev->glb_func_id; + + if (inbuf.cmd.cmd_type != ROCE_CMD_ROCE_SET) { + ROCE_DEV_ERR(rdev, "Not support this type(%d).", + inbuf.cmd.cmd_type); + return -EINVAL; + } + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, ROCE_MPU_CMD_DFX_ROCE_SET, + (void *)&inbuf, (u16)sizeof(inbuf), outbuf, + (u16 *)(void *)out_size); + if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) { + ROCE_ERR( + "Failed to set roce configs, err(%d), out size(0x%x) status(0x%x)", + ret, *out_size, outbuf->head.status); + return -EINVAL; + } + return 0; +} + +int roce5_adm_dfx_roce_set_debug(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) + +{ + roce_dfx_set_debug_cmd_s inbuf; + roce_dfx_set_debug_outbuf_s *outbuf = + (roce_dfx_set_debug_outbuf_s *)buf_out; + int ret; + + if (in_size < sizeof(roce_dfx_set_debug_inbuf_s)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, (int)sizeof(roce_dfx_set_debug_inbuf_s)); + return -ENOMEM; + } + + memset_s(&inbuf, sizeof(inbuf), 0, sizeof(inbuf)); + inbuf.cmd = *(roce_dfx_set_debug_inbuf_s *)buf_in; + inbuf.cmd.func_id = rdev->glb_func_id; + + if (inbuf.cmd.cmd_type != ROCE_CMD_ROCE_SET_DEBUG) { + ROCE_DEV_ERR(rdev, "Not support this type(%d).", + inbuf.cmd.cmd_type); + return -EINVAL; + } + + ret = roce5_msg_to_mgmt_sync(rdev->hwdev, + ROCE_MPU_CMD_DFX_ROCE_SET_DEBUG, + (void *)&inbuf, (u16)sizeof(inbuf), outbuf, + (u16 *)(void *)out_size); + if ((ret != 0) || (*out_size == 0) || (outbuf->head.status != 0)) { + ROCE_ERR( + "Failed to set roce configs, err(%d), out size(0x%x) status(0x%x)", + ret, *out_size, outbuf->head.status); + return -EINVAL; + } + return 0; +} +#endif /* __ROCE_DFX__ */ \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h new file mode 100644 index 000000000..139867581 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx.h @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. + + +#ifndef ROCE_DFX_H +#define ROCE_DFX_H + +#include <linux/types.h> + +#include "hinic5_rdma.h" + +#include "roce_sysfs.h" +#include "roce.h" +#include "roce_gid.h" +#include "roce_compat.h" +#include "roce_pub_cmd.h" +#include "roce_npu_cmd_dfx_defs.h" +#include "roce_debug.h" + +#include "securec.h" + +#define MR_KEY_2_INDEX_SHIFT 8 + +#define ROCE_IO_DFX_CFG_VADDR_ID 0 +#define ROCE_IO_DFX_CFG_PADDR_ID 1 +#define ROCE_IO_DFX_CFG_ADDR_NUM 2 + +#define roce5_dfx_print(fmt, args...) \ + do { \ + (void)printk("[roce5_dfx] : " fmt "\n", ##args); \ + } while (0) + +typedef struct tag_roce_cmd_mpt_query { + roce_verbs_cmd_com_s com; +} roce_cmd_mpt_query_s; + +typedef struct roce5_mpt_query_outbuf { + u32 value[16]; +} roce5_mpt_query_outbuf_s; + +typedef struct tag_roce_cmd_roce_cq_query { + roce_verbs_cmd_com_s com; +} roce_cmd_cq_query_s; + +typedef struct roce5_cq_query_outbuf { + u32 value[32]; +} roce5_cq_query_outbuf_s; + +typedef struct tag_roce_cmd_qp_query { + roce_verbs_cmd_com_s com; +} roce_cmd_qp_query_s; + +typedef struct roce5_qp_query_outbuf { + u32 context[128]; +} roce5_qp_query_outbuf_s; + +typedef struct roce5_srq_query_outbuf { + u32 value[24]; +} roce5_srq_query_outbuf_s; + +struct roce5_dfx_query_inbuf { + u32 cmd_type; + union { + u32 qpn; + u32 cqn; + u32 srqn; + u32 mpt_key; + u32 gid_index; + struct { + u32 qpn; + u32 cqn; + } query_pi_ci; + }; +}; + +#define ROCE_DEV_NAME_LEN_MAX 64 + +struct roce5_dfx_dbg_inbuf { + u32 cmd_type; + union { + u32 qpn; + u32 srqn; + }; + char dev_name[ROCE_DEV_NAME_LEN_MAX]; +}; + +#define ROCE_CTX_MAX_SIZE 512 // adaptation lock store referencein of structure + +union roce5_dfx_dbg_outbuf { + roce_verbs_qp_attr_s qp_attr; + roce_verbs_srq_attr_dfx_s srq_attr; + u32 buf[ROCE_CTX_MAX_SIZE]; + u32 algo_type; + u32 dev_type; +}; + +struct roce5_dfx_common_inbuf { + u32 cmd_type; + char dev_name[ROCE_DEV_NAME_LEN_MAX]; +}; + +union roce5_dfx_common_outbuf { + u32 dev_type; +}; + +struct roce5_dfx_pi_ci { + u32 qpc_sq_pi_on_chip; + u32 qpc_sq_pi; + u32 qpc_sq_load_pi; + u32 qpc_rq_pi_on_chip; + u32 qpc_rq_load_pi; + u32 qpc_rq_pi; + u32 qpc_rc_pi; + u32 qpc_sq_ci; + u32 qpc_sq_wqe_prefetch_ci; + u32 qpc_sq_mtt_prefetch_wqe_ci; + u32 qpc_sqa_ci; + u32 qpc_sqa_wqe_prefetch_ci; + u32 qpc_rq_ci; + u32 qpc_rq_wqe_prefetch_ci; + u32 qpc_rq_mtt_prefetch_wqe_ci; + u32 qpc_rq_base_ci; + u32 qpc_rc_ci; + u32 qpc_rc_prefetch_ci; + u32 cq_ci_on_chip; + u32 cq_ci; + u32 cq_load_ci; + u64 cq_ci_record_gpa_at_hop_num; + u32 cq_last_solicited_pi; + u32 cq_pi; + u32 cq_last_notified_pi; +}; + +struct roce5_dfx_qp_count { + u32 qp_alloced; + u32 qp_deleted; + u32 qp_alive; +}; + +#define ROCE_GID_ENTRY_NUM 17 // 1 host fe + 16 shadow fe +union roce5_dfx_query_outbuf { + /* query cache context */ + roce_verbs_qp_attr_s qp_attr; + roce_verbs_cq_attr_s cq_attr; + roce_verbs_srq_attr_s srq_attr; + roce_verbs_mr_attr_s mr_attr; + /* query host context */ + roce5_qp_query_outbuf_s qp_query; + roce5_cq_query_outbuf_s cq_query; + roce5_srq_query_outbuf_s srq_query; + struct roce_mpt_context mpt; + struct rdma_gid_entry gid_entry[ROCE_GID_ENTRY_NUM]; + struct roce5_dfx_pi_ci pi_ci; + struct roce5_dfx_qp_count qp_count; + u32 algo_type; + struct roce_uni_cmd_latch_data latch_data_outbuf; +}; + +enum roce5_bw_ctrl_cmd_e { + ROCE_BW_CTRL_DIS, + ROCE_BW_CTRL_EN, + ROCE_BW_CTRL_RESET +}; + +enum roce5_qos_wl_ctrl_cmd_e { + ROCE_QOS_WL_CTRL_CMD_SET, + ROCE_QOS_WL_CTRL_CMD_GET +}; + +struct rdma_gid_query_outbuf { + struct rdma_gid_entry gid_entry[ROCE_GID_ENTRY_NUM]; +}; + +struct roce5_bw_ctrl_inbuf { + u32 cmd_type; + struct { + u32 cir; + u32 pir; + u32 cnp; + } ctrl_param; +}; + +struct roce5_bw_set_inbuf { + u32 cmd_type; + struct { + u32 mod; + u32 cir; + u32 cbs; + u32 pir; + u32 pbs; + } set_param; +}; + +struct roce5_bw_ctrl_param { + u8 color_type; + u16 ptype; + u8 hw_wred_mode; + + u32 cir; + u32 pir; + u32 cbs; + u32 xbs; + u32 cnp; + u32 enable; +}; + +struct roce5_bw_set_param { + u16 ptype; + u8 color_type; + u8 hw_wred_mode; + + u32 cir; + u32 cbs; + u32 pir; + u32 pbs; + u32 enable; +}; + +struct roce5_ldcp_param { + u32 cmd_type; + struct { + u8 alpha; + u8 beta; + u8 gamma; + u8 eta; + u8 set_flag; + } ldcp_param; +}; + +struct roce5_bw_ctrl_outbuf { + struct roce5_bw_ctrl_param bw_ctrl_param; +}; + +struct roce5_bw_set_outbuf { + struct roce5_bw_set_param bw_set_param; +}; + +enum roce5_dfx_io_cmd_type { ROCE_IO_CTRL_DIS, ROCE_IO_CTRL_EN }; + +struct roce5_dfx_io_alarm { + enum roce5_dfx_io_cmd_type en_flag; + u16 pf_id; + u16 rsvd; + u16 io_latency_thd; + u16 exec_time; + u32 exp_qpn; + void *rcd_uaddr; + struct timespec64 start; + struct mutex io_alarm_mutex; +}; + +struct roce5_dfx_io_inbuf { + u32 cmd_type; + struct roce5_dfx_io_alarm io_alarm; +}; + +struct roce5_dfx_io_outbuf { + struct roce5_dfx_io_alarm io_alarm; +}; + +typedef struct roce5_adm_generic_cmd_proc { + u32 cmd_type; + void *proc_func; +} roce5_adm_generic_cmd_proc_t; + +typedef int (*roce5_adm_func_t)(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +typedef struct roce5_adm_cmd_proc { + u32 cmd_type; + roce5_adm_func_t proc_func; +} roce5_adm_cmd_proc_t; + +typedef int (*roce5_adm_dfx_dbg_t)(struct roce5_device *rdev, + const struct roce5_dfx_dbg_inbuf *inbuf, + union roce5_dfx_dbg_outbuf *outbuf); + +typedef struct roce5_adm_dfx_dbg_cmd_proc { + u32 cmd_type; + roce5_adm_dfx_dbg_t proc_func; +} roce5_adm_dfx_dbg_cmd_proc_t; + +typedef int (*roce5_adm_dfx_query_t)(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf); + +typedef struct roce5_adm_dfx_query_cmd_proc { + u32 cmd_type; + roce5_adm_dfx_query_t proc_func; +} roce5_adm_dfx_query_cmd_proc_t; + +typedef int (*roce5_adm_dfx_bw_ctrl_func_t)(struct roce5_device *rdev, + struct roce5_bw_ctrl_inbuf *inbuf, + struct roce5_bw_ctrl_outbuf *outbuf); + +typedef struct roce5_adm_dfx_bw_ctrl_cmd_proc { + u32 cmd_type; + roce5_adm_dfx_bw_ctrl_func_t proc_func; +} roce5_adm_dfx_bw_ctrl_cmd_proc_t; + +#ifdef ROCE_CHIP_TEST +typedef struct roce5_vnic_waterline_ctrl_param { + u16 red_waterline; + u16 yellow_waterline; + u32 ip_val; +} roce5_vnic_waterline_ctrl_param_s; + +struct roce5_vnic_waterline_ctrl_buf { + u32 cmd_type; + roce5_vnic_waterline_ctrl_param_s waterline_param; +}; + +typedef int (*roce5_adm_dfx_vnic_waterline_ctrl_func_t)( + struct roce5_device *rdev, struct roce5_vnic_waterline_ctrl_buf *inbuf, + struct roce5_vnic_waterline_ctrl_buf *outbuf); + +typedef struct roce5_adm_dfx_vnic_waterline_cmd_proc { + u32 cmd; + roce5_adm_dfx_vnic_waterline_ctrl_func_t proc_func; +} roce5_adm_dfx_vnic_waterline_cmd_proc_t; + +int roce5_adm_dfx_vnic_wl_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +#endif /* ROCE_CHIP_TEST */ + +/* and above next is V100 compatible data structure */ +typedef struct tag_roce_qurey_gid { + roce_verbs_cmd_com_s com; + __be32 port; +} roce_cmd_query_gid_s; + +typedef int (*roce_adm_dfx_attack_func_t)(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf); +struct roce_dfx_attack_cmd_func { + u32 cmd_id; + roce_adm_dfx_attack_func_t cmd_func; +}; + +DEBUG_REPLACE int roce5_get_drv_version(void *buf_out, u32 *out_size); +void * +roce5_adm_generic_find_cmd_func(const roce5_adm_generic_cmd_proc_t *cmd_array, + int elem_count, u32 target_cmd); +int roce5_adm_dfx_query(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_bw_ctrl(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_set_ldcp_param(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_port_traffic(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_dbg(struct roce5_device *rdev, const void *buf_in, + void *buf_out, u32 *out_size); +int roce5_adm_dfx_roce_set(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int roce5_adm_dfx_roce_set_debug(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +int roce5_dfx_attack_func(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +void roce5_dfx_clean_up(struct roce5_device *rdev); + +void *global_roce5_io_alarm_va_get(void); + +void global_roce5_io_alarm_va_set(u64 va); + +void global_roce5_io_alarm_pa_set(dma_addr_t pa); + +dma_addr_t global_roce5_io_alarm_pa_get(void); + +int roce5_dfx_query_cq(struct roce5_device *rdev, u32 cqn, u32 *context, + int cqc_size); + +int roce5_dfx_cmd_query_srq(struct roce5_device *rdev, u32 srqn, + roce_verbs_srq_attr_s *srq_attr); + +#endif // __ROCE_DFX_H__ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c new file mode 100644 index 000000000..84cc1455b --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_attack.c @@ -0,0 +1,828 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/semaphore.h> + +#include "hinic5_hw.h" +#include "hinic5_hwdev.h" +#include "hisdk5_hwif.h" +#include "hinic5_mt.h" +#include "rdma_comp.h" +#include "roce.h" +#include "roce_qp.h" +#include "roce_infra.h" +#include "roce_mr.h" +#include "roce_dfx.h" +#include "roce_kernel_compat_gen.h" + +#define DB_BUFF_SIZE 2048 +#define DWQE_SIZE 256 +#define DWQE_TYPE_NUM 4 +#define ALIGN_UP_8B(length) (((length) + 7) >> 3) +#define ALIGN_DOWN_8B(offset) ((offset)&0xfffffff8) +#define IS_OFFSET_8B_ALIGN(offset) (((offset)&7) != 0) +#define UPPER_HALF_BIT_OFFSET 32 +#define U32S_PER_U64 2 + +static void roce5_copy_mask_attack_value_to_addr(u32 *addr, u32 attack_value, + u32 mask) +{ + u32 val; + + val = *addr; + val &= ~mask; + val |= (attack_value & mask); + *addr = val; +} + +static void roce5_copy_mask_attack_value_to_u64_addr(u64 *addr, + u32 attack_value, u32 mask, + bool is_upper_half) +{ + u64 val; + + val = *addr; + + if (is_upper_half) { + val &= ~((u64)mask << UPPER_HALF_BIT_OFFSET); + val |= ((u64)(attack_value & mask) << UPPER_HALF_BIT_OFFSET); + } else { + val &= ~mask; + val |= (attack_value & mask); + } + + *addr = val; +} + +static int roce5_copy_data_to_page(struct roce5_device *rdev, const u32 *val, + struct ib_umem *umem, u32 copy_len, + u32 offset, u32 attack_len, u32 mask) +{ + struct scatterlist *sg = NULL; + u32 *inbuf_tmp = NULL; + u32 sg_len; + u32 page_copy_len = 0; + u32 copied_len = 0; + u32 sgl_offset = 0; + u64 buf_offset = 0; + void *va = NULL; + u32 i = 0; + int ret; + + inbuf_tmp = (u32 *)kzalloc(PAGE_SIZE, GFP_KERNEL); + if (inbuf_tmp == NULL) { + ROCE_ERR("Failed to alloc inbuf temp."); + return -ENOMEM; + } + + if (mask == 0) { + for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) { + inbuf_tmp[i] = val[i % attack_len]; + } + } + + ROCE_INFO( + "copy_data_to_page va:%lx, length:%lu, offset:%u, copy_len:%u", + umem->address, umem->length, offset, copy_len); + +#if defined(IB_UMEM_HAVE_SG_APPEND_TABLE) + for_each_sg(umem->sgt_append.sgt.sgl, sg, umem->sgt_append.sgt.nents, i) +#else + //scatterlist of traverse use nents, orig_nents >= nents >= nmap + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, i) +#endif + { + sg_len = sg->length; + if (buf_offset + sg_len <= + offset) { //not found offset of scatterlist, accumulate sg_len, continue 往 next find + buf_offset += sg_len; + } else { //found offset of scatterlist + sgl_offset = + offset - + (u32)buf_offset; //get based on current scatterlist of offset + buf_offset = + offset; //found current page after, later of page of offset all max at offset + + va = sg_virt(sg); + if (va == NULL) { + ROCE_ERR("Failed to get va"); + kfree(inbuf_tmp); + return -ENOMEM; + } + + va += sgl_offset; + page_copy_len = ((copy_len - copied_len) < + (sg_len - sgl_offset)) ? + (copy_len - copied_len) : + (sg_len - sgl_offset); + + if (mask != 0) { + roce5_copy_mask_attack_value_to_addr( + (u32 *)va, val[0], mask); + ROCE_INFO( + "copy mask data to buf, buf_offset:%llu, sgl_offset:%u, mask_val:%u\n", + buf_offset, sgl_offset, + (val[0] & mask)); + break; + } + + ret = memcpy_s(va, page_copy_len, inbuf_tmp, + page_copy_len); + if (ret != 0) { + ROCE_ERR( + "Failed to memcpy attack value, ret(%d)", + ret); + kfree(inbuf_tmp); + return -EFAULT; + } + + ROCE_INFO( + "copy_data_from_page buf_offset:%llu, sgl_offset:%u, copied_len:%u, page_copy_len:%u, i:%u\n", + buf_offset, sgl_offset, copied_len, + page_copy_len, i); + + copied_len += page_copy_len; + if (copied_len >= copy_len) { + break; + } + } + } + + kfree(inbuf_tmp); + return 0; +} + +static void roce5_copy_attack_value_to_addr(u64 *addr, u64 attack_value, + u32 length) +{ + int i; + int count = ALIGN_UP_8B(length); + + for (i = 0; i < count; i++) { + wmb(); + *((u64 *)addr + i) = attack_value; + wmb(); + } +} + +static int roce5_dfx_attack_sqdb_hard(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_ucontext *ucontext = NULL; + u64 db_phy_start; + void __iomem *db_virt_start = NULL; + void __iomem *db_virt_offset = NULL; + void *hwdev = NULL; + struct roce5_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + union roce_sq_db sq_db; + u32 db_offset; + u32 db_8B_offset; + bool is_upper_half; + u32 length; + + hwdev = rdev->hwdev; + if (hwdev == NULL) { + ROCE_ERR("Roce hwdev is NULL"); + return -ENOMEM; + } + + ucontext = rdev->ucontext; + if (ucontext == NULL) { + ROCE_ERR("Roce device context is null"); + return -ENOMEM; + } + + db_offset = inbuf->offset; + if (db_offset >= DB_BUFF_SIZE) { + ROCE_ERR( + "roce attack sqdb_hard offset is too large, should be less than %u", + DB_BUFF_SIZE); + return -EINVAL; + } + + db_8B_offset = ALIGN_DOWN_8B(db_offset); + db_phy_start = ucontext->db_dma_addr; + db_virt_start = (db_phy_start - + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base_phy) + + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base; + db_virt_offset = db_virt_start + db_8B_offset; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + if (inbuf->mask != 0) { + is_upper_half = IS_OFFSET_8B_ALIGN(db_offset); + roce5_copy_mask_attack_value_to_u64_addr((u64 *)db_virt_offset, + inbuf->attack_value[0], + inbuf->mask, + is_upper_half); + ROCE_INFO("roce attack sqdb, mask_val:0x%x, offset:%u", + (inbuf->attack_value[0] & inbuf->mask), db_offset); + } else { + sq_db.sq_db_val = 0; + sq_db.sq_db_seg.dw0.value = + roce5_convert_be32(inbuf->attack_value[0]); + sq_db.sq_db_seg.dw1.value = + roce5_convert_be32(inbuf->attack_value[1]); + + length = (db_offset + inbuf->length > DB_BUFF_SIZE) ? + (DB_BUFF_SIZE - db_offset) : + inbuf->length; + roce5_copy_attack_value_to_addr((u64 *)db_virt_offset, + sq_db.sq_db_val, length); + + ROCE_INFO("roce attack sqdb length:%u, offset:%u", length, + db_offset); + ROCE_INFO("roce attack sqdb type:%u, cos:%u, qpn:%u\n", + sq_db.sq_db_seg.dw0.bs.type, + sq_db.sq_db_seg.dw0.bs.cos, + sq_db.sq_db_seg.dw0.bs.qpn); + ROCE_INFO("roce attack sqdb mtu_shift:%u, pi:%u\n", + sq_db.sq_db_seg.dw1.bs.mtu_shift, + sq_db.sq_db_seg.dw1.bs.pi); + } + + roce5_rqp_put(rqp); + return 0; +} + +static int roce5_copy_attack_value_to_dwqe_addr(u64 *addr, const u32 *val, + u32 length, u32 attack_len) +{ + int i; + int count = ALIGN_UP_8B(length); + u64 *inbuf_tmp = NULL; + u32 step = 2; + + inbuf_tmp = (u64 *)kzalloc(DWQE_SIZE, GFP_KERNEL); + if (inbuf_tmp == NULL) { + ROCE_ERR("Failed to alloc inbuf temp."); + return -ENOMEM; + } + + for (i = 0; i < count * U32S_PER_U64; i += step) { + inbuf_tmp[i / step] = ((u64)val[(i + 1) % attack_len] + << UPPER_HALF_BIT_OFFSET) | + val[i % attack_len]; + } + + for (i = 0; i < count; i++) { + wmb(); + *((u64 *)addr + i) = inbuf_tmp[i]; + wmb(); + } + + kfree(inbuf_tmp); + return 0; +} + +static int roce5_dfx_attack_dwqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_ucontext *ucontext = NULL; + u64 dwqe_phy_start; + void __iomem *dwqe_virt_start = NULL; + void __iomem *dwqe_virt_offset = NULL; + void *hwdev = NULL; + struct roce5_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + u32 dwqe_offset; + u32 length; + u32 dwqe_8B_offset; + bool is_upper_half; + int i; + int ret; + + hwdev = rdev->hwdev; + if (hwdev == NULL) { + ROCE_ERR("Roce hwdev is NULL"); + return -ENOMEM; + } + + ucontext = rdev->ucontext; + if (ucontext == NULL) { + ROCE_ERR("Roce device context is null"); + return -ENOMEM; + } + + dwqe_offset = inbuf->offset; + if (dwqe_offset >= DWQE_SIZE) { + ROCE_ERR( + "roce attack dwqe offset is too large, should be less than %u", + DWQE_SIZE); + return -EINVAL; + } + + dwqe_phy_start = ucontext->dwqe_dma_addr; + //dwqe physical base address and virtual base address all is based on db perform offset of offset, so corresponding when at db of va basebase on to before offset offset + dwqe_virt_start = + (dwqe_phy_start - + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base_phy) + + (((struct hinic5_hwdev *)hwdev)->hwif)->db_base; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + length = (dwqe_offset + inbuf->length > DWQE_SIZE) ? + (DWQE_SIZE - dwqe_offset) : + (inbuf->length); + dwqe_8B_offset = ALIGN_DOWN_8B(dwqe_offset); + for (i = 0; i < DWQE_TYPE_NUM; i++) { + dwqe_virt_offset = + dwqe_virt_start + i * DWQE_SIZE + dwqe_8B_offset; + if (inbuf->mask != 0) { + is_upper_half = IS_OFFSET_8B_ALIGN(dwqe_offset); + roce5_copy_mask_attack_value_to_u64_addr( + (u64 *)dwqe_virt_offset, inbuf->attack_value[0], + inbuf->mask, is_upper_half); + } else { + ret = roce5_copy_attack_value_to_dwqe_addr( + (u64 *)dwqe_virt_offset, inbuf->attack_value, + length, inbuf->attack_len); + if (ret != 0) { + ROCE_ERR( + "Failed to copy attack value to dwqe, ret(%d)", + ret); + return -EFAULT; + } + } + } + + if (inbuf->mask != 0) { + ROCE_INFO("roce attack dwqe, mask_val:0x%x, offset:%u", + (inbuf->attack_value[0] & inbuf->mask), dwqe_offset); + } else { + ROCE_INFO("roce attack dwqe, length:%u, offset:%u", length, + dwqe_offset); + } + + roce5_rqp_put(rqp); + return 0; +} + +static int roce5_dfx_attack_sqwqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + u32 sq_buf_sz; + u32 length; + u32 sq_wqe_offset; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + sq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->sq.wqebb_cnt + << (unsigned)rqp->sq.wqe_shift, + PAGE_SIZE); + sq_wqe_offset = inbuf->offset; + if (sq_wqe_offset >= sq_buf_sz) { + ROCE_ERR( + "roce attack sq wqe offset is too large, should be less than %u", + sq_buf_sz); + ret = -EINVAL; + goto sq_wqe_err; + } + + length = (sq_wqe_offset + inbuf->length > sq_buf_sz) ? + (sq_buf_sz - sq_wqe_offset) : + inbuf->length; + ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, rqp->umem, + length, sq_wqe_offset, inbuf->attack_len, + inbuf->mask); + if (ret != 0) { + ROCE_ERR("Failed to copy to sq page, qpn:0x%x", qpn); + } + +sq_wqe_err: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_attack_rqwqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + u32 rq_buf_sz; + u32 length; + u32 rq_attack_offset; + u32 rq_wqe_offset; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + rq_buf_sz = (u32)ROCE_ALIGN((u64)rqp->rq.wqebb_cnt + << (unsigned)rqp->rq.wqe_shift, + PAGE_SIZE); + rq_attack_offset = inbuf->offset; + if (rq_attack_offset >= rq_buf_sz) { + ROCE_ERR( + "roce attack rq wqe offset is too large, should be less than %u", + rq_buf_sz); + ret = -EINVAL; + goto rq_wqe_err; + } + + length = (rq_attack_offset + inbuf->length > rq_buf_sz) ? + (rq_buf_sz - rq_attack_offset) : + inbuf->length; + rq_wqe_offset = rqp->rq.offset + rq_attack_offset; + ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, rqp->umem, + length, rq_wqe_offset, inbuf->attack_len, + inbuf->mask); + if (ret != 0) { + ROCE_ERR("Failed to copy to rq page, qpn:0x%x", qpn); + } + +rq_wqe_err: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_attack_mtt(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct mtt *roce_mtt) +{ + struct hmm_comp_priv *comp_priv = NULL; + u64 *mtts = NULL; + u64 *mtts_offset = NULL; + u64 page_size; + u32 mtt_offset; + u32 length; + u64 val = ((u64)inbuf->attack_value[1] << 32) | + (u64)inbuf->attack_value[0]; + u32 mtt_8B_offset; + bool is_upper_half; + + if (roce_mtt == NULL) { + ROCE_ERR("roce qp mtt is null"); + return -ENOMEM; + } + + if (roce_mtt->mtt_vaddr == NULL) { + ROCE_ERR("roce mtt vaddr is null, layers:%u", + roce_mtt->mtt_layers); + return -ENOMEM; + } + + comp_priv = get_hmm_comp_priv(rdev->hwdev, SERVICE_T_ROCE); + if (comp_priv == NULL) { + ROCE_ERR("Comp_priv is null"); + return -ENOMEM; + } + + mtts = (u64 *)hmm_em_table_find( + &comp_priv->mtt_em_table, + roce_mtt->mtt_seg[roce_mtt->mtt_layers - 1]->offset, NULL); + if (mtts == NULL) { + ROCE_ERR("Can't find va and pa of mtt entry"); + return -ENOMEM; + } + + page_size = 1L << roce_mtt->mtt_page_shift; + mtt_offset = inbuf->offset; + if (mtt_offset >= page_size) { + ROCE_ERR( + "roce attack mtt offset is too large, should be less than %llu", + page_size); + return -EINVAL; + } + + mtt_8B_offset = ALIGN_DOWN_8B(mtt_offset); + mtts_offset = mtts + mtt_8B_offset / sizeof(u64); + if (inbuf->mask != 0) { + is_upper_half = IS_OFFSET_8B_ALIGN(mtt_offset); + roce5_copy_mask_attack_value_to_u64_addr(mtts_offset, + inbuf->attack_value[0], + inbuf->mask, + is_upper_half); + ROCE_INFO("roce attack mtt, mask_val:0x%x, offset:%u\n", + (inbuf->attack_value[0] & inbuf->mask), mtt_offset); + } else { + length = (inbuf->length + mtt_offset > page_size) ? + (page_size - mtt_offset) : + inbuf->length; + roce5_copy_attack_value_to_addr(mtts_offset, val, length); + ROCE_INFO("roce attack mtt, attack_len:%u, offset:%u", length, + mtt_offset); + } + + return 0; +} + +static int roce5_dfx_attack_cmtt(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + ret = roce5_dfx_attack_mtt(rdev, inbuf, &(rqp->mtt)); + if (ret != 0) { + ROCE_ERR("Failed to attack cmtt"); + } + + roce5_rqp_put(rqp); + return ret; +} + +struct roce5_mr *roce5_mr_find(struct roce5_device *rdev, u32 key_id) +{ + struct roce5_mr *mr = NULL; + + mutex_lock(&rdev->device_res->mr_list.node_lock); + list_for_each_entry(mr, &rdev->device_res->mr_list.node, node) { + if (mr->rdmamr.mpt.mpt_index == key_id) { + mutex_unlock(&rdev->device_res->mr_list.node_lock); + ROCE_INFO("Find mr, it's mpt index is %u", + mr->rdmamr.mpt.mpt_index); + return mr; + } + } + + mutex_unlock(&rdev->device_res->mr_list.node_lock); + ROCE_ERR("Don't find correct mr, key_id:%u", key_id); + + return NULL; +} + +static int roce5_dfx_attack_dmtt(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_mr *mr = NULL; + int ret = 0; + + mr = roce5_mr_find(rdev, inbuf->mpt_index); + if (mr == NULL) { + ROCE_ERR( + "Failed to attack dmtt, don't find valid mr, mpt_index:%u", + inbuf->mpt_index); + return -EINVAL; + } + + ret = roce5_dfx_attack_mtt(rdev, inbuf, &(mr->rdmamr.mtt)); + if (ret != 0) { + ROCE_ERR("Failed to attack dmtt"); + } + + return ret; +} + +static void roce5_attack_value_to_rdmarc(u32 *vaddr, u32 length, const u32 *val, + u32 attack_len) +{ + int cnt = (length + 3) >> 2; + int i; + + for (i = 0; i < cnt; i++) { + vaddr[i] = val[i % attack_len]; + } +} + +static int roce5_dfx_attack_rdmarc(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + u32 qpn = inbuf->qpn; + struct rdma_rdmarc *rdmarc = NULL; + void *rdmarc_vaddr = NULL; + u32 *rdmarc_offset_vaddr = NULL; + u32 rdmarc_entry_num; + u32 rdmarc_offset; + u32 rdmarc_size; + u32 length; + int ret = 0; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + rdmarc = &(rqp->rdmarc); + rdmarc_vaddr = rdmarc->vaddr; + if (rdmarc_vaddr == NULL) { + ROCE_ERR("Rdmarc vaddr is null"); + ret = -ENOMEM; + goto rdmarc_err; + } + + rdmarc_entry_num = (1 << rdmarc->order); + rdmarc_size = rdmarc_entry_num * ROCE_RDMARC_ENTRY_SZ; + rdmarc_offset = inbuf->offset; + if (rdmarc_offset >= rdmarc_size) { + ROCE_ERR( + "roce attack rdmarc offset is too large, should be less than %u", + rdmarc_size); + ret = -EINVAL; + goto rdmarc_err; + } + + rdmarc_offset_vaddr = (u32 *)((u8 *)rdmarc_vaddr + rdmarc_offset); + + if (inbuf->mask != 0) { + roce5_copy_mask_attack_value_to_addr(rdmarc_offset_vaddr, + inbuf->attack_value[0], + inbuf->mask); + ROCE_INFO("roce attack rdmarc, mask_val:0x%x, offset:%u", + (inbuf->attack_value[0] & inbuf->mask), + rdmarc_offset); + } else { + length = (rdmarc_offset + inbuf->length > rdmarc_size) ? + (rdmarc_size - rdmarc_offset) : + inbuf->length; + roce5_attack_value_to_rdmarc(rdmarc_offset_vaddr, length, + inbuf->attack_value, + inbuf->attack_len); + ROCE_INFO("roce attack rdmarc, attack_length:%u, offset:%u", + length, rdmarc_offset); + } + +rdmarc_err: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_attack_cq(struct roce5_cq *cq, struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf) +{ + u32 ret; + u32 cq_attack_offset; + u32 cq_buf_sz; + u32 length; + u32 cq_depth = cq->cqe_mask + 1; + u32 cqe_size = rdev->rdma_cap.cqe_size; + + cq_buf_sz = cqe_size * cq_depth; + if (inbuf->offset >= cq_buf_sz) { + ROCE_ERR( + "Roce attack cq cqe offset is too large, should be less than %u", + cq_buf_sz); + return -EINVAL; + } + + cq_attack_offset = inbuf->offset; + length = ((cq_attack_offset + inbuf->length) > cq_buf_sz) ? + (cq_buf_sz - cq_attack_offset) : + inbuf->length; + ret = roce5_copy_data_to_page(rdev, inbuf->attack_value, cq->umem, + length, cq_attack_offset, + inbuf->attack_len, inbuf->mask); + if (ret != 0) { + ROCE_ERR("Failed to copy to cq page"); + } + + return ret; +} + +static int roce5_dfx_attack_cqe(struct roce5_device *rdev, + const struct roce_attack_inbuf *inbuf, + struct roce_attack_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + struct roce5_cq *send_cq = NULL; + struct roce5_cq *recv_cq = NULL; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + ROCE_ERR("Failed to get roce qp, qpn:0x%x", qpn); + return -EINVAL; + } + + roce5_get_cqs(rqp, &send_cq, &recv_cq); + if (send_cq != NULL) { + ret = roce5_dfx_attack_cq(send_cq, rdev, inbuf); + if (ret != 0) { + ROCE_ERR("Failed to attack send_cq, qpn:0x%x", qpn); + goto cq_wqe_err; + } + } + + if (recv_cq != NULL) { + ret = roce5_dfx_attack_cq(recv_cq, rdev, inbuf); + if (ret != 0) { + ROCE_ERR("Failed to attack recv_cq, qpn:0x%x", qpn); + goto cq_wqe_err; + } + } + +cq_wqe_err: + roce5_rqp_put(rqp); + return ret; +} + +static struct roce_dfx_attack_cmd_func g_roce_attack_cmd_funcs[] = { + { ROCE_ATTACK_TYPE_SQDB_HARD, roce5_dfx_attack_sqdb_hard }, + { ROCE_ATTACK_TYPE_DWQE, roce5_dfx_attack_dwqe }, + { ROCE_ATTACK_TYPE_SQWQE, roce5_dfx_attack_sqwqe }, + { ROCE_ATTACK_TYPE_RQWQE, roce5_dfx_attack_rqwqe }, + { ROCE_ATTACK_TYPE_CMTT, roce5_dfx_attack_cmtt }, + { ROCE_ATTACK_TYPE_DMTT, roce5_dfx_attack_dmtt }, + { ROCE_ATTACK_TYPE_RDMARC, roce5_dfx_attack_rdmarc }, + { ROCE_ATTACK_TYPE_CQE, roce5_dfx_attack_cqe }, +}; + +void roce5_cpu_to_be32(void *data, u32 len) +{ + u32 chunk_sz = sizeof(u32); + u32 *mem = (u32 *)data; + u32 data_len; + u32 i; + + if (data == NULL) + return; + + data_len = len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = cpu_to_be32(*mem); + mem++; + } +} + +int roce5_dfx_attack_func(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct roce_attack_inbuf *inbuf = (struct roce_attack_inbuf *)buf_in; + struct roce_attack_outbuf *outbuf = + (struct roce_attack_outbuf *)buf_out; + roce_adm_dfx_attack_func_t func = NULL; + u32 cmd_type; + u32 i; + + if ((inbuf == NULL) || (outbuf == NULL)) { + ROCE_ERR("Roce dfx attack buf in or buf out is null."); + return -EINVAL; + } + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct roce_attack_inbuf)) { + ROCE_ERR("roce dfx attack buffer in size too small"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(struct roce_attack_outbuf)) { + ROCE_ERR("roce dfx attack buffer out size too small"); + return -EINVAL; + } + + *out_size = 0; + + if (inbuf->convert_endian != 0) + roce5_cpu_to_be32(&(inbuf->attack_value), + sizeof(inbuf->attack_value)); + + for (i = 0; i < sizeof(g_roce_attack_cmd_funcs) / + sizeof(g_roce_attack_cmd_funcs[0]); + i++) { + cmd_type = g_roce_attack_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->attack_type) { + func = g_roce_attack_cmd_funcs[i].cmd_func; + if (func == NULL) { + ROCE_ERR( + "function of attack cmd(0x%x) is not supported.", + cmd_type); + return -EINVAL; + } + + *out_size = sizeof(struct roce_attack_outbuf); + + return func(rdev, inbuf, outbuf); + } + } + + ROCE_ERR("attack cmd(0x%x) does't exist.", cmd_type); + + return -EINVAL; +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c new file mode 100644 index 000000000..371fa3211 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_debug.c @@ -0,0 +1,35 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/semaphore.h> +#include "hinic5_mt.h" +#include "securec.h" +#include "roce_infra.h" +#include "roce.h" + +int roce5_get_drv_version(void *buf_out, u32 *out_size) +{ + struct drv_version_info *ver_info = buf_out; + int rc; + + if (!buf_out) { + ROCE_ERR("Buf_out is NULL."); + return -EINVAL; + } + + if (*out_size != sizeof(*ver_info)) { + ROCE_ERR("Unexpect out buf size from user :%u, expect: %lu", + *out_size, sizeof(*ver_info)); + return -EINVAL; + } + + rc = snprintf_s(ver_info->ver, sizeof(ver_info->ver), + sizeof(ver_info->ver) - 1, "%s %s(debug)", + HIROCE5_DRV_VERSION, __TIME_STR__); + if (rc == -1) { + ROCE_ERR("Snprintf roce version err"); + return -EFAULT; + } + + return 0; +} diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c new file mode 100644 index 000000000..ef4b2da57 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/dfx/roce_dfx_query.c @@ -0,0 +1,851 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. + + +#ifdef __ROCE_DFX__ + +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/kthread.h> + +#include <rdma/ib_verbs.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "hinic5_mt.h" + +#include "roce_compat.h" +#include "roce.h" +#include "roce_dfx.h" +#include "roce_srq.h" +#include "roce_qp.h" +#include "roce_cq.h" +#include "roce_cmd.h" +#include "roce_mix.h" +#include "roce_infra.h" +#include "roce_npu_cmd.h" +#include "roce_cqm_cmd.h" +#include "roce_restore.h" +#include "roce_main_extension.h" +#include "roce_com.h" +#include "securec.h" +#include "hmm_common.h" +#include "roce_npu_cmd_gid_defs.h" + +static int roce5_dfx_get_dev_algo(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + u32 cc_algo; + struct roce5_ecn_ctx ecn_ctx; + u32 *algo_type = &outbuf->algo_type; + + if (rdev == NULL) { + ROCE_ERR("Failed to get roce device."); + return -EINVAL; + } + ecn_ctx = rdev->ecn_ctx; + cc_algo = ecn_ctx.cc_algo; + *algo_type = cc_algo; + + return 0; +} + +static int roce5_dfx_get_qp_count(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_dfx_qp_count *qp_count = &outbuf->qp_count; + + qp_count->qp_alloced = rdev->qp_cnt.alloc_qp_cnt; + qp_count->qp_deleted = rdev->qp_cnt.del_qp_cnt; + qp_count->qp_alive = + rdev->qp_cnt.alloc_qp_cnt - rdev->qp_cnt.del_qp_cnt; + + return 0; +} + +/* QPC is statically allocated, so it is not necessary to judge whether it exists when querying the cache content */ +static int roce5_dfx_query_cache_qpc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + u32 qpn = inbuf->qpn; + roce_verbs_qp_attr_s *qp_attr = &outbuf->qp_attr; + + ret = memset_s(qp_attr, sizeof(roce_verbs_qp_attr_s), 0, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)qp_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + roce5_dfx_print("Failed to query QPC from cache!"); + roce5_dfx_print( + "******************From Cache: qpn(%#x) ********************", + qpn); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY QPC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: qpn(%#x) ********************", + qpn); + + return ret; + } + + return 0; +} + +static int roce5_dfx_query_host_qpc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_qp *rqp = NULL; + int ret = 0; + u32 qpn = inbuf->qpn; + roce5_qp_query_outbuf_s *qp_query = &outbuf->qp_query; + + rqp = roce5_get_rqp_by_qpn(rdev, qpn); + if (rqp == NULL) { + roce5_dfx_print("Failed to get roce qp."); + roce5_dfx_print( + "******************From Host: qpn(%#x) ********************", + qpn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY QPC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: qpn(%#x) ********************", + qpn); + return -EINVAL; + } + + ret = memcpy_s(qp_query, sizeof(roce5_qp_query_outbuf_s), + rqp->qpc_info->vaddr, sizeof(roce5_qp_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy qp_ctx, ret(%d).", ret); + goto err_out; + } + +err_out: + roce5_rqp_put(rqp); + return ret; +} + +static int roce5_dfx_uni_cmd_query_cq(struct roce5_device *rdev, u32 cqn, + u32 *verbs_cq_attr, int cq_attr_size) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_cq_query_s *cq_query_inbuf = NULL; + roce_uni_cmd_cq_query_outbuf_s *cq_query_outbuf = NULL; + struct roce5_cq *rcq = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_cq_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_cq_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + rcq = roce5_get_rcq_by_cqn(rdev, cqn); + if (rcq == NULL) { + roce5_dfx_print("Failed to get roce cq."); + goto err_rcq; + } + cq_query_inbuf = (roce_uni_cmd_cq_query_s *)cqm_cmd_inbuf->buf; + cq_query_inbuf->com.index = cpu_to_be32(cqn); + cq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_CQ_BITMASK)); + if (rcq->shadow != 0) { + cq_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + cq_query_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rcq->shadow_vfid); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, ROCE_CMD_QUERY_CQ, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + ROCE_CMD_TIME_CLASS_B, HINIC5_CHANNEL_ROCE); + if (ret != 0) { + roce5_dfx_print("Failed to send cmd QUERY_CQ, ret=%d", ret); + ret = -1; + goto out; + } + + cq_query_outbuf = (roce_uni_cmd_cq_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)verbs_cq_attr, (size_t)cq_attr_size, + (void *)&cq_query_outbuf->cq_info.context[0], + (size_t)cq_attr_size); + if (ret != 0) { + ROCE_ERR("Failed to memcpy verbs_cq_attr.(ret:%d)", ret); + ret = -EINVAL; + goto out; + } +out: + roce5_rcq_put(rcq); +err_rcq: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int roce5_dfx_query_cq(struct roce5_device *rdev, u32 cqn, u32 *context, + int cqc_size) +{ + if (ROCE5_SUPPORT_DFX_CMD_QUERY_CQ_MSG_V2(rdev) != 0) { + return roce5_dfx_uni_cmd_query_cq(rdev, cqn, context, cqc_size); + } + return -EINVAL; +} + +/* CQC is dynamic allocation. When querying cache content, you need to determine whether it exists or not. */ +static int roce5_dfx_query_cache_cqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + cqm_object_s *cqm_obj_cq = NULL; + u32 cqn = inbuf->cqn; + roce5_cq_query_outbuf_s *cq_query = &outbuf->cq_query; + + cqm_obj_cq = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SCQ, cqn, false); + if (cqm_obj_cq == NULL) { + roce5_dfx_print("Failed to get cqm_obj_cq."); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY CQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + + return -EINVAL; + } + + cqm5_object_put(cqm_obj_cq); + + ret = memset_s(cq_query, sizeof(roce5_cq_query_outbuf_s), 0, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset cq_context.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)cq_query, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to query cq from cache!"); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY CQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: cqn(%#x) ********************", + cqn); + + return -EINVAL; + } + + return 0; +} + +static int roce5_dfx_query_host_cqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_cq *rcq = NULL; + int ret = 0; + u32 cqn = inbuf->cqn; + roce5_cq_query_outbuf_s *cq_query = &outbuf->cq_query; + + rcq = roce5_get_rcq_by_cqn(rdev, cqn); + if (rcq == NULL) { + roce5_dfx_print("Failed to get roce cq."); + roce5_dfx_print( + "******************From Host: cqn(%#x) ********************", + cqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY CQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: cqn(%#x) ********************", + cqn); + + return -EINVAL; + } + + ret = memcpy_s(cq_query, sizeof(roce5_cq_query_outbuf_s), + rcq->cqm_cq->q_ctx_vaddr, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy cq_ctx.(ret:%d)", ret); + goto err_out; + } + +err_out: + roce5_rcq_put(rcq); + return ret; +} + +int roce5_dfx_cmd_query_srq(struct roce5_device *rdev, u32 srqn, + roce_verbs_srq_attr_s *srq_attr) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_srq_query_s *srq_query_inbuf = NULL; + roce_uni_cmd_srq_query_outbuf_s *srq_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_srq_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_srq_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + srq_query_inbuf = (roce_uni_cmd_srq_query_s *)cqm_cmd_inbuf->buf; + srq_query_inbuf->com.index = cpu_to_be32(srqn); + srq_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SRQ_BITMASK)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_SRQ, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + roce5_dfx_print("Failed to send cmd QUERY_SRQ, ret=%d", ret); + ret = -1; + goto out; + } + + srq_query_outbuf = + (roce_uni_cmd_srq_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s(srq_attr, sizeof(roce_verbs_srq_attr_s), + &srq_query_outbuf->com.srq_attr, + sizeof(roce_verbs_srq_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy srq_attr.(ret:%d)", ret); + ret = -EINVAL; + goto out; + } +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +/* SRQC is dynamic allocation. When querying the cache content, you need to determine whether it exists or not. */ +static int roce5_dfx_query_cache_srqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + cqm_object_s *cqm_obj_srq = NULL; + u32 srqn = inbuf->srqn; + roce_verbs_srq_attr_s *srq_attr = &outbuf->srq_attr; + + cqm_obj_srq = + cqm5_object_get(rdev->hwdev, CQM_OBJECT_RDMA_SRQ, srqn, false); + if (cqm_obj_srq == NULL) { + roce5_dfx_print("Failed to get cqm_obj_srq"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY SRQC FROM CACHE FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + cqm5_object_put(cqm_obj_srq); + + ret = memset_s(srq_attr, sizeof(roce_verbs_srq_attr_s), 0, + sizeof(roce_verbs_srq_attr_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset srq_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_cmd_query_srq(rdev, srqn, srq_attr); + if (ret != 0) { + roce5_dfx_print("Failed to query srq from cache"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY SRQC FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + return 0; +} + +static int roce5_dfx_query_host_srqc(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + struct roce5_srq *rsrq = NULL; + int ret = 0; + u32 srqn = inbuf->srqn; + roce5_srq_query_outbuf_s *srq_query = &outbuf->srq_query; + + rsrq = roce5_get_rsrq_by_srqn(rdev, srqn); + if (rsrq == NULL) { + roce5_dfx_print("Failed to get roce srq."); + roce5_dfx_print( + "******************From Host: srqn(%#x) ********************", + srqn); + roce5_dfx_print( + ">>>>>>>>>>> QUERY SRQC FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: srqn(%#x) ********************", + srqn); + + return -EINVAL; + } + + ret = memcpy_s(srq_query, sizeof(roce5_srq_query_outbuf_s), + rsrq->cqm_srq->q_ctx_vaddr, + sizeof(roce5_srq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy srq_ctx.(ret:%d)", ret); + goto err_out; + } + +err_out: + roce5_rsrq_put(rsrq); + return ret; +} + +static int roce5_dfx_uni_cmd_query_mpt(struct roce5_device *rdev, u32 key, + struct roce_mpt_context *mpt_entry) +{ + int ret; + u32 mpt_index; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_mpt_query_s *mpt_query_inbuf = NULL; + roce_uni_cmd_mpt_query_outbuf_s *mpt_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_mpt_query_s), &cqm_cmd_outbuf, + (u16)sizeof(roce_uni_cmd_mpt_query_outbuf_s)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + mpt_query_inbuf = (roce_uni_cmd_mpt_query_s *)cqm_cmd_inbuf->buf; + mpt_index = (key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + mpt_query_inbuf->com.index = cpu_to_be32(mpt_index); + mpt_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_MR_BITMASK)); + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_MPT, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + roce5_dfx_print("Failed to send cmd QUERY_MPT"); + ret = -1; + goto out; + } + + mpt_query_outbuf = + (roce_uni_cmd_mpt_query_outbuf_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s(mpt_entry, sizeof(struct roce_mpt_context), + &mpt_query_outbuf->mpt_info.context[0], + sizeof(struct roce_mpt_context)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mpt_entry.(ret:%d)", ret); + ret = -EINVAL; + goto out; + } + +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static int roce5_dfx_query_mpt(struct roce5_device *rdev, u32 key, + struct roce_mpt_context *mpt_entry) +{ + if (ROCE5_SUPPORT_DFX_CMD_QUERY_MPT_MSG_V2(rdev) != 0) { + return roce5_dfx_uni_cmd_query_mpt(rdev, key, mpt_entry); + } + return -EINVAL; +} + +static int roce5_dfx_uni_cmd_query_gid(struct roce5_device *rdev, u32 port, + u32 gid_index, + struct rdma_gid_entry *gid_entry) +{ + int ret; + cqm_cmd_buf_s *cqm_cmd_inbuf = NULL; + cqm_cmd_buf_s *cqm_cmd_outbuf = NULL; + roce_uni_cmd_query_gid_s *gid_query_inbuf = NULL; + struct rdma_gid_query_outbuf *gid_query_outbuf = NULL; + + ret = roce5_cqm_cmd_zalloc_inoutbuf( + rdev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(roce_uni_cmd_query_gid_s), &cqm_cmd_outbuf, + (u16)sizeof(struct rdma_gid_query_outbuf)); + if (ret != 0) { + roce5_dfx_print("Failed to alloc cqm_cmd_inoutbuf, ret=%d", + ret); + return ret; + } + + gid_query_inbuf = (roce_uni_cmd_query_gid_s *)cqm_cmd_inbuf->buf; + gid_query_inbuf->com.index = cpu_to_be32(gid_index); + gid_query_inbuf->com.dw0.bs.cmd_bitmask = + cpu_to_be16(ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_GID_BITMASK)); + gid_query_inbuf->com.opt = cpu_to_be32(port); + + if (rdev->shadow != 0) { + gid_query_inbuf->com.dw3.bs.shadow_vfid = + cpu_to_be16(rdev->shadow_vfid); + gid_query_inbuf->com.dw0.bs.cmd_bitmask = cpu_to_be16( + ROCE5_ULP_CMD(rdev, VERBS_CMD_TYPE_SHADOW_BITMASK)); + gid_query_inbuf->gid_query.dw0.shadow.shadow_vf_num = + rdev->shadow_vf_num; + gid_query_inbuf->gid_query.dw0.value = + cpu_to_be32(gid_query_inbuf->gid_query.dw0.value); + } + + ret = cqm5_send_cmd_box(rdev->hwdev, HINIC5_MOD_ROCE, + ROCE_CMD_QUERY_GID, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, ROCE_CMD_TIME_CLASS_B, + HINIC5_CHANNEL_ROCE); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to send cmd QUERY_GID, ret(%d)", + ret); + ret = -1; + goto out; + } + + gid_query_outbuf = (struct rdma_gid_query_outbuf *)cqm_cmd_outbuf->buf; + ret = memcpy_s(gid_entry, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM, + gid_query_outbuf->gid_entry, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM); + if (ret != 0) { + ROCE_DEV_ERR(rdev, "Failed to memcpy gid_entry.(ret:%d)", ret); + } +out: + roce5_cqm5_cmd_free_inoutbuf(rdev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int roce5_dfx_query_gid(struct roce5_device *rdev, u32 port, + u32 gid_index, struct rdma_gid_entry *gid_entry) +{ + if (ROCE5_SUPPORT_DFX_CMD_QUERY_GID_MSG_V2(rdev) != 0) { + return roce5_dfx_uni_cmd_query_gid(rdev, port, gid_index, + gid_entry); + } + return -EINVAL; +} + +/* MPT is statically allocated, so it is not necessary to judge whether it exists when querying the cache content */ +static int roce5_dfx_query_cache_mpt(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + ret = memset_s(mpt_entry, sizeof(struct roce_mpt_context), 0, + sizeof(struct roce_mpt_context)); + if (ret != 0) { + ROCE_ERR("Failed to memset mpt_entry.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_mpt(rdev, mpt_key, mpt_entry); + if (ret != 0) { + roce5_dfx_print("Failed to query mpt from cache!"); + roce5_dfx_print( + "******************From Cache: mpt_key(%#x) ********************", + mpt_key); + roce5_dfx_print( + ">>>>>>>>>>>>>>>> QUERY MPT FROM CACHE FAILED <<<<<<<<<<<<<<<<<"); + roce5_dfx_print( + "******************From Cache: mpt_key(%#x) ********************", + mpt_key); + + return ret; + } + + return 0; +} + +static int roce5_dfx_query_host_mpt(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + u32 mpt_index = 0; + struct mpt *rmpt = NULL; + int ret = 0; + u32 mpt_key = inbuf->mpt_key; + struct roce_mpt_context *mpt_entry = &outbuf->mpt; + + mpt_index = (mpt_key >> MR_KEY_2_INDEX_SHIFT) & 0xFFFFFF; + rmpt = roce5_get_mpt_by_index(rdev, mpt_index); + if (rmpt == NULL) { + roce5_dfx_print("Failed to get roce mpt."); + roce5_dfx_print( + "******************From Host: mpt_key(%#x) ********************", + mpt_key); + roce5_dfx_print( + ">>>>>>>>>>> QUERY MPT FROM HOST FAILED, NOT EXIST <<<<<<<<<<<"); + roce5_dfx_print( + "******************From Host: mpt_key(%#x) ********************", + mpt_key); + + return -EINVAL; + } + + ret = memcpy_s(mpt_entry, sizeof(struct roce_mpt_context), rmpt->vaddr, + sizeof(struct roce_mpt_context)); + if (ret != 0) { + ROCE_ERR("Failed to memcpy mpt_entry.(ret:%d)", ret); + goto err_out; + } + +err_out: + roce5_mpt_put(rmpt); + return ret; +} + +static int roce5_dfx_query_cache_gid(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + u32 gid_index = inbuf->gid_index; + struct rdma_gid_entry *gid_entry = outbuf->gid_entry; + + ret = memset_s(gid_entry, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM, 0, + sizeof(struct rdma_gid_entry) * ROCE_GID_ENTRY_NUM); + if (ret != 0) { + ROCE_ERR("Failed to memset gid_entry.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_gid(rdev, 0, gid_index, gid_entry); + if (ret != 0) { + roce5_dfx_print( + "***************port(1), gid_index(%d) *****************", + gid_index); + roce5_dfx_print("Failed to query gid from cache!"); + roce5_dfx_print( + "***************port(1), gid_index(%d) *****************", + gid_index); + + return -EINVAL; + } + + return 0; +} + +static void roce5_dfx_query_pi_ci_set(const roce_verbs_qp_attr_s *qp_attr, + struct roce5_dfx_pi_ci *pi_ci) +{ + pi_ci->qpc_sq_pi_on_chip = qp_attr->chip_seg.dw5.bs.sq_pi_on_chip; + pi_ci->qpc_sq_pi = qp_attr->ext_seg.dfx_info.dw0.bs.sq_pi; + pi_ci->qpc_sq_load_pi = qp_attr->ext_seg.dfx_info.dw1.bs.sq_load_pi; + pi_ci->qpc_rq_pi_on_chip = qp_attr->chip_seg.dw8.bs.rq_pi_on_chip; + pi_ci->qpc_rq_load_pi = qp_attr->ext_seg.dfx_info.dw1.bs.rq_load_pi; + pi_ci->qpc_rq_pi = qp_attr->ext_seg.dfx_info.dw4.bs.rq_pi; + pi_ci->qpc_rc_pi = qp_attr->ext_seg.dfx_info.dw5.bs.rc_pi; + pi_ci->qpc_sq_ci = qp_attr->ext_seg.dfx_info.dw0.bs.sq_ci; + pi_ci->qpc_sq_wqe_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw1.bs.sq_wqe_prefetch_ci; + pi_ci->qpc_sq_mtt_prefetch_wqe_ci = + qp_attr->ext_seg.dfx_info.dw3.bs.sq_mtt_prefetch_wqe_ci; + pi_ci->qpc_sqa_ci = qp_attr->ext_seg.dfx_info.dw2.bs.sqa_ci; + pi_ci->qpc_sqa_wqe_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw2.bs.sqa_wqe_prefetch_ci; + pi_ci->qpc_rq_ci = qp_attr->ext_seg.dfx_info.dw4.bs.rq_ci; + pi_ci->qpc_rq_wqe_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw5.bs.rq_wqe_prefetch_ci; + pi_ci->qpc_rq_mtt_prefetch_wqe_ci = + qp_attr->ext_seg.dfx_info.dw3.bs.rq_mtt_prefetch_wqe_ci; + pi_ci->qpc_rq_base_ci = qp_attr->chip_seg.dw4.bs.rq_base_ci; + pi_ci->qpc_rc_ci = qp_attr->ext_seg.dfx_info.dw5.bs.rc_ci; + pi_ci->qpc_rc_prefetch_ci = + qp_attr->ext_seg.dfx_info.dw1.bs.rc_prefetch_ci; +} + +static int roce5_dfx_query_pi_ci(const struct roce5_dfx_query_inbuf *inbuf, + struct roce5_device *rdev, + union roce5_dfx_query_outbuf *outbuf) +{ + int ret; + roce_verbs_qp_attr_s be_attr; /* BE context structure */ + roce_verbs_qp_attr_s qp_attr; /* LE context structure */ + roce_verbs_cq_attr_s cq_attr; + roce5_cq_query_outbuf_s cq_query; + u32 qpn = inbuf->query_pi_ci.qpn; + u32 cqn = inbuf->query_pi_ci.cqn; + struct roce5_dfx_pi_ci *pi_ci = &outbuf->pi_ci; + + ret = memset_s(pi_ci, sizeof(struct roce5_dfx_pi_ci), 0, + sizeof(struct roce5_dfx_pi_ci)); + if (ret != 0) { + ROCE_ERR("Failed to memset pi_ci.(ret:%d)", ret); + return -ENOMEM; + } + + ret = memset_s(&qp_attr, sizeof(qp_attr), 0, sizeof(qp_attr)); + if (ret != 0) { + ROCE_ERR("Failed to memset qp_attr.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_qp_query(rdev, qpn, (u32 *)(void *)&be_attr, + sizeof(roce_verbs_qp_attr_s)); + if (ret != 0) { + return ret; + } + + roce5_be32_2_le32(&be_attr, (u32 *)(void *)&qp_attr, + (sizeof(roce_verbs_qp_attr_s) / sizeof(u32))); + + roce5_dfx_query_pi_ci_set(&qp_attr, pi_ci); + + ret = memset_s(&cq_query, sizeof(roce5_cq_query_outbuf_s), 0, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + ROCE_ERR("Failed to memset cq_query.(ret:%d)", ret); + return -ENOMEM; + } + ret = roce5_dfx_query_cq(rdev, cqn, (u32 *)(void *)&cq_query, + sizeof(roce5_cq_query_outbuf_s)); + if (ret != 0) { + return ret; + } + + if (ROCE5_SUPPORT_DFX_CMD_QUERY_PI_CI_MSG_V2(rdev) != 0) { + (void)memcpy_s(&cq_attr, sizeof(roce_verbs_cq_attr_s), + &cq_query, sizeof(roce_verbs_cq_attr_s)); + cq_attr.dw0.value = be32_to_cpu(cq_attr.dw0.value); + cq_attr.dfx_info.dw0.value = + be32_to_cpu(cq_attr.dfx_info.dw0.value); + cq_attr.dfx_info.dw1.value = + be32_to_cpu(cq_attr.dfx_info.dw1.value); + cq_attr.dfx_info.dw2.value = + be32_to_cpu(cq_attr.dfx_info.dw2.value); + cq_attr.dfx_info.dw3.value = + be32_to_cpu(cq_attr.dfx_info.dw3.value); + cq_attr.ci_record_gpa_at_hop_num = + be64_to_cpu(cq_attr.ci_record_gpa_at_hop_num); + + pi_ci->cq_ci_on_chip = cq_attr.dw0.bs.ci_on_chip; + pi_ci->cq_ci = cq_attr.dfx_info.dw0.bs.ci; + pi_ci->cq_load_ci = cq_attr.dfx_info.dw2.bs.load_ci; + pi_ci->cq_ci_record_gpa_at_hop_num = + cq_attr.ci_record_gpa_at_hop_num; + pi_ci->cq_last_solicited_pi = + cq_attr.dfx_info.dw2.bs.last_solicited_pi; + pi_ci->cq_pi = cq_attr.dfx_info.dw1.bs.pi; + pi_ci->cq_last_notified_pi = + cq_attr.dfx_info.dw3.bs.last_notified_pi; + } + return 0; +} + +static roce5_adm_dfx_query_cmd_proc_t g_roce5_adm_dfx_query_cmd_proc[] = { + { ROCE_CMD_GET_QPC_FROM_CACHE, roce5_dfx_query_cache_qpc }, + { ROCE_CMD_GET_QPC_FROM_HOST, roce5_dfx_query_host_qpc }, + { ROCE_CMD_GET_CQC_FROM_CACHE, roce5_dfx_query_cache_cqc }, + { ROCE_CMD_GET_CQC_FROM_HOST, roce5_dfx_query_host_cqc }, + { ROCE_CMD_GET_SRQC_FROM_CACHE, roce5_dfx_query_cache_srqc }, + { ROCE_CMD_GET_SRQC_FROM_HOST, roce5_dfx_query_host_srqc }, + { ROCE_CMD_GET_MPT_FROM_CACHE, roce5_dfx_query_cache_mpt }, + { ROCE_CMD_GET_MPT_FROM_HOST, roce5_dfx_query_host_mpt }, + { ROCE_CMD_GET_GID_FROM_CACHE, roce5_dfx_query_cache_gid }, + { ROCE_CMD_GET_QPC_CQC_PI_CI, roce5_dfx_query_pi_ci }, + { ROCE_CMD_GET_QP_COUNT, roce5_dfx_get_qp_count }, + { ROCE_CMD_GET_DEV_ALGO, roce5_dfx_get_dev_algo }, +}; + +int roce5_adm_dfx_query(struct roce5_device *rdev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + int ret; + const struct roce5_dfx_query_inbuf *inbuf = + (struct roce5_dfx_query_inbuf *)buf_in; + union roce5_dfx_query_outbuf *outbuf = + (union roce5_dfx_query_outbuf *)buf_out; + const roce5_adm_generic_cmd_proc_t *generic_array = + (const roce5_adm_generic_cmd_proc_t *) + g_roce5_adm_dfx_query_cmd_proc; + int elem_count = sizeof(g_roce5_adm_dfx_query_cmd_proc) / + sizeof(g_roce5_adm_dfx_query_cmd_proc[0]); + roce5_adm_dfx_query_t roce5_adm_dfx_query_func = NULL; + + if (in_size < sizeof(struct roce5_dfx_query_inbuf)) { + ROCE_ERR( + "Input parameter(in_size: %dB) is shorter than input buffer(allocated: %dB)", + (int)in_size, + (int)sizeof(struct roce5_dfx_query_inbuf)); + return -ENOMEM; + } + if (*out_size < sizeof(union roce5_dfx_query_outbuf)) { + ROCE_ERR( + "Input parameter(out_size: %dB) is shorter than allocated buffer(allocated: %dB)", + (int)*out_size, + (int)sizeof(union roce5_dfx_query_outbuf)); + return -ENOMEM; + } + ret = memset_s(buf_out, sizeof(union roce5_dfx_query_outbuf), 0, + sizeof(union roce5_dfx_query_outbuf)); + if (ret != 0) { + ROCE_ERR("Failed to memset buf_out.(ret:%d)", ret); + return -ENOMEM; + } + *out_size = (u32)sizeof(union roce5_dfx_query_outbuf); + + if (inbuf->cmd_type >= VM_COMPAT_TEST) { + roce5_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + roce5_adm_dfx_query_func = + (roce5_adm_dfx_query_t)roce5_adm_generic_find_cmd_func( + generic_array, elem_count, inbuf->cmd_type); + if (roce5_adm_dfx_query_func == NULL) { + roce5_dfx_print("Not support this type(%d)", inbuf->cmd_type); + return -EINVAL; + } + + return roce5_adm_dfx_query_func(inbuf, rdev, outbuf); +} + +#endif diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c new file mode 100644 index 000000000..7c641f422 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.c @@ -0,0 +1,83 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/slab.h> +#include "securec.h" +#include "roce_infra.h" +#include "roce_obj_ext.h" + +int roce5_obj_ext_init(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext) +{ + int i, ret; + + ret = memset_s(obj_ext->bufs, sizeof(void *) * ROCE_EXT_FEATURE_MAX, 0, + sizeof(void *) * ROCE_EXT_FEATURE_MAX); + if (ret != 0) { + ROCE_ERR( + "Failed to execute memset to object extension buffer, ret(%d).", + ret); + return ret; + } + + for (i = 0; i < item_num; ++i) { + if (reg_items[i].ext_id >= ROCE_EXT_FEATURE_MAX) { + ROCE_ERR("Invalid extension id(%u).", + reg_items[i].ext_id); + ret = -EINVAL; + goto err_out; + } + + if (obj_ext->bufs[reg_items[i].ext_id] != NULL) { + ROCE_ERR("Repeated register, extension id(%u).", + reg_items[i].ext_id); + ret = -EINVAL; + goto err_out; + } + + obj_ext->bufs[reg_items[i].ext_id] = + kzalloc(reg_items[i].data_size, GFP_KERNEL); + if (obj_ext->bufs[reg_items[i].ext_id] == NULL) { + ROCE_ERR("Failed alloc memory for extension."); + ret = -ENOMEM; + goto err_out; + } + + ret = reg_items[i].init_func( + parent, obj_ext->bufs[reg_items[i].ext_id]); + if (ret != 0) { + ROCE_ERR("Failed to init extension, ret(%d).", ret); + goto err_out; + } + } + + return 0; + +err_out: + if (obj_ext->bufs[reg_items[i].ext_id] != NULL) { + kfree(obj_ext->bufs[reg_items[i].ext_id]); + } + + for (--i; i >= 0; --i) { + reg_items[i].cleanup_func(parent, + obj_ext->bufs[reg_items[i].ext_id]); + kfree(obj_ext->bufs[reg_items[i].ext_id]); + } + return ret; +} + +void roce5_obj_ext_deinit(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext) +{ + int i; + for (i = 0; i < item_num; ++i) { + if (reg_items[i].ext_id >= ROCE_EXT_FEATURE_MAX || + obj_ext->bufs[reg_items[i].ext_id] == NULL) { + continue; + } + + reg_items[i].cleanup_func(parent, + obj_ext->bufs[reg_items[i].ext_id]); + kfree(obj_ext->bufs[reg_items[i].ext_id]); + } +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h new file mode 100644 index 000000000..3e5f7196c --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/obj_ext/roce_obj_ext.h @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_OBJ_EXT_H +#define ROCE_OBJ_EXT_H + +#include <linux/types.h> +#include <linux/stddef.h> + +typedef int (*ext_init_func)(void *parent, void *ext_data); +typedef void (*ext_cleanup_func)(void *parent, void *ext_data); + +enum ROCE_EXT_ID { ROCE_EXT_HYPER_ROCE, ROCE_EXT_FEATURE_MAX }; + +struct roce5_ext_reg { + u8 ext_id; + size_t data_size; + ext_init_func init_func; + ext_cleanup_func cleanup_func; +}; + +struct roce5_obj_ext { + void *bufs[ROCE_EXT_FEATURE_MAX]; +}; + +static inline void *roce5_obj_ext_get_buf(struct roce5_obj_ext *obj_ext, + enum ROCE_EXT_ID ext_id) +{ + if (ext_id >= ROCE_EXT_FEATURE_MAX) { + return NULL; + } + return obj_ext->bufs[ext_id]; +} + +int roce5_obj_ext_init(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext); +void roce5_obj_ext_deinit(void *parent, const struct roce5_ext_reg *reg_items, + u8 item_num, struct roce5_obj_ext *obj_ext); + +#endif /* ROCE_OBJ_EXT_H */ diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c new file mode 100644 index 000000000..6cfed12be --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.c @@ -0,0 +1,125 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/jiffies.h> + +#include "roce_infra.h" +#include "securec.h" +#include "roce_option.h" + +#define ROCE_WAIT_TIME_SEC 60 + +struct roce5_option *roce5_some(void *value) +{ + int ret; + + struct roce5_option *opt = + kmalloc(sizeof(struct roce5_option), GFP_KERNEL); + if (opt == NULL) { + ROCE_ERR("Failed to alloc memory."); + return NULL; + } + + ret = strcpy_s(opt->waiting_msg, ROCE_WAITING_MSG_MAX_SIZE, + "Waiting for object release..."); + if (ret != EOK) { + ROCE_ERR("Failed to init waiting message."); + kfree(opt); + return NULL; + } + + spin_lock_init(&opt->spin_lock); + kref_init(&opt->refcount); + opt->value = value; + + return opt; +} + +static void roce5_wait_release(struct kref *refcount) +{ + struct roce5_option *opt = + container_of(refcount, struct roce5_option, refcount); + + opt->value = NULL; + complete(&opt->release_comp); +} + +static void roce5_timer_print_wait_msg(struct timer_list *tl) +{ + struct roce5_option *opt = from_timer(opt, tl, msg_timer); + + ROCE_WARN("%s", opt->waiting_msg); + + mod_timer(&opt->msg_timer, jiffies + ROCE_WAIT_TIME_SEC * HZ); +} + +void roce5_option_unset(struct roce5_option *opt, char *waiting_msg, + size_t waiting_msg_size) +{ + int ret; + unsigned long flags; /* type required by kernel API */ + + if (opt == NULL) { + ROCE_ERR("Invalid parameter, opt is NULL."); + return; + } + + if (waiting_msg != NULL && waiting_msg_size > 0) { + ret = strcpy_s(opt->waiting_msg, ROCE_WAITING_MSG_MAX_SIZE, + waiting_msg); + if (ret != EOK) { + ROCE_WARN("Failed to set waiting message, ret(%d).", + ret); + } + } + + init_completion(&opt->release_comp); + + timer_setup(&opt->msg_timer, roce5_timer_print_wait_msg, 0); + mod_timer(&opt->msg_timer, jiffies + ROCE_WAIT_TIME_SEC * HZ); + + spin_lock_irqsave(&opt->spin_lock, flags); + kref_put(&opt->refcount, roce5_wait_release); + spin_unlock_irqrestore(&opt->spin_lock, flags); + + wait_for_completion(&opt->release_comp); + + del_timer_sync(&opt->msg_timer); +} + +void *roce5_option_get_value(struct roce5_option *opt) +{ + void *ans = NULL; + unsigned long flags; /* type required by kernel API */ + + if (opt == NULL) { + ROCE_ERR("Invalid parameter, opt is NULL."); + return NULL; + } + + spin_lock_irqsave(&opt->spin_lock, flags); + if (opt->value != NULL) { + kref_get(&opt->refcount); + ans = opt->value; + } + spin_unlock_irqrestore(&opt->spin_lock, flags); + + return ans; +} + +void roce5_option_put_value(struct roce5_option *opt) +{ + unsigned long flags; /* type required by kernel API */ + if (opt == NULL) { + ROCE_ERR("Invalid parameter, opt is NULL."); + return; + } + + spin_lock_irqsave(&opt->spin_lock, flags); + if (opt->value != NULL) { + kref_put(&opt->refcount, roce5_wait_release); + } + spin_unlock_irqrestore(&opt->spin_lock, flags); +} \ No newline at end of file diff --git a/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h new file mode 100644 index 000000000..dba51cd68 --- /dev/null +++ b/drivers/infiniband/hw/hiroce5/kernel/util/option/roce_option.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef ROCE_OPTION_H +#define ROCE_OPTION_H + +#include <linux/kref.h> +#include <linux/completion.h> +#include <linux/timer.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> + +#define ROCE_WAITING_MSG_MAX_SIZE 100 + +struct roce5_option { + spinlock_t spin_lock; + struct kref refcount; /* count for the value ptr. */ + struct completion release_comp; + struct timer_list msg_timer; + char waiting_msg[ROCE_WAITING_MSG_MAX_SIZE]; + void *value; +}; + +struct roce5_option *roce5_some(void *value); +void roce5_option_unset(struct roce5_option *opt, char *waiting_msg, + size_t waiting_msg_size); +void *roce5_option_get_value(struct roce5_option *opt); +void roce5_option_put_value(struct roce5_option *opt); + +#endif /* ROCE_OPTION_H */ -- 2.37.7

2 1

[PATCH OLK-6.6] ub/hiudma5: Add hiudma5 driver
by Maogangpin 03 Aug '26

03 Aug '26

From: maogangpin <maogangpin(a)huawei.com<mailto:maogangpin@huawei.com>> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9708 CVE: NA --------------------------------- Add hiudma5 driver to support ub device of hi1825 chip. Signed-off-by: maogangpin <maogangpin(a)huawei.com<mailto:maogangpin@huawei.com>> Reviewed-by: Qiujun <roland.qiu(a)huawei.com<mailto:roland.qiu@huawei.com>> Reviewed-by: chiqijun <chiqijun(a)huawei.com<mailto:chiqijun@huawei.com>> --- .../ub/urma/hw/hiudma5/base/third/ubdevshm.h | 177 ++ .../urma/hw/hiudma5/base/udma_base_common.h | 19 + .../ub/urma/hw/hiudma5/base/udma_base_sq.c | 20 + .../ub/urma/hw/hiudma5/base/udma_base_sq.h | 17 + .../ub/urma/hw/hiudma5/kernel/CMakeLists.txt | 36 + drivers/ub/urma/hw/hiudma5/kernel/Makefile | 239 ++ .../kernel/module/base/cmdq/udma_cmd_com.c | 225 ++ .../kernel/module/base/cmdq/udma_cmd_com.h | 97 + .../kernel/module/base/cmdq/udma_cmd_eid.c | 596 ++++ .../kernel/module/base/cmdq/udma_cmd_eid.h | 37 + .../kernel/module/base/cmdq/udma_cmd_event.c | 194 ++ .../kernel/module/base/cmdq/udma_cmd_event.h | 26 + .../kernel/module/base/cmdq/udma_cmd_jetty.c | 1198 ++++++++ .../kernel/module/base/cmdq/udma_cmd_jetty.h | 69 + .../kernel/module/base/cmdq/udma_cmd_jfc.c | 334 +++ .../kernel/module/base/cmdq/udma_cmd_jfc.h | 23 + .../kernel/module/base/cmdq/udma_cmd_jfr.c | 422 +++ .../kernel/module/base/cmdq/udma_cmd_jfr.h | 20 + .../kernel/module/base/cmdq/udma_cmd_jfrc.c | 480 +++ .../kernel/module/base/cmdq/udma_cmd_jfrc.h | 28 + .../kernel/module/base/cmdq/udma_cmd_jfs.c | 341 +++ .../kernel/module/base/cmdq/udma_cmd_jfs.h | 23 + .../kernel/module/base/cmdq/udma_cmd_mig.c | 60 + .../kernel/module/base/cmdq/udma_cmd_mig.h | 14 + .../module/base/cmdq/udma_cmd_segment.c | 418 +++ .../module/base/cmdq/udma_cmd_segment.h | 30 + .../kernel/module/base/cmdq/udma_cmd_sip.c | 183 ++ .../kernel/module/base/cmdq/udma_cmd_sip.h | 20 + .../kernel/module/base/cmdq/udma_cmd_srq.c | 326 ++ .../kernel/module/base/cmdq/udma_cmd_srq.h | 22 + .../kernel/module/base/cmdq/udma_cmd_tp.c | 1885 ++++++++++++ .../kernel/module/base/cmdq/udma_cmd_tp.h | 72 + .../kernel/module/base/cmdq/udma_cmd_tpg.c | 333 +++ .../kernel/module/base/cmdq/udma_cmd_tpg.h | 32 + .../kernel/module/base/cmdq/udma_cmd_utp.c | 206 ++ .../kernel/module/base/cmdq/udma_cmd_utp.h | 19 + .../kernel/module/base/cmdq/udma_cmd_vtp.c | 656 ++++ .../kernel/module/base/cmdq/udma_cmd_vtp.h | 57 + .../kernel/module/base/common/udma_abi.h | 62 + .../kernel/module/base/common/udma_bitmap.c | 84 + .../kernel/module/base/common/udma_bitmap.h | 27 + .../kernel/module/base/common/udma_common.h | 489 +++ .../kernel/module/base/common/udma_cqm.c | 67 + .../kernel/module/base/common/udma_cqm.h | 19 + .../kernel/module/base/common/udma_log.c | 20 + .../kernel/module/base/common/udma_log.h | 179 ++ .../kernel/module/base/common/udma_misc.c | 1386 +++++++++ .../kernel/module/base/common/udma_misc.h | 131 + .../kernel/module/base/common/udma_mtt.c | 154 + .../kernel/module/base/common/udma_mtt.h | 19 + .../hiudma5/kernel/module/base/db/udma_db.c | 81 + .../hiudma5/kernel/module/base/db/udma_db.h | 40 + .../hw/hiudma5/kernel/module/bond/udma_bond.h | 10 + .../kernel/module/bond/udma_bond_common.c | 163 + .../kernel/module/context/udma_ucontext.c | 325 ++ .../kernel/module/context/udma_ucontext.h | 51 + .../hiudma5/kernel/module/device/udma_cmd.c | 281 ++ .../hiudma5/kernel/module/device/udma_cmd.h | 78 + .../hiudma5/kernel/module/device/udma_comp.h | 46 + .../kernel/module/device/udma_device.c | 242 ++ .../kernel/module/device/udma_device.h | 15 + .../kernel/module/device/udma_fast_msg.c | 787 +++++ .../kernel/module/device/udma_fast_msg.h | 35 + .../hiudma5/kernel/module/device/udma_main.c | 1044 +++++++ .../hiudma5/kernel/module/device/udma_query.c | 332 +++ .../hiudma5/kernel/module/device/udma_query.h | 20 + .../kernel/module/device/udma_ubrc_comp.c | 153 + .../kernel/module/device/udma_ubrc_comp.h | 15 + .../kernel/module/device/udma_xid_cfg.c | 72 + .../kernel/module/device/udma_xid_cfg.h | 16 + .../hw/hiudma5/kernel/module/dfx/udma_dfx.c | 2625 +++++++++++++++++ .../hw/hiudma5/kernel/module/dfx/udma_dfx.h | 71 + .../kernel/module/dfx/udma_dfx_attack.c | 831 ++++++ .../kernel/module/dfx/udma_dfx_attack.h | 24 + .../kernel/module/dfx/udma_dfx_capture.c | 312 ++ .../kernel/module/dfx/udma_dfx_capture.h | 114 + .../kernel/module/dfx/udma_dfx_cc_inject.c | 40 + .../kernel/module/dfx/udma_dfx_cc_inject.h | 24 + .../kernel/module/dfx/udma_dfx_cqm_modify.c | 362 +++ .../kernel/module/dfx/udma_dfx_cqm_modify.h | 28 + .../hiudma5/kernel/module/dfx/udma_dfx_drop.c | 249 ++ .../hiudma5/kernel/module/dfx/udma_dfx_drop.h | 34 + .../hiudma5/kernel/module/dfx/udma_dfx_l2d.c | 187 ++ .../hiudma5/kernel/module/dfx/udma_dfx_l2d.h | 28 + .../kernel/module/dfx/udma_dfx_latch.c | 671 +++++ .../kernel/module/dfx/udma_dfx_latch.h | 34 + .../kernel/module/dfx/udma_dfx_modify.c | 649 ++++ .../kernel/module/dfx/udma_dfx_modify.h | 15 + .../kernel/module/dfx/udma_dfx_perf_ctr.c | 108 + .../kernel/module/dfx/udma_dfx_perf_ctr.h | 24 + .../kernel/module/dfx/udma_dfx_perf_query.c | 152 + .../kernel/module/dfx/udma_dfx_perf_query.h | 29 + .../kernel/module/dfx/udma_dfx_port_sta.c | 182 ++ .../kernel/module/dfx/udma_dfx_port_sta.h | 24 + .../kernel/module/dfx/udma_dfx_query_tp.c | 861 ++++++ .../kernel/module/dfx/udma_dfx_query_tp.h | 16 + .../kernel/module/event/udma_resource.c | 1457 +++++++++ .../kernel/module/event/udma_resource.h | 31 + .../hiudma5/kernel/module/jetty/udma_jetty.c | 2326 +++++++++++++++ .../hiudma5/kernel/module/jetty/udma_jetty.h | 551 ++++ .../hw/hiudma5/kernel/module/jetty/udma_jfc.c | 1358 +++++++++ .../hw/hiudma5/kernel/module/jetty/udma_jfc.h | 77 + .../hw/hiudma5/kernel/module/jetty/udma_jfr.c | 1042 +++++++ .../hw/hiudma5/kernel/module/jetty/udma_jfs.c | 2107 +++++++++++++ .../hw/hiudma5/kernel/module/jetty/udma_srq.c | 528 ++++ .../hw/hiudma5/kernel/module/jetty/udma_srq.h | 205 ++ .../kernel/module/rc_table/udma_rc_table.c | 1049 +++++++ .../kernel/module/rc_table/udma_rc_table.h | 46 + .../hiudma5/kernel/module/segment/udma_key.c | 73 + .../hiudma5/kernel/module/segment/udma_key.h | 29 + .../kernel/module/segment/udma_segment.c | 788 +++++ .../kernel/module/segment/udma_segment.h | 78 + .../kernel/module/segment/udma_shm_manage.c | 67 + .../kernel/module/segment/udma_shm_manage.h | 25 + .../hiudma5/kernel/module/sysfs/udma_sysfs.c | 476 +++ .../hiudma5/kernel/module/sysfs/udma_sysfs.h | 158 + .../kernel/module/sysfs/udma_sysfs_api.h | 216 ++ .../kernel/module/sysfs/udma_sysfs_common.c | 715 +++++ .../kernel/module/sysfs/udma_sysfs_common.h | 88 + .../kernel/module/sysfs/udma_sysfs_config.c | 688 +++++ .../kernel/module/sysfs/udma_sysfs_config.h | 13 + .../kernel/module/table_manage/udma_net.c | 330 +++ .../kernel/module/table_manage/udma_net.h | 77 + .../hw/hiudma5/kernel/module/tp/udma_tp.c | 1809 ++++++++++++ .../hw/hiudma5/kernel/module/tp/udma_tp.h | 184 ++ .../hw/hiudma5/kernel/module/tp/udma_tpg.c | 677 +++++ .../hw/hiudma5/kernel/module/tp/udma_tpg.h | 54 + .../hw/hiudma5/kernel/module/tp/udma_utp.c | 209 ++ .../hw/hiudma5/kernel/module/tp/udma_utp.h | 32 + .../hw/hiudma5/kernel/module/tp/udma_vtp.c | 1011 +++++++ .../hw/hiudma5/kernel/module/tp/udma_vtp.h | 66 + .../kernel/module/userctl/udma_userctl.c | 96 + .../kernel/module/userctl/udma_userctl.h | 51 + .../kernel/module/userctl/udma_userctl_dfx.c | 243 ++ .../kernel/module/userctl/udma_userctl_dfx.h | 14 + .../kernel/module/userctl/udma_userctl_uvs.c | 364 +++ .../kernel/module/userctl/udma_userctl_uvs.h | 23 + .../hw/hiudma5/kernel/module/vram/udma_vram.c | 225 ++ .../hw/hiudma5/kernel/module/vram/udma_vram.h | 95 + .../kernel/module/vram/udma_vram_base.c | 104 + .../kernel/module/vram/udma_vram_base.h | 22 + .../kernel/module/vram/udma_vram_cfg.c | 108 + .../kernel/module/vram/udma_vram_cfg.h | 20 + .../kernel/module/vram/udma_vram_common.h | 25 + .../kernel/module/vram/udma_vram_mem.c | 412 +++ .../kernel/module/vram/udma_vram_mem.h | 44 + .../kernel/module/vram/udma_vram_type_dev.c | 37 + .../kernel/module/vram/udma_vram_type_dev.h | 10 + .../kernel/module/vram/udma_vram_type_tp.c | 89 + .../kernel/module/vram/udma_vram_type_tp.h | 10 + .../kernel/module/vram/udma_vram_type_tpg.c | 69 + .../kernel/module/vram/udma_vram_type_tpg.h | 10 + .../kernel/module/vram/udma_vram_type_vtp.c | 45 + .../kernel/module/vram/udma_vram_type_vtp.h | 10 + 154 files changed, 43857 insertions(+) create mode 100644 drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h create mode 100644 drivers/ub/urma/hw/hiudma5/base/udma_base_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c create mode 100644 drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/Makefile create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c create mode 100644 drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h diff --git a/drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h b/drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h new file mode 100644 index 000000000..79da16b5d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/third/ubdevshm.h @@ -0,0 +1,177 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Hisilicon Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef _UBDEVSHM_H +#define _UBDEVSHM_H + +#include <linux/kernel.h> + +struct uba_eid { + u64 eid : 20; /* bus_instance_eid */ + u64 reserved0 : 44; + u64 reserved1 : 64; +}; + +union uba_attr { + struct { + u64 readable : 1; + u64 writeable : 1; + u64 executeable : 1; + u64 token_value_required : 1; + u64 reserved : 60; + } bs; + u64 value; +}; + +union acquire_attr { + struct { + u64 require_pin : 1; + u64 require_invalidate : 1; + u64 reserved : 62; + } bs; + u64 value; +}; + +struct mem_uva { + u64 va; + u64 size; + u64 invalidate_tag; +}; + +struct mem_uba { + u64 uba; + u64 size; + u32 token_id; /* BIT[0:19] is valid for token_id */ + struct uba_eid eid; + union uba_attr attr; + + void *mem_handle; /* used by ubdevshm inner */ +}; + +enum identity_type { + /* + * IDENTIY_TGID constraint: + * The shared and user processes must reside within the same PID namespace. + */ + IDENTIY_TGID, + IDENTIY_TYPE_MAX, +}; + +struct shm_user { + enum identity_type type; + u32 user_len; + void *user; +}; + +struct access_ctx { + u32 shm_container_id; + u32 access_ctx_id; +}; + +typedef int (*invalidate)(u64 invalidate_tag); + +#define UBDEV_SHM_DRIVER_NAME_LENGTH 128 + +struct ubdevshm_mem_ops { + char name[UBDEV_SHM_DRIVER_NAME_LENGTH]; + u32 version; + + /* Request access to the virtual address range of the given VA */ + int (*acquire)(struct mem_uva *va, union acquire_attr *attr, + invalidate func, struct mem_uba *uba); + + /* Release the uba address */ + int (*release)(struct mem_uba *uba); +}; + +#ifdef ushm_handle_t +#error "ushm_handle_t already defined!" +#else +typedef unsigned long ushm_handle_t; +#endif + +/** + * ubdevshm initialization check interface + * @return: true indicates ubdevshm is avaialbe, false indicates not + * @note: this interface should be called before any other ubdevshm interfaces + */ +bool ubdevshm_is_inited(void); + +/* + * NOTE: + * pid/tgid is implicit in the calling context and not passed explicitly + * in the following interfaces. + */ + +/** + * Memory provider registration interface + * @input: struct ubdevshm_mem_ops *ops, memory provider specific alloc/free callbacks + * @output: ushm_handle_t *handle, corresponding memory provider handle + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_register_ops(struct ubdevshm_mem_ops *ops, ushm_handle_t *handle); + +/** + * Memory provider unregistration interface + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_unregister_ops(ushm_handle_t *handle); + +/** + * Share memory segment registration interface + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @input: struct mem_uva *va, virtual address range of the given VA + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_register_segment(ushm_handle_t *handle, struct mem_uva *va); + +/** + * Share memory segment unregistration interface + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @input: struct mem_uva *va, virtual address range of the given VA + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_unregister_segment(ushm_handle_t *handle, struct mem_uva *va); + +/** + * Authorize the memory of mem_uva to shm_user + * @input: ushm_handle_t *handle, corresponding memory provider handle + * @input: struct shm_user *user, corresponding authorized process information + * @input: struct mem_uva *va, virtual address range of the given VA + * @output: struct access_ctx *ctx, the context of the segment obtained after authorization + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_grant_access(ushm_handle_t *handle, struct shm_user *user, + struct mem_uva *va, struct access_ctx *ctx); + +/** + * De-authorize the shared memory associated with access_ctx + * @input: struct access_ctx *ctx, the context of the segment obtained after authorization + * @return: 0 indicates success, while any other value indicates failure + */ +int ubdevshm_ungrant_access(struct access_ctx *ctx); + +/** + * Request to acqurie the address corresponding to mem_uva and convert it into + * a ub address + * @input: struct access_ctx *ctx, the context of the segment + * @input: struct mem_uva *va, virtual address range of the given VA + * @input: union acquire_attr *attr, request attributes + * @input: func, When require_invalidate in acquire_attr is set to 1, + * this function must not be NULL, and is used as a callback + * for the memory provider to release memory. + * @output: struct mem_uba uba, the ub address corresponding to the virtual address va + */ +int ubdevshm_acquire_uba(struct access_ctx *ctx, struct mem_uva *va, + union acquire_attr *attr, invalidate func, + struct mem_uba *uba); + +/** + * Request to release the address corresponding to mem_uba + * @input: struct access_ctx *ctx, the context of the segment + * @input: struct mem_uba *uba, the ub address to be released + */ +int ubdevshm_release_uba(struct access_ctx *ctx, struct mem_uba *uba); +#endif /* _UBDEVSHM_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/base/udma_base_common.h b/drivers/ub/urma/hw/hiudma5/base/udma_base_common.h new file mode 100644 index 000000000..172203737 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/udma_base_common.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_BASE_COMMON_H +#define UDMA_BASE_COMMON_H + +#ifndef BIT +#define BIT(n) (1U << (n)) +#endif + +enum udma_device_type { + UDMA_DEV_TYPE_UNKNOWN = 0U, /**< unknown device type */ + UDMA_DEV_TYPE_HI1825_V100 = BIT(0), /**< HI1825 V100 device type */ +}; + +#define UDMA_DEV_TYPE_ALL (UDMA_DEV_TYPE_HI1825_V100) + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c new file mode 100644 index 000000000..79f9ad1a4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.c @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "udma_base_sq.h" +#include <stddef.h> + +void udma_base_construct_sq_db(union tag_ub_sq_db_u *send_queue_db, u32 xid, + u32 pi, u8 cos) +{ + if (send_queue_db == NULL) { + return; + } + send_queue_db->sq_db_value = 0; + send_queue_db->sq_db.dw0.bs.xqn = xid; + send_queue_db->sq_db.dw0.bs.type = UB_SERVICE_TYPE; + send_queue_db->sq_db.dw1.bs.pi = (pi >> UB_SQ_DB_SHIFT) & 0xFF; + send_queue_db->sq_db.dw0.bs.cntx_size = UB_CNTX_SIZE; + send_queue_db->sq_db.dw0.bs.cos = cos; +} diff --git a/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h new file mode 100644 index 000000000..b43a7c092 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/base/udma_base_sq.h @@ -0,0 +1,17 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_BASE_SQ_H +#define UDMA_BASE_SQ_H + +#include "ub_xqe_format.h" + +#define UB_SERVICE_TYPE 2 +#define UB_SQ_DB_SHIFT 8 +#define UB_CNTX_SIZE 2 + +void udma_base_construct_sq_db(union tag_ub_sq_db_u *send_queue_db, u32 xid, + u32 pi, u8 cos); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt b/drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt new file mode 100644 index 000000000..832ac9aa9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/CMakeLists.txt @@ -0,0 +1,36 @@ +if("${BUILD_VERSION}" MATCHES "ub_ascend") + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core-v1) +elseif("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(UBUS_DIR ${TOP_DIR}/drivers/ubus) + set(KDIR ${KERNEL_WORK_DIR}/../linux-4.19) + set(UMMU_CORE_BUILD_DIR ${UBUS_DIR}/kernel/ummu-core-v1) +else() + set(UMMU_CORE_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ummu-core) +endif() + +set(UBUS_UBC_BUILD_DIR ${UBUS_BUILD_DIR}/kernel/ubus) +set(HISDK5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/sdk/knldk/lld) +set(HINIC5_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/ChipSolution/src/dpu_platform_library/host/service/nic/linux) +set(UBCORE_BUILD_DIR ${BUILD_CACHE_TOP_DIR}/cmake/component/kernel/urma/code/output/kmod/ubcore) + +if("${PRODUCT}" STREQUAL "ascend910D" OR "${PRODUCT}" STREQUAL "ascend910Desl") + set(udma_depends ubus URMA) + add_device_ko(LOCAL_MODULE hiudma5 + KO_SRC_FOLDER ${CMAKE_CURRENT_SOURCE_DIR} + MAKE_ARGS "IS_ASCEND=true" "URMA_SOURCE_DIR=${URMA_SOURCE_DIR}" + TARGETE_DPENDS "${udma_depends}") +else() + add_custom_target(hiudma5_ko + COMMENT echo "build ${CMAKE_CURRENT_SOURCE_DIR} start." + COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/Makefile ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${MAKE} -j64 -C ${KDIR} M=${CMAKE_CURRENT_BINARY_DIR} src=${CMAKE_CURRENT_SOURCE_DIR} UBUS_UBC_BUILD_DIR=${UBUS_UBC_BUILD_DIR} UBCORE_SYMVERS=${UBCORE_BUILD_DIR}/Module.symvers UMMU_CORE_BUILD_DIR=${UMMU_CORE_BUILD_DIR} HI1823_TRUNK_DIR=${TOP_DIR}/ChipSolution HI1823_BUILD_DIR=${TOP_DIR}/ChipSolution/build CONFIG_UBUS_DEVICE=y HI1823_OS_TYPE=openEuler24.03 VERBOSE=1 HISDK5_SYMVERS=${HISDK5_BUILD_DIR}/Module.symvers HINIC5_SYMVERS=${HINIC5_BUILD_DIR}/Module.symvers CHIP_VERSION=hi1825v100 + COMMAND cp -f *.ko ${CMAKE_INSTALL_PREFIX}/ko + DEPENDS kernel + ) + + add_dependencies(hiudma5_ko URMA) + add_dependencies(hiudma5_ko hisdk5_ko) + add_dependencies(hiudma5_ko hinic5_ko) + add_dependencies(hiudma5_ko ubus_ko) + add_dependencies(hiudma5_ko ummu_core_ko) +endif() \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/Makefile b/drivers/ub/urma/hw/hiudma5/kernel/Makefile new file mode 100644 index 000000000..7230ad40d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/Makefile @@ -0,0 +1,239 @@ +# +# Makefile for the Hisilicon ubep drivers. +# +# +mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +KDIR=/lib/modules/$(shell uname -r)/build +TOP_DIR := $(src)/../../../../../../ +KUDMA_DIR := $(src) + +ifeq ($(CONFIG_UBUS_DEVICE), y) +TOP_DIR := $(HI1823_TRUNK_DIR) +UMDK_DIR := $(HI1823_TRUNK_DIR)/src/dpu_platform_library/host/service/urma/code +KUDMA_DIR := $(HI1823_TRUNK_DIR)/src/dpu_platform_library/host/service/udma/kernel +endif + +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/public \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfg_mgmt \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mpu \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/rdma/hi182x \ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/roce \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/nic \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/nic \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/mag \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/migrate \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/bond \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/cfm \ + -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/fast_msg \ + -I$(TOP_DIR)/src/dpu_platform_library/host/include/sdk/knldk \ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/uboe \ + -I$(TOP_DIR)/src/dpu_develop_interface/fw_msg_intf/ubc_net \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/hisdk \ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_tool_msg \ + -I$(TOP_DIR)/src/dpu_platform_library/include\ + -I$(TOP_DIR)/src/dpu_platform_library/include/drv_fw_msg/uboe \ + -I$(TOP_DIR)/src/dpu_platform_library/host/service/include \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_sdk_intf/ossl \ + -I$(TOP_DIR)/src/dpu_develop_interface/drv_srvc_intf/udma \ + -I$(TOP_DIR)/platform/huawei_secure_c/include \ + -I$(TOP_DIR)/platform/huawei_secure_c/src + +ifeq ($(CONFIG_UBUS_DEVICE), y) +ccflags-y += -I$(TOP_DIR)/src/dpu_develop_interface \ + -I$(TOP_DIR)/../opensource/openeuler/kernel/olk-6.6/include/linux +endif + +ccflags-y += -I$(UMDK_DIR)/kmod/ubcore \ + -I$(UMDK_DIR)/kmod/ubcore/include \ + -I$(UMDK_DIR)/include \ + -I$(UMDK_DIR)/include/common \ + -I$(KUDMA_DIR)/module/base/common \ + -I$(KUDMA_DIR)/module \ + -I$(KUDMA_DIR)/module/base/cmdq \ + -I$(KUDMA_DIR)/module/bond \ + -I$(KUDMA_DIR)/module/vram \ + -I$(KUDMA_DIR)/module/userctl \ + -I$(TOP_DIR)/src/dpu_platform_library/host/include/cfm/bond \ + -I$(KUDMA_DIR)/../../../../../dpu_develop_interface/fw_msg_intf/cfm \ + -I$(KUDMA_DIR)/../../../../../dpu_develop_interface/drv_cfm_intf/hmm \ + -I$(TOP_DIR)/src/dpu_platform_library/host/cfm/hmm \ + -I$(KUDMA_DIR)/module/dfx \ + -I$(KUDMA_DIR)/../base \ + -I$(KUDMA_DIR)/../base/third \ + -I$(KUDMA_DIR)/module/segment \ + -I$(KUDMA_DIR)/module/base/db \ + -I$(KUDMA_DIR)/module/context \ + -I$(KUDMA_DIR)/module/sysfs \ + -I$(KUDMA_DIR)/module/device \ + -I$(KUDMA_DIR)/module/tp \ + -I$(KUDMA_DIR)/module/jetty \ + -I$(KUDMA_DIR)/module/rc_table \ + -I$(KUDMA_DIR)/module/event \ + -I$(KUDMA_DIR)/module/table_manage + +ifeq ($(VERSION),UDMA_STANDARD) + ccflags-y += -DSIGN_MTT_EN +endif + +ccflags-y += -I$(UMDK_DIR)/src/urma/kmod/include \ + -I$(UMDK_DIR)/src/urma/kmod/include/ub/urma \ + -I$(UMDK_DIR)/src/urma/kmod/drivers/ub/urma/ubcore + +ccflags-y += -DLINUX_KERNEL -Wno-implicit-fallthrough +ccflags-y += -DSECUREC_EXPORT_KERNEL_SYMBOL=0 + +##Safe compilation options +ccflags-y += -fstack-protector-all -Wl,-z,noexecstack +ifneq ($(CONFIG_UBUS_DEVICE), y) +ccflags-y += -Werror -std=gnu11 +endif + +KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/ci/lib/UBCORE_Module.symvers +ifeq ($(CONFIG_UBUS_DEVICE), y) +KBUILD_EXTRA_SYMBOLS += $(HISDK5_SYMVERS) +KBUILD_EXTRA_SYMBOLS += $(HINIC5_SYMVERS) +else +KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/ci/lib/SDK_Module.symvers +KBUILD_EXTRA_SYMBOLS += $(TOP_DIR)/ci/lib/NIC_Module.symvers +endif + +UDMA_OBJS := module/tp/udma_tp.o \ + module/tp/udma_tpg.o \ + module/device/udma_query.o \ + module/tp/udma_vtp.o \ + module/tp/udma_utp.o \ + module/device/udma_fast_msg.o \ + module/device/udma_xid_cfg.o \ + module/device/udma_device.o \ + module/device/udma_cmd.o \ + module/event/udma_resource.o \ + module/table_manage/udma_net.o \ + module/device/udma_ubrc_comp.o \ + module/segment/udma_segment.o \ + module/segment/udma_key.o \ + module/segment/udma_shm_manage.o \ + module/context/udma_ucontext.o \ + module/base/db/udma_db.o \ + module/sysfs/udma_sysfs.o \ + module/sysfs/udma_sysfs_common.o \ + module/sysfs/udma_sysfs_config.o \ + module/jetty/udma_jetty.o \ + module/jetty/udma_jfs.o \ + module/jetty/udma_jfr.o \ + module/jetty/udma_jfc.o \ + module/jetty/udma_srq.o \ + module/rc_table/udma_rc_table.o + +UDMA_CMDQ_OBJS := module/base/cmdq/udma_cmd_jetty.o \ + module/base/cmdq/udma_cmd_com.o \ + module/base/cmdq/udma_cmd_eid.o \ + module/base/cmdq/udma_cmd_event.o \ + module/base/cmdq/udma_cmd_jfs.o \ + module/base/cmdq/udma_cmd_jfc.o \ + module/base/cmdq/udma_cmd_segment.o \ + module/base/cmdq/udma_cmd_srq.o \ + module/base/cmdq/udma_cmd_jfrc.o \ + module/base/cmdq/udma_cmd_sip.o \ + module/base/cmdq/udma_cmd_utp.o \ + module/base/cmdq/udma_cmd_tp.o \ + module/base/cmdq/udma_cmd_vtp.o \ + module/base/cmdq/udma_cmd_tpg.o \ + module/base/cmdq/udma_cmd_jfr.o \ + module/base/cmdq/udma_cmd_mig.o + +COM_OBJS := module/base/common/udma_bitmap.o \ + module/base/common/udma_cqm.o \ + module/base/common/udma_mtt.o \ + module/base/common/udma_misc.o \ + module/base/common/udma_log.o + + +BOND_OBJS := module/bond/udma_bond_common.o + +UDMA_VRAM := module/vram/udma_vram.o \ + module/vram/udma_vram_base.o \ + module/vram/udma_vram_cfg.o \ + module/vram/udma_vram_mem.o \ + module/vram/udma_vram_type_dev.o \ + module/vram/udma_vram_type_tp.o \ + module/vram/udma_vram_type_tpg.o \ + module/vram/udma_vram_type_vtp.o + + +USERCTL_OBJS := module/userctl/udma_userctl.o \ + module/userctl/udma_userctl_uvs.o \ + module/userctl/udma_userctl_dfx.o + +BASE_OBJS := ../base/udma_base_sq.o + +HMM_OBJS := ../../../cfm/hmm/hmm_buddy.o \ + ../../../cfm/hmm/hmm_common.o \ + ../../../cfm/hmm/hmm_mtt.o \ + ../../../cfm/hmm/hmm_mpt.o \ + ../../../cfm/hmm/hmm_em.o \ + +DFX_OBJS := module/dfx/udma_dfx.o \ + module/dfx/udma_dfx_attack.o \ + module/dfx/udma_dfx_capture.o \ + module/dfx/udma_dfx_drop.o \ + module/dfx/udma_dfx_port_sta.o \ + module/dfx/udma_dfx_cc_inject.o \ + module/dfx/udma_dfx_cqm_modify.o \ + module/dfx/udma_dfx_query_tp.o \ + module/dfx/udma_dfx_perf_ctr.o \ + module/dfx/udma_dfx_modify.o \ + module/dfx/udma_dfx_latch.o \ + module/dfx/udma_dfx_l2d.o \ + module/dfx/udma_dfx_perf_query.o + +HW_SECUREC_OBJS := ../../../../../../platform/huawei_secure_c/src/securecutil.o \ + ../../../../../../platform/huawei_secure_c/src/secureinput_a.o \ + ../../../../../../platform/huawei_secure_c/src/secureprintoutput_a.o \ + ../../../../../../platform/huawei_secure_c/src/memset_s.o \ + ../../../../../../platform/huawei_secure_c/src/memcpy_s.o \ + ../../../../../../platform/huawei_secure_c/src/strncpy_s.o \ + ../../../../../../platform/huawei_secure_c/src/strncat_s.o \ + ../../../../../../platform/huawei_secure_c/src/sprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/vsprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/vsnprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/snprintf_s.o \ + ../../../../../../platform/huawei_secure_c/src/vsscanf_s.o \ + ../../../../../../platform/huawei_secure_c/src/sscanf_s.o \ + ../../../../../../platform/huawei_secure_c/src/memmove_s.o \ + ../../../../../../platform/huawei_secure_c/src/strcpy_s.o \ + ../../../../../../platform/huawei_secure_c/src/strcat_s.o \ + ../../../../../../platform/huawei_secure_c/src/strtok_s.o + +MAIN_OBJS := module/device/udma_main.o + +hiudma5-objs := $(MAIN_OBJS) \ + $(UDMA_OBJS) \ + $(UDMA_CMDQ_OBJS) \ + $(COM_OBJS) \ + $(BOND_OBJS) \ + $(UDMA_VRAM) \ + $(HMM_OBJS) \ + $(DFX_OBJS) \ + $(HW_SECUREC_OBJS) \ + $(USERCTL_OBJS) \ + $(BASE_OBJS) + +ifeq ($(CONFIG_UBUS_DEVICE), y) +KBUILD_EXTRA_SYMBOLS += $(UBUS_UBC_BUILD_DIR)/Module.symvers +ccflags-y += -D__UBUS_DRIVER__ +endif + +obj-m := hiudma5.o + +all:build_info default + +build_info: + mkdir -p build + @echo "CURDIR=$(CURDIR)" > build/build_src.txt + @for obj in $(hiudma5-objs); do \ + echo $$(realpath $$obj) >> build/build_src.txt; \ + done + +include $(HI1823_BUILD_DIR)/host/linux/Makefile.ko diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c new file mode 100644 index 000000000..c3706ab75 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/sched.h> +#include <linux/ktime.h> +#include "comm_defs.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" + +#define UDMA_CMD_DPATH_CHECK_VAL 0xa5 + +#define UDMA_CMD_WQEBB_INDEX_BASE 4 +#define UDMA_CMD_WQEBB_INDEX_MIN 2 +#define UDMA_CMD_WQEBB_INDEX_MAX 4 +#define UDMA_CMD_DEFAULT_WQEBB_SIZE 64 +#define UDMA_US_TO_NS(x) ((x)*1000) + +struct udma_page_shift { + u32 size; + u32 shift; +}; + +void udma_cpu_to_be32(void *data, u32 len) +{ + u32 chunk_sz = sizeof(u32); + u32 *mem = (u32 *)data; + u32 data_len; + u32 i; + + if (data == NULL) + return; + + data_len = len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = cpu_to_be32(*mem); + mem++; + } +} + +void udma_be32_to_cpu(void *data, u32 len) +{ + u32 i, chunk_sz = sizeof(u32); + u32 data_len = len; + u32 *mem = (u32 *)data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = be32_to_cpu(*mem); + mem++; + } +} + +void udma_be64_to_cpu(void *data, u32 len) +{ + u32 i, chunk_sz = sizeof(u64); + u32 data_len = len; + u64 *mem = (u64 *)data; + + if (!data) + return; + + data_len = data_len / chunk_sz; + + for (i = 0; i < data_len; i++) { + *mem = be64_to_cpu(*mem); + mem++; + } +} + +void udma_fill_cmd_header(struct ub_cmd_com_header *header, u8 cmd_type, + u8 cmd_subtype, u16 cmd_len, u32 id) +{ + header->major_version = 1; + header->minor_version = 0; + header->cmd_type = cmd_type; + header->cmd_subtype = cmd_subtype; + header->cmd_len = cmd_len; + header->idx = id; +} + +static u32 udma_get_timeout_for_dpath(enum ub_env_type env_type) +{ + return env_type == UB_ENV_ESL ? UB_CMD_TIME_CLASS_B_FOR_ESL : + UB_CMD_TIME_CLASS_B; +} + +int udma_cmd_check_dpath_dma(struct udma_device *udma_dev, u8 *buf, u32 size, + u32 step) +{ + u32 timeout = udma_get_timeout_for_dpath(udma_dev->env_type); + ulong end = jiffies + msecs_to_jiffies(timeout); + u32 i; + + step = step == 0 ? 1 : step; + size = size * step; + + while (time_before(jiffies, end) != 0) { + for (i = 0; i < size; i = i + step) { + if (buf[i] != UDMA_CMD_DPATH_CHECK_VAL) + break; + } + + if (i == size) + return 0; + + cond_resched(); + } + udma_err(udma_dev->dev, UDMA_M_COM, + "check udma cmd dpath failed, size(%u), timeout(%u)\n", size, + timeout); + udma_dump_byte_err_dfx(udma_dev, buf, size); + + return -EIO; +} + +int udma_dpath_drain_handle(struct udma_device *udma_dev, u8 *buf, + u32 dpath_oq_num) +{ + if (dpath_oq_num > UDMA_DPATH_MAX_OQ_NUM) { + udma_err(udma_dev->dev, UDMA_M_COM, + "dpath oq num(%u) is over max oq num(%u)\n", + dpath_oq_num, UDMA_DPATH_MAX_OQ_NUM); + return -EINVAL; + } + + /* dpath_oq_num return actually pending flush of oq num data, driver check magic number of corresponding count; */ + return udma_cmd_check_dpath_dma(udma_dev, buf, dpath_oq_num, + sizeof(u32)); +} + +uint32_t udma_cal_page_shift(uint32_t buf_size) +{ + const struct udma_page_shift page_shift_map[] = { + { PAGE_SIZE_4K, PAGE_SHIFT_4K }, + { PAGE_SIZE_64K, PAGE_SHIFT_64K }, + { PAGE_SIZE_2M, PAGE_SHIFT_2M }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(page_shift_map); i++) { + if (buf_size == page_shift_map[i].size) { + return page_shift_map[i].shift; + } + } + + return PAGE_SHIFT_4K; +} + +/* + * Page size of Queue. + * 0: 4KB + * 4: 64KB + * 9: 2MB + * others: reserved + */ +u32 udma_get_queue_kernel_page_size(u8 mtt_pro_en, u32 queue_size) +{ + u32 page_shift = PAGE_SHIFT; + + if (mtt_pro_en == 1) { + page_shift = udma_cal_page_shift(queue_size); + } + + return ((u32)(page_shift - PAGE_SHIFT_4K)) & UB_PAGE_SIZE_MASK; +} + +static u32 udma_get_timeout_for_cmd(enum ub_env_type env_type) +{ + return env_type == UB_ENV_ESL ? UB_CMD_TIME_CLASS_C_FOR_ESL : + UB_CMD_TIME_CLASS_C; +} + +int udma_send_cmd_box(struct udma_device *udma_dev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cmd_buf *buf_out, + ub_cmdq_cqe_udata_s *out_param, struct cmdq_rtt *rtt) +{ + int ret; + ktime_t time_start = 0; + ktime_t time_cost = 0; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + time_start = ktime_get(); + } + + ret = cqm5_send_cmd_box(udma_dev->hwdev, HINIC5_MOD_UB, cmd, buf_in, + buf_out, (u64 *)(void *)&cmdq_cqe, + udma_get_timeout_for_cmd(udma_dev->env_type), + HINIC5_CHANNEL_UB); + if (unlikely(ret != 0)) { + udma_err( + udma_dev->dev, UDMA_M_COM, + "cmdq failed! cmd_type: 0x%x, sub_type: 0x%x, err_status: 0x%x, err_value: 0x%x, dw0: %x, dw1: 0x%x", + cmdq_cqe.bs.cmd_type, cmdq_cqe.bs.sub_type, + cmdq_cqe.bs.err_status, cmdq_cqe.bs.err_value, + cmdq_cqe.bs.dw0_value, cmdq_cqe.bs.dw1_value); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1) && + rtt != NULL) { + /* debug when enabled, statistics cmdq latency on success */ + if (unlikely(ret != 0)) { + return ret; + } + time_cost = ktime_to_ns(ktime_sub(ktime_get(), time_start)); + atomic64_inc(&rtt->times); + atomic64_add(UDMA_US_TO_NS(cmdq_cqe.bs.exec_time), + &rtt->cmdq_npu_rtt_sum); + atomic64_add(time_cost, &rtt->cmdq_rtt_sum); + } + + if (out_param != NULL) { + (void)memcpy_s(out_param, sizeof(ub_cmdq_cqe_udata_s), + &cmdq_cqe, sizeof(ub_cmdq_cqe_udata_s)); + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h new file mode 100644 index 000000000..c4f90d742 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_com.h @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_COM_H +#define UDMA_CMD_COM_H + +#include "hinic5_cqm.h" +#include "udma_common.h" +#include "ub_npu_base_cmd.h" + +#define UB_RC_HI_PADDR_SHIFT 40 // high:24bit, 64-24= 40 +#define UB_RC_HI_PADDR_MASK 0xffffff +#define UB_RC_LOW_PADDR_SHIFT 8 +#define UB_DPATH_CHECK_FLAG ((u32)(1 << 31)) +#define UB_PAGE_SIZE_MASK 0xf +#define UDMA_DPATH_MAX_OQ_NUM 32 + +#define DATALEN_RATIO 2 // user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 // user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +/* cmdq timeout for ASIC, FPGA */ +enum { + UB_CMD_TIME_CLASS_A = 3000, + UB_CMD_TIME_CLASS_B = 6000, + UB_CMD_TIME_CLASS_C = 10000, + UB_CMD_TIME_CLASS_D = 15000, + UB_CMD_TIME_CLASS_E = 20000 +}; + +/* cmdq timeout for ESL */ +enum { + UB_CMD_TIME_CLASS_A_FOR_ESL = 3000000, + UB_CMD_TIME_CLASS_B_FOR_ESL = 6000000, + UB_CMD_TIME_CLASS_C_FOR_ESL = 10000000, + UB_CMD_TIME_CLASS_D_FOR_ESL = 15000000, + UB_CMD_TIME_CLASS_E_FOR_ESL = 20000000, +}; + +enum jetty_type { + JETTY_RAW_OR_NIC, + JETTY_UM, + JETTY_RC, + JETTY_RM, + JETTY_TYPE_RESERVED, +}; + +typedef struct udma_cqm_cmd_type_info { + u8 cmd_type; + u8 cmd_subtype; +} udma_cqm_cmd_type_info_t; + +void udma_cpu_to_be32(void *data, u32 len); +void udma_be32_to_cpu(void *data, u32 len); +void udma_be64_to_cpu(void *data, u32 len); + +void udma_fill_cmd_header(struct ub_cmd_com_header *header, u8 cmd_type, + u8 cmd_subtype, u16 cmd_len, u32 id); +int udma_cmd_check_dpath_dma(struct udma_device *udma_dev, u8 *buf, u32 size, + u32 step); +u32 udma_get_queue_kernel_page_size(u8 mtt_pro_en, u32 queue_size); +int udma_send_cmd_box(struct udma_device *udma_dev, u8 cmd, + struct tag_cqm_cmd_buf *buf_in, + struct tag_cqm_cmd_buf *buf_out, + ub_cmdq_cqe_udata_s *out_param, struct cmdq_rtt *rtt); +int udma_dpath_drain_handle(struct udma_device *udma_dev, u8 *buf, + u32 dpath_oq_num); + +/** + * @brief data convert cmdq buf + * + * @param[in] in_buf - tool passed in of data + * @param[in] in_len - tool passed in of data length + * @param[out] out_buf - cmdq inbuf in of buf + * @param[out] out_len - cmdq inbuf in of buf length + * @param[in] offset - in ctx in offset + * + * @details user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again + * todo pending tool passed in of data in each u8 all in normal use when change is memory cpy + * + * @return NULL + * */ +static inline void udma_cmdq_data_to_context_buf(const u8 *in_buf, u32 in_len, + u8 *out_buf, u32 out_len, + u32 offset) +{ + u32 i = 0; + + for (i = 0; (i < (in_len / DATALEN_RATIO)) && (i + offset < out_len); + ++i) { + *(out_buf + i + offset) = + (in_buf[i * DATALEN_RATIO] << DATA_SHIFT) + + in_buf[i * DATALEN_RATIO + 1]; + } +} +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c new file mode 100644 index 000000000..c5ab7660a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.c @@ -0,0 +1,596 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include <linux/workqueue.h> +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "ub_npu_eid_cmd_defs.h" +#include "ub_npu_eid_cmd.h" +#include "ub_npu_upi_cmd_defs.h" +#include "ub_npu_upi_cmd.h" +#include "udma_cmd_eid.h" + +#define EID_ERR_VFID 0xffff + +static void udma_fill_ueid_cmd(enum UB_CMD_UPI_TYPE cmd_type, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + ub_upi_mig_state_e mig_state, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_upi_update); + struct ub_cmd_upi_update *ueid_inbuf = NULL; + + ueid_inbuf = (struct ub_cmd_upi_update *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&ueid_inbuf->header, UB_CMD_TYPE_UPI, cmd_type, + cmd_len, 0); + ueid_inbuf->body.upi = cfg->upi; + ueid_inbuf->body.vfid = vf_id; + ueid_inbuf->body.mig_state = mig_state; + + udma_cpu_to_be32(ueid_inbuf, cmd_len); + (void)memcpy_s(ueid_inbuf->body.id, sizeof(ueid_inbuf->body.id), + cfg->eid.raw, + sizeof(cfg->eid.raw)); //id not need endian conversion +} + +static int udma_add_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + ub_upi_mig_state_e mig_state) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_EID, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_update), &cqm_cmd_outbuf, + (u16)(sizeof(int))); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for add_ueid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_ueid_cmd(UB_CMD_UPI_UPDATE, vf_id, cfg, mig_state, + cqm_cmd_inbuf); + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_UPDATE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send add_ueid command, ret = %d.\n", ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int udma_delete_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_EID, "%s to_udma_dev failed.\n", __func__); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_update), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for delete_ueid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_ueid_cmd(UB_CMD_UPI_CLEAR, vf_id, cfg, UB_UPI_NORM, + cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_CLEAR, cqm_cmd_inbuf, NULL, + NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send delete_ueid command, ret = %d.\n", + ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_update_upi(struct udma_device *udma_dev, u16 vfid, + ub_seid_ctx_info_s *seid_ctx, + ub_upi_mig_state_e mig_state) +{ + u16 cmd_len = sizeof(struct ub_cmd_upi_update); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_upi_update *ueid_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_update), &cqm_cmd_outbuf, + (u16)(sizeof(int))); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_EID, + "Update upi: failed to alloc inbuf for add_ueid, ret = %d.\n", + ret); + return -ENOMEM; + } + + ueid_inbuf = (struct ub_cmd_upi_update *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&ueid_inbuf->header, UB_CMD_TYPE_UPI, + UB_CMD_UPI_UPDATE, cmd_len, 0); + ueid_inbuf->body.upi = seid_ctx->upi; + ueid_inbuf->body.vfid = vfid; + ueid_inbuf->body.mig_state = mig_state; + udma_cpu_to_be32(ueid_inbuf, cmd_len); + ret = memcpy_s(ueid_inbuf->body.id, sizeof(ueid_inbuf->body.id), + seid_ctx->seid, + sizeof(seid_ctx->seid)); //id not need endian conversion + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Update upi: memcpy failed, ret:%d.\n", ret); + ret = -EFAULT; + goto out; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_UPDATE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_EID, + "Update upi: failed to send add_ueid command, ret = %d.\n", + ret); + ret = -EFAULT; + } + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void udma_fill_add_seid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_add); + struct ub_cmd_seid_add *seid_inbuf = NULL; + + seid_inbuf = (struct ub_cmd_seid_add *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_ADD, cmd_len, 0); + seid_inbuf->body.idx = cfg->eid_index; + seid_inbuf->body.upi = cfg->upi; + seid_inbuf->body.vfid = vf_id; + + udma_cpu_to_be32(seid_inbuf, cmd_len); + (void)memcpy_s((void *)seid_inbuf->body.eid, + sizeof(seid_inbuf->body.eid), (void *)cfg->eid.raw, + sizeof(seid_inbuf->body.eid)); //eid not need endian conversion +} + +static int udma_cmd_seid_add(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_EID, "Invalid parameter. dev(%d) cfg(%d).\n", + !!dev, !!cfg); + return -ENXIO; + } + + udma_dev = to_udma_dev(dev); + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_add), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for add seid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_add_seid_cmd(dev, vf_id, cfg, cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_ADD, cqm_cmd_inbuf, NULL, + NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for add seid, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_ueid_add_npu_cmdq(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_EID, + "%s Invalid parameter: dev(%d), cfg(%d).\n", + __func__, !!dev, !!cfg); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + ret = udma_add_ueid_cmd(dev, vf_id, cfg, UB_UPI_NORM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Set ueid failed, ret = %d\n", ret); + return ret; + } + + ret = udma_cmd_seid_add(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Set seid failed, ret = %d\n", ret); + udma_delete_ueid_cmd(dev, vf_id, cfg); + return ret; + } + + return 0; +} + +static void udma_fill_del_seid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_del); + struct ub_cmd_seid_del *seid_inbuf = NULL; + + seid_inbuf = (struct ub_cmd_seid_del *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_DEL, cmd_len, 0); + seid_inbuf->body.idx = cfg->eid_index; + seid_inbuf->body.vfid = vf_id; + + udma_cpu_to_be32(seid_inbuf, cmd_len); +} + +static int udma_cmd_seid_delete(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_EID, + "%s Invalid parameter: dev(%d), cfg(%d).\n", + __func__, !!dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_EID, "%s to_udma_dev failed.\n", __func__); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_del), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for del seid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_del_seid_cmd(dev, vf_id, cfg, cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_DEL, cqm_cmd_inbuf, NULL, + NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for del seid, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int delete_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_EID, + "%s Invalid parameter: dev(%d), cfg(%d).\n", + __func__, !!dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + + ret = udma_delete_ueid_cmd(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, "Delete ueid failed: %d\n", + ret); + return ret; + } + + ret = udma_cmd_seid_delete(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, "Delete seid failed: %d\n", + ret); + return ret; + } + + return 0; +} + +static void udma_fill_query_seid_cmd(u16 vf_id, u32 idx, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_query); + + struct ub_cmd_seid_query *seid_inbuf = NULL; + seid_inbuf = (struct ub_cmd_seid_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_QUERY, cmd_len, idx); + seid_inbuf->body.vfid = vf_id; + seid_inbuf->body.idx = idx; + + udma_cpu_to_be32(seid_inbuf, cmd_len); +} + +int udma_cmd_seid_query_ctx_from_cache(struct udma_device *udma_dev, u16 vf_id, + u32 idx, struct ub_seid_ctx_query *ctx, + u32 len) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + int ret; + + if (len < sizeof(*ctx)) { + udma_err(udma_dev->dev, UDMA_M_EID, + "out len(%u) < ctx size(%u)\n", len, + (u32)(sizeof(*ctx))); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_EID, + "Start query seid, vfid: %u, idx: %u\n", vf_id, idx); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_query), &cqm_cmd_outbuf, + sizeof(struct ub_seid_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for del seid, ret = %d.\n", + ret); + return -ENOMEM; + } + + udma_fill_query_seid_cmd(vf_id, idx, cqm_cmd_inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_QUERY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for query seid, ret = %d.\n", + ret); + } else { + ret = memcpy_s((u8 *)ctx, sizeof(*ctx), + (u8 *)cqm_cmd_outbuf->buf, sizeof(*ctx)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "memcpy_s failed, ret = %d.\n", ret); + ret = -EFAULT; + } + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_seid_query_ignore_ub_disable_err(struct udma_device *udma_dev, + u16 vf_id, u32 idx, + ub_seid_ctx_info_s *ctx, u32 len) +{ + u16 cmd_len = sizeof(struct ub_cmd_seid_query); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_seid_query *seid_inbuf = NULL; + int ret; + + if (len < sizeof(*ctx)) { + udma_err(udma_dev->dev, UDMA_M_EID, + "seid query, out len(%u) < ctx size(%u)\n", len, + (u32)(sizeof(*ctx))); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_EID, + "Start query seid ignore ub disable, vfid: %u, idx: %u\n", + vf_id, idx); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_seid_query), &cqm_cmd_outbuf, + sizeof(ub_seid_ctx_info_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to alloc inbuf for seid query, ret = %d.\n", + ret); + return -ENOMEM; + } + + seid_inbuf = (struct ub_cmd_seid_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&seid_inbuf->header, UB_CMD_TYPE_EID, + UB_CMD_SEID_QUERY, cmd_len, idx); + seid_inbuf->body.vfid = vf_id; + seid_inbuf->body.idx = idx; + seid_inbuf->body.ignore_ub_disable_err = 1; + udma_cpu_to_be32(seid_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SEID_QUERY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "Failed to send command for seid query, ret = %d.\n", + ret); + } else { + ret = memcpy_s((u8 *)ctx, sizeof(*ctx), + (u8 *)cqm_cmd_outbuf->buf, sizeof(*ctx)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_EID, + "memcpy_s failed, ret = %d.\n", ret); + ret = -EFAULT; + } + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_cmd_upi_query(struct udma_device *udma_dev, + ub_cmd_upi_key_attr_s *upi_idx, + ub_cmd_upi_value_attr_s *upi_value) +{ + struct ub_cmd_upi_query *cmd_upi_inbuf = NULL; + ub_cmd_upi_value_attr_s *cmd_upi_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + u16 cmd_len = sizeof(struct ub_cmd_upi_query); + int i = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_upi_query), &cqm_cmd_outbuf, + (u16)sizeof(ub_cmd_upi_value_attr_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Alloc cmd_buf failed, err(%d)\n", ret); + return ret; + } + + cmd_upi_outbuf = (ub_cmd_upi_value_attr_s *)cqm_cmd_outbuf->buf; + cmd_upi_inbuf = (struct ub_cmd_upi_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_upi_inbuf->header, UB_CMD_TYPE_UPI, + UB_CMD_UPI_QUERY, sizeof(struct ub_cmd_upi_query), + 0); + cmd_upi_inbuf->body.dw0.value = upi_idx->dw0.value; + for (i = 0; i < UB_EID_LEN; i++) + cmd_upi_inbuf->body.id_dw[i] = upi_idx->id_dw[i]; + udma_cpu_to_be32(cmd_upi_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_UPI_QUERY, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + ret = -EINVAL; + goto out; + } + + ret = memcpy_s(upi_value, sizeof(ub_cmd_upi_value_attr_s), + cmd_upi_outbuf, sizeof(ub_cmd_upi_value_attr_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s failed, ret(%d)\n", ret); + ret = -EFAULT; + goto out; + } + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_config_function_migrate_state(struct ubcore_device *dev, u16 fe_idx, + u32 cnt, struct ubcore_ueid_cfg *cfg, + enum ubcore_mig_state state) +{ + int ret = -EINVAL; + int cfg_cnt = 0; + uint32_t i; + struct udma_device *udma_dev = to_udma_dev(dev); + + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_EID, + "config function migrate state invalid parameter. udma_dev(%d),cfg(%d) .\n", + !!udma_dev, !!cfg); + return -EINVAL; + } + + udma_info( + udma_dev->dev, UDMA_M_EID, + "config function migrate state, vfid: %u, cnt: %u, state %u\n", + fe_idx, cnt, state); + for (i = 0; i < cnt; i++) { + switch (state) { + case UBCORE_MIG_STATE_START: + ret = udma_add_ueid_cmd(dev, fe_idx, cfg, UB_UPI_MIGR); + break; + + case UBCORE_MIG_STATE_ROLLBACK: + ret = udma_add_ueid_cmd(dev, fe_idx, cfg, + UB_UPI_ROLLBACK); + break; + + case UBCORE_MIG_STATE_FINISH: + ret = udma_add_ueid_cmd(dev, fe_idx, cfg, UB_UPI_NORM); + break; + + default: + return -1; + } + + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_EID, + "config function migrate state failed, vfid: %u, cnt: %d, ret %d\n", + fe_idx, cfg_cnt, ret); + return -1; + } + + cfg_cnt++; + cfg++; + } + + return cfg_cnt; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h new file mode 100644 index 000000000..43e4e77e7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_eid.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_EID_H +#define UDMA_CMD_EID_H + +#include <linux/types.h> + +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_npu_upi_cmd_defs.h" +#include "ub_npu_eid_cmd_defs.h" + +#define UDMA_SEID_LOOP \ + 256 // this value need and ub_max_eid_tbl_cnt keep same as, used for traverse SEID + +int udma_cmd_seid_query_ctx_from_cache(struct udma_device *udma_dev, u16 vf_id, + u32 idx, struct ub_seid_ctx_query *ctx, + u32 len); +int udma_cmd_seid_query_ignore_ub_disable_err(struct udma_device *udma_dev, + u16 vf_id, u32 idx, + ub_seid_ctx_info_s *ctx, u32 len); +int udma_cmd_upi_query(struct udma_device *udma_dev, + ub_cmd_upi_key_attr_s *upi_idx, + ub_cmd_upi_value_attr_s *upi_value); +int udma_ueid_add_npu_cmdq(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); +int delete_ueid_cmd(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); +int udma_config_function_migrate_state(struct ubcore_device *dev, u16 fe_idx, + u32 cnt, struct ubcore_ueid_cfg *cfg, + enum ubcore_mig_state state); +int udma_cmd_update_upi(struct udma_device *udma_dev, u16 vfid, + ub_seid_ctx_info_s *seid_ctx, + ub_upi_mig_state_e mig_state); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c new file mode 100644 index 000000000..cc38eadbb --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.c @@ -0,0 +1,194 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/device.h> +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" +#include "ub_npu_event_cmd.h" +#include "ub_npu_event_cmd_defs.h" +#include "udma_cmd_event.h" + +static void udma_fill_async_event_credit_cmd( + u32 credit, struct ub_cmd_async_event_info *async_event_inbuf, + enum UB_CMD_ASYNC_EVENT_TYPE type) +{ + u16 cmd_len = sizeof(struct ub_cmd_async_event_info); + + udma_fill_cmd_header(&async_event_inbuf->header, + UB_CMD_TYPE_ASYNC_EVENT, type, cmd_len, 0); + async_event_inbuf->body.credit = credit; + + udma_cpu_to_be32(async_event_inbuf, cmd_len); +} + +int udma_cmd_async_event_credit_init(struct udma_device *udma_dev) +{ + struct ub_cmd_async_event_info *async_event_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_async_event_info), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to alloc inbuf for init_async_event_credit, ret = %d.\n", + ret); + return -ENOMEM; + } + + async_event_inbuf = + (struct ub_cmd_async_event_info *)cqm_cmd_inbuf->buf; + udma_fill_async_event_credit_cmd(UDMA_ASYNC_EVENT_CREDIT_MAX, + async_event_inbuf, + UB_CMD_ASYNC_EVENT_CREDIT_INIT); + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_CREDIT_INIT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to send init_async_event_credit command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +int udma_cmd_async_event_credit_return(struct udma_device *udma_dev, + u32 return_credit) +{ + struct ub_cmd_async_event_info *async_event_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_async_event_info), NULL, 0); + if (ret != 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "Failed to alloc inbuf for return_async_event_credit, ret = %d.\n", + ret); + return -ENOMEM; + } + + async_event_inbuf = + (struct ub_cmd_async_event_info *)cqm_cmd_inbuf->buf; + udma_fill_async_event_credit_cmd(return_credit, async_event_inbuf, + UB_CMD_ASYNC_EVENT_CREDIT_RETURN); + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_CREDIT_RETURN, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "Failed to send return_async_event_credit command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return 0; +} + +int udma_cmd_async_event_credit_query(struct udma_device *udma_dev, + u32 return_credit) +{ + struct ub_cmd_async_event_info *async_event_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_async_event_info), &cqm_cmd_outbuf, + (u16)(sizeof(int))); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to alloc inbuf for return_async_event_credit, ret = %d.\n", + ret); + return -ENOMEM; + } + + async_event_inbuf = + (struct ub_cmd_async_event_info *)cqm_cmd_inbuf->buf; + udma_fill_async_event_credit_cmd(return_credit, async_event_inbuf, + UB_CMD_ASYNC_EVENT_CREDIT_QUERY); + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_CREDIT_QUERY, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "Failed to send return_async_event_credit command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return 0; +} + +int udma_cmd_dfx_aeqe_inject(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_async_event_inject_info *aeqe_info_cmd = NULL; + u16 cmd_len = sizeof(struct ub_cmd_async_event_inject_info); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_async_event_inject_info), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + aeqe_info_cmd = + (struct ub_cmd_async_event_inject_info *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&aeqe_info_cmd->header, UB_CMD_TYPE_ASYNC_EVENT, + UB_CMD_ASYNC_EVENT_DFX_INJECT, cmd_len, 0); + + aeqe_info_cmd->body.xid = inbuf->xid; + aeqe_info_cmd->body.event_type = inbuf->event_type; + aeqe_info_cmd->body.queue_type = inbuf->queue_type; + aeqe_info_cmd->body.event_sub_type = inbuf->event_sub_type; + aeqe_info_cmd->body.vfid = inbuf->vfid; + + udma_cpu_to_be32(aeqe_info_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_ASYNC_EVENT_DFX_INJECT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to send aeqe inject command, ret = %d.\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + udma_info(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "send aeqe(%u, %u,%u) inject command success\n", inbuf->xid, + inbuf->event_type, inbuf->event_sub_type); + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h new file mode 100644 index 000000000..e5d5f0746 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_event.h @@ -0,0 +1,26 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_EVENT_H +#define UDMA_CMD_EVENT_H + +#include "udma_common.h" + +/* + * The depth of AEQ is 64k, set credit to 63k for the overmeasure. + * Because the credit return is faster than aqe consume idx, + * it will lead event dropped when aeq is full. So overmeasure set + * is needed. + */ +#define UDMA_ASYNC_EVENT_CREDIT_MAX 0xFC00 + +int udma_cmd_async_event_credit_return(struct udma_device *udma_dev, + u32 return_credit); +int udma_cmd_async_event_credit_init(struct udma_device *udma_dev); +int udma_cmd_async_event_credit_query(struct udma_device *udma_dev, + u32 return_credit); +int udma_cmd_dfx_aeqe_inject(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c new file mode 100644 index 000000000..c4bec3a04 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.c @@ -0,0 +1,1198 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_srq.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" +#include "ub_npu_jetty_cmd.h" +#include "udma_cmd_jetty.h" + +#define DATALEN_RATIO 2 // user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 // user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit +#define UB_RNR_TIMER_MASK 0x1f +#define UB_JETTY_BATCH_MODIFY_MAX_LEN UB_JETTY_CONTEXT_SIZE + +void udma_fill_cmd_jetty_chip_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + u8 pi_on_chip_en, u8 mtt_pro_en) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_sq_attr_s *sq_attr = &chip_attr->sq_attr; + ub_srq_attr_s *srq_attr = &chip_attr->srq_attr; + ub_rc_attr_s *rc_attr = &chip_attr->rc_attr; + u64 baseblk_cnt = 0; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(jetty->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(jetty->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = jetty->ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = jetty->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jetty->sdb.dma); + chip_attr->db_paddr_l = lower_32_bits(jetty->sdb.dma); + chip_attr->srq_en = 1; + + if (cfg->jfs_depth != 0) { + baseblk_cnt = roundup_pow_of_two(jetty->sq.baseblk_cnt); + sq_attr->sq_size = (u32)((u32)UB_ILOG2(baseblk_cnt)) & + UB_JETTY_SIZE_MASK; + } + + sq_attr->sq_wqebb_size = + ((u32)(JETTY_WQE_SHIFT - UB_JETTY_WQEBB_SIZE_SHIFT)) & + UB_JETTY_WQEBB_SIZE_MASK; + sq_attr->sq_page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, jetty->sq_buf_len); + if (cfg->send_jfc != NULL) + sq_attr->cqn = cfg->send_jfc->id; + sq_attr->sq_inline_en = 1; + sq_attr->sq_pi_on_chip = pi_on_chip_en; + + srq_attr->container_mode = jetty->ub_jfr->srq.container_mode; + srq_attr->srqn = jetty->ub_jfr->srq.srqn; + srq_attr->srq_size = (u32)UB_ILOG2(jetty->ub_jfr->srq.max_depth); + srq_attr->srq_wqebb_size = + jetty->ub_jfr->srq.wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->srq_page_size = udma_get_queue_kernel_page_size( + mtt_pro_en, jetty->ub_jfr->srq.buf_size); + if ((cfg->jfr != NULL) && (cfg->jfr->jfr_cfg.jfc != NULL)) + srq_attr->srq_cqn = cfg->jfr->jfr_cfg.jfc->id; + srq_attr->container_en = 1; + + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) { + rc_attr->rc_paddr_h = + (jetty->ubrc.dma_addr >> UB_RC_HI_PADDR_SHIFT) & + UB_RC_HI_PADDR_MASK; + rc_attr->rc_paddr_l = (u32)(jetty->ubrc.dma_addr >> + UB_RC_LOW_PADDR_SHIFT); // bit[39:8] + rc_attr->rc_size = depth_align_power2( + udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_depth); + rc_attr->ext_order = + jetty->ubrc.ext_order - UB_RC_EXT_ORDER_SHIFT; + } +} + +void udma_fill_cmd_jetty_sw_attr(struct udma_device *udma_dev, + struct ub_jetty_sw_attr *sw_attr, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + ub_jetty_com_attr_s *jetty_com_attr = &sw_attr->jetty_com_attr; + ub_jetty_sq_attr_s *jetty_sq_attr = &sw_attr->jetty_sq_attr; + ub_jetty_rq_attr_s *jetty_rq_attr = &sw_attr->jetty_rq_attr; + + jetty_com_attr->jetty_state = UBCORE_JETTY_STATE_READY; + jetty_com_attr->jetty_type = JETTY; + jetty_com_attr->priority = cfg->priority; + jetty_com_attr->db_cos = jetty->cos; + jetty_com_attr->transport_mode = + (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + ((cfg->trans_mode == UBCORE_TP_RM) ? + UB_TRANSPORT_MODE_RM : + UB_TRANSPORT_MODE_RSC); + jetty_com_attr->jetty_id = jetty->jetty_id; + jetty_com_attr->eid_index = cfg->eid_index; + jetty_com_attr->slice_size_shift = udma_dev->slice; + jetty_com_attr->slice_en = udma_get_jetty_slice_en(udma_dev); + if (jetty_com_attr->transport_mode == UB_TRANSPORT_MODE_RSC) + jetty_com_attr->hash = + (u32)atomic_inc_return(&udma_dev->rs_hash); + else + jetty_com_attr->hash = + (u32)atomic_inc_return(&udma_dev->jfs_hash); + + if (cfg->send_jfc != NULL) + jetty_sq_attr->tx_jfcn = cfg->send_jfc->id; + else + udma_warn(udma_dev->dev, UDMA_M_JETTY, + "cfg->send_jfc is null, tx_jfcn default 0.\n"); + jetty_sq_attr->ta_timeout = cfg->err_timeout; + jetty_sq_attr->rnr_max_retry_num = cfg->rnr_retry; + jetty_sq_attr->err_mode = UB_ERROR_SUSPEND; + + if ((cfg->jfr != NULL) && (cfg->jfr->jfr_cfg.jfc != NULL)) { + jetty_rq_attr->rx_jfcn = cfg->jfr->jfr_cfg.jfc->id; + jetty_rq_attr->jfc_inline = + udma_get_jfc_inline(udma_dev, cfg->jfr->jfr_cfg.jfc); + } else { + udma_warn(udma_dev->dev, UDMA_M_JETTY, + "recv_jfc is null, rx_jfcn default 0.\n"); + } + + jetty_rq_attr->rnr_timer = + jetty->ub_jfr->ubcore_jfr.jfr_cfg.min_rnr_timer & + UB_RNR_TIMER_MASK; + + udma_debug(udma_dev->dev, UDMA_M_JETTY, "jetty rnr_timer:%u.\n", + jetty_rq_attr->rnr_timer); + + jetty_rq_attr->jfrn = jetty->ub_jfr->jfr_id; + jetty_rq_attr->jfr_round = jetty->ub_jfr->round; + jetty_rq_attr->token_value = + jetty->ub_jfr->ubcore_jfr.jfr_cfg.token_value.token; + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "slice = %u, err_mode: %u, hash: %u\n", udma_dev->slice, + jetty_sq_attr->err_mode, jetty_com_attr->hash); +} + +void udma_fill_cmd_jetty_create_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct ub_cmd_jetty_create *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + u8 sq_pi_on_chip = 0; + + if (udma_dev->ub_cap.dev_cap.feature.bs.pi_on_chip_en != 0) + sq_pi_on_chip = + (u8)(jetty->jetty_id < + udma_dev->ub_cap.dev_cap.max_pi_on_chip_num); + + udma_fill_cmd_jetty_chip_attr(udma_dev, &cmd_buf->body.chip_attr, jetty, + cfg, sq_pi_on_chip, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + udma_fill_cmd_jetty_sw_attr(udma_dev, &cmd_buf->body.sw_attr, jetty, + cfg); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CREATE_CTX, cmd_len, jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_create_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_create), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_create_ctx(udma_dev, jetty, cfg, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CREATE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jetty->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + jetty->round = (u32)ntohl(*(u32 *)cqm_cmd_outbuf->buf); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "cmdq create jetty(%u) success and get round(0x%08x)\n", + jetty->jetty_id, jetty->round); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jetty_modify_ctx(struct udma_jetty *jetty, + enum ubcore_jetty_state state, + struct ub_cmd_jetty_modify *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.jetty_type = JETTY; + cmd_buf->body.mask.state = UDMA_MASK_VALID; + cmd_buf->body.old_state = jetty->state; + cmd_buf->body.new_state = state; + cmd_buf->body.sq_pi = jetty->sq.pi; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_MODIFY_CTX, cmd_len, jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_modify_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + enum ubcore_jetty_state state, + struct ubcore_udata *udata) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_modify *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_modify_ctx(jetty, state, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void +udma_fill_cmd_jetty_modify_dfx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct udma_jetty_modify_dfx *jetty_dfx, + struct ub_cmd_jetty_modify *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.jetty_type = JETTY; + cmd_buf->body.mask.next_send_tassn_flg = + jetty_dfx->mask.next_send_tassn; + cmd_buf->body.next_send_tassn = jetty_dfx->attr.next_send_tassn; + cmd_buf->body.mask.next_ack_tassn_flg = jetty_dfx->mask.next_ack_tassn; + cmd_buf->body.next_ack_tassn = jetty_dfx->attr.next_ack_tassn; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "%s: next_send_tassn: mask(0x%x), value(0x%x)\n", __func__, + jetty_dfx->mask.next_send_tassn, + jetty_dfx->attr.next_send_tassn); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "%s: next_ack_tassn: mask(0x%x), value(0x%x)\n", __func__, + jetty_dfx->mask.next_ack_tassn, + jetty_dfx->attr.next_ack_tassn); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_MODIFY_CTX, cmd_len, jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_modify_dfx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct udma_jetty_modify_dfx *jetty_dfx) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_modify *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_modify_dfx(udma_dev, jetty, jetty_dfx, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_delete_ctx(u32 jetty_id, + struct ub_cmd_jetty_delete *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_DELETE_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_delete *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_delete), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_delete_ctx(jetty->jetty_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_DELETE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jetty->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + else + ret = udma_cmd_check_dpath_dma(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + sizeof(u8), sizeof(u8)); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_fill_cmd_body(struct udma_device *udma_dev, + struct ub_cmd_jetty_cache_invld *cmd_buf, u32 buf_len, + u64 wb_dma) +{ + struct udma_comp_priv *comp_priv = NULL; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_COM, + "Failed to get hmm comp priv.\n"); + return -EINVAL; + } + + cmd_buf->body.sq_buf_len = ALIGN(buf_len, PAGE_SIZE); + cmd_buf->body.rq_buf_len = ALIGN(buf_len, PAGE_SIZE); + cmd_buf->body.syn_gpa_hi32 = upper_32_bits(wb_dma); + cmd_buf->body.syn_gpa_lo32 = lower_32_bits(wb_dma); + cmd_buf->body.mtt_cache_line_start = comp_priv->ub_cap.cmtt_cl_start; + cmd_buf->body.mtt_cache_line_end = comp_priv->ub_cap.cmtt_cl_end; + cmd_buf->body.mtt_cache_line_size = comp_priv->ub_cap.cmtt_cl_sz; + cmd_buf->body.mtt_flags = 0; + cmd_buf->body.mtt_num = 0; + cmd_buf->body.wqe_cache_line_start = comp_priv->ub_cap.wqe_cl_start; + cmd_buf->body.wqe_cache_line_end = comp_priv->ub_cap.wqe_cl_end; + cmd_buf->body.wqe_cache_line_size = comp_priv->ub_cap.wqe_cl_sz; + cmd_buf->body.wqe_flags = 0; + cmd_buf->body.wqe_num = 0; + + return 0; +} + +static int +udma_fill_cmd_jetty_cache_invld_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u64 wb_dma, + struct ub_cmd_jetty_cache_invld *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + int ret; + + ret = udma_fill_cmd_body(udma_dev, cmd_buf, jetty->sq_buf_len, wb_dma); + if (ret != 0) { + return ret; + } + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CACHE_INVLD_CTX, cmd_len, + jetty->jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + return 0; +} + +int udma_send_cache_invld_ctx(struct udma_device *udma_dev, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u32 *times, + u8 cmd_subtype, struct cmdq_rtt *cmdq_rtt) +{ + u64 jiffies_now; + int ret = 0; + u64 end; + + jiffies_now = get_jiffies_64(); + end = jiffies_now + (UB_DESTROY_FLOW_TIMEOUT * HZ); + while (time_before64(jiffies_now, end)) { + (*times)++; + ret = udma_send_cmd_box(udma_dev, cmd_subtype, cqm_cmd_inbuf, + NULL, NULL, cmdq_rtt); + if (ret <= 0) + break; + + cond_resched(); + jiffies_now = get_jiffies_64(); + } + + return ret; +} + +int udma_cmd_jetty_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u64 wb_dma) +{ + struct ub_cmd_jetty_cache_invld *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 times = 0; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + ret = udma_fill_cmd_jetty_cache_invld_ctx(udma_dev, jetty, wb_dma, + cmd_buf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed fill jetty cacheout inbuf, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + ret = udma_send_cache_invld_ctx(udma_dev, cqm_cmd_inbuf, &times, + UB_CMD_JETTY_CACHE_INVLD_CTX, + &jetty->cmdq_rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box after try times(%u), jetty(%u), ret(%d)\n", + times, jetty->jetty_id, ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_query_ctx(u32 jetty_id, + struct ub_cmd_jetty_query *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_ctx(struct udma_device *udma_dev, u32 jetty_id, + struct cmdq_rtt *rtt, + struct ub_ta_context_query *outbuf) +{ + struct ub_cmd_jetty_query *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_ctx(jetty_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cmd QUERY_JETTY, ret=%d", ret); + ret = -EIO; + goto out; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_ta_context_query), + cqm_cmd_outbuf->buf, sizeof(struct ub_ta_context_query)); + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void +udma_fill_cmd_jetty_query_rdma_rc(u32 jetty_id, u32 entry_index, + struct ub_cmd_jetty_query_rdma_rc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.entry_index = entry_index; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_RDMA_RC, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_rdma_rc(struct udma_device *udma_dev, u32 jetty_id, + u32 entry_index, + struct ub_ta_rdma_rc_outbuf *outbuf) +{ + struct ub_cmd_jetty_query_rdma_rc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + u32 entry_size = udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_rdma_rc), &cqm_cmd_outbuf, + sizeof(struct ub_ta_rdma_rc_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_rdma_rc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_rdma_rc(jetty_id, entry_index, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_RDMA_RC, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, jetty_id:%u, ret = %d.\n", + jetty_id, ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf->rdmarc, entry_size, + (void *)cqm_cmd_outbuf->buf, entry_size); + outbuf->rc_table_size = entry_size; + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void +udma_fill_cmd_jetty_query_ta_msn_entry(u32 xid, u32 entry_index, u8 msn_type, + struct ub_cmd_jetty_query_tamsn *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.index = entry_index; + cmd_buf->body.type = msn_type; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_TAMSN_ENTRY, cmd_len, xid); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_ta_msn_entry(struct udma_device *udma_dev, u32 xid, + u32 entry_index, u8 msn_type, + struct ub_msn_entry_outbuf *outbuf) +{ + struct ub_cmd_jetty_query_tamsn *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_tamsn), &cqm_cmd_outbuf, + sizeof(struct ub_msn_entry_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_tamsn *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_ta_msn_entry(xid, entry_index, msn_type, + cmd_buf); + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_TAMSN_ENTRY, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, xid:%u, ret = %d.\n", xid, + ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf->msn_entry, UB_MSN_ENTRY_SIZE, + (void *)cqm_cmd_outbuf->buf, UB_MSN_ENTRY_SIZE); + outbuf->msn_entry_size = UB_MSN_ENTRY_SIZE; + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jetty_clear_dfx_lock_ctx( + u32 jetty_id, struct ub_cmd_jetty_clear_dfx_lock *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CLEAR_DFX_LOCK_CTX, cmd_len, + jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_clear_dfx_lock_ctx(struct udma_device *udma_dev, + u32 jetty_id) +{ + struct ub_cmd_jetty_clear_dfx_lock *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_clear_dfx_lock), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_clear_dfx_lock *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_clear_dfx_lock_ctx(jetty_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CLEAR_DFX_LOCK_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "failed to send jetty clear dfx lock ctx command, ret = %d.\n", + ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_batch_modify_ctx(u32 jetty_id, u32 offset, u32 len, + const u8 *data, + struct ub_cmd_jetty_batch_modify *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + u32 i; + + cmd_buf->body.offset = offset; + cmd_buf->body.length = len; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_BATCH_MODIFY, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + for (i = 0; i < len; ++i) + cmd_buf->body.ctx[i + offset] = + (data[i * DATALEN_RATIO] << DATA_SHIFT) + + data[i * DATALEN_RATIO + 1]; +} + +int udma_cmd_jetty_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_jetty_batch_modify *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (inbuf->length > UB_JETTY_BATCH_MODIFY_MAX_LEN || + inbuf->offset > UB_JETTY_BATCH_MODIFY_MAX_LEN - inbuf->length) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "offset(%u) + length(%u) exceed max len(%u)\n", + inbuf->offset, inbuf->length, + UB_JETTY_BATCH_MODIFY_MAX_LEN); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_batch_modify_ctx( + inbuf->xid, inbuf->offset, inbuf->length, inbuf->data, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_BATCH_MODIFY, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_warn(udma_dev->dev, UDMA_M_JETTY, + "can't send jetty batch modify command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_grp_create_ctx(enum ubcore_jetty_grp_policy policy, + struct udma_jetty_grp *jetty_grp, + struct ub_cmd_jetty_grp_create *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.policy = policy; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_CREATE_CTX, cmd_len, + jetty_grp->jetty_grp_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_create_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp, + struct ubcore_jetty_grp_cfg *cfg) +{ + struct ub_cmd_jetty_grp_create *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_create), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_grp_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_create_ctx(cfg->policy, jetty_grp, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_CREATE_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send jetty grp create command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_grp_update_ctx(enum UB_JETTY_GRP_UPDATE_TYPE_E type, + u32 jetty_id, u32 jetty_grp_id, + struct ub_cmd_jetty_grp_update *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.update_type = type; + cmd_buf->body.jetty_id = jetty_id; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_UPDATE_CTX, cmd_len, + jetty_grp_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_update_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 jetty_grp_id, + enum UB_JETTY_GRP_UPDATE_TYPE_E type) +{ + struct ub_cmd_jetty_grp_update *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_update), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate mtt, ret:%d\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_grp_update *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_update_ctx(type, jetty->jetty_id, jetty_grp_id, + cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_UPDATE_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void +udma_fill_cmd_jetty_grp_delete_ctx(u32 jetty_grp_id, + struct ub_cmd_jetty_grp_delete *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_DELETE_CTX, cmd_len, + jetty_grp_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp) +{ + struct ub_cmd_jetty_grp_delete *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_delete), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_grp_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_delete_ctx(jetty_grp->jetty_grp_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_DELETE_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to send jetty grp delete command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_jetty_grp_query_ctx(u32 jetty_group_id, + struct ub_cmd_jetty_grp_query *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_GRP_QUERY_CTX, cmd_len, + jetty_group_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_grp_query_ctx(struct udma_device *udma_dev, + u32 jetty_group_id, + struct ub_jetty_grp_ctx_query *outbuf) +{ + struct ub_cmd_jetty_grp_query *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_grp_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_jetty_grp_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to alloc cqm_cmd_inoutbuf, ret=%d", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_grp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_grp_query_ctx(jetty_group_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_GRP_QUERY_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Failed to send cmd QUERY_JETTY_GROUP_CONTEXT, ret = %d.\n", + ret); + ret = -EIO; + goto out; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_jetty_grp_ctx_query), + cqm_cmd_outbuf->buf, + sizeof(struct ub_jetty_grp_ctx_query)); + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void udma_fill_cmd_jetty_query_rdma_rc_ext( + u32 jetty_id, u32 idx, struct ub_cmd_jetty_query_rdma_rc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.idx = idx; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_RDMA_RC_EXT, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_rdma_rc_ext(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, + struct ub_ta_rdma_rc_ext_outbuf *outbuf) +{ + struct ub_cmd_jetty_query_rdma_rc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 rc_ext_table_size; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_rdma_rc), &cqm_cmd_outbuf, + sizeof(struct ub_ta_rdma_rc_ext_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_rdma_rc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_rdma_rc_ext(jetty_id, idx, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_RDMA_RC_EXT, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, jetty_id:%u, idx:%u, ret = %d.\n", + jetty_id, idx, ret); + goto err_send_cmd; + } + + rc_ext_table_size = udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100 ? + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1825 : + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1823; + (void)memcpy_s((void *)outbuf->rdmarc_ext, rc_ext_table_size, + (void *)cqm_cmd_outbuf->buf, rc_ext_table_size); + outbuf->rc_ext_table_size = rc_ext_table_size; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "query ta rdma rc ext success, jetty_id:%u, idx:%u\n", + jetty_id, idx); + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_jetty_query_driver_ctr(struct udma_device *udma_dev, u32 jetty_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + struct mutex *lock = &udma_dev->device_res->jetty_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jetty, &udma_dev->device_res->jetty_list.node, + node) { + if (ub_jetty->jetty_id == jetty_id) { + mutex_unlock(lock); + outbuf->post_send_wr_num = (u64)atomic64_read( + &ub_jetty->sq.udma_driver_sq_ctr + .post_send_wr_num); + outbuf->post_recv_wr_num = (u64)atomic64_read( + &ub_jetty->ub_jfr->srq.udma_driver_rq_ctr + .post_recv_wr_num); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JETTY, + "query driver ctr fail, jetty_id:%u\n", jetty_id); + return -EINVAL; +} + +static void udma_fill_cmd_jetty_bind_ctx(u32 jetty_id, u32 tjetty_id, + struct ub_cmd_jetty_bind *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.peer_jetty_id = tjetty_id; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_BIND_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_bind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 tjetty_id) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_bind *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_bind), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_bind *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_bind_ctx(jetty->ubcore_jetty.jetty_id.id, tjetty_id, + cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_BIND_CTX, cqm_cmd_inbuf, + NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + ret = -EFAULT; + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_unbind_ctx(u32 jetty_id, + struct ub_cmd_jetty_unbind *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_UNBIND_CTX, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_unbind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_unbind *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_unbind), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_unbind *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_unbind_ctx(jetty->ubcore_jetty.jetty_id.id, + cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_UNBIND_CTX, + cqm_cmd_inbuf, NULL, NULL, &jetty->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + ret = -EFAULT; + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jetty_query_ta_oor_info( + u32 jetty_id, u32 idx, struct ub_cmd_jetty_query_rdma_rc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + + cmd_buf->body.idx = idx; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_QUERY_TA_OOR_INFO, cmd_len, jetty_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jetty_query_ta_oor_info(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, struct ub_oor_ta_info *outbuf) +{ + struct ub_cmd_jetty_query_rdma_rc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_query_rdma_rc), &cqm_cmd_outbuf, + sizeof(struct ub_oor_ta_info)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_query_rdma_rc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jetty_query_ta_oor_info(jetty_id, idx, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_QUERY_TA_OOR_INFO, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "failed to send cqm cmd, jetty_id:%u, idx:%u, ret = %d.\n", + jetty_id, idx, ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf, sizeof(struct ub_oor_ta_info), + (void *)cqm_cmd_outbuf->buf, + sizeof(struct ub_oor_ta_info)); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "query ta oor info success, jetty_id:%u, idx:%u\n", jetty_id, + idx); + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h new file mode 100644 index 000000000..517fecd0b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jetty.h @@ -0,0 +1,69 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JETTY_H +#define UDMA_CMD_JETTY_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "ub_pub_cmd.h" + +int udma_cmd_jetty_create_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg); +int udma_cmd_jetty_modify_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + enum ubcore_jetty_state state, + struct ubcore_udata *udata); +int udma_cmd_jetty_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty); +int udma_cmd_jetty_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u64 wb_dma); +int udma_cmd_jetty_bind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 tjetty_id); +int udma_cmd_jetty_unbind_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty); +int udma_cmd_jetty_query_ctx(struct udma_device *udma_dev, u32 jetty_id, + struct cmdq_rtt *rtt, + struct ub_ta_context_query *outbuf); +int udma_cmd_jetty_query_rdma_rc(struct udma_device *udma_dev, u32 jetty_id, + u32 entry_index, + struct ub_ta_rdma_rc_outbuf *outbuf); +int udma_cmd_jetty_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_jetty_clear_dfx_lock_ctx(struct udma_device *udma_dev, + u32 jetty_id); +int udma_cmd_jetty_query_ta_oor_info(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, struct ub_oor_ta_info *outbuf); + +int udma_cmd_jetty_grp_create_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp, + struct ubcore_jetty_grp_cfg *cfg); +int udma_cmd_jetty_grp_update_ctx(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 jetty_grp_id, + enum UB_JETTY_GRP_UPDATE_TYPE_E type); +int udma_cmd_jetty_grp_delete_ctx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp); +int udma_cmd_jetty_grp_query_ctx(struct udma_device *udma_dev, + u32 jetty_group_id, + struct ub_jetty_grp_ctx_query *outbuf); +int udma_cmd_jetty_query_rdma_rc_ext(struct udma_device *udma_dev, u32 jetty_id, + u32 idx, + struct ub_ta_rdma_rc_ext_outbuf *outbuf); +int udma_send_cache_invld_ctx(struct udma_device *udma_dev, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf, u32 *times, + u8 cmd_subtype, struct cmdq_rtt *cmdq_rtt); +int udma_fill_cmd_body(struct udma_device *udma_dev, + struct ub_cmd_jetty_cache_invld *cmd_buf, u32 buf_len, + u64 wb_dma); +int udma_cmd_jetty_query_driver_ctr(struct udma_device *udma_dev, u32 jetty_id, + ub_driver_comm_ctr_t *outbuf); +int udma_cmd_jetty_query_ta_msn_entry(struct udma_device *udma_dev, u32 xid, + u32 entry_index, u8 msn_type, + struct ub_msn_entry_outbuf *outbuf); +int udma_cmd_jetty_modify_dfx(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct udma_jetty_modify_dfx *jetty_dfx); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c new file mode 100644 index 000000000..0cfc27af9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.c @@ -0,0 +1,334 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_jetty.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jfc.h" + +#define DATALEN_RATIO 2 //user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 //user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +static u16 get_cmdq_len[] = { + [UB_CMD_JFC_CREATE_CTX] = (u16)sizeof(struct ub_cmd_jfc_create), + [UB_CMD_JFC_MODIFY_CTX] = (u16)sizeof(struct ub_cmd_jfc_modify), + [UB_CMD_JFC_DELETE_CTX] = (u16)sizeof(struct ub_cmd_jfc_delete), + [UB_CMD_JFC_CACHE_INVLD_CTX] = (u16)sizeof(struct ub_cmd_jfc_invld), + + [UB_CMD_JFC_MAX] = 0, +}; + +static void udma_fill_cmd_jfc_chip_attr(struct ub_jfc_chip_attr *chip_attr, + struct udma_jfc *jfc, u8 mtt_pro_en) +{ + ub_mtt_attr_s *cq_mtt_attr = &chip_attr->cq_mtt_attr; + + cq_mtt_attr->mtt_paddr_h = upper_32_bits(jfc->ub_mtt.mtt_paddr); + cq_mtt_attr->mtt_paddr_l = lower_32_bits(jfc->ub_mtt.mtt_paddr); + cq_mtt_attr->mtt_layers = jfc->ub_mtt.mtt_layers; + cq_mtt_attr->mtt_page_shift = jfc->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jfc->db.dma); + chip_attr->db_paddr_l = lower_32_bits(jfc->db.dma); + + chip_attr->cq_size = (u32)ilog2(jfc->base.jfc_cfg.depth); + chip_attr->cqe_size = UB_JFC_CQE_SIZE; + chip_attr->cq_page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, jfc->cq_buf_size); + + chip_attr->ceqn = jfc->base.jfc_cfg.ceqn; + + chip_attr->cnt_adjust_en = 1; + chip_attr->cnt_clear_en = 1; + chip_attr->ceqe_en = 1; +} + +static void udma_fill_cmd_jfc_sw_attr(struct ub_jfc_sw_attr *sw_attr, + struct udma_jfc *jfc) +{ + sw_attr->jfc_inline_en = jfc->base.jfc_cfg.flag.bs.jfc_inline; +} + +static void udma_fill_cmd_jfc_create_ctx(struct udma_jfc *jfc, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf, + u8 mtt_pro_en) +{ + struct ub_cmd_jfc_create *cmd_buf = + (struct ub_cmd_jfc_create *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jfc_create); + + udma_fill_cmd_jfc_chip_attr(&cmd_buf->body.chip_attr, jfc, mtt_pro_en); + + udma_fill_cmd_jfc_sw_attr(&cmd_buf->body.sw_attr, jfc); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_CREATE_CTX, cmd_len, jfc->base.id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_modify_ctx(u32 jfc_id, + const struct ubcore_jfc_attr *attr, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jfc_modify *cmd_buf = + (struct ub_cmd_jfc_modify *)cqm_cmd_inbuf->buf; + struct ub_cmd_body_jfc_modify *body = &cmd_buf->body; + u16 cmd_len = sizeof(struct ub_cmd_jfc_modify); + + body->mask = attr->mask; + body->max_cnt = attr->moderate_count; + body->timeout = attr->moderate_period; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_MODIFY_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_delete_ctx(u32 jfc_id, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jfc_delete *cmd_buf = + (struct ub_cmd_jfc_delete *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jfc_delete); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_DELETE_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_invld_ctx(u32 jfc_id, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jfc_invld *cmd_buf = + (struct ub_cmd_jfc_invld *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jfc_invld); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_CACHE_INVLD_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_jfc_query_ctx(u32 jfc_id, + struct ub_cmd_jfc_query *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jfc_query); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_QUERY_CTX, cmd_len, jfc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfc_ctx(struct udma_device *udma_dev, struct udma_jfc *jfc, + const struct ubcore_jfc_attr *attr, + enum UB_CMD_JFC_TYPE type) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + get_cmdq_len[type], &cqm_cmd_outbuf, + sizeof(u32) * UDMA_DPATH_MAX_OQ_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_cqm_alloc; + } + + switch (type) { + case UB_CMD_JFC_CREATE_CTX: + udma_fill_cmd_jfc_create_ctx( + jfc, cqm_cmd_inbuf, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + break; + + case UB_CMD_JFC_MODIFY_CTX: + udma_fill_cmd_jfc_modify_ctx(jfc->base.id, attr, cqm_cmd_inbuf); + break; + + case UB_CMD_JFC_DELETE_CTX: + udma_fill_cmd_jfc_delete_ctx(jfc->base.id, cqm_cmd_inbuf); + break; + + case UB_CMD_JFC_CACHE_INVLD_CTX: + udma_fill_cmd_jfc_invld_ctx(jfc->base.id, cqm_cmd_inbuf); + break; + + default: + ret = -EINVAL; + udma_err(udma_dev->dev, UDMA_M_JFC, + "invalid jfc cmd type(%u)\n", type); + goto err_cqm_send; + } + + ret = udma_send_cmd_box(udma_dev, type, cqm_cmd_inbuf, cqm_cmd_outbuf, + &cmdq_cqe, &jfc->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to send cqm cmd, ret(%d)\n", ret); + ret = -EIO; + goto err_cqm_send; + } + + /* 删 JFC when need according to actual return of oq num flush data generalpath */ + if (type == UB_CMD_JFC_DELETE_CTX && + udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + ret = udma_dpath_drain_handle(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + cmdq_cqe.bs.dpath_oq_num); + } + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "sub_type(%u), cmdq_cqe.bs.dpath_oq_num(%u), cmdq_cqe.value(0x%llx), ret:%d\n", + type, cmdq_cqe.bs.dpath_oq_num, cmdq_cqe.value, ret); + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + +err_cqm_alloc: + return ret; +} + +int udma_cmd_jfc_query_ctx(struct udma_device *udma_dev, u32 jfc_id, + struct ub_jfc_ctx_query *jfc_ctx) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jfc_query *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jfc_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_jfc_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_cqm_alloc; + } + + cmd_buf = (struct ub_cmd_jfc_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfc_query_ctx(jfc_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFC_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(jfc_ctx, sizeof(struct ub_jfc_ctx_query), + cqm_cmd_outbuf->buf, sizeof(struct ub_jfc_ctx_query)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + +err_cqm_alloc: + return ret; +} + +static void +udma_fill_cmd_jfc_batch_modify_ctx(struct ub_cmd_jfc_batch_modify *modify_cmd, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jfc_batch_modify); + u32 i; + + modify_cmd->body.offset = cpu_to_be32(inbuf->offset); + modify_cmd->body.length = cpu_to_be32(inbuf->length); + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; i < inbuf->offset + inbuf->length && + i < UB_CQM_MODIFY_DATA_LEN_JFC / DATALEN_RATIO; + ++i) { + *(((u8 *)&modify_cmd->body.ctx) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_JFC, + UB_CMD_JFC_BATCH_MODIFY_CTX, cmd_len, inbuf->xid); + /* here only convert header endian */ + udma_cpu_to_be32(modify_cmd, sizeof(ub_cmd_com_header_s)); +} + +int udma_cmd_jfc_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_jfc_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfc_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "cqm inbuf allocation failed\n"); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_jfc_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfc_batch_modify_ctx(modify_cmd, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFC_BATCH_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "send cmd failed, ret(%d)\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "send cmd success, jfc_id(%u)\n", inbuf->xid); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfc_query_driver_ctr(struct udma_device *udma_dev, u32 jfc_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct udma_jfc *ub_jfc = NULL; + struct mutex *lock = &udma_dev->device_res->jfc_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jfc, &udma_dev->device_res->jfc_list.node, + node) { + if (ub_jfc->base.id == jfc_id) { + udma_jfc_get(ub_jfc); + mutex_unlock(lock); + outbuf->poll_total_num = (u64)atomic64_read( + &ub_jfc->udma_driver_jfc_ctr.poll_total_num); + outbuf->poll_sq_num = (u64)atomic64_read( + &ub_jfc->udma_driver_jfc_ctr.poll_sq_num); + outbuf->poll_rq_num = (u64)atomic64_read( + &ub_jfc->udma_driver_jfc_ctr.poll_rq_num); + udma_jfc_put(ub_jfc); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JFC, + "query driver ctr fail, jfc_id:%u\n", jfc_id); + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h new file mode 100644 index 000000000..b45bcb046 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfc.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFC_H +#define UDMA_CMD_JFC_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_jfc.h" +#include "ub_pub_cmd.h" +#include "ub_npu_jfc_cmd.h" + +int udma_cmd_jfc_ctx(struct udma_device *udma_dev, struct udma_jfc *jfc, + const struct ubcore_jfc_attr *attr, + enum UB_CMD_JFC_TYPE type); +int udma_cmd_jfc_query_ctx(struct udma_device *udma_dev, u32 jfc_id, + struct ub_jfc_ctx_query *jfc_ctx); +int udma_cmd_jfc_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_jfc_query_driver_ctr(struct udma_device *udma_dev, u32 jfc_id, + ub_driver_comm_ctr_t *outbuf); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c new file mode 100644 index 000000000..de492b427 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.c @@ -0,0 +1,422 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_srq.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "ub_npu_jfr_cmd.h" +#include "udma_cmd_jfr.h" + +static inline u32 to_jetty_misc_state(enum ubcore_jfr_state state) +{ + switch (state) { + case UBCORE_JFR_STATE_RESET: + return UB_JETTY_STATE_RESET; + case UBCORE_JFR_STATE_READY: + return UB_JETTY_STATE_READY; + case UBCORE_JFR_STATE_ERROR: + return UB_JETTY_STATE_ERROR; + default: + return UB_JETTY_STATE_RESERVED; + } +} + +static void udma_fill_cmd_jfr_chip_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_srq_attr_s *srq_attr = &chip_attr->srq_attr; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(jfr->srq.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(jfr->srq.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = jfr->srq.ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = jfr->srq.ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jfr->sdb.dma); + chip_attr->db_paddr_l = lower_32_bits(jfr->sdb.dma); + chip_attr->srq_en = 1; + + srq_attr->container_mode = jfr->srq.container_mode; + srq_attr->srqn = jfr->srq.srqn; + srq_attr->srq_size = (u32)ilog2(jfr->srq.max_depth); + srq_attr->srq_wqebb_size = + jfr->srq.wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + srq_attr->srq_page_size = udma_get_queue_kernel_page_size( + udma_dev->ub_cap.dev_cap.mtt_pro_en, jfr->srq.buf_size); + if (cfg->jfc != NULL) + srq_attr->srq_cqn = cfg->jfc->id; + else + udma_warn(udma_dev->dev, UDMA_M_JFR, + "cfg->recv_jfc is null, srq_cqn default 0.\n"); + srq_attr->container_en = 1; +} + +static void udma_fill_cmd_jfr_sw_attr(struct udma_device *udma_dev, + struct ub_jetty_sw_attr *sw_attr, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + ub_jetty_com_attr_s *jetty_com_attr = &sw_attr->jetty_com_attr; + ub_jetty_rq_attr_s *jetty_rq_attr = &sw_attr->jetty_rq_attr; + + jetty_com_attr->jetty_state = UBCORE_JETTY_STATE_READY; + jetty_com_attr->jetty_type = JFR; + jetty_com_attr->transport_mode = + (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + ((cfg->trans_mode == UBCORE_TP_RM) ? + UB_TRANSPORT_MODE_RM : + UB_TRANSPORT_MODE_RSC); + jetty_com_attr->jetty_id = jfr->jfr_id; + jetty_com_attr->eid_index = cfg->eid_index; + jetty_com_attr->hash = (u32)atomic_inc_return(&udma_dev->jfr_hash); + + if (cfg->jfc != NULL) { + jetty_rq_attr->rx_jfcn = cfg->jfc->id; + jetty_rq_attr->jfc_inline = + udma_get_jfc_inline(udma_dev, cfg->jfc); + } else { + udma_warn(udma_dev->dev, UDMA_M_JFR, + "cfg->recv_jfc is null, rx_jfcn default 0.\n"); + } + jetty_rq_attr->rnr_timer = cfg->min_rnr_timer & + 0x1f; //rnr_timer: 5bits + jetty_rq_attr->jfrn = jfr->jfr_id; + jetty_rq_attr->token_value = cfg->token_value.token; + jetty_rq_attr->jfr_round = jfr->round; +} + +static void udma_fill_cmd_jfr_index_queue_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_index_q_attr_s *index_q_attr = &chip_attr->index_q_attr; + __be64 first_page_addr; + + sq_mtt_attr->mtt_paddr_ext_h = + upper_32_bits(jfr->index_queue.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_ext_l = + lower_32_bits(jfr->index_queue.ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers_ext = jfr->index_queue.ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift_ext = + jfr->index_queue.ub_mtt.mtt_page_shift; + + index_q_attr->idx_queue_size = (u32)ilog2(jfr->index_queue.max_depth); + index_q_attr->idx_queue_pi_on_chip = 0; + index_q_attr->idx_queue_lth_pre_en = 1; + index_q_attr->idx_queue_lth_gap = + 1; /* PI - CI <= idx_queue_lth_gap + idx_queue_lth when update PI */ + index_q_attr->user_data0 = + 0; /* todo index_q_attr->user_data0 = cfg->user_data0 */ + index_q_attr->user_data1 = + 0; /* todo index_q_attr->user_data1 = cfg->user_data1 */ + if (sq_mtt_attr->mtt_layers_ext == 0) { + index_q_attr->first_page_addr_h = sq_mtt_attr->mtt_paddr_ext_h; + index_q_attr->first_page_addr_l = sq_mtt_attr->mtt_paddr_ext_l; + } else { + first_page_addr = *(jfr->index_queue.ub_mtt.mtt_vaddr); + index_q_attr->first_page_addr_h = + cpu_to_be32(lower_32_bits(first_page_addr)); + index_q_attr->first_page_addr_l = + cpu_to_be32(upper_32_bits(first_page_addr)); + } +} + +static void udma_fill_cmd_jfr_create_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg, + struct ub_cmd_jetty_create *cmd_buf, + udma_cqm_cmd_type_info_t cmd_info) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_create); + + udma_fill_cmd_jfr_chip_attr(udma_dev, &cmd_buf->body.chip_attr, jfr, + cfg); + + udma_fill_cmd_jfr_sw_attr(udma_dev, &cmd_buf->body.sw_attr, jfr, cfg); + + if (jfr->srq.index_queue_en) { + udma_fill_cmd_jfr_index_queue_attr( + udma_dev, &cmd_buf->body.chip_attr, jfr, cfg); + } + + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfr_create_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *cmd_buf = NULL; + udma_cqm_cmd_type_info_t cmd_info; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_CREATE_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_create), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfr_create_ctx(udma_dev, jfr, cfg, cmd_buf, cmd_info); + + ret = udma_send_cmd_box(udma_dev, cmd_info.cmd_subtype, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, &jfr->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "failed to send cqm cmd, ret(%d)\n", ret); + ret = -EIO; + goto err_cqm_send; + } + + jfr->round = (u32)ntohl(*(u32 *)cqm_cmd_outbuf->buf); + udma_debug(udma_dev->dev, UDMA_M_JFR, + "cmdq create jfr(%d) success and get round(0x%08x)\n", + jfr->jfr_id, jfr->round); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jfr_modify_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, + struct udma_jfr_modify_cfg *cfg, + struct ub_cmd_jetty_modify *cmd_buf, + udma_cqm_cmd_type_info_t cmd_info) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_modify); + + if (cfg->mask.state > 0) { + cmd_buf->body.jetty_type = JFR; + cmd_buf->body.old_state = to_jetty_misc_state(jfr->state); + cmd_buf->body.new_state = to_jetty_misc_state(cfg->state); + cmd_buf->body.mask.state = 1; + } + /* 1825 newly added jfr modify idx_wqe_cnt_lth function */ + if (cfg->mask.idx_queue_lth > 0) { + cmd_buf->body.idx_queue_lth = cfg->idx_queue_lth; + cmd_buf->body.idx_queue_lth_pre_en = 1; + cmd_buf->body.mask.idx_queue_lth = 1; + } + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfr_modify_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + struct udma_jfr_modify_cfg *cfg) +{ + struct ub_cmd_jetty_modify *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + udma_cqm_cmd_type_info_t cmd_info; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_MODIFY_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfr_modify_ctx(udma_dev, jfr, cfg, cmd_buf, cmd_info); + + ret = udma_send_cmd_box(udma_dev, cmd_info.cmd_subtype, cqm_cmd_inbuf, + NULL, NULL, &jfr->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfr_delete_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_delete); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_delete *cmd_buf = NULL; + udma_cqm_cmd_type_info_t cmd_info; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_DELETE_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_delete), &cqm_cmd_outbuf, + sizeof(u32) * UDMA_DPATH_MAX_OQ_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, cmd_info.cmd_subtype, cqm_cmd_inbuf, + cqm_cmd_outbuf, &cmdq_cqe, &jfr->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + else + ret = udma_dpath_drain_handle(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + cmdq_cqe.bs.dpath_oq_num); + + udma_debug( + udma_dev->dev, UDMA_M_JFR, + "cmdq_cqe.bs.dpath_oq_num(%u), cmdq_cqe.value(0x%llx), ret:%d\n", + cmdq_cqe.bs.dpath_oq_num, cmdq_cqe.value, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int +udma_fill_cmd_jfr_cache_invld_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, u64 wb_dma, + struct ub_cmd_jetty_cache_invld *cmd_buf, + udma_cqm_cmd_type_info_t cmd_info) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_cache_invld); + struct udma_comp_priv *comp_priv = NULL; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to get hmm comp priv.\n"); + return -EINVAL; + } + + cmd_buf->body.rq_buf_len = ALIGN(jfr->srq.buf_size, PAGE_SIZE); + cmd_buf->body.mtt_flags = 0; + cmd_buf->body.mtt_num = 0; + cmd_buf->body.mtt_cache_line_start = comp_priv->ub_cap.cmtt_cl_start; + cmd_buf->body.mtt_cache_line_end = comp_priv->ub_cap.cmtt_cl_end; + cmd_buf->body.mtt_cache_line_size = comp_priv->ub_cap.cmtt_cl_sz; + cmd_buf->body.syn_gpa_hi32 = upper_32_bits(wb_dma); + cmd_buf->body.syn_gpa_lo32 = lower_32_bits(wb_dma); + cmd_buf->body.wqe_flags = 0; + cmd_buf->body.wqe_num = 0; + cmd_buf->body.wqe_cache_line_start = comp_priv->ub_cap.wqe_cl_start; + cmd_buf->body.wqe_cache_line_end = comp_priv->ub_cap.wqe_cl_end; + cmd_buf->body.wqe_cache_line_size = comp_priv->ub_cap.wqe_cl_sz; + + udma_fill_cmd_header(&cmd_buf->header, cmd_info.cmd_type, + cmd_info.cmd_subtype, cmd_len, jfr->jfr_id); + udma_cpu_to_be32(cmd_buf, cmd_len); + + return 0; +} + +int udma_cmd_jfr_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, u64 wb_dma) +{ + struct ub_cmd_jetty_cache_invld *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 times = 0; + udma_cqm_cmd_type_info_t cmd_info; + int ret; + + cmd_info.cmd_type = UB_CMD_TYPE_JFR; + cmd_info.cmd_subtype = UB_CMD_JFR_CACHE_INVLD_CTX; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + ret = udma_fill_cmd_jfr_cache_invld_ctx(udma_dev, jfr, wb_dma, cmd_buf, + cmd_info); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed fill jfr cacheout inbuf, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + ret = udma_send_cache_invld_ctx(udma_dev, cqm_cmd_inbuf, &times, + cmd_info.cmd_subtype, &jfr->cmdq_rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "Failed to send cqm_cmd_box after try times(%u), jfr(%u), ret(%d)\n", + times, jfr->jfr_id, ret); + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfr_query_driver_ctr(struct udma_device *udma_dev, u32 jfr_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct mutex *lock = NULL; + struct udma_jfr *ub_jfr = NULL; + lock = &udma_dev->device_res->jfr_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jfr, &udma_dev->device_res->jfr_list.node, + node) { + if (ub_jfr->jfr_id == jfr_id) { + mutex_unlock(lock); + outbuf->post_recv_wr_num = (u64)atomic64_read( + &ub_jfr->srq.udma_driver_rq_ctr + .post_recv_wr_num); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JFR, + "query driver ctr fail, jfr_id:%u\n", jfr_id); + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h new file mode 100644 index 000000000..3876dafe0 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfr.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFR_H +#define UDMA_CMD_JFR_H + +#include "udma_common.h" +#include "udma_jetty.h" + +int udma_cmd_jfr_create_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg); +int udma_cmd_jfr_modify_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr, + struct udma_jfr_modify_cfg *cfg); +int udma_cmd_jfr_delete_ctx(struct udma_device *udma_dev, struct udma_jfr *jfr); +int udma_cmd_jfr_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfr *jfr, u64 wb_dma); +int udma_cmd_jfr_query_driver_ctr(struct udma_device *udma_dev, u32 jfr_id, + ub_driver_comm_ctr_t *outbuf); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c new file mode 100644 index 000000000..4d37b0594 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "ub_npu_jetty_cmd_defs.h" +#include "ub_npu_jetty_cmd.h" +#include "ub_npu_jfrc_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_jetty.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfrc.h" + +static void udma_fill_cmd_jfrc_sw_attr(struct udma_rc_queue *udma_rcq, + struct ub_jetty_sw_attr *sw_attr) +{ + sw_attr->jetty_com_attr.transport_mode = UB_TRANSPORT_MODE_RCQ; + sw_attr->jetty_com_attr.jetty_type = JFRC; +} + +static void udma_fill_cmd_jfrc_chip_attr(struct udma_rc_queue *udma_rcq, + struct ub_chip_attr *chip_attr) +{ +#define UDMA_MTT_PRO_EN 1 + + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_sq_attr_s *sq_attr = &chip_attr->sq_attr; + ub_rc_attr_s *rc_attr = &chip_attr->rc_attr; + u64 tmp = 0; + u64 rcq_depth = 0; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(udma_rcq->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(udma_rcq->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = udma_rcq->ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = udma_rcq->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = + upper_32_bits(udma_rcq->rc_queue->q_header_paddr); + chip_attr->db_paddr_l = + lower_32_bits(udma_rcq->rc_queue->q_header_paddr); + + if (udma_rcq->total_depth != 0) { + /* driver to rcq allocate of 2MB memory page, but agreed chip uses of 1MB(i.e. depth is 8k), so conversion is done here */ + rcq_depth = udma_rcq->total_depth >> 1; + + /* microcode sq_wqebb_size is 64B, so depth sent to chip needs * UB_PER_RCQE_WQEBB_NUM */ + tmp = roundup_pow_of_two(rcq_depth * UB_PER_RCQE_WQEBB_NUM); + sq_attr->sq_size = (u32)((u32)ilog2((u32)tmp)) & + UB_RC_QUEUE_DEPTH_MASK; + } + sq_attr->sq_wqebb_size = /* value is 2, wqebb_size equal to 2^sq_wqebb_size * 16 = 64 byte */ + ((u32)(UB_RC_QUEUE_WQE_SHIFT - UB_RC_QUEUE_WQEBB_SIZE_SHIFT)) & + UB_JETTY_WQEBB_SIZE_MASK; + sq_attr->sq_page_size = udma_get_queue_kernel_page_size( + UDMA_MTT_PRO_EN, udma_rcq->total_size); + sq_attr->sq_inline_en = 0; + sq_attr->sq_pi_on_chip = 0; + + rc_attr->rc_paddr_h = + (udma_rcq->ubrc.dma_addr >> UB_RC_HI_PADDR_SHIFT) & + UB_RC_HI_PADDR_MASK; + rc_attr->rc_paddr_l = (u32)(udma_rcq->ubrc.dma_addr >> + UB_RC_LOW_PADDR_SHIFT); // bit[39:8] + rc_attr->rc_size = UB_RC_TABLE_SIZE_128B; + rc_attr->ext_order = udma_rcq->ubrc.ext_order - UB_RC_EXT_ORDER_SHIFT; +} + +static void +udma_fill_cmd_jfrc_create_body(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq, + struct ub_cmd_body_jetty_create *body) +{ + udma_fill_cmd_jfrc_chip_attr(udma_rcq, &body->chip_attr); + udma_fill_cmd_jfrc_sw_attr(udma_rcq, &body->sw_attr); +} + +int udma_cmd_jfrc_create_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *rc_cmd = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jetty_create); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jetty_create), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_CREATE_CTX, cmd_len, + udma_rcq->rc_ctx->xid); + + udma_fill_cmd_jfrc_create_body(udma_dev, udma_rcq, &rc_cmd->body); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_CREATE_CTX, cqm_cmd_inbuf, + NULL, NULL, &udma_rcq->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm send cmd failed, ret(%d), xid(%u)\n", + __func__, ret, udma_rcq->rc_ctx->xid); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfrc_delete_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + struct ub_cmd_jfrc_delete *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_delete); + u32 *cmd_out_num = NULL; + u32 cmd_out_val = 0; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_delete), + &cqm_cmd_outbuf, + sizeof(u64) + sizeof(u64)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return ret; + } + + rc_cmd = (struct ub_cmd_jfrc_delete *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_DESTROY_CTX, cmd_len, + udma_rcq->rc_ctx->xid); + + rc_cmd->body.bs.dpath_flag = 1; + + udma_cpu_to_be32(rc_cmd, cmd_len); + cmd_out_num = (u32 *)cqm_cmd_outbuf->buf; + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_DESTROY_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &udma_rcq->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, ret = %d.\n", __func__, ret); + else + cmd_out_val = be32_to_cpu(*cmd_out_num); + if ((cmd_out_val & UB_DPATH_CHECK_FLAG) != 0) { + ret = udma_cmd_check_dpath_dma( + udma_dev, ((u8 *)cqm_cmd_outbuf->buf) + sizeof(u64), + sizeof(u8), sizeof(u8)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: check_dpath_dma failed\n", __func__); + } + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void +udma_fill_cmd_jfrc_modify_pcqc(u32 jfrc_id, u32 offset, u32 len, const u8 *data, + u32 data_len, + struct ub_cmd_jfrc_modify_pcqc *cmd_buf) +{ + u16 cmd_len = sizeof(*cmd_buf); + u32 i; + + cmd_buf->body.offset = offset; + cmd_buf->body.length = len; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_MODIFY_PCQC, cmd_len, jfrc_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + for (i = 0; i < len; ++i) { + if (i + offset < UB_PCQ_CTX_SIZE && + (i * DATALEN_RATIO + 1) < data_len) { + cmd_buf->body.ctx[i + offset] = + (data[i * DATALEN_RATIO] << DATA_SHIFT) + + data[i * DATALEN_RATIO + 1]; + } + } +} + +int udma_cmd_jfrc_modify_pcqc(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_jfrc_modify_pcqc *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jfrc_modify_pcqc), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "failed to alloc buffer, ret = %d.\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jfrc_modify_pcqc *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_jfrc_modify_pcqc(inbuf->xid, inbuf->offset, inbuf->length, + inbuf->data, inbuf->data_len, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_MODIFY_PCQC, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_warn(udma_dev->dev, UDMA_M_RC_TABLE, + "can't send jfrc modify pcqc command, ret = %d.\n", + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static int +udma_cmd_jfrc_cache_invalidate_fill(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq, u64 wb_dma, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct ub_cmd_jetty_cache_invld *rc_cmd = + (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + u16 cmd_len = sizeof(struct ub_cmd_jetty_cache_invld); + int ret; + + ret = udma_fill_cmd_body(udma_dev, rc_cmd, udma_rcq->total_size, + wb_dma); + if (ret != 0) { + return ret; + } + + /* reuse Jetty cache invalidate microcode */ + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CACHE_INVLD_CTX, cmd_len, + udma_rcq->rc_ctx->xid); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + return 0; +} + +static int +udma_rc_queue_send_cache_invalidate_cmd(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq, + u64 wb_dma) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + /* reuse jetty command */ + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: alloc cqm_cmd_inoutbuf failed, ret(%d)\n", + __func__, ret); + return -ENOMEM; + } + + ret = udma_cmd_jfrc_cache_invalidate_fill(udma_dev, udma_rcq, wb_dma, + cqm_cmd_inbuf); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "%s: fill rc queue cache invalidate inbuf failed, ret(%d)\n", + __func__, ret); + goto fill_inbuf_failed; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CACHE_INVLD_CTX, + cqm_cmd_inbuf, NULL, NULL, &udma_rcq->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cqm_cmd_box failed, ret(%d)\n", __func__, + ret); + +fill_inbuf_failed: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfrc_cache_invalidate(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + void *wb_va = NULL; + u64 wb_dma = 0; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: alloc wb_va failed\n", __func__); + return -ENOMEM; + } + + ret = udma_rc_queue_send_cache_invalidate_cmd(udma_dev, udma_rcq, + wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +int udma_cmd_jfrc_query_qpc_ctx(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf) +{ + struct ub_ta_context_query *query_outbuf = NULL; + struct ub_cmd_jfrc_query *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_query), + &cqm_cmd_outbuf, + sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jfrc_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_QUERY_QPC_CTX, cmd_len, rc_id); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_QUERY_QPC_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_ta_context_query *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, sizeof(struct ub_ta_context_query), + (void *)query_outbuf, + sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: memcpy_s failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, "%s success, rc_id(%u)\n", + __func__, rc_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_jfrc_query_pcqc_ctx(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf) +{ + ub_pcqc_query_rsp_s *query_outbuf = NULL; + struct ub_cmd_jfrc_query *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_query), + &cqm_cmd_outbuf, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jfrc_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_QUERY_PCQC_CTX, cmd_len, rc_id); + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_QUERY_PCQC_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + query_outbuf = (ub_pcqc_query_rsp_s *)cqm_cmd_outbuf->buf; + (void)memcpy_s((void *)outbuf, sizeof(ub_pcqc_query_rsp_s), + (void *)query_outbuf, sizeof(ub_pcqc_query_rsp_s)); + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, "%s success, rc_id(%u)\n", + __func__, rc_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_jfrc_query_wqe(struct udma_device *udma_dev, u32 rc_id, u32 ci, + struct ub_rc_queue_wqe *outbuf) +{ + struct ub_rc_queue_wqe *query_outbuf = NULL; + struct ub_cmd_jfrc_query *rc_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_jfrc_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_jfrc_query), + &cqm_cmd_outbuf, + sizeof(struct ub_rc_queue_wqe)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + rc_cmd = (struct ub_cmd_jfrc_query *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&rc_cmd->header, UB_CMD_TYPE_JFRC, + UB_CMD_JFRC_QUERY_WQE, cmd_len, rc_id); + + rc_cmd->body.ci = ci; + + udma_cpu_to_be32(rc_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JFRC_QUERY_WQE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "%s: send cmd failed, rc_buf_id(%u) ci(%u) ret(%d)\n", + __func__, rc_id, ci, ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_rc_queue_wqe *)cqm_cmd_outbuf->buf; + (void)memcpy_s((void *)outbuf, sizeof(struct ub_rc_queue_wqe), + (void *)query_outbuf, sizeof(struct ub_rc_queue_wqe)); + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "%s success, rc_id(%u) ci(%u)\n", __func__, rc_id, ci); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h new file mode 100644 index 000000000..d59e80050 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfrc.h @@ -0,0 +1,28 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFRC_H +#define UDMA_CMD_JFRC_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_npu_jfrc_cmd_defs.h" +#include "ub_pub_cmd.h" + +int udma_cmd_jfrc_create_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq); +int udma_cmd_jfrc_delete_ctx(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq); +int udma_cmd_jfrc_cache_invalidate(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq); +int udma_cmd_jfrc_query_qpc_ctx(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf); +int udma_cmd_jfrc_query_pcqc_ctx(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf); +int udma_cmd_jfrc_query_wqe(struct udma_device *udma_dev, u32 rc_id, u32 ci, + struct ub_rc_queue_wqe *outbuf); +int udma_cmd_jfrc_modify_pcqc(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c new file mode 100644 index 000000000..64116330c --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.c @@ -0,0 +1,341 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "ub_npu_jetty_cmd.h" +#include "udma_cmd_jfs.h" + +static void udma_fill_cmd_jfs_chip_attr(struct udma_device *udma_dev, + struct ub_chip_attr *chip_attr, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, + u8 pi_on_chip_en, u8 mtt_pro_en) +{ + ub_mtt_attr_s *sq_mtt_attr = &chip_attr->sq_mtt_attr; + ub_sq_attr_s *sq_attr = &chip_attr->sq_attr; + ub_rc_attr_s *rc_attr = &chip_attr->rc_attr; + + sq_mtt_attr->mtt_paddr_h = upper_32_bits(jfs->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_paddr_l = lower_32_bits(jfs->ub_mtt.mtt_paddr); + sq_mtt_attr->mtt_layers = jfs->ub_mtt.mtt_layers; + sq_mtt_attr->mtt_page_shift = jfs->ub_mtt.mtt_page_shift; + + chip_attr->db_paddr_h = upper_32_bits(jfs->sdb.dma); + chip_attr->db_paddr_l = lower_32_bits(jfs->sdb.dma); + + if (cfg->depth != 0) { + u64 tmp = (u64)roundup_pow_of_two(jfs->sq.baseblk_cnt); + sq_attr->sq_size = + (u32)((UB_ILOG2((u32)tmp)) & UB_JETTY_SIZE_MASK); + } + + sq_attr->sq_wqebb_size = + ((u32)(JETTY_WQE_SHIFT - UB_JETTY_WQEBB_SIZE_SHIFT)) & + UB_JETTY_WQEBB_SIZE_MASK; + sq_attr->sq_page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, jfs->sq_buf_len); + sq_attr->cqn = cfg->jfc->id; + sq_attr->sq_inline_en = 1; + sq_attr->sq_pi_on_chip = pi_on_chip_en; + + if (jfs->trans_mode == UBCORE_TP_RM) { + rc_attr->rc_paddr_h = (jfs->ubrc.dma_addr >> 40) & + 0xffffff; //high:24bit, 64-24= 40 + rc_attr->rc_paddr_l = + (u32)(jfs->ubrc.dma_addr >> 8); //bit[39:8] + rc_attr->rc_size = depth_align_power2( + udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_depth); + rc_attr->ext_order = + jfs->ubrc.ext_order - UB_RC_EXT_ORDER_SHIFT; + } +} + +static void udma_fill_cmd_jfs_sw_attr(struct udma_device *udma_dev, + struct ub_jetty_sw_attr *sw_attr, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + ub_jetty_com_attr_s *com_attr = &sw_attr->jetty_com_attr; + ub_jetty_sq_attr_s *sq_attr = &sw_attr->jetty_sq_attr; + + com_attr->jetty_state = UBCORE_JETTY_STATE_READY; + com_attr->jetty_type = JFS; + com_attr->priority = cfg->priority; + com_attr->db_cos = jfs->cos; + com_attr->transport_mode = (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + UB_TRANSPORT_MODE_RM; + com_attr->jetty_id = jfs->jfs_id; + com_attr->eid_index = cfg->eid_index; + com_attr->slice_size_shift = udma_dev->slice; + com_attr->slice_en = udma_get_jetty_slice_en(udma_dev); + com_attr->hash = (u32)atomic_inc_return(&udma_dev->jfs_hash); + + sq_attr->tx_jfcn = cfg->jfc->id; + sq_attr->ta_timeout = cfg->err_timeout; + sq_attr->rnr_max_retry_num = cfg->rnr_retry; + sq_attr->err_mode = UB_ERROR_SUSPEND; + + udma_info(udma_dev->dev, UDMA_M_JFS, + "slice = %u, err_mode: %u, hash: %u\n", udma_dev->slice, + sq_attr->err_mode, com_attr->hash); +} + +static void udma_fill_cmd_jfs_create_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, + struct ub_cmd_jetty_create *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_create); + u8 sq_pi_on_chip = 0; + if (udma_dev->ub_cap.dev_cap.feature.bs.pi_on_chip_en != 0) + sq_pi_on_chip = + (u8)(jfs->jfs_id < + udma_dev->ub_cap.dev_cap.max_pi_on_chip_num); + + udma_fill_cmd_jfs_chip_attr(udma_dev, &cmd_buf->body.chip_attr, jfs, + cfg, sq_pi_on_chip, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + udma_fill_cmd_jfs_sw_attr(udma_dev, &cmd_buf->body.sw_attr, jfs, cfg); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CREATE_CTX, cmd_len, jfs->jfs_id); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfs_create_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_create *cmd_buf = NULL; + + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_create), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_jetty_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfs_create_ctx(udma_dev, jfs, cfg, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_CREATE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jfs->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + jfs->round = (u32)ntohl(*(u32 *)cqm_cmd_outbuf->buf); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "cmdq create jfs(%d) success and get round(0x%08x)\n", + jfs->jfs_id, jfs->round); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void udma_fill_cmd_jfs_modify_ctx(struct udma_jfs *jfs, + enum ubcore_jetty_state state, + struct ub_cmd_jetty_modify *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_modify); + + cmd_buf->body.jetty_type = JFS; + cmd_buf->body.mask.state = UDMA_MASK_VALID; + cmd_buf->body.old_state = jfs->state; + cmd_buf->body.new_state = state; + cmd_buf->body.sq_pi = jfs->sq.pi; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_MODIFY_CTX, cmd_len, jfs->jfs_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfs_modify_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + enum ubcore_jetty_state state, + struct ubcore_udata *udata) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_modify *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfs_modify_ctx(jfs, state, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, &jfs->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_fill_cmd_jfs_delete_ctx(u32 idx, + struct ub_cmd_jetty_delete *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_delete); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_DELETE_CTX, cmd_len, idx); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_jfs_delete_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_jetty_delete *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_delete), &cqm_cmd_outbuf, + sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_jfs_delete_ctx(jfs->jfs_id, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_JETTY_DELETE_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, + &jfs->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + else + ret = udma_cmd_check_dpath_dma(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + sizeof(u8), sizeof(u8)); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int +udma_fill_cmd_jfs_cache_invld_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, u64 wb_dma, + struct ub_cmd_jetty_cache_invld *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_jetty_cache_invld); + int ret; + + ret = udma_fill_cmd_body(udma_dev, cmd_buf, jfs->sq_buf_len, wb_dma); + if (ret != 0) { + return ret; + } + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_JETTY, + UB_CMD_JETTY_CACHE_INVLD_CTX, cmd_len, + jfs->jfs_id); + + udma_cpu_to_be32(cmd_buf, cmd_len); + + return 0; +} + +int udma_cmd_jfs_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, u64 wb_dma) +{ + struct ub_cmd_jetty_cache_invld *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u32 times = 0; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_jetty_cache_invld), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc cqm_cmd_inoutbuf, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_jetty_cache_invld *)cqm_cmd_inbuf->buf; + ret = udma_fill_cmd_jfs_cache_invld_ctx(udma_dev, jfs, wb_dma, cmd_buf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed fill jetty cacheout inbuf, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + ret = udma_send_cache_invld_ctx(udma_dev, cqm_cmd_inbuf, &times, + UB_CMD_JETTY_CACHE_INVLD_CTX, + &jfs->cmdq_rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Failed to send cqm_cmd_box after try times(%u), jfs(%u), ret(%d)\n", + times, jfs->jfs_id, ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_jfs_query_driver_ctr(struct udma_device *udma_dev, u32 jfs_id, + ub_driver_comm_ctr_t *outbuf) +{ + struct udma_jfs *ub_jfs = NULL; + struct mutex *lock = &udma_dev->device_res->jfs_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jfs, &udma_dev->device_res->jfs_list.node, + node) { + if (ub_jfs->jfs_id == jfs_id) { + udma_jfs_get(ub_jfs); + mutex_unlock(lock); + outbuf->post_send_wr_num = (u64)atomic64_read( + &ub_jfs->sq.udma_driver_sq_ctr.post_send_wr_num); + udma_jfs_put(ub_jfs); + return 0; + } + } + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_JFS, + "query driver ctr fail, jfs_id:%u\n", jfs_id); + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h new file mode 100644 index 000000000..e271672a8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_jfs.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_JFS_H +#define UDMA_CMD_JFS_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_jetty.h" + +int udma_cmd_jfs_create_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg); +int udma_cmd_jfs_modify_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs, + enum ubcore_jetty_state state, + struct ubcore_udata *udata); +int udma_cmd_jfs_delete_ctx(struct udma_device *udma_dev, struct udma_jfs *jfs); +int udma_cmd_jfs_cache_out_ctx(struct udma_device *udma_dev, + struct udma_jfs *jfs, u64 wb_dma); +int udma_cmd_jfs_query_driver_ctr(struct udma_device *udma_dev, u32 jfs_id, + ub_driver_comm_ctr_t *outbuf); + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c new file mode 100644 index 000000000..8ea1f9aea --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.c @@ -0,0 +1,60 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> + +#include "ub_npu_mig_cmd.h" +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "udma_cmd_mig.h" + +int udma_cmd_mig_query_counter(struct udma_device *udma_dev, u32 func_id, + struct ub_cmd_mig_drain_status_outbuf *out) +{ + struct ub_cmd_mig_query_drain_status *cmd_in = NULL; + struct ub_cmd_mig_drain_status_outbuf *cmd_out = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mig_query_drain_status), + &cqm_cmd_outbuf, + (u16)sizeof(struct ub_cmd_mig_drain_status_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "Alloc cmd_buf failed. ret: %d\n", ret); + return ret; + } + + cmd_in = (struct ub_cmd_mig_query_drain_status *)cqm_cmd_inbuf->buf; + cmd_len = sizeof(struct ub_cmd_mig_query_drain_status); + cmd_in->body.func_id = func_id; + udma_fill_cmd_header(&cmd_in->header, UB_CMD_TYPE_MIGRATE, + UB_CMD_MIG_QUERY_DRAIN_STATUS, cmd_len, 0); + udma_cpu_to_be32(cmd_in, cmd_len); + cmd_out = (struct ub_cmd_mig_drain_status_outbuf *)cqm_cmd_outbuf->buf; + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MIG_QUERY_DRAIN_STATUS, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return -EBUSY; + } + + udma_be32_to_cpu(cmd_out, + sizeof(struct ub_cmd_mig_drain_status_outbuf)); + (void)memcpy_s((void *)out, + sizeof(struct ub_cmd_mig_drain_status_outbuf), + (void *)cmd_out, + sizeof(struct ub_cmd_mig_drain_status_outbuf)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h new file mode 100644 index 000000000..5c68d81de --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_mig.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_MIG_H +#define UDMA_CMD_MIG_H + +#include "udma_common.h" +#include "ub_npu_mig_cmd_defs.h" + +int udma_cmd_mig_query_counter(struct udma_device *udma_dev, u32 func_id, + struct ub_cmd_mig_drain_status_outbuf *out); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c new file mode 100644 index 000000000..8be510539 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.c @@ -0,0 +1,418 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_cmd_com.h" +#include "ub_npu_mapt_cmd.h" +#include "udma_cmd_segment.h" + +#define MAPT_MAX_RETRY 200 +#define HMM_MS_DELAY 5 +#define DATALEN_RATIO 2 //user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 //user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +static void udma_set_seg_access(struct ub_mapt_chip_attr *chip_attr, + const struct udma_segment *udma_seg) +{ + chip_attr->local_write = 1; + if ((UBCORE_ACCESS_LOCAL_ONLY & udma_seg->access) != 0) + return; + + chip_attr->remote_invld = 1; + if ((UBCORE_ACCESS_READ & udma_seg->access) != 0) + chip_attr->remote_read = 1; + + if ((UBCORE_ACCESS_WRITE & udma_seg->access) != 0) + chip_attr->remote_write = 1; + + if ((UBCORE_ACCESS_ATOMIC & udma_seg->access) != 0) + chip_attr->remote_atomic = 1; +} + +static void udma_fill_cmd_seg_chip_attr(struct ub_mapt_chip_attr *chip_attr, + struct udma_segment *udma_seg) +{ + struct ub_mtt_attr *mtt_attr = &chip_attr->mtt_attr; + + if (!udma_seg->david_hbm_en) { + mtt_attr->mtt_paddr_h = upper_32_bits(udma_seg->mtt.mtt_paddr); + mtt_attr->mtt_paddr_l = lower_32_bits(udma_seg->mtt.mtt_paddr); + mtt_attr->mtt_layers = udma_seg->mtt.mtt_layers; + mtt_attr->mtt_page_shift = udma_seg->mtt.mtt_page_shift; + chip_attr->buf_page_size = udma_seg->mtt.buf_page_shift; + } + + chip_attr->iova_hi = upper_32_bits(udma_seg->va); + chip_attr->iova_lo = lower_32_bits(udma_seg->va); + chip_attr->length_hi = upper_32_bits(udma_seg->size); + chip_attr->length_lo = lower_32_bits(udma_seg->size); + + chip_attr->block_size = udma_seg->block_size; + chip_attr->so_ro = 0; + chip_attr->mkey = udma_seg->key; + + udma_set_seg_access(chip_attr, udma_seg); +} + +static void udma_fill_cmd_seg_user_attr(struct ub_mapt_sw_attr *attr, + struct udma_segment *udma_seg) +{ + attr->token_value = udma_seg->token_value; + attr->key_type = udma_seg->key_type; + if (udma_seg->david_hbm_en) { + attr->eid_d = udma_seg->uba.eid.eid; + attr->tid_d = udma_seg->uba.token_id; + } +} + +static void udma_fill_cmd_seg_create_ctx(struct udma_segment *udma_seg, + struct udma_keyid *udma_key, + struct ub_cmd_mapt_create *cmd_buf, + u8 cmd) +{ + struct ub_cmd_body_mapt_create *body = &cmd_buf->body; + u16 cmd_len = sizeof(struct ub_cmd_mapt_create); + + udma_fill_cmd_seg_chip_attr(&body->chip_attr, udma_seg); + + udma_fill_cmd_seg_user_attr(&body->sw_attr, udma_seg); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, cmd, + cmd_len, udma_key->mpt.mpt_index); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_seg_query_ctx(u32 tokenid, + struct ub_cmd_mapt_query *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_mapt_query); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_QUERY_CTX, cmd_len, tokenid); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_seg_delete_ctx(u32 tokenid, + struct ub_cmd_mapt_delete *cmd_buf, + u8 cmd) +{ + u16 cmd_len = sizeof(struct ub_cmd_body_mapt_delete); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, cmd, + cmd_len, tokenid); + + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +static void udma_fill_cmd_seg_query_safe_dmtt_bitmap( + const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_bitmap *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_safe_dmtt_bitmap); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_QUERY_SAFE_DMTT_BITMAP, cmd_len, + inbuf->xid); + cmd_buf->func_id = inbuf->vfid; + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_seg_query_safe_dmtt_bitmap( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_bitmap_outbuf *outbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_safe_dmtt_bitmap *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_bitmap), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_safe_dmtt_bitmap *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_query_safe_dmtt_bitmap(inbuf, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_QUERY_SAFE_DMTT_BITMAP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf), + cqm_cmd_outbuf->buf, + sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void +udma_fill_cmd_seg_query_safe_dmtt_ctx(const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_ctx *cmd_buf) +{ + u16 cmd_len = sizeof(struct ub_cmd_safe_dmtt_ctx); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_QUERY_SAFE_DMTT_CTX, cmd_len, + inbuf->xid); + cmd_buf->func_id = inbuf->vfid; + cmd_buf->gpa_hi = upper_32_bits(inbuf->gpa); + cmd_buf->gpa_lo = lower_32_bits(inbuf->gpa); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_seg_query_safe_dmtt_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_ctx_outbuf *outbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_safe_dmtt_ctx *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_ctx), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_safe_dmtt_ctx *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_query_safe_dmtt_ctx(inbuf, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_QUERY_SAFE_DMTT_CTX, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(outbuf, sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf), + cqm_cmd_outbuf->buf, + sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_cmd_seg_create_ctx(struct udma_device *udma_dev, + struct udma_segment *udma_seg, + struct udma_keyid *udma_key) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_mapt_create *cmd_buf = NULL; + u8 cmd = udma_seg->david_hbm_en ? UB_CMD_MAPT_CREATE_DAVID_CTX : + UB_CMD_MAPT_CREATE_CTX; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mapt_create), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_mapt_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_create_ctx(udma_seg, udma_key, cmd_buf, cmd); + + ret = udma_send_cmd_box(udma_dev, cmd, cqm_cmd_inbuf, NULL, NULL, + &udma_seg->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_seg_query_ctx(struct udma_device *udma_dev, u32 tokenid, + struct ub_mapt_ctx_query *ctx) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_mapt_query *cmd_buf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mapt_query), &cqm_cmd_outbuf, + (u16)sizeof(struct ub_mapt_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_mapt_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_query_ctx(tokenid, cmd_buf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } + + (void)memcpy_s(ctx, sizeof(struct ub_mapt_ctx_query), + cqm_cmd_outbuf->buf, sizeof(struct ub_mapt_ctx_query)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +int udma_cmd_seg_delete_ctx(struct udma_device *udma_dev, + struct udma_keyid *udma_key, + struct udma_segment *udma_seg) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_mapt_delete *cmd_buf = NULL; + ub_cmdq_cqe_udata_s cmdq_cqe = { 0 }; + u8 cmd = udma_seg->david_hbm_en ? UB_CMD_MAPT_DELETE_DAVID_CTX : + UB_CMD_MAPT_DELETE_CTX; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_mapt_delete), &cqm_cmd_outbuf, + sizeof(u32) * UDMA_DPATH_MAX_OQ_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_mapt_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_seg_delete_ctx(udma_key->mpt.mpt_index, cmd_buf, cmd); + + ret = udma_send_cmd_box(udma_dev, cmd, cqm_cmd_inbuf, cqm_cmd_outbuf, + &cmdq_cqe, &udma_seg->cmdq_rtt); + if (ret != 0) { + ret = -EIO; + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_cqm_send; + } else { + ret = udma_dpath_drain_handle(udma_dev, + (u8 *)cqm_cmd_outbuf->buf, + cmdq_cqe.bs.dpath_oq_num); + if (ret != 0) { + goto err_cqm_send; + } + } + udma_debug( + udma_dev->dev, UDMA_M_SEG, + "cmdq_cqe.bs.dpath_oq_num(%u), cmdq_cqe.value(0x%llx), ret:%d\n", + cmdq_cqe.bs.dpath_oq_num, cmdq_cqe.value, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return 0; + +err_cqm_send: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static void +udma_fill_cmd_mapt_batch_modify_ctx(struct ub_cmd_mapt_batch_modify *modify_cmd, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_mapt_batch_modify); + u32 i; + + modify_cmd->body.offset = cpu_to_be32(inbuf->offset); + modify_cmd->body.length = cpu_to_be32(inbuf->length); + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; i < inbuf->offset + inbuf->length && + i < UB_CQM_MODIFY_DATA_LEN_MAPT / DATALEN_RATIO; + ++i) { + *(((u8 *)&modify_cmd->body.ctx) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_SEGMENT, + UB_CMD_MAPT_BATCH_MODIFY_CTX, cmd_len, inbuf->xid); + /* here only convert header endian */ + udma_cpu_to_be32(modify_cmd, sizeof(ub_cmd_com_header_s)); +} + +int udma_cmd_seg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct ub_cmd_mapt_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_mapt_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "cqm inbuf allocation failed\n"); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_mapt_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_mapt_batch_modify_ctx(modify_cmd, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MAPT_BATCH_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "send cmd failed, ret(%d)\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "seg batch modify success, mapt_id(%u)\n", inbuf->xid); + +err_send_cmd: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h new file mode 100644 index 000000000..0810ff720 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_segment.h @@ -0,0 +1,30 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_SEGMENT_H +#define UDMA_CMD_SEGMENT_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_key.h" +#include "udma_segment.h" +#include "ub_pub_cmd.h" + +int udma_cmd_seg_create_ctx(struct udma_device *udma_dev, + struct udma_segment *udma_seg, + struct udma_keyid *udma_key); +int udma_cmd_seg_query_ctx(struct udma_device *udma_dev, u32 tokenid, + struct ub_mapt_ctx_query *ctx); +int udma_cmd_seg_delete_ctx(struct udma_device *udma_dev, + struct udma_keyid *udma_key, + struct udma_segment *udma_seg); +int udma_cmd_seg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_seg_query_safe_dmtt_bitmap( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_bitmap_outbuf *outbuf); +int udma_cmd_seg_query_safe_dmtt_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + struct ub_cmd_safe_dmtt_ctx_outbuf *outbuf); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c new file mode 100644 index 000000000..acbdaba88 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "ub_npu_sip_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_net.h" +#include "udma_cmd_com.h" +#include "udma_cmd_sip.h" + +static int udma_sip_cmd(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr, u8 cmd) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_sip_info); + struct ub_cmd_sip_info *cmd_buf = NULL; + int ret; + + if (sip_idx >= UDMA_SIP_NUM_MAX) { + udma_err(udma_dev->dev, UDMA_M_DEV, "sip_idx:%u >= %u err\n", + sip_idx, UDMA_SIP_NUM_MAX); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Alloc cmd_buf failed, err(%d)\n", __func__, ret); + return ret; + } + + // construct cmd data + cmd_buf = (struct ub_cmd_sip_info *)cqm_cmd_inbuf->buf; + if (sip_attr != NULL) { + (void)memcpy_s(&cmd_buf->body.sip_attr, + sizeof(struct ub_sip_attr), sip_attr, + sizeof(struct ub_sip_attr)); + } else { + ret = -EINVAL; + udma_err(udma_dev->dev, UDMA_M_DEV, + "sip attr is null, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SIP, cmd, cmd_len, + sip_idx); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, cmd, cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s:Failed to send cqm_cmd_box, ret(%d)\n", __func__, + ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_sip_add(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr) +{ + return udma_sip_cmd(udma_dev, sip_idx, sip_attr, UB_CMD_ADD_SIP); +} + +int udma_cmd_sip_del(struct udma_device *udma_dev, u32 sip_idx) +{ + struct ub_sip_attr sip_attr = { 0 }; + + return udma_sip_cmd(udma_dev, sip_idx, &sip_attr, UB_CMD_DEL_SIP); +} + +int udma_cmd_sip_modify(struct udma_device *udma_dev, u32 sip_idx, + const struct ub_cqm_modify_inbuf *inbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_sip_info); + struct ub_cmd_sip_info *cmd_buf = NULL; + int ret; + u32 i; + + if (sip_idx >= UDMA_SIP_NUM_MAX) { + udma_err(udma_dev->dev, UDMA_M_DEV, "sip_idx:%u >= %u err\n", + sip_idx, UDMA_SIP_NUM_MAX); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Alloc cmd_buf failed, err(%d)\n", __func__, ret); + return ret; + } + + cmd_buf = (struct ub_cmd_sip_info *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SIP, + UB_CMD_MODIFY_SIP, cmd_len, inbuf->xid); + cmd_buf->body.offset = inbuf->offset; + cmd_buf->body.length = inbuf->length; + udma_cpu_to_be32(cmd_buf, cmd_len); + + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; + i < inbuf->offset + inbuf->length && i < sizeof(ub_sip_attr_s); + ++i) { + *(((u8 *)&cmd_buf->body.sip_attr) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_MODIFY_SIP, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "[ERR] %s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "[INFO] %s: success, tpn(%u)\n", __func__, inbuf->xid); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_sip_query(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr) +{ + struct ub_cmd_sip_info *cmd_buf = NULL; + struct ub_sip_attr *cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_sip_info); + u16 out_len = sizeof(struct ub_sip_attr); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, out_len); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Alloc cmd_buf failed, err(%d)\n", ret); + return ret; + } + + // construct cmd data + cmd_buf = (struct ub_cmd_sip_info *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SIP, + UB_CMD_QUERY_SIP, cmd_len, sip_idx); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_QUERY_SIP, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to send cqm_cmd_box, ret(%d)\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return -EBUSY; + } + + cmd_outbuf = (struct ub_sip_attr *)cqm_cmd_outbuf->buf; + (void)memcpy_s(sip_attr, sizeof(struct ub_sip_attr), cmd_outbuf, + sizeof(struct ub_sip_attr)); + sip_attr->dw4.dw4_value = be32_to_cpu(sip_attr->dw4.dw4_value); + sip_attr->dw5.dw5_value = be32_to_cpu(sip_attr->dw5.dw5_value); + sip_attr->dw6.dw6_value = be32_to_cpu(sip_attr->dw6.dw6_value); + sip_attr->smac_l32 = be32_to_cpu(sip_attr->smac_l32); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h new file mode 100644 index 000000000..5682aaa96 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_sip.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_SIP_H +#define UDMA_CMD_SIP_H + +#include "udma_common.h" +#include "ub_npu_sip_cmd_defs.h" +#include "ub_pub_cmd.h" + +int udma_cmd_sip_add(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr); +int udma_cmd_sip_del(struct udma_device *udma_dev, u32 sip_idx); +int udma_cmd_sip_query(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr); +int udma_cmd_sip_modify(struct udma_device *udma_dev, u32 sip_idx, + const struct ub_cqm_modify_inbuf *inbuf); + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c new file mode 100644 index 000000000..e6ea67377 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.c @@ -0,0 +1,326 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ub_npu_srq_cmd.h" +#include "ub_npu_srq_cmd_defs.h" +#include "udma_srq.h" +#include "udma_comp.h" +#include "udma_cqm.h" +#include "udma_cmd_com.h" +#include "udma_cmd_srq.h" + +static void udma_fill_cmd_srq_sw_attr(struct ub_srq_sw_attr *attr, + struct udma_srq *srq, + const struct udma_db *sdb, u8 mtt_pro_en) +{ + attr->dw0.bs.size = (u32)ilog2(srq->max_depth); + attr->dw0.bs.page_size = + udma_get_queue_kernel_page_size(mtt_pro_en, srq->buf_size); + attr->dw0.bs.wqebb_size = srq->wqe_shift - SRQ_WQEBB_SIZE_CAL_SECTTOR; + attr->dw0.bs.mtt_page_size = srq->ub_mtt.mtt_page_shift - PAGE_SHIFT_4K; + + attr->dw1.bs.dma_attr_idx = 0; + attr->dw1.bs.so_ro = 1; + attr->dw1.bs.state = UB_SRQ_STATE_VALID; + attr->dw1.bs.container = 1; //ub use container mode + attr->dw1.bs.lth_pre_en = 0; //rdma engine prefetch + attr->dw1.bs.pcnt_on_chip = 0; + attr->dw1.bs.rkey_en = 1; + + attr->srqn = srq->srqn; + + attr->cont.dw0.bs.head_idx = 0; + attr->cont.dw0.bs.warn_th = 0; //Do NOT init warn_th until user arm srq + attr->cont.dw0.bs.cont_size = (srq->container_mode - 1) & 0x3; + + attr->mtt_paddr_h = upper_32_bits(srq->ub_mtt.mtt_paddr); + attr->mtt_paddr_l = lower_32_bits(srq->ub_mtt.mtt_paddr); + + attr->db_paddr_h = upper_32_bits(sdb->dma); + attr->db_paddr_l_at_hop_num = lower_32_bits(sdb->dma) & (~0x3ULL); + attr->db_paddr_l_at_hop_num |= srq->ub_mtt.mtt_layers & 0x3; + return; +} + +void udma_fill_cmd_srq_create_ctx(struct ub_cmd_srq_create *cmd_buf, + struct udma_srq *srq, struct udma_db *sdb, + u8 mtt_pro_en) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_create); + udma_fill_cmd_srq_sw_attr(&cmd_buf->body.sw_attr, srq, sdb, mtt_pro_en); + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_CREATE_CTX, cmd_len, srq->srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_srq_create_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb) +{ + struct ub_cmd_srq_create *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret = 0; + + udma_debug(udma_dev->dev, UDMA_M_SRQ, + "srq cmdq refactor start, srqn(%u)\n", srq->srqn); + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_create), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_create *)cqm_cmd_inbuf->buf; + udma_fill_cmd_srq_create_ctx(cmd_buf, srq, sdb, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_CREATE_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Send CREATE_SRQ command failed, ret(%d)\n", ret); + ret = -EIO; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +void udma_fill_cmd_srq_delete_ctx(struct ub_cmd_srq_delete *cmd_buf, + struct udma_srq *srq, + struct udma_comp_priv *comp_priv) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_delete); + + cmd_buf->body.srq_buf_len = srq->buf_size; + cmd_buf->body.mtt_flags = 0; + cmd_buf->body.mtt_num = 0; + cmd_buf->body.mtt_cache_line_start = comp_priv->ub_cap.cmtt_cl_start; + cmd_buf->body.mtt_cache_line_end = comp_priv->ub_cap.cmtt_cl_end; + cmd_buf->body.mtt_cache_line_size = comp_priv->ub_cap.cmtt_cl_sz; + + cmd_buf->body.wqe_cache_line_start = comp_priv->ub_cap.wqe_cl_start; + cmd_buf->body.wqe_cache_line_end = comp_priv->ub_cap.wqe_cl_end; + cmd_buf->body.wqe_cache_line_size = comp_priv->ub_cap.wqe_cl_sz; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_DELETE_CTX, cmd_len, srq->srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_srq_delete_ctx(struct udma_device *udma_dev, struct udma_srq *srq) +{ + struct ub_cmd_srq_delete *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct udma_comp_priv *comp_priv = NULL; + int ret = 0; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Get hmm comp priv failed. srqn(%u)\n", srq->srqn); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_delete), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", + srq->srqn, ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_delete *)cqm_cmd_inbuf->buf; + udma_fill_cmd_srq_delete_ctx(cmd_buf, srq, comp_priv); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_DELETE_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Send DELETE_SRQ command failed, ret(%d)\n", + srq->srqn, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static u8 udma_cal_srq_container_num(u32 avail_wr, struct udma_srq *srq) +{ + u32 container_num = 0; + u32 srqe_num = 0; + + /* cal container_num by avail_wr from user */ + srqe_num = srq->container_size - 1; + + /* explain the formula: round up by srqe_num */ + container_num = + (srqe_num == 0) ? 0 : ((avail_wr + srqe_num - 1) / srqe_num); + if (container_num == 0) + return 0; + + container_num = (u32)roundup_pow_of_two(container_num); + return (u8)ilog2(container_num); +} + +void udma_fill_cmd_srq_modify_ctx(struct ub_cmd_srq_modify *cmd_buf, + struct udma_srq *srq, u8 container_warn_th) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_modify); + u8 warh = 0; + + cmd_buf->body.jfr_id = srq->jfr_id; + cmd_buf->body.dw0.bs.cont_en = srq->container_flag; + cmd_buf->body.dw0.bs.th_up_en = + 1; //Valid for setting up container warn_threshold + warh = udma_cal_srq_container_num(container_warn_th, srq); + cmd_buf->body.dw0.bs.warth = warh & UB_SRQ_CONTAINER_WARTH_MASK; + + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_MODIFY_CTX, cmd_len, srq->srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); +} + +int udma_cmd_srq_modify_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + u8 container_warn_th) +{ + struct ub_cmd_srq_modify *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", + srq->srqn, ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_srq_modify_ctx(cmd_buf, srq, container_warn_th); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_MODIFY_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Send MODIFY_SRQ command failed, ret(%d)\n", + srq->srqn, ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_srq_query_ctx(struct udma_device *udma_dev, u32 srqn, + ub_cmd_srq_query_rsp_s *srq_entry) +{ + struct ub_cmd_srq_query *cmd_buf = NULL; + ub_cmd_srq_query_rsp_s *srq_query_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_srq_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_srq_query), + &cqm_cmd_outbuf, + (u16)sizeof(ub_cmd_srq_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "(%u)Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", srqn, + ret); + return -ENOMEM; + } + + cmd_buf = (struct ub_cmd_srq_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&cmd_buf->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_QUERY_CTX, cmd_len, srqn); + udma_cpu_to_be32(cmd_buf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Failed to send cmd QUERY_SRQ, ret=%d", ret); + ret = -EIO; + goto out; + } + + srq_query_outbuf = (ub_cmd_srq_query_rsp_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s(srq_entry, sizeof(ub_cmd_srq_query_rsp_s), + srq_query_outbuf, sizeof(ub_cmd_srq_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, "memcpy_s failed, ret=%d\n", + ret); + ret = -EFAULT; + goto out; + } + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + return ret; +} + +static void +udma_fill_cmd_srq_batch_modify_ctx(struct ub_cmd_body_srq_batch_modify *body, + const struct ub_cqm_modify_inbuf *inbuf) +{ + body->offset = inbuf->offset; + body->length = inbuf->length; + + udma_cpu_to_be32(body, sizeof(struct ub_cmd_body_srq_batch_modify)); + + udma_cmdq_data_to_context_buf(inbuf->data, inbuf->data_len, + (u8 *)&body->srq_ctx, + sizeof(body->srq_ctx) / sizeof(u8), + inbuf->offset); +} + +int udma_cmd_srq_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_srq_batch_modify); + struct ub_cmd_srq_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Alloc cqm_cmd_inoutbuf failed, ret(%d)\n", ret); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_srq_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_SRQ, + UB_CMD_SRQ_BATCH_MODIFY_CTX, cmd_len, inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(modify_cmd->header)); + + udma_fill_cmd_srq_batch_modify_ctx(&modify_cmd->body, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_SRQ_BATCH_MODIFY_CTX, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Failed to send cmd SRQ_BATCH_MODIFY_CTX, ret=%d", + ret); + goto err_send_cmd; + } + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h new file mode 100644 index 000000000..981843091 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_srq.h @@ -0,0 +1,22 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_SRQ_H +#define UDMA_CMD_SRQ_H + +#include "ub_npu_srq_cmd_defs.h" +#include "udma_common.h" +#include "udma_srq.h" +#include "ub_pub_cmd.h" + +int udma_cmd_srq_create_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb); +int udma_cmd_srq_delete_ctx(struct udma_device *udma_dev, struct udma_srq *srq); +int udma_cmd_srq_modify_ctx(struct udma_device *udma_dev, struct udma_srq *srq, + u8 container_warn_th); +int udma_cmd_srq_query_ctx(struct udma_device *udma_dev, u32 srqn, + ub_cmd_srq_query_rsp_s *srq_entry); +int udma_cmd_srq_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c new file mode 100644 index 000000000..06f268570 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.c @@ -0,0 +1,1885 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include <linux/sched.h> + +#include "ubcore_types.h" +#include "ub_npu_tp_cmd.h" +#include "ub_npu_tp_cmd_defs.h" +#include "udma_cqm.h" +#include "udma_db.h" +#include "udma_net.h" +#include "udma_tpg.h" +#include "udma_cmd_com.h" +#include "udma_tp.h" +#include "udma_cmd_tp.h" + +#define TP_DIP_MASK 0xffffffff //32 bit + +#define TP_DEFAULT_OOR_BS 4 +#define TP_RC_NUM 0x2 + +bool g_must_check_token_en = 0; + +module_param(g_must_check_token_en, bool, + (S_IRUSR | S_IRGRP | S_IROTH)); //0444 + +static int udma_tp_get_oor_bs(struct udma_device *udma_dev, u32 oor_cnt) +{ + u32 udma_tp_bitmap_level[] = { UDMA_TP_OOR_64, UDMA_TP_OOR_128, + UDMA_TP_OOR_256, UDMA_TP_OOR_512, + UDMA_TP_OOR_1024 }; + int i; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter. udma_dev(%d) .\n", + !!udma_dev); + return -EINVAL; + } + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: tp oor_cnt:%u\n", __func__, + oor_cnt); + + for (i = 0; + i < sizeof(udma_tp_bitmap_level) / sizeof(udma_tp_bitmap_level[0]); + i++) { + if (oor_cnt == udma_tp_bitmap_level[i]) + return i; + } + + return TP_DEFAULT_OOR_BS; //default:4 contrast:1024 +} + +static void udma_cmd_tp_create_chip_seg_fill(struct tag_ub_tp_chip *chip_seg, + struct udma_tp *ub_tp) +{ + u64 tmp = 0; + + /* QPCC */ + if (ub_tp->kdb != NULL) { + chip_seg->dw2.bs.sq_rq_pi_record_gpa_hi = + upper_32_bits(ub_tp->kdb->dma); + chip_seg->dw2.bs.sq_rq_pi_record_gpa_lo_at_hop_num = + lower_32_bits(ub_tp->kdb->dma) & (~0x3ULL); //bits:31-2 + } + + chip_seg->dw0.bs.sq_rq_l0mtt_gpa_hi = + upper_32_bits(ub_tp->ub_mtt.mtt_paddr); + chip_seg->dw0.bs.sq_rq_l0mtt_gpa_lo = + lower_32_bits(ub_tp->ub_mtt.mtt_paddr); + chip_seg->dw2.bs.sq_rq_pi_record_gpa_lo_at_hop_num |= + ub_tp->ub_mtt.mtt_layers & 0x3; //bits:1-0 + chip_seg->dw4.bs.sq_rq_mtt_page_size = + ub_tp->ub_mtt.mtt_page_shift - PAGE_SHIFT_4K; + + if (ub_tp->entry_cnt != 0) { + tmp = roundup_pow_of_two(ub_tp->entry_cnt); + chip_seg->dw4.bs.rq_base_ci = ((u32)ilog2((u32)tmp)) & + TP_SQ_SIZE_MASK; + } + chip_seg->dw4.bs.rrw_mtt_prefetch_maxlen = 0; + chip_seg->dw4.bs.sq_wqe_prefetch_mode = 0; + chip_seg->dw4.bs.sq_mtt_prefetch_maxlen = 0; + chip_seg->dw4.bs.sqa_mtt_prefetch_maxlen = 0; + chip_seg->dw4.bs.rc_entry_prefetch_maxnum = 0; + chip_seg->dw4.bs.rc_mtt_prefetch_maxlen = TP_QP_MAX_PFETCH_MTT_LAYER; + chip_seg->dw4.bs.qp_rkey_en = 1; + chip_seg->dw4.bs.sq_wqebb_size = + ((u32)(TP_WQE_SHIFT - TP_WQEBB_SIZE_SHIFT)) & + TP_WQEBB_SIZE_MASK; + chip_seg->dw4.bs.sq_rkey_en = 1; + + if (ub_tp->entry_cnt != 0) { + tmp = roundup_pow_of_two(ub_tp->entry_cnt); + chip_seg->dw5.bs.sq_size = ((u32)ilog2((u32)tmp)) & + TP_SQ_SIZE_MASK; + } + chip_seg->dw5.bs.sq_pi_on_chip = 0; + + chip_seg->dw5.bs.sq_wqecnt_lth = TP_SQ_WQECNT_LTH; + chip_seg->dw5.bs.sq_wqecnt_rctl_en = 0; + chip_seg->dw5.bs.sq_wqecnt_rctl = 0; + chip_seg->dw5.bs.sq_wqe_prefetch_minnum = TP_SQ_PREFETCH_MINNUM; + chip_seg->dw5.bs.sq_wqe_prefetch_maxnum = TP_SQ_PREFETCH_MAXNUM; + chip_seg->dw5.bs.sq_wqe_cache_thd_sel = 0; + chip_seg->dw5.bs.sqa_wqe_prefetch_minnum = TP_SQ_PREFETCH_MINNUM; + chip_seg->dw5.bs.sqa_wqe_prefetch_maxnum = TP_WQE_PREFETCH_MAXNUM; + chip_seg->dw5.bs.sqa_wqe_cache_thd_sel = 0; + chip_seg->dw5.bs.sq_inline_en = 1; +} + +static void udma_cmd_tp_create_sw_fill(struct ubcore_device *ub_dev, + struct tag_tp_sw_ctx *tp_sw, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_tpg *ub_tpg = NULL; + u32 i; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return; + } + + /* dw0 */ + if (ub_tp->tp_info != NULL) { + tp_sw->srctpn = ub_tp->tp_info->xid; + } + tp_sw->retry_factor = cfg->retry_factor; + tp_sw->state = UB_TPC_STATE_RST; + tp_sw->transport_mode = (cfg->trans_mode == UBCORE_TP_UM) ? + UB_TRANSPORT_MODE_UM : + ((cfg->trans_mode == UBCORE_TP_RM) ? + UB_TRANSPORT_MODE_RM : + UB_TRANSPORT_MODE_RC); + /* dw1 */ + tp_sw->tp_retry_num = cfg->retry_num; + tp_sw->tp_timeout = cfg->ack_timeout; + tp_sw->oor_bitmap_size = + (u32)udma_tp_get_oor_bs(udma_dev, cfg->oor_cnt); + /* dw3 */ + tp_sw->loopback = cfg->flag.bs.loopback; + tp_sw->ep_id = udma_dev->hw_info.ep_id; + tp_sw->must_check_token_en = g_must_check_token_en; + tp_sw->jetty_id = cfg->local_jetty.id; + + if (cfg->flag.bs.ack_resp == 0 && cfg->tpg != NULL) { + for (i = 0; i < TP_RC_NUM; i++) { + ub_tpg = container_of(cfg->tpg, struct udma_tpg, tpg); + if (ub_tpg->tpinfo_list[i]->xid != ub_tp->tp.tpn) { + tp_sw->rsp_tpgn = ub_tpg->tpinfo_list[i]->xid; + break; + } + } + udma_debug(udma_dev->dev, UDMA_M_TP, "rsp_tpgn: %u\n", + tp_sw->rsp_tpgn); + } + /* dw5 */ + tp_sw->suspend_cnt = udma_dev->suspend_cnt; + /* dw6 */ + tp_sw->suspend_period = udma_dev->suspend_period; + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw0_value(0x%08x)\n", + __func__, tp_sw->dw0_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw1_value(0x%08x)\n", + __func__, tp_sw->dw1_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw2_value(0x%08x)\n", + __func__, tp_sw->dw2_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw3_value(0x%08x)\n", + __func__, tp_sw->dw3_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw4_value(0x%08x)\n", + __func__, tp_sw->dw4_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: suspend_cnt(0x%08x)\n", + __func__, tp_sw->suspend_cnt); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: suspend_period(0x%08x)\n", + __func__, tp_sw->suspend_period); +} + +static void udma_cmd_tp_create_body_fill(struct ubcore_device *ub_dev, u32 cnt, + struct ub_cmd_tp_create_body *body, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp[]) +{ + u32 i; + + body->num = cnt; + + for (i = 0; i < cnt; i++) { + udma_cmd_tp_create_chip_seg_fill(&body->tp_attr[i].chip_seg, + ub_tp[i]); + udma_cmd_tp_create_sw_fill(ub_dev, &body->tp_attr[i].tpc_com, + cfg + i, ub_tp[i]); + } +} + +/* return value: cmd tp create success of count */ +u32 udma_cmd_tp_create_ctx(struct ubcore_device *ub_dev, u32 cnt, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp[]) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + u16 cmd_len = sizeof(struct ub_cmd_tp_create); + struct ub_cmd_tp_create *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u32 *cmd_out_num = NULL; + u32 ret_num; + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return 0; + } + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s to_udma_dev failed.\n", __func__); + return 0; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_create), + &buf_out, sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: allocate cqm inbuf failed\n", __func__); + return 0; + } + + tp_cmd_in = (struct ub_cmd_tp_create *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_CREATE_CTX, cmd_len, + 0); //xid fill 0 not affect, actual xid in body in passed in + + udma_cmd_tp_create_body_fill(ub_dev, cnt, &tp_cmd_in->body, cfg, ub_tp); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + cmd_out_num = (u32 *)buf_out->buf; + + cmdq_rtt = (cfg->tpg == NULL) ? &ub_tp[0]->cmdq_rtt : + &to_udma_tpg(cfg->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_CREATE_CTX, buf_in, buf_out, + NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: send cqm cmd failed, ret(%d)\n", __func__, ret); + } + + ret_num = be32_to_cpu(*cmd_out_num); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret_num; +} + +static void +udma_fill_update_attr_mask(const union ubcore_tp_attr_mask *mask, + struct tag_ub_tp_modify_attr *modify_attr) +{ + modify_attr->attr_mask.bs.flag = mask->bs.flag; + modify_attr->attr_mask.bs.peer_tpn = mask->bs.peer_tpn; + modify_attr->attr_mask.bs.state = mask->bs.state; + modify_attr->attr_mask.bs.tx_psn = mask->bs.tx_psn; + modify_attr->attr_mask.bs.rx_psn = mask->bs.rx_psn; + modify_attr->attr_mask.bs.mtu = mask->bs.mtu; + modify_attr->attr_mask.bs.cc_pattern_idx = mask->bs.cc_pattern_idx; + modify_attr->attr_mask.bs.oos_cnt = mask->bs.oos_cnt; + modify_attr->attr_mask.bs.local_net_addr_idx = + mask->bs.local_net_addr_idx; + modify_attr->attr_mask.bs.peer_net_addr = mask->bs.peer_net_addr; + modify_attr->attr_mask.bs.data_udp_start = mask->bs.data_udp_start; + modify_attr->attr_mask.bs.ack_udp_start = mask->bs.ack_udp_start; + modify_attr->attr_mask.bs.udp_range = mask->bs.udp_range; + modify_attr->attr_mask.bs.hop_limit = mask->bs.hop_limit; + modify_attr->attr_mask.bs.flow_label = mask->bs.flow_label; + modify_attr->attr_mask.bs.port = mask->bs.port_id; + modify_attr->attr_mask.bs.mn = mask->bs.mn; +} + +static ub_cc_algo_e udma_get_ub_cc_alg(struct udma_device *udma_dev, + enum ubcore_tp_cc_alg cc_alg) +{ + switch (cc_alg) { + case UBCORE_TP_CC_DCQCN: + return UB_CC_ALGO_DCQCN; + case UBCORE_TP_CC_LDCP: + return UB_CC_ALGO_LDCP; + case UBCORE_TP_CC_CAQM: + return UB_CC_ALGO_CAQM; + case UBCORE_TP_CC_LDCP_L2_HEADER: + return UB_CC_ALGO_LDCP_L2H; + case UBCORE_TP_CC_LDCP_TP_HEADER: + return UB_CC_ALGO_LDCP_TPH; + case UBCORE_TP_CC_CUSTOM_1: + return UB_CC_ALGO_USER_1; + case UBCORE_TP_CC_CUSTOM_2: + return UB_CC_ALGO_USER_2; + default: + udma_debug( + udma_dev->dev, UDMA_M_TP, + "the configured cc_alg is currently not supported\n"); + return UB_CC_ALGO_MAX; + } +} + +static void udma_fill_droute_attr(struct udma_device *udma_dev, + const struct udma_tp *ub_tp, + const union ubcore_tp_attr_mask *mask, + const struct ubcore_tp_attr *attr, + struct tag_ub_tp_modify_attr *modify_attr) +{ + vlan_u vlan_value = { 0 }; + + if (mask->bs.peer_net_addr == 1) { + /* Note: cmd send before will uniformly perform cpu_to_be32 convert, here dip original this then is big-endian sequence, no need convert to big-endian; + * 因 this here 先 be32_to_cpu, 这样 after cpu_to_be32 就 keep 最初 of value (done) */ + if (attr->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: peer is call ipv4\n", __func__); + modify_attr->droute.dip3 = be32_to_cpu( + attr->peer_net_addr.net_addr.in4.addr); + } else { + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: peer is call ipv6\n", __func__); + /* dip0 - dip3 */ + modify_attr->droute.dip0 = be32_to_cpu( + attr->peer_net_addr.net_addr.in6.subnet_prefix & + TP_DIP_MASK); + modify_attr->droute.dip1 = + be32_to_cpu((attr->peer_net_addr.net_addr.in6 + .subnet_prefix >> + 32) & + TP_DIP_MASK); //take upper 32 bit + modify_attr->droute.dip2 = be32_to_cpu( + attr->peer_net_addr.net_addr.in6.interface_id & + TP_DIP_MASK); + modify_attr->droute.dip3 = be32_to_cpu( + (attr->peer_net_addr.net_addr.in6.interface_id >> + 32) & + TP_DIP_MASK); //take upper 32 bit + } + + /* dw0 */ + modify_attr->droute.ack_udp_srcport = + (mask->bs.ack_udp_start == 1) ? + attr->ack_udp_start : + modify_attr->droute.ack_udp_srcport; + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: modify ack_udp_start:0x%x\n", __func__, + attr->ack_udp_start); + modify_attr->droute.dmac_h16 = + UDMA_GET_MAC_HI(attr->peer_net_addr.mac[0x0], + attr->peer_net_addr.mac[0x1]); + /* dmac */ + modify_attr->droute.dmac_l32 = + UDMA_GET_MAC_LO(attr->peer_net_addr.mac[0x2], + attr->peer_net_addr.mac[0x3], + attr->peer_net_addr.mac[0x4], + attr->peer_net_addr.mac[0x5]); + /* dw6 */ + modify_attr->droute.tclass = + (ub_tp->pri & 0x3f) << 0x2 | + 0x2; //ub and roce return one, high 6 bit store dscp, low 2 bit is ecn + modify_attr->droute.flow_label = + (mask->bs.flow_label == 1) ? + attr->flow_label : + modify_attr->droute.flow_label; + /* dw7 */ + vlan_value.value = (u16)attr->peer_net_addr.vlan; + modify_attr->droute.sl = ub_tp->pri & 0x7; + modify_attr->droute.vlan_id = vlan_value.bs.vlan; + modify_attr->droute.hoplmt = (mask->bs.hop_limit == 1) ? + attr->hop_limit : + modify_attr->droute.hoplmt; + /* enable vlan */ + if (modify_attr->droute.vlan_id != 0) { + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: vlan en\n", + __func__); + modify_attr->attr_mask.bs.vlan_en = 1; + } + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: peer vlan:%llu\n", + __func__, attr->peer_net_addr.vlan); + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw0:0x%x\n", __func__, + modify_attr->droute.dw0_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dmac_l32:0x%x\n", + __func__, modify_attr->droute.dmac_l32); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip0:0x%x\n", + __func__, modify_attr->droute.dip0); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip1:0x%x\n", + __func__, modify_attr->droute.dip1); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip2:0x%x\n", + __func__, modify_attr->droute.dip2); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dsip3:0x%x\n", + __func__, modify_attr->droute.dip3); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw6:0x%x\n", __func__, + modify_attr->droute.dw6_value); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: dw7:0x%x\n", __func__, + modify_attr->droute.dw7_value); + } +} + +static void udma_fill_update_attr(struct udma_device *udma_dev, + struct udma_tp *ub_tp, + const union ubcore_tp_attr_mask *mask, + const struct ubcore_tp_attr *attr, + struct tag_ub_tp_modify_attr *modify_attr) +{ + modify_attr->tpn = ub_tp->tp.tpn; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter. udma_dev(%d) .\n", + !!udma_dev); + return; + } + + /* determine no need cmdq microcode handle, other attr then no need continue assignment */ + if (mask->bs.state == 1 && ub_tp->tp.state == UBCORE_TP_STATE_ERR && + attr->state == UBCORE_TP_STATE_ERR) { + modify_attr->attr_mask.bs.cmd_ignore = 1; + udma_warn(udma_dev->dev, UDMA_M_TP, + "%s: tp state is already ERR\n", __func__); + return; + } + + if (mask->bs.flag == 1) { + modify_attr->oor_en = attr->flag.bs.oor_en; + modify_attr->sr_en = attr->flag.bs.sr_en; + modify_attr->cc_en = attr->flag.bs.cc_en; + modify_attr->cc_alg = + udma_get_ub_cc_alg(udma_dev, attr->flag.bs.cc_alg); + modify_attr->spray_en = attr->flag.bs.spray_en; + } + + if (mask->bs.state == 1) { + modify_attr->state = attr->state; + udma_debug(udma_dev->dev, UDMA_M_TP, + "tpn:%u, change state:%u\n", ub_tp->tp.tpn, + attr->state); + } + + if (mask->bs.local_net_addr_idx == 1) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: local index(%d), cos(%d)\n", __func__, + attr->local_net_addr_idx, ub_tp->cos); + modify_attr->local_net_addr_idx = attr->local_net_addr_idx; + modify_attr->cos = ub_tp->cos; + } + + modify_attr->tpgn = ((ub_tp->tp.tpg != NULL) && + (ub_tp->tp.trans_mode == UBCORE_TP_RM)) ? + ub_tp->tp.tpg->tpgn : + UB_INVALID_TPGN; + modify_attr->peer_tpn = (mask->bs.peer_tpn == 1) ? + attr->peer_tpn : + modify_attr->peer_tpn; + modify_attr->tx_psn = (mask->bs.tx_psn == 1) ? attr->tx_psn : + modify_attr->tx_psn; + modify_attr->rx_psn = (mask->bs.rx_psn == 1) ? attr->rx_psn : + modify_attr->rx_psn; + modify_attr->mtu = (mask->bs.mtu == 1) ? attr->mtu : modify_attr->mtu; + modify_attr->cc_pattern_idx = (mask->bs.cc_pattern_idx == 1) ? + attr->cc_pattern_idx : + modify_attr->cc_pattern_idx; + modify_attr->oos_cnt = (mask->bs.oos_cnt == 1) ? attr->oos_cnt : + modify_attr->oos_cnt; + modify_attr->data_udp_start = (mask->bs.data_udp_start == 1) ? + attr->data_udp_start : + modify_attr->data_udp_start; + modify_attr->udp_range = (mask->bs.udp_range == 1) ? + attr->udp_range : + modify_attr->udp_range; + modify_attr->port = (mask->bs.port_id == 1) ? attr->port_id : + modify_attr->port; + modify_attr->mn = (mask->bs.mn == 1) ? attr->mn : modify_attr->mn; + + udma_fill_update_attr_mask(mask, modify_attr); + + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: peer_tpn(0x%x), mask->peer_tpn(0x%x), srctpn(0x%x)\n", + __func__, modify_attr->peer_tpn, mask->bs.peer_tpn, + ub_tp->tp.tpn); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: port(%u)\n", __func__, + modify_attr->port); + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: modify attr 0x%x\n", __func__, + modify_attr->dw3_value); +} + +static void udma_fill_cmd_tp_modify(u32 cnt, struct ubcore_tp **tp, + const struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask, + struct ub_cmd_tp_modify *tp_cmd) +{ + struct tag_ub_tp_modify_attr *modify_attr = tp_cmd->body.attr; + struct udma_device *udma_dev = to_udma_dev(tp[0]->ub_dev); + struct udma_tp *ub_tp = NULL; + u32 i; + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!tp[0]->ub_dev); + return; + } + + tp_cmd->body.num = cnt; + + for (i = 0; i < cnt; ++i) { + ub_tp = container_of(tp[i], struct udma_tp, tp); + udma_fill_update_attr(udma_dev, ub_tp, &mask[i], &attr[i], + &modify_attr[i]); + if (modify_attr[i].attr_mask.bs.cmd_ignore == + 0) //need cmdq microcode handle, then continue assignment droute + udma_fill_droute_attr(udma_dev, ub_tp, &mask[i], + &attr[i], &modify_attr[i]); + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s:dw0:0x%x\n", __func__, + modify_attr[i].droute.dw0_value); + } +} + +static void udma_fill_local_en(struct udma_device *udma_dev, u32 cnt, + struct ub_cmd_tp_modify *tp_cmd) +{ + u32 i; + + for (i = 0; i < cnt; ++i) + tp_cmd->body.attr[i].attr_mask.bs.local_en = udma_dev->local_en; + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: TP fill local en:%u\n", + __func__, udma_dev->local_en); +} + +static u32 udma_cmd_tp_cmd_ignore_num_cal(const struct ub_cmd_tp_modify *tp_cmd, + u32 cnt) +{ + u32 cmd_ignore_num = 0; + u32 i; + + for (i = 0; i < cnt; i++) { + if (tp_cmd->body.attr[i].attr_mask.bs.cmd_ignore == 1) + ++cmd_ignore_num; + } + + return cmd_ignore_num; +} + +/* input @cnt: send down modify cmd of tp count + * return: send down cmd success of tp count + * note: input parameter tp, attr, mask may is alreadypast offset of address, non- ubcore passed in of start address, specific see function be call of code + */ +u32 udma_cmd_tp_modify_ctx(u32 cnt, struct ubcore_tp **tp, + const struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask) +{ + struct udma_device *udma_dev = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_modify); + struct ub_cmd_tp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u32 *cmd_out_num = NULL; + u32 modify_num = 0; + int ret; + + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s Invalid parameter: ub_dev(%d).\n", + __func__, !!tp[0]->ub_dev); + return 0; + } + + udma_dev = to_udma_dev(tp[0]->ub_dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s to_udma_dev failed.\n", __func__); + return 0; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_modify), + &buf_out, sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp: failed to allocate cqm inbuf\n", + __func__); + return 0; + } + + cmd_out_num = (u32 *)buf_out->buf; + tp_cmd = (struct ub_cmd_tp_modify *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_CTX, cmd_len, + 0); // actual xid in body in, here only is default fill 0 + + udma_fill_local_en(udma_dev, cnt, tp_cmd); + udma_fill_cmd_tp_modify(cnt, tp, attr, mask, tp_cmd); + + /* if all tp all no need cmdq microcode handle, then directly return */ + if (udma_cmd_tp_cmd_ignore_num_cal(tp_cmd, cnt) == cnt) { + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: all tp don't need send cmd, cnt(%u)\n", __func__, + cnt); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + return cnt; + } + + udma_cpu_to_be32(tp_cmd, cmd_len); + cmdq_rtt = (tp[0]->tpg == NULL) ? &to_udma_tp(tp[0])->cmdq_rtt : + &to_udma_tpg(tp[0]->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_CTX, buf_in, buf_out, + NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp ctx failed, ret(%d)\n", __func__, ret); + } + + modify_num = be32_to_cpu(*cmd_out_num); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + udma_debug(udma_dev->dev, UDMA_M_TP, "%s: modify_num(%u), cnt(%u)\n", + __func__, modify_num, cnt); + + return modify_num; +} + +static void +udma_cmd_tp_modify_dfx_print(struct udma_device *udma_dev, + const struct tag_ub_tp_modify_attr *modify_attr) +{ + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.state, modify_attr->state); + udma_info( + udma_dev->dev, UDMA_M_TP, + "%s: flag: mask(0x%x), oor_en(0x%x), sr_en(0x%x), spray_en(0x%x)\n", + __func__, modify_attr->attr_mask.bs.flag, modify_attr->oor_en, + modify_attr->sr_en, modify_attr->spray_en); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: flag: mask(0x%x), cc_en(0x%x), cc_alg(0x%x)\n", __func__, + modify_attr->attr_mask.bs.flag, modify_attr->cc_en, + modify_attr->cc_alg); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: cc_pattern_idx: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.cc_pattern_idx, + modify_attr->cc_pattern_idx); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: mtu: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.mtu, modify_attr->mtu); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: rx_psn: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.rx_psn, modify_attr->rx_psn); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: tx_psn: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.tx_psn, modify_attr->tx_psn); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: peer_tpn: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.peer_tpn, modify_attr->peer_tpn); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: sq_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.sq_err_state, + modify_attr->sq_err_state); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: sqa_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.sqa_err_state, + modify_attr->sqa_err_state); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: rq_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.rq_err_state, + modify_attr->rq_err_state); + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: rqa_err_state: mask(0x%x), value(0x%x)\n", __func__, + modify_attr->attr_mask.bs.rqa_err_state, + modify_attr->rqa_err_state); +} + +static void +udma_fill_cmd_tp_modify_dfx(struct udma_device *udma_dev, struct udma_tp *ub_tp, + const struct udma_tp_modify_dfx *tp_dfx, + struct tag_ub_tp_modify_attr *modify_attr) +{ + modify_attr->tpn = ub_tp->tp.tpn; + + /* status modify will cause other field of modify, is ensure modify of 准correct validity, constraint: modify status when cannot modify other field */ + if (tp_dfx->mask.state == 1) { + modify_attr->attr_mask.bs.state = tp_dfx->mask.state; + modify_attr->state = tp_dfx->attr.state; + modify_attr->tpgn = ((ub_tp->tp.tpg != NULL) && + (ub_tp->tp.trans_mode == UBCORE_TP_RM)) ? + ub_tp->tp.tpg->tpgn : + UB_INVALID_TPGN; + udma_cmd_tp_modify_dfx_print(udma_dev, modify_attr); + return; + } + + modify_attr->attr_mask.bs.flag = tp_dfx->mask.flag; + modify_attr->oor_en = tp_dfx->attr.oor_en; + modify_attr->sr_en = tp_dfx->attr.sr_en; + modify_attr->cc_en = tp_dfx->attr.cc_en; + modify_attr->cc_alg = udma_get_ub_cc_alg(udma_dev, tp_dfx->attr.cc_alg); + modify_attr->spray_en = tp_dfx->attr.spray_en; + + modify_attr->attr_mask.bs.cc_pattern_idx = tp_dfx->mask.cc_pattern_idx; + modify_attr->cc_pattern_idx = tp_dfx->attr.cc_pattern_idx; + modify_attr->attr_mask.bs.mtu = tp_dfx->mask.mtu; + modify_attr->mtu = tp_dfx->attr.mtu; + modify_attr->attr_mask.bs.rx_psn = tp_dfx->mask.rx_psn; + modify_attr->rx_psn = tp_dfx->attr.rx_psn; + modify_attr->attr_mask.bs.tx_psn = tp_dfx->mask.tx_psn; + modify_attr->tx_psn = tp_dfx->attr.tx_psn; + modify_attr->attr_mask.bs.peer_tpn = tp_dfx->mask.peer_tpn; + modify_attr->peer_tpn = tp_dfx->attr.peer_tpn; + modify_attr->attr_mask.bs.sq_err_state = tp_dfx->mask.sq_err_state; + modify_attr->sq_err_state = tp_dfx->attr.sq_err_state; + modify_attr->attr_mask.bs.sqa_err_state = tp_dfx->mask.sqa_err_state; + modify_attr->sqa_err_state = tp_dfx->attr.sqa_err_state; + modify_attr->attr_mask.bs.rq_err_state = tp_dfx->mask.rq_err_state; + modify_attr->rq_err_state = tp_dfx->attr.rq_err_state; + modify_attr->attr_mask.bs.rqa_err_state = tp_dfx->mask.rqa_err_state; + modify_attr->rqa_err_state = tp_dfx->attr.rqa_err_state; + udma_cmd_tp_modify_dfx_print(udma_dev, modify_attr); +} + +/* dfx path, used for tool modify tp ctx in specifies field */ +int udma_cmd_tp_modify_dfx(struct udma_device *udma_dev, struct udma_tp *ub_tp, + const struct udma_tp_modify_dfx *tp_dfx) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_modify); + struct ub_cmd_tp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_out = + NULL; /* microcode will return modify success count of need out buf */ + struct tag_cqm_cmd_buf *buf_in = NULL; + u32 *cmd_out_num = NULL; + int ret = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_modify), + &buf_out, sizeof(u32)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp: failed to allocate cqm inbuf\n", + __func__); + return -ENOMEM; + } + + cmd_out_num = (u32 *)buf_out->buf; + tp_cmd = (struct ub_cmd_tp_modify *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_CTX, cmd_len, + 0); //actual xid in body in, here only is default fill 0 + tp_cmd->body.num = 1; + udma_fill_cmd_tp_modify_dfx(udma_dev, ub_tp, tp_dfx, + &tp_cmd->body.attr[0]); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_CTX, buf_in, buf_out, + NULL, NULL); + if (ret != 0 || *cmd_out_num == 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: modify tp ctx failed, ret(%d) modify_num(%d)\n", + __func__, ret, *cmd_out_num); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +static void udma_fill_cmd_tp_cc_modify(struct tag_ub_cmd_tp_cc_modify *tp_cmd, + const struct tag_ub_cc_attr *cc_attr) +{ + tp_cmd->body.attr.inject_mode = cc_attr->cc_inject_mode; + tp_cmd->body.attr.percent = cc_attr->cc_percent; + tp_cmd->body.attr.prm_limit_l = cc_attr->cc_limit_low; + tp_cmd->body.attr.prm_limit_h = cc_attr->cc_limit_high; +} + +int udma_cmd_tp_cc_modify(struct udma_device *udma_dev, + const struct tag_ub_cc_attr *cc_attr) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_cc_modify); + struct tag_ub_cmd_tp_cc_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_cc_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_cc_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_TP_CC, cmd_len, cc_attr->tpn); + udma_fill_cmd_tp_cc_modify(tp_cmd, cc_attr); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_TP_CC, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn(0x%x), ret = %d.\n", + cc_attr->tpn, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "%s: success, tpn(0x%x), inject mode: %u\n", __func__, + cc_attr->tpn, cc_attr->cc_inject_mode); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_modify_tp_srp_bitmap(struct udma_device *udma_dev, + u32 tp_srp_index, u32 bitmap_index, + u32 bitmap_mode) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_srp_modify); + struct tag_ub_cmd_tp_srp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_srp_modify), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "udma_cmd_tp_modify_tp_srp_bitmap failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_srp_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_SRP_BITMAP, cmd_len, 0); + tp_cmd->body.attr.bitmap_index = bitmap_index; + tp_cmd->body.attr.tp_srp_index = tp_srp_index; + tp_cmd->body.attr.bitmap_mode = bitmap_mode; + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_SRP_BITMAP, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret = %d.\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, "modify tp srp bitmap success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_modify_srp_ctr_bitmap(struct udma_device *udma_dev, + u32 ctr_bitmap_index, u32 bitmap_mode) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_srp_modify); + struct tag_ub_cmd_tp_srp_modify *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_srp_modify), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "udma_cmd_tp_modify_srp_ctr_bitmap failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_srp_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_MODIFY_SRP_CTR_BITMAP, cmd_len, 0); + tp_cmd->body.attr.ctr_bitmap_index = ctr_bitmap_index; + tp_cmd->body.attr.bitmap_mode = bitmap_mode; + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_MODIFY_SRP_CTR_BITMAP, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret = %d.\n", ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "modify tp srp ctr bitmap success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_query_cc_info(struct udma_device *udma_dev, u32 tpn, + ub_cc_dfx_info_s *cc_attr) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_cc_query); + ub_cc_dfx_info_s *query_outbuf = NULL; + struct tag_ub_cmd_tp_cc_query *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, sizeof(struct tag_ub_cmd_tp_cc_query), + &buf_out, sizeof(ub_cc_dfx_info_s)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp cc info res: failed to allocate cqm inoutbuf, tpn:0x%x, ret:%u\n", + tpn, ret); + return -ENOMEM; + } + + tp_cmd = (struct tag_ub_cmd_tp_cc_query *)buf_in->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CC_INFO, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CC_INFO, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp cc info res: failed to query tp cc info res, tpn:0x%x, ret:%u\n", + tpn, ret); + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, + buf_out); + return -EBUSY; + } + + query_outbuf = (ub_cc_dfx_info_s *)buf_out->buf; + query_outbuf->timestamp = be32_to_cpu(query_outbuf->timestamp); + query_outbuf->cwnd = be32_to_cpu(query_outbuf->cwnd); + query_outbuf->send_out = be32_to_cpu(query_outbuf->send_out); + query_outbuf->lost_out = be32_to_cpu(query_outbuf->lost_out); + query_outbuf->send_credit = be32_to_cpu(query_outbuf->send_credit); + query_outbuf->send_credit_rate = + be32_to_cpu(query_outbuf->send_credit_rate); + query_outbuf->srtt = be32_to_cpu(query_outbuf->srtt); + query_outbuf->ldcp.acked_seq = + be32_to_cpu(query_outbuf->ldcp.acked_seq); + query_outbuf->ldcp.acked_ce_seq = + be32_to_cpu(query_outbuf->ldcp.acked_ce_seq); + + ret = memcpy_s(cc_attr, sizeof(ub_cc_dfx_info_s), query_outbuf, + sizeof(ub_cc_dfx_info_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(ub_cc_dfx_info_s)); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + return ret; + } + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + udma_info(udma_dev->dev, UDMA_M_TP, "query tp cc info success\n"); + + return 0; +} + +int udma_cmd_tp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct tag_ub_cmd_tp_batch_modify); + struct tag_ub_cmd_tp_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + u32 i; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct tag_ub_cmd_tp_batch_modify), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + modify_cmd = (struct tag_ub_cmd_tp_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_BATCH_MODIFY_CONTEXT, cmd_len, + inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(struct ub_cmd_com_header)); + modify_cmd->body.attr.offset = cpu_to_be32(inbuf->offset); + modify_cmd->body.attr.length = cpu_to_be32(inbuf->length); + /* user input copy to context corresponding position, Note: user input a character occupy 4bit, and user inputs in big-endian format, no need to convert again */ + for (i = inbuf->offset; i < inbuf->offset + inbuf->length && + i < UB_CQM_MODIFY_DATA_LEN_TP / DATALEN_RATIO; + ++i) { + *(((u8 *)&modify_cmd->body.attr.tp_ctx) + i) = + (inbuf->data[(i - inbuf->offset) * DATALEN_RATIO] + << DATA_SHIFT) + + inbuf->data[(i - inbuf->offset) * DATALEN_RATIO + 1]; + } + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_BATCH_MODIFY_CONTEXT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, "%s: success, tpn(%u)\n", + __func__, inbuf->xid); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +void udma_fill_cmd_tp_create_ubrc(struct ub_cmd_tp_create_ubrc *create_cmd, + struct udma_tp *ub_tp) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_create_ubrc); + + create_cmd->body.rc_page_gpa_h = (ub_tp->rc_table.dma_addr >> 40) & + 0xffffff; //high:24bit, 64-24= 40 + create_cmd->body.rc_page_gpa_l = + (u32)(ub_tp->rc_table.dma_addr >> 8); //low address right shift 8 bit + + /* dw0 */ + create_cmd->body.rc_max_size = + ub_tp->rc_table.ext_order - ub_tp->rc_log_ubrc_seg; + + /* dw1 */ + create_cmd->body.rc_entry_size = 1; //32B + create_cmd->body.rc_size = ub_tp->rc_table.order; //size as 128 corresponding is 7 + + if (ub_tp->tp_info != NULL) { + udma_fill_cmd_header(&create_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_CREATE_UBRC, cmd_len, + ub_tp->tp_info->xid); + } + + udma_cpu_to_be32(create_cmd, cmd_len); +} + +int udma_cmd_tp_create_ubrc(struct udma_device *udma_dev, struct udma_tp *ub_tp) +{ + struct ub_cmd_tp_create_ubrc *create_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_tp_create_ubrc), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate cqm inbuf\n"); + return ret; + } + + create_cmd = (struct ub_cmd_tp_create_ubrc *)cqm_cmd_inbuf->buf; + udma_fill_cmd_tp_create_ubrc(create_cmd, ub_tp); + cmdq_rtt = (ub_tp->cfg.tpg == NULL) ? + &ub_tp->cmdq_rtt : + &to_udma_tpg(ub_tp->cfg.tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_CREATE_UBRC, cqm_cmd_inbuf, + NULL, NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp ubrc: failed to send cqm cmd, ret = %d.\n", + ret); + } + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int udma_cmd_tp_query_srp_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_bitmap_outbuf *outbuf, u16 outbuf_size) +{ + struct ub_cmd_tp_query_srp_bitmap_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_srp_bitmap); + struct ub_cmd_tp_query_srp_bitmap *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_tp_query_srp_bitmap), &cqm_cmd_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query_srp_bitmap *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_SRP_BITMAP, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_SRP_BITMAP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn:%u, ret:%d\n", tpn, ret); + goto err_send_cmd; + } + + query_outbuf = + (struct ub_cmd_tp_query_srp_bitmap_outbuf *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query srp bitmap from cache success, tpn:%u\n", tpn); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_query_srp_ctr_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *outbuf, u16 outbuf_size) +{ + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_srp_ctr_bitmap); + struct ub_cmd_tp_query_srp_ctr_bitmap *tp_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_tp_query_srp_ctr_bitmap), &cqm_cmd_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query_srp_ctr_bitmap *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_SRP_CTR_BITMAP, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_SRP_CTR_BITMAP, + cqm_cmd_inbuf, cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret:%d\n", ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *) + cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query srp ctr bitmap from cache success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_query_oor_info(struct udma_device *udma_dev, u32 tpn, u32 msn, + struct ub_oor_tp_info *oor_tp_info, + u32 outbuf_size) +{ + struct ub_cmd_tp_query_oor_info_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_oor_info); + struct ub_cmd_tp_query_oor_info *query_inbuf = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_query_oor_info), &buf_out, + sizeof(struct ub_cmd_tp_query_oor_info_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to allocate cqm inoutbuf, tpn:0x%x\n", + tpn); + return -ENOMEM; + } + + query_outbuf = (struct ub_cmd_tp_query_oor_info_outbuf *)buf_out->buf; + query_inbuf = (struct ub_cmd_tp_query_oor_info *)buf_in->buf; + + query_inbuf->body.attr.msn = msn; + udma_fill_cmd_header(&query_inbuf->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_OOR_INFO, cmd_len, tpn); + udma_cpu_to_be32(query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_OOR_INFO, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp res, tpn:0x%x, ret = %d.\n", + tpn, ret); + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, + buf_out); + return -EBUSY; + } + + ret = memcpy_s(oor_tp_info, sizeof(struct ub_oor_tp_info), + &query_outbuf->oor_tp_info, + sizeof(struct ub_oor_tp_info)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(struct ub_oor_tp_info)); + } + + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_msn_entry(struct udma_device *udma_dev, u32 tpn, + u32 entry_index, + struct ub_msn_entry_outbuf *tp_msn_entry, + u32 outbuf_size) +{ + struct ub_msn_entry_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_msn_entry_info); + struct ub_cmd_tp_query_msn_entry_info *query_inbuf = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_query_msn_entry_info), &buf_out, + sizeof(struct ub_msn_entry_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to allocate cqm inoutbuf, tpn:0x%x\n", + tpn); + return -ENOMEM; + } + + query_outbuf = (struct ub_msn_entry_outbuf *)buf_out->buf; + query_inbuf = (struct ub_cmd_tp_query_msn_entry_info *)buf_in->buf; + + query_inbuf->body.entry_index = entry_index; + udma_fill_cmd_header(&query_inbuf->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_TPMSN_TABLE, cmd_len, tpn); + udma_cpu_to_be32(query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_TPMSN_TABLE, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp res, tpn:0x%x, ret = %d.\n", + tpn, ret); + goto err_send_cmd; + } + + ret = memcpy_s(tp_msn_entry->msn_entry, UB_CMD_TP_MSN_ENTRY_SIZE, + query_outbuf, UB_CMD_TP_MSN_ENTRY_SIZE); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "memcpy failed, size:%u, ret:%d", outbuf_size, ret); + goto err_send_cmd; + } + tp_msn_entry->msn_entry_size = UB_CMD_TP_MSN_ENTRY_SIZE; + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_wqe(struct udma_device *udma_dev, u32 tpn, u32 ci, + struct ub_tp_wqe *outbuf, u32 outbuf_size) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query); + struct ub_cmd_tp_query *tp_cmd = NULL; + struct ub_tp_wqe *query_outbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + sizeof(struct ub_tp_wqe)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query *)cqm_cmd_inbuf->buf; + tp_cmd->body.ci = ci; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_WQE, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_WQE, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn:%u, ci:%u, ret:%d\n", tpn, + ci, ret); + goto err_send_cmd; + } + + query_outbuf = (struct ub_tp_wqe *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp wqe from cache success, tpn:%u, ci:%u\n", tpn, ci); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_query_pcqc(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf, u32 outbuf_size) +{ + ub_pcqc_query_rsp_s *query_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query); + struct ub_cmd_tp_query *tp_cmd = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + (u16)outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_PCQC, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_PCQC, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, tpn:%u, ret = %d.\n", tpn, + ret); + goto err_send_cmd; + } + + query_outbuf = (ub_pcqc_query_rsp_s *)cqm_cmd_outbuf->buf; + ret = memcpy_s((void *)outbuf, outbuf_size, (void *)query_outbuf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp pcqc form cache success, tpn:%u\n", tpn); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +static int udma_tp_fill_cacheout_inbuf(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, u64 wb_dma, + struct tag_cqm_cmd_buf *cqm_cmd_inbuf) +{ + struct udma_tp *ub_tp = container_of(tp[0], struct udma_tp, tp); + struct ub_cmd_tp_cache_invalid *tp_cacheout_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_cache_invalid); + struct udma_comp_priv *comp_priv = NULL; + u32 i; + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get hmm comp priv info\n"); + return -EINVAL; + } + + tp_cacheout_inbuf = + (struct ub_cmd_tp_cache_invalid *)cqm_cmd_inbuf->buf; + tp_cacheout_inbuf->body.tp_cache.sq_buf_len = + ALIGN(ub_tp->buf_size, PAGE_SIZE); + tp_cacheout_inbuf->body.tp_cache.rq_buf_len = + ALIGN(ub_tp->buf_size, PAGE_SIZE); + tp_cacheout_inbuf->body.tp_cache.mtt_flags = 0; + tp_cacheout_inbuf->body.tp_cache.mtt_num = 0; + tp_cacheout_inbuf->body.tp_cache.mtt_cache_line_start = + comp_priv->ub_cap.cmtt_cl_start; + tp_cacheout_inbuf->body.tp_cache.mtt_cache_line_end = + comp_priv->ub_cap.cmtt_cl_end; + tp_cacheout_inbuf->body.tp_cache.mtt_cache_line_size = + comp_priv->ub_cap.cmtt_cl_sz; + tp_cacheout_inbuf->body.tp_cache.gpa_dw.syn_gpa_hi32 = + upper_32_bits(wb_dma); + tp_cacheout_inbuf->body.tp_cache.gpa_dw.syn_gpa_lo32 = + lower_32_bits(wb_dma); + tp_cacheout_inbuf->body.tp_cache.wqe_flags = 0; + tp_cacheout_inbuf->body.tp_cache.wqe_num = 0; + tp_cacheout_inbuf->body.tp_cache.wqe_cache_line_start = + comp_priv->ub_cap.wqe_cl_start; + tp_cacheout_inbuf->body.tp_cache.wqe_cache_line_end = + comp_priv->ub_cap.wqe_cl_end; + tp_cacheout_inbuf->body.tp_cache.wqe_cache_line_size = + comp_priv->ub_cap.wqe_cl_sz; + tp_cacheout_inbuf->body.tp_cnt = cnt; + for (i = 0; i < cnt; i++) + tp_cacheout_inbuf->body.ub_tpn[i] = tp[i]->tpn; + + udma_fill_cmd_header(&tp_cacheout_inbuf->header, UB_CMD_TYPE_TP, + UB_CMD_TP_CACHE_INVLD, cmd_len, ub_tp->tp.tpn); + udma_cpu_to_be32(tp_cacheout_inbuf, cmd_len); + + return 0; +} + +static u32 udma_get_timeout_for_cache_out(enum ub_env_type env_type) +{ +#define UB_DESTROY_FLOW_TIMEOUT 10 +#define UB_DESTROY_FLOW_TIMEOUT_FOR_ESL 1000 + + return env_type == UB_ENV_ESL ? UB_DESTROY_FLOW_TIMEOUT_FOR_ESL : + UB_DESTROY_FLOW_TIMEOUT; +} + +int udma_cmd_tp_send_cache_out(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, u64 wb_dma) +{ + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u64 jiffies_now; + u32 times = 0; + u64 end; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_tp_cache_invalid), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm cmd inoutbuf, ret:%d\n", ret); + return -ENOMEM; + } + + ret = udma_tp_fill_cacheout_inbuf(udma_dev, cnt, tp, wb_dma, + cqm_cmd_inbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed fill tp cacheout inbuf, ret:%d\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + NULL); + return ret; + } + + jiffies_now = get_jiffies_64(); + end = jiffies_now + + (udma_get_timeout_for_cache_out(udma_dev->env_type) * HZ); + while (time_before64(jiffies_now, end)) { + times++; + cmdq_rtt = (tp[0]->tpg == NULL) ? + &to_udma_tp(tp[0])->cmdq_rtt : + &to_udma_tpg(tp[0]->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_CACHE_INVLD, + cqm_cmd_inbuf, NULL, NULL, cmdq_rtt); + if (ret <= 0) + break; + + cond_resched(); + jiffies_now = get_jiffies_64(); + } + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "Failed to send cqm_cmd_box after try times(%u), tpn(%u), ret(%d)\n", + times, tp[0]->tpn, ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_tp_query_cnt(struct udma_device *udma_dev, u32 *tp_state_num, + u32 outbuf_size) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query); + struct ub_cmd_tp_query *tp_cmd = NULL; + int ret; + u32 i; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + (u16)outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CNT, cmd_len, 0); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CNT, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, ret = %d.\n", ret); + goto err_send_cmd; + } + + ret = memcpy_s((void *)tp_state_num, outbuf_size, cqm_cmd_outbuf->buf, + outbuf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%u", + outbuf_size); + goto err_send_cmd; + } + + for (i = UB_TPC_STATE_RST; i < UB_TPC_STATE_MAX; i++) { + tp_state_num[i] = be32_to_cpu(tp_state_num[i]); + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp cnt from cache success\n"); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_set_drop_level(struct udma_device *udma_dev, const u32 tpn, + const u32 drop_level) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_dfx_drop); + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_tp_dfx_drop *tp_cmd = NULL; + struct tag_cqm_object *cqm_obj = NULL; + int ret; + + /* ensure has tpn corresponding to tp context exist */ + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "get cqm object failed, tpn(0x%x)\n", tpn); + return -EINVAL; + } + cqm5_object_put(cqm_obj); + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tp_dfx_drop *)cqm_cmd_inbuf->buf; + tp_cmd->body.drop_level = drop_level; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP, + UB_CMD_TP_SET_DROP_LEVEL, cmd_len, tpn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_SET_DROP_LEVEL, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "failed to send cqm cmd, tpn(0x%x), ret = %d.\n", tpn, + ret); + goto err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: success, tpn(0x%x), drop_level(%u)\n", __func__, tpn, + drop_level); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +static void udma_cmd_tp_destroy_body_fill(u32 cnt, + struct ub_cmd_tp_destroy_body *body, + struct ubcore_tp **tp) +{ + u32 i; + + body->num = cnt; + body->bs.dpath_flag = 1; + + for (i = 0; i < cnt; i++) + body->tp_attr[i].tpn = tp[i]->tpn; +} + +#define UB_DPATH_FLAG_NUM (1 << 2) +#define UB_DESTROY_OUT_NUM \ + (1 + UB_DPATH_FLAG_NUM) //destroy_num(1) + dpath_flag(4) + +/* return value: delete cmd destroy of tp count */ +u32 udma_cmd_tp_destroy_ctx(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u16 cmd_len = sizeof(struct ub_cmd_tp_destroy); + struct ub_cmd_tp_destroy *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + struct cmdq_rtt *cmdq_rtt = NULL; + u32 *cmd_out_num = NULL; + u32 destroy_num = 0; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + return 0; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(struct ub_cmd_tp_destroy), + &buf_out, + sizeof(u64) * UB_DESTROY_OUT_NUM); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: failed to allocate cqm inbuf\n", __func__); + return 0; + } + + tp_cmd_in = (struct ub_cmd_tp_destroy *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_DESTROY_CTX, cmd_len, + 0); //xid fill 0 not affect, actual xid in body in passed in + + udma_cmd_tp_destroy_body_fill(cnt, &tp_cmd_in->body, tp); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + cmd_out_num = (u32 *)buf_out->buf; + + cmdq_rtt = (tp[0]->tpg == NULL) ? &to_udma_tp(tp[0])->cmdq_rtt : + &to_udma_tpg(tp[0]->tpg)->cmdq_rtt; + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_DESTROY_CTX, buf_in, + buf_out, NULL, cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: send cqm cmd failed, ret(%d)\n", __func__, ret); + } + + destroy_num = be32_to_cpu(*cmd_out_num); + if ((destroy_num & UB_DPATH_CHECK_FLAG) != 0) { + ret = udma_cmd_check_dpath_dma( + udma_dev, ((u8 *)buf_out->buf) + sizeof(u64), + UB_DPATH_FLAG_NUM, sizeof(u64)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: check_dpath_dma failed\n", __func__); + } + } + destroy_num = destroy_num & (UB_DPATH_CHECK_FLAG - 1); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + udma_info(udma_dev->dev, UDMA_M_TP, "%s: destroy_num(%u)\n", __func__, + destroy_num); + + return destroy_num; +} + +int udma_cmd_tp_query_ctx(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *outbuf) +{ + struct tag_ub_tpc_query_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_ctx); + struct ub_cmd_tp_query_ctx *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, sizeof(struct ub_cmd_tp_query_ctx), + &buf_out, sizeof(struct tag_ub_tpc_query_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: failed to allocate cqm inoutbuf, key(0x%x)\n", + __func__, key->key); + return -ENOMEM; + } + + query_outbuf = (struct tag_ub_tpc_query_outbuf *)buf_out->buf; + tp_cmd_in = (struct ub_cmd_tp_query_ctx *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CTX, cmd_len, key->key); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CTX, buf_in, buf_out, + NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: query tp ctx failed, key(0x%x), ret(%d)\n", + __func__, key->key, ret); + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, + buf_out); + return -EBUSY; + } + + ret = memcpy_s(outbuf, sizeof(struct ub_dfx_tp_ctx_query), + &query_outbuf->res_val, sizeof(ub_tpc_query_rsp_s)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(struct ub_dfx_tp_ctx_query)); + } + + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_ctx_ext(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_tp_ctx_ext_query *outbuf) +{ + cqm_cmd_buf_s *buf_out = NULL; + cqm_cmd_buf_s *buf_in = NULL; + ub_cmd_tp_query_ctx_ext_s *tp_cmd_in = NULL; + ub_tpc_query_ext_outbuf_s *data_out = NULL; + int ret = 0; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, + sizeof(ub_cmd_tp_query_ctx_ext_s), + &buf_out, + sizeof(ub_tpc_query_ext_outbuf_s)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "[%s] Failed to allocate cqm buffer for command, key(0x%x)\n", + __func__, key->key); + return ret; + } + + tp_cmd_in = (ub_cmd_tp_query_ctx_ext_s *)(buf_in->buf); + data_out = (ub_tpc_query_ext_outbuf_s *)(buf_out->buf); + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CTX_EXT, + sizeof(ub_cmd_tp_query_ctx_ext_s), key->key); + udma_cpu_to_be32(tp_cmd_in, sizeof(ub_cmd_tp_query_ctx_ext_s)); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CTX_EXT, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "[%s] Failed to query extern TP context, key(0x%x), ret(%d).\n", + __func__, key->key, ret); + goto err_out; + } + + ret = memcpy_s(outbuf, sizeof(struct ub_mapt_ctx_query), + &data_out->ret_val, sizeof(struct ub_mapt_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "[%s] Failed to copy memory, size(%zu), ret(%d).\n", + __func__, sizeof(struct ub_mapt_ctx_query), ret); + goto err_out; + } + +err_out: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} + +int udma_cmd_tp_query_ctx_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *outbuf) +{ + struct tag_ub_tp_query_info_outbuf *query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_tp_query_ctx); + struct ub_cmd_tp_query_ctx *tp_cmd_in = NULL; + struct tag_cqm_cmd_buf *buf_out = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &buf_in, sizeof(struct ub_cmd_tp_query_ctx), + &buf_out, sizeof(struct tag_ub_tp_query_info_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: failed to allocate cqm inoutbuf, key(0x%x)\n", + __func__, key->key); + return -ENOMEM; + } + + query_outbuf = (struct tag_ub_tp_query_info_outbuf *)buf_out->buf; + tp_cmd_in = (struct ub_cmd_tp_query_ctx *)buf_in->buf; + + udma_fill_cmd_header(&tp_cmd_in->header, UB_CMD_TYPE_TP, + UB_CMD_TP_QUERY_CTX_FOR_UDMA, cmd_len, key->key); + + udma_cpu_to_be32(tp_cmd_in, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TP_QUERY_CTX_FOR_UDMA, buf_in, + buf_out, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: query tp ctx info failed, key(0x%x), ret(%d)\n", + __func__, key->key, ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + return -EBUSY; + } + udma_be32_to_cpu(query_outbuf, + sizeof(struct tag_ub_tp_query_info_outbuf)); + + ret = memcpy_s(outbuf, sizeof(struct tag_ub_tp_query_info_outbuf), + query_outbuf, + sizeof(struct tag_ub_tp_query_info_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "memcpy failed, size:%lu", + sizeof(struct tag_ub_tp_query_info_outbuf)); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, buf_out); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h new file mode 100644 index 000000000..9615d3085 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tp.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_TP_H +#define UDMA_CMD_TP_H + +#include "udma_common.h" +#include "udma_tp.h" +#include "udma_dfx_cc_inject.h" +#include "ub_pub_cmd.h" + +#define DATALEN_RATIO 2 // user input of data is 4bit, one byte 8bit / 4 = 2 +#define DATA_SHIFT \ + 4 // user input of data is 4bit, to convert to byte unit, even index need left shift 4 bit + +u32 udma_cmd_tp_create_ctx(struct ubcore_device *ub_dev, u32 cnt, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp[]); +u32 udma_cmd_tp_modify_ctx(u32 cnt, struct ubcore_tp **tp, + const struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask); +u32 udma_cmd_tp_destroy_ctx(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp); +int udma_cmd_tp_query_ctx(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *outbuf); +int udma_cmd_tp_cc_modify(struct udma_device *udma_dev, + const struct tag_ub_cc_attr *cc_attr); +int udma_cmd_tp_modify_tp_srp_bitmap(struct udma_device *udma_dev, + u32 tp_srp_index, u32 bitmap_index, + u32 bitmap_mode); +int udma_cmd_tp_modify_srp_ctr_bitmap(struct udma_device *udma_dev, + u32 ctr_bitmap_index, u32 bitmap_mode); +int udma_cmd_tp_query_cc_info(struct udma_device *udma_dev, u32 tpn, + ub_cc_dfx_info_s *cc_attr); +int udma_cmd_tp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_tp_query_srp_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_bitmap_outbuf *outbuf, u16 outbuf_size); +int udma_cmd_tp_query_srp_ctr_bitmap( + struct udma_device *udma_dev, u32 tpn, + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *outbuf, u16 outbuf_size); +int udma_cmd_tp_query_oor_info(struct udma_device *udma_dev, u32 tpn, u32 msn, + struct ub_oor_tp_info *oor_tp_info, + u32 outbuf_size); +int udma_cmd_tp_create_ubrc(struct udma_device *udma_dev, + struct udma_tp *ub_tp); +int udma_cmd_tp_query_wqe(struct udma_device *udma_dev, u32 tpn, u32 ci, + struct ub_tp_wqe *outbuf, u32 outbuf_size); +int udma_cmd_tp_query_pcqc(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf, u32 outbuf_size); +int udma_cmd_tp_send_cache_out(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, u64 wb_dma); +int udma_cmd_tp_query_cnt(struct udma_device *udma_dev, u32 *tp_state_num, + u32 outbuf_size); +int udma_cmd_tp_set_drop_level(struct udma_device *udma_dev, const u32 tpn, + const u32 drop_level); +int udma_cmd_tp_query_ctx_ext(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct ub_tp_ctx_ext_query *outbuf); +int udma_cmd_tp_query_ctx_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *outbuf); +int udma_cmd_tp_query_msn_entry(struct udma_device *udma_dev, u32 tpn, + u32 entry_index, + struct ub_msn_entry_outbuf *tp_msn_entry, + u32 outbuf_size); +int udma_cmd_tp_modify_dfx(struct udma_device *udma_dev, struct udma_tp *ub_tp, + const struct udma_tp_modify_dfx *tp_dfx); +#endif // UDMA_CMD_TP_H diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c new file mode 100644 index 000000000..1db269711 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.c @@ -0,0 +1,333 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "ubcore_uapi.h" +#include "ub_npu_tpg_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" +#include "udma_cmd_tpg.h" + +u32 g_udma_multi_fast_tp_enable = UDMA_MULTI_FAST_TP_DISABLE; +module_param(g_udma_multi_fast_tp_enable, uint, 0664); +MODULE_PARM_DESC(g_udma_multi_fast_tp_enable, + "0 for disable multi fast tp, 1 enable multi fast tp"); + +#define RSV_TP_WQE_CNT 2 +#define WQEBB_IN_TP_WQE_NUM 2 +#define CALC_TP_PCQ_DEPTH(tp_q_depth) \ + ((tp_q_depth) / WQEBB_IN_TP_WQE_NUM - RSV_TP_WQE_CNT) + +int udma_cmd_tpg_query_ctx_from_cache(struct udma_device *udma_dev, u32 tpg_id, + struct ub_tpg_ctx_query *outbuf, + u32 outbuf_size) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_query_inbuf); + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_tpg_query_inbuf *tp_cmd = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, &cqm_cmd_outbuf, + sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_query_inbuf *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_QUERY_CTX, cmd_len, tpg_id); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to send cqm cmd, tpg_id:%u, ret = %d.\n", + tpg_id, ret); + goto err_send_cmd; + } + + (void)memcpy_s((void *)outbuf, outbuf_size, (void *)cqm_cmd_outbuf->buf, + outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_TPG, + "query tpg ctx form cache success, tpg_id:%u\n", tpg_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_tp_latch_tpg_create(struct udma_device *udma_dev, u32 tpgn) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_create); + struct ub_cmd_tpg_create *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, cmd_len, + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Cqm inbuf allocation failed, ret=%d.\n", ret); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_create *)buf_in->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CREATE, cmd_len, tpgn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CREATE, buf_in, NULL, NULL, + NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Send tpg cmd failed, ret = %d.\n", ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, NULL); + + return ret; +} + +int udma_cmd_clear_tp_latch_lock(struct udma_device *udma_dev, u32 xid) +{ + struct ub_cmd_com_header *cmd_buf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_com_header), + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "failed to allocate cqm inbuf, ret = %d\n", ret); + return ret; + } + + cmd_buf = (struct ub_cmd_com_header *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(cmd_buf, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CLEAR_TP_LATCH_LOCK, + sizeof(struct ub_cmd_com_header), xid); + udma_cpu_to_be32(cmd_buf, sizeof(struct ub_cmd_com_header)); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CLEAR_TP_LATCH_LOCK, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "failed to send clear tp latch lock command, ret = %d.\n", + ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int udma_cmd_tpg_create(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_create); + struct ub_cmd_tpg_create *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + u32 i; + u32 tp_pcq_depth; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, cmd_len, + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "cqm inbuf allocation failed.\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_create *)buf_in->buf; + tp_cmd->body.attr.sl = cfg->dscp; + tp_cmd->body.attr.all_tp_num = cfg->tp_cnt; + tp_cmd->body.attr.ep_id = udma_dev->hw_info.ep_id; + + ub_tpg->tp_q_depth = udma_sysfs_get_tp_depth(&udma_dev->sysfs_res); + tp_pcq_depth = CALC_TP_PCQ_DEPTH(ub_tpg->tp_q_depth); + + for (i = 0; i < cfg->tp_cnt; i++) { + tp_cmd->body.attr.tpn[i].bs.tpn = ub_tpg->tpinfo_list[i]->xid; + tp_cmd->body.attr.tpn[i].bs.depth = + tp_pcq_depth; //indicates current depth + } + + if (g_udma_multi_fast_tp_enable == UDMA_MULTI_FAST_TP_ENABLE) { + tp_cmd->body.attr.multi_fast_tp_en = UDMA_MULTI_FAST_TP_ENABLE; + } + + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CREATE, cmd_len, ub_tpg->tpg.tpgn); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CREATE, buf_in, NULL, NULL, + &ub_tpg->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Send tpg cmd failed, ret = %d.\n", ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, NULL); + + return ret; +} + +int udma_cmd_tpg_delete(struct udma_device *udma_dev, u32 idx, + struct cmdq_rtt *rtt) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_destroy); + struct ub_cmd_tpg_destroy *tp_cmd = NULL; + struct tag_cqm_cmd_buf *buf_in = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &buf_in, cmd_len, + NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "cqm inbuf allocation failed.\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_destroy *)buf_in->buf; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_DESTROY, cmd_len, idx); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_DESTROY, buf_in, NULL, + NULL, rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Destroy tpg cmd failed, id: %u, ret = %d.\n", idx, + ret); + ret = -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf_in, NULL); + + return ret; +} + +int udma_cmd_tpg_modify_multipath(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + u16 cmd_len = sizeof(struct ub_cmd_tpg_modify); + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_tpg_modify *tp_cmd = NULL; + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + tp_cmd = (struct ub_cmd_tpg_modify *)cqm_cmd_inbuf->buf; + tp_cmd->body.attr.multi_tp_en = inbuf->multi_tp_en; + tp_cmd->body.attr.default_tpn = inbuf->default_tpn; + tp_cmd->body.attr.dw0_value = 0; + udma_fill_cmd_header(&tp_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_CTX_SELECT_TP, cmd_len, inbuf->tpg_id); + udma_cpu_to_be32(tp_cmd, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_CTX_SELECT_TP, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "failed to send cqm cmd, ret = %d.\n", ret); + goto out; + } + udma_info(udma_dev->dev, UDMA_M_TPG, + "query tp cnt from cache success\n"); + +out: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +static void +udma_fill_cmd_tpg_batch_modify_ctx(struct ub_cmd_tpg_batch_modify_body *body, + const struct ub_cqm_modify_inbuf *inbuf) +{ + body->attr.offset = inbuf->offset; + body->attr.length = inbuf->length; + + udma_cpu_to_be32(body, sizeof(struct ub_cmd_tpg_batch_modify_body)); + + udma_cmdq_data_to_context_buf(inbuf->data, inbuf->data_len, + (u8 *)&body->attr.tpg_ctx, + sizeof(body->attr.tpg_ctx) / sizeof(u8), + inbuf->offset); +} + +int udma_cmd_tpg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_tpg_batch_modify); + struct ub_cmd_tpg_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev->device_type < UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "%s: device_type(%d) not support tpg batch modify ctx\n", + __func__, udma_dev->device_type); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_tpg_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_TP_GROUP, + UB_CMD_TPG_BATCH_MODIFY_CONTEXT, cmd_len, + inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(modify_cmd->header)); + + udma_fill_cmd_tpg_batch_modify_ctx(&modify_cmd->body, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_TPG_BATCH_MODIFY_CONTEXT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto batch_err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_TPG, "%s: success, tpn(%u)\n", __func__, + inbuf->xid); + +batch_err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h new file mode 100644 index 000000000..df822ba2f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_tpg.h @@ -0,0 +1,32 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_TPG_H +#define UDMA_CMD_TPG_H + +#include "ubcore_types.h" +#include "udma_tpg.h" +#include "ub_npu_tpg_cmd_defs.h" +#include "ub_pub_cmd.h" +#include "udma_common.h" + +#define UB_TP_PCQ_INIT_DEPTH 62u +enum { UDMA_MULTI_FAST_TP_DISABLE = 0, UDMA_MULTI_FAST_TP_ENABLE }; + +int udma_cmd_tpg_query_ctx_from_cache(struct udma_device *udma_dev, u32 tpg_id, + struct ub_tpg_ctx_query *outbuf, + u32 outbuf_size); +int udma_cmd_tpg_create(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg); +int udma_cmd_tpg_delete(struct udma_device *udma_dev, u32 idx, + struct cmdq_rtt *rtt); +int udma_cmd_tpg_modify_multipath(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int udma_cmd_tpg_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); +int udma_cmd_tp_latch_tpg_create(struct udma_device *udma_dev, u32 tpgn); +int udma_cmd_clear_tp_latch_lock(struct udma_device *udma_dev, u32 xid); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c new file mode 100644 index 000000000..2043cf66b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "ubcore_uapi.h" +#include "ub_npu_utp_cmd.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_net.h" +#include "udma_utp.h" +#include "udma_cmd_com.h" +#include "udma_cmd_utp.h" + +#define UB_CMD_UTP_DIP_MASK 0xffffffff +#define UB_CMD_UTP_DIP_OFFSET 0x20 +#define UB_CMD_UTP_TCLASS_DSCP 0x3f +#define UB_CMD_UTP_TCLASS_ECN 0x2 + +static void udma_fill_create_utp_cmd(struct udma_device *udma_dev, + struct tag_ub_create_utp_attr *utp_attr, + const struct ubcore_utp_cfg *cfg) +{ + vlan_u vlan_value = { 0 }; + struct ub_sip_attr sip = { 0 }; + u8 cos = 0; + + /* dw0 */ + utp_attr->dmac_l32 = UDMA_GET_MAC_LO(cfg->peer_net_addr.mac[0x2], + cfg->peer_net_addr.mac[0x3], + cfg->peer_net_addr.mac[0x4], + cfg->peer_net_addr.mac[0x5]); + /* dw1 */ + utp_attr->dmac_h16 = UDMA_GET_MAC_HI(cfg->peer_net_addr.mac[0], + cfg->peer_net_addr.mac[1]); + utp_attr->port_id = cfg->port_id; + utp_attr->mtu = cfg->mtu; + + /* dw2 - dw5 */ + if (cfg->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + utp_attr->dip3 = cfg->peer_net_addr.net_addr.in4.addr; + } else { + utp_attr->dip0 = cfg->peer_net_addr.net_addr.in6.subnet_prefix & + UB_CMD_UTP_DIP_MASK; + utp_attr->dip1 = + (cfg->peer_net_addr.net_addr.in6.subnet_prefix >> + UB_CMD_UTP_DIP_OFFSET) & + UB_CMD_UTP_DIP_MASK; + utp_attr->dip2 = cfg->peer_net_addr.net_addr.in6.interface_id & + UB_CMD_UTP_DIP_MASK; + utp_attr->dip3 = + (cfg->peer_net_addr.net_addr.in6.interface_id >> + UB_CMD_UTP_DIP_OFFSET) & + UB_CMD_UTP_DIP_MASK; + } + utp_attr->dip0 = be32_to_cpu(utp_attr->dip0); + utp_attr->dip1 = be32_to_cpu(utp_attr->dip1); + utp_attr->dip2 = be32_to_cpu(utp_attr->dip2); + utp_attr->dip3 = be32_to_cpu(utp_attr->dip3); + + /* dw6 */ + utp_attr->dip_type = + (cfg->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) ? 0 : 1; + utp_attr->loop_back = cfg->flag.bs.loopback; + + /* dw7 */ + vlan_value.value = (u16)cfg->peer_net_addr.vlan; + utp_attr->vlan_pri = + cfg->dscp; // pcp scenario next, user passed in dscp field treated as pri + utp_attr->vlan_cfi = vlan_value.bs.cfi; + utp_attr->vlan_id = vlan_value.bs.vlan; + utp_attr->hoplmt = cfg->hop_limit; + + /* dw8 */ + utp_attr->flow_label = cfg->flow_label; + utp_attr->tclass = (cfg->dscp & UB_CMD_UTP_TCLASS_DSCP) + << UB_CMD_UTP_TCLASS_ECN | + UB_CMD_UTP_TCLASS_ECN; + /* get fails then use priority 0, stub sip field */ + sip.dw6.tag = VLAN_VALID; + sip.dw6.dw6_value = le32_to_cpu(sip.dw6.dw6_value); + if (udma_get_dcb_cfg_cos(udma_dev, &sip, cfg->dscp, &cos, 0) == 0) { + utp_attr->cos = cos; + } + + /* dw9 */ + utp_attr->sip_idx = cfg->local_net_addr_idx; +} + +int udma_cmd_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx) +{ + struct tag_ub_cmd_query_utp *utp_query_inbuf = NULL; + struct ub_dip_ctx_query *utpc = NULL; + struct tag_cqm_cmd_buf *cmd_inbuf = NULL; + struct tag_cqm_cmd_buf *cmd_outbuf = NULL; + u16 cmd_len = sizeof(struct tag_ub_cmd_query_utp); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cmd_inbuf, + (u16)sizeof(struct tag_ub_cmd_query_utp), &cmd_outbuf, + (u16)sizeof(struct ub_dip_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Failed to alloc in_buffer, ret = %d.\n", ret); + return ret; + } + + utp_query_inbuf = (struct tag_ub_cmd_query_utp *)cmd_inbuf->buf; + udma_fill_cmd_header(&utp_query_inbuf->header, UB_CMD_TYPE_UTP, + UB_CMD_QUERY_DIP, cmd_len, utpn); + udma_cpu_to_be32(utp_query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_QUERY_DIP, cmd_inbuf, + cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Send query utp cmd msg failed: %d\n", ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, + cmd_outbuf); + return ret; + } + + utpc = (struct ub_dip_ctx_query *)cmd_outbuf->buf; + ret = memcpy_s(utp_ctx, sizeof(struct ub_dip_ctx_query), utpc, + sizeof(struct ub_dip_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, "memcpy_s failed, ret:%d\n", + ret); + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, + cmd_outbuf); + return -EFAULT; + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, cmd_outbuf); + + return ret; +} + +int udma_cmd_create_utp(struct udma_device *udma_dev, + const struct ubcore_utp_cfg *cfg, struct udma_utp *utp) +{ + struct tag_ub_cmd_create_utp *utp_create_inbuf = NULL; + struct tag_cqm_cmd_buf *cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct tag_ub_cmd_create_utp); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cmd_inbuf, + (u16)sizeof(struct tag_ub_cmd_create_utp), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Failed to alloc in_buffer, ret = %d.\n", ret); + return ret; + } + + utp_create_inbuf = (struct tag_ub_cmd_create_utp *)cmd_inbuf->buf; + udma_fill_create_utp_cmd(udma_dev, &utp_create_inbuf->body.utp_attr, + cfg); + udma_fill_cmd_header(&utp_create_inbuf->header, UB_CMD_TYPE_UTP, + UB_CMD_ADD_DIP, cmd_len, utp->cqm_utpn->index); + udma_cpu_to_be32(utp_create_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_ADD_DIP, cmd_inbuf, NULL, NULL, + NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_UTP, + "Send create utp cmd msg failed: %d\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_destroy_utp(struct udma_device *udma_dev, struct udma_utp *utp) +{ + struct tag_ub_cmd_destroy_utp *utp_destroy_inbuf = NULL; + struct tag_cqm_cmd_buf *cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct tag_ub_cmd_destroy_utp); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cmd_inbuf, + (u16)sizeof(struct tag_ub_cmd_destroy_utp), NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Failed to alloc in_buffer, ret = %d.\n", ret); + return ret; + } + + utp_destroy_inbuf = (struct tag_ub_cmd_destroy_utp *)cmd_inbuf->buf; + udma_fill_cmd_header(&utp_destroy_inbuf->header, UB_CMD_TYPE_UTP, + UB_CMD_DEL_DIP, cmd_len, utp->cqm_utpn->index); + udma_cpu_to_be32(utp_destroy_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_DEL_DIP, cmd_inbuf, NULL, NULL, + NULL); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_UTP, + "Send destroy dip cmd msg failed: %d\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h new file mode 100644 index 000000000..35fc73f98 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_utp.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_CMD_UTP_H +#define UDMA_CMD_UTP_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_net.h" +#include "udma_utp.h" + +int udma_cmd_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx); +int udma_cmd_create_utp(struct udma_device *udma_dev, + const struct ubcore_utp_cfg *cfg, struct udma_utp *utp); +int udma_cmd_destroy_utp(struct udma_device *udma_dev, struct udma_utp *utp); + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c new file mode 100644 index 000000000..47608237b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.c @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include "ubcore_uapi.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_vtp.h" +#include "udma_comp.h" +#include "ub_npu_vtp_cmd.h" +#include "udma_misc.h" +#include "udma_cmd_com.h" +#include "udma_cqm.h" +#include "udma_cmd_vtp.h" + +int udma_get_rc_tpn(struct udma_device *udma_dev, struct ubcore_tpg *tpg) +{ + u32 i; + + if (tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, "tpg is null\n"); + return -EINVAL; + } + + for (i = 0; (i < tpg->tpg_cfg.tp_cnt) && (i < UBCORE_MAX_TP_CNT_IN_GRP); + i++) { + if (tpg->tp_list[i] == NULL) { + udma_warn(udma_dev->dev, UDMA_M_VTP, + "tp_list(%u) in tpg(%u) is null\n", i, + tpg->tpgn); + continue; + } + + udma_info(udma_dev->dev, UDMA_M_VTP, + "tp_list(%u) ack_resp: %u\n", i, + tpg->tp_list[i]->flag.bs.ack_resp); + if (tpg->tp_list[i]->flag.bs.ack_resp == 0) { + return (int)tpg->tp_list[i]->tpn; + } + } + + return -EINVAL; +} + +static int udma_fill_cmd_vtp_create_body(struct udma_device *udma_dev, + struct ub_cmd_body_vtp_create *body, + const struct ubcore_vtp_cfg *cfg) +{ + int tpn; + u32 i; + + body->func_id = adapted_ue_idx(cfg); + (void)memcpy_s((void *)body->seid, UBCORE_EID_SIZE, cfg->local_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + body->seid[i] = be32_to_cpu(body->seid[i]); + (void)memcpy_s((void *)body->deid, UBCORE_EID_SIZE, cfg->peer_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + body->deid[i] = be32_to_cpu(body->deid[i]); + body->upi = cfg->upi; + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + body->type = UB_VTP_TYPE_TPG; + body->id = cfg->tpg->tpgn; + break; + case UBCORE_TP_RC: + tpn = udma_get_rc_tpn(udma_dev, cfg->tpg); + if (tpn < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get rc tpn failed\n"); + return -EINVAL; + } + body->type = UB_VTP_TYPE_TP; + body->id = (u32)tpn; + break; + case UBCORE_TP_UM: + body->type = UB_VTP_TYPE_DIP; + body->id = cfg->utp->utpn; + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, "Unknown trans_mode: %d\n", + cfg->trans_mode); + return -EINVAL; + } + + return 0; +} + +int udma_cmd_create_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt) +{ + struct ub_cmd_vtp_create *vtp_create_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_body_vtp_create *body = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_create); + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, "Recv create vtp: vtpn:%u vf:%u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_vtp_create), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for vtp_create_inbuf, ret = %d, vtpn: %u, vfid: %u\n", + ret, cfg->vtpn, adapted_ue_idx(cfg)); + return ret; + } + + vtp_create_inbuf = (struct ub_cmd_vtp_create *)cqm_cmd_inbuf->buf; + + udma_fill_cmd_header(&vtp_create_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_CREATE_CTX, cmd_len, cfg->vtpn); + + body = &vtp_create_inbuf->body; + ret = udma_fill_cmd_vtp_create_body(udma_dev, body, cfg); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to create vtp body cmdq msg, vtpn: %u, vfid: %u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + goto out; + } + + udma_cpu_to_be32(vtp_create_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_CREATE_CTX, cqm_cmd_inbuf, + NULL, NULL, rtt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Send create vtp cmd msg failed: %d, vtpn: %u, vfid: %u\n", + ret, cfg->vtpn, adapted_ue_idx(cfg)); + goto out; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, + "create vtpn cmd success, vtpn: %u, vfid: %u, id: %u\n", + cfg->vtpn, adapted_ue_idx(cfg), be32_to_cpu(body->id)); + +out: + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +void udma_fill_cmd_vtp_restore_body_node( + struct udma_device *udma_dev, struct udma_vtp_buf *buf, + struct ub_cmd_restore_vtp_node *body_node) +{ + u32 idx; + u64 node_size = sizeof(struct ub_cmd_restore_vtp_node); + struct ub_cmd_restore_vtp_node *src_node = NULL; + + src_node = (struct ub_cmd_restore_vtp_node *)((u8 *)buf->addr + + buf->cur_idx * node_size); + for (idx = buf->cur_idx; idx < UB_MAX_VTP_NUM; idx++) { + if (src_node->vld == UDMA_MIG_BUF_USED) { + src_node->vld = UDMA_MIG_BUF_IDLE; + (void)memcpy_s((void *)body_node, node_size, src_node, + node_size); + buf->cur_idx = idx; + buf->total--; + return; + } + src_node++; + } +} + +int udma_fill_vtp_node(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ub_cmd_restore_vtp_node *node) +{ + int tpn, i; + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + node->type = UB_VTP_TYPE_TPG; + node->id = cfg->tpg->tpgn; + break; + case UBCORE_TP_RC: + tpn = udma_get_rc_tpn(udma_dev, cfg->tpg); + if (tpn < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "get rc tpn failed, ret:%d\n", tpn); + return -EINVAL; + } + node->type = UB_VTP_TYPE_TP; + node->id = (u32)tpn; + break; + case UBCORE_TP_UM: + node->type = UB_VTP_TYPE_DIP; + node->id = cfg->utp->utpn; + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, + "Unknown trans_mode:%d, vf:%u vtpn:%u\n", + cfg->trans_mode, adapted_ue_idx(cfg), cfg->vtpn); + return -EINVAL; + } + node->vtpn = cfg->vtpn; + node->destroy = UDMA_RESTORE_VTPN_CREATE; + + (void)memcpy_s((void *)node->seid, UBCORE_EID_SIZE, cfg->local_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + node->seid[i] = be32_to_cpu(node->seid[i]); + (void)memcpy_s((void *)node->deid, UBCORE_EID_SIZE, cfg->peer_eid.raw, + UBCORE_EID_SIZE); + for (i = 0; i < UB_EID_SW_DW_LEN; i++) + node->deid[i] = be32_to_cpu(node->deid[i]); + return 0; +} + +static inline void udma_cmd_init_inoutbuf(struct ub_cmd_vtp_restore *inbuf, + u32 vfid) +{ + u16 cmd_len = sizeof(struct ub_cmd_vtp_restore); + + udma_fill_cmd_header(&inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_RESTORE_CTX, cmd_len, 0); + inbuf->body.vfid = vfid; + inbuf->body.num = 0; +} + +static int udma_cmd_send_restore_ctx(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, + struct ub_cmd_vtp_restore *inbuf) +{ + int ret; + + udma_cpu_to_be32(inbuf, sizeof(struct ub_cmd_vtp_restore)); + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_RESTORE_CTX, + buf->cqm_cmd_buf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "send restore vtp cmd msg failed: %d\n", ret); + } + udma_cmd_init_inoutbuf(inbuf, buf->func_id); + return ret; +} + +static inline int +udma_cmd_check_body_full_proc(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, + struct ub_cmd_vtp_restore *inbuf) +{ + if (inbuf->body.num >= UB_VTP_RESTORE_NUM_MAX) { + return udma_cmd_send_restore_ctx(udma_dev, buf, inbuf); + } + return 0; +} + +int udma_cmd_modify_get_new_tpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ubcore_vtp_attr *attr, u32 *new_tpn) +{ + int ret; + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + *new_tpn = attr->tp.tpg->tpgn; + break; + case UBCORE_TP_RC: + ret = udma_get_rc_tpn(udma_dev, attr->tp.tpg); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get rc tpn failed, vf:%u vtpn:%u\n", + adapted_ue_idx(cfg), cfg->vtpn); + return -EINVAL; + } + *new_tpn = (u32)ret; + break; + case UBCORE_TP_UM: + *new_tpn = attr->tp.utp->utpn; + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify: invalid transmode:0x%x, vf:%u vtpn:%u\n", + cfg->trans_mode, adapted_ue_idx(cfg), cfg->vtpn); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_VTP, "New tpn:0x%x. vtpn:%u vfid:%u\n", + *new_tpn, cfg->vtpn, adapted_ue_idx(cfg)); + return 0; +} + +// full VTP flush down complete, received incremental create VTP request +int udma_cmd_restore_vtp_create_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + // create vtp, entry checks cfg in of parameters whether valid + if (udma_fill_vtp_node(udma_dev, cfg, + &inbuf->body.nodes[inbuf->body.num]) != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, "fill vtp node failed\n"); + return -EINVAL; + } + inbuf->body + .num++; // reaching 32 then will flush down, this count convenient will return 0; nodes to be flushed to microcode stored in order + + return udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); +} + +// full VTP flush down complete, received incremental destroy VTP request, convert to create operation +int udma_cmd_restore_vtp_modify_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf, + struct ubcore_vtp_attr *attr) +{ + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + u32 new_tpn; + int ret; + + ret = udma_cmd_modify_get_new_tpn(udma_dev, cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify vtpn: get new tpn failed, ret:%d.\n", ret); + return ret; + } + if (udma_fill_vtp_node(udma_dev, cfg, + &inbuf->body.nodes[inbuf->body.num]) != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, "fill vtp node failed\n"); + return -EINVAL; + } + inbuf->body.nodes[inbuf->body.num].id = new_tpn; + inbuf->body.num++; + + return udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); +} + +// full VTP flush down complete, received incremental destroy VTP request +int udma_cmd_restore_vtp_destroy_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + ret = udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "destroy vtpn: send restore vtp cmd msg failed: %d\n", + ret); + return ret; + } + + inbuf->body.nodes[inbuf->body.num].vtpn = + vtp->cfg.vtpn; // only care about vtpn and destroy i.e. can, no need to care about other parameters + inbuf->body.nodes[inbuf->body.num].destroy = + UDMA_RESTORE_VTPN_DESTROY; // matching microcode processing + inbuf->body + .num++; // nodes to be flushed to microcode stored in order, 32(count word) elements may contain both chain-delete and chain-create requests + + return udma_cmd_check_body_full_proc(udma_dev, buf, inbuf); +} + +int udma_cmd_restore_vtp_create_multi_vld(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, u32 num) +{ + int ret; + u32 i; + + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + inbuf->body.num = num; + for (i = 0; i < num; i++) { + udma_fill_cmd_vtp_restore_body_node(udma_dev, buf, + &inbuf->body.nodes[i]); + } + udma_cpu_to_be32(inbuf, sizeof(struct ub_cmd_vtp_restore)); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_RESTORE_CTX, cqm_cmd_buf, + NULL, NULL, NULL); + if (ret != 0) + udma_err( + udma_dev->dev, UDMA_M_VTP, + "create all vld: send restore vtp cmd msg failed: %d\n", + ret); + udma_cmd_init_inoutbuf(inbuf, buf->func_id); + + return ret; +} + +int udma_cmd_restore_vtp_remain_vtpn(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_buf = + (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + struct ub_cmd_vtp_restore *inbuf = + (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + + if (inbuf->body.num != 0) { + return udma_cmd_send_restore_ctx(udma_dev, buf, inbuf); + } + return 0; +} + +void *udma_cmd_restore_vtp_get_cqm_cmd_buf(struct udma_device *udma_dev, + u32 vfid) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_buf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_restore); + struct ub_cmd_vtp_restore *vtp_restore_inbuf = NULL; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_buf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for vtp_restore_inbuf, ret = %d.\n", + ret); + return NULL; + } + + vtp_restore_inbuf = (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + udma_fill_cmd_header(&vtp_restore_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_RESTORE_CTX, cmd_len, 0); + vtp_restore_inbuf->body.vfid = vfid; + + return cqm_cmd_buf; +} + +int udma_cmd_query_vtp(struct udma_device *udma_dev, u32 vfid, u32 vtp_id, + struct ub_vtp_ctx_query *outbuf) +{ + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + struct ub_cmd_vtp_query *vtp_query_inbuf = NULL; + struct ub_vtp_ctx_query *vtp_query_outbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_query); + int ret; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + sizeof(struct ub_cmd_vtp_query), + &cqm_cmd_outbuf, + sizeof(struct ub_vtp_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "failed to allocate cqm inbuf\n"); + return -ENOMEM; + } + + vtp_query_inbuf = (struct ub_cmd_vtp_query *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&vtp_query_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_QUERY_CTX, cmd_len, vtp_id); + + vtp_query_inbuf->body.func_id = vfid; + + udma_cpu_to_be32(vtp_query_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_QUERY_CTX, cqm_cmd_inbuf, + cqm_cmd_outbuf, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "failed to send cqm cmd, vtp_id:%u, ret = %d.\n", + vtp_id, ret); + goto err_send_cmd; + } + + vtp_query_outbuf = (struct ub_vtp_ctx_query *)cqm_cmd_outbuf->buf; + (void)memcpy_s((void *)outbuf, sizeof(struct ub_vtp_ctx_query), + (void *)vtp_query_outbuf, + sizeof(struct ub_vtp_ctx_query)); + + udma_info(udma_dev->dev, UDMA_M_VTP, + "query vtp ctx form cache success, vtp_id:%u, func_id:%u\n", + vtp_id, vtp_query_inbuf->body.func_id); + +err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, + cqm_cmd_outbuf); + + return ret; +} + +int udma_cmd_destroy_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp) +{ + struct ub_cmd_vtp_destory *vtp_destroy_inbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_destory); + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Recv destroy vtp, vtpn: %u, vfid: %u\n", vtp->cfg.vtpn, + adapted_ue_idx(&vtp->cfg)); + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udma_dev->hwdev, &cqm_cmd_inbuf, + (u16)sizeof(struct ub_cmd_vtp_destory), NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for destroy vtp, ret = %d.\n", + ret); + return ret; + } + + vtp_destroy_inbuf = (struct ub_cmd_vtp_destory *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&vtp_destroy_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_DESTORY_CTX, cmd_len, vtp->cfg.vtpn); + + vtp_destroy_inbuf->body.func_id = adapted_ue_idx(&vtp->cfg); + + udma_cpu_to_be32(vtp_destroy_inbuf, cmd_len); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_DESTORY_CTX, cqm_cmd_inbuf, + NULL, NULL, &to_udma_vtp(vtp)->cmdq_rtt); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_VTP, + "Send destroy vtp cmd msg failed: %d\n", ret); + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} + +int udma_cmd_send_modify(struct udma_device *udma_dev, u32 vfid, u32 vtpn, + u32 new_tpn) +{ + int ret; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + u16 cmd_len = sizeof(struct ub_cmd_vtp_modify); + struct ub_cmd_vtp_modify *vtp_modify_inbuf = NULL; + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to alloc in_buffer for vtp_modify_inbuf, ret=%d.\n", + ret); + return ret; + } + vtp_modify_inbuf = (struct ub_cmd_vtp_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&vtp_modify_inbuf->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_MODIFY_CTX, cmd_len, vtpn); + vtp_modify_inbuf->body.func_id = vfid; + vtp_modify_inbuf->body.index = new_tpn; + + udma_cpu_to_be32(vtp_modify_inbuf, cmd_len); + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_MODIFY_CTX, cqm_cmd_inbuf, + NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Send modify vtp cmd msg failed, ret=%d\n", ret); + } + + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + return ret; +} + +int udma_cmd_modify_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr) +{ + u32 new_tpn; + int ret; + + ret = udma_cmd_modify_get_new_tpn(udma_dev, &vtp->cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify vtp: get new tpn failed, ret:%d vf:%u.\n", ret, + adapted_ue_idx(&vtp->cfg)); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_VTP, + "Modify vtp: vf:%u vtpn:%u tpn:%u\n", + adapted_ue_idx(&vtp->cfg), vtp->cfg.vtpn, new_tpn); + + return udma_cmd_send_modify(udma_dev, adapted_ue_idx(&vtp->cfg), + vtp->cfg.vtpn, new_tpn); +} + +static void +udma_fill_cmd_vtp_batch_modify_ctx(struct ub_cmd_vtp_batch_modify_body *body, + const struct ub_cqm_modify_inbuf *inbuf) +{ + body->attr.offset = inbuf->offset; + body->attr.length = inbuf->length; + body->func_id = inbuf->vf_id; + + udma_cpu_to_be32(body, sizeof(struct ub_cmd_vtp_batch_modify_body)); + + udma_cmdq_data_to_context_buf(inbuf->data, inbuf->data_len, + (u8 *)&body->attr.vtp_ctx, + sizeof(body->attr.vtp_ctx) / sizeof(u8), + inbuf->offset); +} + +int udma_cmd_vtp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + u16 cmd_len = sizeof(struct ub_cmd_vtp_batch_modify); + struct ub_cmd_vtp_batch_modify *modify_cmd = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + int ret; + + if (udma_dev->device_type < UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "%s: device_type(%d) not support vtp batch modify ctx\n", + __func__, udma_dev->device_type); + return -EINVAL; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf(udma_dev->hwdev, &cqm_cmd_inbuf, + cmd_len, NULL, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "%s: cqm inbuf allocation failed\n", __func__); + return -ENOMEM; + } + + modify_cmd = (struct ub_cmd_vtp_batch_modify *)cqm_cmd_inbuf->buf; + udma_fill_cmd_header(&modify_cmd->header, UB_CMD_TYPE_VTP, + UB_CMD_VTP_BATCH_MODIFY_CONTEXT, cmd_len, + inbuf->xid); + udma_cpu_to_be32(modify_cmd, sizeof(modify_cmd->header)); + + udma_fill_cmd_vtp_batch_modify_ctx(&modify_cmd->body, inbuf); + + ret = udma_send_cmd_box(udma_dev, UB_CMD_VTP_BATCH_MODIFY_CONTEXT, + cqm_cmd_inbuf, NULL, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "%s: send cmd failed, ret(%d)\n", __func__, ret); + goto batch_err_send_cmd; + } + + udma_info(udma_dev->dev, UDMA_M_VTP, "%s: success, tpn(%u)\n", __func__, + inbuf->xid); + +batch_err_send_cmd: + (void)udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, cqm_cmd_inbuf, NULL); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h new file mode 100644 index 000000000..7889b7717 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/cmdq/udma_cmd_vtp.h @@ -0,0 +1,57 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_CMD_VTP_H +#define UDMA_CMD_VTP_H + +#include <linux/types.h> + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" + +#define UB_MAX_VTP_NUM (128 * 1024) + +typedef void (*udma_cmd_restore_vtp_handler)(struct udma_device *udma_dev, + struct ub_cmd_vtp_restore *inbuf, + struct udma_vtp_buf *buf, + void *para); + +int udma_fill_vtp_node(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ub_cmd_restore_vtp_node *node); +int udma_cmd_restore_vtp_create_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf); +int udma_cmd_restore_vtp_modify_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf, + struct ubcore_vtp_attr *attr); +int udma_cmd_restore_vtp_destroy_vtpn(struct udma_device *udma_dev, + const struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf); +int udma_cmd_restore_vtp_create_multi_vld(struct udma_device *udma_dev, + struct udma_vtp_buf *buf, u32 num); +int udma_cmd_restore_vtp_remain_vtpn(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +void *udma_cmd_restore_vtp_get_cqm_cmd_buf(struct udma_device *udma_dev, + u32 vfid); +int udma_cmd_create_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt); +int udma_cmd_query_vtp(struct udma_device *udma_dev, u32 vfid, u32 vtp_id, + struct ub_vtp_ctx_query *outbuf); +int udma_cmd_destroy_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp); +int udma_get_rc_tpn(struct udma_device *udma_dev, struct ubcore_tpg *tpg); +int udma_cmd_modify_get_new_tpn(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct ubcore_vtp_attr *attr, u32 *new_tpn); +int udma_cmd_send_modify(struct udma_device *udma_dev, u32 vfid, u32 vtpn, + u32 new_tpn); +int udma_cmd_modify_vtp(struct udma_device *udma_dev, struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr); +int udma_cmd_vtp_batch_modify_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h new file mode 100644 index 000000000..9764af30f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_abi.h @@ -0,0 +1,62 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_ABI_H +#define UDMA_ABI_H + +#include <linux/types.h> +#include "udma_common.h" + +typedef struct ub_create_ctx_resp { + u32 async_fd; + u32 num_comp_vectors; + u32 cap_flags; + u64 db_addr; + u32 max_jfc; + u32 max_jfs; + u32 max_jfr; + u32 max_jetty; + u32 max_jfs_inline_size; + u32 max_jfs_sge; + u32 max_jfs_rsge; + u32 max_jfr_sge; + u32 dwqe_size; + u16 db_offset; + u16 dwqe_offset; + u32 dwqe_dis : 1; + u32 mtt_pro_en : 1; + u32 debug_en : 1; + u32 dcs_en : 1; + u32 index_queue_en : 1; + u32 fis_en : 1; + u32 rsv : 26; + enum udma_device_type device_type; + enum ub_env_type env_type; +} ub_create_ctx_resp_t; + +struct ub_create_jetty_ucmd { + u64 buf_addr; + u64 db_addr; + u64 sdb_addr; + u32 buf_size; + u32 jfr_id; + u64 index_q_buf_addr; + u64 srq_sdb_addr; + u32 index_q_buf_size; + u32 sq_wqebb_cnt; + u32 rsvd[32]; +}; + +struct ub_modify_jetty_uinfo { + u32 id; + u32 sq_pi; +}; + +struct ub_create_jetty_resp { + u32 cos; + u32 slice; + u32 jetty_round; + u8 pattern; +}; + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c new file mode 100644 index 000000000..1d102eb15 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.c @@ -0,0 +1,84 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#include "udma_log.h" +#include "udma_bitmap.h" + +struct udma_bitmap *udma_bitmap_alloc(u32 bitmap_size) +{ + struct udma_bitmap *bitmap = NULL; + + bitmap = (struct udma_bitmap *)kzalloc(sizeof(struct udma_bitmap), + GFP_KERNEL); + if (bitmap == NULL) { + udma_pr_err(UDMA_M_BITMAP, "%s: unable to alloc bitmap", + __func__); + return NULL; + } + + spin_lock_init(&bitmap->lock); + bitmap->bits = (unsigned long *)kzalloc( + BITS_TO_LONGS(bitmap_size) * sizeof(unsigned long), GFP_KERNEL); + if (bitmap->bits == NULL) { + udma_pr_err(UDMA_M_BITMAP, "%s: unable to alloc bitmap->bits", + __func__); + kfree(bitmap); + return NULL; + } + + bitmap->size = bitmap_size; + return bitmap; +} + +void udma_bitmap_free(struct udma_bitmap *bitmap) +{ + if (bitmap != NULL) { + kfree(bitmap->bits); + bitmap->bits = NULL; + kfree(bitmap); + } +} + +u32 udma_bitmap_alloc_id(struct udma_bitmap *bitmap) +{ + u32 index = 0; + u32 max_num = bitmap->size; + u32 offset = bitmap->last; + unsigned long *table = bitmap->bits; + + spin_lock(&bitmap->lock); + + /* Search for an idle bit from the offset position. */ + index = (u32)find_next_zero_bit(table, max_num, offset); + /* The preceding search fails. Search for an idle bit + * from the beginning. + */ + if (index >= max_num) { + offset = 0; + index = (u32)find_next_zero_bit(table, max_num, offset); + } + + /* Set the found bit to 1 and reset offset. */ + if (index < max_num) { + set_bit(index, table); + + bitmap->last = index + 1; + } + + spin_unlock(&bitmap->lock); + return index; +} + +void udma_bitmap_free_id(struct udma_bitmap *bitmap, u32 id) +{ + spin_lock(&bitmap->lock); + + if (id >= bitmap->size) { + spin_unlock(&bitmap->lock); + return; + } + + clear_bit(id, bitmap->bits); + + spin_unlock(&bitmap->lock); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h new file mode 100644 index 000000000..704c472e1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_bitmap.h @@ -0,0 +1,27 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef CPU_UDMA_BITMAP_H +#define CPU_UDMA_BITMAP_H + +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/slab.h> + +#define UDMA_NUM_BIT_BYTE 8 +#define UDMA_BYTE_BIT_SHIFT 3 + +typedef struct udma_bitmap { + unsigned long *bits; + u32 size; + u32 last; + spinlock_t lock; +} udma_bitmap_t; + +udma_bitmap_t *udma_bitmap_alloc(u32 bitmap_size); +void udma_bitmap_free(udma_bitmap_t *bitmap); +u32 udma_bitmap_alloc_id(udma_bitmap_t *bitmap); +void udma_bitmap_free_id(udma_bitmap_t *bitmap, u32 id); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h new file mode 100644 index 000000000..91e2d2747 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_common.h @@ -0,0 +1,489 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_COMMON_H +#define UDMA_COMMON_H + +#include <asm/byteorder.h> +#include <linux/fwnode.h> +#include <linux/irqdomain.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <linux/iommu.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/notifier.h> +#include <linux/uaccess.h> + +#include "securec.h" +#include "hinic5_crm.h" +#include "hinic5_cqm.h" +#include "vram_common.h" +#include "hinic5_common.h" +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" +#include "hinic5_lld.h" +#include "ubcore_api.h" +#include "udma_base_common.h" +#include "udma_bitmap.h" +#include "udma_comp.h" +#include "udma_log.h" +#include "udma_sysfs_api.h" +#include "ub_mpu_cmd_defs.h" + +extern u32 g_mtt_page_size; + +#define UDMA_DRV_DESC "UBus UDMA Driver" +#define UDMA_DRV_VERSION GLOBAL_VERSION_STR + +#define MAX_CEQ_NEED 256 +#define TP_POOL_QUEUE_NUM 8192 + +#define UB_DEFAULT_SLICE_SIZE (0x10000) /* 64K */ + +#define UB_DEFAULT_PORT_NUM 1 + +#define UB_BDF_BUS_BIT 8 +#define UB_BDF_DEVICE_ID_BIT 3 +#define UB_BDF_DEVICE_ID_MASK 0x1F +#define UB_BDF_FUNC_ID_MASK 0x7 +#define UB_FUNCTION_MAX_NUM 2048 +#define UB_DEVID_BUS_U8_OFF 2 +#define UB_DEVID_DEVICE_U8_OFF 1 +#define UB_SPU_HOST_ID 0x4 + +#define UDMA_DSCP_NUM 64 +#define UDMA_DSCP_MAX 63 +#define UDMA_COS_MAX 7 + +#define SCQC_VTP_START(udma_dev) \ + ((udma_dev)->ub_cap.dev_cap_res.udma_res.max_jfrc_num) +#define SCQC_JETTY_GRP_START(udma_dev) \ + ((udma_dev)->ub_cap.dev_cap_res.sdk_res.max_vtp) +#define SCQC_JFC_START(udma_dev) \ + (SCQC_JETTY_GRP_START(udma_dev) + \ + (udma_dev)->ub_cap.dev_cap_res.sdk_res.max_jetty_grp) +#define SCQC_UTP_START(udma_dev) \ + ((udma_dev)->ub_cap.dev_cap.max_tp + (udma_dev)->ub_cap.dev_cap.max_tpg) + +#define UDMA_MAX_MIG_VF_NUM 2 +#define UDMA_MAX_MIG_VF_ID_NUM 64 //live migration of vfid range in [64, 127], total 64(count word) +#define UDMA_MIG_VFID_MIN 64 +#define UDMA_MIG_VFID_MAX 127 +#define UDMA_IS_MIG_VFID(vfid) \ + (((vfid) >= UDMA_MIG_VFID_MIN) && ((vfid) <= UDMA_MIG_VFID_MAX)) +#define UDMA_GET_MIG_VF_IDX(vfid) \ + ((vfid)-UDMA_MIG_VFID_MIN) //used for get live migration manage buffer array of index + +#define EID_ERR_VFID 0xffff + +#define UDMA_JETTY_CTX_DW4 4 +#define UDMA_JETTY_CTX_STATE_OFFSET 80 + +#define UDMA_STATE_IN_DW_OFFSET 27 +#define UDMA_STATE_IN_DW_MASK 0x7 + +#define UDMA_IDX_QUEUE_LTH_IN_DW_OFFSET 16 + +#define UDMA_MASK_VALID 1 +#define UDMA_MASK_INVALID 0 + +#define CLOCK_SIZE_OF_INT 32 + +enum udma_restore_vtpn_type { + UDMA_RESTORE_VTPN_CREATE = 0, + UDMA_RESTORE_VTPN_DESTROY, +}; + +enum udma_vtpn_vld { + UDMA_MIG_BUF_IDLE = 0, + UDMA_MIG_BUF_USED, +}; + +enum udma_vtpn_restore_type { + UDMA_MIG_VTPN_STORE_CMDQ = 0, + UDMA_MIG_VTPN_STORE_BUF, +}; + +enum udma_mig_status { + UDMA_MIG_IDLE = 0, //idle status, not start live migration + //1 - full cache: start same step VTP ~ VTP full flush down complete before (pre_activate stage segment complete flush down) + UDMA_MIG_FULL_VOLUME, + //2 - incremental cache: full VTP flush down complete after ~ incremental VTP flush down complete before (restore stage segment complete flush down) + UDMA_MIG_INCREMENTAL, + //3 - incremental modify cache: incremental VTP flush down complete after ~ incremental modify_vtp flush down complete before (restore stage segment complete flush down) + UDMA_MIG_INCREMENTAL_MODIFY, + //4 - normal flush down: incremental modify_vtp flush down complete after ~ live migration buf free before (resource free by UVS via config_device control) + UDMA_MIG_NORMAL, +}; + +enum ub_transport_mode { + UB_TRANSPORT_MODE_RC = 0, + UB_TRANSPORT_MODE_RM, + UB_TRANSPORT_MODE_UM, + UB_TRANSPORT_MODE_RCQ, + UB_TRANSPORT_MODE_RSC, +}; + +enum udma_cqmobj_priv_type { + UDMA_JETTY_TYPE_INVALID = 0, + UDMA_JETTY_TYPE_JETTY, + UDMA_JETTY_TYPE_JFS, + UDMA_JETTY_TYPE_JFR, + UDMA_JETTY_TYPE_RC, + UDMA_CQM_PRIV_TYPE_TP, + UDMA_CQM_PRIV_TYPE_JFC, + UDMA_CQM_PRIV_TYPE_JETTY_GRP, + UDMA_CQM_PRIV_TYPE_UTP, + UDMA_CQM_PRIV_TYPE_VTPN, + UDMA_CQM_PRIV_TYPE_SRQ, +}; + +enum udma_vf_mig_status_e { + UDMA_VF_MIG_END = 0, + UDMA_VF_MIG_START, +}; + +enum ub_env_type { + UB_ENV_ASIC = 0, /* asic */ + UB_ENV_ESL = 1, /* esl */ + UB_ENV_FPGA = 2, /* fpga */ + UB_ENV_EMU = 3, /* emu */ +}; + +struct udma_vf_mig_status_in { //user ctrl opcode 0x3: and UVS agreement of data structure, used for live migration when notify current VF whether start migration + u16 vfid; + u16 sts; //0: non- live migration status; 1: migration state +}; + +struct udma_vf_mig_status_out { //user ctrl opcode 0x3: command result + s32 result; //0 : success ; other : fail +}; + +struct udma_mig_vf_status_in { //user ctrl opcode 0x4: and UVS agreement of data structure, used for live migration when query VF whether passthrough into VM + u16 vfid; +}; + +struct udma_mig_vf_status_out { //user ctrl opcode 0x4: query result + s32 result; //0: MIG_VF_IN_VM; 1: MIG_VF_NOT_IN_VM; 2: other error +}; + +struct udma_dev_cfg_info { + u8 scenes_id; /* cfg data scenes ID */ + u8 lb_en; /* load balance enable */ + u8 lb_mode; /* load balance mode */ + u8 rsvd; + + u8 fake_en; /* whether enable fake_vf */ + u8 page_bit; /* db page address offset */ + u8 pf_start_bit; /* fake vf in pf start bit bit */ + u8 pf_end_bit; /* fake vf in pf end bit bit */ +}; + +struct udma_dev_hw_info { + u32 config_num_ports; /* Number of ports from configuration file */ + u32 hca_type; /* HCA version: 4x25G or 2x100G */ + u8 phy_port; /* from pci_priv_dev in mention take out roce device corresponding to physical port */ + u8 ep_id; /* EP ID */ + u8 cpu_endian; /* 0: little-endian,1: big-endian */ + u8 rsvd; + bool is_vf; /* PF/VF of indicate, from pci_priv_dev get to */ +}; + +struct udma_tp_sq { + struct tag_cqm_queue *sq_info; + struct mtt ub_mtt; +}; + +struct udma_tp_rq { + struct ubrc rc_table; +}; + +struct udma_tp_pool { + struct udma_tp_sq *sq_list[TP_POOL_QUEUE_NUM]; + struct udma_tp_rq *rq_list[TP_POOL_QUEUE_NUM]; + bool is_created_tx; + bool is_created_rx; +}; + +struct ub_dev_cap_res { + struct ub_dev_cap_sdk_res sdk_res; /* definition in sdk */ + struct ub_dev_cap_udma_res udma_res; +}; + +struct udma_device_cap { + struct ub_dev_cap_res dev_cap_res; + struct ub_dev_cap dev_cap; + struct ub_net_dev_cap net_dev_cap; +}; + +struct udma_vtp_buf { + struct mutex vf_mutex; + void *addr; //cache VTPN of buffer(capacity amount 128K), used for " full cache " and " incremental modify cache " stage segment, refer to enum udma_mig_status + void *cqm_cmd_buf; //used for " incremental cache " stage segment, via DMA way to microcode send request, single time maximum carry 32 node, struct tag_cqm_cmd_buf + u32 total; //current 128K buffer in status is " occupy " of VTP node total data, each VF maximum 128K - 64(count word) + u32 cur_idx; //full flush down VTP table, used to indicate each one 轮 flush down (each time at most flush down 32) after already traverse to of VTP table entry bit position, used for mention high traverse valid率 + u8 mig_sts; //flag: current VF migration state enum udma_mig_status + u8 rsvd; + u16 func_id; //current VF of ID +}; + +struct udma_mig_vf_mgt { + struct mutex + mig_vf_mgt_mutex; //global lock, newly added live migration VM or VM migration end when need add lock + struct udma_vtp_buf + buf_array[UDMA_MAX_MIG_VF_ID_NUM]; //via index index corresponding VF + u8 migrating_num; //current currently at at live migration process in of VM count + u8 rsv[3]; +}; + +typedef struct udma_dev_bdf_info { + u8 bus_id; + u8 device_id; + u8 func_id; + u8 bdf_info_vld; + u8 host_id; +} udma_dev_bdf_info_s; + +/* current increasedevice queue(such as wqe/cqe buffer etc.) of one kind new allocate way, via dma interface allocate continuous physical memory, + * newly added this data structure store continuous physical memory base this information. + */ +struct queue_addr_info { + void *va; /* queue va */ + dma_addr_t pa; /* queue pa */ + u32 size; /* queue size */ + struct hinic5_dma_addr_align mem_align; +}; + +/* statistics cmdq latency */ +struct cmdq_rtt { + atomic64_t cmdq_rtt_sum; /* cmdq send start to cmdq complete return driver of total time */ + atomic64_t times; /* driver send down cmdq return success count */ + atomic64_t cmdq_npu_rtt_sum; /* microcode from wake cmdq to complete handle generate CQE of total time */ +}; + +/* remote command queue */ +struct udma_rc_queue { + u32 cqm_priv_type; + struct tag_cqm_qpc_mpt *rc_ctx; + struct mtt ub_mtt; + struct tag_cqm_queue *rc_queue; + struct tag_cqm_queue *pcqc; /* producer consumer queue context */ + struct queue_addr_info queue_addr; + void *rcq_va; /* rcq actual virtual addr */ + u32 total_depth; /* rc queue total number of wqes */ + u32 total_size; /* queue total size = UB_RCQE_SIZE * total_depth */ + struct list_head node; + struct ubrc ubrc; + u32 rsp_depth; + struct cmdq_rtt cmdq_rtt; +}; + +struct udma_object_n_record { + struct mutex mutex; + u32 num; +}; + +struct udma_default_cos { + u8 cfg_cos_en; /* supports user configuration cos value, enable: 1(supports user configuration) disable: 0(not support user configuration, use default cos value) default: 0(same disable) */ + u8 func_cos; + u16 vf_default_cos; /* default value: 5 */ + u16 pf_default_cos; /* default value: 4 */ + u16 jfrc_cos; /* default value: 2 */ +}; + +typedef struct udma_driver_device_ctr { + atomic64_t post_send_wr_num; + atomic64_t post_recv_wr_num; + atomic64_t poll_total_num; + atomic64_t poll_sq_num; + atomic64_t poll_rq_num; +} udma_driver_device_ctr_t; + +typedef struct udma_device { + struct ubcore_device ub_dev; + struct device *dev; + struct net_device *ndev; + void *hwdev; + union ubcore_eid eid; + u32 tp_num; + atomic_t task_jfs_num; + struct mutex dev_mutex; + struct mutex tp_mutex; + struct udma_device_cap ub_cap; + u64 feat_bitmap[UB_FEAT_BITMAP_U64_MAX]; + int numa_original_node_id; + bool numa_original_node_id_valid; + + int ceq_num; + int ceqn[MAX_CEQ_NEED]; + + void __iomem *kernel_db_map; + void __iomem *kernel_dwqe_map; + + struct udma_dev_cfg_info cfg_info; + struct udma_dev_hw_info hw_info; + + struct udma_rc_queue rsvd_qpc; /* reserved xid 0 of qpc */ + struct udma_rc_queue *rc_queue; + u32 rc_queue_cnt; /* already configured RC queue of count */ + u32 is_rc_queue_configured; /* whether already configuration RC flag, avoid repeat configuration */ + + u32 jfs_current_cnt; + atomic_t jfs_hash; + atomic_t jfr_hash; + atomic_t rs_hash; + udma_driver_device_ctr_t udma_driver_device_ctr; + + char dev_name[UBCORE_MAX_DEV_NAME]; + + u32 local_en; + u8 capture_en; /* 1: capture packet enable 0: capture packet disable */ + u8 host_id; + + struct udma_device_res *device_res; + struct udma_bitmap *bitmap; /* tpg bitmap */ + struct udma_sysfs_res sysfs_res; + + u8 slice_cfg_en; /* 1: slice can configure 0: slice cannot configure */ + u8 latch_tpg_configured; /* whether already configuration TP Latch flag, avoid repeat configuration */ + u32 slice; + u32 suspend_period; + u32 suspend_cnt; + + u8 mtu_1k_en; /* 1: supports DFX configuration 1K mtu of value 0: default udma only supports configuration 4K, 8K MTU */ + bool index_queue_en; /* todo pending ub mpu adaptation feat_bitmap after delete, use feat_bitmap to get enable */ + + unsigned long status; + + struct udma_tp_pool tp_pool; + + struct udma_mig_vf_mgt mig_vf_mgt; + u32 fastmsg_connect; + + struct notifier_block netdev_nb; + + enum mag_cmd_port_speed speed; + struct udma_dev_bdf_info *bdf_info; + struct mutex bdf_mutex; + u32 bdf_cnt; + u64 dscp_bitmap; + u8 dscp2cos[UDMA_DSCP_NUM]; + struct udma_default_cos default_cos; + + struct udma_object_n_record jettyn_record; + struct udma_object_n_record tpn_record; + struct udma_object_n_record jfcn_record; + struct udma_object_n_record vtpn_record; + struct udma_object_n_record utpn_record; + struct udma_object_n_record tpgn_record; + struct udma_object_n_record jetty_grpn_record; + struct udma_object_n_record srqn_record; + + spinlock_t dwqe_lock; + + enum udma_device_type device_type; /* device type */ + enum ub_env_type env_type; /* environment type */ + enum hinic5_dev_type dev_bus_type; + u64 card_id; + struct { + u64 pa; + void *va; + } fake_dma_addr; +} udma_device_t; + +struct udma_bus_addr { + uint32_t domain; /* Device domain */ + uint8_t bus; /* Device bus */ + uint8_t devid; /* Device ID */ + uint8_t function; /* Device function */ + uint8_t rsvd; /* Rsvd field for align */ +}; + +#define UDMA_IS_VF_MIGRATING(status) ((status) != UDMA_MIG_IDLE) + +#define UDMA_PCI_DEV_DBDF(pdev) \ + (((u32)pci_domain_nr((pdev)->bus) << 16) | \ + ((pdev)->bus->number << 8) | ((pdev)->devfn)) + +//PF and VF of domain id all is 0000 +#define UDMA_QUERY_DBDF_BY_BDF_INFO(bus_addr) \ + (((bus_addr)->domain << 16) | ((bus_addr)->bus << 8) | \ + ((bus_addr)->devid << 3) | ((bus_addr)->function)) + +#define UDMA_PCI_DBDF_DOM(dbdf) (((dbdf) >> 16) & 0xFFFF) +#define UDMA_PCI_DBDF_BUS(dbdf) (((dbdf) >> 8) & 0xFF) +#define UDMA_PCI_DBDF_DEVID(dbdf) (((dbdf) >> 3) & 0x1F) +#define UDMA_PCI_DBDF_FUNCTION(dbdf) ((dbdf)&0x7) +#define UDMA_PCI_DBDF_DEVFN(dbdf) ((dbdf)&0xFF) + +#define udma_container_of(ptr, type, member) \ + (((ptr) == NULL) ? NULL : container_of(ptr, type, member)) + +static inline struct udma_vtp_buf * +udma_mig_get_vf_buf(struct udma_device *udma_dev, u32 vfid) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + return &mig_mgt->buf_array[UDMA_GET_MIG_VF_IDX(vfid)]; +} + +static inline struct udma_device * +to_udma_dev(const struct ubcore_device *ub_dev) +{ + return (struct udma_device *)udma_container_of( + ub_dev, struct udma_device, ub_dev); +} + +static inline u32 udma_dev_is_tpf(struct udma_device *udma_dev) +{ + return udma_dev->ub_cap.net_dev_cap.is_tpf; +} + +static inline void +udma_fill_kernel_queue_addr_info(u32 queue_len, + struct queue_addr_info *queue_addr) +{ + queue_addr->va = queue_addr->mem_align.align_vaddr; + queue_addr->pa = queue_addr->mem_align.align_paddr; + queue_addr->size = queue_len; +} + +void udma_get_dev_name(struct udma_device *udma_dev); + +static inline bool udma_get_index_queue_en(struct udma_device *udma_dev) +{ + return udma_dev->index_queue_en; +} + +#define adapted_ue_idx(cfg) (cfg)->ue_idx +#define adapted_type_ue_idx(cfg, type) (cfg)->type##_ue_idx +typedef struct ubcore_ue_stats adapted_ubcore_ue_stats; + +#if defined(__x86_64__) +static inline uint64_t udma_get_cycles_dfx(void) +{ + uint32_t low, high; + uint64_t val; + asm volatile("rdtsc" : "=a"(low), "=d"(high)); + val = high; + val = (val << CLOCK_SIZE_OF_INT) | low; + return val; +} +#elif defined(__aarch64__) +static inline uint64_t udma_get_cycles_dfx(void) +{ + uint64_t freq; + asm volatile("isb" : : : "memory"); + asm volatile("mrs %0, cntvct_el0" : "=r"(freq)); + return freq; +} +#else +#warning get_cycles not implemented +#endif + +#define UDMA_MAX_DEV_NAME 64 +#define UDMA_MAX_CHIP_NAME 64 + +#endif /* UDMA_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c new file mode 100644 index 000000000..b208421dd --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.c @@ -0,0 +1,67 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Hisilicon Technologies Co., Ltd. 2021. All rights reserved. + +#include <linux/version.h> +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 19, 90)) +#include <linux/types.h> +#else +#include <linux/auxiliary_bus.h> +#endif + +#include "securec.h" +#include "hinic5_hw.h" +#include "udma_cqm.h" + +void udma_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf) +{ + if (cqm_cmd_outbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_outbuf); + } + + if (cqm_cmd_inbuf != NULL) { + cqm5_cmd_free(ex_handle, cqm_cmd_inbuf); + } +} + +int udma_cqm_cmd_zalloc_inoutbuf(void *ex_handle, cqm_cmd_buf_s **cqm_cmd_inbuf, + u16 inbuf_size, cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size) +{ + int ret; + + if (cqm_cmd_inbuf != NULL) { + *cqm_cmd_inbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_inbuf == NULL) { + return -ENOMEM; + } + (*cqm_cmd_inbuf)->size = inbuf_size; + + (void)memset_s((*cqm_cmd_inbuf)->buf, inbuf_size, 0, + inbuf_size); + } + + if (cqm_cmd_outbuf != NULL) { + *cqm_cmd_outbuf = cqm5_cmd_alloc(ex_handle); + + if (*cqm_cmd_outbuf == NULL) { + ret = -ENOMEM; + goto err_alloc_outbuf; + } + (*cqm_cmd_outbuf)->size = outbuf_size; + + (void)memset_s((*cqm_cmd_outbuf)->buf, outbuf_size, 0, + outbuf_size); + } + + return 0; + +err_alloc_outbuf: + if (cqm_cmd_inbuf != NULL && *cqm_cmd_inbuf != NULL) { + cqm5_cmd_free(ex_handle, *cqm_cmd_inbuf); + *cqm_cmd_inbuf = NULL; + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h new file mode 100644 index 000000000..f06c352bf --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_cqm.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Hisilicon Technologies Co., Ltd. 2021. All rights reserved. + +#ifndef _UDMA_CQM_H +#define _UDMA_CQM_H + +#include <linux/types.h> +#include "hinic5_hw.h" +#include "hinic5_cqm.h" +#include "hmm_common.h" +#include "hmm_em_inner.h" + +int udma_cqm_cmd_zalloc_inoutbuf(void *ex_handle, cqm_cmd_buf_s **cqm_cmd_inbuf, + u16 inbuf_size, cqm_cmd_buf_s **cqm_cmd_outbuf, + u16 outbuf_size); +void udma_cqm5_cmd_free_inoutbuf(void *ex_handle, cqm_cmd_buf_s *cqm_cmd_inbuf, + cqm_cmd_buf_s *cqm_cmd_outbuf); + +#endif /* _UDMA_CQM_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c new file mode 100644 index 000000000..f7a8ccdc1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.c @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/moduleparam.h> + +#include "udma_log.h" + +u32 g_udma_log_level = UDMA_LOG_LEVEL_INFO; + +module_param(g_udma_log_level, uint, 0664); +MODULE_PARM_DESC(g_udma_log_level, + "0 for debug, 1 for info, 2 for warn, 3 for error"); + +bool udma_kernel_log_level_prior(int level) +{ + return (((u32)level < g_udma_log_level) ? true : false); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h new file mode 100644 index 000000000..893bb4056 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_log.h @@ -0,0 +1,179 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_LOG_H +#define UDMA_LOG_H + +#include <linux/device.h> +#include <linux/sched.h> + +enum { + UDMA_LOG_LEVEL_DEBUG = 0, + UDMA_LOG_LEVEL_INFO, + UDMA_LOG_LEVEL_WARN, + UDMA_LOG_LEVEL_ERR +}; + +extern u32 g_udma_log_level; + +#define UDMA_M_COM "COM" +#define UDMA_M_JETTY "Jetty" +#define UDMA_M_JFC "JFC" +#define UDMA_M_JFCE "JFCE" +#define UDMA_M_JFR "JFR" +#define UDMA_M_SRQ "SRQ" +#define UDMA_M_JFS "JFS" +#define UDMA_M_KEY "Key" +#define UDMA_M_QUERY "Query" +#define UDMA_M_SEG "Segment" +#define UDMA_M_TP "TP" +#define UDMA_M_TPG "TPG" +#define UDMA_M_VTP "VTP" +#define UDMA_M_UTP "UTP" +#define UDMA_M_FMSG "FMSG" +#define UDMA_M_JFRC "JFRC" +#define UDMA_M_TPG "TPG" +#define UDMA_M_MAIN "Main" +#define UDMA_M_EID "EID" +#define UDMA_M_MISC "Misc" +#define UDMA_M_DFX "DFX" +#define UDMA_M_DEV "DEV" +#define UDMA_M_SRQ "SRQ" +#define UDMA_M_SYSFS "SYSFS" +#define UDMA_M_RX_RATE_LIMITE "RX_RATE_LIMITE" +#define UDMA_M_CAPTURE_DFX "Capture_DFX" +#define UDMA_M_CC_INJECT_DFX "CC_INJECT_DFX" +#define UDMA_M_PORT_STA_DFX "PORT_STA_DFX" +#define UDMA_M_DROP_DFX "Drop_DFX" +#define UDMA_M_CQM_MOD_DFX "CQM_MODIFY_DFX" +#define UDMA_M_GLB_XID_CFG "GLB_XID_CFG" +#define UDMA_M_DB "DB" +#define UDMA_M_ASYNC_EVENT "ASYNC_EVENT" +#define UDMA_M_DCA "DCA" +#define UDMA_M_RC_TABLE "RC_TABLE" +#define UDMA_M_RESOURCE "RESOURCE" +#define UDMA_M_UBRC_COMP "UBRC_COMP" +#define UDMA_M_USER_CTL "USER_CTL" +#define UDMA_M_CMD "CMD" +#define UDMA_M_BOND "BOND" +#define UDMA_M_SHM "SHM" +#define UDMA_M_VRAM "VRAM" +#define UDMA_M_BITMAP "BITMAP" + +#define G_TGID task_tgid_nr(current) +#define G_PID task_pid_nr(current) + +#define udma_debug(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_DEBUG) \ + dev_info(dev, \ + "[tgid:%u,pid:%u][UDMA5][DEBUG][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_info(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_INFO) \ + dev_info(dev, \ + "[tgid:%u,pid:%u][UDMA5][INFO][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_warn(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_WARN) \ + dev_warn(dev, \ + "[tgid:%u,pid:%u][UDMA5][WARN][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_err(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + dev_err(dev, \ + "[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_info_limit(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_INFO) \ + dev_info_ratelimited( \ + dev, \ + "[tgid:%u,pid:%u][UDMA5][INFO][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_warn_limit(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_WARN) \ + dev_warn_ratelimited( \ + dev, \ + "[tgid:%u,pid:%u][UDMA5][WARN][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_err_limit(dev, module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + dev_err_ratelimited( \ + dev, \ + "[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_info(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_INFO) \ + pr_info("[tgid:%u,pid:%u][UDMA5][INFO][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_warn(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_WARN) \ + pr_warn("[tgid:%u,pid:%u][UDMA5][WARN][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_err(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + pr_err("[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +#define udma_pr_err_limit(module, fmt, ...) \ + do { \ + if (g_udma_log_level <= UDMA_LOG_LEVEL_ERR) \ + pr_err_ratelimited( \ + "[tgid:%u,pid:%u][UDMA5][ERR][" module \ + "][%s:%u]: " fmt, \ + G_TGID, G_PID, __func__, __LINE__, \ + ##__VA_ARGS__); \ + } while (0) + +bool udma_kernel_log_level_prior(int level); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c new file mode 100644 index 000000000..0a04a680a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.c @@ -0,0 +1,1386 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <net/ipv6.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/iommu.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/netdevice.h> +#include <linux/spinlock.h> +#include "ubcore_api.h" +/* + * Generally, a driver should not include ubcore_uapi.h for kernel users, + * the udma_pci driver for simluation calls ubcore_find_jfs/jfrin the ubcore_uapi.h to reduce complexity + */ +#include "comm_defs.h" +#include "ubcore_uapi.h" +#include "ubcore_types.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "ub_mpu_cmd.h" +#include "nic_mpu_cmd.h" +#include "ub_pub_cmd.h" +#include "ub_dw_index.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_net.h" +#include "udma_segment.h" +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_jetty.h" +#include "udma_jfc.h" +#include "udma_utp.h" +#include "udma_vtp.h" +#include "udma_rc_table.h" +#include "udma_cmd_com.h" +#include "udma_misc.h" + +#define MAX_DATA_LEN_UB_DFX 8192 +#define DUMP_LINE_BYTE_NUM 16 +#define DUMP_LINE_BYTE_HALF_NUM (DUMP_LINE_BYTE_NUM / 4) +#define CHAR_BUF_LEN 64 +#define CHAR_WORD_LEN 9 +#define WORD_SIZE 16 +#define FORMAT_SIZE 8 +#define WORD_LEN 4 +#define COS_TA_MASK 0x7 +#define COS_TP_MASK 0x7 +#define COS_TP_DSCP_MASK 0x3f + +/* temporary use global change amount configuration cfg_cos_en, iterate fifth modify is udma_dev->default_cos.cfg_cos_en */ +enum { CFG_COS_DISABLE = 0, CFG_COS_ENABLE }; + +u32 g_udma_cfg_cos_en = CFG_COS_DISABLE; +module_param(g_udma_cfg_cos_en, uint, 0664); +MODULE_PARM_DESC(g_udma_cfg_cos_en, + "0 for can't config jetty_cos, 1 for can config jetty_cos"); + +void udma_store_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list) +{ + if (udma_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "store list node fail, head node is null\n"); + return; + } + mutex_lock(&udma_list->node_lock); + list_add(node, &udma_list->node); + udma_list->cnt++; + mutex_unlock(&udma_list->node_lock); +} + +void udma_delete_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list) +{ + if (udma_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "delete list node fail, head node is null\n"); + return; + } + mutex_lock(&udma_list->node_lock); + if (node->prev != NULL) { + list_del(node); + node->prev = NULL; + udma_list->cnt--; + } + mutex_unlock(&udma_list->node_lock); +} + +static void list_lock_init(struct udma_device_res *device_res) +{ + mutex_init(&device_res->tpn_list.node_lock); + INIT_LIST_HEAD(&device_res->tpn_list.node); + mutex_init(&device_res->jfs_list.node_lock); + INIT_LIST_HEAD(&device_res->jfs_list.node); + mutex_init(&device_res->jfr_list.node_lock); + INIT_LIST_HEAD(&device_res->jfr_list.node); + mutex_init(&device_res->jetty_list.node_lock); + INIT_LIST_HEAD(&device_res->jetty_list.node); + mutex_init(&device_res->jfc_list.node_lock); + INIT_LIST_HEAD(&device_res->jfc_list.node); + mutex_init(&device_res->seg_list.node_lock); + INIT_LIST_HEAD(&device_res->seg_list.node); + mutex_init(&device_res->jetty_group_list.node_lock); + INIT_LIST_HEAD(&device_res->jetty_group_list.node); + mutex_init(&device_res->tpg_list.node_lock); + INIT_LIST_HEAD(&device_res->tpg_list.node); + mutex_init(&device_res->rc_list.node_lock); + INIT_LIST_HEAD(&device_res->rc_list.node); + mutex_init(&device_res->vtp_list.node_lock); + INIT_LIST_HEAD(&device_res->vtp_list.node); + mutex_init(&device_res->utp_list.node_lock); + INIT_LIST_HEAD(&device_res->utp_list.node); + device_res->tpn_list.cnt = 0; + device_res->jfs_list.cnt = 0; + device_res->jfr_list.cnt = 0; + device_res->jetty_list.cnt = 0; + device_res->jfc_list.cnt = 0; + device_res->seg_list.cnt = 0; + device_res->jetty_group_list.cnt = 0; + device_res->tpg_list.cnt = 0; + device_res->rc_list.cnt = 0; + device_res->vtp_list.cnt = 0; + device_res->utp_list.cnt = 0; +} + +int udma_dfx_device_init(struct udma_device *udma_dev) +{ + udma_dev->device_res = devm_kzalloc( + udma_dev->dev, sizeof(struct udma_device_res), GFP_KERNEL); + if (udma_dev->device_res == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "alloc device_res failed.\n"); + return -ENOMEM; + } + + list_lock_init(udma_dev->device_res); + + udma_info(udma_dev->dev, UDMA_M_MISC, "init device_res successful\n"); + + return 0; +} + +void udma_dfx_device_deinit(struct udma_device *udma_dev) +{ + struct udma_device_res *dfx = udma_dev->device_res; + + if (list_empty(&dfx->jetty_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, + "jetty list is not empty\n"); + + if (list_empty(&dfx->jetty_group_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, + "jetty group list is not empty\n"); + + if (list_empty(&dfx->jfs_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "jfs list is not empty\n"); + + if (list_empty(&dfx->jfr_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "jfr list is not empty\n"); + + if (list_empty(&dfx->jfc_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "jfc list is not empty\n"); + + if (list_empty(&dfx->seg_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "seg list is not empty\n"); + + if (list_empty(&dfx->rc_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "rc list is not empty\n"); + + if (list_empty(&dfx->tpn_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "tpn list is not empty\n"); + + if (list_empty(&dfx->tpg_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "tpg list is not empty\n"); + + if (list_empty(&dfx->vtp_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "vtp list is not empty\n"); + + if (list_empty(&dfx->utp_list.node) == 0) + udma_err(udma_dev->dev, UDMA_M_MISC, "utp list is not empty\n"); + + devm_kfree(udma_dev->dev, udma_dev->device_res); + + udma_info(udma_dev->dev, UDMA_M_MISC, + "udma_dfx_device_deinit successful\n"); +} + +int udma_query_res_dev_tp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_tp *tpn_now = NULL; + u32 *tp_list = NULL; + u32 cnt; + size_t size; + + mutex_lock(&udma_dev->device_res->tpn_list.node_lock); + cnt = udma_dev->device_res->tpn_list.cnt; + if (cnt > 0) { + size = (size_t)cnt * sizeof(u32); + tp_list = (u32 *)vmalloc(size); + if (tp_list == NULL) { + mutex_unlock(&udma_dev->device_res->tpn_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "tp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = tp_list; + list_for_each_entry( + tpn_now, &udma_dev->device_res->tpn_list.node, node) { + *tp_list = tpn_now->tp_info->xid; + tp_list++; + } + } + mutex_unlock(&udma_dev->device_res->tpn_list.node_lock); + + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jfs_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jfs *jfs_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jfs_list.node_lock); + cnt = udma_dev->device_res->jfs_list.cnt; + if (cnt > 0) { + u32 *jfs_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jfs_list == NULL) { + mutex_unlock(&udma_dev->device_res->jfs_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jfs_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jfs_list; + list_for_each_entry( + jfs_now, &udma_dev->device_res->jfs_list.node, node) { + *jfs_list = jfs_now->jfs_id; + jfs_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jfs_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jfr_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jfr *jfr_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jfr_list.node_lock); + cnt = udma_dev->device_res->jfr_list.cnt; + if (cnt > 0) { + u32 *jfr_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jfr_list == NULL) { + mutex_unlock(&udma_dev->device_res->jfr_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jfr_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jfr_list; + list_for_each_entry( + jfr_now, &udma_dev->device_res->jfr_list.node, node) { + *jfr_list = jfr_now->jfr_id; + jfr_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jfr_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jetty_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jetty *jetty_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jetty_list.node_lock); + cnt = udma_dev->device_res->jetty_list.cnt; + if (cnt > 0) { + u32 *jetty_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jetty_list == NULL) { + mutex_unlock( + &udma_dev->device_res->jetty_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jetty_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jetty_list; + list_for_each_entry(jetty_now, + &udma_dev->device_res->jetty_list.node, + node) { + *jetty_list = jetty_now->jettyc_info->xid; + jetty_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jetty_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jfc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jfc *jfc_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->jfc_list.node_lock); + cnt = udma_dev->device_res->jfc_list.cnt; + if (cnt > 0) { + u32 *jfc_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (jfc_list == NULL) { + mutex_unlock(&udma_dev->device_res->jfc_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "jfc_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = jfc_list; + list_for_each_entry( + jfc_now, &udma_dev->device_res->jfc_list.node, node) { + *jfc_list = jfc_now->base.id; + jfc_list++; + } + } + + mutex_unlock(&udma_dev->device_res->jfc_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_seg_list(struct udma_device *udma_dev, + struct ubcore_res_seg_val *ret_val) +{ + struct udma_segment *udma_seg = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->seg_list.node_lock); + cnt = udma_dev->device_res->seg_list.cnt; + if (cnt > 0) { + struct ubcore_seg_info *seg_list = + (struct ubcore_seg_info *)vmalloc( + sizeof(struct ubcore_seg_info) * cnt); + if (seg_list == NULL) { + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "seg_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->seg_list = seg_list; + list_for_each_entry( + udma_seg, &udma_dev->device_res->seg_list.node, node) { + seg_list->ubva.eid = udma_dev->eid; + seg_list->ubva.va = udma_seg->va; + seg_list->len = udma_seg->size; + seg_list->token_id = udma_seg->mpt->mpt_index; + seg_list++; + } + } + + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + ret_val->seg_cnt = cnt; + + return 0; +} + +int udma_query_res_dev_tpg_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_tpg *tpg_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->tpg_list.node_lock); + cnt = udma_dev->device_res->tpg_list.cnt; + if (cnt > 0) { + u32 *val_tpg_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_tpg_list == NULL) { + mutex_unlock(&udma_dev->device_res->tpg_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "tpg_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_tpg_list; + list_for_each_entry( + tpg_now, &udma_dev->device_res->tpg_list.node, node) { + *val_tpg_list = tpg_now->tpg.tpgn; + val_tpg_list++; + } + } + + mutex_unlock(&udma_dev->device_res->tpg_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_rc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_rc_queue *rc_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->rc_list.node_lock); + cnt = udma_dev->device_res->rc_list.cnt; + if (cnt > 0) { + u32 *val_rc_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_rc_list == NULL) { + mutex_unlock(&udma_dev->device_res->rc_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_rc_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_rc_list; + list_for_each_entry(rc_now, &udma_dev->device_res->rc_list.node, + node) { + *val_rc_list = rc_now->rc_ctx->xid; + val_rc_list++; + } + } + + mutex_unlock(&udma_dev->device_res->rc_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_vtp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_vtp *vtp_now = NULL; + struct udma_vtpn *vtpn_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->vtp_list.node_lock); + cnt = udma_dev->device_res->vtp_list.cnt; + if (cnt > 0) { + u32 *val_vtp_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_vtp_list == NULL) { + mutex_unlock(&udma_dev->device_res->vtp_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_vtp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_vtp_list; + if (udma_dev_is_tpf(udma_dev) == 1) { + list_for_each_entry( + vtp_now, &udma_dev->device_res->vtp_list.node, + node) { + *val_vtp_list = vtp_now->base.cfg.vtpn; + val_vtp_list++; + } + } else { + list_for_each_entry( + vtpn_now, &udma_dev->device_res->vtp_list.node, + node) { + *val_vtp_list = vtpn_now->base.vtpn; + val_vtp_list++; + } + } + } + + mutex_unlock(&udma_dev->device_res->vtp_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_utp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_utp *utp_now = NULL; + u32 cnt; + + mutex_lock(&udma_dev->device_res->utp_list.node_lock); + cnt = udma_dev->device_res->utp_list.cnt; + if (cnt > 0) { + u32 *val_utp_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_utp_list == NULL) { + mutex_unlock(&udma_dev->device_res->utp_list.node_lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_utp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_utp_list; + list_for_each_entry( + utp_now, &udma_dev->device_res->utp_list.node, node) { + *val_utp_list = utp_now->base.utpn; + val_utp_list++; + } + } + + mutex_unlock(&udma_dev->device_res->utp_list.node_lock); + ret_val->cnt = cnt; + + return 0; +} + +int udma_query_res_dev_jetty_group_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val) +{ + struct udma_jetty_grp *grp_now = NULL; + struct mutex *lock = NULL; + u32 cnt; + + lock = &udma_dev->device_res->jetty_group_list.node_lock; + mutex_lock(lock); + cnt = udma_dev->device_res->jetty_group_list.cnt; + if (cnt > 0) { + u32 *val_grp_list = (u32 *)vmalloc(sizeof(u32) * cnt); + if (val_grp_list == NULL) { + mutex_unlock(lock); + udma_err(udma_dev->dev, UDMA_M_MISC, + "val_grp_list vmalloc fail\n"); + return -ENOMEM; + } + + ret_val->list = val_grp_list; + list_for_each_entry( + grp_now, &udma_dev->device_res->jetty_group_list.node, + node) { + *val_grp_list = grp_now->jetty_grp_id; + val_grp_list++; + } + } + + mutex_unlock(lock); + ret_val->cnt = cnt; + + return 0; +} + +static void udma_free_res_list(struct ubcore_res_list_val *res_list) +{ + if (res_list == NULL || res_list->list == NULL) { + return; + } + + vfree(res_list->list); + res_list->list = NULL; + res_list->cnt = 0; +} + +static void udma_free_res_seg(struct ubcore_res_seg_val *res_seg) +{ + if (res_seg == NULL || res_seg->seg_list == NULL) { + return; + } + + vfree(res_seg->seg_list); + res_seg->seg_list = NULL; + res_seg->seg_cnt = 0; +} + +void udma_free_dev_ctx_res(struct udma_device_ctx_res *dev_ctx_res) +{ + udma_free_res_list(&dev_ctx_res->tp_list); + udma_free_res_list(&dev_ctx_res->jfs_list); + udma_free_res_list(&dev_ctx_res->jfr_list); + udma_free_res_list(&dev_ctx_res->jetty_list); + udma_free_res_list(&dev_ctx_res->jfc_list); + udma_free_res_seg(&dev_ctx_res->seg_list); + udma_free_res_list(&dev_ctx_res->tpg_list); + udma_free_res_list(&dev_ctx_res->rc_list); + udma_free_res_list(&dev_ctx_res->vtp_list); + udma_free_res_list(&dev_ctx_res->utp_list); + udma_free_res_list(&dev_ctx_res->jetty_group_list); +} + +int udma_query_res_dev(struct udma_device *udma_dev, + struct udma_device_ctx_res *dev_ctx_res) +{ + int ret = 0; + + udma_info(udma_dev->dev, UDMA_M_MISC, "start to query device res.\n"); + + ret |= udma_query_res_dev_tp_list(udma_dev, &dev_ctx_res->tp_list); + ret |= udma_query_res_dev_jfs_list(udma_dev, &dev_ctx_res->jfs_list); + ret |= udma_query_res_dev_jfr_list(udma_dev, &dev_ctx_res->jfr_list); + ret |= udma_query_res_dev_jetty_list(udma_dev, + &dev_ctx_res->jetty_list); + ret |= udma_query_res_dev_jfc_list(udma_dev, &dev_ctx_res->jfc_list); + ret |= udma_query_res_dev_seg_list(udma_dev, &dev_ctx_res->seg_list); + ret |= udma_query_res_dev_tpg_list(udma_dev, &dev_ctx_res->tpg_list); + ret |= udma_query_res_dev_rc_list(udma_dev, &dev_ctx_res->rc_list); + ret |= udma_query_res_dev_vtp_list(udma_dev, &dev_ctx_res->vtp_list); + ret |= udma_query_res_dev_utp_list(udma_dev, &dev_ctx_res->utp_list); + ret |= udma_query_res_dev_jetty_group_list( + udma_dev, &dev_ctx_res->jetty_group_list); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "Failed to query device resources.\n"); + udma_free_dev_ctx_res(dev_ctx_res); + return -ENOMEM; + } + + udma_info(udma_dev->dev, UDMA_M_MISC, "query device res successful!\n"); + return 0; +} + +void udma_dump_dfx(struct udma_device *udma_dev, u64 istart, u32 ilen, + u64 buf_addr, u32 buf_size) +{ + u8 printf_buf[CHAR_BUF_LEN]; + u32 *line_start = NULL; + u32 *dw_current = NULL; + int ret = 0; + u64 i = 0; + u32 row; + + if (istart == 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "The start addr is null.\n"); + return; + } + + if ((ilen == 0) || (ilen > MAX_DATA_LEN_UB_DFX)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "The dump mem len is error.\n"); + return; + } + + line_start = (u32 *)(uintptr_t)istart; + row = ilen / WORD_SIZE; + while (row != 0) { + row--; + if ((uintptr_t)(void *)line_start >= buf_addr + buf_size) + line_start = (u32 *)(uintptr_t)buf_addr; + dw_current = line_start; + (void)memset_s(printf_buf, CHAR_BUF_LEN, 0x0, CHAR_BUF_LEN); + for (i = 0; i < WORD_LEN; i++) { + ret = sprintf_s( + (char *)(printf_buf + CHAR_WORD_LEN * i), + CHAR_BUF_LEN - CHAR_WORD_LEN * i, "%08X ", + be32_to_cpu(*dw_current)); + if (ret < 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "format dump dfx string failed, ret:%d .\n", + ret); + return; + } + dw_current++; + } + udma_debug(udma_dev->dev, UDMA_M_MISC, "%s\n", printf_buf); + line_start += WORD_LEN; + } +} + +void udma_dump_byte_err_dfx(struct udma_device *udma_dev, const u8 *buf, + u32 ilen) +{ + char line[CHAR_BUF_LEN] = { 0 }; + size_t line_len = 0; + int ret = 0; + u32 i; + + if (buf == NULL) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, + "The buf addr is NULL\n"); + return; + } + + if ((ilen == 0) || (ilen > MAX_DATA_LEN_UB_DFX)) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, + "The dump mem len is error.\n"); + return; + } + + for (i = 0; i < ilen; i++) { + if ((i != 0) && (i % DUMP_LINE_BYTE_NUM == 0)) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, "%08x:%s\n", + i - DUMP_LINE_BYTE_NUM, line); + line[0] = '\0'; + } else if (i % DUMP_LINE_BYTE_HALF_NUM == 0) { + line_len = strlen(line); + ret = sprintf_s(line + line_len, + CHAR_BUF_LEN - line_len, " "); + if (ret < 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "format dump byte string separator failed, ret:%d .\n", + ret); + return; + } + } + + line_len = strlen(line); + ret = sprintf_s(line + line_len, CHAR_BUF_LEN - line_len, + "%02x", buf[i]); + if (ret < 0) { + udma_err_limit( + udma_dev->dev, UDMA_M_MISC, + "format dump byte string failed, ret:%d .\n", + ret); + return; + } + } + + if (strlen(line) != 0) { + udma_err_limit(udma_dev->dev, UDMA_M_MISC, "%08x: %s\n", + ((i - DUMP_LINE_BYTE_NUM) / DUMP_LINE_BYTE_NUM) * + DUMP_LINE_BYTE_NUM, + line); + } +} + +int udma_cos_valid_check(struct udma_device *udma_dev, u8 *cos) +{ + struct hinic5_dcb_state dcb = { 0 }; + u8 func_cos_bitmap, port_cos_bitmap; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_MISC, "Invalid parameter. udma_dev(%d) .\n", + !!udma_dev); + return -ENXIO; + } + + ret = hinic5_cos_valid_bitmap(udma_dev->hwdev, &func_cos_bitmap, + &port_cos_bitmap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "get cos valid bitmap failed, ret: %d\n", ret); + return -EFAULT; + } + + if (*cos == udma_dev->default_cos.jfrc_cos) { + *cos = (u8)fls(func_cos_bitmap) - 1; + udma_info( + udma_dev->dev, UDMA_M_MISC, + "equal to jfrc_cos, mapped to the maximum effective cos %u\n", + *cos); + } + + if ((func_cos_bitmap & BIT((*cos))) != 0) { + udma_info(udma_dev->dev, UDMA_M_MISC, + "cos %u, func cos bitmap %u\n", *cos, + func_cos_bitmap); + return 0; + } + + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF) { + udma_info(udma_dev->dev, UDMA_M_MISC, + "vf cos %u, func cos bitmap %u\n", *cos, + func_cos_bitmap); + *cos = (u8)fls(func_cos_bitmap) - 1; + return 0; + } + + ret = hinic5_get_dcb_state(udma_dev->hwdev, &dcb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "hinic5_get_dcb_state failed.ret: %d\n", ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_MISC, + "cos %u, func cos bitmap %u, default cos %u\n", *cos, + func_cos_bitmap, dcb.default_cos); + + *cos = dcb.default_cos; + + return 0; +} + +int udma_get_dcb_cfg_default_cos(struct udma_device *udma_dev, u8 *cos) +{ + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF) { + *cos = (u8)(udma_dev->default_cos.vf_default_cos); + } else { + *cos = (u8)(udma_dev->default_cos.pf_default_cos); + } + + udma_info(udma_dev->dev, UDMA_M_MISC, "default cos: %u\n", *cos); + + return 0; +} + +int udma_get_dcb_cfg_cos(struct udma_device *udma_dev, struct ub_sip_attr *sip, + u8 pri, u8 *cos, u8 flag) +{ + int ret; + struct hinic5_dcb_state dcb = { 0 }; + u8 ta_flag = 1; + u8 mpri = pri; + + if (flag == ta_flag) { + if (udma_dev->default_cos.cfg_cos_en == 0) { + return udma_get_dcb_cfg_default_cos(udma_dev, cos); + } + *cos = pri & COS_TA_MASK; + return udma_cos_valid_check(udma_dev, cos); + } + + if (sip == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, "sip is null\n"); + return -EINVAL; + } + + ret = hinic5_get_dcb_state(udma_dev->hwdev, &dcb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "hinic5_get_dcb_state failed.ret: %d\n", ret); + return -EINVAL; + } + + *cos = dcb.default_cos; + + sip->dw6.dw6_value = cpu_to_le32(sip->dw6.dw6_value); + if (sip->dw6.tag == VLAN_INVALID && dcb.trust == UDMA_DCB_PCP) { + udma_debug(udma_dev->dev, UDMA_M_MISC, + "pcp cfg & no vlan should use default cos\n"); + return 0; + } + + mpri = (dcb.trust == UDMA_DCB_PCP) ? (pri & COS_TP_MASK) : + (pri & COS_TP_DSCP_MASK); + //PCP mode or not send down past dscp when 依然 from nic get + if (dcb.trust == UDMA_DCB_PCP || + (udma_dev->dscp_bitmap & BIT((u64)mpri)) == 0) { + ret = hinic5_get_cos_by_pri(udma_dev->hwdev, mpri, cos); + } else { + *cos = udma_dev->dscp2cos[mpri]; + } + + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "get_cos_by_pri failed.ret: %d, mask_pri:%u, dcb_on:%u, trust:%u.\n", + ret, mpri, dcb.dcb_on, dcb.trust); + return -EINVAL; + } + return 0; +} + +int udma_get_cos_by_pri(struct udma_device *udma_dev, u8 pri, u8 *cos) +{ + struct hinic5_dcb_state dcb = { 0 }; + u8 mpri = pri; + + int ret = hinic5_get_dcb_state(udma_dev->hwdev, &dcb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "hinic5_get_dcb_state failed.ret: %d, fe_idx: %u\n", + ret, adapted_ue_idx(&udma_dev->ub_dev.cfg)); + return -EINVAL; + } + + mpri = (dcb.trust == UDMA_DCB_PCP) ? (pri & COS_TP_MASK) : + (pri & COS_TP_DSCP_MASK); + ret = hinic5_get_cos_by_pri(udma_dev->hwdev, mpri, cos); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "[UDMA]: get_cos_by_pri failed.ret: %d, mask_pri:%u, dcb_on:%u, trust:%u.\n", + ret, mpri, dcb.dcb_on, dcb.trust); + return -EINVAL; + } + + return 0; +} + +int udma_rx_rate_limit_set_para(struct udma_device *udma_dev, + struct ub_rx_rate_limit_params *params) +{ + struct ub_rx_rate_limit_acc_cmd rx_rate_limit = { 0 }; + u16 out_size = (u16)sizeof(rx_rate_limit); + int ret; + + rx_rate_limit.para.pattern.down_period = params->down_period; + rx_rate_limit.para.pattern.up_period = params->up_period; + rx_rate_limit.para.pattern.alpha1 = params->alpha1; + rx_rate_limit.para.pattern.n1 = params->n1; + rx_rate_limit.para.pattern.alpha2 = params->alpha2; + rx_rate_limit.para.pattern.n2 = params->n2; + rx_rate_limit.para.pattern.alpha3 = params->alpha3; + rx_rate_limit.para.pattern.beta = params->beta; + rx_rate_limit.para.pattern.min_rate = params->min_rate; + rx_rate_limit.para.pattern.init_rate = params->init_rate; + + udma_info( + udma_dev->dev, UDMA_M_MISC, + "set rx rate limit params ,dec_period:%u inc_period:%u alpha1:%u n1:%u alpha2:%u n2:%u alpha3:%u beta:%u min_rate:%u init_rate:%u.\n", + params->down_period, params->up_period, params->alpha1, + params->n1, params->alpha2, params->n2, params->alpha3, + params->beta, params->min_rate, params->init_rate); + + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, UB_MPU_CMD_CFG_RRLT_ACC_PARA, + &rx_rate_limit, (u16)sizeof(rx_rate_limit), &rx_rate_limit, + &out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (rx_rate_limit.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "set rx rate limit params failed. ret: %d, status(0x%x)\n", + ret, rx_rate_limit.head.status); + return -EINVAL; + } + return 0; +} + +static void udma_cc_para_cmd_update(struct ub_cc_para *para, + struct ub_cc_params *params) +{ + switch (params->ub_cc_algo) { + case UB_CC_ALGO_LDCP: + para->ldcp.wnd_min = params->wnd_min; + para->ldcp.init_wnd = params->init_wnd; + para->ldcp.alpha = params->alpha; + para->ldcp.beta = params->beta; + para->ldcp.gamma = params->gamma; + para->ldcp.eta = params->eta; + break; + case UB_CC_ALGO_CAQM: + para->caqm.alpha = params->alpha; + para->caqm.beta = params->beta; + para->caqm.eta = params->eta; + para->caqm.wnd_min = params->wnd_min; + para->caqm.init_wnd = params->init_wnd; + para->caqm.cc_unit = params->cc_unit; + break; + case UB_CC_ALGO_USER_1: + case UB_CC_ALGO_USER_2: + para->dw[DW_IDX0] = params->dw[DW_IDX0]; + para->dw[DW_IDX1] = params->dw[DW_IDX1]; + para->dw[DW_IDX2] = params->dw[DW_IDX2]; + para->dw[DW_IDX3] = params->dw[DW_IDX3]; + break; + default: + break; + } +} + +int udma_rx_rate_limit_cfg_wred(struct udma_device *udma_dev, u32 cmd, u32 idx, + u32 *start_val, u32 *end_val) +{ + int ret; + struct ub_rx_rate_limit_wred_cmd cfg_wred_cmd = { 0 }; + u16 buf_size = (u16)sizeof(cfg_wred_cmd); + + if (udma_dev == NULL || start_val == NULL || end_val == NULL) { + return -EINVAL; + } + cfg_wred_cmd.cmd = cmd; + cfg_wred_cmd.wred_idx = idx; + cfg_wred_cmd.start_val = *start_val; + cfg_wred_cmd.end_val = *end_val; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CFG_RRLT_WRED, &cfg_wred_cmd, + buf_size, &cfg_wred_cmd, &buf_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (buf_size == 0) || (cfg_wred_cmd.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_MISC, + "set rx rate limit cfg wred failed. ret: %d, status(0x%x)\n", + ret, cfg_wred_cmd.head.status); + return -EIO; + } + + if (cmd != 0) { + *start_val = cfg_wred_cmd.start_val; + *end_val = cfg_wred_cmd.end_val; + } + + return 0; +} + +int udma_cc_set_para(struct udma_device *udma_dev, struct ub_cc_params *params) +{ + struct ub_cc_para_cmd cc_para = { 0 }; + u16 out_size = (u16)sizeof(cc_para); + int ret; + u8 i; + + cc_para.algo = params->ub_cc_algo; + cc_para.index = params->index; + udma_cc_para_cmd_update(&cc_para.para, params); + + for (i = 0; i < (u8)ARRAY_SIZE(cc_para.para.dw); i++) { + cc_para.para.dw[i] = cpu_to_be32(cc_para.para.dw[i]); + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_CC_PARA, &cc_para, + (u16)sizeof(cc_para), &cc_para, &out_size, + 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (cc_para.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "set cc params failed. ret: %d, status(0x%x)\n", ret, + cc_para.head.status); + return -EINVAL; + } + return 0; +} + +static void udma_cc_params_update(struct ub_cc_params *params, + struct ub_cc_para *para) +{ + switch (params->ub_cc_algo) { + case UB_CC_ALGO_LDCP: + params->wnd_min = para->ldcp.wnd_min; + params->init_wnd = para->ldcp.init_wnd; + params->alpha = para->ldcp.alpha; + params->beta = para->ldcp.beta; + params->gamma = para->ldcp.gamma; + params->eta = para->ldcp.eta; + break; + case UB_CC_ALGO_CAQM: + params->alpha = para->caqm.alpha; + params->beta = para->caqm.beta; + params->eta = para->caqm.eta; + params->wnd_min = para->caqm.wnd_min; + params->init_wnd = para->caqm.init_wnd; + params->cc_unit = para->caqm.cc_unit; + break; + default: + break; + } +} + +int udma_cc_query_para(struct udma_device *udma_dev, + struct ub_cc_params *params) +{ + struct ub_cc_para_cmd cc_para = { 0 }; + struct ub_cc_para_cmd para_out = { 0 }; + u16 out_size = (u16)sizeof(para_out); + int ret; + u8 i; + + cc_para.index = params->index; + cc_para.algo = params->ub_cc_algo; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CC_PARA, &cc_para, + (u16)sizeof(cc_para), &para_out, + &out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (para_out.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "query cc params failed. ret: %d, status(0x%x)\n", ret, + para_out.head.status); + return -EINVAL; + } + + for (i = 0; i < (u8)ARRAY_SIZE(para_out.para.dw); i++) { + para_out.para.dw[i] = be32_to_cpu(para_out.para.dw[i]); + } + + udma_cc_params_update(params, &para_out.para); + + return 0; +} + +static void udma_cc_entry_set(struct ubcore_cc_entry *cc_entry, u32 cnt, + enum ubcore_tp_cc_alg alg, u32 length) +{ + u32 j = 0, index = cnt; + + for (j = 0; j < length; j++) { + cc_entry[index].alg = alg; + cc_entry[index].cc_pattern_idx = (u8)j; + cc_entry[index++].cc_priority = (u8)j; + } +} + +struct ubcore_cc_entry *udma_query_cc(struct ubcore_device *ub_dev, + u32 *cc_entry_cnt) +{ + struct udma_device *udma_dev = NULL; + struct ubcore_cc_entry *cc_entry = NULL; + u32 cnt = 0; + int i = 0, array_num = 0; + + struct { + enum ubcore_tp_cc_alg algo; + u32 pattern_num; + u32 support_ver; + } support_cc[] = { + { UBCORE_TP_CC_LDCP, LDCP_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_LDCP_L2_HEADER, LDCP_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_LDCP_TP_HEADER, LDCP_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_CUSTOM_1, USER_DEFINED_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + { UBCORE_TP_CC_CUSTOM_2, USER_DEFINED_PATTERN_ARRAY_LEN, + UDMA_DEV_TYPE_HI1825_V100 }, + }; + + if (ub_dev == NULL || cc_entry_cnt == NULL) { + udma_pr_err( + UDMA_M_MISC, + "%s: Invalid parameter, ub_dev(%d), cc_entry_cnt(%d)\n", + __func__, !!ub_dev, !!cc_entry_cnt); + return NULL; + } + + udma_dev = to_udma_dev(ub_dev); + array_num = (int)ARRAY_SIZE(support_cc); + for (i = 0; i < array_num; ++i) { + if (((u32)udma_dev->device_type & support_cc[i].support_ver) != + 0) { + *cc_entry_cnt += support_cc[i].pattern_num; + } + } + if (*cc_entry_cnt == 0) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "no support cc for device type(%u).\n", + udma_dev->device_type); + return NULL; + } + cc_entry = (struct ubcore_cc_entry *)kzalloc( + *cc_entry_cnt * sizeof(struct ubcore_cc_entry), GFP_KERNEL); + if (cc_entry == NULL) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "cc_entry alloc failed.\n"); + return NULL; + } + + for (i = 0; i < array_num; ++i) { + if (((u32)udma_dev->device_type & support_cc[i].support_ver) != + 0) { + udma_cc_entry_set(cc_entry, cnt, support_cc[i].algo, + support_cc[i].pattern_num); + cnt += support_cc[i].pattern_num; + } + } + + return cc_entry; +} + +int udma_dscp_cos_map(struct udma_device *udma_dev, u8 *dscp2cos, u32 size, + u8 opcode) +{ + struct hinic5_cmd_qos_map_cfg qos_msp = { 0 }; + u16 out_size = (u16)sizeof(struct hinic5_cmd_qos_map_cfg); + int ret; + + qos_msp.op_code = opcode; + qos_msp.cfg_bitmap = CMD_QOS_MAP_DSCP2COS; + + ret = memcpy_s(qos_msp.dscp2cos, sizeof(qos_msp.dscp2cos), dscp2cos, + size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy failed, size:%u .\n", size); + return ret; + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_L2NIC, + HINIC5_NIC_CMD_QOS_MAP_CFG, &qos_msp, + out_size, &qos_msp, &out_size, 0, + HINIC5_CHANNEL_NIC); + if ((ret != 0) || (out_size == 0) || (qos_msp.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set qos failed, ret:%d, out_size:%u, status:%u .\n", + ret, out_size, qos_msp.head.status); + return -EFAULT; + } + + ret = memcpy_s(dscp2cos, size, qos_msp.dscp2cos, + sizeof(qos_msp.dscp2cos)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy failed, size:%lu .\n", + sizeof(qos_msp.dscp2cos)); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_DFX, "set qos success\n"); + + return 0; +} + +struct tag_cqm_queue *udma_object_queue_create( + struct udma_device *udma_dev, enum cqm_object_type object_type, + u32 object_size, void *object_priv, struct udma_object_n_record *record, + bool room_header_alloc, u32 xid, u32 bitmap_start, u32 bitmap_end) +{ + struct tag_cqm_queue *obj = NULL; + u32 record_start = record->num; + + if ((bitmap_start > bitmap_end) || (xid == CQM_INDEX_INVALID)) + return cqm5_object_rdma_queue_create(udma_dev->hwdev, + SERVICE_T_UB, object_type, + object_size, object_priv, + room_header_alloc, xid, + bitmap_start, bitmap_end); + + udma_debug(udma_dev->dev, UDMA_M_MISC, + "Alloc %u object, record(%u), xid(%u), start(%u), end(%u)\n", + object_type, record->num, xid, bitmap_start, bitmap_end); + mutex_lock(&record->mutex); + + if ((record_start < bitmap_start) || (record_start >= bitmap_end)) + record_start = bitmap_start; + + obj = cqm5_object_rdma_queue_create(udma_dev->hwdev, SERVICE_T_UB, + object_type, object_size, + object_priv, room_header_alloc, xid, + record_start, bitmap_end); + if ((obj != NULL) || (record_start == bitmap_start)) + goto leave; + + obj = cqm5_object_rdma_queue_create(udma_dev->hwdev, SERVICE_T_UB, + object_type, object_size, + object_priv, room_header_alloc, xid, + bitmap_start, bitmap_end); + +leave: + if (obj != NULL) { + record->num = obj->index + 1; + } + mutex_unlock(&record->mutex); + udma_debug(udma_dev->dev, UDMA_M_MISC, + "Alloced %u object, record(%u)\n", object_type, record->num); + + return obj; +} + +struct tag_cqm_qpc_mpt * +udma_object_qpc_create(struct udma_device *udma_dev, u32 object_size, + void *object_priv, struct udma_object_n_record *record, + u32 xid, u32 bitmap_start, u32 bitmap_end) +{ + struct tag_cqm_qpc_mpt *obj = NULL; + u32 record_start = record->num; + + if ((bitmap_start > bitmap_end) || (xid == CQM_INDEX_INVALID)) + return cqm5_object_qpc_mpt_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_SERVICE_CTX, + object_size, object_priv, xid, + bitmap_start, bitmap_end); + + udma_debug( + udma_dev->dev, UDMA_M_MISC, + "Alloc qpc object, record(%u), xid(%u), start(%u), end(%u)\n", + record->num, xid, bitmap_start, bitmap_end); + mutex_lock(&record->mutex); + + if ((record_start < bitmap_start) || (record_start >= bitmap_end)) + record_start = bitmap_start; + + obj = cqm5_object_qpc_mpt_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_SERVICE_CTX, object_size, + object_priv, xid, record_start, + bitmap_end); + if ((obj != NULL) || (record_start == bitmap_start)) + goto leave; + + obj = cqm5_object_qpc_mpt_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_SERVICE_CTX, object_size, + object_priv, xid, bitmap_start, + bitmap_end); + +leave: + if (obj != NULL) { + record->num = obj->xid + 1; + } + mutex_unlock(&record->mutex); + udma_debug(udma_dev->dev, UDMA_M_MISC, + "Alloced qpc object, record(%u)\n", record->num); + + return obj; +} + +int udma_get_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size) +{ + int ret; + ub_feat_nego_s nego_data; + u16 out_size = sizeof(ub_feat_nego_s); + + if (feat_bitmap == NULL || size < UB_FEAT_BITMAP_U64_MAX) { + udma_err(udev->dev, UDMA_M_MISC, + "Invalid parameters, feat_bitmap(%u), size(%u).", + !!feat_bitmap, size); + return -EINVAL; + } + + (void)memset_s(&nego_data, sizeof(ub_feat_nego_s), 0, + sizeof(ub_feat_nego_s)); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_FEAT_NEGO_GET, &nego_data, + sizeof(ub_feat_nego_s), &nego_data, + &out_size, 0, HINIC5_CHANNEL_UB); + if (nego_data.head.status == + HINIC5_MGMT_CMD_UNSUPPORTED) { //compatible with older firmware version + udma_warn(udev->dev, UDMA_M_MISC, + "The firmware version is too old!\n"); + (void)memset_s(feat_bitmap, sizeof(u64) * size, 0, + sizeof(u64) * size); + return 0; + } else if (ret != 0 || nego_data.head.status != 0 || out_size == 0) { + udma_err( + udev->dev, UDMA_M_MISC, + "Failed to get feature bitmap, ret(%d), status(%u), out_size(%u).\n", + ret, nego_data.head.status, out_size); + return -EIO; + } + + (void)memcpy_s(feat_bitmap, sizeof(u64) * size, nego_data.feat_bitmap, + sizeof(u64) * UB_FEAT_BITMAP_U64_MAX); + udev->index_queue_en = + udev->device_type == UDMA_DEV_TYPE_HI1825_V100 ? true : false; + + return 0; +} + +int udma_set_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size) +{ + int ret; + ub_feat_nego_s nego_data; + u16 out_size = sizeof(ub_feat_nego_s); + + if (feat_bitmap == NULL || size > UB_FEAT_BITMAP_U64_MAX) { + udma_err(udev->dev, UDMA_M_MISC, + "Invalid parameters, feat_bitmap(%u), size(%u).", + !!feat_bitmap, size); + return -EINVAL; + } + + (void)memset_s(&nego_data, sizeof(ub_feat_nego_s), 0, + sizeof(ub_feat_nego_s)); + (void)memcpy_s(nego_data.feat_bitmap, + sizeof(u64) * UB_FEAT_BITMAP_U64_MAX, feat_bitmap, + sizeof(u64) * size); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_FEAT_NEGO_SET, &nego_data, + sizeof(ub_feat_nego_s), &nego_data, + &out_size, 0, HINIC5_CHANNEL_UB); + if (nego_data.head.status == + HINIC5_MGMT_CMD_UNSUPPORTED) { //compatible with older firmware version + udma_warn(udev->dev, UDMA_M_MISC, + "The firmware version is too old!\n"); + return 0; + } else if (ret != 0 || nego_data.head.status != 0 || out_size == 0) { + udma_err( + udev->dev, UDMA_M_MISC, + "Failed to set feature bitmap, ret(%d), status(%u), out_size(%u).\n", + ret, nego_data.head.status, out_size); + return -EIO; + } + + return 0; +} + +u32 udma_get_qpn_mod(struct udma_device *udma_dev, u32 default_mode, + u32 default_xid_low) +{ + u32 low_bit_mode = default_mode; + u32 xid_low = default_xid_low; + + /* dfx set low bit align */ + if (udma_sysfs_get_xid_low_3_bit(&udma_dev->sysfs_res, &xid_low)) { + low_bit_mode = CQM_XID_LOW_BIT_1_1_1; + } + + return CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, low_bit_mode, xid_low); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h new file mode 100644 index 000000000..39d29071e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_misc.h @@ -0,0 +1,131 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_MISC_H +#define UDMA_MISC_H + +#include "udma_common.h" +#include "ub_pub_cmd.h" + +#define UDMA_DCB_PCP 0 +#define UDMA_DCB_DSCP 1 + +#define U64_BIT_SIZE 64 +#define UDMA_VERITY_FEAT_BIT(udev, shift) \ + ({ \ + u8 _ret = (shift) < (U64_BIT_SIZE * UB_FEAT_BITMAP_U64_MAX); \ + _ret = _ret ? (((udev)->feat_bitmap[(shift) / U64_BIT_SIZE] >> \ + ((shift) % U64_BIT_SIZE)) & \ + 0x1) : \ + 0; \ + _ret; \ + }) + +struct udma_list { + struct list_head node; + struct mutex node_lock; + u32 cnt; +}; + +struct udma_device_res { + struct udma_list tpn_list; + struct udma_list jfs_list; + struct udma_list jfr_list; + struct udma_list jetty_list; + struct udma_list jfc_list; + struct udma_list seg_list; + struct udma_list jetty_group_list; + struct udma_list tpg_list; + struct udma_list rc_list; + struct udma_list vtp_list; + struct udma_list utp_list; +}; + +struct udma_device_ctx_res { + struct ubcore_res_list_val tp_list; + struct ubcore_res_list_val jfs_list; + struct ubcore_res_list_val jfr_list; + struct ubcore_res_list_val jetty_list; + struct ubcore_res_list_val jfc_list; + struct ubcore_res_seg_val seg_list; + struct ubcore_res_list_val jetty_group_list; + struct ubcore_res_list_val tpg_list; + struct ubcore_res_list_val rc_list; + struct ubcore_res_list_val vtp_list; + struct ubcore_res_list_val utp_list; +}; + +static inline u64 align_size(u32 size, u64 align) +{ + return ((size + align - 1) & (~(align - 1))); +} + +int udma_dfx_device_init(struct udma_device *udma_dev); +void udma_dfx_device_deinit(struct udma_device *udma_dev); +int udma_query_res_dev(struct udma_device *udma_dev, + struct udma_device_ctx_res *dev_ctx_res); +void udma_free_dev_ctx_res(struct udma_device_ctx_res *dev_ctx_res); +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *user_ctl); +void udma_dump_dfx(struct udma_device *udma_dev, u64 istart, u32 ilen, + u64 buf_addr, u32 buf_size); +void udma_dump_byte_err_dfx(struct udma_device *udma_dev, const u8 *buf, + u32 ilen); +int udma_get_dcb_cfg_default_cos(struct udma_device *udma_dev, u8 *cos); +int udma_get_dcb_cfg_cos(struct udma_device *udma_dev, struct ub_sip_attr *sip, + u8 pri, u8 *cos, u8 flag); +int udma_get_cos_by_pri(struct udma_device *udma_dev, u8 pri, u8 *cos); +int udma_cc_set_para(struct udma_device *udma_dev, struct ub_cc_params *params); +int udma_rx_rate_limit_set_para(struct udma_device *udma_dev, + struct ub_rx_rate_limit_params *params); +int udma_rx_rate_limit_cfg_wred(struct udma_device *udma_dev, u32 cmd, u32 idx, + u32 *start_val, u32 *end_val); +int udma_cc_query_para(struct udma_device *udma_dev, + struct ub_cc_params *params); +struct ubcore_cc_entry *udma_query_cc(struct ubcore_device *ub_dev, + u32 *cc_entry_cnt); +void udma_store_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list); +void udma_delete_list(struct udma_device *udma_dev, struct list_head *node, + struct udma_list *udma_list); +void udma_init_ub_dev_attr(struct udma_device *udma_dev, + struct ubcore_device_attr *attr); +int udma_dscp_cos_map(struct udma_device *udma_dev, u8 *dscp2cos, u32 size, + u8 opcode); + +struct tag_cqm_queue *udma_object_queue_create( + struct udma_device *udma_dev, enum cqm_object_type object_type, + u32 object_size, void *object_priv, struct udma_object_n_record *record, + bool room_header_alloc, u32 xid, u32 bitmap_start, u32 bitmap_end); + +struct tag_cqm_qpc_mpt * +udma_object_qpc_create(struct udma_device *udma_dev, u32 object_size, + void *object_priv, struct udma_object_n_record *record, + u32 xid, u32 bitmap_start, u32 bitmap_end); +int udma_query_res_dev_tp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jfs_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jfr_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jetty_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jfc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_seg_list(struct udma_device *udma_dev, + struct ubcore_res_seg_val *ret_val); +int udma_query_res_dev_tpg_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_rc_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_vtp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_utp_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_query_res_dev_jetty_group_list(struct udma_device *udma_dev, + struct ubcore_res_list_val *ret_val); +int udma_get_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size); +int udma_set_feat_bitmap(struct udma_device *udev, u64 *feat_bitmap, u16 size); +u32 udma_get_qpn_mod(struct udma_device *udma_dev, u32 default_mode, + u32 default_xid_low); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c new file mode 100644 index 000000000..0f9f03e8f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> + +#include "udma_mtt.h" + +static int udma_umem_write_mtt_check(const struct udma_device *udma_dev, + const struct mtt *mtt, + const struct ubcore_umem *umem) +{ + if ((udma_dev == NULL) || (mtt == NULL) || (umem == NULL)) { + udma_pr_err(UDMA_M_SEG, "%s: Rdev or mtt or umem is null\n", + __func__); + return -EINVAL; + } + + return 0; +} + +int udma_mem_write_mtt_update(struct udma_device *udma_dev, struct mtt *mtt, + struct scatterlist *sgl, u32 nr) +{ + struct scatterlist *sg = NULL; + u32 pages_in_chunk = 0; /* umem_chunk in single (count word) memory block of page count */ + u64 *page_list = NULL; + u32 start_index = 0; /* need write mtt of page */ + u32 npages = 0; /* already record of page count */ + int ret = 0; + u32 i = 0; + u32 j = 0; + + page_list = (u64 *)kzalloc(PAGE_SIZE, GFP_KERNEL); + if (page_list == NULL) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "failed to allocate memory for page list, func_id(%u)\n", + hinic5_global_func_id(udma_dev->hwdev)); + return -ENOMEM; + } + + /* max page scenario sg_dma_len(sg) equal to page size, cause calculate of pages_in_chunk is 0, not will de execute hmm_mtt_write, need extra handle */ + if (mtt->mtt_layers == MTT_ZERO_LAYER && nr != 0) { + page_list[0] = sg_dma_address(sgl); + ret = hmm_mtt_write(udma_dev->hwdev, mtt, 0, 0, page_list, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d, func_id(%u)\n", + ret, hinic5_global_func_id(udma_dev->hwdev)); + } + goto out; + } + + /* calculate 1(count word) memory block in page count, page size as 1 << page_shift */ + for_each_sg(sgl, sg, nr, i) { + pages_in_chunk = UDMA_ROUND_UP(sg_dma_len(sg), PAGE_SIZE) >> + mtt->buf_page_shift; + for (j = 0; j < pages_in_chunk; ++j) { + page_list[npages] = + sg_dma_address(sg) + (PAGE_SIZE * j); + npages++; + + /* page_list pointed by memory size as a page, at most can store PAGE_SIZE / sizeof(u64) pa, full one page after need write mtt */ + if (npages != PAGE_SIZE / sizeof(u64)) { + continue; + } + ret = hmm_mtt_write(udma_dev->hwdev, mtt, start_index, + npages, page_list, SERVICE_T_UB); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d, start_index:%u, func_id(%u)\n", + ret, start_index, + hinic5_global_func_id(udma_dev->hwdev)); + goto out; + } + start_index += npages; + npages = 0; + } + } + + if (npages != 0) { + ret = hmm_mtt_write(udma_dev->hwdev, mtt, start_index, npages, + page_list, SERVICE_T_UB); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d, start_index:%u, func_id(%u)\n", + ret, start_index, + hinic5_global_func_id(udma_dev->hwdev)); + goto out; + } + } + +out: + kfree(page_list); + + return ret; +} + +int udma_umem_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct ubcore_umem *umem) +{ + int ret; + + ret = udma_umem_write_mtt_check(udma_dev, mtt, umem); + if (ret != 0) { + return ret; + } + + return udma_mem_write_mtt_update(udma_dev, mtt, umem->sg_head.sgl, + umem->nmap); +} + +int udma_buf_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct tag_cqm_buf *buf) +{ + u64 *page_list = NULL; + int ret = 0; + u32 i = 0; + + page_list = + (u64 *)kzalloc(buf->buf_number * sizeof *page_list, GFP_KERNEL); + if (page_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Failed to alloc memory for page list, func_id(%u)\n", + hinic5_global_func_id(udma_dev->hwdev)); + return (-ENOMEM); + } + + /* Each buf is written to MTT as a page, buf_size is 2^n times PAGE_SIZE */ + for (i = 0; i < buf->buf_number; ++i) + page_list[i] = buf->buf_list[i].pa; + + ret = hmm_mtt_write(udma_dev->hwdev, mtt, 0, buf->buf_number, page_list, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Failed to write mtt, func_id(%u)\n", + hinic5_global_func_id(udma_dev->hwdev)); + } + + kfree(page_list); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h new file mode 100644 index 000000000..8e81eae51 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/common/udma_mtt.h @@ -0,0 +1,19 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_MTT_H +#define UDMA_MTT_H + +#include "udma_common.h" + +#define UDMA_ROUND_UP(n, d) (DIV_ROUND_UP(n, d) * (d)) + +int udma_umem_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct ubcore_umem *umem); +int udma_mem_write_mtt_update(struct udma_device *udma_dev, struct mtt *mtt, + struct scatterlist *sgl, u32 nr); +int udma_buf_write_mtt(struct udma_device *udma_dev, struct mtt *mtt, + struct tag_cqm_buf *buf); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c new file mode 100644 index 000000000..e80c7bc23 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.c @@ -0,0 +1,81 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/kernel.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_db.h" + +int udma_db_map_user(struct udma_context *ctx, u64 virt, struct udma_db *db) +{ + u64 page_addr = virt & PAGE_MASK; + union ubcore_umem_flag access = { 0 }; + struct udma_user_db_page *page = NULL; + u64 offset; + int ret = 0; + + if (ctx == NULL) { + udma_pr_err(UDMA_M_DB, "Invalid parameter. ctx(%d) .\n", !!ctx); + return -EINVAL; + } + + mutex_lock(&ctx->db_page_mutex); + + list_for_each_entry(page, &ctx->db_page_list, list) { + if (page->user_virt == page_addr) + goto found; + } + + page = (struct udma_user_db_page *)kmalloc(sizeof(*page), GFP_KERNEL); + if (page == NULL) { + udma_err(ctx->udma_dev->dev, UDMA_M_DB, + "alloc doorbell page failed.\n"); + ret = -ENOMEM; + goto out; + } + + page->refcount = 0; + page->user_virt = page_addr; + access.bs.non_pin = 0; + access.bs.writable = 1; + page->umem = ubcore_umem_get(&ctx->udma_dev->ub_dev, page_addr, + PAGE_SIZE, access); + if (IS_ERR_OR_NULL(page->umem)) { + udma_err(ctx->udma_dev->dev, UDMA_M_DB, + "umem get db page failed.\n"); + ret = -EINVAL; + kfree(page); + goto out; + } + + list_add(&page->list, &ctx->db_page_list); + +found: + offset = virt - page_addr; + db->dma = sg_dma_address(page->umem->sg_head.sgl) + offset; + db->virt_addr = (char *)sg_virt(page->umem->sg_head.sgl) + offset; + db->user_page = page; + ++page->refcount; + +out: + mutex_unlock(&ctx->db_page_mutex); + + return ret; +} + +void udma_db_unmap_user(struct udma_context *ctx, struct udma_db *db) +{ + mutex_lock(&ctx->db_page_mutex); + + if ((--db->user_page->refcount) == 0) { + list_del(&db->user_page->list); + ubcore_umem_release(db->user_page->umem); + kfree(db->user_page); + db->user_page = NULL; + } + + mutex_unlock(&ctx->db_page_mutex); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h new file mode 100644 index 000000000..37d05d32b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/base/db/udma_db.h @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#ifndef _UDMA_DB_H +#define _UDMA_DB_H + +#include <linux/types.h> +#include <linux/list.h> + +#include "ubcore_api.h" +#include "udma_misc.h" +#include "udma_ucontext.h" + +#define TP_DB_SIZE 8 + +struct udma_db { + u32 *db_record; + struct udma_user_db_page *user_page; + u64 dma; + void *virt_addr; +}; + +struct udma_user_db_page { + struct list_head list; + struct ubcore_umem *umem; + u64 user_virt; + int refcount; +}; + +struct udma_kdb { + u32 *db_record; + dma_addr_t dma; + void *virt_addr; + u16 idx; //only for dca +}; + +int udma_db_map_user(struct udma_context *ctx, u64 virt, struct udma_db *db); +void udma_db_unmap_user(struct udma_context *ctx, struct udma_db *db); + +#endif /* _UDMA_DB_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h new file mode 100644 index 000000000..49a68f472 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_BOND_H +#define UDMA_BOND_H + +int udma_bond_init(void); +void udma_bond_uninit(void); +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c new file mode 100644 index 000000000..991c54833 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/bond/udma_bond_common.c @@ -0,0 +1,163 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. + +#include <net/bonding.h> +#include <linux/netdevice.h> +#include "hinic5_hw.h" +#include "hinic5_srv_nic.h" + +#include "drv_bond_api.h" +#include "udma_log.h" +#include "udma_bond.h" + +struct udma_bond_resource { + atomic_t bond_id; +}; + +#define UDMA_BOND_2_FUNC_NUM 2 +#define UDMA_INVALID_BOND_ID 0xFF + +static struct udma_bond_resource g_udma_bond_res; + +static enum netdev_lag_tx_type udma_get_tx_type_by_bond_mode(u16 bond_mode) +{ + switch (bond_mode) { + case BOND_MODE_8023AD: + return NETDEV_LAG_TX_TYPE_HASH; + case BOND_MODE_ACTIVEBACKUP: + return NETDEV_LAG_TX_TYPE_ACTIVEBACKUP; + default: + return NETDEV_LAG_TX_TYPE_UNKNOWN; + } +} + +static bool udma_bond_mode_is_supported(u16 bond_mode) +{ + enum netdev_lag_tx_type tx_type = + udma_get_tx_type_by_bond_mode(bond_mode); + if ((tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) && + (tx_type != NETDEV_LAG_TX_TYPE_HASH)) { + udma_pr_err(UDMA_M_BOND, "Failed to support bond mode(%d)\n", + bond_mode); + return false; + } + return true; +} + +static bool udma_can_do_bond(struct bonding *bond) +{ + bool ret = false; + int slave_cnt = 0; + struct slave *slave = NULL; + struct list_head *iter = NULL; + struct hinic5_lld_dev *lld_dev = NULL; + struct hinic5_lld_dev *ppf_dev = NULL; + + if (!bond || !udma_bond_mode_is_supported((u16)(bond->params.mode))) { + return ret; + } + + rcu_read_lock(); + bond_for_each_slave_rcu(bond, slave, iter) { + lld_dev = hinic5_get_lld_dev_by_netdev(slave->dev); + if (!lld_dev) { + goto out; + } + + if (!ppf_dev) { + ppf_dev = hinic5_get_ppf_lld_dev_unsafe(lld_dev); + if (!ppf_dev) { + goto out; + } + } + + if (hinic5_get_ppf_lld_dev_unsafe(lld_dev) != ppf_dev) { + goto out; + } + slave_cnt++; + udma_pr_info(UDMA_M_BOND, + "%s:can do bond? slave_cnt(%d), slave_name(%s)", + __func__, slave_cnt, slave->dev->name); + } + + ret = (slave_cnt == UDMA_BOND_2_FUNC_NUM); +out: + rcu_read_unlock(); + return ret; +} + +static void udma_before_bond_active(const char *bond_name, + struct bond_attr *attr, int err) +{ + atomic_set(&g_udma_bond_res.bond_id, attr->bond_id); + return; +} + +static void udma_after_bond_modify(const char *bond_name, + struct bond_attr *attr, int err) +{ + struct bond_tracker tracker; + int ret; + + if (bond_name == NULL || attr == NULL) { + udma_pr_err(UDMA_M_BOND, "udma bond_name or attr is NULL."); + return; + } + + ret = hinic5_bond_get_tracker_by_name(bond_name, &tracker); + if (ret != 0) { + udma_pr_err( + UDMA_M_BOND, + "Failed to get bond tracker, bond name(%s), ret(%d).", + bond_name, ret); + return; + } +} + +static void udma_before_bond_deactive(const char *bond_name, + struct bond_attr *attr, int err) +{ + atomic_set(&g_udma_bond_res.bond_id, UDMA_INVALID_BOND_ID); +} + +int udma_bond_init(void) +{ + int ret; + struct bond_srv_func ub_bond_attr = { + .before_active = udma_before_bond_active, + .after_active = NULL, + .before_modify = NULL, + .after_modify = udma_after_bond_modify, + .before_deactive = udma_before_bond_deactive, + .after_deactive = NULL, + .can_attach = udma_can_do_bond + }; + + atomic_set(&g_udma_bond_res.bond_id, UDMA_INVALID_BOND_ID); + + ret = hinic5_bond_register(HINIC5_BOND_USER_UB, &ub_bond_attr); + if (ret != 0) { + udma_pr_err( + UDMA_M_BOND, + "%s: Failed to register bond service func, ret(%d)\n", + __func__, ret); + return ret; + } + udma_pr_info(UDMA_M_BOND, "udma bond init success\n"); + + return 0; +} + +void udma_bond_uninit(void) +{ + int ret; + + ret = hinic5_bond_unregister(HINIC5_BOND_USER_UB); + if (ret != 0) { + udma_pr_err( + UDMA_M_BOND, + "%s: Failed to unregister bond service func, ret(%d).", + __func__, ret); + return; + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c new file mode 100644 index 000000000..31e08f760 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.c @@ -0,0 +1,325 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/kernel.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/semaphore.h> +#include <linux/mm.h> + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_resource.h" +#include "udma_abi.h" +#include "udma_ucontext.h" + +static void udma_alloc_ucontext_resp_set(struct udma_device *udma_dev, + const struct udma_context *udma_ctx, + struct ub_create_ctx_resp *resp) +{ + struct ub_dev_cap *ub_cap = &udma_dev->ub_cap.dev_cap; + + resp->max_jfc = ub_cap->max_jfc; + resp->max_jfs = ub_cap->max_jfs; + resp->max_jfr = ub_cap->max_jfr; + resp->max_jetty = ub_cap->max_jetty; + resp->max_jfs_inline_size = ub_cap->max_jfs_inline_size; + resp->max_jfs_sge = ub_cap->max_jfs_sge; + resp->max_jfs_rsge = ub_cap->max_jfs_rsge; + resp->max_jfr_sge = ub_cap->max_jfr_sge; + resp->dwqe_size = udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size; + + resp->db_offset = udma_ctx->db_dma_addr & ((1 << PAGE_SHIFT) - 1); + resp->dwqe_offset = udma_ctx->dwqe_dma_addr & ((1 << PAGE_SHIFT) - 1); + if (udma_ctx->dwqe_dma_addr == 0) + resp->dwqe_size = 0; + resp->dwqe_dis = udma_sysfs_get_dwqe_dis(&udma_dev->sysfs_res); + resp->device_type = udma_dev->device_type; + resp->env_type = udma_dev->env_type; + resp->mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + resp->debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + resp->dcs_en = udma_dev->ub_cap.dev_cap.dcs_en; + resp->index_queue_en = udma_get_index_queue_en(udma_dev); + resp->fis_en = udma_sysfs_get_fis_en(&udma_dev->sysfs_res); +} + +struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *ub_dev, + u32 eid_index, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_device *udma_dev = NULL; + struct udma_context *udma_ctx = NULL; + struct ub_create_ctx_resp resp = { 0 }; + u32 mtt_pro_en = 0; + int ret; + + if (ub_dev == NULL || udrv_data == NULL) { + udma_pr_err( + UDMA_M_DEV, + "parameters is incorrect, ub_dev(%d), udrv_data(%d).\n", + !!ub_dev, !!udrv_data); + return ERR_PTR(-EINVAL); + } + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "tpf not support this function.\n"); + return ERR_PTR(-EINVAL); + } + + udma_ctx = kzalloc(sizeof(struct udma_context), GFP_KERNEL); + if (udma_ctx == NULL) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "udma_ctx context allocation failed.\n"); + return ERR_PTR(-ENOMEM); + } + + udma_ctx->udma_dev = udma_dev; + + /* Alloc user space context Doorbell and DWQE */ + ret = hinic5_alloc_db_phy_addr(udma_dev->hwdev, &udma_ctx->db_dma_addr, + &udma_ctx->dwqe_dma_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc DB pa, ret(%d)\n", ret); + goto err_db_phy_addr_alloc; + } + + /* init software doorbell */ + INIT_LIST_HEAD(&udma_ctx->db_page_list); + mutex_init(&udma_ctx->db_page_mutex); + + ret = udma_sysfs_get_mtt_pro_en(&udma_dev->sysfs_res, &mtt_pro_en); + if (ret != 0) { + goto err_resp; + } + udma_dev->ub_cap.dev_cap.mtt_pro_en = mtt_pro_en; + udma_alloc_ucontext_resp_set(udma_dev, udma_ctx, &resp); + + /* fill create ctx cmd resp */ + if (copy_to_user((void *)(uintptr_t)udrv_data->out_addr, &resp, + min((u32)udrv_data->out_len, (u32)sizeof(resp))) != + 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "fill create ctx resp failed\n"); + goto err_resp; + } + + return &udma_ctx->base; +err_resp: + mutex_destroy(&udma_ctx->db_page_mutex); + + hinic5_free_db_phy_addr(udma_dev->hwdev, udma_ctx->db_dma_addr, + udma_ctx->dwqe_dma_addr); +err_db_phy_addr_alloc: + kfree(udma_ctx); + return ERR_PTR(-EFAULT); +} + +int udma_free_ucontext(struct ubcore_ucontext *ucontext) +{ + struct udma_context *udma_ctx = NULL; + struct udma_device *udma_dev = NULL; + + if (ucontext == NULL) { + udma_pr_err(UDMA_M_DEV, + "parameters is incorrect, ucontext(%d).\n", + !!ucontext); + return -EINVAL; + } + udma_ctx = to_udma_ctx(ucontext); + udma_dev = udma_ctx->udma_dev; + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_DEV, "udma_dev is null\n"); + return -EINVAL; + } + + hinic5_free_db_phy_addr(udma_dev->hwdev, udma_ctx->db_dma_addr, + udma_ctx->dwqe_dma_addr); + mutex_destroy(&udma_ctx->db_page_mutex); + udma_info(udma_dev->dev, UDMA_M_DEV, "free ucontext success.\n"); + kfree(udma_ctx); + + return 0; +} + +static void udma_vma_close(struct vm_area_struct *vma) +{ + struct udma_queue_buffer_info *buf = vma->vm_private_data; + + if (buf != NULL) { + hinic5_dma_free_coherent_align(buf->dev, &buf->mem_align); + kfree(buf); + } +} + +static const struct vm_operations_struct udma_vm_ops = { + .close = udma_vma_close, +}; + +static bool udma_buffer_is_page_align(struct udma_device *udma_dev, + struct vm_area_struct *vma) +{ + size_t size = ((size_t)vma->vm_end - vma->vm_start); + + if (size % PAGE_SIZE != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "(Vm_end - vm_start) is not align to PAGE_SIZE\n"); + return false; + } + + return true; +} + +static void udma_fill_mmap_queue_buf_info(struct udma_device *udma_dev, + size_t buf_size, + struct udma_queue_buffer_info *buf) +{ + buf->buf_va = buf->mem_align.align_vaddr; + buf->dma_addr = buf->mem_align.align_paddr; + buf->size = buf_size; + buf->dev = udma_dev->dev; +} + +static int udma_queue_dma_alloc(struct udma_device *udma_dev, + struct vm_area_struct *vma, + struct udma_queue_buffer_info *buf) +{ + size_t size; + int ret; + + size = ((size_t)vma->vm_end - vma->vm_start); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, size, size, + GFP_KERNEL, &buf->mem_align); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + + udma_fill_mmap_queue_buf_info(udma_dev, size, buf); + return 0; +} + +int udma_mmap_queue_buffer(struct udma_device *udma_dev, + struct vm_area_struct *vma) +{ + struct udma_queue_buffer_info *buf = NULL; + size_t size; + struct iommu_domain *dom = NULL; + phys_addr_t phy_addr; + + if (!udma_buffer_is_page_align(udma_dev, vma)) { + return -EINVAL; + } + + buf = kzalloc(sizeof(struct udma_queue_buffer_info), GFP_KERNEL); + if (buf == NULL) { + udma_err(udma_dev->dev, UDMA_M_DEV, "Failed to alloc buf\n"); + return -ENOMEM; + } + + if (udma_queue_dma_alloc(udma_dev, vma, buf) != 0) { + kfree(buf); + return -ENOMEM; + } + + size = ((size_t)vma->vm_end - vma->vm_start); +#ifndef __aarch64__ + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + dom = iommu_get_domain_for_dev(udma_dev->dev); + phy_addr = dom ? iommu_iova_to_phys(dom, buf->dma_addr) : buf->dma_addr; + if (io_remap_pfn_range(vma, vma->vm_start, phy_addr >> PAGE_SHIFT, size, + vma->vm_page_prot) != 0) { + hinic5_dma_free_coherent_align(udma_dev->dev, &buf->mem_align); + kfree(buf); + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to do queue buf io remap\n"); + return -EAGAIN; + } + + vma->vm_private_data = buf; + vma->vm_ops = &udma_vm_ops; + return 0; +} + +int udma_mmap(struct ubcore_ucontext *ucontext, struct vm_area_struct *vma) +{ + struct udma_device *udma_dev = NULL; + struct udma_context *udma_ctx = NULL; + unsigned long db_pfn = 0; + unsigned long dwqe_pfn = 0; + s64 size; + + if (ucontext == NULL || ucontext->ub_dev == NULL || vma == NULL) { + udma_pr_err(UDMA_M_DEV, + "ucontext(%d), vma(%d) or ub_dev is NULL.\n", + !!ucontext, !!vma); + return -EINVAL; + } + + udma_dev = to_udma_dev(ucontext->ub_dev); + udma_ctx = to_udma_ctx(ucontext); + + db_pfn = udma_ctx->db_dma_addr >> PAGE_SHIFT; + dwqe_pfn = udma_ctx->dwqe_dma_addr >> PAGE_SHIFT; + size = (s64)vma->vm_end - (s64)vma->vm_start; + + /* Map hardware doorbell from physical address to virtual address */ + if (vma->vm_pgoff == USR_MMAP_DB_OFFSET) { + if (size % PAGE_SIZE != 0) { + udma_err( + udma_dev->dev, UDMA_M_DEV, + "(Vm_end - vm_start) is not equal to PAGE_SIZE\n"); + return -EINVAL; + } + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* construct vm_start~vm_start+PAGE_SIZE page table + * db_pfn is page number + * vm_page_prot means attr + */ + if (io_remap_pfn_range(vma, vma->vm_start, db_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Failed to do db io remap\n", __func__); + return -EAGAIN; + } + return 0; + } + + /* Map dwqe from physical address to virtual address */ + if ((vma->vm_pgoff == USR_MMAP_DWQE_OFFSET) && + (udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size != 0)) { + if (size % PAGE_SIZE != 0) { + udma_err( + udma_dev->dev, UDMA_M_DEV, + "%s: (Vm_end - vm_start) is not equal to PAGE_SIZE\n", + __func__); + return -EINVAL; + } + +#ifdef __aarch64__ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); +#endif + + if (io_remap_pfn_range(vma, vma->vm_start, dwqe_pfn, PAGE_SIZE, + vma->vm_page_prot) != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "%s: Failed to do dwqe io remap\n", __func__); + return -EAGAIN; + } + return 0; + } + + if (vma->vm_pgoff == USR_MMAP_QBUF_OFFSET) { + return udma_mmap_queue_buffer(udma_dev, vma); + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h new file mode 100644 index 000000000..1d6bbdd6b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/context/udma_ucontext.h @@ -0,0 +1,51 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_UCONTEXT_H +#define UDMA_UCONTEXT_H + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" + +enum { USR_MMAP_DB_OFFSET = 0, USR_MMAP_DWQE_OFFSET, USR_MMAP_QBUF_OFFSET }; + +struct udma_context { + struct ubcore_ucontext base; + struct udma_device *udma_dev; + u64 db_dma_addr; + u64 dwqe_dma_addr; + u32 uasid; + struct list_head db_page_list; /* software doorbell page list */ + struct mutex db_page_mutex; /* software doorbell page mutex */ +}; + +struct udma_queue_buffer_info { + dma_addr_t dma_addr; /* queue dma addr */ + void *buf_va; /* queue va */ + u64 size; /* queue size */ + struct hinic5_dma_addr_align mem_align; + struct device *dev; +}; + +static inline struct udma_context * +to_udma_ctx(const struct ubcore_ucontext *ucontext) +{ + return (struct udma_context *)udma_container_of( + ucontext, struct udma_context, base); +} + +static inline struct ubcore_ucontext *udma_get_uctx(struct ubcore_udata *udata) +{ + return ((udata == NULL) || (udata->udrv_data == NULL)) ? NULL : + udata->uctx; +} + +struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *ub_dev, + u32 eid_index, + struct ubcore_udrv_priv *udrv_data); +int udma_free_ucontext(struct ubcore_ucontext *ucontext); +int udma_mmap(struct ubcore_ucontext *ucontext, struct vm_area_struct *vma); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c new file mode 100644 index 000000000..84c9a87f9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#include <linux/device.h> +#include <linux/iommu.h> +#include <linux/delay.h> + +#include "udma_common.h" +#include "ub_mpu_cmd.h" +#include "comm_defs.h" +#include "udma_srq.h" +#include "udma_cqm.h" +#include "udma_cmd.h" + +int udma_set_func_attr(struct udma_device *udma_dev, u16 func_id, + struct ub_cmd_func_attr *func_attr) +{ + u16 out_size = sizeof(struct ub_cmd_func_attr); + int ret; + + if (udma_dev == NULL || func_attr == NULL) { + udma_pr_err( + UDMA_M_CMD, + "parameters is incorrect, ub_dev(%d), func_attr(%d).\n", + !!udma_dev, !!func_attr); + return -EINVAL; + } + + func_attr->func_id = func_id; + + if (func_attr->attr1_mask.value == 0 && + func_attr->attr2_mask.value == 0) { + udma_info(udma_dev->dev, UDMA_M_CMD, + "mask is zero, return 0\n"); + return 0; + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_FUNC_ATTR, func_attr, + out_size, func_attr, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (func_attr->head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "cmd set func attr failed, ret:%d, out_size:%u, status:%u", + ret, out_size, func_attr->head.status); + return -EFAULT; + } + udma_info(udma_dev->dev, UDMA_M_CMD, "set func(%d) table success,", + func_id); + return 0; +} + +int udma_cfg_ipsu_tbl_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_addr_info *addr_info, + enum cmd_mpu_ipsu_type type) +{ + struct ub_cmd_ipsu_ip_entry ipsu_ip_entry_param = { 0 }; + struct ub_cmd_ipsu_ip_entry ipsu_ip_entry_resp = { 0 }; + union ub_ip_info *ip_addr = &addr_info->ip_addr; + u16 out_size = sizeof(ipsu_ip_entry_resp); + u16 ipsu_type; + int ret; + u32 i; + + for (i = 0; i < ARRAY_SIZE(ip_addr->dw); i++) + ip_addr->dw[i] = htonl(ip_addr->dw[i]); + + ipsu_ip_entry_param.func_id = hinic5_global_func_id(udma_dev->hwdev); + ipsu_ip_entry_param.ip_ver = addr_info->ip_ver; + ipsu_ip_entry_param.port_id = 0; // from tpf send down request, port_id no has meaning + (void)memcpy_s(&ipsu_ip_entry_param.ip, sizeof(union ub_ip_info), + ip_addr, sizeof(union ub_ip_info)); + + udma_info(udma_dev->dev, UDMA_M_CMD, + "udma cfg ipsu tbl ipaddr: ip_type:%d,portid=%u\n", + ipsu_ip_entry_param.ip_ver, ipsu_ip_entry_param.port_id); + + ipsu_type = (type == TYPE_MPU_IPSU_ADD) ? UB_MPU_CMD_ADD_IP_ENTRY : + UB_MPU_CMD_DEL_IP_ENTRY; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + (u16)ipsu_type, &ipsu_ip_entry_param, + sizeof(ipsu_ip_entry_param), + &ipsu_ip_entry_resp, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || + (ipsu_ip_entry_resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "Failed to type:%d ip entry, err(%d), status(0x%x), out size(0x%x)\n", + ipsu_type, ret, ipsu_ip_entry_resp.head.status, + out_size); + if (ipsu_ip_entry_resp.head.status == ENXIO) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "ip entry exist. \n"); + return -EEXIST; + } + return -EFAULT; + } + + return 0; +} + +int udma_del_func_res(struct udma_device *udma_dev) +{ + struct ub_cmd_func_attr func_state_info = { 0 }; + u16 out_size = sizeof(func_state_info); + int ret; + + func_state_info.func_id = hinic5_global_func_id(udma_dev->hwdev); + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DEL_FUNC_RES, &func_state_info, + out_size, &func_state_info, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || + (func_state_info.head.status != 0)) { + udma_warn( + udma_dev->dev, UDMA_M_CMD, + "cmd set func res failed, ret:%d, out_size:%u, status:%u", + ret, out_size, func_state_info.head.status); + return -EFAULT; + } + if (udma_dev->fake_dma_addr.va != NULL) { + dma_free_coherent(udma_dev->dev, (size_t)PAGE_SIZE_4K, + udma_dev->fake_dma_addr.va, + udma_dev->fake_dma_addr.pa); + udma_dev->fake_dma_addr.va = NULL; + udma_dev->fake_dma_addr.pa = 0; + } + udma_info(udma_dev->dev, UDMA_M_CMD, "udma_del_func_res success"); + return 0; +} + +void udma_get_bus_info(struct device *dev, struct udma_bus_addr *bus_addr) +{ + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + uint32_t dbdf = UDMA_PCI_DEV_DBDF(pdev); + + bus_addr->domain = UDMA_PCI_DBDF_DOM(dbdf); + bus_addr->bus = UDMA_PCI_DBDF_BUS(dbdf); + bus_addr->devid = UDMA_PCI_DBDF_DEVID(dbdf); + bus_addr->function = UDMA_PCI_DBDF_FUNCTION(dbdf); +} + +int udma_query_mig_vf_status(struct udma_device *udma_dev, u16 func_id) +{ + struct udma_dev_bdf_info *bdf_info = udma_dev->bdf_info; + struct hinic5_lld_dev *spu_lld_dev = NULL; + struct hinic5_lld_dev *ppf_lld_dev = NULL; + struct udma_bus_addr bus_addr; + u8 vm_status = HIMIGR_VF_FREE; + u16 size = sizeof(vm_status); + void *ppf_hw_dev = NULL; + u32 dbdf; + int ret; + + if (bdf_info == NULL || func_id >= UB_FUNCTION_MAX_NUM || + bdf_info[func_id].bdf_info_vld == 0) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "function %d bdf info is not valid\n", func_id); + return MIG_VF_INVAL; + } + + udma_get_bus_info(udma_dev->dev, &bus_addr); + dbdf = UDMA_QUERY_DBDF_BY_BDF_INFO(&bus_addr); + + udma_info(udma_dev->dev, UDMA_M_CMD, "[DFX] dbdf is %d\n", dbdf); + spu_lld_dev = hinic5_get_lld_dev_by_netdev(udma_dev->ndev); + if (spu_lld_dev == NULL) { + udma_pr_err(UDMA_M_CMD, + "get spu lld dev failed, lld dev is null.\n"); + return MIG_VF_INVAL; + } + ppf_lld_dev = hinic5_get_ppf_lld_dev(spu_lld_dev); + if (ppf_lld_dev == NULL) { + udma_pr_err(UDMA_M_CMD, + "get ppf lld dev failed, lld dev is null.\n"); + return MIG_VF_INVAL; + } + ppf_hw_dev = ppf_lld_dev->hwdev; + if (ppf_hw_dev == NULL) { + udma_pr_err(UDMA_M_CMD, "get ppf failed, ppf dev is null.\n"); + hinic5_lld_dev_put(ppf_lld_dev); + return MIG_VF_INVAL; + } + ret = hinic5_mbox_ppf_to_host(ppf_hw_dev, HINIC5_MOD_MIGRATE, 0, 0, + &dbdf, sizeof(dbdf), &vm_status, &size, 0, + HINIC5_CHANNEL_UB); + hinic5_lld_dev_put(ppf_lld_dev); + if (ret != 0 || size != sizeof(vm_status)) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "failed send msg to host %u ret:%d\n", SPU_HOST_ID, + ret); + return MIG_VF_INVAL; + } + if (vm_status == HIMIGR_VF_IN_VM) { + return MIG_VF_IN_VM; + } + return MIG_VF_NOT_IN_VM; +} + +int udma_ipsu_ta_vf_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_ueid_addr_info *addr_info, + enum cmd_mpu_ipsu_type type, u16 vf_id) +{ + struct ub_cmd_ipsu_ta_ip_entry ipsu_ip_entry_param = { 0 }; + struct ub_cmd_ipsu_ta_ip_entry ipsu_ip_entry_resp = { 0 }; + union ub_ip_info *ip_addr = &addr_info->ip; + u16 out_size = sizeof(ipsu_ip_entry_resp); + u16 ipsu_type; + int ret; + u32 i; + + for (i = 0; i < ARRAY_SIZE(ip_addr->dw); i++) { + ip_addr->dw[i] = htonl(ip_addr->dw[i]); + } + (void)memcpy_s(&ipsu_ip_entry_param.ip, sizeof(union ub_ip_info), + ip_addr, sizeof(union ub_ip_info)); + + if (udma_dev->ub_dev.cfg.pattern == + UBCORE_PATTERN_1) { /** pattern1 mode, send down ip */ + ipsu_ip_entry_param.nlp = 0; + if (udma_dev->ub_dev.sip_table.entry->sip_info.addr.type == + UBCORE_NET_ADDR_TYPE_IPV4) { + ipsu_ip_entry_param.ip_ver = 0; + } else { + ipsu_ip_entry_param.ip_ver = 1; + } + } else { /** pattern3 mode, send down upi+eid */ + ipsu_ip_entry_param.nlp = 1; + ipsu_ip_entry_param.upi = addr_info->upi; + } + ipsu_ip_entry_param.func_id = vf_id; + + ipsu_type = (type == TYPE_MPU_IPSU_ADD) ? UB_MPU_CMD_ADD_TA_IP_ENTRY : + UB_MPU_CMD_DEL_TA_IP_ENTRY; + udma_info(udma_dev->dev, UDMA_M_CMD, + "ipsu_type:%d, upi:0x%x, vf_id:0x%x\n", ipsu_type, + ipsu_ip_entry_param.upi, ipsu_ip_entry_param.func_id); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + (u16)ipsu_type, &ipsu_ip_entry_param, + sizeof(ipsu_ip_entry_param), + &ipsu_ip_entry_resp, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || + (ipsu_ip_entry_resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "Failed to type:%d ip entry, err(%d), status(0x%x), out size(0x%x)\n", + ipsu_type, ret, ipsu_ip_entry_resp.head.status, + out_size); + if (ipsu_ip_entry_resp.head.status == ENXIO) { + udma_err(udma_dev->dev, UDMA_M_CMD, + "ip entry exist.\n"); + return -EEXIST; + } + return -EFAULT; + } + return 0; +} + +void udma_cmdq_rtt_statistic_init(struct cmdq_rtt *rtt) +{ + atomic64_set(&rtt->times, 0); + atomic64_set(&rtt->cmdq_rtt_sum, 0); + atomic64_set(&rtt->cmdq_npu_rtt_sum, 0); +} + +void udma_print_cmdq_rtt_stat(struct udma_device *udma_dev, const char *mod, + u32 idx, struct cmdq_rtt *rtt) +{ + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_err( + udma_dev->dev, UDMA_M_CMD, + "module[%s] idx(%u) times(%llu) cmdq_rtt_sum(%llu) cmdq_npu_rtt_sum(%llu)\n", + mod, idx, (u64)atomic64_read(&rtt->times), + (u64)atomic64_read(&rtt->cmdq_rtt_sum), + (u64)atomic64_read(&rtt->cmdq_npu_rtt_sum)); + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h new file mode 100644 index 000000000..0d5548733 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_cmd.h @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_CMD_H +#define UDMA_CMD_H + +#include <linux/fwnode.h> +#include <linux/irqdomain.h> +#include <linux/pci.h> +#include <linux/types.h> +#include <linux/iommu.h> + +#include "udma_common.h" + +#define HIMIGR_VF_FREE 0 +#define HIMIGR_VF_IN_VM 1 +#define MIG_VF_IN_VM 0 // user ctrl opcode 0x4 message return value, 0: indicates VF is in VM in +#define MIG_VF_NOT_IN_VM \ + 1 // user ctrl opcode 0x4 message return value, 1: indicates VF not is in VM in +#define MIG_VF_INVAL 2 // user ctrl opcode 0x4 message return value, 2: other error + +/** + * @brief set func table attribute value and same when set corresponding to mask bit + * @param func_attr must is struct ub_cmd_func_attr type of change amount (cannot is pointer) + * @param dw func_attr struct pointed by in some (count word) union member of name, the union contains specific attribute fields. + * @param attr name of attribute field to set (exists in both dw struct member pointed by and dw##_mask.bs bitfield in). + * @param value value to assign to attribute field. + */ +#define SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, dw, attr, value) \ + do { \ + (func_attr).dw.attr = (value); \ + (func_attr).dw##_mask.bs.attr = 1; \ + } while (0) + +/** + * @brief via pointer set func table attribute value and set corresponding to mask bit + * @param func_attr pointed by struct ub_cmd_func_attr type of pointer (cannot is empty) + * @param dw func_attr struct pointed by in some (count word) union member of name, the union contains specific attribute fields. + * @param attr name of attribute field to set (exists in both dw struct member pointed by and dw##_mask.bs bitfield in). + * @param value value to assign to attribute field. + */ +#define SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, dw, attr, value) \ + do { \ + (func_attr)->dw.attr = (value); \ + (func_attr)->dw##_mask.bs.attr = 1; \ + } while (0) + +enum cmd_mpu_ipsu_type { TYPE_MPU_IPSU_ADD, TYPE_MPU_IPSU_DEL }; + +struct cmd_mpu_addr_info { + u8 ip_ver; // 0x0：IPv4; 0x1：IPv6。 + u8 rsvd[3]; + union ub_ip_info ip_addr; +}; + +struct cmd_mpu_ueid_addr_info { + u8 ip_ver; // 0x0：IPv4; 0x1：IPv6; 0x2：upi。 + u8 nlp; + u8 rsvd[2]; + union ub_ip_info ip; + u32 upi; +}; + +int udma_set_func_attr(struct udma_device *udma_dev, u16 func_id, + struct ub_cmd_func_attr *func_attr); +int udma_cfg_ipsu_tbl_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_addr_info *addr_info, + enum cmd_mpu_ipsu_type type); +int udma_del_func_res(struct udma_device *udma_dev); +int udma_ipsu_ta_vf_ipaddr(struct udma_device *udma_dev, + struct cmd_mpu_ueid_addr_info *addr_info, + enum cmd_mpu_ipsu_type type, u16 vf_id); +void udma_cmdq_rtt_statistic_init(struct cmdq_rtt *rtt); +void udma_print_cmdq_rtt_stat(struct udma_device *udma_dev, const char *mod, + u32 idx, struct cmdq_rtt *rtt); +int udma_query_mig_vf_status(struct udma_device *udma_dev, u16 func_id); +void udma_get_bus_info(struct device *dev, struct udma_bus_addr *bus_addr); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h new file mode 100644 index 000000000..275533f2a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_comp.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Hisilicon Technologies Co., Ltd. 2021. All rights reserved. + +#ifndef _UDMA_COMP_H +#define _UDMA_COMP_H + +#include "hmm_common.h" +#include "hmm_em_inner.h" + +#define UBRC_MIN_DEPTH 1 +#define UBRC_MAX_DEPTH 512 +#define UBRC_EXT_ENTRY 16 +#define UBRC_MIN_ENTRY 8 /* min entry aligned required by engine */ +#define UBRC_MAXSIZE_OFFSET 8 +#define UB_INVALID_INDEX 0xFFFFFFFF +#define MAX_DEST_RM_ATOMIC 128 + +struct ub_own_cap { + u32 cmtt_cl_start; + u32 cmtt_cl_end; + u32 cmtt_cl_sz; + u32 wqe_cl_start; + u32 wqe_cl_end; + u32 wqe_cl_sz; + u32 ubrc_depth; + u32 ubrc_entry_sz; + u32 log_ubrc_seg; +}; + +struct udma_comp_priv { + struct hmm_comp_priv hmm_priv; + struct ub_own_cap ub_cap; + struct hmm_buddy ubrc_buddy; + struct hmm_em_table ubrc_em_table; + void *udma_dev; +}; + +struct ubrc { + u32 offset; /* allocate continuous index of first index */ + u32 order; /* allocate of ubrc of order, represents (done) count */ + u32 ext_order; /* contain rc table and extension table count */ + u64 dma_addr; + void *vaddr; +}; + +#endif \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c new file mode 100644 index 000000000..ab11c5ad4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/netdevice.h> + +#include "ubcore_api.h" +#include "ubcore_types.h" +#include "udma_common.h" +#include "udma_net.h" +#include "udma_rc_table.h" + +#define MAX_IO_SIZE 0x80000000 /** supports UB device read/write maximum max IO specification 2G */ + +enum ub_port_speed { + UB_SPEED_SDR = 1, + UB_SPEED_DDR = 2, + UB_SPEED_QDR = 4, + UB_SPEED_FDR10 = 8, + UB_SPEED_FDR = 16, + UB_SPEED_EDR = 32 +}; + +enum ub_port_width { + UB_WIDTH_1X = 1, + UB_WIDTH_4X = 2, + UB_WIDTH_8X = 4, + UB_WIDTH_12X = 8 +}; + +static void udma_init_dev_feature(struct udma_device *udma_dev, + struct ubcore_device_attr *attr) +{ + struct ub_dev_cap *dev_cap = &udma_dev->ub_cap.dev_cap; + + attr->dev_cap.feature.bs.oor = dev_cap->feature.bs.oor; + attr->dev_cap.feature.bs.selective_retrans = + dev_cap->feature.bs.selective_retrans; + attr->dev_cap.feature.bs.dca = + (dev_cap->feature.bs.dca_rx & dev_cap->feature.bs.dca_tx); + attr->dev_cap.feature.bs.jfc_inline = dev_cap->feature.bs.jfc_inline; +} + +void udma_init_dev_atomic(struct udma_device *udma_dev, + struct ubcore_device_attr *attr) +{ + attr->dev_cap.atomic_feat.value = 0; + attr->dev_cap.max_cas_size = 0; + attr->dev_cap.max_swap_size = 0; + attr->dev_cap.max_fetch_and_add_size = 0; + attr->dev_cap.max_fetch_and_sub_size = 0; + attr->dev_cap.max_fetch_and_and_size = 0; + attr->dev_cap.max_fetch_and_or_size = 0; + attr->dev_cap.max_fetch_and_xor_size = 0; +} + +void udma_init_ub_dev_attr(struct udma_device *udma_dev, + struct ubcore_device_attr *attr) +{ + struct ub_dev_cap *dev_cap = &udma_dev->ub_cap.dev_cap; + struct ub_dev_cap_udma_res *udma_res = + &udma_dev->ub_cap.dev_cap_res.udma_res; + struct ub_net_dev_cap *net_dev_cap = &udma_dev->ub_cap.net_dev_cap; + int i; + + udma_init_dev_feature(udma_dev, attr); + udma_init_dev_atomic(udma_dev, attr); + + attr->dev_cap.max_jfc = dev_cap->max_jfc; + attr->dev_cap.max_jfs = dev_cap->max_jfs; + attr->dev_cap.max_jfr = dev_cap->max_jfr; + attr->dev_cap.max_jetty = dev_cap->max_jetty; + attr->dev_cap.max_jetty_grp = dev_cap->max_jetty_grp; + attr->dev_cap.max_jetty_in_jetty_grp = dev_cap->max_jetty_in_jetty_grp; + attr->dev_cap.max_jfc_depth = dev_cap->max_jfc_depth; + attr->dev_cap.max_jfs_depth = dev_cap->max_jfs_depth; + attr->dev_cap.max_jfr_depth = dev_cap->max_jfr_depth; + attr->dev_cap.max_jfs_inline_size = dev_cap->max_jfs_inline_size; + attr->dev_cap.max_jfs_sge = dev_cap->max_jfs_sge; + attr->dev_cap.max_jfs_rsge = dev_cap->max_jfs_rsge; + attr->dev_cap.max_jfr_sge = dev_cap->max_jfr_sge; + attr->dev_cap.max_msg_size = dev_cap->max_msg_size; + attr->dev_cap.max_rc_outstd_cnt = dev_cap->max_rc_outstd_cnt; + attr->dev_cap.trans_mode = UBCORE_TP_RM | UBCORE_TP_RC | UBCORE_TP_UM; + attr->dev_cap.ceq_cnt = (uint16_t)net_dev_cap->comp_vector_cnt; + attr->dev_cap.max_rc = udma_res->max_jfrc_num; + attr->dev_cap.max_rc_depth = udma_res->jfrc_depth; + attr->dev_cap.min_slice = dev_cap->min_slice; + attr->dev_cap.max_slice = dev_cap->max_slice; + attr->dev_cap.max_read_size = MAX_IO_SIZE; + attr->dev_cap.max_write_size = MAX_IO_SIZE; + + attr->dev_cap.max_sip_cnt_per_ue = dev_cap->max_sip_cnt_per_vf; + attr->dev_cap.max_dip_cnt_per_ue = dev_cap->max_dip_cnt_per_vf; + attr->dev_cap.max_seid_cnt_per_ue = dev_cap->max_seid_cnt_per_vf; + attr->dev_cap.max_ue_cnt = net_dev_cap->vf_cnt; + + attr->dev_cap.max_oor_cnt = dev_cap->max_oor_cnt; + attr->dev_cap.max_tp_in_tpg = dev_cap->max_tp_in_tpg; + attr->virtualization = (dev_cap->virtualization == 1) ? true : false; + adapted_ue_idx(attr) = hinic5_global_func_id(udma_dev->hwdev); + attr->dev_cap.max_upi_cnt = dev_cap->max_upi_cnt; + attr->port_cnt = (uint8_t)net_dev_cap->port_cnt; + + for (i = 0; i < UBCORE_MAX_PORT_CNT; i++) { + attr->port_attr[i].max_mtu = + udma_mtu_int_to_enum(net_dev_cap->max_mtu); + } + attr->tp_maintainer = (net_dev_cap->is_tpf == 1) ? true : false; + + attr->dev_cap.max_eid_cnt = dev_cap->max_eid_cnt; + attr->pattern = + (udma_dev->ub_cap.dev_cap.pattern_type == PATTERN_TYPE_3) ? + UBCORE_PATTERN_3 : + UBCORE_PATTERN_1; +} + +int udma_query_device_attr(struct ubcore_device *ub_dev, + struct ubcore_device_attr *attr) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + if (udma_dev == NULL || attr == NULL) { + udma_pr_err( + UDMA_M_DEV, + "query device attr invalid parameter. udma_dev(%d), attr(%d).\n", + !!udma_dev, !!attr); + return -EINVAL; + } + + udma_init_ub_dev_attr(udma_dev, attr); + + return 0; +} + +static void udma_set_port_attr(struct ubcore_port_status *port_status) +{ + port_status->active_mtu = UBCORE_MTU_256; + port_status->state = UBCORE_PORT_DOWN; +} + +static void udma_set_speed(enum mag_cmd_port_speed speed, + struct ubcore_port_status *port_status) +{ + switch (speed) { + /* 10G <==> 1X x 10G */ + case PORT_SPEED_10GB: + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_10G; + break; + + /* 25G <==> 1X x 25G */ + case PORT_SPEED_25GB: + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_25G; + break; + + /* 40G <==> 4X x 10G */ + case PORT_SPEED_40GB: + port_status->active_width = UBCORE_LINK_X4; + port_status->active_speed = UBCORE_SP_40G; + break; + + /* 100G <==> 4X x 25G */ + case PORT_SPEED_100GB: + port_status->active_width = UBCORE_LINK_X4; + port_status->active_speed = UBCORE_SP_100G; + break; + + default: + port_status->active_width = 0; + port_status->active_speed = 0; + break; + } +} + +int udma_query_device_status(struct ubcore_device *dev, + struct ubcore_device_status *status) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + enum mag_cmd_port_speed speed = PORT_SPEED_10GB; + struct net_device *net_dev = NULL; + enum ubcore_mtu net_dev_mtu; + enum ubcore_mtu mtu; + u32 port_nums; + u32 length; + int ret; + u32 i; + + if (udma_dev == NULL || status == NULL) { + udma_pr_err( + UDMA_M_DEV, + "query query device status invalid parameter. udma_dev(%d), status(%d).\n", + !!udma_dev, !!status); + return -EINVAL; + } + + port_nums = udma_dev->ub_cap.net_dev_cap.port_cnt; + length = (sizeof(status->port_status) / + sizeof(struct ubcore_port_status)); + if (port_nums > length) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "port_nums is %u, port_status length is %u!\n", + port_nums, length); + return -EINVAL; + } + + for (i = 0; i < port_nums; i++) { + net_dev = udma_dev->ndev; + udma_set_port_attr(&status->port_status[i]); + + if (udma_dev->speed == PORT_SPEED_NOT_SET) { + ret = hinic5_get_speed(udma_dev->hwdev, &speed, + HINIC5_CHANNEL_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "failed to get speed!\n"); + status->port_status[i].active_speed = 0; + status->port_status[i].active_mtu = 0; + return ret; + } + udma_dev->speed = speed; + } + speed = udma_dev->speed; + udma_set_speed(speed, &status->port_status[i]); + + if (netif_running(net_dev)) { + status->port_status[i].state = UBCORE_PORT_ACTIVE; + } + status->port_status[i].active_mtu = UBCORE_MTU_1024; + net_dev_mtu = udma_mtu_int_to_enum( + udma_dev->ub_cap.net_dev_cap.max_mtu); + mtu = ubcore_get_mtu((int)net_dev->mtu); + if (mtu != 0) + status->port_status[i].active_mtu = + udma_get_support_mtu( + min((u32)net_dev_mtu, (u32)mtu)); + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h new file mode 100644 index 000000000..3c30e8cd4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_device.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DEVICE_H +#define UDMA_DEVICE_H + +#include "ubcore_types.h" + +int udma_query_device_attr(struct ubcore_device *ub_dev, + struct ubcore_device_attr *attr); +int udma_query_device_status(struct ubcore_device *dev, + struct ubcore_device_status *status); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c new file mode 100644 index 000000000..705c2025b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> + +#include "comm_defs.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "hinic5_common.h" +#include "ub_mpu_cmd.h" +#include "ub_npu_upi_cmd_defs.h" +#include "ub_npu_eid_cmd_defs.h" +#include "ubcore_api.h" +#include "ubcore_msg.h" +#include "udma_common.h" +#include "udma_comp.h" +#include "udma_cqm.h" +#include "udma_query.h" +#include "udma_misc.h" +#include "udma_rc_table.h" +#include "udma_resource.h" +#include "udma_vtp.h" +#include "udma_cmd_eid.h" +#include "udma_cmd_com.h" +#include "fast_msg_common_define.h" +#include "hinic5_fast_msg.h" +#include "udma_fast_msg.h" + +#define MAX_DEV_REG_NUM 128 +#define FUNC_ID_BOUND_FOR_PF 32 +#define UDMA_IS_FST_MSG_START(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_START)) +#define UDMA_IS_FST_MSG_END(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_END)) +#define UDMA_IS_FST_MSG_FLS_MODIFY(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_FLS_MODIFY)) +#define UDMA_FST_MSG_NEED_RSP(opcode) \ + (UDMA_IS_FST_MSG_START(opcode) || UDMA_IS_FST_MSG_END(opcode) || \ + UDMA_IS_FST_MSG_FLS_MODIFY(opcode)) +#define UDMA_IS_FST_MSG_RESTORE_VTP(opcode) \ + (((opcode) >= ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_START)) && \ + ((opcode) <= \ + ((enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_FLS_MODIFY))) +#define UDMA_IS_FST_UPDATE_UPI_NORM(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_NORM)) +#define UDMA_IS_FST_UPDATE_UPI_MIG(opcode) \ + ((opcode) == ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_MIG)) +#define UDMA_IS_FST_UPDATE_UPI(opcode) \ + (((opcode) >= ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_NORM)) && \ + ((opcode) <= ((enum ubcore_msg_opcode)UB_MIG_UPDATE_UPI_MIG))) + +struct udma_msg_dev_info { + u32 used; + u32 func_id; + struct udma_device *udma_dev; +}; + +static struct udma_msg_dev_info g_devs_info[MAX_DEV_REG_NUM]; + +int udma_get_func_id(struct udma_device *udma_dev, u32 *func_id) +{ + int ret; + struct ub_cmd_get_func_id cmd_in = { 0 }; + struct ub_cmd_get_func_id_resp cmd_out = { 0 }; + u16 out_size = sizeof(struct ub_cmd_get_func_id_resp); + u16 in_size = sizeof(struct ub_cmd_get_func_id); + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_FUNC_ID, &cmd_in, in_size, + &cmd_out, &out_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (cmd_out.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_FMSG, + "cmd get func id failed, ret:%d, out_size:%u, status:%u\n", + ret, out_size, cmd_out.head.status); + return -EFAULT; + } + + udma_debug(udma_dev->dev, UDMA_M_FMSG, + "cmd get func id success, func id:%u, tpf id:%u\n", + cmd_out.func_id, cmd_out.tpf_id); + + *func_id = cmd_out.func_id; + + return 0; +} + +void udma_fast_msg_handler(struct hisdk5_fast_msg_buf *fast_msg, + void *priv_data); +int udma_mbox_msg_vf_handler(void *pri_handle, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size); + +int udma_msg_dev_register(struct udma_device *udma_dev) +{ + int i; + + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if (g_devs_info[i].used != 0) { + continue; + } + + g_devs_info[i].used = 1; + g_devs_info[i].udma_dev = udma_dev; + g_devs_info[i].func_id = hinic5_global_func_id(udma_dev->hwdev); + break; + } + + /* new VM driver compatible with old firmware and SPU driver */ + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF && + hinic5_support_fast_msg(udma_dev->hwdev) == false) { + return hinic5_register_vf_mbox_cb(udma_dev->hwdev, + HINIC5_MOD_FAKE_FMSG, + udma_dev, + udma_mbox_msg_vf_handler); + } + + return hinic5_fast_msg_register_cb(udma_dev->hwdev, HINIC5_MOD_UB, + udma_fast_msg_handler, udma_dev); +} + +int udma_msg_dev_unregister(struct udma_device *udma_dev) +{ + int i; + + udma_info(udma_dev->dev, UDMA_M_FMSG, "msg dev unregister: %s\n", + udma_dev->ub_dev.dev_name); + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if ((g_devs_info[i].used != 0) && + (g_devs_info[i].udma_dev == udma_dev)) { + g_devs_info[i].used = 0; + break; + } + } + + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF && + hinic5_support_fast_msg(udma_dev->hwdev) == false) { + hinic5_unregister_vf_mbox_cb(udma_dev->hwdev, + HINIC5_MOD_FAKE_FMSG); + } else { + hinic5_fast_msg_unregister_cb(udma_dev->hwdev, HINIC5_MOD_UB); + } + + return 0; +} + +#define FAST_MSG_BUFFER_NOT_ENOUGH 2 +#define FAST_MSG_RETRY_CNT 10 +static int udma_send_fast_msg(struct udma_device *udma_dev, + struct hinic5_cmd_buf *cmd_buf) +{ + struct hisdk5_fast_msg_header *fmsg_head = cmd_buf->buf; + u64 out_param = 0; + u16 retry_cnt; + int ret; + + cmd_buf->size = + fmsg_head->data_len + sizeof(struct hisdk5_fast_msg_header); + hinic5_cpu_to_be32(cmd_buf->buf, cmd_buf->size); + + retry_cnt = 0; + do { + ret = hinic5_fast_msg_send(udma_dev->hwdev, cmd_buf, + &out_param); + if (out_param != FAST_MSG_BUFFER_NOT_ENOUGH && ret != -EBUSY) { + break; + } + msleep(1); + retry_cnt++; + } while (retry_cnt < FAST_MSG_RETRY_CNT); + + if (ret != 0 || out_param != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "fast msg send failed, ret:%d, out_param: 0x%llx\n", + ret, out_param); + return -EIO; + } + + return 0; +} + +static bool udma_is_from_same_card(struct udma_device *dev_a, + struct udma_device *dev_b) +{ + return (dev_a->dev_bus_type == dev_b->dev_bus_type) && + (dev_a->card_id == dev_b->card_id); +} + +static struct udma_device * +udma_msg_find_tp_function(struct udma_device *udma_dev, u32 src_func_id) +{ + struct udma_device *tmp_tpf = NULL; + int i; + + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if ((g_devs_info[i].used != 0) && + (g_devs_info[i].udma_dev->ub_dev.attr.tp_maintainer) && + udma_is_from_same_card(g_devs_info[i].udma_dev, udma_dev) && + (g_devs_info[i].udma_dev->host_id == udma_dev->host_id)) { + tmp_tpf = g_devs_info[i].udma_dev; + /* tuningfirst per odd/even func_id match tpf, compatible single tpf scenario */ + if ((g_devs_info[i].func_id & 0x1) == + (src_func_id & 0x1)) { + return tmp_tpf; + } + } + } + + return tmp_tpf; +} + +struct udma_device *udma_find_dev_by_fe_idx(struct udma_device *udma_dev, + u32 func_id) +{ + int i; + + for (i = 0; i < MAX_DEV_REG_NUM; i++) { + if ((g_devs_info[i].used != 0) && + (g_devs_info[i].func_id == func_id) && + udma_is_from_same_card(udma_dev, g_devs_info[i].udma_dev) && + (g_devs_info[i].udma_dev->host_id == udma_dev->host_id)) { + return g_devs_info[i].udma_dev; + } + } + + return NULL; +} + +int udma_rsp_msg_to_mig(struct udma_device *dst_dev, + struct ubcore_req_host *req_host, int status) +{ + int ret; + struct ubcore_resp_host *resp_host = NULL; + struct ubcore_function_mig_resp *resp_msg = NULL; + struct ubcore_function_mig_req *req_msg = + (struct ubcore_function_mig_req *)req_host->req.data; + struct hisdk5_fast_msg_header *fmsg_head = NULL; + struct hinic5_cmd_buf *cmd_buf = NULL; + + udma_info(dst_dev->dev, UDMA_M_FMSG, + "[dst dev] opcode: %u, func id: %u\n", req_host->req.opcode, + adapted_type_ue_idx(req_msg, mig)); + + cmd_buf = hinic5_alloc_cmd_buf(dst_dev->hwdev); + if (cmd_buf == NULL) { + udma_err(dst_dev->dev, UDMA_M_FMSG, "alloc cmd buf failed\n"); + return -ENOMEM; + } + + fmsg_head = cmd_buf->buf; + resp_host = cmd_buf->buf + sizeof(struct hisdk5_fast_msg_header); + adapted_type_ue_idx(resp_host, dst) = 0; + resp_host->resp.msg_id = req_host->req.msg_id; + resp_host->resp.opcode = req_host->req.opcode; + resp_host->resp.len = sizeof(struct ubcore_function_mig_resp); + + resp_msg = (struct ubcore_function_mig_resp *)resp_host->resp.data; + adapted_type_ue_idx(resp_msg, mig) = adapted_type_ue_idx(req_msg, mig); + resp_msg->status = status; + + fmsg_head->status = 0; + fmsg_head->mod = HINIC5_MOD_UB; + fmsg_head->cmd = UB_MSG_WITH_MIG; + fmsg_head->dst_func_id = adapted_type_ue_idx(resp_host, dst); + fmsg_head->send = false; + fmsg_head->data_len = + resp_host->resp.len + sizeof(struct ubcore_resp_host); + fmsg_head->nack = 0; + fmsg_head->ulp_format = HISDK5_FAST_MSG_ULP_FROMAT_NONE; + fmsg_head->rsvd = 0; + fmsg_head->rsvd1 = 0; + ret = udma_send_fast_msg(dst_dev, cmd_buf); + hinic5_free_cmd_buf(dst_dev->hwdev, cmd_buf); + udma_info(dst_dev->dev, UDMA_M_FMSG, + "[FMSG] send mig msg resp, ret: %d\n", ret); + + return ret; +} + +static int udma_fastmsg_restore_start(struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + int ret; + + ret = udma_restore_all_vtp_cmd(dst_dev, buf); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: [START], restore all vtp failed, ret:%d\n", + ret); + } else { + buf->mig_sts = UDMA_MIG_INCREMENTAL; + } + return ret; +} + +static int udma_fastmsg_restore_end(struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + int ret; + + ret = udma_cmd_restore_vtp_remain_vtpn(dst_dev, buf); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: [END] restore remain vtp failed, ret:%d\n", + ret); + } else { + buf->mig_sts = UDMA_MIG_INCREMENTAL_MODIFY; + } + return ret; +} + +int udma_fastmsg_restore_fls_modify(struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + int ret; + + ret = udma_flush_remain_modify(dst_dev, buf); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: [FLUSH MODIFY] Fls remain failed, ret:%d\n", + ret); + } else { + buf->mig_sts = UDMA_MIG_NORMAL; + } + return ret; +} + +int udma_vtp_restore_handle(void *buf_in, struct udma_device *dst_dev, + struct udma_vtp_buf *buf) +{ + struct ubcore_req_host *req_host = (struct ubcore_req_host *)buf_in; + u16 vfid = buf->func_id; + int rsp_msg_ret = 0; + int ret = 0; + int status; + + if (req_host->req.opcode == + (enum ubcore_msg_opcode)UB_MIG_VTP_RESTORE_CANCEL) { + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: vf:%u recv CANCEL msg.\n", vfid); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: vf:%u is idle.\n", vfid); + return 0; + } + udma_set_vtp_buf_idle(dst_dev, buf); + return 0; + } + + if (UDMA_FST_MSG_NEED_RSP(req_host->req.opcode)) { + mutex_lock(&buf->vf_mutex); + if (!UDMA_IS_VF_MIGRATING( + buf->mig_sts)) { // may this when UVS already trigger (done) rollback free (done) buf, because this in this determine status + mutex_unlock(&buf->vf_mutex); + udma_warn(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: vf:%u is already idle.\n", vfid); + return udma_rsp_msg_to_mig(dst_dev, req_host, + UBCORE_MIG_MSG_PROC_SUCCESS); + } + + if (UDMA_IS_FST_MSG_START(req_host->req.opcode)) { + ret = udma_fastmsg_restore_start(dst_dev, buf); + } else if (UDMA_IS_FST_MSG_END(req_host->req.opcode)) { + ret = udma_fastmsg_restore_end(dst_dev, buf); + } else if (UDMA_IS_FST_MSG_FLS_MODIFY(req_host->req.opcode)) { + ret = udma_fastmsg_restore_fls_modify(dst_dev, buf); + } + mutex_unlock(&buf->vf_mutex); + status = (ret == 0) ? UBCORE_MIG_MSG_PROC_SUCCESS : + UBCORE_MIG_MSG_PROC_FAILURE; + rsp_msg_ret = udma_rsp_msg_to_mig(dst_dev, req_host, status); + if (UDMA_IS_FST_MSG_FLS_MODIFY(req_host->req.opcode)) { + udma_set_vtp_buf_idle(dst_dev, buf); + } + } + + return (ret | rsp_msg_ret); +} + +static int udma_update_upi_mig_status(struct ubcore_req_host *req_host, + struct udma_device *dst_dev, u16 vfid, + ub_upi_mig_state_e upi_mig_sts) +{ + ub_seid_ctx_info_s seid_ctx = { 0 }; + int status, ret; + u32 idx; + + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Start to update upi vf:%u mig_sts:%u\n", vfid, upi_mig_sts); + + for (idx = 0; idx < UDMA_SEID_LOOP; idx++) { + (void)memset_s(&seid_ctx, sizeof(ub_seid_ctx_info_s), 0, + sizeof(ub_seid_ctx_info_s)); + ret = udma_cmd_seid_query_ignore_ub_disable_err( + dst_dev, vfid, idx, &seid_ctx, + sizeof(ub_seid_ctx_info_s)); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Update upi: get seid failed, vf:%u, ret:%d\n", + vfid, ret); + goto out; + } + // seid from microcode read out to of time already is big-endian, later send to microcode also need is big-endian, this at not do convert + udma_be32_to_cpu(&seid_ctx, sizeof(seid_ctx.dw0_value) + + sizeof(seid_ctx.upi)); + + if (seid_ctx.vld == 0) { + continue; + } + + ret = udma_cmd_update_upi(dst_dev, vfid, &seid_ctx, + upi_mig_sts); + if (ret != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Update upi failed, vf:%u, ret:%d\n", vfid, + ret); + goto out; + } + udma_info(dst_dev->dev, UDMA_M_FMSG, + "Update upi success, vf:%u seid_idx:%u\n", vfid, idx); + } + +out: + status = (ret == 0) ? UBCORE_MIG_MSG_PROC_SUCCESS : + UBCORE_MIG_MSG_PROC_FAILURE; + return udma_rsp_msg_to_mig(dst_dev, req_host, status); +} + +int udma_mig_msg_handler(struct udma_device *udma_dev, + struct hisdk5_fast_msg_buf *fast_msg) +{ + struct ubcore_function_mig_req *req_msg = NULL; + struct ubcore_req_host *req_host = NULL; + struct udma_device *dst_dev = udma_dev; + struct udma_vtp_buf *buf = NULL; + u32 upi_mig_sts; + int opcode; + + if ((udma_dev == NULL) || (fast_msg == NULL)) { + udma_pr_err(UDMA_M_FMSG, + "ub mig msg handler input para is null.\n"); + return -EINVAL; + } + + req_host = (struct ubcore_req_host *)fast_msg->fast_msg_data; + req_msg = (struct ubcore_function_mig_req *)req_host->req.data; + + opcode = req_host->req.opcode; + udma_info(dst_dev->dev, UDMA_M_FMSG, "Fast msg: opcode:0x%x, vf:%u\n", + opcode, adapted_type_ue_idx(req_msg, mig)); + + if (fast_msg->fast_msg_header.src_func_id != 0) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: src_func_id:%u is not migrate, vfid:%u\n", + fast_msg->fast_msg_header.src_func_id, + adapted_type_ue_idx(req_msg, mig)); + return -EINVAL; + } + + if (!UDMA_IS_MIG_VFID(adapted_type_ue_idx(req_msg, mig))) { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: func id:%u invalid\n", + adapted_type_ue_idx(req_msg, mig)); + return -EINVAL; + } + + if (UDMA_IS_FST_MSG_RESTORE_VTP(opcode)) { + buf = udma_mig_get_vf_buf(dst_dev, + adapted_type_ue_idx(req_msg, mig)); + return udma_vtp_restore_handle(req_host, dst_dev, buf); + } else if (UDMA_IS_FST_UPDATE_UPI(opcode)) { + upi_mig_sts = UDMA_IS_FST_UPDATE_UPI_NORM(opcode) ? + UB_UPI_NORM : + UB_UPI_MIGR; + return udma_update_upi_mig_status( + req_host, dst_dev, adapted_type_ue_idx(req_msg, mig), + upi_mig_sts); + } else { + udma_err(dst_dev->dev, UDMA_M_FMSG, + "Fast msg: opcode:%u invalid, vf:%u\n", opcode, + adapted_type_ue_idx(req_msg, mig)); + return -EINVAL; + } +} + +void udma_fast_msg_drain_handler(struct ubcore_fastmsg_buf *fastmsg_buf) +{ + ubcore_recv_req(&fastmsg_buf->udma_dev->ub_dev, &fastmsg_buf->req_host); + udma_info(fastmsg_buf->udma_dev->dev, UDMA_M_FMSG, + "[FMSG] drain handler callback\n"); + kfree(fastmsg_buf); + return; +} + +void udma_fast_msg_handler(struct hisdk5_fast_msg_buf *fast_msg, + void *priv_data) +{ + struct udma_device *udma_dev = priv_data; + struct ubcore_req_host *req_host = NULL; + struct ubcore_resp_host *resp_host = NULL; + struct ubcore_function_mig_req *req_msg = NULL; + struct ubcore_fastmsg_buf *fastmsg_buf = NULL; + u16 fastmsg_buf_len; + u16 req_host_len; + + if (udma_dev == NULL || fast_msg == NULL) { + udma_pr_err(UDMA_M_FMSG, + "fast msg handler input para is null.\n"); + return; + } + + req_host = (struct ubcore_req_host *)fast_msg->fast_msg_data; + req_msg = (struct ubcore_function_mig_req *)req_host->req.data; + if (fast_msg->fast_msg_header.ulp_format == + HISDK5_FAST_MSG_ULP_FROMAT_MIG) { + /* live migration message from pf0 send out, even data tpf receive, per live migration message format parse get vf, find corresponding odd/even tpf */ + udma_dev = udma_msg_find_tp_function( + udma_dev, adapted_type_ue_idx(req_msg, mig)); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_FMSG, + "udma tpf not found, src fe idx: %u.\n", + adapted_type_ue_idx(req_msg, mig)); + return; + } + } + + /* intercept VM sent live migration message */ + if ((req_host->req.opcode <= UBCORE_MSG_MIG_VM_START && + req_host->req.opcode >= UBCORE_MSG_STOP_PROC_VTP_MSG) || + req_host->req.opcode == UBCORE_MSG_NOTIFY_FASTMSG_DRAIN) { + if (fast_msg->fast_msg_header.src_func_id != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] err src_func_id(%u).\n", + fast_msg->fast_msg_header.src_func_id); + return; + } + } + + if (fast_msg->fast_msg_header.cmd == UB_MSG_WITH_MIG) { + udma_mig_msg_handler(udma_dev, fast_msg); + return; + } + + if (req_host->req.opcode == UBCORE_MSG_NOTIFY_FASTMSG_DRAIN) { + udma_info( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] func %u recv req msg, src func: %u, opcode: %u\n", + hinic5_global_func_id(udma_dev->hwdev), + adapted_type_ue_idx(req_host, src), + req_host->req.opcode); + req_host_len = sizeof(struct ubcore_req_host) + + sizeof(struct ubcore_function_mig_req); + fastmsg_buf_len = sizeof(struct ubcore_fastmsg_buf) + + sizeof(struct ubcore_function_mig_req); + fastmsg_buf = kzalloc(fastmsg_buf_len, GFP_KERNEL); + if (fastmsg_buf == NULL) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] fast msg mem alloc failed.\n"); + return; + } + + fastmsg_buf->udma_dev = udma_dev; + (void)memcpy_s(&fastmsg_buf->req_host, req_host_len, req_host, + req_host_len); + hinic5_fast_msg_forward(udma_dev->hwdev, + adapted_type_ue_idx(req_msg, mig), + fastmsg_buf, + (void *)udma_fast_msg_drain_handler); + return; + } + + if (fast_msg->fast_msg_header.send == 1) { + if (adapted_type_ue_idx(req_host, src) != + fast_msg->fast_msg_header.src_func_id) { + udma_err( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] src_fe_idx(%u) src_func_id(%u) is inconsistent.\n", + adapted_type_ue_idx(req_host, src), + fast_msg->fast_msg_header.src_func_id); + return; + } + + udma_info( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] func %u recv req msg, src func: %u, opcode: %u\n", + hinic5_global_func_id(udma_dev->hwdev), + adapted_type_ue_idx(req_host, src), + req_host->req.opcode); + ubcore_recv_req(&udma_dev->ub_dev, req_host); + return; + } + + resp_host = (struct ubcore_resp_host *)fast_msg->fast_msg_data; + udma_info( + udma_dev->dev, UDMA_M_FMSG, + "[FMSG] func %u recv resp msg, dst func: %u opcode:%u data:0x%llx len:%u\n", + hinic5_global_func_id(udma_dev->hwdev), + adapted_type_ue_idx(resp_host, dst), resp_host->resp.opcode, + *((u64 *)resp_host->resp.data), (u32)(resp_host->resp.len)); + ubcore_recv_resp(&udma_dev->ub_dev, &resp_host->resp); +} + +int udma_mbox_msg_vf_handler(void *pri_handle, u16 cmd, void *buf_in, + u16 in_size, void *buf_out, u16 *out_size) +{ + struct udma_device *udma_dev = pri_handle; + struct ubcore_resp_host *resp_host = (struct ubcore_resp_host *)buf_in; + + if (pri_handle == NULL || buf_in == NULL) { + udma_pr_err( + UDMA_M_FMSG, + "Invalid input parameter, pri_handle: %d, buf_in: %d.\n", + !!pri_handle, !!buf_in); + return -EINVAL; + } + + if (in_size < sizeof(struct ubcore_resp_host)) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[=MB=] Invalid in_size: %u, expect: %zu\n", in_size, + sizeof(struct ubcore_resp_host)); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_FMSG, + "[=MB=] [dst dev] ep: %u, opcode: %u\n", + adapted_type_ue_idx(resp_host, dst), resp_host->resp.opcode); + + return ubcore_recv_resp(&udma_dev->ub_dev, &resp_host->resp); +} + +static void udma_build_fast_msg_header(struct hisdk5_fast_msg_header *fmsg_head, + bool is_req, u16 dst_func_id, + u16 data_len) +{ + fmsg_head->status = 0; + fmsg_head->mod = HINIC5_MOD_UB; + fmsg_head->cmd = UB_MSG_WITH_UVS; + fmsg_head->dst_func_id = dst_func_id; /* microcode will for this field do check */ + fmsg_head->send = is_req; + fmsg_head->data_len = data_len; + fmsg_head->nack = 0; + fmsg_head->ulp_format = HISDK5_FAST_MSG_ULP_FROMAT_NONE; + fmsg_head->rsvd = 0; + fmsg_head->rsvd1 = 0; +} + +int udma_send_msg_to_tpf(struct ubcore_device *dev, struct ubcore_req *req) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + struct udma_device *peer_dev = NULL; + struct ubcore_req_host *req_host = NULL; + struct hinic5_cmd_buf *cmd_buf = NULL; + u16 req_len, reqh_len, fmsg_max_size; + u32 src_fe_idx = 0; + int ret; + + if (udma_dev == NULL || req == NULL) { + udma_pr_err( + UDMA_M_FMSG, + "send msg to tpf invalid parameter. udma_dev(%d), req(%d).\n", + !!udma_dev, !!req); + return -EINVAL; + } + + ret = udma_get_func_id(udma_dev, &src_fe_idx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] get func id failed\n"); + return -EINVAL; + } + + cmd_buf = hinic5_alloc_cmd_buf(udma_dev->hwdev); + if (cmd_buf == NULL) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] alloc cmd buf failed\n"); + return -ENOMEM; + } + + req_len = (u16)req->len + sizeof(struct ubcore_req); + reqh_len = (u16)req->len + sizeof(struct ubcore_req_host); + fmsg_max_size = cmd_buf->size - sizeof(struct hisdk5_fast_msg_header); + req_host = cmd_buf->buf + sizeof(struct hisdk5_fast_msg_header); + adapted_type_ue_idx(req_host, src) = src_fe_idx & 0xffff; + + ret = memcpy_s(&req_host->req, + fmsg_max_size - + sizeof(adapted_type_ue_idx(req_host, src)), + req, req_len); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] memcpy err, ret:%d, destMax:%lu, count:%u\n", + ret, + fmsg_max_size - + sizeof(adapted_type_ue_idx(req_host, src)), + req_len); + goto free_cmd_buf; + } + + peer_dev = udma_msg_find_tp_function(udma_dev, src_fe_idx); + if (adapted_type_ue_idx(req_host, src) < FUNC_ID_BOUND_FOR_PF && + peer_dev != NULL) { + ret = ubcore_recv_req(&peer_dev->ub_dev, req_host); + } else { + /* new VM driver compatible with old firmware and SPU driver */ + if (hinic5_func_type(udma_dev->hwdev) == TYPE_VF && + hinic5_support_fast_msg(udma_dev->hwdev) == false) { + ret = hinic5_mbox_to_pf(udma_dev->hwdev, + HINIC5_MOD_FAKE_FMSG, 0, + (void *)req_host, reqh_len, + NULL, NULL, 0, + HINIC5_CHANNEL_UB); + } else { + /* primary dynamic sent message, dst_func_id by microcode to cover fill in */ + udma_build_fast_msg_header(cmd_buf->buf, true, 0xFFFF, + reqh_len); + ret = udma_send_fast_msg(udma_dev, cmd_buf); + } + } + + udma_info(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] send msg to tpf from func_%u, ret: %d\n", src_fe_idx, + ret); + +free_cmd_buf: + hinic5_free_cmd_buf(udma_dev->hwdev, cmd_buf); + return ret; +} + +int udma_send_msg_from_tpf(struct ubcore_device *dev, + struct ubcore_resp_host *resp_host) +{ + struct udma_device *udma_dev = NULL; + struct udma_device *peer_dev = NULL; + struct hinic5_cmd_buf *cmd_buf = NULL; + u16 resp_len, fmsg_max_size; + int ret; + + if (dev == NULL || resp_host == NULL) { + udma_pr_err(UDMA_M_FMSG, + "Invalid parameter. dev(%d), resp_host(%d).\n", + !!dev, !!resp_host); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + + peer_dev = udma_find_dev_by_fe_idx(udma_dev, + adapted_type_ue_idx(resp_host, dst)); + if (adapted_type_ue_idx(resp_host, dst) < FUNC_ID_BOUND_FOR_PF && + peer_dev != NULL) { + ret = ubcore_recv_resp(&peer_dev->ub_dev, &resp_host->resp); + } else { + cmd_buf = hinic5_alloc_cmd_buf(udma_dev->hwdev); + if (cmd_buf == NULL) { + udma_err(udma_dev->hwdev, UDMA_M_FMSG, + "[FMSG] alloc cmd buf failed\n"); + return -ENOMEM; + } + + resp_len = + resp_host->resp.len + sizeof(struct ubcore_resp_host); + fmsg_max_size = + cmd_buf->size - sizeof(struct hisdk5_fast_msg_header); + ret = memcpy_s(cmd_buf->buf + + sizeof(struct hisdk5_fast_msg_header), + fmsg_max_size, resp_host, resp_len); + if (ret != 0) { + udma_err( + udma_dev->hwdev, UDMA_M_FMSG, + "[FMSG] memcpy err, ret:%d, destMax:%u, count:%u\n", + ret, fmsg_max_size, resp_len); + } else { + udma_build_fast_msg_header( + cmd_buf->buf, false, + adapted_type_ue_idx(resp_host, dst), resp_len); + ret = udma_send_fast_msg(udma_dev, cmd_buf); + } + + hinic5_free_cmd_buf(udma_dev->hwdev, cmd_buf); + } + + udma_info(udma_dev->dev, UDMA_M_FMSG, + "[FMSG] send msg from tpf to func_%u, ret: %u\n", + adapted_type_ue_idx(resp_host, dst), ret); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h new file mode 100644 index 000000000..652a2d289 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_fast_msg.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_FAST_MSG_H +#define UDMA_FAST_MSG_H + +#include "udma_common.h" + +#define UB_MSG_WITH_UVS 0 +#define UB_MSG_WITH_MIG 1 +#define MIG_UB_CMD 1 + +enum { + UB_MIG_VTP_RESTORE_START = 0x20, // notify UDMA perform VTP full flush down + UB_MIG_VTP_RESTORE_END, // notify UDMA remaining of incremental request flush down + UB_MIG_VTP_RESTORE_CANCEL, // notify UDMA rollback + UB_MIG_VTP_RESTORE_FLS_MODIFY, // complete FLUSH DMA, notify UDMA remaining of modify request flush down + UB_MIG_UPDATE_UPI_NORM, // live migration process in, activate and cancel handle in, if notify UVS failure, primary dynamic update UPI is NORM + UB_MIG_UPDATE_UPI_MIG, // live migration process in, deactivate stage segment primary dynamic update UPI is MIG(later adaptation) +}; + +struct ubcore_fastmsg_buf { + struct udma_device *udma_dev; + struct ubcore_req_host req_host; +}; + +int udma_msg_dev_register(struct udma_device *udma_dev); +int udma_msg_dev_unregister(struct udma_device *udma_dev); +int udma_send_msg_to_tpf(struct ubcore_device *dev, struct ubcore_req *req); +int udma_send_msg_from_tpf(struct ubcore_device *dev, + struct ubcore_resp_host *resp_host); +struct udma_device *udma_find_dev_by_fe_idx(struct udma_device *udma_dev, + u32 func_id); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c new file mode 100644 index 000000000..1b6704146 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_main.c @@ -0,0 +1,1044 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/netdevice.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include "linux/nodemask.h" + +#include "securec.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "ubcore_api.h" +#include "udma_bond.h" +#include "udma_common.h" +#include "udma_device.h" +#include "udma_comp.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_query.h" +#include "udma_jfc.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_key.h" +#include "udma_rc_table.h" +#include "udma_mtt.h" +#include "udma_net.h" +#include "udma_dfx.h" +#include "udma_misc.h" +#include "udma_rc_table.h" +#include "udma_resource.h" +#include "udma_sysfs_api.h" +#include "udma_vtp.h" +#include "udma_utp.h" +#include "udma_fast_msg.h" +#include "udma_ucontext.h" +#include "udma_device.h" +#include "udma_cmd_event.h" +#include "udma_cmd_eid.h" +#include "udma_dfx_capture.h" +#include "udma_dfx_drop.h" +#include "udma_dfx_modify.h" +#include "udma_vram.h" +#include "udma_vram_mem.h" +#include "udma_shm_manage.h" + +#define UDMA_IS_EVEN_NUM(num) (((num)&0x1) == 0) +#define PARITY_DIVISOR 2 + +u32 g_mtt_page_size = 0U; +u32 g_double_tpf_en = 1U; +u32 g_env_type = UB_ENV_FPGA; +u32 g_udma_fast_path_timer_disable = 0U; +static char g_ub_uld_dev_name[IFNAMSIZ] = { 0 }; +module_param(g_mtt_page_size, uint, S_IRUGO); +module_param(g_double_tpf_en, uint, S_IRUGO); +module_param(g_env_type, uint, S_IRUGO); +module_param(g_udma_fast_path_timer_disable, uint, 0664); +MODULE_PARM_DESC(g_mtt_page_size, "0:4K,1:64K,2:2M,default:4K"); +MODULE_PARM_DESC(g_double_tpf_en, "use for enable double TPF"); +MODULE_PARM_DESC(g_env_type, "0:asic, 1:esl, 2:fpga, 3: emu, default:fpga"); +MODULE_PARM_DESC(g_udma_fast_path_timer_disable, "0: enable, 1: disable"); + +static struct ubcore_ops g_dev_ops = { + .owner = THIS_MODULE, + .driver_name = "ub_hal", + .abi_version = 0, + .add_ueid = udma_cmd_ueid_add, + .delete_ueid = udma_cmd_ueid_delete, + .query_device_attr = udma_query_device_attr, + .query_device_status = udma_query_device_status, + .alloc_ucontext = udma_alloc_ucontext, + .free_ucontext = udma_free_ucontext, + .mmap = udma_mmap, + .config_device = udma_config_device, + .add_net_addr = udma_add_netaddr, + .delete_net_addr = udma_del_netaddr, + .create_jetty = udma_create_jetty, + .destroy_jetty = udma_destroy_jetty, + .modify_jetty = udma_modify_jetty, + .flush_jetty = udma_flush_jetty, + .query_jetty = udma_query_jetty, + .create_jfc = udma_create_jfc, + .modify_jfc = udma_modify_jfc, + .destroy_jfc = udma_destroy_jfc, + .rearm_jfc = udma_rearm_jfc, + .create_jfs = udma_create_jfs, + .query_jfs = udma_query_jfs, + .destroy_jfs = udma_destroy_jfs, + .modify_jfs = udma_modify_jfs, + .flush_jfs = udma_flush_jfs, + .create_jfr = udma_create_jfr, + .query_jfr = udma_query_jfr, + .destroy_jfr = udma_destroy_jfr, + .import_jfr = udma_import_jfr, + .unimport_jfr = udma_unimport_jfr, + .modify_jfr = udma_modify_jfr, + .import_jetty = udma_import_jetty, + .unimport_jetty = udma_unimport_jetty, + .bind_jetty = udma_bind_jetty, + .unbind_jetty = udma_unbind_jetty, + .create_jetty_grp = udma_create_jetty_grp, + .delete_jetty_grp = udma_destroy_jetty_grp, + .alloc_token_id = udma_alloc_key_id, + .free_token_id = udma_free_key_id, + .register_seg = udma_register_seg, + .unregister_seg = udma_unregister_seg, + .import_seg = udma_import_seg, + .unimport_seg = udma_unimport_seg, + .create_tp = udma_create_tp, + .modify_tp = udma_modify_tp, + .modify_user_tp = NULL, + .create_vtp = udma_create_vtp, + .alloc_vtpn = udma_alloc_vtpn, + .destroy_vtp = udma_destroy_vtp, + .free_vtpn = udma_free_vtpn, + .modify_vtp = udma_modify_vtp, + .create_utp = udma_create_utp, + .destroy_utp = udma_destroy_utp, + .query_res = udma_query_res, + .query_ue_idx = udma_query_fe_idx, + .destroy_tp = udma_destroy_tp, + .create_multi_tp = udma_create_multi_tp, + .destroy_multi_tp = udma_destroy_multi_tp, + .modify_multi_tp = udma_modify_multi_tp, + .create_tpg = udma_create_tpg, + .destroy_tpg = udma_destroy_tpg, + .post_jetty_recv_wr = udma_post_jetty_recv_wr, + .post_jetty_send_wr = udma_post_jetty_send_wr, + .post_jfs_wr = udma_post_jfs_wr, + .post_jfr_wr = udma_post_jfr_wr, + .poll_jfc = udma_poll_jfc, + .user_ctl = udma_user_ctl, + .send_req = udma_send_msg_to_tpf, + .send_resp = udma_send_msg_from_tpf, + .query_cc = udma_query_cc, + .config_dscp_vl = udma_config_dscp_vl, + .query_dscp_vl = udma_query_dscp_vl, + .config_function_migrate_state = udma_config_function_migrate_state, + .query_ue_stats = udma_query_fe_stats, + .disassociate_ucontext = NULL, +}; + +static int udma_register_netdev_event(struct udma_device *udma_dev) +{ + int ret; + + udma_dev->netdev_nb.notifier_call = udma_dev_event; + ret = register_netdevice_notifier(&udma_dev->netdev_nb); + if (ret != 0) { + udma_err(udma_dev->hwdev, UDMA_M_MAIN, + "register netdev notifier failed\n"); + udma_dev->netdev_nb.notifier_call = NULL; + } + + return ret; +} + +static void udma_unregister_netdev_event(struct udma_device *udma_dev) +{ + unregister_netdevice_notifier(&udma_dev->netdev_nb); + if (udma_dev->netdev_nb.notifier_call != NULL) { + udma_dev->netdev_nb.notifier_call = NULL; + } +} + +void udma_get_dev_name(struct udma_device *udma_dev) +{ + struct udma_bus_addr bus_addr = { 0 }; + + if (udma_dev->dev_bus_type == HINIC5_DEVICE_T_PCI) { + udma_get_bus_info(udma_dev->dev, &bus_addr); + } else if (udma_dev->dev_bus_type == + HINIC5_DEVICE_T_UB) { /* todo ubc scenario need adaptation */ + bus_addr.function = hinic5_global_func_id(udma_dev->hwdev) & + 0xff; + } + + scnprintf(udma_dev->dev_name, UDMA_MAX_DEV_NAME, "hiudma5_p%us%uf%u", + bus_addr.bus, bus_addr.devid, bus_addr.function); + + /* uld_dev_name is for internal use */ + scnprintf(g_ub_uld_dev_name, sizeof(g_ub_uld_dev_name) - 1, + "ub%us%uf%u", bus_addr.bus, bus_addr.devid, + bus_addr.function); +} + +static void udma_init_ub_dev(struct ubcore_device *ub_dev, + struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev) +{ + ub_dev->transport_type = UBCORE_TRANSPORT_UB; + ub_dev->ops = &g_dev_ops; + ub_dev->dev.parent = udma_dev->dev; + ub_dev->netdev = udma_dev->ndev; + ub_dev->dma_dev = ub_dev->dev.parent; + + udma_get_dev_name(udma_dev); + + strlcpy(ub_dev->dev_name, udma_dev->dev_name, UDMA_MAX_DEV_NAME); + + udma_init_ub_dev_attr(udma_dev, &ub_dev->attr); + + ub_dev->cfg.pattern = + (udma_dev->ub_cap.dev_cap.pattern_type == PATTERN_TYPE_3) ? + UBCORE_PATTERN_3 : + UBCORE_PATTERN_1; + ub_dev->cfg.virtualization = + (udma_dev->ub_cap.dev_cap.virtualization == 1) ? true : false; +} + +static int udma_register_ub_device_base(struct udma_device *udma_dev) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + int ret; + + if (udma_vram5_get_kexec_flag() == 0) { + udma_pr_info(UDMA_M_MAIN, + "first OS: about to register dev: %s\n", + udma_dev->dev_name); + ret = ubcore_register_device(ub_dev); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "register udma dev to ubcore failed, ret(%d)\n", + ret); + } + return ret; + } + +#ifdef UBCORE_EN + udma_pr_err(UDMA_M_VRAM, "second OS, about to register dev: %s\n", + udma_dev->dev_name); + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_DEV, udma_dev); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "ubcore recover failed failed. type:%d\n", + UDMA_VRAM_TYPE_DEV); + return -EINVAL; + } +#endif + + return 0; +} + +static void ubcore_unregister_device_base(struct ubcore_device *ub_dev) +{ + if (udma_vram5_get_kexec_flag() == 0) { + ubcore_unregister_device(ub_dev); + } +} + +static int udma_register_ub_device(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev) +{ + int ret; + u8 port_id; + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + + udma_init_ub_dev(ub_dev, lld_dev, udma_dev); + + ret = udma_msg_dev_register(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "register udma dev failed, ret(%d)\n", ret); + return ret; + } + + ret = udma_register_ub_device_base(udma_dev); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "register udma dev to ub device base failed, ret(%d)\n", + ret); + goto err_ubcore_register; + } + + ret = udma_register_netdev_event(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "register netdev event failed, ret(%d)\n", ret); + goto err_register_netdev_event; + } + + port_id = hinic5_physical_port_id(udma_dev->hwdev); + udma_info(udma_dev->hwdev, UDMA_M_MAIN, "find port id:%u\n", port_id); + + ret = ubcore_set_port_netdev(ub_dev, ub_dev->netdev, port_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "set port netdev failed, ret(%d)\n", ret); + goto err_set_port; + } + + if (udma_dev->ub_dev.attr.tp_maintainer && + udma_vram5_get_kexec_flag() == 0) { + ret = udma_cmd_async_event_credit_init(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "init asyn event credit failed, ret(%d)\n", + ret); + goto err_init_credit; + } + } + + return ret; + +err_init_credit: + ubcore_put_port_netdev(&udma_dev->ub_dev); +err_set_port: + udma_unregister_netdev_event(udma_dev); +err_register_netdev_event: + udma_delete_rc_table( + udma_dev); //ubcore_register_device of config_device called in, in when destroying device destroy RC table + udma_destroy_tp_latch_tpg(udma_dev); /* config_device created in */ + ubcore_unregister_device_base(&udma_dev->ub_dev); +err_ubcore_register: + udma_msg_dev_unregister(udma_dev); + + return ret; +} + +static void udma_unregister_ub_device(struct udma_device *udma_dev) +{ + ubcore_put_port_netdev(&udma_dev->ub_dev); + udma_unregister_netdev_event(udma_dev); + udma_delete_rc_table( + udma_dev); //ubcore_register_device of config_device called in, in when destroying device destroy RC table + udma_destroy_tp_latch_tpg(udma_dev); /* config_device created in */ + ubcore_unregister_device(&udma_dev->ub_dev); + udma_msg_dev_unregister(udma_dev); +} + +static u64 udma_get_card_id(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_device_info info = { 0 }; + + hinic5_get_device_info(lld_dev, &info); + return info.id; +} + +static int udma_get_dev(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev) +{ + udma_dev->dev = lld_dev->dev; + udma_dev->hwdev = lld_dev->hwdev; + udma_dev->ndev = hinic5_get_netdev_by_lld(lld_dev); + if (udma_dev->ndev == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, "failed to get net dev\n"); + return -EINVAL; + } + udma_dev->device_type = UDMA_DEV_TYPE_HI1825_V100; + udma_dev->env_type = g_env_type; + udma_dev->dev_bus_type = lld_dev->dev_type; + udma_dev->card_id = udma_get_card_id(lld_dev); + udma_dev->host_id = hinic5_host_id(lld_dev->hwdev); + + udma_info(udma_dev->dev, UDMA_M_MAIN, + "card_id:%llu, host_id:%u, function_id:%u\n", + udma_dev->card_id, udma_dev->host_id, + hinic5_global_func_id(udma_dev->hwdev)); + return 0; +} + +static int udma_tpg_bitmap_alloc(struct udma_device *udma_dev) +{ + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + udma_dev->bitmap = + udma_bitmap_alloc(udma_dev->ub_cap.dev_cap.max_tpg); + if (udma_dev->bitmap == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to allocate bitmap.\n"); + return -ENOMEM; + } + } + + return 0; +} + +static void udma_tpg_bitmap_free(struct udma_device *udma_dev) +{ + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + udma_bitmap_free(udma_dev->bitmap); + udma_dev->bitmap = NULL; + } +} + +static int udma_create_tp_res(struct udma_device *udma_dev) +{ + int ret; + + ret = udma_tpg_bitmap_alloc(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to allocate tpg bitmap.\n"); + ret = -ENOMEM; + return ret; + } + + return 0; +} + +static void udma_free_tp_res(struct udma_device *udma_dev) +{ + udma_tpg_bitmap_free(udma_dev); +} + +void udma_init_mig_vf_mgt(struct udma_device *udma_dev) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = NULL; + int i; + + mutex_init(&mig_mgt->mig_vf_mgt_mutex); + mig_mgt->migrating_num = 0; + + for (i = 0; i < ARRAY_SIZE(mig_mgt->buf_array); i++) { + buf = &mig_mgt->buf_array[i]; + mutex_init(&buf->vf_mutex); + buf->addr = NULL; + buf->cqm_cmd_buf = NULL; + buf->total = 0; + buf->cur_idx = 0; + buf->mig_sts = UDMA_MIG_IDLE; + buf->func_id = UDMA_MIG_VFID_MIN + i; + } +} + +void udma_deinit_mig_vf_mgt(struct udma_device *udma_dev) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = NULL; + int i; + + for (i = 0; i < ARRAY_SIZE(mig_mgt->buf_array); i++) { + buf = &mig_mgt->buf_array[i]; + mutex_destroy(&buf->vf_mutex); + } + mutex_destroy(&mig_mgt->mig_vf_mgt_mutex); +} + +static void udma_misc_init(struct udma_device *udma_dev) +{ + if (udma_vram5_get_kexec_flag() == 1) { + return; + } + + udma_dev->jfs_current_cnt = 0; + atomic_set(&udma_dev->jfs_hash, 0); + atomic_set(&udma_dev->jfr_hash, 0); + atomic_set(&udma_dev->rs_hash, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.post_send_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.post_recv_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_total_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_sq_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_rq_num, 0); + + atomic_set(&udma_dev->task_jfs_num, 0); + + mutex_init(&udma_dev->jettyn_record.mutex); + mutex_init(&udma_dev->tpn_record.mutex); + mutex_init(&udma_dev->jfcn_record.mutex); + mutex_init(&udma_dev->vtpn_record.mutex); + mutex_init(&udma_dev->utpn_record.mutex); + mutex_init(&udma_dev->tpgn_record.mutex); + mutex_init(&udma_dev->jetty_grpn_record.mutex); + mutex_init(&udma_dev->srqn_record.mutex); + mutex_init(&udma_dev->dev_mutex); + mutex_init(&udma_dev->tp_mutex); + spin_lock_init(&udma_dev->dwqe_lock); +} + +static void udma_misc_deinit(struct udma_device *udma_dev) +{ + if (udma_vram5_get_kexec_flag() == 1) { + return; + } + + mutex_destroy(&udma_dev->jettyn_record.mutex); + mutex_destroy(&udma_dev->tpn_record.mutex); + mutex_destroy(&udma_dev->jfcn_record.mutex); + mutex_destroy(&udma_dev->vtpn_record.mutex); + mutex_destroy(&udma_dev->utpn_record.mutex); + mutex_destroy(&udma_dev->tpgn_record.mutex); + mutex_destroy(&udma_dev->jetty_grpn_record.mutex); + mutex_destroy(&udma_dev->srqn_record.mutex); + mutex_destroy(&udma_dev->dev_mutex); + mutex_destroy(&udma_dev->tp_mutex); +} + +static int udma_reload_device_cap(struct hinic5_lld_dev *lld_dev, + struct ub_service_cap *ub_cap) +{ + int ret; + + if (udma_vram5_get_kexec_flag() == 1) { + return 0; + } + + /* VF scenario from MPU get ub resource information and update to SDK, avoid resource dynamic allocate not generatevalid */ + if (hinic5_func_type(lld_dev->hwdev) == TYPE_VF) { + ret = hinic5_get_dev_cap(lld_dev->hwdev); + if (ret != 0) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to get dev cap, ret:%d\n", ret); + return ret; + } + /* update localpartchange amount ub_cap, this when hwdev one certain valid and supports UB */ + (void)hinic5_support_ub(lld_dev->hwdev, ub_cap); + } + + return 0; +} + +static int udma_device_cap_init(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev, + struct ub_service_cap *ub_cap) +{ + int ret; + + ret = udma_reload_device_cap(lld_dev, ub_cap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "Failed to reload device cap, ret:%d\n", ret); + return ret; + } + + ret = udma_get_device_cap(udma_dev, ub_cap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "Failed to get device cap\n"); + return ret; + } + + ret = udma_get_feat_bitmap(udma_dev, udma_dev->feat_bitmap, + UB_FEAT_BITMAP_U64_MAX); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "Failed to get feature bitmap.\n"); + return ret; + } + + return 0; +} + +static int udma_dev_init(struct hinic5_lld_dev *lld_dev, + struct udma_device *udma_dev, + struct ub_service_cap *ub_cap) +{ + int ret; + + udma_misc_init(udma_dev); + + ret = udma_get_dev(lld_dev, udma_dev); + if (ret != 0) { + goto error_init_res; + } + + ret = udma_device_cap_init(lld_dev, udma_dev, ub_cap); + if (ret != 0) { + goto error_init_res; + } + + udma_init_mig_vf_mgt(udma_dev); + ret = udma_init_sysfs(&udma_dev->sysfs_res); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to init sysfs, ret:%d\n", ret); + goto error_init_mig_vf_mgt; + } + + ret = udma_init_res(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "udma init resource failed, ret:%d", ret); + goto error_init_sysfs; + } + + ret = udma_dfx_device_init(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "udma init dfx resource failed, ret:%d", ret); + goto error_init_dfx; + } + + ret = udma_create_tp_res(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to create tp res, ret:%d\n", ret); + goto error_tp_res; + } + + ret = udma_register_ub_device(lld_dev, udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to register udma dev, ret:%d\n", ret); + goto error_register; + } + + return 0; + +error_register: + udma_free_tp_res(udma_dev); +error_tp_res: + udma_dfx_device_deinit(udma_dev); +error_init_dfx: + udma_free_res(udma_dev); +error_init_sysfs: + udma_remove_sysfs(&udma_dev->sysfs_res); +error_init_mig_vf_mgt: + udma_deinit_mig_vf_mgt(udma_dev); +error_init_res: + udma_misc_deinit(udma_dev); + return ret; +} + +/* TODO this interface only use in vram scenario, not in c00 interact付 range, current only supports pcie scenario */ +static void udma_get_dev_name_by_dev(char device_name[], u32 dev_name_len, + struct device *dev) +{ + struct udma_bus_addr bus_addr = { 0 }; + + udma_get_bus_info(dev, &bus_addr); + + scnprintf((char *)device_name, dev_name_len, "hiudma5_p%us%uf%u", + bus_addr.bus, bus_addr.devid, bus_addr.function); +} + +static struct udma_device *udma_dev_vram_find(struct device *dev) +{ + struct udma_vram_header *vram_header = NULL; + struct udma_vram_block *block = NULL; + struct udma_device *node = NULL; + char device_name[UDMA_MAX_DEV_NAME]; + u32 max_num; + u32 index; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_err(dev, UDMA_M_MAIN, + "vram_header get is NULL in udma_dev_vram_find\n"); + return NULL; + } + block = vram_header->header[UDMA_VRAM_TYPE_DEV].block; + max_num = block->node_num; + + udma_err(dev, UDMA_M_MAIN, "max_num is %d\n", max_num); + + udma_get_dev_name_by_dev(device_name, UDMA_MAX_DEV_NAME, dev); + + for (index = 0; index < max_num; index++) { + if (test_bit(index, (const unsigned long *)block->bitmap) == + 0) { + continue; + } + node = (void *)((u8 *)block->buf + index * block->node_size); + if (strncmp((const char *)node->dev_name, + (const char *)device_name, + sizeof(node->dev_name)) == 0) { + return node; + } + } + + return NULL; +} + +static struct udma_device *udma_dev_create(struct hinic5_lld_dev *lld_dev, + struct ub_service_cap *ub_cap) +{ + struct device *dev = lld_dev->dev; + struct udma_device *udma_dev = NULL; + int numa_original_node_id = 0; + bool numa_original_node_id_valid = false; + + if ((hinic5_host_id(lld_dev->hwdev) == UB_SPU_HOST_ID) && + (num_online_nodes() > 1)) { + numa_original_node_id = dev_to_node(dev); + numa_original_node_id_valid = true; + set_dev_node( + dev, + hinic5_global_func_id(lld_dev->hwdev) % + PARITY_DIVISOR); //set numa node by the parity of function id. + } + + if (udma_vram5_get_kexec_flag() == 0) { + udma_dev = (struct udma_device *)udma_vram_kzalloc( + dev, UDMA_VRAM_TYPE_DEV, sizeof(struct udma_device), + GFP_KERNEL); + } else { + udma_dev = udma_dev_vram_find(dev); + } + if (udma_dev == NULL) { + udma_err(dev, UDMA_M_MAIN, + "failed to allocate udma dev, kexec flag: %u\n", + udma_vram5_get_kexec_flag()); + return NULL; + } + + if (udma_dev_init(lld_dev, udma_dev, ub_cap) != 0) { + goto error_create_dev; + } + + udma_dev->numa_original_node_id_valid = numa_original_node_id_valid; + if (numa_original_node_id_valid == true) { + udma_dev->numa_original_node_id = numa_original_node_id; + } + + return udma_dev; + +error_create_dev: + udma_vram_kfree(dev, udma_dev, UDMA_VRAM_TYPE_DEV); + return NULL; +} + +static void udma_dev_deinit(struct udma_device *udma_dev) +{ + udma_unregister_ub_device(udma_dev); + udma_free_tp_res(udma_dev); + udma_dfx_device_deinit(udma_dev); + udma_free_res(udma_dev); + udma_remove_sysfs(&udma_dev->sysfs_res); + udma_misc_deinit(udma_dev); +} + +static void udma_dev_destroy(struct device *dev, struct udma_device *udma_dev) +{ + int numa_original_node_id = udma_dev->numa_original_node_id; + bool numa_original_node_id_valid = + udma_dev->numa_original_node_id_valid; + + /* capture packet still in perform, first disable capture packet */ + if (udma_dev->capture_en == true) { + udma_warn(udma_dev->dev, UDMA_M_MAIN, + "%s: capture is still running, now stop\n", __func__); + udma_dfx_stop_cap_pkt(udma_dev, NULL, NULL); + } + + /* disable packet drop */ + udma_stop_all_drop_pkt(udma_dev); + udma_dev_deinit(udma_dev); + + udma_info(udma_dev->dev, UDMA_M_MAIN, "udma free resource success\n"); + + /* offload driver when none (classifier) file free memory resource */ + udma_vram_kfree(dev, udma_dev, UDMA_VRAM_TYPE_DEV); + + if (numa_original_node_id_valid == true) { + set_dev_node(dev, numa_original_node_id); + } +} + +static int udma_get_plug_attach_en(struct hinic5_lld_dev *lld_dev) +{ + struct device *dev = lld_dev->dev; + void *hwdev = lld_dev->hwdev; + u8 attach_en = 0; + int ret; + + if (!hinic5_is_slave_host(hwdev)) { + udma_info(dev, UDMA_M_MAIN, + "no need to check attach_en flag.\n"); + return 0; + } + + ret = hisdk5_get_plug_srv_bitmap(hwdev, COMM_PLUG_SRV_UB, + hinic5_global_func_id(hwdev), + &attach_en); + if (ret != 0) { + udma_err(dev, UDMA_M_MAIN, + "get plug service bitmap failed, ret:%d\n", ret); + return ret; + } + + if (attach_en == 0) { + udma_err(dev, UDMA_M_MAIN, "attach_en flag is invalid.\n"); + return -EFAULT; + } + + udma_info(dev, UDMA_M_MAIN, + "attach_en flag is valid, udma probe begin.\n"); + return 0; +} + +static void cleanup_udma_device(struct hinic5_lld_dev *lld_dev) +{ + if (hinic5_func_type(lld_dev->hwdev) == TYPE_PF) { + if (!hinic5_is_slave_host(lld_dev->hwdev) && + !hinic5_is_master_host(lld_dev->hwdev)) { + struct hinic5_lld_dev *ppf_lld_dev = + hinic5_get_ppf_lld_dev_unsafe(lld_dev); + hinic5_stateful_deinit(ppf_lld_dev->hwdev); + } + } +} + +static bool udma_double_tpf_invalid(struct hinic5_lld_dev *lld_dev, + struct ub_service_cap *ub_cap) +{ + return g_double_tpf_en == 0 && + UDMA_IS_EVEN_NUM(hinic5_global_func_id(lld_dev->hwdev)) && + !!ub_cap->net_dev_cap.is_tpf; +} + +static int udma_ppf_init(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_lld_dev *ppf_lld_dev = NULL; + int ret; + + if (hinic5_func_type(lld_dev->hwdev) != TYPE_PF) { + return 0; + } + + if (hinic5_is_slave_host(lld_dev->hwdev) || + hinic5_is_master_host(lld_dev->hwdev)) { + return 0; + } + + /* hwdev device type is TPYE_PF, hwdev device not is primary or prepare type, need get ppf, for ppf device perform init stateful */ + ppf_lld_dev = hinic5_get_ppf_lld_dev_unsafe(lld_dev); + /* get ppf when exist return NULL of exception scenario, need check non-null avoid oops */ + if (ppf_lld_dev == NULL) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to find ppf resource\n"); + return -ENXIO; + } + ret = hinic5_stateful_init(ppf_lld_dev->hwdev); + if (ret != 0) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to init stateful ppf resource ret:%d\n", ret); + return ret; + } + return 0; +} + +static void udma_ppf_deinit(struct hinic5_lld_dev *lld_dev) +{ + struct hinic5_lld_dev *ppf_lld_dev = NULL; + + if (hinic5_func_type(lld_dev->hwdev) != TYPE_PF) { + return; + } + + if (hinic5_is_slave_host(lld_dev->hwdev) || + hinic5_is_master_host(lld_dev->hwdev)) { + return; + } + + /* hwdev device type is TPYE_PF, hwdev device not is primary or prepare type, need get ppf, for ppf device perform deinit stateful */ + ppf_lld_dev = hinic5_get_ppf_lld_dev_unsafe(lld_dev); + /* get ppf when exist return NULL of exception scenario, need check non-null avoid oops */ + if (ppf_lld_dev == NULL) { + udma_err(lld_dev->dev, UDMA_M_MAIN, + "Failed to find ppf resource\n"); + return; + } + hinic5_stateful_deinit(ppf_lld_dev->hwdev); +} + +static int udma_probe(struct hinic5_lld_dev *lld_dev, void **uld_dev, + char *uld_dev_name) +{ + struct device *dev = lld_dev->dev; + struct ub_service_cap ub_cap = { 0 }; + struct udma_device *udma_dev = NULL; + int ret = 0; + + udma_info(dev, UDMA_M_MAIN, + "udma(function id:%u, dev type:%d) probe begin\n", + hinic5_global_func_id(lld_dev->hwdev), lld_dev->dev_type); + + if (!hinic5_support_ub(lld_dev->hwdev, &ub_cap)) { + if (hinic5_func_type(lld_dev->hwdev) == + TYPE_PPF) { //version number from ppf on read + udma_info(dev, UDMA_M_MAIN, + "register ppf for udma version\n"); + *uld_dev = lld_dev; + return 0; + } + udma_info(dev, UDMA_M_MAIN, "hardware NOT support udma\n"); + *uld_dev = NULL; + return 0; + } + + if (udma_double_tpf_invalid(lld_dev, &ub_cap)) { + udma_info(dev, UDMA_M_MAIN, + "tpf with even function id not supported\n"); + *uld_dev = NULL; + return 0; + } + + if (udma_get_plug_attach_en(lld_dev) != 0) { + return -EINVAL; + } + + ret = udma_ppf_init(lld_dev); + if (ret != 0) { + udma_err(dev, UDMA_M_MAIN, "Failed to init ppf, ret = %d\n", + ret); + return ret; + } + + udma_dev = udma_dev_create(lld_dev, &ub_cap); + if (udma_dev == NULL) { + cleanup_udma_device(lld_dev); + udma_err(dev, UDMA_M_MAIN, "failed to create udma dev\n"); + return -ENOMEM; + } + + *uld_dev = udma_dev; + ret = memcpy_s(uld_dev_name, IFNAMSIZ, g_ub_uld_dev_name, + strlen(g_ub_uld_dev_name) + 1); + if (ret != 0) { + udma_err(dev, UDMA_M_MAIN, "memcpy_s failed, ret = %d\n", ret); + return ret; + } + + udma_info(dev, UDMA_M_MAIN, "udma probe ret = %d\n", ret); + + return 0; +} + +static void udma_remove(struct hinic5_lld_dev *lld_dev, void *adapter) +{ + struct device *dev = lld_dev->dev; + + udma_info(dev, UDMA_M_MAIN, "udma remove, func_id:%u\n", + hinic5_global_func_id(lld_dev->hwdev)); + + if (!hinic5_support_ub(lld_dev->hwdev, NULL)) { + if (hinic5_func_type(lld_dev->hwdev) == TYPE_PPF) { + udma_info(dev, UDMA_M_MAIN, + "udma remove, func_id:%u is ppf\n", + hinic5_global_func_id(lld_dev->hwdev)); + } + return; + } + + if (adapter == NULL) { + udma_pr_err(UDMA_M_MAIN, "udma remove failed\n"); + return; + } + + udma_dev_destroy(dev, adapter); + + udma_ppf_deinit(lld_dev); + + udma_pr_info(UDMA_M_MAIN, "udma remove success\n"); + + return; +} + +static struct hinic5_uld_info g_udma_uld_info = { + .probe = udma_probe, + .remove = udma_remove, + .suspend = NULL, + .resume = NULL, + .event = udma_event, + .ioctl = udma_ioctl, +}; + +static int __init udma_lld_init(void) +{ + int ret; + + udma_pr_info(UDMA_M_MAIN, "%s - version %s\n", UDMA_DRV_DESC, + UDMA_DRV_VERSION); + + ret = udma_init_shm_related_symbols(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, + "shm related symbols init failed, ret:%d\n", ret); + return ret; + } + + ret = udma_vram_mem_manager_init(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, + "vram mem manager init failed, ret:%d\n", ret); + return ret; + } + + ret = hinic5_register_uld(SERVICE_T_UB, &g_udma_uld_info); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, "register hinic5 uld failed, ret:%d\n", + ret); + goto error_register_uld; + } + + ret = udma_vram_mem_obj_recover(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, + "vram mem obj recover failed, ret:%d\n", ret); + goto error_mem_obj_recover; + } + + ret = hinic5_module_pre_init(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, "hinic5 pre-init failed, ret:%d\n", + ret); + goto error_mem_obj_recover; + } + + ret = udma_bond_init(); + if (ret != 0) { + udma_pr_err(UDMA_M_MAIN, "udma_bond_init failed, ret:%d\n", + ret); + goto error_bond_init; + } + + return 0; + +error_bond_init: + hinic5_module_post_exit(); +error_mem_obj_recover: + hinic5_unregister_uld(SERVICE_T_UB); +error_register_uld: + udma_vram_mem_manager_uninit(); + + return ret; +} + +static void __exit udma_lld_exit(void) +{ + udma_bond_uninit(); + hinic5_unregister_uld(SERVICE_T_UB); + udma_vram_mem_manager_uninit(); + hinic5_module_post_exit(); + udma_uninit_shm_related_symbols(); +} + +module_init(udma_lld_init); +module_exit(udma_lld_exit); + +MODULE_AUTHOR("Huawei Technologies CO., Ltd"); +MODULE_DESCRIPTION(UDMA_DRV_DESC); +MODULE_VERSION(UDMA_DRV_VERSION); +MODULE_LICENSE("GPL"); diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c new file mode 100644 index 000000000..7be0eae84 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.c @@ -0,0 +1,332 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "comm_defs.h" +#include "ub_mpu_cmd.h" +#include "udma_vtp.h" +#include "udma_tp.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_misc.h" +#include "udma_tpg.h" +#include "udma_resource.h" +#include "udma_query.h" + +static int udma_query_res_list(struct udma_device *udma_dev, + const struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_list_val *ret_val = + (struct ubcore_res_list_val *)(uintptr_t)val->addr; + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_res_dev_tp_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JFS: + return udma_query_res_dev_jfs_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JFR: + return udma_query_res_dev_jfr_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JFC: + return udma_query_res_dev_jfc_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JETTY: + return udma_query_res_dev_jetty_list(udma_dev, ret_val); + case UBCORE_RES_KEY_JETTY_GROUP: + return udma_query_res_dev_jetty_group_list(udma_dev, ret_val); + case UBCORE_RES_KEY_TPG: + return udma_query_res_dev_tpg_list(udma_dev, ret_val); + case UBCORE_RES_KEY_VTP: + return udma_query_res_dev_vtp_list(udma_dev, ret_val); + case UBCORE_RES_KEY_UTP: + return udma_query_res_dev_utp_list(udma_dev, ret_val); + case UBCORE_RES_KEY_RC: + return udma_query_res_dev_rc_list(udma_dev, ret_val); + default: + udma_warn( + udma_dev->dev, UDMA_M_QUERY, + "key type:%u is not supported currently for query res list.\n", + key->type); + return -EINVAL; + } +} + +static int udma_query_res_dev_ta(struct udma_device *udma_dev, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_ta_val *ret_val = + (struct ubcore_res_dev_ta_val *)(uintptr_t)val->addr; + + ret_val->seg_cnt = udma_dev->device_res->seg_list.cnt; + ret_val->jfs_cnt = udma_dev->device_res->jfs_list.cnt; + ret_val->jfr_cnt = udma_dev->device_res->jfr_list.cnt; + ret_val->jfc_cnt = udma_dev->device_res->jfc_list.cnt; + ret_val->jetty_cnt = udma_dev->device_res->jetty_list.cnt; + ret_val->jetty_group_cnt = udma_dev->device_res->jetty_group_list.cnt; + ret_val->rc_cnt = udma_dev->device_res->rc_list.cnt; + + return 0; +} + +static int udma_query_res_dev_tp(struct udma_device *udma_dev, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_tp_val *ret_val = + (struct ubcore_res_dev_tp_val *)(uintptr_t)val->addr; + + ret_val->vtp_cnt = udma_dev->device_res->vtp_list.cnt; + ret_val->tp_cnt = udma_dev->device_res->tpn_list.cnt; + ret_val->tpg_cnt = udma_dev->device_res->tpg_list.cnt; + ret_val->utp_cnt = udma_dev->device_res->utp_list.cnt; + + return 0; +} + +int udma_query_res_pre_check(const struct ubcore_device *dev, + const struct ubcore_res_key *key, + const struct ubcore_res_val *val) + +{ + struct udma_device *udma_dev = NULL; + if (dev == NULL || key == NULL || val == NULL) { + udma_pr_err( + UDMA_M_QUERY, + "parameters is incorrect, dev(%d), key(%d), val(%d).\n", + !!dev, !!key, !!val); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + + if (key->type < UBCORE_RES_KEY_VTP || + key->type > UBCORE_RES_KEY_DEV_TP) { + udma_err(udma_dev->dev, UDMA_M_QUERY, + "key type:%u is invalid.\n", key->type); + return -EINVAL; + } + + return 0; +} + +int udma_query_res(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct udma_device *udma_dev = NULL; + int ret = 0; + + ret = udma_query_res_pre_check(dev, key, val); + if (ret != 0) { + return ret; + } + udma_dev = to_udma_dev(dev); + + if (key->key_cnt == 0 && key->type != UBCORE_RES_KEY_DEV_TA && + key->type != UBCORE_RES_KEY_DEV_TP && + key->type != UBCORE_RES_KEY_SEG) { + return udma_query_res_list(udma_dev, key, val); + } + + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_res_tp(udma_dev, key, val); + case UBCORE_RES_KEY_JFS: + return udma_query_res_jfs(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JFR: + return udma_query_res_jfr(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JFC: + return udma_query_res_jfc(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JETTY: + return udma_query_res_jetty(udma_dev, dev, key, val); + case UBCORE_RES_KEY_JETTY_GROUP: + return udma_query_res_jetty_group(udma_dev, dev, key, val); + case UBCORE_RES_KEY_SEG: + return udma_query_res_segment(udma_dev, dev, key, val); + case UBCORE_RES_KEY_TPG: + return udma_query_res_tpg(udma_dev, key, val); + case UBCORE_RES_KEY_VTP: + return udma_query_res_vtp(udma_dev, key, val); + case UBCORE_RES_KEY_DEV_TA: + return udma_query_res_dev_ta(udma_dev, val); + case UBCORE_RES_KEY_DEV_TP: + return udma_query_res_dev_tp(udma_dev, val); + default: + udma_warn( + udma_dev->dev, UDMA_M_QUERY, + "key type:%u is not supported currently for query res.\n", + key->type); + return -EINVAL; + } +} + +int udma_query_ctx(const struct ubcore_device *dev, struct ubcore_res_key *key, + void *val) +{ + struct udma_device *udma_dev = NULL; + + if (dev == NULL) { + udma_pr_err(UDMA_M_QUERY, "Invalid parameter: ub_dev(%d).\n", + !!dev); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_QUERY, "to_udma_dev failed.\n"); + return -EINVAL; + } + + if (key->type < UBCORE_RES_KEY_VTP || + key->type > UBCORE_RES_KEY_DEV_TP) { + udma_err(udma_dev->dev, UDMA_M_QUERY, + "key type:%u is invalid.\n", key->type); + return -EINVAL; + } + + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_tpc(udma_dev, key, + (struct ub_dfx_tp_ctx_query *)val); + default: + udma_warn(udma_dev->dev, UDMA_M_QUERY, + "key type:%u is not supported currently.\n", + key->type); + return -EINVAL; + } +} + +static int udma_query_fe_idx_from_bdf_list(struct ubcore_device *dev, + const struct ubcore_devid *devid, + u16 *fe_idx) +{ + struct udma_device *udma_dev = to_udma_dev(dev); + u8 bus_id, device_id, func_id; + u32 i; + if (dev == NULL) { + udma_pr_err(UDMA_M_QUERY, "Invalid parameter: ub_dev(%d).\n", + !!dev); + return -EINVAL; + } + + if (udma_dev == NULL || udma_dev->bdf_info == NULL) { + udma_pr_err( + UDMA_M_QUERY, + "query fe_idx invalid parameter. udma_dev(%d), udma_bdf_info(%d).\n", + !!udma_dev, !!udma_dev->bdf_info); + return -EINVAL; + } + + bus_id = *(u8 *)(&devid->raw[UB_DEVID_BUS_U8_OFF]); + device_id = *(u8 *)(&devid->raw[UB_DEVID_DEVICE_U8_OFF]); + func_id = *(u8 *)(&devid->raw[0]); + + udma_info(udma_dev->dev, UDMA_M_QUERY, + "bus_id:%u, device_id:%u, func_id:%u.\n", bus_id, device_id, + func_id); + + mutex_lock(&udma_dev->bdf_mutex); + for (i = 0; i < udma_dev->bdf_cnt; i++) { + if (udma_dev->bdf_info[i].bdf_info_vld != 0 && + udma_dev->bdf_info[i].bus_id == bus_id && + udma_dev->bdf_info[i].device_id == device_id && + udma_dev->bdf_info[i].func_id == func_id) { + *fe_idx = (u16)i; + mutex_unlock(&udma_dev->bdf_mutex); + return 0; + } + } + + udma_err(udma_dev->dev, UDMA_M_QUERY, + "devid(bus_id:%u, device_id:%u, func_id:%u) not found.\n", + bus_id, device_id, func_id); + mutex_unlock(&udma_dev->bdf_mutex); + + return -EINVAL; +} + +int udma_query_fe_idx(struct ubcore_device *dev, struct ubcore_devid *devid, + u16 *fe_idx) +{ + struct ubcore_devid zero_devid = { 0 }; + struct udma_device *udma_dev = to_udma_dev(dev); + int ret = 0; + + if (udma_dev == NULL || devid == NULL || fe_idx == NULL) { + udma_pr_err( + UDMA_M_QUERY, + "query fe_idx invalid parameter. udma_dev(%d), devid(%d), fe_idx(%d).\n", + !!udma_dev, !!devid, !!fe_idx); + return -EINVAL; + } + + if (udma_dev->dev_bus_type != HINIC5_DEVICE_T_PCI) { + udma_pr_err(UDMA_M_QUERY, + "dev(%s) with dev bus type(%d) is not supported.\n", + udma_dev->dev_name, udma_dev->dev_bus_type); + return -EINVAL; + } + + if (!memcmp(devid->raw, zero_devid.raw, UBCORE_DEVID_SIZE)) { + *fe_idx = hinic5_global_func_id(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_QUERY, + "all zero devid get fe_idx:%u\n", *fe_idx); + return 0; + } + + ret = udma_query_fe_idx_from_bdf_list(dev, devid, fe_idx); + if (ret != 0) { + udma_warn(udma_dev->dev, UDMA_M_QUERY, + "bdf not found, reload bdf info\n"); + ret = udma_get_pcie_bdf_info(udma_dev); + if (ret == 0) + ret = udma_query_fe_idx_from_bdf_list(dev, devid, + fe_idx); + } + + return ret; +} + +int udma_query_fe_stats(struct ubcore_device *dev, u32 cnt, u16 *fe_idx, + adapted_ubcore_ue_stats *stats) +{ + struct ub_dfx_vf_statistics_cmd_inbuf cmd_in = { 0 }; + struct ub_dfx_port_statistics_cmd cmd_out = { 0 }; + u16 out_size = (u16)sizeof(struct ub_dfx_port_statistics_cmd); + struct udma_device *udma_dev = NULL; + u32 i = 0; + int ret = 0; + + if (dev == NULL || fe_idx == NULL || stats == NULL || cnt == 0) { + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + for (i = 0; i < cnt; i++) { + cmd_in.vf_id = (u8)fe_idx[i]; + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_READ_VF_STA, &cmd_in, + sizeof(ub_dfx_vf_statistics_cmd_inbuf_s), &cmd_out, + &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_out.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_QUERY, + "query vf statistics counter failed, err(%d), status(0x%x), " + "func id(0x%x), idx(0x%x)\n", + ret, cmd_out.head.status, fe_idx[i], i); + return -EFAULT; + } + + stats[i].rx_bytes = cmd_out.ctr_array_cur.dp_rx.pkt_bytes; + stats[i].rx_pkt = cmd_out.ctr_array_cur.dp_rx.pkt_num; + stats[i].tx_bytes = cmd_out.ctr_array_cur.dp_tx.pkt_bytes; + stats[i].tx_pkt = cmd_out.ctr_array_cur.dp_tx.pkt_num; + + udma_info( + udma_dev->dev, UDMA_M_QUERY, + "query vf statistics counter rx_bytes(0x%llx), rx_pkt(0x%llx), " + "tx_bytes(0x%llx), tx_pkt(0x%llx) func id(0x%x)\n", + stats[i].rx_bytes, stats[i].rx_pkt, stats[i].tx_bytes, + stats[i].tx_pkt, fe_idx[i]); + } + + return 0; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h new file mode 100644 index 000000000..d193e4675 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_query.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_QUERY_H +#define UDMA_QUERY_H + +#include "ubcore_types.h" +#include "udma_common.h" + +int udma_query_res(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_query_ctx(const struct ubcore_device *dev, struct ubcore_res_key *key, + void *val); +int udma_query_fe_idx(struct ubcore_device *dev, struct ubcore_devid *devid, + u16 *fe_idx); +int udma_query_fe_stats(struct ubcore_device *dev, u32 cnt, u16 *fe_idx, + adapted_ubcore_ue_stats *stats); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c new file mode 100644 index 000000000..a4351dfa6 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.c @@ -0,0 +1,153 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "hinic5_hw.h" +#include "hmm_em_inner.h" +#include "hmm_buddy.h" +#include "udma_common.h" +#include "udma_comp.h" + +static int udma_ubrc_alloc_check(struct udma_comp_priv **comp_priv, + struct udma_device *udma_dev, u32 num) +{ + *comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (*comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Comp_priv is null\n"); + return -EINVAL; + } + + if ((num < UBRC_MIN_DEPTH) || (num > UBRC_MAX_DEPTH)) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Num is invalid, ret(%d)\n", -EINVAL); + *comp_priv = NULL; + return -EINVAL; + } + + return 0; +} + +void udma_ubrc_attr_init(struct udma_device *udma_dev, struct ubrc *ubrc, + u32 num, u32 log_ubrc_per_seg) +{ + u32 chip_num = (num < UBRC_MIN_ENTRY) ? UBRC_MIN_ENTRY : num; + u32 ext_num = num + UBRC_EXT_ENTRY; + u32 ext_order = 0; + u32 order = 0; + u32 i = 0; + + for (i = 1; i < chip_num; i <<= 1) + order++; + + for (i = 1; i < ext_num; i <<= 1) + ext_order++; + + ubrc->order = order; + ubrc->ext_order = (ext_order > log_ubrc_per_seg) ? ext_order : + log_ubrc_per_seg; +} + +int udma_ubrc_alloc(struct udma_device *udma_dev, u32 num, struct ubrc *ubrc) +{ + struct udma_comp_priv *comp_priv = NULL; + void *vaddr = NULL; + u32 log_ubrc_per_seg = 0; + u32 ext_order; + u32 offset = 0; + int ret = 0; + + if (ubrc == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, "ubrc is null\n"); + return -EINVAL; + } + + ret = udma_ubrc_alloc_check(&comp_priv, udma_dev, num); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "udma ubrc alloc check failed\n"); + return ret; + } + + log_ubrc_per_seg = comp_priv->ub_cap.log_ubrc_seg; + + udma_ubrc_attr_init(udma_dev, ubrc, num, log_ubrc_per_seg); + ext_order = ubrc->ext_order - log_ubrc_per_seg; + + offset = hmm_buddy_alloc(&comp_priv->ubrc_buddy, ext_order); + if (offset == UB_INVALID_INDEX) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Alloc ubrc index failed, ret(%d)\n", -ENOMEM); + return -ENOMEM; + } + + ubrc->offset = offset << log_ubrc_per_seg; + + ret = hmm_em_table_get_range( + comp_priv->hmm_priv.dev, &comp_priv->ubrc_em_table, + ubrc->offset, ubrc->offset + (1U << ubrc->ext_order) - 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Alloc ubrc entry failed, ret(%d)\n", -ENOMEM); + goto err_table_get; + } + + vaddr = hmm_em_table_find(&comp_priv->ubrc_em_table, ubrc->offset, + &ubrc->dma_addr); + if (vaddr == NULL) { + ret = -ENOMEM; + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Can't find va and pa of ubrc entry, ret(%d)\n", + -ENOMEM); + goto err_ubrc_find; + } + ubrc->vaddr = vaddr; + (void)memset_s(vaddr, ((u64)1U << (ext_order + UBRC_MAXSIZE_OFFSET)), 0, + ((u64)1U << (ext_order + UBRC_MAXSIZE_OFFSET))); + + return 0; + +err_ubrc_find: + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table, ubrc->offset, + ubrc->offset + (1U << ubrc->ext_order) - 1); +err_table_get: + hmm_buddy_free(&comp_priv->ubrc_buddy, offset, ext_order); + + return ret; +} + +void udma_ubrc_free(struct udma_device *udma_dev, struct ubrc *ubrc) +{ + struct udma_comp_priv *comp_priv = NULL; + u32 log_ubrc_per_seg = 0; + u32 offset = 0; + u32 order = 0; + + if (ubrc == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, "ubrc is null\n"); + return; + } + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_UBRC_COMP, + "Comp_priv is null\n"); + return; + } + + hmm_em_table_put_range(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table, ubrc->offset, + ubrc->offset + (1U << ubrc->ext_order) - 1); + log_ubrc_per_seg = comp_priv->ub_cap.log_ubrc_seg; + + order = ubrc->ext_order - log_ubrc_per_seg; + offset = ubrc->offset >> log_ubrc_per_seg; + + hmm_buddy_free(&comp_priv->ubrc_buddy, offset, order); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h new file mode 100644 index 000000000..64e67b45e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_ubrc_comp.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_UBRC_COMP_H +#define UDMA_UBRC_COMP_H + +#include "udma_common.h" + +int udma_ubrc_alloc(struct udma_device *udma_dev, u32 num, struct ubrc *ubrc); +void udma_ubrc_free(struct udma_device *udma_dev, struct ubrc *ubrc); +void udma_ubrc_attr_init(struct udma_device *udma_dev, struct ubrc *ubrc, + u32 num, u32 log_ubrc_per_seg); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c new file mode 100644 index 000000000..0b60d76a9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> + +#include "comm_defs.h" +#include "ub_mpu_cmd.h" +#include "ubcore_types.h" +#include "ub_pub_cmd.h" +#include "udma_common.h" +#include "udma_xid_cfg.h" + +int udma_vm_glb_xid_cfg_set(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + struct ub_vm_global_xid_cfg_cmd xid_cfg_cmd = { 0 }; + u16 out_size = sizeof(struct ub_vm_global_xid_cfg_cmd); + int ret; + + if (!udma_dev->ub_dev.attr.tp_maintainer) { + /* only has tp maintainer has permission set xid cfg, normal return, not affect outside layer call function */ + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, + "device of func_id(%u) is not tp maintainer\n", + adapted_ue_idx(cfg)); + return 0; + } + + if (adapted_ue_idx(cfg) < PF_MAX_NUM) { + /* description not is vf scenario, normal return, not affect outside layer call function */ + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, + "func_id(%u) is not vfid\n", adapted_ue_idx(cfg)); + return 0; + } + + if (cfg->mask.bs.max_jetty_cnt == 0 && + cfg->mask.bs.min_jetty_cnt == 0) { + /* no has set jetty_cnt, normal return, not affect outside layer call function */ + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, + "func_id(%u) jetty mask is not set!\n", + adapted_ue_idx(cfg)); + return 0; + } + + xid_cfg_cmd.vfid = adapted_ue_idx(cfg); + + xid_cfg_cmd.xid_cfg.min_jetty_num = cfg->min_jetty_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.min_jetty_num = cfg->mask.bs.min_jetty_cnt; + xid_cfg_cmd.xid_cfg.max_jetty_num = cfg->max_jetty_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.max_jetty_num = cfg->mask.bs.max_jetty_cnt; + xid_cfg_cmd.xid_cfg.min_jfr_num = cfg->min_jfr_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.min_jfr_num = cfg->mask.bs.min_jfr_cnt; + xid_cfg_cmd.xid_cfg.max_jfr_num = cfg->max_jfr_cnt; + xid_cfg_cmd.xid_cfg.mask.bs.max_jfr_num = cfg->mask.bs.max_jfr_cnt; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_VM_XID_CFG, &xid_cfg_cmd, + sizeof(xid_cfg_cmd), &xid_cfg_cmd, + &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || xid_cfg_cmd.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_GLB_XID_CFG, + "set xid cfg failed, ret(%d), status(0x%x), out size(0x%x)\n", + ret, xid_cfg_cmd.head.status, out_size); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_GLB_XID_CFG, "success, vfid(%u)\n", + adapted_ue_idx(cfg)); + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h new file mode 100644 index 000000000..75c5fdee1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/device/udma_xid_cfg.h @@ -0,0 +1,16 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_XID_CFG_H +#define UDMA_XID_CFG_H + +#include "ubcore_types.h" +#include "udma_common.h" + +#define PF_MAX_NUM 32 + +int udma_vm_glb_xid_cfg_set(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg); + +#endif //UDMA_XID_CFG_H diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c new file mode 100644 index 000000000..2964a1e1a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.c @@ -0,0 +1,2625 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <asm/io.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> +#include <linux/security.h> + +#include "hinic5_mt.h" +#include "mig_mpu_cmd.h" +#include "mig_mpu_cmd_defs.h" +#include "mpu_inband_cmd.h" +#include "comm_defs.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "udma_cmd_segment.h" +#include "udma_dfx_attack.h" +#include "udma_dfx_capture.h" +#include "udma_cqm.h" +#include "udma_cmd_event.h" +#include "udma_cmd_jfc.h" +#include "udma_jetty.h" +#include "udma_net.h" +#include "ub_npu_jfrc_cmd_defs.h" +#include "udma_rc_table.h" +#include "ubcore_uapi.h" +#include "udma_misc.h" +#include "udma_device.h" +#include "udma_ucontext.h" +#include "ubcore_types.h" +#include "udma_query.h" +#include "udma_tp.h" +#include "udma_srq.h" +#include "udma_vtp.h" +#include "udma_tpg.h" +#include "udma_utp.h" +#include "udma_dfx_port_sta.h" +#include "udma_dfx_perf_ctr.h" +#include "udma_dfx_cqm_modify.h" +#include "udma_dfx_modify.h" +#include "udma_dfx_query_tp.h" +#include "udma_cmd_srq.h" +#include "udma_cmd_eid.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_com.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_vtp.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfs.h" +#include "udma_cmd_jfc.h" +#include "udma_cmd_jfr.h" +#include "udma_cmd_jfrc.h" +#include "udma_resource.h" +#include "udma_cmd_mig.h" +#include "ub_mpu_cmd.h" +#include "ub_mpu_cmd_defs.h" +#include "udma_dfx_latch.h" +#include "udma_dfx_l2d.h" +#include "udma_dfx_perf_query.h" +#include "udma_dfx.h" + +int udma_dfx_obj_put(struct udma_device *udev, u32 type, void *obj, u32 xid) +{ + if (obj == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "xid:%u not find in type: %u\n", + xid, type); + return -EINVAL; + } + switch (type) { + case UB_QUERY_TYPE_JFS: + udma_jfs_put(obj); + break; + case UB_QUERY_TYPE_JFR: + udma_jfr_put(obj); + break; + case UB_QUERY_TYPE_JFC: + udma_jfc_put(obj); + break; + case UB_QUERY_TYPE_JETTY: + udma_jetty_put(obj); + break; + case UB_QUERY_TYPE_JETTY_GROUP: + udma_jetty_group_put(obj); + break; + case UB_QUERY_TYPE_TP: + udma_tp_put(obj); + break; + case UB_QUERY_TYPE_TPG: + udma_tpg_put(obj); + break; + case UB_QUERY_TYPE_VTP: + udma_vtp_put(obj); + break; + case UB_QUERY_TYPE_SEG: + udma_seg_put(obj); + break; + default: + udma_err(udev->dev, UDMA_M_DFX, + "Put obj error, type is (%d).\n", type); + return -EINVAL; + } + return 0; +} + +int udma_dfx_check_xid(struct udma_device *udev, u32 type, + struct ub_query_inbuf *query_inbuf) +{ + void *ptr = NULL; + int ret = 0; + + if (query_inbuf == NULL) { + udma_pr_err(UDMA_M_DFX, + "Invalid parameter. query_inbuf(%d) .\n", + !!query_inbuf); + return -EINVAL; + } + switch (type) { + case UB_QUERY_TYPE_JFS: + ptr = (void *)udma_find_jfs(udev, query_inbuf->jfsid); + break; + case UB_QUERY_TYPE_JFR: + ptr = (void *)udma_find_jfr(udev, query_inbuf->jfrid); + break; + case UB_QUERY_TYPE_JFC: + ptr = (void *)udma_find_jfc(udev, query_inbuf->jfcid); + break; + case UB_QUERY_TYPE_JETTY: + ptr = (void *)udma_find_jetty(udev, query_inbuf->jettyid); + break; + case UB_QUERY_TYPE_JETTY_GROUP: + ptr = (void *)udma_find_jetty_group( + udev, query_inbuf->jetty_group_id); + break; + case UB_QUERY_TYPE_TP: + ptr = (void *)udma_find_tp(udev, query_inbuf->tpn); + break; + case UB_QUERY_TYPE_TPG: + ptr = (void *)udma_get_tpg_by_id(udev, query_inbuf->tpg_id); + break; + case UB_QUERY_TYPE_VTP: + ptr = (void *)udma_get_vtp_by_id(udev, query_inbuf->vtp_id, + query_inbuf->vfid); + break; + default: + return ret; + } + ret = udma_dfx_obj_put(udev, type, ptr, query_inbuf->xid); + return ret; +} + +static int udma_dfx_get_drv_version(const void *buf_in, u32 in_size, + void *buf_out, const u32 *out_size) +{ + struct drv_version_info *info = buf_out; + int rc; + + if (!buf_out || !out_size) { + udma_pr_err(UDMA_M_DFX, "Input params is NULL.\n"); + return -EINVAL; + } + + if (*out_size != sizeof(*info)) { + udma_pr_err( + UDMA_M_DFX, + "Unexpect out buf size from user :%u, expect: %lu\n", + *out_size, sizeof(*info)); + return -EINVAL; + } + + rc = snprintf_s(info->ver, sizeof(info->ver), sizeof(info->ver) - 1, + "%s %s", UDMA_DRV_VERSION, __TIME_STR__); + if (rc == -1) { + udma_pr_err(UDMA_M_DFX, "Snprintf udma version err\n"); + return -EFAULT; + } + + return 0; +} + +int udma_dfx_query_prepare(struct udma_device *udma_dev, void *buf_out, + u32 *out_size) +{ + int ret = 0; + + ret = memset_s(buf_out, *out_size, 0, *out_size); + if (ret != EOK) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query prepare memset error ret:%d\n", ret); + return -EFAULT; + } + + if (*out_size < (u32)sizeof(union ub_query_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query prepare buffer out size too small\n"); + return -EFAULT; + } + *out_size = (u32)sizeof(union ub_query_outbuf); + + return 0; +} + +static int udma_dfx_query_seg_mpt_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_mapt_ctx_query *mpt_entry = &outbuf->mpt; + u32 tokenid = inbuf->tokenid; + int ret; + + (void)memset_s(mpt_entry, sizeof(struct ub_mapt_ctx_query), 0, + sizeof(struct ub_mapt_ctx_query)); + + ret = udma_cmd_seg_query_ctx(udev, tokenid, mpt_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query mpt from cache! tokenid:%u", tokenid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_seg_mpt_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_mapt_ctx_query *mpt_entry = &outbuf->mpt; + struct udma_segment *udma_seg = NULL; + struct mpt *mpt = NULL; + + udma_seg = udma_seg_find(udev, inbuf->tokenid); + if (udma_seg == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ubcore_seg, tokenid:%u", + inbuf->tokenid); + return -EINVAL; + } + + mpt = udma_seg->mpt; + memcpy_fromio(mpt_entry, mpt->vaddr, sizeof(struct ub_mapt_ctx_query)); + udma_seg_put(udma_seg); + return 0; +} + +static int udma_dfx_query_seg_safe_dmtt_bitmap( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_cmd_safe_dmtt_bitmap_outbuf *safe_dmtt_bitmap = + &outbuf->safe_dmtt_bitmap; + int ret; + + (void)memset_s(safe_dmtt_bitmap, + sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf), 0, + sizeof(struct ub_cmd_safe_dmtt_bitmap_outbuf)); + + ret = udma_cmd_seg_query_safe_dmtt_bitmap(udma_dev, inbuf, + safe_dmtt_bitmap); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query safe dmtt bitmap func_id:%u", + inbuf->vfid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_seg_safe_dmtt_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_cmd_safe_dmtt_ctx_outbuf *safe_dmtt_ctx = + &outbuf->safe_dmtt_ctx; + int ret; + + (void)memset_s(safe_dmtt_ctx, + sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf), 0, + sizeof(struct ub_cmd_safe_dmtt_ctx_outbuf)); + + ret = udma_cmd_seg_query_safe_dmtt_ctx(udma_dev, inbuf, safe_dmtt_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query safe dmtt ctx func_id:%u", + inbuf->vfid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_dmtt(struct udma_device *udev, struct mtt *ub_mtt, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct mtt_seg *mtt_seg = NULL; + void *vaddr = NULL; + u32 max_mtt_entry; + u32 offset; + u32 length; + u32 i; + + if (ub_mtt->mtt_layers == 0) { + udma_err(udev->dev, UDMA_M_DFX, "mtt layer is 0\n"); + return -EINVAL; + } + + if (inbuf->layers > ub_mtt->mtt_layers || inbuf->layers == 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "inbuf layer should be less than %u and not equal zero!\n", + ub_mtt->mtt_layers); + return -EINVAL; + } + + length = inbuf->length; + offset = inbuf->offset; + mtt_seg = ub_mtt->mtt_seg[inbuf->layers - 1]; + + if (mtt_seg->order > UDMA_MTT_MAX_ORDER) { + udma_err(udev->dev, UDMA_M_DFX, "mtt_seg order[%u] is err\n", + mtt_seg->order); + return -EINVAL; + } + + max_mtt_entry = 1 << mtt_seg->order; + + if (length >= HINIC5_MAX_BUF_SIZE) { + udma_err(udev->dev, UDMA_M_DFX, + "length is too large, HINIC5_MAX_BUF_SIZE[%u]\n", + HINIC5_MAX_BUF_SIZE); + return -EINVAL; + } + if (offset + length > max_mtt_entry) { + udma_err(udev->dev, UDMA_M_DFX, + "offset or length is too large, max_mtt_entry[%u]\n", + max_mtt_entry); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(udev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "Comp_priv is null\n"); + return -EINVAL; + } + + for (i = 0; i < length; i++) { + vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, + mtt_seg->offset + offset + i, + (dma_addr_t *)&outbuf->dmtt[i]); + if (vaddr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "mtt entry is not found\n"); + return -EINVAL; + } + } + + return 0; +} + +static int udma_dfx_query_seg_dmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct mtt *mtt = NULL; + struct udma_segment *udma_seg = NULL; + int ret = 0; + + udma_seg = udma_seg_find(udev, inbuf->tokenid); + if (udma_seg == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get udma_segment, tokenid = %u", + inbuf->tokenid); + return -EINVAL; + } + + mtt = &udma_seg->mtt; + + ret = udma_dfx_query_dmtt(udev, mtt, inbuf, outbuf); + udma_seg_put(udma_seg); + return ret; +} + +int udma_dfx_query_cmtt(struct udma_device *udev, struct mtt *ub_mtt, + union ub_query_outbuf *outbuf) +{ + struct hmm_comp_priv *comp_priv = NULL; + struct mtt_seg *mtt_seg = NULL; + void *vaddr = NULL; + int i; + + if (ub_mtt->mtt_layers == 0) { + udma_err(udev->dev, UDMA_M_DFX, "mtt layer is 0\n"); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(udev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "Comp_priv is null\n"); + return -EINVAL; + } + + mtt_seg = ub_mtt->mtt_seg[ub_mtt->mtt_layers - 1]; + + if (mtt_seg->order > UDMA_MTT_MAX_ORDER) { + udma_err(udev->dev, UDMA_M_DFX, "mtt_seg order[%u] is err\n", + mtt_seg->order); + return -EINVAL; + } + + for (i = 0; i < (1 << mtt_seg->order); i++) { + vaddr = hmm_em_table_find(&comp_priv->mtt_em_table, + mtt_seg->offset + i, + (dma_addr_t *)&outbuf->cmtt[i]); + if (vaddr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "mtt entry is not found\n"); + return -EINVAL; + } + } + + udma_info(udev->dev, UDMA_M_DFX, "query mmt entry success\n"); + + return 0; +} + +static int udma_dfx_query_jetty_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jetty_entry = &outbuf->jetty_ctx; + struct udma_jetty *jetty = NULL; + u32 jettyid = inbuf->jettyid; + int ret; + + (void)memset_s(jetty_entry, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + jetty = udma_find_jetty(udev, jettyid); + if (jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jetty, jettyid(%u)\n", jettyid); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udev, jettyid, &jetty->cmdq_rtt, + jetty_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query jetty from cache! jettyid(%u)", + jettyid); + return ret; + } + + udma_jetty_put(jetty); + return 0; +} + +static int udma_dfx_query_jetty_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jetty_entry = &outbuf->jetty_ctx; + struct udma_jetty *ub_jetty = NULL; + u32 jettyid = inbuf->jettyid; + + ub_jetty = udma_find_jetty(udev, jettyid); + if (ub_jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jettyid(%u)", jettyid); + return -EINVAL; + } + + memcpy_fromio(jetty_entry, ub_jetty->jettyc_info->vaddr, + sizeof(struct ub_ta_context_query)); + udma_jetty_put(ub_jetty); + return 0; +} + +static int +udma_dfx_query_jetty_group_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jetty_grp_ctx_query *jetty_grp_entry = &outbuf->jetty_grp_ctx; + u32 jetty_group_id = inbuf->jetty_group_id; + int ret; + + (void)memset_s(jetty_grp_entry, sizeof(struct ub_jetty_grp_ctx_query), + 0, sizeof(struct ub_jetty_grp_ctx_query)); + + ret = udma_cmd_jetty_grp_query_ctx(udev, jetty_group_id, + jetty_grp_entry); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query jetty group from cache! jetty group id(%u)", + jetty_group_id); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_jetty_group_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jetty_grp_ctx_query *jetty_grp_entry = &outbuf->jetty_grp_ctx; + struct udma_jetty_grp *ub_jetty_grp = NULL; + u32 jetty_grp_id = inbuf->jetty_group_id; + + ub_jetty_grp = udma_find_jetty_group(udev, jetty_grp_id); + if (ub_jetty_grp == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ubcore_jetty_group, jetty group id(%u)", + jetty_grp_id); + return -EINVAL; + } + + memcpy_fromio(jetty_grp_entry, ub_jetty_grp->cqm_jetty_grp->q_ctx_vaddr, + sizeof(struct ub_jetty_grp_ctx_query)); + udma_jetty_group_put(ub_jetty_grp); + return 0; +} + +static int copy_data_from_page(struct udma_device *udev, u8 *outbuf, + struct ubcore_umem *umem, u32 copy_len, + u32 offset) +{ + struct scatterlist *sg = NULL; + u8 *outbuf_tmp = outbuf; + u32 page_copy_len = 0; + u32 has_copy_len = 0; + u32 page_offset = 0; + u32 buf_offset = 0; + void *va = NULL; + u32 i = 0; + + udma_info(udev->dev, UDMA_M_DFX, + "copy_data_from_page va:%llx, length:%llu, offset:%u, \ + copy_len:%u", + umem->va, umem->length, offset, copy_len); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + if (buf_offset + PAGE_SIZE > offset) { + page_offset = offset - buf_offset; + buf_offset = offset; + va = sg_virt(sg); + if (va == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get va"); + return -ENOMEM; + } + + va += page_offset; + page_copy_len = ((copy_len - has_copy_len) < + (PAGE_SIZE - page_offset)) ? + (copy_len - has_copy_len) : + (PAGE_SIZE - page_offset); + udma_info( + udev->dev, UDMA_M_DFX, + "copy_data_from_page buf_offset:%u, page_offset:%u, has_copy_len:%u, \ + page_copy_len:%u, i:%u", + buf_offset, page_offset, has_copy_len, + page_copy_len, i); + (void)memcpy_s(outbuf_tmp, page_copy_len, va, + page_copy_len); + outbuf_tmp += page_copy_len; + has_copy_len += page_copy_len; + if (has_copy_len >= copy_len) { + break; + } + } else { + buf_offset += PAGE_SIZE; + } + } + return 0; +} + +static int udma_dfx_query_jetty_sqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + u8 *jetty_sqe = (u8 *)outbuf->sqe; + struct udma_jetty *ub_jetty = NULL; + u32 jettyid = inbuf->jettyid; + u32 wqe_offset; + u32 copy_len; + int ret = 0; + + if (source_type != UB_QUERY_SQ) { + udma_err(udev->dev, UDMA_M_DFX, + "query jetty sqe, source type should be UB_QUERY_SQ"); + return -EINVAL; + } + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + (void)memset_s(jetty_sqe, UB_MAX_JFS_DFX_BUF, 0x0, UB_MAX_JFS_DFX_BUF); + ub_jetty = udma_find_jetty(udev, jettyid); + if (ub_jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jettyid = %u", jettyid); + return -EINVAL; + } + + if (ub_jetty->ubcore_jetty.uctx == NULL) { + wqe_offset = inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % + ub_jetty->sq.buf_size; + copy_len = ((UB_MAX_JFS_DFX_BUF + wqe_offset) < + ub_jetty->sq.buf_size) ? + UB_MAX_JFS_DFX_BUF : + (ub_jetty->sq.buf_size - wqe_offset); + udma_info(udev->dev, UDMA_M_DFX, + "copy sq buf from kernel copy_len:%u, wqe_offset:%u", + copy_len, wqe_offset); + (void)memcpy_s(jetty_sqe, copy_len, + ub_jetty->sq.buf + wqe_offset, copy_len); + goto OUT; + } + wqe_offset = + inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % ub_jetty->umem->length; + + ret = copy_data_from_page(udev, jetty_sqe, ub_jetty->umem, + UB_MAX_JFS_DFX_BUF, wqe_offset); +OUT: + udma_jetty_put(ub_jetty); + return ret; +} + +static int +udma_dfx_query_rc_queue_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *rcq_ctx = &outbuf->rcq_ctx; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(rcq_ctx, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + ret = udma_query_rc_queue_ctx_from_cache( + udev, rcq_id, rcq_ctx, sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue context from cache! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_ctx_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *rcq_ctx = &outbuf->rcq_ctx; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(rcq_ctx, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + ret = udma_query_rc_queue_ctx_from_host( + udev, rcq_id, rcq_ctx, sizeof(struct ub_ta_context_query)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue context from host! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_pcqc_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(pcqc, sizeof(ub_pcqc_query_rsp_s), 0, + sizeof(ub_pcqc_query_rsp_s)); + + ret = udma_query_rc_queue_pcqc_from_cache(udev, rcq_id, pcqc, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue pcqc from cache! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_pcqc_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + u32 rcq_id = inbuf->rcq_id; + int ret; + + (void)memset_s(pcqc, sizeof(ub_pcqc_query_rsp_s), 0, + sizeof(ub_pcqc_query_rsp_s)); + + ret = udma_query_rc_queue_pcqc_from_host(udev, rcq_id, pcqc, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query rc queue pcqc from cache! rcq_id(%u) ret(%d)", + rcq_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_wqe_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_rc_queue_wqe *rcqe = &outbuf->rcqe; + u32 rcq_id = inbuf->rcq_id; + u32 ci = inbuf->wqe_id; + int ret; + + (void)memset_s(rcqe, sizeof(struct ub_rc_queue_wqe), 0, + sizeof(struct ub_rc_queue_wqe)); + + ret = udma_query_rc_queue_wqe_from_cache( + udev, rcq_id, ci, rcqe, sizeof(struct ub_rc_queue_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_rc_queue_wqe_from_cache_func failed! rcq_id(%u) ci(%u) ret(%d)", + rcq_id, ci, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_rc_queue_wqe_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_rc_queue_wqe *rcqe = &outbuf->rcqe; + u32 rcq_id = inbuf->rcq_id; + u32 ci = inbuf->wqe_id; + int ret; + + (void)memset_s(rcqe, sizeof(struct ub_rc_queue_wqe), 0, + sizeof(struct ub_rc_queue_wqe)); + + ret = udma_query_rc_queue_wqe_from_host(udev, rcq_id, ci, rcqe, + sizeof(struct ub_rc_queue_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_rc_queue_wqe_from_host_func failed! rcq_id(%u) ci(%u) ret(%d)", + rcq_id, ci, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_jfc_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfc *udma_jfc = NULL; + struct mtt *ub_mtt = NULL; + int ret = 0; + + udma_jfc = udma_find_jfc(udev, inbuf->jfcid); + if (udma_jfc == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "Failed to get jfc, jfcid = %u", + inbuf->jfcid); + return -EINVAL; + } + + ub_mtt = (struct mtt *)&udma_jfc->ub_mtt; + + ret = udma_dfx_query_cmtt(udev, ub_mtt, outbuf); + udma_jfc_put(udma_jfc); + return ret; +} + +static int udma_dfx_query_jfc_cqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfc *udma_jfc = NULL; + u32 jfcid = inbuf->jfcid; + u8 *jfc_cqe = outbuf->cqe; + u32 cqe_offset; + int ret = 0; + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + (void)memset_s(jfc_cqe, UB_MAX_JFC_DFX_BUF, 0x0, UB_MAX_JFC_DFX_BUF); + udma_jfc = udma_find_jfc(udev, jfcid); + if (udma_jfc == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfc_cqe_func: ubcore find jfc failed, jfcid(%u)", + jfcid); + return -EINVAL; + } + + if (udma_jfc->umem == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfc_cqe_func: udma_jfc->umem is null"); + ret = -EINVAL; + goto OUT; + } + + cqe_offset = + inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % udma_jfc->umem->length; + ret = copy_data_from_page(udev, jfc_cqe, udma_jfc->umem, + UB_MAX_JFC_DFX_BUF, cqe_offset); + udma_be32_to_cpu(jfc_cqe, UB_MAX_JFC_DFX_BUF); +OUT: + udma_jfc_put(udma_jfc); + return ret; +} + +static int udma_dfx_query_jfr_rqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + u8 *jfr_rqe = outbuf->rqe; + struct udma_srq *srq = NULL; + u32 whole_container_size; // ci corresponding to container of size + u32 wqe_offset; + u64 wqe_offset_tmp; + int ret = 0; + + (void)memset_s(jfr_rqe, UB_MAX_JFR_DFX_BUF, 0x0, UB_MAX_JFR_DFX_BUF); + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_rqe_func: ubcore find jfr failed, jfrid(%u)", + jfrid); + return -EINVAL; + } + + srq = &udma_jfr->srq; + + if (srq->umem == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_rqe_func: srq->umem is null"); + ret = -EINVAL; + goto OUT; + } + + whole_container_size = (1u << srq->wqe_shift) * srq->container_size; + if (inbuf->wqe_id > (UINT_MAX / whole_container_size)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause overflow", + inbuf->wqe_id); + ret = -EINVAL; + goto OUT; + } + wqe_offset_tmp = (u64)inbuf->wqe_id * whole_container_size; + wqe_offset = (u32)(wqe_offset_tmp % srq->umem->length); + + udma_info( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_rqe_func: whole_container_size(%u), wqe_shift(%u), container_size(%u), umem->length(%llu)", + whole_container_size, srq->wqe_shift, srq->container_size, + srq->umem->length); + + ret = copy_data_from_page(udev, jfr_rqe, srq->umem, UB_MAX_JFR_DFX_BUF, + wqe_offset); +OUT: + udma_jfr_put(udma_jfr); + return ret; +} + +static int udma_dfx_query_idx_wqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfr *udma_jfr = NULL; + u32 jfr_id = inbuf->jfrid; + u8 *idx_wqe = outbuf->idx_wqe; + struct udma_index_queue *index_queue = NULL; + u32 wqe_size; + u32 wqe_offset; + u64 wqe_offset_tmp; + int ret = 0; + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err( + udev->dev, UDMA_M_DFX, + "Invalid parameter: inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + (void)memset_s(idx_wqe, UB_MAX_IDX_WQE_DFX_BUF, 0x0, + UB_MAX_IDX_WQE_DFX_BUF); + udma_jfr = udma_find_jfr(udev, jfr_id); + if (udma_jfr == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_idx_wqe: ubcore find jfr failed, jfr_id(%u)", + jfr_id); + return -EINVAL; + } + + index_queue = &udma_jfr->index_queue; + + if (inbuf->wqe_id > index_queue->max_depth) { + udma_err(udev->dev, UDMA_M_DFX, + "Invalid parameter: inbuf->wqe_id %u, jfr depth: %u", + inbuf->wqe_id, index_queue->max_depth); + ret = -EINVAL; + goto OUT; + } + + if (index_queue->umem == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "udma_dfx_query_idx_wqe: index_queue->umem is null"); + ret = -EINVAL; + goto OUT; + } + + wqe_size = 1u << index_queue->wqe_shift; + if (inbuf->wqe_id > (UINT_MAX / wqe_size)) { + udma_err( + udev->dev, UDMA_M_DFX, + "Invalid parameter: inbuf->wqe_id %u too large may cause overflow", + inbuf->wqe_id); + ret = -EINVAL; + goto OUT; + } + wqe_offset_tmp = (u64)inbuf->wqe_id * wqe_size; + wqe_offset = (u32)(wqe_offset_tmp % index_queue->umem->length); + + udma_info( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_idx_wqe: wqe_size(%u), wqe_shift(%u), umem->length(%llu)", + wqe_size, index_queue->wqe_shift, index_queue->umem->length); + + ret = copy_data_from_page(udev, idx_wqe, index_queue->umem, + UB_MAX_IDX_WQE_DFX_BUF, wqe_offset); +OUT: + udma_jfr_put(udma_jfr); + return ret; +} + +static int udma_dfx_query_sip(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_sip_attr sip_attr = { 0 }; + u32 sip_index = inbuf->sipid; + int ret; + + ret = udma_query_sip(udev, sip_index, &sip_attr); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query sip, sip_index:%u, ret:%d", sip_index, + ret); + return ret; + } + (void)memcpy_s(&outbuf->sip_attr, sizeof(sip_attr), &sip_attr, + sizeof(sip_attr)); + return 0; +} + +static void copy_dev_res_to_dfx(struct ub_dev_res *dev_res, u32 offset, + struct udma_device_ctx_res *res_val) +{ + u32 i; + + for (i = offset; i < offset + UB_QUERY_RES_CNT; i++) { + if (i < res_val->seg_list.seg_cnt) { + (void)memcpy_s( + dev_res->seg_list[i - offset].eid, + UBCORE_EID_SIZE, + res_val->seg_list.seg_list[i].ubva.eid.raw, + UBCORE_EID_SIZE); + dev_res->seg_list[i - offset].va = + res_val->seg_list.seg_list[i].ubva.va; + dev_res->seg_list[i - offset].key_id = + res_val->seg_list.seg_list[i].token_id; + dev_res->seg_list[i - offset].len = + res_val->seg_list.seg_list[i].len; + } + if (i < res_val->jfs_list.cnt) { + dev_res->jfs_list[i - offset] = + res_val->jfs_list.list[i]; + } + if (i < res_val->jfr_list.cnt) { + dev_res->jfr_list[i - offset] = + res_val->jfr_list.list[i]; + } + if (i < res_val->jfc_list.cnt) { + dev_res->jfc_list[i - offset] = + res_val->jfc_list.list[i]; + } + if (i < res_val->jetty_list.cnt) { + dev_res->jetty_list[i - offset] = + res_val->jetty_list.list[i]; + } + if (i < res_val->jetty_group_list.cnt) { + dev_res->jetty_group_list[i - offset] = + res_val->jetty_group_list.list[i]; + } + if (i < res_val->tp_list.cnt) { + dev_res->tp_list[i - offset] = res_val->tp_list.list[i]; + } + if (i < res_val->tpg_list.cnt) { + dev_res->tpg_list[i - offset] = + res_val->tpg_list.list[i]; + } + if (i < res_val->rc_list.cnt) { + dev_res->rc_list[i - offset] = res_val->rc_list.list[i]; + } + if (i < res_val->vtp_list.cnt) { + dev_res->vtp_list[i - offset] = + res_val->vtp_list.list[i]; + } + if (i < res_val->utp_list.cnt) { + dev_res->utp_list[i - offset] = + res_val->utp_list.list[i]; + } + } +} + +static void udma_dfx_query_dev_member_cnt(struct udma_device *udma_dev, + struct ub_dev_res *dev_res, + struct udma_device_ctx_res *res_val) +{ + dev_res->jfs_cnt = res_val->jfs_list.cnt; + dev_res->jfr_cnt = res_val->jfr_list.cnt; + dev_res->seg_cnt = res_val->seg_list.seg_cnt; + dev_res->jfc_cnt = res_val->jfc_list.cnt; + dev_res->jetty_cnt = res_val->jetty_list.cnt; + dev_res->jetty_group_cnt = res_val->jetty_group_list.cnt; + dev_res->tp_cnt = res_val->tp_list.cnt; + dev_res->tpg_cnt = res_val->tpg_list.cnt; + dev_res->rc_cnt = res_val->rc_list.cnt; + dev_res->vtp_cnt = res_val->vtp_list.cnt; + dev_res->utp_cnt = res_val->utp_list.cnt; + + udma_info(udma_dev->dev, UDMA_M_DFX, "get tp_cnt:%u\n", + dev_res->tp_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jfs_cnt:%u\n", + dev_res->jfs_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jfr_cnt:%u\n", + dev_res->jfr_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jetty_cnt:%u\n", + dev_res->jetty_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jfc_cnt:%u\n", + dev_res->jfc_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get seg_cnt:%u\n", + dev_res->seg_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get jetty_group_cnt:%u\n", + dev_res->jetty_group_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get tpg_cnt:%u\n", + dev_res->tpg_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get rc_cnt:%u\n", + dev_res->rc_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get vtp_cnt:%u\n", + dev_res->vtp_cnt); + udma_info(udma_dev->dev, UDMA_M_DFX, "get utp_cnt:%u\n", + dev_res->utp_cnt); +} + +static void udma_dfx_query_dev_status(struct udma_device *udma_dev, + struct ub_dev_res *ub_dev_res) +{ + struct ubcore_device_status status = { 0 }; + int ret; + int i; + + ub_dev_res->port_cnt = udma_dev->ub_cap.net_dev_cap.port_cnt; + ret = udma_query_device_status(&(udma_dev->ub_dev), &status); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_dev_host failed.\n"); + return; + } + for (i = 0; i < UB_MAX_PORT_CNT; i++) { + ub_dev_res->port_status[i].state = + (ub_port_state_t)status.port_status[i].state; + ub_dev_res->port_status[i].active_speed = + (ub_speed_t)status.port_status[i].active_speed; + ub_dev_res->port_status[i].active_width = + (ub_link_width_t)status.port_status[i].active_width; + ub_dev_res->port_status[i].active_mtu = + (ub_mtu_t)status.port_status[i].active_mtu; + } +} + +static int udma_dfx_query_dev_res(struct udma_device *udma_dev, + struct ub_dev_res *ub_dev_res, u32 offset) +{ + struct udma_device_ctx_res dev_ctx_res = { 0 }; + int ret; + + ret = udma_query_res_dev(udma_dev, &dev_ctx_res); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query device resources, ret(%d).\n", ret); + return ret; + } + + udma_dfx_query_dev_member_cnt(udma_dev, ub_dev_res, &dev_ctx_res); + copy_dev_res_to_dfx(ub_dev_res, offset, &dev_ctx_res); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_dev_res success.\n"); + udma_free_dev_ctx_res(&dev_ctx_res); + + return 0; +} + +static int udma_dfx_query_dev_host(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_dev_res *ub_dev_res = &outbuf->dev_res; + u32 offset = inbuf->offset; + int ret; + + udma_dfx_query_dev_status(udma_dev, ub_dev_res); + + ret = udma_dfx_query_dev_res(udma_dev, ub_dev_res, offset); + + return ret; +} + +static int udma_dfx_query_dscp(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + int ret; + + if (inbuf->dscp < UDMA_DSCP_MAX_NUM && inbuf->cos <= UDMA_COS_MAX) { + ret = udma_config_dscp_vl(&udma_dev->ub_dev, (u8 *)&inbuf->dscp, + (u8 *)&inbuf->cos, 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "config dscp cos failed.\n"); + return ret; + } + } + + ret = udma_get_dscp_cos(udma_dev, outbuf->dscp_dev, UDMA_DSCP_MAX_NUM, + 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "query dscp dev failed.\n"); + return ret; + } + ret = udma_get_dscp_cos(udma_dev, outbuf->dscp_port, UDMA_DSCP_MAX_NUM, + 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "query dscp port failed.\n"); + return ret; + } + return 0; +} + +static int udma_dfx_query_seg_mpt(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_seg_mpt_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_seg_mpt_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int +udma_dfx_query_jfc_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jfc_ctx_query *jfc_ctx = &outbuf->jfc_ctx; + u32 jfcid = inbuf->jfcid; + int ret; + + (void)memset_s(jfc_ctx, sizeof(struct ub_jfc_ctx_query), 0, + sizeof(struct ub_jfc_ctx_query)); + + ret = udma_cmd_jfc_query_ctx(udev, jfcid, jfc_ctx); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query jfc_ctx from cache! jfcid:(%#x)", + jfcid); + return ret; + } + + return 0; +} + +static int udma_dfx_query_jfc_ctx_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_jfc_ctx_query *jfc_ctx = &outbuf->jfc_ctx; + struct udma_jfc *ub_jfc = NULL; + u32 jfcid = inbuf->jfcid; + + ub_jfc = udma_find_jfc(udev, jfcid); + if (ub_jfc == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, jfcid = %u", jfcid); + return -EINVAL; + } + + memcpy_fromio(jfc_ctx, ub_jfc->cqm_jfc->q_ctx_vaddr, + sizeof(struct ub_jfc_ctx_query)); + udma_jfc_put(ub_jfc); + return 0; +} + +static int udma_dfx_query_jfc_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jfc_ctx_from_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jfc_ctx_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_upi(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + ub_cmd_upi_value_attr_s upi_value = outbuf->upi_value; + ub_cmd_upi_key_attr_s upi_index = inbuf->upi_key; + int ret; + + ret = udma_cmd_upi_query(udev, &upi_index, &upi_value); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, "Failed to query upi, ret:%d", + ret); + return ret; + } + (void)memcpy_s(&outbuf->upi_value, sizeof(ub_cmd_upi_value_attr_s), + &upi_value, sizeof(ub_cmd_upi_value_attr_s)); + return 0; +} + +static int udma_dfx_query_jetty_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jetty_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jetty_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_jetty_group_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jetty_group_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jetty_group_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_dev_dip(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_dip_ctx_query dip_ctx = { 0 }; + u32 dip_index = inbuf->dipid; + int ret; + + ret = udma_query_utp(udma_dev, dip_index, &dip_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to query dip, dip_index:%u, ret:%d", dip_index, + ret); + return ret; + } + (void)memcpy_s(&outbuf->dip_ctx, sizeof(dip_ctx), &dip_ctx, + sizeof(dip_ctx)); + return 0; +} + +static void udma_dfx_query_dev_cap_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->feature = udev->ub_cap.dev_cap.feature.value; + out_cap->max_jfc_num = udev->ub_cap.dev_cap.max_jfc; + out_cap->max_jfs_num = udev->ub_cap.dev_cap.max_jfs; + out_cap->max_jfr_num = udev->ub_cap.dev_cap.max_jfr; + out_cap->max_jetty_num = udev->ub_cap.dev_cap.max_jetty; + out_cap->max_tp_num = udev->ub_cap.dev_cap.max_tp; + out_cap->max_tpg_num = + (udev->device_type == UDMA_DEV_TYPE_HI1825_V100 && + udev->ub_dev.attr.tp_maintainer) ? + udev->ub_cap.dev_cap.max_tpg - UB_TP_LATCH_RSV_TPG_NUM : + udev->ub_cap.dev_cap.max_tpg; + out_cap->max_jetty_group_num = udev->ub_cap.dev_cap.max_jetty_grp; + out_cap->max_vtp_num = udev->ub_cap.dev_cap.max_vtp; + out_cap->max_jfc_depth = udev->ub_cap.dev_cap.max_jfc_depth; + out_cap->max_jfs_depth = udev->ub_cap.dev_cap.max_jfs_depth; + out_cap->max_jfr_depth = udev->ub_cap.dev_cap.max_jfr_depth; + out_cap->max_jfs_inline_size = udev->ub_cap.dev_cap.max_jfs_inline_size; + out_cap->max_jfc_inline_size = udev->ub_cap.dev_cap.max_jfc_inline_size; + out_cap->max_jfs_sge = udev->ub_cap.dev_cap.max_jfs_sge; + out_cap->max_jfs_rsge = udev->ub_cap.dev_cap.max_jfs_rsge; + out_cap->max_jfr_sge = udev->ub_cap.dev_cap.max_jfr_sge; + out_cap->max_dma_msg_sz = UB_MAX_WRITE_READ_LEN; + out_cap->max_send_msg_sz = (u32)udev->ub_cap.dev_cap.max_msg_size; + out_cap->max_rc_outstd_cnt = + (u32)udev->ub_cap.dev_cap.max_rc_outstd_cnt; + + out_cap->utp_cnt = 0; + out_cap->max_eid_cnt = udev->ub_cap.dev_cap.max_eid_cnt; + out_cap->guid = 0; + out_cap->max_upi_cnt = udev->ub_cap.dev_cap.max_upi_cnt; + + out_cap->max_oor_cnt = udev->ub_cap.dev_cap.max_oor_cnt; + out_cap->max_sip_cnt_per_vf = udev->ub_cap.dev_cap.max_sip_cnt_per_vf; + out_cap->max_dip_cnt_per_vf = udev->ub_cap.dev_cap.max_dip_cnt_per_vf; + out_cap->max_seid_cnt_per_vf = udev->ub_cap.dev_cap.max_seid_cnt_per_vf; + out_cap->max_jetty_cnt_per_jtg = + udev->ub_cap.dev_cap.max_jetty_in_jetty_grp; + out_cap->max_tp_cnt_per_tpg = udev->ub_cap.dev_cap.max_tp_in_tpg; + + out_cap->seid_idx_start = udev->ub_cap.dev_cap.seid_idx_start; + out_cap->max_rc_num = UB_MAX_RC_NUM; + + out_cap->slice_min_size = UB_SLICE_NUM64K; + out_cap->slice_max_size = UB_SLICE_NUM128K; + out_cap->max_pi_on_chip_num = udev->ub_cap.dev_cap.max_pi_on_chip_num; + + out_cap->max_jetty_dest_ub = 0; + out_cap->device_type = udev->device_type; + out_cap->fe_idx = hinic5_global_func_id(udev->hwdev); +} + +static void udma_dfx_query_dev_cap_net_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->port_cnt = udev->ub_cap.net_dev_cap.port_cnt; + out_cap->vf_cnt = udev->ub_cap.net_dev_cap.vf_cnt; + out_cap->max_mtu = udev->ub_cap.net_dev_cap.max_mtu; + out_cap->is_tpf = udev->ub_cap.net_dev_cap.is_tpf; +} + +static void udma_dfx_query_dev_attr_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->trans_mode = udev->ub_dev.attr.dev_cap.trans_mode; + out_cap->congestion_ctrl_alg = + UB_CC_ALGO_LDCP; /* TP congestion control algorithm, 1823 and 1825 only supports UB_CC_ALGO_LDCP */ + out_cap->comp_vector_cnt = udev->ub_dev.attr.dev_cap.ceq_cnt; + out_cap->virtualization = udev->ub_dev.attr.virtualization; +} + +static void udma_dfx_query_dev_cap_res_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_query_dev_cap *out_cap = &outbuf->dev_cap; + + out_cap->max_rc_depth = udev->ub_cap.dev_cap_res.udma_res.ubrc_depth; + out_cap->rc_entry_size = + udev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + out_cap->max_mpts = udev->ub_cap.dev_cap_res.sdk_res.max_mpts; + out_cap->num_ceq_vectors = + udev->ub_cap.dev_cap_res.udma_res.num_ceq_vectors; + out_cap->cqc_entry_size = udev->ub_cap.dev_cap_res.sdk_res.cqc_entry_sz; + out_cap->qpc_entry_size = udev->ub_cap.dev_cap_res.sdk_res.qpc_entry_sz; + out_cap->srqc_entry_size = + udev->ub_cap.dev_cap_res.sdk_res.srqc_entry_sz; + out_cap->direct_wqe_size = + udev->ub_cap.dev_cap_res.udma_res.direct_wqe_size; + out_cap->log_mtt = udev->ub_cap.dev_cap_res.udma_res.log_mtt; + out_cap->num_mtts = udev->ub_cap.dev_cap_res.udma_res.num_mtts; + out_cap->log_mtt_seg = udev->ub_cap.dev_cap_res.udma_res.log_mtt_seg; + out_cap->mtt_entry_sz = udev->ub_cap.dev_cap_res.udma_res.mtt_entry_sz; + out_cap->mpt_entry_sz = udev->ub_cap.dev_cap_res.sdk_res.mpt_entry_sz; + out_cap->dmtt_cl_start = udev->ub_cap.dev_cap_res.sdk_res.dmtt_cl_start; + out_cap->dmtt_cl_end = udev->ub_cap.dev_cap_res.sdk_res.dmtt_cl_end; + out_cap->dmtt_cl_sz = udev->ub_cap.dev_cap_res.sdk_res.dmtt_cl_sz; + out_cap->ubrc_entry_sz = + udev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + out_cap->log_ubrc_seg = udev->ub_cap.dev_cap_res.udma_res.log_ubrc_seg; + out_cap->cmtt_cl_start = udev->ub_cap.dev_cap_res.sdk_res.cmtt_cl_start; + out_cap->cmtt_cl_end = udev->ub_cap.dev_cap_res.sdk_res.cmtt_cl_end; + out_cap->cmtt_cl_sz = udev->ub_cap.dev_cap_res.sdk_res.cmtt_cl_sz; + out_cap->wqe_cl_start = udev->ub_cap.dev_cap_res.sdk_res.wqe_cl_start; + out_cap->wqe_cl_end = udev->ub_cap.dev_cap_res.sdk_res.wqe_cl_end; + out_cap->wqe_cl_sz = udev->ub_cap.dev_cap_res.sdk_res.wqe_cl_sz; +} + +static int udma_dfx_query_dev_cap(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + udma_dfx_query_dev_cap_func(udma_dev, inbuf, outbuf); + udma_dfx_query_dev_cap_net_func(udma_dev, inbuf, outbuf); + udma_dfx_query_dev_attr_func(udma_dev, inbuf, outbuf); + udma_dfx_query_dev_cap_res_func(udma_dev, inbuf, outbuf); + + return 0; +} + +static int +udma_dfx_query_sec_addr_pcie_hole(struct udma_device *udev, + struct ub_secure_addr_info *secure_addr_info) +{ + struct vmsec_pcie_hole_cmd cmd = { 0 }; + u16 out_size = sizeof(cmd); + int ret; + + cmd.vfid = hinic5_global_func_id(udev->hwdev); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_MIGRATE, + MIG_MPU_CMD_PCIE_HOLE_QUERY, &cmd, + sizeof(cmd), &cmd, &out_size, 0, + HINIC5_CHANNEL_MIG); + if ((ret != 0) || (out_size == 0) || (cmd.head.status != 0)) { + udma_err( + udev->dev, UDMA_M_DFX, + "query pcie hole failed, err: %d, status: 0x%x, out_size: 0x%x", + ret, cmd.head.status, out_size); + return -EFAULT; + } + + secure_addr_info->pcie_hole.pcie_hole0_addr_hi = + cmd.pcie_hole.pcie_hole0_addr_hi; + secure_addr_info->pcie_hole.pcie_hole0_addr_lo = + cmd.pcie_hole.pcie_hole0_addr_lo; + secure_addr_info->pcie_hole.pcie_hole0_len = + cmd.pcie_hole.pcie_hole0_len; + secure_addr_info->pcie_hole.pcie_hole1_addr_hi = + cmd.pcie_hole.pcie_hole1_addr_hi; + secure_addr_info->pcie_hole.pcie_hole1_addr_lo = + cmd.pcie_hole.pcie_hole1_addr_lo; + secure_addr_info->pcie_hole.pcie_hole1_len = + cmd.pcie_hole.pcie_hole1_len; + secure_addr_info->pcie_hole.domain_id = cmd.pcie_hole.domain_id; + + return 0; +} + +static int +udma_dfx_query_sec_addr_ctx_gpa(struct udma_device *udev, + struct ub_secure_addr_info *secure_addr_info) +{ + struct vmsec_ctx_gpa_cmd cmd = { 0 }; + u16 out_size = sizeof(cmd); + int ret; + + cmd.vfid = hinic5_global_func_id(udev->hwdev); + + ret = hinic5_msg_to_mgmt_sync(udev->hwdev, HINIC5_MOD_MIGRATE, + MIG_MPU_CMD_CTX_GPA_QUERY, &cmd, + sizeof(cmd), &cmd, &out_size, 0, + HINIC5_CHANNEL_MIG); + if ((ret != 0) || (out_size == 0) || (cmd.head.status != 0)) { + udma_err( + udev->dev, UDMA_M_DFX, + "query ctx gpa failed, err: %d, status: 0x%x, out_size: 0x%x", + ret, cmd.head.status, out_size); + return -EFAULT; + } + + secure_addr_info->ctx_gpa.ctx_gpa_hi = cmd.ctx_gpa.ctx_gpa_hi; + secure_addr_info->ctx_gpa.ctx_gpa_lo = cmd.ctx_gpa.ctx_gpa_lo; + secure_addr_info->ctx_gpa.ctx_len = cmd.ctx_gpa.ctx_len; + secure_addr_info->ctx_gpa.ctx_gpa_mode = cmd.ctx_gpa.ctx_gpa_mode; + + return 0; +} + +static int udma_dfx_query_sec_addr_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_secure_addr_info *secure_addr_info = &outbuf->sec_addr_info; + int ret; + + ret = (udma_dfx_query_sec_addr_ctx_gpa(udma_dev, secure_addr_info) != + 0) || + (udma_dfx_query_sec_addr_pcie_hole(udma_dev, secure_addr_info) != + 0); + + return ret; +} + +static int +udma_dfx_query_jfs_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfs_ctx = &outbuf->jfs_ctx; + struct udma_jfs *jfs = NULL; + u32 jfsid = inbuf->jfsid; + int ret; + + (void)memset_s(jfs_ctx, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + jfs = udma_find_jfs(udev, jfsid); + if (jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jfs, jfsid(%u)\n", jfsid); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udev, jfsid, &jfs->cmdq_rtt, jfs_ctx); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query jfs_ctx from cache! jfsid:(%#x)", + jfsid); + return ret; + } + + udma_jfs_put(jfs); + return 0; +} + +static int udma_dfx_query_jfs_ctx_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfs_ctx = &outbuf->jfs_ctx; + struct udma_jfs *ub_jfs = NULL; + u32 jfsid = inbuf->jfsid; + + ub_jfs = udma_find_jfs(udev, jfsid); + if (ub_jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfsid = %u", jfsid); + return -EINVAL; + } + + memcpy_fromio(jfs_ctx, ub_jfs->jfsc_info->vaddr, + sizeof(struct ub_ta_context_query)); + udma_jfs_put(ub_jfs); + return 0; +} + +static int udma_dfx_query_jfs_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jfs_ctx_from_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jfs_ctx_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_jetty_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udev, inbuf->jettyid); + if (ub_jetty == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jetty, jetty_id = %u", inbuf->jettyid); + return -EINVAL; + } + + ret = udma_dfx_query_cmtt(udev, &ub_jetty->ub_mtt, outbuf); + udma_jetty_put(ub_jetty); + return ret; +} + +static int udma_dfx_query_jfs_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udev, inbuf->jfsid); + if (ub_jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jfs, jfs_id = %u", inbuf->jfsid); + return -EINVAL; + } + + ret = udma_dfx_query_cmtt(udev, &ub_jfs->ub_mtt, outbuf); + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_query_jfr_cmtt(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct udma_jfr *udma_jfr = NULL; + int ret = 0; + + udma_jfr = udma_find_jfr(udev, inbuf->jfrid); + if (udma_jfr == NULL) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_jfr_cmtt_func, ubcore find jfr failed, jfr_id(%u)", + inbuf->jfrid); + return -EINVAL; + } + + ret = udma_dfx_query_cmtt(udev, &udma_jfr->srq.ub_mtt, outbuf); + udma_jfr_put(udma_jfr); + return ret; +} + +static int +udma_dfx_query_jfr_ctx_from_host_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfr_entry = &outbuf->jetty_ctx; + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "query jfr from host failed!, jfrid(%u)", jfrid); + return -EINVAL; + } + + memcpy_fromio(jfr_entry, udma_jfr->jfrc_info->vaddr, + sizeof(struct ub_ta_context_query)); + udma_jfr_put(udma_jfr); + return 0; +} + +static int +udma_dfx_query_jfr_ctx_from_cache_func(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_ta_context_query *jfr_entry = &outbuf->jetty_ctx; + struct udma_jfr *jfr = NULL; + u32 jfrid = inbuf->jfrid; + int ret; + + (void)memset_s(jfr_entry, sizeof(struct ub_ta_context_query), 0, + sizeof(struct ub_ta_context_query)); + + jfr = udma_find_jfr(udev, jfrid); + if (jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get jfr, jfrid(%u)\n", jfrid); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udev, jfrid, &jfr->cmdq_rtt, jfr_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "query jfr from cache failed!! jfrid(%u)", jfrid); + return ret; + } + + udma_jfr_put(jfr); + return 0; +} + +static int +udma_dfx_query_index_queue_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + if (!udma_get_index_queue_en(udma_dev)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "device type(%d) does not support the command for query index wqe.", + udma_dev->device_type); + return -EINVAL; + } + + /* TODO: later need to consider ICSL submission for invalid xid queue query of intercept */ + return 0; +} + +static int udma_dfx_query_jfr_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_jfr_ctx_from_cache_func(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_jfr_ctx_from_host_func(udma_dev, buf_in, + buf_out); + } +} + +static int udma_dfx_query_srq_ctx_from_host(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_cmd_srq_query_rsp_s *srq_entry = &outbuf->srq_ctx; + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "query srq from host failed!, jfrid(%u)", jfrid); + return -EINVAL; + } + + memcpy_fromio(srq_entry, udma_jfr->srq.srqc_info->q_ctx_vaddr, + sizeof(ub_cmd_srq_query_rsp_s)); + udma_jfr_put(udma_jfr); + return 0; +} + +static int udma_dfx_query_srq_ctx_from_cache(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_cmd_srq_query_rsp_s *srq_entry = &outbuf->srq_ctx; + struct udma_jfr *udma_jfr = NULL; + u32 jfrid = inbuf->jfrid; + int ret; + + udma_jfr = udma_find_jfr(udev, jfrid); + if (udma_jfr == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "query jfr from host failed!, jfrid(%u)", jfrid); + return -EINVAL; + } + + (void)memset_s(srq_entry, sizeof(ub_cmd_srq_query_rsp_s), 0, + sizeof(ub_cmd_srq_query_rsp_s)); + + ret = udma_cmd_srq_query_ctx(udev, udma_jfr->srq.srqn, srq_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "query srq from cache failed!! jfrid(%u)", jfrid); + udma_jfr_put(udma_jfr); + return ret; + } + udma_jfr_put(udma_jfr); + return 0; +} + +static int udma_dfx_query_srq_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_srq_ctx_from_cache(udma_dev, buf_in, + buf_out); + } else { + return udma_dfx_query_srq_ctx_from_host(udma_dev, buf_in, + buf_out); + } +} + +static int +udma_dfx_query_rc_queue_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + u32 xid = check_args->xid; + bool tpf = check_args->tpf; + struct udma_rc_queue *rcq = NULL; + + if (tpf) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Should not query rcq in TPF"); + return -EINVAL; + } + + rcq = udma_find_rcq(udma_dev, xid); + if (rcq == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get rcq, xid = %u", xid); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_query_rc_queue_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_rc_queue_ctx_from_cache_func( + udma_dev, buf_in, buf_out); + } else { + return udma_dfx_query_rc_queue_ctx_from_host_func( + udma_dev, buf_in, buf_out); + } +} + +static int udma_dfx_query_rc_queue_cmtt(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + u32 rc_id = buf_in->rcq_id; + struct tag_cqm_object *cqm_obj = NULL; + struct udma_rc_queue *udma_rcq = NULL; + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + rc_id, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + rcq_ctx = container_of(cqm_obj, struct tag_cqm_qpc_mpt, object); + udma_rcq = (struct udma_rc_queue *)rcq_ctx->priv; + if (udma_rcq->cqm_priv_type != UDMA_JETTY_TYPE_RC) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "cqmobj is not rc, rc_id(%u), type(%u)\n", rc_id, + udma_rcq->cqm_priv_type); + ret = -EINVAL; + goto cleanup; + } + + ret = udma_dfx_query_cmtt(udma_dev, &udma_rcq->ub_mtt, buf_out); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +static int udma_dfx_query_rc_queue_pcqc_ctx(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_rc_queue_pcqc_from_cache_func( + udma_dev, buf_in, buf_out); + } else { + return udma_dfx_query_rc_queue_pcqc_from_host_func( + udma_dev, buf_in, buf_out); + } +} + +static int udma_dfx_query_jfs_sqe(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct tag_ub_sq_wqe_s *jfs_sqe = &outbuf->jfs_sqe; + struct udma_jfs *ub_jfs = NULL; + u32 jfsid = inbuf->jfsid; + u32 wqe_offset; + int ret = 0; + + if (inbuf->wqe_id > (UINT_MAX / UB_SQ_WRITE_WQEBB_LEN)) { + udma_err(udev->dev, UDMA_M_DFX, + "inbuf->wqe_id %u too large may cause looping", + inbuf->wqe_id); + return -EINVAL; + } + + ub_jfs = udma_find_jfs(udev, jfsid); + if (ub_jfs == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfsid = %u", jfsid); + return -EINVAL; + } + + ub_jfs->sq_buf_len = (u32)ub_jfs->umem->length; + if (ub_jfs->sq_buf_len == 0) { + udma_err(udev->dev, UDMA_M_DFX, "JFS sq buf length is 0"); + ret = -EINVAL; + goto OUT; + } + + wqe_offset = + inbuf->wqe_id * UB_SQ_WRITE_WQEBB_LEN % ub_jfs->umem->length; + + ret = copy_data_from_page(udev, (u8 *)jfs_sqe, ub_jfs->umem, + UB_MAX_JFS_DFX_BUF, wqe_offset); +OUT: + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_query_rc_queue_wqe(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type) +{ + if (source_type == UB_QUERY_FROM_CACHE) { + return udma_dfx_query_rc_queue_wqe_from_cache_func( + udma_dev, buf_in, buf_out); + } else { + return udma_dfx_query_rc_queue_wqe_from_host_func( + udma_dev, buf_in, buf_out); + } +} + +static int udma_dfx_query_seid_ctx_from_cache( + struct udma_device *udma_dev, const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_seid_ctx_query *seid_ctx = &outbuf->seid_ctx; + int ret; + + ret = udma_cmd_seid_query_ctx_from_cache( + udma_dev, (u16)inbuf->vfid, inbuf->seid_id, seid_ctx, + sizeof(struct ub_seid_ctx_query)); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DFX, + "query seid ctx failed, ret=%d.\n", ret); + + return ret; +} + +static int udma_dfx_query_cc(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + struct ub_cc_params *para = &outbuf->cc_params; + int ret; + + para->index = (u8)inbuf->pattern_index; + para->ub_cc_algo = (u8)inbuf->cc_alg; + ret = udma_cc_query_para(udma_dev, para); + + return ret; +} + +static int udma_dfx_query_func_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_cmd_func_resp *para = &outbuf->func_info; + struct ub_cmd_get_cfg_info get_cfg_info = { 0 }; + u16 resp_size = sizeof(struct ub_cmd_func_resp); + int ret; + + get_cfg_info.func_id = (u16)inbuf->vfid; + para->func_id = (u16)inbuf->vfid; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_FUNC_ATTR, &get_cfg_info, + sizeof(get_cfg_info), para, &resp_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (get_cfg_info.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "failed to get func info, ret:%d, status:0x%x, out size:0x%x", + ret, get_cfg_info.head.status, resp_size); + return -EIO; + } + + return 0; +} + +static int +udma_dfx_query_func_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + bool tpf = check_args->tpf; + + if (!tpf) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Should query func filter in TPF"); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_query_func_filter_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_cmd_get_ub_filter_info_resp *para = &outbuf->func_filter; + struct ub_cmd_get_cfg_info get_cfg_info = { 0 }; + u16 resp_size = sizeof(struct ub_cmd_get_ub_filter_info_resp); + int ret; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_GET_FUNC_FILTER, + &get_cfg_info, sizeof(get_cfg_info), para, + &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (get_cfg_info.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "failed to get func filter info, ret:%d, status:0x%x, out size:0x%x", + ret, get_cfg_info.head.status, resp_size); + return -EIO; + } + + return 0; +} + +static int udma_dfx_query_mig_counter(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + return udma_cmd_mig_query_counter(udma_dev, inbuf->vfid, + &outbuf->mig_ctr); +} + +static int udma_dfx_query_ta_rdma_rc(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_ta_rdma_rc_outbuf *ta_rdma_rc = &outbuf->ta_rdma_rc; + int ret; + + ret = udma_cmd_jetty_query_rdma_rc(udma_dev, inbuf->jettyid, + inbuf->entry_index, ta_rdma_rc); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_rdma_rc_exec failed, ret=%d!\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_ta_msn_entry(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_msn_entry_outbuf *ta_msn_entry = &outbuf->msn_entry_info; + int ret; + + ret = udma_cmd_jetty_query_ta_msn_entry(udma_dev, inbuf->xid, + inbuf->entry_index, + inbuf->msn_type, ta_msn_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_msn_entry failed, ret=%d!\n", ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_ta_oor_info(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_oor_ta_info *oor_ta_info = &outbuf->oor_ta_info; + int ret; + + ret = udma_cmd_jetty_query_ta_oor_info(udma_dev, inbuf->jettyid, + inbuf->idx, oor_ta_info); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_oor_info_exec failed, ret=%d!\n", + ret); + return ret; + } + udma_cpu_to_be32(&oor_ta_info->start_tassn, + sizeof(oor_ta_info->start_tassn)); + udma_cpu_to_be32(&oor_ta_info->start_bit, + sizeof(oor_ta_info->start_bit)); + udma_cpu_to_be32(&oor_ta_info->end_bit, sizeof(oor_ta_info->end_bit)); + udma_cpu_to_be32(&oor_ta_info->total_bit_num, + sizeof(oor_ta_info->total_bit_num)); + udma_cpu_to_be32(&oor_ta_info->acked_bit, + sizeof(oor_ta_info->acked_bit)); + udma_cpu_to_be32(&oor_ta_info->tamsnc_pi, + sizeof(oor_ta_info->tamsnc_pi)); + udma_cpu_to_be32(&oor_ta_info->tamsnc_ci, + sizeof(oor_ta_info->tamsnc_ci)); + udma_cpu_to_be32(&oor_ta_info->next_ack_tassn, + sizeof(oor_ta_info->next_ack_tassn)); + udma_cpu_to_be32(&oor_ta_info->next_send_tassn, + sizeof(oor_ta_info->next_send_tassn)); + oor_ta_info->bitmap.rc_ext_table_size = + udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100 ? + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1825 : + UB_TA_RDMARC_EXT_TABLE_SIZE_HI1823; + + return 0; +} + +static int udma_dfx_query_ta_rdma_rc_ext(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + struct ub_ta_rdma_rc_ext_outbuf *ta_rdma_rc_ext = + &outbuf->ta_rdma_rc_ext; + int ret; + + ret = udma_cmd_jetty_query_rdma_rc_ext(udma_dev, inbuf->jettyid, + inbuf->idx, ta_rdma_rc_ext); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_ta_rdma_rc_ext_exec failed, ret=%d!\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_driver_ctr(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type) +{ + ub_driver_comm_ctr_t *ub_driver_ctr = &outbuf->ub_driver_comm_ctr; + u32 cmd_type; + int ret = 0; + cmd_type = inbuf->cmd_type >> UB_QUERY_TYPE_BIT; + + switch (cmd_type) { + case UB_QUERY_TYPE_JFS: + ret = udma_cmd_jfs_query_driver_ctr(udma_dev, inbuf->jfsid, + ub_driver_ctr); + break; + case UB_QUERY_TYPE_JETTY: + ret = udma_cmd_jetty_query_driver_ctr(udma_dev, inbuf->jettyid, + ub_driver_ctr); + break; + case UB_QUERY_TYPE_DEV: + ub_driver_ctr->post_send_wr_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.post_send_wr_num); + ub_driver_ctr->post_recv_wr_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.post_recv_wr_num); + ub_driver_ctr->poll_total_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.poll_total_num); + ub_driver_ctr->poll_sq_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.poll_sq_num); + ub_driver_ctr->poll_rq_num = (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr.poll_rq_num); + break; + case UB_QUERY_TYPE_JFC: + ret = udma_cmd_jfc_query_driver_ctr(udma_dev, inbuf->jfcid, + ub_driver_ctr); + break; + case UB_QUERY_TYPE_JFR: + ret = udma_cmd_jfr_query_driver_ctr(udma_dev, inbuf->jfrid, + ub_driver_ctr); + break; + default: + udma_err(udma_dev->dev, UDMA_M_DFX, "err cmd_type=%d!\n", + cmd_type); + return -EINVAL; + } + + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_driver_ctr failed!, ret=%d\n", ret); + } + + return ret; +} + +/* TODO: later need to consider ICSL submission for invalid xid queue query of intercept */ +static struct udma_dfx_query_cmd_cb g_udma_query_cmd_funcs[] = { + /* query segment */ + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_seg_mpt }, + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_seg_dmtt }, + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_SAFE_DMTT_BITMAP, NULL, + udma_dfx_query_seg_safe_dmtt_bitmap }, + { UB_QUERY_TYPE_SEG, UB_QUERY_SUB_TYPE_SAFE_DMTT_CTX, NULL, + udma_dfx_query_seg_safe_dmtt_ctx }, + + /* query jfc */ + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jfc_ctx }, + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jfc_cmtt }, + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jfc_cqe }, + { UB_QUERY_TYPE_JFC, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + + /* query jetty */ + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jetty_ctx }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jetty_cmtt }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jetty_sqe }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_RDMARC, NULL, + udma_dfx_query_ta_rdma_rc }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_RDMARC_EXT, NULL, + udma_dfx_query_ta_rdma_rc_ext }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_TA_OOR_INFO, NULL, + udma_dfx_query_ta_oor_info }, + { UB_QUERY_TYPE_JETTY_GROUP, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jetty_group_ctx }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + { UB_QUERY_TYPE_JETTY, UB_QUERY_SUB_TYPE_MSN_ENTRY, NULL, + udma_dfx_query_ta_msn_entry }, + + /* query jfs */ + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jfs_ctx }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jfs_cmtt }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jfs_sqe }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + { UB_QUERY_TYPE_JFS, UB_QUERY_SUB_TYPE_MSN_ENTRY, NULL, + udma_dfx_query_ta_msn_entry }, + + /* query jfr */ + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_jfr_ctx }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_MTT, NULL, + udma_dfx_query_jfr_cmtt }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_WQE, NULL, + udma_dfx_query_jfr_rqe }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_SRQ, NULL, + udma_dfx_query_srq_ctx }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_IDX_WQE, + udma_dfx_query_index_queue_check, udma_dfx_query_idx_wqe }, + { UB_QUERY_TYPE_JFR, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + + /* query table config */ + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_dev_host }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_SIP, NULL, udma_dfx_query_sip }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_DIP, NULL, + udma_dfx_query_dev_dip }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_CAP, NULL, + udma_dfx_query_dev_cap }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_SECURE_MEM_INFO, NULL, + udma_dfx_query_sec_addr_info }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_DSCP, NULL, + udma_dfx_query_dscp }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_IO_STAT, NULL, + udma_dfx_query_driver_ctr }, + { UB_QUERY_TYPE_UPI, UB_QUERY_SUB_TYPE_INFO, NULL, udma_dfx_query_upi }, + + /* query rc queue */ + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_CTX, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_ctx }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_MTT, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_cmtt }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_WQE, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_wqe }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_PCQC, + udma_dfx_query_rc_queue_check, udma_dfx_query_rc_queue_pcqc_ctx }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_RDMARC, + udma_dfx_query_rc_queue_check, udma_dfx_query_ta_rdma_rc }, + { UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_SUB_TYPE_RDMARC_EXT, + udma_dfx_query_rc_queue_check, udma_dfx_query_ta_rdma_rc_ext }, + + /* query misc */ + { UB_QUERY_TYPE_SEID, UB_QUERY_SUB_TYPE_CTX, NULL, + udma_dfx_query_seid_ctx_from_cache }, + { UB_QUERY_TYPE_CC, UB_QUERY_SUB_TYPE_INVALID, NULL, + udma_dfx_query_cc }, + { UB_QUERY_TYPE_MIG, UB_QUERY_SUB_TYPE_FUNC_INFO, NULL, + udma_dfx_query_func_info }, + { UB_QUERY_TYPE_MIG, UB_QUERY_SUB_TYPE_MIG_COUNTER_INFO, NULL, + udma_dfx_query_mig_counter }, + { UB_QUERY_TYPE_DEV, UB_QUERY_SUB_TYPE_FUNC_INFO, + udma_dfx_query_func_check, udma_dfx_query_func_filter_info }, + + /* query ta latch info */ + { UB_QUERY_TYPE_LATCH, UB_QUERY_SUB_TYPE_TA_LATCH, + udma_dfx_query_ta_latch_check, udma_dfx_query_ta_latch_data } +}; + +static inline int udma_dfx_query_args_size_check(struct udma_device *udma_dev, + u32 out_size) +{ + if (out_size < (u32)sizeof(union ub_query_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query buffer out size too small\n"); + return -EINVAL; + } + + return 0; +} + +static inline int udma_dfx_query_outbuf_clear(struct udma_device *udma_dev, + void *buf_out, u32 *out_size) +{ + int ret = 0; + + ret = memset_s(buf_out, *out_size, 0, + (u32)sizeof(union ub_query_outbuf)); + if (ret != EOK) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query out buf memset error ret:%d\n", ret); + return -EINVAL; + } + + *out_size = (u32)sizeof(union ub_query_outbuf); + return 0; +} + +static inline int +udma_dfx_query_subcmd_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + udma_check_func_t check_func) +{ + struct udma_dfx_check_args check_args = { 0 }; + + /* check: 1.xid;2.ta/tp func;3.debug/release version(to add in subcmd) */ + check_args.xid = inbuf->xid; + check_args.tpf = udma_dev->ub_dev.attr.tp_maintainer; + + if (check_func == NULL) { + udma_info(udma_dev->dev, UDMA_M_DFX, + "Subcmd(%u) do not need validity-check.\n", + inbuf->cmd_type); + return 0; + } + + if (check_func(udma_dev, inbuf, &check_args) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Check func not null and check failed.\n"); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_query_ta_func(struct udma_device *udma_dev, + const void *buf_in, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + udma_query_func_t cmd_func = NULL; + udma_check_func_t check_func = NULL; + u32 cmd_type; + u32 i; + + /* check in buf size & out buf size, memset out buf */ + if (udma_dfx_query_args_size_check(udma_dev, *out_size) != 0 || + udma_dfx_query_outbuf_clear(udma_dev, buf_out, out_size) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query buf check or clear failed.\n"); + return -EINVAL; + } + + /* look for sub cmd and exec */ + for (i = 0; i < sizeof(g_udma_query_cmd_funcs) / + sizeof(g_udma_query_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_query_cmd_funcs[i].cmd_type, + g_udma_query_cmd_funcs[i].sub_type, 0); + if (cmd_type == + (inbuf->cmd_type & UB_DFX_QUERY_SOURCE_TYPE_SHIELD_MASK)) { + check_func = g_udma_query_cmd_funcs[i].check_func; + cmd_func = g_udma_query_cmd_funcs[i].cmd_func; + + if (cmd_func == NULL) { + udma_warn( + udma_dev->dev, UDMA_M_DFX, + "Subcmd(%u) not support yet, check cmd_type!\n", + inbuf->cmd_type); + return -EINVAL; + } + + if (udma_dfx_query_subcmd_check(udma_dev, inbuf, + check_func) != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Check func not null and check failed.\n"); + return -EINVAL; + } + return cmd_func(udma_dev, inbuf, outbuf, + inbuf->cmd_type & + UB_DFX_QUERY_SOURCE_TYPE_BIT); + } + } + + udma_err(udma_dev->dev, UDMA_M_DFX, + "Subcmd(%u) not found, check cmd_type!\n", inbuf->cmd_type); + return -EINVAL; +} + +int udma_query_npu_dfx_version(struct udma_device *udma_dev, int *npu_version) +{ + struct comm_cmd_get_fw_version fw_npu_version; + u16 out_size = sizeof(fw_npu_version); + u16 channel = HINIC5_CHANNEL_UB; + int ret; + + if (udma_dev == NULL || udma_dev->hwdev == NULL) { + return -EINVAL; + } + + (void)memset_s(&fw_npu_version, sizeof(fw_npu_version), 0, + sizeof(fw_npu_version)); + fw_npu_version.fw_type = HINIC5_FW_VER_TYPE_NPU; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_COMM, + COMM_MGMT_CMD_GET_FW_VERSION, + &fw_npu_version, sizeof(fw_npu_version), + &fw_npu_version, &out_size, 0, channel); + if ((ret != 0) || (out_size == 0) || + (fw_npu_version.head.status) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "fail to query npu version, ret:%d, status:0x%x\n", + ret, fw_npu_version.head.status); + return -EIO; + } + *npu_version = fw_npu_version.fw_dfx_vld; + return ret; +} + +int udma_verify_dfx_version(struct udma_device *udma_dev, u32 cmd) +{ + int npu_version = -1; + int ret; + + ret = udma_query_npu_dfx_version(udma_dev, &npu_version); + if (ret < 0) { + udma_pr_err(UDMA_M_DFX, "UDMA fail to query version.\n"); + return ret; + } + + if (npu_version == 0 && (cmd == UB_DFX_PACKET || cmd == UB_DFX_ATTACK || + cmd == UB_DFX_CQM_MODIFY)) { + udma_pr_err( + UDMA_M_DFX, + "Not support this cmd type (%u) for release version", + cmd); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_query_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_query_inbuf *query_inbuf = NULL; + u32 cmd_type; + int ret; + + // check in buf size + if (in_size < (u32)sizeof(struct ub_query_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx query buffer in size too small\n"); + return -EINVAL; + } + query_inbuf = (struct ub_query_inbuf *)buf_in; + cmd_type = query_inbuf->cmd_type >> UB_QUERY_TYPE_BIT; + + if (cmd_type == UB_QUERY_TYPE_TP || cmd_type == UB_QUERY_TYPE_TPG || + cmd_type == UB_QUERY_TYPE_VTP) + ret = udma_dfx_query_tp_func(udma_dev, buf_in, in_size, buf_out, + out_size); + else + ret = udma_dfx_query_ta_func(udma_dev, buf_in, buf_out, + out_size); + + return ret; +} + +static udma_ioctl_func_t g_udma_ioctl_funcs[VM_COMPAT_TEST] = { + [UB_DFX_QUERY] = udma_dfx_query_func, + [UB_DFX_MODIFY] = udma_dfx_modify_func, + [UB_DFX_PACKET] = udma_dfx_packet_func, + [UB_DFX_PORT_STA] = udma_dfx_port_sta_func, + [UB_DFX_ATTACK] = udma_dfx_attack_func, + [UB_DFX_CQM_MODIFY] = udma_dfx_cqm_modify_func, + [UB_DFX_PERF_CTR] = udma_dfx_perf_ctr_func, + [UB_DFX_L2D_MEM] = udma_dfx_l2d_func, + [UB_DFX_PERF_QUERY] = udma_dfx_perf_query_func, +}; + +int udma_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct udma_device *udma_dev = (struct udma_device *)uld_dev; + udma_ioctl_func_t func = NULL; + int ret; + + /* check whether is root user */ + if (!capable(CAP_SYS_ADMIN)) { + udma_pr_err(UDMA_M_DFX, "Invalid permission."); + return -EPERM; + } + + /* ppf device on perform version number get */ + if (cmd == GET_DRV_VERSION) { + return udma_dfx_get_drv_version(buf_in, in_size, buf_out, + out_size); + } + + if (uld_dev == NULL || buf_in == NULL || buf_out == NULL || + out_size == NULL) { + udma_pr_err(UDMA_M_DFX, "UDMA ioctl. input params is null\n"); + return -EINVAL; + } + + if (cmd >= VM_COMPAT_TEST) { + udma_pr_err(UDMA_M_DFX, "Not support this cmd type(%u)", cmd); + return -EINVAL; + } + + if (!hinic5_support_ub(udma_dev->hwdev, NULL)) { + udma_pr_err(UDMA_M_DFX, "UDMA ioctl. Hw don't support UB\n"); + return -EINVAL; + } + + ret = udma_verify_dfx_version(udma_dev, cmd); + if (ret != 0) { + udma_pr_err(UDMA_M_DFX, "UDMA dfx version verify failed.\n"); + return ret; + } + + func = g_udma_ioctl_funcs[cmd]; + if (func == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Not support this cmd type(%u)", cmd); + return -EINVAL; + } + + return func(udma_dev, buf_in, in_size, buf_out, out_size); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h new file mode 100644 index 000000000..6b9aa40bc --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx.h @@ -0,0 +1,71 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_DFX_H +#define UDMA_DFX_H + +#include "udma_common.h" +#include "ub_pub_cmd.h" + +int udma_ioctl(void *uld_dev, u32 cmd, const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); + +#define UDMA_MTT_MAX_ORDER 27 + +#define UDMA_DFX_GET_RES_CNT(member, list_head, member_name, cnt) \ + do { \ + (cnt) = 0; \ + list_for_each_entry(member, &(list_head), member_name) { \ + (cnt)++; \ + } \ + } while (0) + +struct udma_dfx_res_dev_len { + void **addr; + u32 len; +}; + +struct udma_dfx_check_args { + u32 xid; + bool dfx_fw_version; + bool tpf; +}; + +typedef int (*udma_ioctl_func_t)(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size); + +typedef int (*udma_query_func_t)(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + union ub_query_outbuf *buf_out, + u32 source_type); + +typedef int (*udma_check_func_t)(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args); + +struct udma_dfx_query_cmd_cb { + u32 cmd_type; + u32 sub_type; + udma_check_func_t check_func; + udma_query_func_t cmd_func; +}; + +struct udma_dfx_query_cmd_func { + u32 cmd_type; + u32 sub_type; + u32 source_type; + udma_ioctl_func_t cmd_func; +}; + +int udma_dfx_query_prepare(struct udma_device *udma_dev, void *buf_out, + u32 *out_size); +int udma_dfx_query_cmtt(struct udma_device *udev, struct mtt *ub_mtt, + union ub_query_outbuf *outbuf); +int udma_dfx_check_xid(struct udma_device *udev, u32 type, + struct ub_query_inbuf *query_inbuf); +int udma_query_npu_dfx_version(struct udma_device *udma_dev, int *npu_version); +int udma_verify_dfx_version(struct udma_device *udma_dev, u32 cmd); +int udma_dfx_obj_put(struct udma_device *udev, u32 type, void *obj, u32 xid); +#endif //UDMA_DFX_H diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c new file mode 100644 index 000000000..ece91c09c --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.c @@ -0,0 +1,831 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/highmem.h> + +#include "ub_pub_cmd.h" +#include "ubcore_uapi.h" +#include "ub_xqe_format.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_cmd_com.h" +#include "udma_base_sq.h" + +#include "udma_dfx_attack.h" + +u8 copy_data_by_mask(u8 inbyte, u8 mask, u8 byte) +{ + return (byte & ~mask) | (inbyte & mask); +} + +static int copy_data_to_page(struct udma_device *udev, u8 *inbuf, u8 *mask, + struct ubcore_umem *umem, u32 copy_len, u32 offset, + bool use_mask) +{ + struct scatterlist *sg = NULL; + u8 *inbuf_tmp = inbuf; + u8 *mask_tmp = mask; + u32 page_copy_len = 0; + u32 copied_len = 0; + u32 page_offset = 0; + u64 buf_offset = 0; + void *va = NULL; + u32 i = 0; + + udma_info(udev->dev, UDMA_M_DFX, + "copy_data_to_page va:%llx, length:%llu, offset:%u, \ + copy_len:%u", + umem->va, umem->length, offset, copy_len); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + if (buf_offset + PAGE_SIZE > offset) { + page_offset = offset - (u32)buf_offset; + buf_offset = offset; + va = sg_virt(sg); + if (va == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get va"); + return -ENOMEM; + } + + va += page_offset; + page_copy_len = ((copy_len - copied_len) < + (PAGE_SIZE - page_offset)) ? + (copy_len - copied_len) : + (PAGE_SIZE - page_offset); + udma_info( + udev->dev, UDMA_M_DFX, + "copy_data_from_page buf_offset:%llu, page_offset:%u, copied_len:%u, \ + page_copy_len:%u, i:%u", + buf_offset, page_offset, copied_len, + page_copy_len, i); + + if (use_mask) { + for (u32 j = 0; j < page_copy_len; j++) { + u8 inbyte = inbuf_tmp[j]; + u8 maskbyte = mask_tmp[j]; + u8 outbyte = ((u8 *)va)[j]; + ((u8 *)va)[j] = copy_data_by_mask( + inbyte, maskbyte, outbyte); + } + mask_tmp += page_copy_len; + } else { + (void)memcpy_s(va, page_copy_len, inbuf_tmp, + page_copy_len); + } + inbuf_tmp += page_copy_len; + copied_len += page_copy_len; + if (copied_len >= copy_len) { + break; + } + } else { + buf_offset += PAGE_SIZE; + } + } + return 0; +} + +static int udma_dfx_attack_db(struct udma_device *udma_dev, struct udma_db *sdb, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + union tag_ub_sq_db_u jetty_db; + + jetty_db.sq_db_value = 0; + jetty_db.sq_db.dw0.value = inbuf->attack_value[0]; + jetty_db.sq_db.dw1.value = inbuf->attack_value[1]; + jetty_db.sq_db.dw0.bs.type = inbuf->srv_type; + jetty_db.sq_db.dw1.bs.pi = (inbuf->pi_ci >> UB_SQ_DB_SHIFT) & 0xFF; + + udma_info(udma_dev->dev, UDMA_M_DFX, "type:%u, cos:%u, inbuf->pi:%u\n", + jetty_db.sq_db.dw0.bs.type, jetty_db.sq_db.dw0.bs.cos, + inbuf->pi_ci); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "c:%u, n:%u, cntx_size:%u, xqn:%u\n", jetty_db.sq_db.dw0.bs.c, + jetty_db.sq_db.dw0.bs.n, jetty_db.sq_db.dw0.bs.cntx_size, + jetty_db.sq_db.dw0.bs.xqn); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "queue_id:%u, rsvd0:%u, pi:%u jetty_db.sq_db_value:%llu\n", + jetty_db.sq_db.dw1.bs.queue_id, jetty_db.sq_db.dw1.bs.rsvd0, + jetty_db.sq_db.dw1.bs.pi, jetty_db.sq_db_value); + + outbuf->db = jetty_db.sq_db_value; + if (sdb->db_record != NULL) { + *(sdb->db_record) = cpu_to_be32(inbuf->pi_ci); + } + + wmb(); + return cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + inbuf->pi_ci & 0xff, jetty_db.sq_db_value); +} + +static int udma_dfx_attack_jetty_db(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jettyid(%u)\n", + inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_db(udma_dev, &ub_jetty->sdb, inbuf, outbuf); + + udma_jetty_put(ub_jetty); + return ret; +} + +static int udma_dfx_attack_jfs_db(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udma_dev, inbuf->object); + if (ub_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfsid(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_db(udma_dev, &ub_jfs->sdb, inbuf, outbuf); + + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_attack_jfr_db(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfr *ub_jfr = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, jfr id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_db(udma_dev, &ub_jfr->sdb, inbuf, outbuf); + + udma_jfr_put(ub_jfr); + return ret; +} + +static int udma_dfx_attack_wqe(struct udma_device *udma_dev, + struct ubcore_umem *umem, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf, bool use_mask) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + int ret = 0; + + if (umem == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, "umem is null\n"); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "length:%u, inbuf->offset:%u\n", + length, inbuf->offset); + + ret = copy_data_to_page(udma_dev, (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, umem, length, + inbuf->offset, use_mask); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to copy data to page, ret: %d\n", ret); + } + + return ret; +} + +int copy_data_for_mtt_pro_en(struct udma_device *udma_dev, u8 *inbuf, u8 *mask, + struct queue_addr_info *queue_addr, u32 len, + u32 offset, bool use_mask) +{ + void *va = queue_addr->va; + + if ((offset >= queue_addr->size) || + (offset + len >= queue_addr->size)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "invalid params, offset:%u, len:%u, queue size:%u", + offset, len, queue_addr->size); + return -EINVAL; + } + + if (use_mask) { + u8 *dest = (u8 *)va + offset; + for (u32 i = 0; i < len; i++) { + dest[i] = copy_data_by_mask(inbuf[i], mask[i], dest[i]); + } + } else { + return memcpy_s((void *)((u8 *)va + offset), len, (void *)inbuf, + len); + } + return 0; +} + +int copy_data_for_attack_tamsn(struct udma_device *udma_dev, u8 *inbuf, + struct ubrc *ub_rc, u32 len, u32 offset) +{ + void *va = ub_rc->vaddr; + + if ((offset >= UB_ATTACK_TAMSN_MAX_LEN) || + (offset + len >= UB_ATTACK_TAMSN_MAX_LEN)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "invalid params, offset:%u, len:%u, tamsn max size:%u", + offset, len, UB_ATTACK_TAMSN_MAX_LEN); + return -EINVAL; + } + + return memcpy_s((void *)((u8 *)va + offset), len, (void *)inbuf, len); +} + +static int udma_dfx_attack_jetty_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jetty id(%u)\n", + inbuf->object); + return -EINVAL; + } + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &ub_jetty->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jetty->umem, inbuf, outbuf, + true); +OUT: + udma_jetty_put(ub_jetty); + + return ret; +} + +static int udma_dfx_attack_jfs_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udma_dev, inbuf->object); + if (ub_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, jfs id(%u)\n", inbuf->object); + return -EINVAL; + } + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &ub_jfs->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfs->umem, inbuf, outbuf, true); +OUT: + udma_jfs_put(ub_jfs); + + return ret; +} + +static int udma_dfx_attack_jfr_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jfr *ub_jfr = NULL; + struct udma_srq *srq = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, jfr id(%u)\n", inbuf->object); + return -EINVAL; + } + + srq = &ub_jfr->srq; + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &srq->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfr->srq.umem, inbuf, outbuf, + true); +OUT: + udma_jfr_put(ub_jfr); + + return ret; +} + +static int udma_dfx_attack_jfc_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jfc *ub_jfc = NULL; + int ret = 0; + + ub_jfc = udma_find_jfc(udma_dev, inbuf->object); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, jfc id(%u)\n", inbuf->object); + return -EINVAL; + } + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + ret = copy_data_for_mtt_pro_en(udma_dev, + (u8 *)inbuf->attack_value, + (u8 *)inbuf->attack_mask, + &ub_jfc->queue_addr, length, + inbuf->offset, true); + goto OUT; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfc->umem, inbuf, outbuf, true); +OUT: + udma_jfc_put(ub_jfc); + + return ret; +} + +static int udma_dfx_attack_jfr_index_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfr *ub_jfr = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, jfr id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_wqe(udma_dev, ub_jfr->index_queue.umem, inbuf, + outbuf, false); + + udma_jfr_put(ub_jfr); + + return ret; +} + +static inline struct udma_rc_queue * +cqmobj_to_ub_rc_queue(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + rcq_ctx = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_rc_queue *)rcq_ctx->priv; +} + +static int udma_dfx_attack_jfrc_wqe(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct tag_cqm_object *cqm_obj = NULL; + struct udma_rc_queue *rcq = NULL; + u32 buf_size; + int ret = 0; + + if (inbuf->object >= udma_dev->rc_queue_cnt) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to jfrc id(%u) over rc_queue_cnt(%d)\n", + inbuf->object, udma_dev->rc_queue_cnt); + return -EINVAL; + } + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->object, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, jfrc id(%u)\n", + inbuf->object); + return -EINVAL; + } + + rcq = cqmobj_to_ub_rc_queue(cqm_obj); + if (rcq == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfrc, jfrc id(%u)\n", inbuf->object); + ret = -EINVAL; + goto TO_UB_RC_QUEUE_ERR; + } + + buf_size = rcq->rc_queue->q_room_buf_1.buf_size; + if ((inbuf->offset >= buf_size) || + (inbuf->offset + length > buf_size)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "invalid params, offset:%u, len:%u, buf_size:%u", + inbuf->offset, length, buf_size); + ret = -EINVAL; + goto TO_UB_RC_QUEUE_ERR; + } + + ret = memcpy_s(((u8 *)rcq->rc_queue->q_room_buf_1.direct.va) + + inbuf->offset, + length, inbuf->attack_value, length); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "memcpy_s err ret:%d\n", + ret); + } + +TO_UB_RC_QUEUE_ERR: + cqm5_object_put(cqm_obj); + return ret; +} + +static int udma_dfx_attack_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct mtt *ub_mtt) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct hmm_comp_priv *comp_priv = NULL; + u64 *mtts = NULL; + u64 page_size; + int ret; + + if (ub_mtt == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, "ub_mtt is null\n"); + return -EINVAL; + } + + if (ub_mtt->mtt_vaddr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "mtt_vaddr is null, mtt_layers: %u\n", + ub_mtt->mtt_layers); + return -EINVAL; + } + page_size = 1L << ub_mtt->mtt_page_shift; + + if ((length + inbuf->offset > page_size) || + (inbuf->offset > page_size)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "out-of-bounds address, length:%u, offset:%u, pa:0x%llu\n", + length, inbuf->offset, page_size); + return -EINVAL; + } + + comp_priv = get_hmm_comp_priv(udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, "Comp_priv is null\n"); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "mtt_layers: %u\n", + ub_mtt->mtt_layers); + + mtts = (u64 *)hmm_em_table_find( + &comp_priv->mtt_em_table, + ub_mtt->mtt_seg[ub_mtt->mtt_layers - 1]->offset, NULL); + if (mtts == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Can't find va and pa of mtt entry\n"); + return -EINVAL; + } + + ret = memcpy_s(((void *)mtts) + inbuf->offset, length, + inbuf->attack_value, length); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DFX, "memcpy_s err ret:%d\n", + ret); + + return ret; +} + +static int udma_dfx_attack_jetty_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jetty->ub_mtt); + + udma_jetty_put(ub_jetty); + return ret; +} + +static int udma_dfx_attack_jfs_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfs *ub_jfs = NULL; + int ret = 0; + + ub_jfs = udma_find_jfs(udma_dev, inbuf->object); + if (ub_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfs, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jfs->ub_mtt); + + udma_jfs_put(ub_jfs); + return ret; +} + +static int udma_dfx_attack_jfr_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfr *ub_jfr = NULL; + int ret = 0; + + ub_jfr = udma_find_jfr(udma_dev, inbuf->object); + if (ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfr, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jfr->srq.ub_mtt); + + udma_jfr_put(ub_jfr); + return ret; +} + +static int udma_dfx_attack_jfc_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfc *ub_jfc = NULL; + int ret = 0; + + ub_jfc = udma_find_jfc(udma_dev, inbuf->object); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, id(%u)\n", inbuf->object); + return -EINVAL; + } + + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_jfc->ub_mtt); + + udma_jfc_put(ub_jfc); + return ret; +} + +static int udma_dfx_attack_segment_mtt(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_segment *ub_segment = NULL; + int ret = 0; + + ub_segment = udma_seg_find(udma_dev, inbuf->object); + if (ub_segment == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_segment, id(%u)\n", inbuf->object); + return -EINVAL; + } + ret = udma_dfx_attack_mtt(udma_dev, inbuf, &ub_segment->mtt); + udma_seg_put(ub_segment); + return ret; +} + +static int udma_dfx_attack_jetty_pi(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jetty *ub_jetty = NULL; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jetty id(%u)\n", + inbuf->object); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "inbuf->object:%u, inbuf->pi:%u\n", + inbuf->object, inbuf->pi_ci); + + *((u32 *)ub_jetty->sdb.virt_addr) = cpu_to_be32(inbuf->pi_ci); + + udma_jetty_put(ub_jetty); + return 0; +} + +static int udma_dfx_attack_jfc_ci(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + struct udma_jfc *ub_jfc = NULL; + + ub_jfc = udma_find_jfc(udma_dev, inbuf->object); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jfc, jfc id(%u)\n", inbuf->object); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "inbuf->object:%u, inbuf->ci:%u\n", + inbuf->object, inbuf->pi_ci); + + *((u32 *)ub_jfc->db.virt_addr) = + cpu_to_be32(inbuf->pi_ci & UB_CQ_UPDATE_CI_MASK); + + udma_jfc_put(ub_jfc); + return 0; +} + +static int udma_dfx_attack_jetty_tamsn(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf) +{ + u32 length = (inbuf->length > UB_ATTACK_LEN) ? UB_ATTACK_LEN : + inbuf->length; + struct udma_jetty *ub_jetty = NULL; + int ret = 0; + + ub_jetty = udma_find_jetty(udma_dev, inbuf->object); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Failed to get ub_jetty, jetty id(%u)\n", + inbuf->object); + return -EINVAL; + } + + ret = copy_data_for_attack_tamsn(udma_dev, (u8 *)inbuf->attack_value, + &ub_jetty->ubrc, length, + inbuf->offset); + + udma_jetty_put(ub_jetty); + + return ret; +} + +static struct udma_dfx_attack_cmd_func g_udma_attack_cmd_funcs[] = { + { UB_ATTACK_TYPE_DB, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_db }, + { UB_ATTACK_TYPE_DB, UB_ATTACK_SUB_TYPE_JFS, udma_dfx_attack_jfs_db }, + { UB_ATTACK_TYPE_DB, UB_ATTACK_SUB_TYPE_JFR, + udma_dfx_attack_jfr_db }, //ATTACK PCNT + + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFS, udma_dfx_attack_jfs_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFR, udma_dfx_attack_jfr_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFC, udma_dfx_attack_jfc_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_JFRC, + udma_dfx_attack_jfrc_wqe }, + { UB_ATTACK_TYPE_WQE, UB_ATTACK_SUB_TYPE_IDX_Q, + udma_dfx_attack_jfr_index_wqe }, + + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JFS, udma_dfx_attack_jfs_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JFR, udma_dfx_attack_jfr_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_JFC, udma_dfx_attack_jfc_mtt }, + { UB_ATTACK_TYPE_MTT, UB_ATTACK_SUB_TYPE_SEG, + udma_dfx_attack_segment_mtt }, + + { UB_ATTACK_TYPE_PICI, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_pi }, + { UB_ATTACK_TYPE_PICI, UB_ATTACK_SUB_TYPE_JFC, udma_dfx_attack_jfc_ci }, + + { UB_ATTACK_TYPE_TAMSN, UB_ATTACK_SUB_TYPE_JETTY, + udma_dfx_attack_jetty_tamsn }, +}; + +static inline int udma_dfx_attack_inbuf_outbuf_check_and_clear( + struct udma_device *udma_dev, void *buf_out, u32 in_size, u32 *out_size) +{ + int ret = 0; + + if (in_size < (u32)sizeof(struct ub_attack_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack buffer in size too small\n"); + return -EINVAL; + } + + if (*out_size < (u32)sizeof(struct ub_attack_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack buffer out size too small\n"); + return -EINVAL; + } + + ret = memset_s(buf_out, *out_size, 0, + (u32)sizeof(struct ub_attack_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack out buf memset error ret:%d\n", ret); + return -EINVAL; + } + + return 0; +} + +int udma_dfx_attack_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_attack_inbuf *inbuf = (struct ub_attack_inbuf *)buf_in; + struct ub_attack_outbuf *outbuf = (struct ub_attack_outbuf *)buf_out; + udma_adm_dfx_attack_func_t func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (udma_dfx_attack_inbuf_outbuf_check_and_clear( + udma_dev, buf_out, in_size, out_size) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx attack buf check or clear failed.\n"); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, + "inbuf->object:%u, inbuf->convert_endian:%u\n", inbuf->object, + inbuf->convert_endian); + + if (inbuf->convert_endian != 0) { + udma_cpu_to_be32((void *)inbuf->attack_value, + sizeof(inbuf->attack_value)); + udma_cpu_to_be32((void *)inbuf->attack_mask, + sizeof(inbuf->attack_mask)); + } + + udma_dump_dfx(udma_dev, (u64)(uintptr_t)inbuf->attack_value, + sizeof(inbuf->attack_value), + (u64)(uintptr_t)inbuf->attack_value, + sizeof(inbuf->attack_value)); + udma_dump_dfx(udma_dev, (u64)(uintptr_t)inbuf->attack_mask, + sizeof(inbuf->attack_mask), + (u64)(uintptr_t)inbuf->attack_mask, + sizeof(inbuf->attack_mask)); + + for (i = 0; i < sizeof(g_udma_attack_cmd_funcs) / + sizeof(g_udma_attack_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_attack_cmd_funcs[i].cmd_type, + g_udma_attack_cmd_funcs[i].sub_type, 0); + if (cmd_type == + (inbuf->cmd_type & UB_DFX_QUERY_SOURCE_TYPE_SHIELD_MASK)) { + func = g_udma_attack_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "function of attack cmd(0x%x) is not supported\n", + cmd_type); + return -EINVAL; + } + + *out_size = (u32)sizeof(struct ub_attack_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + udma_err(udma_dev->dev, UDMA_M_DFX, "attack cmd(0x%x) is not exist\n", + cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h new file mode 100644 index 000000000..58e7c6694 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_attack.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_ATTACK_H +#define UDMA_DFX_ATTACK_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_adm_dfx_attack_func_t)(struct udma_device *udma_dev, + const struct ub_attack_inbuf *inbuf, + struct ub_attack_outbuf *outbuf); +struct udma_dfx_attack_cmd_func { + u32 cmd_type; + u32 sub_type; + udma_adm_dfx_attack_func_t cmd_func; +}; + +int udma_dfx_attack_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_ATTACK_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c new file mode 100644 index 000000000..fb51f3522 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> +#include <linux/skbuff.h> +#include <linux/tcp.h> +#include <linux/etherdevice.h> +#include <linux/if_packet.h> +#include <linux/skbuff.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_drop.h" +#include "udma_dfx_cc_inject.h" +#include "udma_dfx_capture.h" + +/* + * capture packet configured SML table specification is 256 * 16B + * 0~254 index of table entry is pa address information, for microcode write capture take to of packet count data, driver can read, by struct udma_dfx_cap_tbl indicates + * 255 index of table entry is configuration information, by struct udma_dfx_cap_cfg_tbl indicates + */ +#define CAP_CFG_ENTRY_INDEX 255 // configuration item in SML table in of index +#define CAP_PA_TBL_ENTRY_NUM 255 // a PA address table table entry count +#define CAP_BLOCK_SIZE 256 // a block of size (Byte) + +#define CAP_ADDR_COMBIN_SHIFT 32 + +#define CLEAR_CAP_TRYING_TIMES 10 +#define CLEAR_SLEEP_TIME 2 +#define CAP_STOP_SLEEP_TIME 50 + +static DEFINE_MUTEX(cap_mutex); + +/* + * structure as use: intomember num/2 indicates a pa address corresponding to space of block count + * block used to store place capture take to of packet, is fixed size, each block size by macro CAP_BLOCK_SIZE specifies + * a pa address corresponding space size = block count * each block of size + * a pa address table one total has CAP_PA_TBL_ENTRY_NUM (count word) pa, total has block count is CAP_PA_TBL_ENTRY_NUM * a pa corresponding to block count + * total has two pa address table, respectively is que_addr[0][1][] and que_addr[1][1][] + */ +static struct udma_cap_block_num_attr g_nattr[8] = { + { .block_num_idx = 0, .shift = 4, .num = 16 }, + { .block_num_idx = 1, .shift = 5, .num = 32 }, + { .block_num_idx = 2, .shift = 6, .num = 64 }, + { .block_num_idx = 3, .shift = 7, .num = 128 }, + { .block_num_idx = 4, .shift = 8, .num = 256 }, + { .block_num_idx = 5, .shift = 9, .num = 512 }, + { .block_num_idx = 6, .shift = 10, .num = 1024 }, + { .block_num_idx = 7, .shift = 11, .num = 2048 }, +}; + +struct udma_pkt_cap_info g_udma_cap_info = { + .cap_status = UDMA_CAPTURE_STOP, + .cap_mode = 1, + .poll_ci = 0, + .udma_dev = NULL, + .func_id = 0, + /* Configurable, currently using 4, when modifying, the size of cfg bl needs to be modified synchronously */ + .block_num_idx = 4, + .mode = 0, + .qp_list_cnt = 0, + .qp_list_head = LIST_HEAD_INIT(g_udma_cap_info.qp_list_head), +}; + +static int udma_clear_cap_counter(struct udma_device *udma_dev, u32 *value) +{ + struct ub_dfx_cap_ctr_cmd clear_cap_counter = { 0 }; + u16 out_size = (u16)sizeof(clear_cap_counter); + int ret; + + clear_cap_counter.index = 0; // UB_DP_DFX_FLOW_CAP_NUM_CNT th 0(count word) counter + + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, UB_MPU_CMD_DFX_CLEAR_CAP_CTR, + &clear_cap_counter, sizeof(clear_cap_counter), + &clear_cap_counter, &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || clear_cap_counter.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear cap counter failed, ret:%d, status:0x%x, out size:0x%x\n", + ret, clear_cap_counter.head.status, out_size); + return -EINVAL; + } + + *value = clear_cap_counter.value; + + udma_info(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear success, counter before clear:%u\n", + clear_cap_counter.value); + + return 0; +} + +static void udma_stop_thread(struct sdk_thread_info *thread_info) +{ + if (thread_info->thread_obj) { + (void)kthread_stop(thread_info->thread_obj); + thread_info->thread_obj = NULL; + } +} + +static int udma_set_cap_func_disable(struct udma_device *udma_dev) +{ + u16 glb_func_id = hinic5_global_func_id(udma_dev->hwdev); + int ret; + + ret = hinic5_set_func_capture_en(udma_dev->hwdev, glb_func_id, 0); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "set capture disable failed, ret:%d\n", ret); + return ret; + } + + return 0; +} + +static void udma_dfx_dma_free_addr(struct udma_device *udma_dev, int cap_index, + int que_index) +{ + u64 v_addr; + u64 p_addr; + + v_addr = g_udma_cap_info.que_addr[que_index][0][cap_index]; + p_addr = g_udma_cap_info.que_addr[que_index][1][cap_index]; + dma_free_coherent( + udma_dev->dev, + (unsigned long)(CAP_BLOCK_SIZE * + ((g_nattr[g_udma_cap_info.block_num_idx].num) >> + 1)), + (void *)(uintptr_t)v_addr, (dma_addr_t)p_addr); + /* resource free after address clear 0 */ + g_udma_cap_info.que_addr[que_index][0][cap_index] = 0; + g_udma_cap_info.que_addr[que_index][1][cap_index] = 0; +} + +static void udma_dfx_free_mem_for_one_cap(struct udma_device *udma_dev, + int cap_index) +{ + udma_dfx_dma_free_addr(udma_dev, cap_index, 1); + udma_dfx_dma_free_addr(udma_dev, cap_index, 0); +} + +static void udma_dfx_free_cap(struct udma_device *udma_dev) +{ + int i; + + for (i = 0; i < CAP_PA_TBL_ENTRY_NUM; i++) + udma_dfx_free_mem_for_one_cap(udma_dev, i); +} + +static int udma_clear_cap_counter_encap(struct udma_device *udma_dev) +{ + u32 counter = 0; + u32 times; + int ret; + + for (times = 0; times < CLEAR_CAP_TRYING_TIMES; times++) { + if (times == CLEAR_CAP_TRYING_TIMES - 1) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear pi failed\n"); + return -EFAULT; + } + + ret = udma_clear_cap_counter(udma_dev, &counter); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear cap counter failed, ret:%d\n", ret); + return -EFAULT; + } + + if (counter == 0) + break; + + msleep(CLEAR_SLEEP_TIME); + } + + return 0; +} + +int udma_dfx_stop_cap_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + u32 glb_func_id = hinic5_global_func_id(udma_dev->hwdev); + int ret = 0; + + udma_info(udma_dev->dev, UDMA_M_CAPTURE_DFX, "start to stop cap\n"); + + mutex_lock(&cap_mutex); + if (g_udma_cap_info.cap_status != UDMA_CAPTURE_START) { // 1:RUNNING + mutex_unlock(&cap_mutex); + udma_warn( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "%s: capture is not running, cap_status(%u), ret(%d)\n", + __func__, g_udma_cap_info.cap_status, ret); + return 0; + } + + if (g_udma_cap_info.func_id != glb_func_id) { + mutex_unlock(&cap_mutex); + udma_warn( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "%s: func_id not match, capture func_id(%u), glb_func_id(%u)\n", + __func__, g_udma_cap_info.func_id, glb_func_id); + return 0; + } + + udma_stop_thread(&(g_udma_cap_info.task)); + + /* stop cap */ + ret = udma_set_cap_func_disable(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "set cap enable failed, ret:%d\n", ret); + ret = -EFAULT; + goto err; + } + + msleep(CAP_STOP_SLEEP_TIME); + + /* release lt mem resource */ + udma_dfx_free_cap(udma_dev); + /* lt index_1 */ + ret = udma_clear_cap_counter_encap(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "clear cap counter failed, ret:%d\n", ret); + goto err; + } + + g_udma_cap_info.cap_status = UDMA_CAPTURE_STOP; + g_udma_cap_info.poll_ci = 0; + g_udma_cap_info.func_id = 0; + g_udma_cap_info.udma_dev = NULL; + + udma_info(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "stop cap success, func_id:%u\n", glb_func_id); + +err: + udma_dev->capture_en = false; + mutex_unlock(&cap_mutex); + return ret; +} + +static struct udma_dfx_packet_cmd_func g_udma_packet_cmd_funcs[] = { + /* packet drop */ + { UB_CMD_START_DROP_CARD_PACKET, udma_dfx_start_card_drop_pkt }, + { UB_CMD_STOP_DROP_CARD_PACKET, udma_dfx_stop_card_drop_pkt }, + { UB_CMD_START_DROP_FUNC_PACKET, udma_dfx_start_func_drop_pkt }, + { UB_CMD_STOP_DROP_FUNC_PACKET, udma_dfx_stop_func_drop_pkt }, + { UB_CMD_START_DROP_TP_PACKET, udma_dfx_start_tp_drop_pkt }, + { UB_CMD_STOP_DROP_TP_PACKET, udma_dfx_stop_tp_drop_pkt }, + /* cc inject */ + { UB_CMD_CC_INJECT, udma_dfx_cc_inject }, +}; + +int udma_dfx_packet_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_packet_inbuf *inbuf = (struct ub_packet_inbuf *)buf_in; + union ub_packet_outbuf *outbuf = (union ub_packet_outbuf *)buf_out; + udma_adm_dfx_packet_func_t func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_packet_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "udma dfx packet buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(union ub_packet_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "udma dfx packet buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_packet_cmd_funcs) / + sizeof(g_udma_packet_cmd_funcs[0]); + i++) { + cmd_type = g_udma_packet_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_packet_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_CAPTURE_DFX, + "error: function of cmd:0x%x is not supported\n", + cmd_type); + return -EINVAL; + } + + *out_size = sizeof(union ub_packet_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + udma_err(udma_dev->dev, UDMA_M_CAPTURE_DFX, + "error: cmd:0x%x is not exist\n", inbuf->cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h new file mode 100644 index 000000000..106c299c8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_capture.h @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_CAPTURE_H +#define UDMA_DFX_CAPTURE_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +/* hardware side packet control header (non- transport packet actualbody data) */ +typedef union udma_cap_hdr { + struct { + u32 rsvd : 16; + u32 pad : 8; // rx direction pad value is 0, tx direction pad value is 2(macro ETH_TX_PKT_PADDING_LEN specifies) + u32 rsvd0 : 2; + u32 col_num : 4; + u32 tx_rx : 1; + u32 vld : 1; + } cap_ctrl_info; + u32 value; +} udma_cap_hdr_u; + +typedef struct udma_dfx_cap_pa_entry { + u32 wr_init_pc_h32; + u32 wr_init_pc_l32; +} udma_dfx_cap_pa_entry_s; + +typedef struct udma_dfx_cap_tbl { + udma_dfx_cap_pa_entry_s pa[2]; +} udma_dfx_cap_tbl_s; + +typedef struct udma_dfx_cap_cfg_tbl { + u32 ci_index; + + union { + struct { + /* Driver configurate bit offset by the number of per pa block, for example, 7 means 128, 8 means 256 */ + u8 cap_block_num_shift; + u8 cap_mode; /* Packet Capture mode */ + u8 qp_mode; /* according to qp Packet Capture */ + u8 rsvd; /* reserved */ + } bs; + u32 value; + } dw1; + + union { + struct { + u32 state : 8; /* Packet Capture rate control */ + u32 rsvd : 8; + u32 cap_func : 16; /* Packet Capture function */ + } bs; + u32 value; + } dw2; + + u32 maxnum; +} udma_dfx_cap_cfg_tbl_s; + +struct udma_cap_block_num_attr { + u32 block_num_idx; + u32 shift; + u32 num; +}; + +struct sdk_thread_info { + struct task_struct *thread_obj; + char *name; + void (*thread_fn)(void *); + void *thread_event; + void *data; +}; + +struct udma_pkt_cap_info { + struct udma_device *udma_dev; + u32 func_id; + u32 poll_ci; /* driver push to protocol stack of packet of total count */ + u32 cap_mode; /* 0 : drop mode 1: overwrite mode */ + u32 mode; /* 0 : func mode 1: qp mode */ + u32 cap_status; /* 0 : stopped 1: running */ + u32 block_num_idx; + u32 block_num_per_entry; + u32 maxnum; + struct sdk_thread_info task; + char thread_name[20]; + u64 que_addr[2][2][256]; + struct list_head + qp_list_head; /* Based on qp Packet Capture linked list */ + u32 qp_list_cnt; +}; + +enum udma_dfx_capture_state { + UDMA_CAPTURE_START = 0, + UDMA_CAPTURE_STOP, +}; + +typedef int (*udma_adm_dfx_packet_func_t)(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); + +struct udma_dfx_packet_cmd_func { + u32 cmd_id; + udma_adm_dfx_packet_func_t cmd_func; +}; + +extern int hinic5_set_func_capture_en(void *hwdev, u16 func_id, bool cap_en); + +int udma_dfx_packet_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int udma_dfx_stop_cap_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); + +#endif /* UDMA_DFX_CAPTURE_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c new file mode 100644 index 000000000..79c77d3a9 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.c @@ -0,0 +1,40 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "hinic5_hw.h" +#include "ub_pub_cmd.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_dfx.h" +#include "udma_cmd_tp.h" + +int udma_dfx_cc_inject(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct tag_ub_cc_attr cc_attr = { 0 }; + struct tag_cqm_object *cqm_obj = NULL; + + /* make sure that the tpn is valid */ + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->tpn, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_CC_INJECT_DFX, + "get cqm object failed, tpn(0x%x)\n", inbuf->tpn); + return -EINVAL; + } + cqm5_object_put(cqm_obj); + + cc_attr.tpn = inbuf->tpn; + cc_attr.cc_inject_mode = inbuf->cc_inject_mode; + cc_attr.cc_percent = inbuf->cc_percent; + cc_attr.cc_limit_low = inbuf->cc_limit_low; + cc_attr.cc_limit_high = inbuf->cc_limit_high; + return udma_cmd_tp_cc_modify(udma_dev, &cc_attr); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h new file mode 100644 index 000000000..ebe05bbf2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cc_inject.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_CC_INJECT_H +#define UDMA_DFX_CC_INJECT_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef struct tag_ub_cc_attr { + u32 tpn; + u32 cc_inject_mode; + u32 cc_percent; + u32 cc_limit_low; + u32 cc_limit_high; +} ub_cc_attr_s; + +int udma_dfx_cc_inject(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); + +#endif /* UDMA_DFX_CC_INJECT_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c new file mode 100644 index 000000000..9daba8765 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include <linux/dma-mapping.h> +#include "hinic5_hw.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "ub_pub_cmd.h" +#include "udma_dfx.h" +#include "udma_cmd_com.h" +#include "udma_cmd_srq.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_jfc.h" +#include "udma_cmd_segment.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfrc.h" +#include "udma_cmd_vtp.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_sip.h" +#include "udma_dfx_cqm_modify.h" + +#define INVALID_VALUE ((u32)~0u) +#define INVALID_XID INVALID_VALUE +#define INVALID_OFFSET INVALID_VALUE +#define INVALID_LENGTH INVALID_VALUE +#define INVALID_VF_ID INVALID_VALUE +#define MAX_LENGTH (INVALID_VALUE >> 1) + +static int udma_dfx_cqm_modify_sip_tbl(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_sip_modify(udma_dev, inbuf->xid, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "[INFO] %s: ret(%d), sip_idx(%u), offset(%u), length(%u)\n", + __func__, ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_jfrc_modify_pcqc(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_jfrc_modify_pcqc(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), jfrc_id(%u), offset(%u), length(%u)\n", + __func__, ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_jetty_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_jetty_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), jetty_id(%u), offset(%u), length(%u)\n", + __func__, ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_tp_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_tp_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), tpn(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_jfr_srq_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_srq_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), srqn(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_jfc_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_jfc_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), jfc_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_seg_mpt_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_seg_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), seg_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_vtp_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_vtp_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), vtp_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int udma_dfx_cqm_modify_tpg_ctx(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + + ret = udma_cmd_tpg_batch_modify_ctx(udma_dev, inbuf); + + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: ret(%d), tpg_id(%u), offset(%u), length(%u)\n", __func__, + ret, inbuf->xid, inbuf->offset, inbuf->length); + + return ret; +} + +static int +udma_dfx_cqm_modify_clear_latch_lock(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf) +{ + int ret; + ret = udma_cmd_jetty_clear_dfx_lock_ctx(udma_dev, inbuf->xid); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: clear latch lock failed, ret(%d)\n", __func__, + ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_cqm_modify_latch_check(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + // TODO: later enable send requirement when add XID validity intercept + if (inbuf->xid == INVALID_XID) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify xid not set\n", __func__); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_cqm_modify_check(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + if (inbuf->xid == INVALID_XID) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify xid not set\n", __func__); + return -EINVAL; + } + + if (inbuf->offset == INVALID_OFFSET) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify offset not set\n", __func__); + return -EINVAL; + } + + if (inbuf->length == INVALID_LENGTH) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify length not set\n", __func__); + return -EINVAL; + } + + if (inbuf->is_data_set == false) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify data not set\n", __func__); + return -EINVAL; + } + + /* length is byte is unit, user input of data array of a intomember is 4bit, length:data_len = 1:2 */ + if ((inbuf->length > MAX_LENGTH) || + (inbuf->length * 2 != inbuf->data_len)) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: length(%u) not match data_len(%u)\n", __func__, + inbuf->length, inbuf->data_len); + return -EINVAL; + } + + return 0; +} + +static int +udma_dfx_cqm_modify_vtp_check(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *inbuf) +{ + int ret = 0; + + ret = udma_dfx_cqm_modify_check(udma_dev, inbuf); + if (ret != 0) { + return ret; + } + + /* driver only check whether passed in, not check valid validity */ + if (inbuf->vf_id == INVALID_VF_ID) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cqm modify vf_id not set\n", __func__); + return -EINVAL; + } + + return ret; +} + +static struct udma_dfx_cqm_modify_cmd_func g_udma_cqm_modify_cmd_funcs[] = { + { UB_CQM_MODIFY_TYPE_JETTY_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, + { UB_CQM_MODIFY_TYPE_TP_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_tp_ctx }, + { UB_CQM_MODIFY_TYPE_JFR_SRQ_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jfr_srq_ctx }, + { UB_CQM_MODIFY_TYPE_JFC_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jfc_ctx }, + { UB_CQM_MODIFY_TYPE_SEG_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_seg_mpt_ctx }, + { UB_CQM_MODIFY_TYPE_JFS_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, // reuse jetty code + { UB_CQM_MODIFY_TYPE_JFR_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, // reuse jetty code + { UB_CQM_MODIFY_TYPE_JFRC_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_jetty_ctx }, // reuse jetty code + { UB_CQM_MODIFY_TYPE_VTP_CTX, udma_dfx_cqm_modify_vtp_check, + udma_dfx_cqm_modify_vtp_ctx }, + { UB_CQM_MODIFY_TYPE_TPG_CTX, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_tpg_ctx }, + { UB_CQM_MODIFY_TYPE_CLEAR_LATCH_LOCK, udma_dfx_cqm_modify_latch_check, + udma_dfx_cqm_modify_clear_latch_lock }, + { UB_CQM_MODIFY_TYPE_JFRC_PCQC, udma_dfx_cqm_modify_check, + udma_dfx_cqm_jfrc_modify_pcqc }, + { UB_CQM_MODIFY_TYPE_SIP_TBL, udma_dfx_cqm_modify_check, + udma_dfx_cqm_modify_sip_tbl }, +}; + +static inline int +udma_dfx_cqm_modify_check_func(struct udma_device *udma_dev, + const struct ub_cqm_modify_inbuf *buf_in, + udma_adm_dfx_cqm_check_func_t check_func) +{ + if (check_func == NULL) { + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "Cmd(%u) do not need validity-check.\n", + buf_in->cmd_type); + return 0; + } + + if (check_func(udma_dev, buf_in) != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "Check func not null and check failed.\n"); + return -EINVAL; + } + + return 0; +} + +int udma_dfx_cqm_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_cqm_modify_inbuf *inbuf = + (struct ub_cqm_modify_inbuf *)buf_in; + union ub_cqm_modify_outbuf *outbuf = + (union ub_cqm_modify_outbuf *)buf_out; + udma_adm_dfx_cqm_modify_func_t cmd_func = NULL; + udma_adm_dfx_cqm_check_func_t check_func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_cqm_modify_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "udma dfx cqm modify buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(union ub_cqm_modify_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "udma dfx cqm modify buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < ARRAY_SIZE(g_udma_cqm_modify_cmd_funcs); i++) { + cmd_type = g_udma_cqm_modify_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + cmd_func = g_udma_cqm_modify_cmd_funcs[i].cmd_func; + check_func = g_udma_cqm_modify_cmd_funcs[i].check_func; + if (cmd_func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + if (udma_dfx_cqm_modify_check_func(udma_dev, inbuf, + check_func) != 0) { + return -EINVAL; + } + *out_size = sizeof(union ub_cqm_modify_outbuf); + + return cmd_func(udma_dev, inbuf, outbuf); + } + } + + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: cmd(0x%x) is not exist\n", __func__, cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h new file mode 100644 index 000000000..ef2d6955f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_cqm_modify.h @@ -0,0 +1,28 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_CQM_MODIFY_H +#define UDMA_DFX_CQM_MODIFY_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_adm_dfx_cqm_modify_func_t)( + struct udma_device *udma_dev, const struct ub_cqm_modify_inbuf *inbuf, + union ub_cqm_modify_outbuf *outbuf); + +typedef int (*udma_adm_dfx_cqm_check_func_t)( + struct udma_device *udma_dev, const struct ub_cqm_modify_inbuf *inbuf); + +struct udma_dfx_cqm_modify_cmd_func { + u32 cmd_id; + udma_adm_dfx_cqm_check_func_t check_func; + udma_adm_dfx_cqm_modify_func_t cmd_func; +}; + +int udma_dfx_cqm_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_CQM_MODIFY_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c new file mode 100644 index 000000000..90b363ed8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/mutex.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include <linux/vmalloc.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_tp.h" +#include "udma_dfx.h" +#include "udma_cmd_tp.h" +#include "udma_dfx_drop.h" + +#define DROP_MODE_SHIFT 3 // packet drop mode occupy the 4, 5 two bit +#define DROP_1823v200_MAX_MODE 3 // packet drop supported maximum mode + +static void fill_dfx_l2d_set_param(struct udma_device *udma_dev, + ub_l2d_mem_cfg_s *cmd_in, + const struct udma_l2d_tbl_dfx *l2d_tbl_dfx) +{ + cmd_in->func_id = hinic5_global_func_id(udma_dev->hwdev); + cmd_in->tbl_mod = UB_L2D_TBL_MOD_DFX; + cmd_in->op_type = UB_L2D_TBL_OP_SET; + cmd_in->tile_id = UB_L2D_DFX_TILE_ID_TILE_ALL; + + memcpy_s(&cmd_in->l2d_tbl.dfx_tbl.attr, sizeof(l2d_ub_dfx_tbl_attr_s), + &l2d_tbl_dfx->attr, sizeof(l2d_ub_dfx_tbl_attr_s)); + memcpy_s(&cmd_in->l2d_tbl.dfx_tbl.mask, sizeof(l2d_ub_dfx_tbl_mask_s), + &l2d_tbl_dfx->mask, sizeof(l2d_ub_dfx_tbl_mask_s)); +} + +int udma_dfx_set_l2d_tbl_filter_param(struct udma_device *udma_dev, + const struct udma_l2d_tbl_dfx *l2d_tbl_dfx, + bool set_drop_level) +{ + ub_l2d_mem_cfg_cmd_s cmd_in = { 0 }; + ub_l2d_mem_cmd_rsp_s cmd_out = { 0 }; + u16 outsize = (u16)sizeof(ub_l2d_mem_cmd_rsp_s); + int ret = 0; + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return ret; + } + + fill_dfx_l2d_set_param(udma_dev, &cmd_in.cfg, l2d_tbl_dfx); + cmd_in.cfg.l2d_tbl.dfx_tbl.mask.dw0.bs.drop_pkt_ratio &= + (u8)set_drop_level; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CFG_L2D_TBL, &cmd_in, + sizeof(ub_l2d_mem_cfg_cmd_s), &cmd_out, + &outsize, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || outsize == 0 || cmd_out.msg_head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "set drop l2dmem failed, err(%d), status(0x%x), out size(0x%x)\n", + ret, cmd_out.msg_head.status, outsize); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "set drop l2dmem success.\n"); + + return 0; +} + +static int udma_set_func_drop_level(struct udma_device *udma_dev, u16 func_id, + u32 drop_level) +{ + struct ub_dfx_drop_cmd drop_cmd = { { 0 } }; + u16 out_size = sizeof(drop_cmd); + int ret; + + drop_cmd.drop_level = drop_level; + drop_cmd.func_port = func_id; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_SET_DROP, &drop_cmd, + sizeof(drop_cmd), &drop_cmd, &out_size, 0, + HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || drop_cmd.head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_DROP_DFX, + "%s: set func drop failed, ret(%d), status(0x%x), out size(0x%x)\n", + __func__, ret, drop_cmd.head.status, out_size); + return -EINVAL; + } + + return 0; +} + +static int udma_dfx_set_func_drop_level(struct udma_device *udma_dev, + u32 drop_level) +{ + u16 glb_func_id = hinic5_global_func_id(udma_dev->hwdev); + int ret; + + ret = udma_set_func_drop_level(udma_dev, glb_func_id, drop_level); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: set drop level(%u) failed, ret(%d)\n", __func__, + drop_level, ret); + return ret; + } + + udma_info(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: success, drop_level(%u), func_id(%u)\n", __func__, + drop_level, glb_func_id); + + return 0; +} + +static int udma_dfx_get_drop_pkt_ratio(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + u32 *drop_pkt_ratio) +{ + *drop_pkt_ratio = inbuf->data.attr.dw15.bs.drop_pkt_ratio; + + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + return 0; + } + + if (inbuf->data.attr.dw15.bs.drop_pkt_type > DROP_1823v200_MAX_MODE) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "%s: drop_mode(%u) exceeds the drop_max_mode(%u)\n\n", + __func__, inbuf->data.attr.dw15.bs.drop_pkt_type, + DROP_1823v200_MAX_MODE); + return -EINVAL; + } + + *drop_pkt_ratio = + *drop_pkt_ratio + + (inbuf->data.attr.dw15.bs.drop_pkt_type << DROP_MODE_SHIFT); + + return 0; +} + +int udma_dfx_start_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + return udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf->data, true); +} + +int udma_dfx_stop_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct udma_l2d_tbl_dfx data = { 0 }; + + data.mask.dw0.value = U32_MAX; + + return udma_dfx_set_l2d_tbl_filter_param(udma_dev, &data, true); +} + +int udma_dfx_start_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + int ret = 0; + + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf->data, false); + if (ret != 0) { + return ret; + } + + return udma_dfx_set_func_drop_level( + udma_dev, inbuf->data.attr.dw15.bs.drop_pkt_ratio); +} + +int udma_dfx_stop_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct udma_l2d_tbl_dfx data = { 0 }; + int ret = 0; + + data.mask.dw0.value = U32_MAX; + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &data, false); + if (ret != 0) { + return ret; + } + + return udma_dfx_set_func_drop_level(udma_dev, 0); +} + +int udma_dfx_start_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + int ret = 0; + u32 drop_pkt_ratio = 0; + + ret = udma_dfx_get_drop_pkt_ratio(udma_dev, inbuf, &drop_pkt_ratio); + if (ret != 0) { + return ret; + } + + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf->data, false); + if (ret != 0) { + return ret; + } + + return udma_cmd_tp_set_drop_level(udma_dev, inbuf->tpn, drop_pkt_ratio); +} + +int udma_dfx_stop_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf) +{ + struct udma_l2d_tbl_dfx data = { 0 }; + int ret = 0; + + data.mask.dw0.value = U32_MAX; + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &data, false); + if (ret != 0) { + return ret; + } + + return udma_cmd_tp_set_drop_level(udma_dev, inbuf->tpn, 0); +} + +void udma_stop_all_drop_pkt(struct udma_device *udma_dev) +{ + int ret; + int npu_version = -1; + + ret = udma_query_npu_dfx_version(udma_dev, &npu_version); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "UDMA fail to query version, ret(%d)\n", ret); + return; + } + if (npu_version == 0) { + return; + } + + ret = udma_dfx_stop_func_drop_pkt(udma_dev, NULL, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DROP_DFX, + "stop func drop pkt failed, ret(%d)\n", ret); + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h new file mode 100644 index 000000000..63f9cbcb2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_drop.h @@ -0,0 +1,34 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_DROP_H +#define UDMA_DFX_DROP_H + +#include "ub_pub_cmd.h" +#include "udma_common.h" + +int udma_dfx_start_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_stop_card_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_start_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_stop_func_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_start_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +int udma_dfx_stop_tp_drop_pkt(struct udma_device *udma_dev, + const struct ub_packet_inbuf *inbuf, + union ub_packet_outbuf *outbuf); +void udma_stop_all_drop_pkt(struct udma_device *udma_dev); +int udma_dfx_set_l2d_tbl_filter_param(struct udma_device *udma_dev, + const struct udma_l2d_tbl_dfx *l2d_tbl_dfx, + bool set_drop_level); + +#endif /* UDMA_DFX_DROP_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c new file mode 100644 index 000000000..b23aeb3ee --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "hinic5_hw.h" +#include "ub_mpu_cmd.h" +#include "comm_defs.h" +#include "udma_dfx_drop.h" +#include "udma_dfx_l2d.h" + +static int udma_dfx_l2d_buf_size_check(struct udma_device *udma_dev, + u32 in_size, u32 *out_size) +{ + if (in_size < (u32)sizeof(struct ub_l2d_dfx_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx buffer in size %u too small\n", in_size); + return -EINVAL; + } + + if (*out_size < (u32)sizeof(union ub_l2d_dfx_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx buffer out size %u too small\n", + *out_size); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_l2d_check(struct udma_device *udma_dev, + const struct ub_l2d_dfx_inbuf *inbuf) +{ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "l2d dfx only support tpf device.\n"); + return -EINVAL; + } + return 0; +} + +static int udma_dfx_l2d_query_perf_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + ub_l2d_mem_cfg_cmd_s cmd_in = { 0 }; + ub_l2d_mem_cmd_rsp_s cmd_out = { 0 }; + u16 outsize = (u16)sizeof(ub_l2d_mem_cmd_rsp_s); + int ret = 0; + + cmd_in.cfg.func_id = hinic5_global_func_id(udma_dev->hwdev); + cmd_in.cfg.tbl_mod = UB_L2D_TBL_MOD_DFX; + cmd_in.cfg.op_type = UB_L2D_TBL_OP_GET; + cmd_in.cfg.tile_id = + ((struct ub_l2d_dfx_inbuf *)buf_in)->l2d_cfg.tile_id; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CFG_L2D_TBL, &cmd_in, + sizeof(ub_l2d_mem_cfg_cmd_s), &cmd_out, + &outsize, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || outsize == 0 || cmd_out.msg_head.status != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx query l2dmem perf data fail, ret=%d.\n", + ret); + return -EFAULT; + } + + /* from MPU obtain of data is little-endian, dump_dfx need seek input is big-endian, in this need convert one next */ + udma_cpu_to_be32(cmd_out.data, sizeof(l2d_ub_dfx_tbl_attr_s)); + ret = memcpy_s( + ((union ub_l2d_dfx_outbuf *)buf_out)->ub_l2d_dfx_data, + sizeof(((union ub_l2d_dfx_outbuf *)buf_out)->ub_l2d_dfx_data), + cmd_out.data, + sizeof(((union ub_l2d_dfx_outbuf *)buf_out)->ub_l2d_dfx_data)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx query l2dmem perf data fail, ret=%d.\n", + ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, "set drop l2dmem success.\n"); + + return 0; +} + +static int udma_dfx_l2d_modify_perf_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + struct udma_l2d_tbl_dfx inbuf = { 0 }; + int ret = 0; + + ret = memcpy_s(&inbuf.attr, sizeof(inbuf.attr), + &(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.attr), + sizeof(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.attr)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s for attr failed, ret=%d.\n", ret); + return -EFAULT; + } + ret = memcpy_s(&inbuf.mask, sizeof(inbuf.mask), + &(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.mask), + sizeof(((struct ub_l2d_dfx_inbuf *)buf_in) + ->l2d_cfg.l2d_tbl.dfx_tbl.mask)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s for mask failed, ret=%d.\n", ret); + return -EFAULT; + } + + ret = udma_dfx_set_l2d_tbl_filter_param(udma_dev, &inbuf, true); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma l2d dfx modify l2dmem perf data fail, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +const static struct udma_dfx_l2d_cmd_cb g_udma_dfx_l2d_cmd_funcs[] = { + { UB_L2D_DFX_MODULE_DFX_TBL, UB_L2D_DFX_SUB_TYPE_SHOW, + udma_dfx_l2d_check, udma_dfx_l2d_query_perf_data }, + { UB_L2D_DFX_MODULE_DFX_TBL, UB_L2D_DFX_SUB_TYPE_CONFIG, + udma_dfx_l2d_check, udma_dfx_l2d_modify_perf_data }, +}; + +int udma_dfx_l2d_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_l2d_dfx_inbuf *inbuf = NULL; + udma_dfx_l2d_check_func_t check_func = NULL; + udma_ioctl_func_t cmd_func = NULL; + u32 cmd_type = 0; + int ret = 0; + int i = 0; + + ret = udma_dfx_l2d_buf_size_check(udma_dev, in_size, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "check buffer size fail, ret=%d.\n", ret); + return -EINVAL; + } + + inbuf = (struct ub_l2d_dfx_inbuf *)buf_in; + for (i = 0; i < sizeof(g_udma_dfx_l2d_cmd_funcs) / + sizeof(g_udma_dfx_l2d_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_dfx_l2d_cmd_funcs[i].type, + g_udma_dfx_l2d_cmd_funcs[i].sub_type, 0); + if (cmd_type == inbuf->cmd_type) { + check_func = g_udma_dfx_l2d_cmd_funcs[i].check_func; + cmd_func = g_udma_dfx_l2d_cmd_funcs[i].cmd_func; + break; + } + } + + if (cmd_func == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "cmd_type %u not support.\n", cmd_type); + return -EINVAL; + } + + if (check_func != NULL) { + ret = check_func(udma_dev, inbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "cmd_type:%u check func fail, ret=%d.\n", + cmd_type, ret); + return ret; + } + } + + ret = cmd_func(udma_dev, buf_in, in_size, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "l2d dfx cmd_type:%u, cmd func fail, ret=%d.\n", + cmd_type, ret); + return ret; + } + + return 0; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h new file mode 100644 index 000000000..1b06091e7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_l2d.h @@ -0,0 +1,28 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_DFX_L2D_H +#define UDMA_DFX_L2D_H + +#include <linux/device.h> + +#include "udma_common.h" +#include "udma_dfx.h" +#include "udma_cmd_com.h" +#include "ub_pub_cmd.h" + +typedef int (*udma_dfx_l2d_check_func_t)(struct udma_device *udma_dev, + const struct ub_l2d_dfx_inbuf *inbuf); + +struct udma_dfx_l2d_cmd_cb { + u32 type; + u32 sub_type; + udma_dfx_l2d_check_func_t check_func; + udma_ioctl_func_t cmd_func; +}; + +int udma_dfx_l2d_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif //UDMA_DFX_L2D_H \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c new file mode 100644 index 000000000..ae1350118 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.c @@ -0,0 +1,671 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "udma_dfx_latch.h" +#include "ub_npu_jetty_cmd_defs.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_tpg.h" + +#define UB_TP_LATCH_TPG_NUM 4 +#define UB_TP_LATCH_START_TPGN(tpgn) ((tpgn) * (UB_TP_LATCH_TPG_NUM)) +#define UB_TP_LATCH_END_TPGN(tpgn) \ + (((tpgn) * (UB_TP_LATCH_TPG_NUM)) + (UB_TP_LATCH_TPG_NUM)) +#define MAX_LIMITED_TIME 10000 +#define LIMIT_PRINT_TIME 3 +#define CHAR_BUF_LEN 64 +#define MAX_DATA_LEN_UB_DFX 8192 +#define DUMP_LINE_BYTE_NUM 16 +#define DUMP_LINE_BYTE_HALF_NUM (DUMP_LINE_BYTE_NUM / 4) + +typedef struct print_latch_limiter { + ulong end_time; + int print_count; +} print_latch_limiter_t; + +static struct print_latch_limiter ta_latch_limiter = { + .end_time = 0, + .print_count = 0, +}; + +static struct print_latch_limiter tp_latch_limiter = { + .end_time = 0, + .print_count = 0, +}; + +/* =================================================LATCH COMMON======================================================= */ +static bool udma_print_info_limited(struct udma_device *udma_dev, + struct print_latch_limiter *limter) +{ + ulong curr_time = jiffies; + if (curr_time > limter->end_time) { + limter->print_count = 0; + limter->end_time = + curr_time + jiffies_to_msecs(MAX_LIMITED_TIME); + } + + if (limter->print_count < LIMIT_PRINT_TIME) { + udma_info_limit(udma_dev->dev, UDMA_M_DFX, + "start print info count: %d\n", + limter->print_count); + limter->print_count++; + return true; + } + + return false; +} + +/** + this function no has print limit, because is is dump information, if use ratelimit related of interface, kernel 10 second 5 time, + then will lose log, so here not limit, by external call add print limit, and use err level ensure in any log level all can print. +*/ +static void udma_print_byte_no_freq_limit(struct udma_device *udma_dev, + const u8 *buf, u32 ilen) +{ + char line[CHAR_BUF_LEN] = { 0 }; + size_t line_len = 0; + int ret = 0; + u32 i; + + if ((buf == NULL) || (ilen == 0) || (ilen > MAX_DATA_LEN_UB_DFX)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "The buf is Null,len=%u is error.\n", ilen); + return; + } + + for (i = 0; i < ilen; i++) { + if ((i != 0) && (i % DUMP_LINE_BYTE_NUM == 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, "%08x:%s\n", + i - DUMP_LINE_BYTE_NUM, line); + line[0] = '\0'; + } else if ((i != 0) && (i % DUMP_LINE_BYTE_HALF_NUM == 0)) { + line_len = strlen(line); + ret = sprintf_s(line + line_len, + CHAR_BUF_LEN - line_len, " "); + if (ret < 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "format dump byte string separator failed, ret:%d .\n", + ret); + return; + } + } + + line_len = strlen(line); + ret = sprintf_s(line + line_len, CHAR_BUF_LEN - line_len, + "%02X", buf[i]); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "format dump byte string failed, ret:%d .\n", + ret); + return; + } + } + + if (strlen(line) != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "%08x:%s\n", + ((i + DUMP_LINE_BYTE_NUM - 1) / DUMP_LINE_BYTE_NUM) * + DUMP_LINE_BYTE_NUM, + line); + } +} + +static void +udma_dfx_parase_latch_common_header(struct udma_device *udma_dev, + union ub_latch_data_common_header *header, + u32 dw0) +{ + udma_be32_to_cpu(&dw0, sizeof(u32)); + + header->dw0 = dw0; + udma_info( + udma_dev->dev, UDMA_M_DFX, + "Latched info dw0=0x%08x, first_seq=%u, last_seq=%u, valid=%u, data_len=0x%04x.\n", + dw0, header->first_seq, header->last_seq, header->valid, + header->latch_total_len); + return; +} + +/* =================================================TA LATCH======================================================= */ +/** + this part is TA lock store query,TA lock store header parse,TA lock store rate-limit print + */ +int udma_dfx_query_ta_latch_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + bool tpf = check_args->tpf; + + if (tpf) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Ta latch should be checked for non-TPF devices."); + return -EINVAL; + } + + return 0; +} + +static void +udma_dfx_parse_ta_latch_header(struct udma_device *udma_dev, + struct ub_ta_rdma_rc_outbuf *ta_rc_entry, + union ub_latch_data_common_header *header) +{ + u32 dw0 = *(u32 *)(ta_rc_entry->rdmarc); + + udma_dfx_parase_latch_common_header(udma_dev, header, dw0); + return; +} + +static int +udma_dfx_get_ta_latch_header(struct udma_device *udma_dev, u32 jetty_id, + union ub_latch_data_common_header *header) +{ + int ret; + struct ub_ta_rdma_rc_outbuf ta_rc_entry = { 0 }; + + (void)memset_s(header, sizeof(union ub_latch_data_common_header), 0, + sizeof(union ub_latch_data_common_header)); + + /* 1. query entry 0 */ + ret = udma_cmd_jetty_query_rdma_rc(udma_dev, jetty_id, 0, &ta_rc_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "Cmd failed, ret=%d!\n", + ret); + return ret; + } + + /* 2. parse get of entry 0 in of dw0 lock store header */ + udma_dfx_parse_ta_latch_header(udma_dev, &ta_rc_entry, header); + return 0; +} + +static int +udma_dfx_query_ta_latch_entry(struct udma_device *udma_dev, + struct ub_ta_latch_data_outbuf *ta_latch_data, + u32 query_entry_num, u32 jetty_id) +{ + int ret = 0; + u32 remaining_size; + struct ub_ta_rdma_rc_outbuf ta_rc_entry = { 0 }; + u32 entry_size = udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + + if (query_entry_num > UB_TA_RDMARC_MAX_SIZE) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "query_entry_num(%u) is greater than (%u)\n", + query_entry_num, (u32)UB_TA_RDMARC_MAX_SIZE); + return -EINVAL; + } + + for (u32 i = 0; i < query_entry_num; i++) { + ret = udma_cmd_jetty_query_rdma_rc(udma_dev, jetty_id, i, + &ta_rc_entry); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Cmd failed, jetty_id=%u, entry_idx=%u, ret=%d!\n", + jetty_id, i, ret); + return ret; + } + remaining_size = UB_TA_LATCH_DATA_MAX_SIZE - + ta_latch_data->ta_latch_size; + if (entry_size > remaining_size) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Buffer overflow: entry_size(%u) > remaining_size(%u)\n", + entry_size, remaining_size); + return -EINVAL; + } + ret = memcpy_s((void *)(ta_latch_data->ta_latch + + ta_latch_data->ta_latch_size), + remaining_size, (void *)(&ta_rc_entry), + entry_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memcpy_s failed, ret=%d!\n", ret); + return -EFAULT; + } + + ta_latch_data->ta_latch_size += entry_size; + } + return ret; +} + +int udma_dfx_query_ta_latch_data_exe( + struct udma_device *udma_dev, u32 jetty_id, + struct ub_ta_latch_data_outbuf *ta_latch_data) +{ + union ub_latch_data_common_header latch_header = { 0 }; + union ub_latch_data_common_header record_latch_header = { 0 }; + u32 query_entry_num = 0; + u32 entry_size = udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_entry_size; + int ret; + + ret = memset_s(ta_latch_data, sizeof(struct ub_ta_latch_data_outbuf), 0, + sizeof(struct ub_ta_latch_data_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "memset_s failed, ret=%d!\n", ret); + return ret; + } + + /* 1. get lock store of common header */ + ret = udma_dfx_get_ta_latch_header(udma_dev, jetty_id, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get latch header fail, ret=%d!\n", ret); + return ret; + } + + /* 2. according to header information do check */ + if (latch_header.valid == 0 || + latch_header.first_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Latched info is invalid, dw0=0x%08x, first_seq=%u, last_seq=%u, valid=%u, data_len=%u.\n", + latch_header.dw0, latch_header.first_seq, + latch_header.last_seq, latch_header.valid, + latch_header.latch_total_len); + return 0; + } + + /* 3. record the one query of header information */ + record_latch_header.first_seq = latch_header.first_seq; + record_latch_header.last_seq = latch_header.last_seq; + + /* 4. according to header calculate need query of entry count */ + query_entry_num = + (latch_header.latch_total_len + entry_size - 1) / entry_size; + ret = udma_dfx_query_ta_latch_entry(udma_dev, ta_latch_data, + query_entry_num, jetty_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Query latch entry data failed, ret=%d!\n", ret); + return ret; + } + + /* 4. again query once header, determine data valid validity */ + ret = udma_dfx_get_ta_latch_header(udma_dev, jetty_id, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get latch header fail, ret=%d!\n", ret); + return ret; + } + + if (record_latch_header.first_seq != latch_header.first_seq || + record_latch_header.last_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Second query latch header seq not equal, record(%u, %u), (%u, %u).\n", + record_latch_header.first_seq, + record_latch_header.last_seq, latch_header.first_seq, + latch_header.last_seq); + ta_latch_data->ta_latch_size = 0; + } + + return 0; +} + +int udma_dfx_query_ta_latch_data(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, u32 source_type) +{ + int ret; + + /* TPF no has TA lock store */ + if (udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s: TA latch info only non-TPF.\n", __func__); + return -EINVAL; + } + + ret = udma_dfx_query_ta_latch_data_exe(udma_dev, inbuf->jettyid, + &outbuf->ta_latch_data); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get ta latch data fail, ret=%d!\n", ret); + return ret; + } + return 0; +} + +static void udma_dfx_query_ta_latch_print(struct udma_device *udma_dev, int xid) +{ + int ret; + struct ub_ta_latch_data_outbuf *ta_latch_data = + (struct ub_ta_latch_data_outbuf *)devm_kzalloc( + udma_dev->dev, sizeof(struct ub_ta_latch_data_outbuf), + GFP_KERNEL); + + if (ta_latch_data == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "devm_kzalloc failed for ta_latch_data\n"); + return; + } + + (void)memset_s(ta_latch_data, sizeof(struct ub_ta_latch_data_outbuf), 0, + sizeof(struct ub_ta_latch_data_outbuf)); + + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid(%d) ta latch info print.\n", xid); + ret = udma_dfx_query_ta_latch_data_exe(udma_dev, (u32)xid, + ta_latch_data); + if (ret != 0) { + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid=%u ta latch data fail, ret=%d.", xid, ret); + devm_kfree(udma_dev->dev, ta_latch_data); + return; + } + udma_print_byte_no_freq_limit(udma_dev, (u8 *)(ta_latch_data->ta_latch), + ta_latch_data->ta_latch_size); + + devm_kfree(udma_dev->dev, ta_latch_data); + return; +} + +void udma_dump_ta_latch_info(struct udma_device *udma_dev) +{ + if (udma_dev->ub_dev.attr.tp_maintainer) { + return; + } + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + if (udma_print_info_limited( + udma_dev, &ta_latch_limiter)) { /* 10s 3 time of frequency */ + udma_dfx_query_ta_latch_print( + udma_dev, 0); /* 1823 only for for xid 0 do (done) lock store information */ + } + return; + } + + if (udma_print_info_limited(udma_dev, + &ta_latch_limiter)) { /* 10s 3 time of frequency */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TA Start Freq(%d)=========\n", + ta_latch_limiter.print_count); + /* log level is INFO and above next: print first occurrence and non- first occurrence xid 0 ~ 3 */ + if (!udma_kernel_log_level_prior(UDMA_LOG_LEVEL_DEBUG)) { + for (int i = 0; i <= LATCH_LOCK_MAX_XID; i++) { + udma_dfx_query_ta_latch_print(udma_dev, i); + } + return; + } + + /* log level is INFO and above on: print first occurrence xid 0, 2 */ + udma_dfx_query_ta_latch_print(udma_dev, 0); + udma_dfx_query_ta_latch_print(udma_dev, 2); /* 2: xid */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TA End Freq(%d)=========", + ta_latch_limiter.print_count); + } + return; +} + +/* =================================================TP LATCH======================================================= */ +/** + this part is TP lock store query,TP lock store header parse,TP lock store rate-limit print + */ +static void +udma_dfx_parse_tp_latch_header(struct udma_device *udma_dev, + struct ub_tpg_ctx_query *tpg_ctx, + union ub_latch_data_common_header *header) +{ + u32 dw0 = *(u32 *)(tpg_ctx->ctx_req); + udma_dfx_parase_latch_common_header(udma_dev, header, dw0); + return; +} + +static int udma_dfx_query_tp_latch_tpg_ctx(struct udma_device *udma_dev, + u32 tpgn, + struct ub_tpg_ctx_query *tpg_ctx) +{ + int ret = 0; + (void)memset_s(tpg_ctx, sizeof(struct ub_tpg_ctx_query), 0, + sizeof(struct ub_tpg_ctx_query)); + + ret = udma_cmd_tpg_query_ctx_from_cache( + udma_dev, tpgn, tpg_ctx, sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Query tpgn(%u) context failed, ret=%d.\n", tpgn, ret); + return ret; + } + return 0; +} + +static int udma_dfx_query_check_tp_latch_header( + struct udma_device *udma_dev, struct ub_tpg_ctx_query *tpg_ctx, + u32 tpgn, union ub_latch_data_common_header *latch_header) +{ + int ret; + + ret = udma_dfx_query_tp_latch_tpg_ctx(udma_dev, tpgn, tpg_ctx); + if (ret != 0) { + return ret; + } + + udma_dfx_parse_tp_latch_header(udma_dev, tpg_ctx, latch_header); + return 0; +} + +static int +udma_dfx_query_tp_latch_data_exe(struct udma_device *udma_dev, u32 xid, + struct ub_tp_latch_data_outbuf *tp_latch_data) +{ + union ub_latch_data_common_header latch_header = { 0 }; + union ub_latch_data_common_header record_header = { 0 }; + struct ub_tpg_ctx_query tpg_ctx = { 0 }; + u32 start_tpgn = UB_TP_LATCH_START_TPGN( + xid); /* a TP lock store occupy 4 TPG CTX, so need mapping 0:0~3 1:4~7 2:8~11 3:12~15 */ + u32 end_tpgn = UB_TP_LATCH_END_TPGN(xid); + int ret; + + /* non- TPF no has TP lock store */ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s: TP latch info only TPF.\n", __func__); + return -EINVAL; + } + + (void)memset_s(tp_latch_data, sizeof(struct ub_tp_latch_data_outbuf), 0, + sizeof(struct ub_tp_latch_data_outbuf)); + + /* 1. get lock store of common header, and determine valid validity */ + ret = udma_dfx_query_check_tp_latch_header(udma_dev, &tpg_ctx, + start_tpgn, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s:First get tp latch header faild, ret=%d.\n", + __func__, ret); + return ret; + } + + if (latch_header.valid == 0 || + latch_header.first_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Latched tp info is invalid, valid=%u, first_seq=%u, last_seq=%u.\n", + latch_header.valid, latch_header.first_seq, + latch_header.last_seq); + return 0; + } + record_header.first_seq = latch_header.first_seq; + record_header.last_seq = latch_header.last_seq; + + /* 2. save the a TPG CTX */ + (void)memcpy_s((void *)(tp_latch_data->tp_latch + + tp_latch_data->tp_latch_size), + sizeof(struct ub_tpg_ctx_query), (void *)(&tpg_ctx), + sizeof(struct ub_tpg_ctx_query)); + tp_latch_data->tp_latch_size += sizeof(struct ub_tpg_ctx_query); + + /* 3. query its remaining剩 next 3 TPG CTX */ + for (u32 i = start_tpgn + 1; i < end_tpgn; i++) { + ret = udma_dfx_query_tp_latch_tpg_ctx(udma_dev, i, &tpg_ctx); + if (ret != 0) { + return ret; + } + (void)memcpy_s((void *)(tp_latch_data->tp_latch + + tp_latch_data->tp_latch_size), + sizeof(struct ub_tpg_ctx_query), + (void *)(&tpg_ctx), + sizeof(struct ub_tpg_ctx_query)); + tp_latch_data->tp_latch_size += sizeof(struct ub_tpg_ctx_query); + } + + /* 4. again query once header, determine data valid validity */ + ret = udma_dfx_query_check_tp_latch_header(udma_dev, &tpg_ctx, + start_tpgn, &latch_header); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "%s:Second get tp latch header faild, ret=%d.\n", + __func__, ret); + return ret; + } + + if (record_header.first_seq != latch_header.first_seq || + record_header.last_seq != latch_header.last_seq) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Second query tp latch header seq not equal, record(%u, %u), cur(%u, %u).\n", + record_header.first_seq, record_header.last_seq, + latch_header.first_seq, latch_header.last_seq); + tp_latch_data->tp_latch_size = 0; + } + + return 0; +} + +int udma_dfx_query_tp_latch_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_tp_latch_data_exe(udma_dev, inbuf->tpg_id, + &outbuf->tp_latch_data); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "Get tp latch data fail, ret=%d.\n", ret); + return ret; + } + + return 0; +} + +static void udma_dfx_query_tp_latch_print(struct udma_device *udma_dev, u32 xid) +{ + int ret; + + struct ub_tp_latch_data_outbuf *tp_latch_data = + (struct ub_tp_latch_data_outbuf *)devm_kzalloc( + udma_dev->dev, sizeof(struct ub_tp_latch_data_outbuf), + GFP_KERNEL); + + if (tp_latch_data == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "devm_kzalloc failed for tp_latch_data\n"); + return; + } + + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid(%u) tp latch info print.\n", xid); + ret = udma_dfx_query_tp_latch_data_exe(udma_dev, xid, tp_latch_data); + if (ret != 0) { + udma_info(udma_dev->dev, UDMA_M_DFX, + "Query xid=%u tp latch data fail, ret=%d.", xid, ret); + devm_kfree(udma_dev->dev, tp_latch_data); + return; + } + udma_print_byte_no_freq_limit(udma_dev, (u8 *)(tp_latch_data->tp_latch), + tp_latch_data->tp_latch_size); + devm_kfree(udma_dev->dev, tp_latch_data); + return; +} + +void udma_dump_tp_latch_info(struct udma_device *udma_dev) +{ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return; + } + + if (udma_print_info_limited(udma_dev, + &tp_latch_limiter)) { /* 10s 3 time of frequency */ + /* log level is INFO and above next: print first occurrence and non- first occurrence xid 0 ~ 3 i.e.: TPG 0 ~ 15 */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TP Start Freq(%d)=========", + tp_latch_limiter.print_count); + if (!udma_kernel_log_level_prior(UDMA_LOG_LEVEL_DEBUG)) { + for (u32 i = 0; i <= LATCH_LOCK_MAX_XID; i++) { + udma_dfx_query_tp_latch_print(udma_dev, i); + } + return; + } + + /* log level is INFO and above on: print first occurrence xid 0, 2 */ + udma_dfx_query_tp_latch_print(udma_dev, 0); + udma_dfx_query_tp_latch_print(udma_dev, 2); /* 2: xid */ + udma_err(udma_dev->dev, UDMA_M_DFX, + "=========TP End Freq(%d)=========", + tp_latch_limiter.print_count); + } +} + +/* =================================================CLEAR LATCH====================================================== */ +/** + this part is TA/TP lock store clear + */ +int udma_dfx_ub_modify_clear_ta_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret; + + /* TA lock store only has non- TPF device */ + if (udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TA latch clear only non-TPF.\n", __func__); + return -EINVAL; + } + + ret = udma_cmd_jetty_clear_dfx_lock_ctx( + udma_dev, 0); /* one clear all of TA lock store, default jetty_id fill 0 */ + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: clear ta latch lock failed, ret(%d)\n", __func__, + ret); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TA latch clear Successful.\n", __func__); + return 0; +} + +int udma_dfx_ub_modify_clear_tp_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret; + + /* TP lock store only has TPF device */ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TP latch clear only HI1825 TPF.\n", __func__); + return -EINVAL; + } + + ret = udma_cmd_clear_tp_latch_lock( + udma_dev, 0); /* one clear all of TP lock store, default xid fill 0 */ + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: clear latch lock failed, ret(%d)\n", __func__, + ret); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_CQM_MOD_DFX, + "%s: TP latch clear Successful.\n", __func__); + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h new file mode 100644 index 000000000..9b3066df2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_latch.h @@ -0,0 +1,34 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_DFX_LATCH_H +#define UDMA_DFX_LATCH_H + +#include <linux/device.h> + +#include "udma_common.h" +#include "udma_dfx.h" +#include "udma_cmd_com.h" + +#define LATCH_LOCK_MAX_XID 3 + +int udma_dfx_query_ta_latch_check(struct udma_device *udma_dev, + const struct ub_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args); +int udma_dfx_query_ta_latch_data(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf, + u32 source_type); +int udma_dfx_query_tp_latch_data(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size); +void udma_dump_ta_latch_info(struct udma_device *udma_dev); +void udma_dump_tp_latch_info(struct udma_device *udma_dev); +int udma_dfx_ub_modify_clear_ta_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +int udma_dfx_ub_modify_clear_tp_latch_lock(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size); +#endif //UDMA_DFX_LATCH_H \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c new file mode 100644 index 000000000..cd085a956 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.c @@ -0,0 +1,649 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_cmd_event.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_tp.h" +#include "udma_resource.h" +#include "udma_tp.h" +#include "udma_jetty.h" +#include "udma_dfx_latch.h" +#include "udma_dfx.h" + +static int udma_dfx_modify_tp(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp failed.\n"); + return ret; + } + return udma_dfx_modify_tp_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_modify_tp_q_depth(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + if (inbuf->q_depth == 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "q_depth cannot be 0.\n"); + return -EINVAL; + } + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_q_depth failed, ret= %d.\n", ret); + return ret; + } + + ret = udma_sysfs_set_tp_depth(&udma_dev->sysfs_res, inbuf->q_depth); + if (ret < 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "q_depth(%u) set failed or depth is not a power of 2, ret: %d.\n", + inbuf->q_depth, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_modify_tp_mtu_1k_en(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_mtu_1k_en failed, ret= %d.\n", + ret); + return ret; + } + udma_dev->mtu_1k_en = inbuf->mtu_1k_en; + return ret; +} + +static int udma_dfx_modify_max_jetty(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_get_cfg_cap_cmd cmd = { 0 }; + struct ub_get_cfg_cap_resp resp = { 0 }; + u16 resp_size = sizeof(resp); + int ret; + + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "tpf not allow modify max jetty num\n"); + return -EINVAL; + } + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_max_jetty failed, ret= %d.\n", ret); + return ret; + } + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_DFX, "modify max jetty func_id:%u\n", + cmd.func_id); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CFG_CAP, &cmd, sizeof(cmd), + &resp, &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EINVAL; + } + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty = inbuf->max_jetty_num; + if (udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty < + resp.udma_res.max_jfrc_num) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "max_jetty_num(%u) less than max_jfrc_num(%u)\n", + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty, + resp.udma_res.max_jfrc_num); + return -EINVAL; + } + udma_dev->ub_cap.dev_cap.max_jetty = + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty - + resp.udma_res.max_jfrc_num; + udma_info(udma_dev->dev, UDMA_M_DFX, "sdk max_jetty:%u\n", + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + return ret; +} + +static int udma_dfx_modify_jetty(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "The version (%d) does not support ub_modify jetty cmd.\n", + udma_dev->device_type); + return -EINVAL; + } + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_mtu_1k_en failed, ret= %d.\n", + ret); + return ret; + } + + return udma_dfx_modify_jetty_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_cc_set_para(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_cc_params para = { 0 }; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "set cc parameters failed, fail to prepare buffers.\n"); + return ret; + } + + para.ub_cc_algo = inbuf->cc_params.ub_cc_algo; + para.init_wnd = inbuf->cc_params.init_wnd; + para.alpha = inbuf->cc_params.alpha; + para.beta = inbuf->cc_params.beta; + para.gamma = inbuf->cc_params.gamma; + para.eta = inbuf->cc_params.eta; + + ret = udma_cc_set_para(udma_dev, &para); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set cc parameters failed.\n"); + } + + return ret; +} + +static int udma_dfx_modify_tp_srp_bitmap(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_srp_bitmap failed, ret= %d.\n", + ret); + return ret; + } + ret = udma_cmd_tp_modify_tp_srp_bitmap(udma_dev, inbuf->tp_srp_index, + inbuf->bitmap_index, + inbuf->bitmap_mode); + + return ret; +} + +static int udma_dfx_modify_tp_srp_ctr_bitmap(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_modify_tp_srp_ctr_bitmap failed, ret= %d.\n", + ret); + return ret; + } + ret = udma_cmd_tp_modify_srp_ctr_bitmap( + udma_dev, inbuf->ctr_bitmap_index, inbuf->bitmap_mode); + + return ret; +} + +static int udma_cmd_dfx_qos_set_func_vnic_mapping(struct udma_device *udma_dev, + const void *buf_in, + u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_qos_cmd qos_cmd = { 0 }; + u16 cmd_out_size = sizeof(struct ub_qos_cmd); + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "set func vnic mapping failed, fail to prepare buffers, ret=%d!\n", + ret); + return ret; + } + + qos_cmd.func_id = (u16)inbuf->qos_params.func_id; + qos_cmd.vnic_id = (u16)inbuf->qos_params.vnic_id; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_QOS_FUNC_MAPPING_SET, &qos_cmd, + sizeof(struct ub_qos_cmd), &qos_cmd, + &cmd_out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_out_size == 0) || (qos_cmd.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "failed to set func vnic mapping, ret:%d, status:0x%x, outsize: %u", + ret, qos_cmd.head.status, cmd_out_size); + return -EIO; + } + + return 0; +} + +static int udma_cmd_dfx_qos_set_bps(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_qos_cmd qos_cmd = { 0 }; + u16 cmd_out_size = sizeof(struct ub_qos_cmd); + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set bps failed, fail to prepare buffers, ret=%d!\n", + ret); + return ret; + } + + qos_cmd.vnic_id = (u16)inbuf->qos_params.vnic_id; + qos_cmd.xir_type = (u16)inbuf->qos_params.xir_type; + qos_cmd.bps = (u32)inbuf->qos_params.bps; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_QOS_BPS_SET, &qos_cmd, + sizeof(struct ub_qos_cmd), &qos_cmd, + &cmd_out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_out_size == 0) || (qos_cmd.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to set bps, ret:%d, status:0x%x, outsize: %u", + ret, qos_cmd.head.status, cmd_out_size); + return -EIO; + } + + return 0; +} + +static int udma_cmd_dfx_qos_set_pps(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + struct ub_qos_cmd qos_cmd = { 0 }; + u16 cmd_out_size = sizeof(struct ub_qos_cmd); + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set pps failed, fail to prepare buffers, ret=%d!\n", + ret); + return ret; + } + + qos_cmd.vnic_id = (u16)inbuf->qos_params.vnic_id; + qos_cmd.xir_type = (u16)inbuf->qos_params.xir_type; + qos_cmd.pps = (u32)inbuf->qos_params.pps; + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_QOS_PPS_SET, &qos_cmd, + sizeof(struct ub_qos_cmd), &qos_cmd, + &cmd_out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_out_size == 0) || (qos_cmd.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to set pps, ret:%d, status:0x%x, outsize: %u", + ret, qos_cmd.head.status, cmd_out_size); + return -EIO; + } + + return 0; +} + +static int udma_dfx_set_multi_mode(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + ub_set_multi_mode_cmd_s para = { 0 }; + u16 out = (u16)sizeof(para); + int ret; + + para.multi_path_mode = inbuf->multi_path_mode; + para.func_id = inbuf->func_id; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_MULTI_PATH_MODE, &para, + (u16)sizeof(para), &para, &out, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out == 0) || (para.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_MISC, + "set multi path mode failed. ret: %d, status(0x%x)\n", + ret, para.head.status); + return -EINVAL; + } + + return ret; +} + +static void fill_modify_func_table_attr(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + ub_dfx_modify_func_table_cmd_s *cmd) +{ + if (inbuf->func_table_modify_attr.dw15_flag.debug_en == 1) { + cmd->func_table_modify_attr.bs.debug_en = + inbuf->func_table_modify_attr.dw15.debug_en; + cmd->func_table_modify_attr.bs_flag.debug_en = + inbuf->func_table_modify_attr.dw15_flag.debug_en; + } + if (inbuf->func_table_modify_attr.dw15_flag.lwb_en == 1) { + cmd->func_table_modify_attr.bs.lwb_en = + inbuf->func_table_modify_attr.dw15.lwb_en; + cmd->func_table_modify_attr.bs_flag.lwb_en = + inbuf->func_table_modify_attr.dw15_flag.lwb_en; + } + if (inbuf->func_table_modify_attr.dw15_flag.traffic_en == 1) { + cmd->func_table_modify_attr.bs.traffic_en = + inbuf->func_table_modify_attr.dw15.traffic_en; + cmd->func_table_modify_attr.bs_flag.traffic_en = + inbuf->func_table_modify_attr.dw15_flag.traffic_en; + } + if ((udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) && + (inbuf->func_table_modify_attr.dw15_flag.ub_link_en == 1)) { + cmd->func_table_modify_attr.bs.ub_link_en = + inbuf->func_table_modify_attr.dw15.ub_link_en; + cmd->func_table_modify_attr.bs_flag.ub_link_en = + inbuf->func_table_modify_attr.dw15_flag.ub_link_en; + } + if (inbuf->func_table_modify_attr.dw15_flag.mtt_pro_en == 1) { + cmd->func_table_modify_attr.bs.mtt_pro_en = + inbuf->func_table_modify_attr.dw15.mtt_pro_en; + cmd->func_table_modify_attr.bs_flag.mtt_pro_en = + inbuf->func_table_modify_attr.dw15_flag.mtt_pro_en; + cmd->func_id = (u16)inbuf->func_id; + } +} + +static int udma_dfx_set_func_table(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + ub_dfx_modify_func_table_cmd_s cmd = { 0 }; + u16 out = (u16)sizeof(cmd); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + fill_modify_func_table_attr(udma_dev, inbuf, &cmd); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_SET_FUNC_TABLE, &cmd, + (u16)sizeof(cmd), &cmd, &out, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (out == 0) || (cmd.head.status != 0)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set func table failed. ret: %d, status(0x%x)\n", ret, + cmd.head.status); + return -EINVAL; + } + + return ret; +} + +static int udma_dfx_set_device(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret = 0; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set device failed, fail to prepare buffers.\n"); + return ret; + } + + udma_err(udma_dev->dev, UDMA_M_DFX, + "slice_cfg_set:%d, slice_cfg_en:%d\n", inbuf->slice_cfg_set, + inbuf->slice_cfg_en); + if (inbuf->slice_cfg_set == 1) { + udma_dev->slice_cfg_en = inbuf->slice_cfg_en; + } + + return 0; +} + +static int udma_dfx_clear_io_stat(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + int ret = 0; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, "fail to clear.\n"); + return ret; + } + + atomic64_set(&udma_dev->udma_driver_device_ctr.post_send_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.post_recv_wr_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_total_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_sq_num, 0); + atomic64_set(&udma_dev->udma_driver_device_ctr.poll_rq_num, 0); + return 0; +} + +static int udma_dfx_set_cfg_cos_en(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "set cos enable failed, fail to prepare buffers.\n"); + return ret; + } + + udma_dev->default_cos.cfg_cos_en = inbuf->cfg_cos_en; + + return ret; +} + +static struct udma_dfx_query_cmd_func g_udma_modify_cmd_funcs[] = { + /* modify */ + { UB_MODIFY_TYPE_TP, 0, 0, udma_dfx_modify_tp }, + { UB_MODIFY_TYPE_TP, UB_MODIFY_SUB_TYPE_Q_DEPTH, 0, + udma_dfx_modify_tp_q_depth }, + { UB_MODIFY_TYPE_TP, UB_MODIFY_SUB_TYPE_MTU_1K_EN, 0, + udma_dfx_modify_tp_mtu_1k_en }, + { UB_MODIFY_TYPE_CC, 0, 0, udma_dfx_cc_set_para }, + { UB_MODIFY_TYPE_TPG_MULTIPATH, 0, 0, udma_cmd_tpg_modify_multipath }, + { UB_MODIFY_TYPE_TP_SRP, UB_MODIFY_SUB_TYPE_SRP_BITMAP, 0, + udma_dfx_modify_tp_srp_bitmap }, + { UB_MODIFY_TYPE_TP_SRP, UB_MODIFY_SUB_TYPE_SRP_CTR_BITMAP, 0, + udma_dfx_modify_tp_srp_ctr_bitmap }, + { UB_MODIFY_TYPE_AEQE, 0, 0, udma_cmd_dfx_aeqe_inject }, + { UB_MODIFY_TYPE_DROP_AEQE, 0, 0, udma_cmd_dfx_aeqe_drop }, + { UB_MODIFY_TYPE_QOS, UB_MODIFY_SUB_TYPE_FUNC_MAPPING, 0, + udma_cmd_dfx_qos_set_func_vnic_mapping }, + { UB_MODIFY_TYPE_QOS, UB_MODIFY_SUB_TYPE_BPS, 0, + udma_cmd_dfx_qos_set_bps }, + { UB_MODIFY_TYPE_QOS, UB_MODIFY_SUB_TYPE_PPS, 0, + udma_cmd_dfx_qos_set_pps }, + { UB_MODIFY_TYPE_MUL_MODE, 0, 0, udma_dfx_set_multi_mode }, + { UB_MODIFY_TYPE_FUNC_TABLE, 0, 0, udma_dfx_set_func_table }, + { UB_MODIFY_TYPE_CFG_COS_EN, 0, 0, udma_dfx_set_cfg_cos_en }, + { UB_MODIFY_TYPE_DEVICE, 0, 0, udma_dfx_set_device }, + { UB_MODIFY_TYPE_DEVICE, UB_MODIFY_SUB_TYPE_CLEAR_IO_STAT, 0, + udma_dfx_clear_io_stat }, + { UB_MODIFY_TYPE_DEVICE, UB_MODIFY_SUB_TYPE_MAX_JETTY, 0, + udma_dfx_modify_max_jetty }, + { UB_MODIFY_TYPE_JETTY, 0, 0, udma_dfx_modify_jetty }, + { UB_MODIFY_TYPE_LATCH_LOCK, UB_MODIFY_SUB_TYPE_CLEAR_TA_LATCH_LOCK, 0, + udma_dfx_ub_modify_clear_ta_latch_lock }, + { UB_MODIFY_TYPE_LATCH_LOCK, UB_MODIFY_SUB_TYPE_CLEAR_TP_LATCH_LOCK, 0, + udma_dfx_ub_modify_clear_tp_latch_lock }, +}; + +static bool +udma_dfx_check_device_notmatch_for_modify(const struct udma_device *udma_dev, + u32 cmd_type, u32 sub_type) +{ + static const u32 tpf_obj_modify[] = { UB_MODIFY_TYPE_TP, + UB_MODIFY_TYPE_TPG_MULTIPATH, + UB_MODIFY_TYPE_TP_SRP }; + bool ret = false; + u32 i = 0; + /* determine tp/tpg/utp must not be on pf device on */ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + for (i = 0; i < ARRAY_SIZE(tpf_obj_modify); i++) { + if (cmd_type == tpf_obj_modify[i]) { + ret = true; + break; + } + } + } + return ret; +} + +static bool udma_modify_cmd_in_white_list(u32 in_cmd_type) +{ + return (in_cmd_type == + ub_dfx_get_cmd_type(UB_MODIFY_TYPE_FUNC_TABLE, 0, 0) || + in_cmd_type == ub_dfx_get_cmd_type(UB_MODIFY_TYPE_TP, + UB_MODIFY_SUB_TYPE_MTU_1K_EN, + 0) || + in_cmd_type == + ub_dfx_get_cmd_type(UB_MODIFY_TYPE_CFG_COS_EN, 0, 0) || + in_cmd_type == + ub_dfx_get_cmd_type(UB_MODIFY_TYPE_DEVICE, 0, 0) || + in_cmd_type == ub_dfx_get_cmd_type( + UB_MODIFY_TYPE_LATCH_LOCK, + UB_MODIFY_SUB_TYPE_CLEAR_TA_LATCH_LOCK, + 0) || + in_cmd_type == ub_dfx_get_cmd_type( + UB_MODIFY_TYPE_LATCH_LOCK, + UB_MODIFY_SUB_TYPE_CLEAR_TP_LATCH_LOCK, + 0)); +} + +int udma_verify_modify_func(struct udma_device *udma_dev, u32 in_cmd_type) +{ + int npu_version = -1; + int ret; + + /* npu_version:0, means release version; npu_version:1, means debug version */ + ret = udma_query_npu_dfx_version(udma_dev, &npu_version); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "UDMA fail to query version.\n"); + return ret; + } + + if (npu_version == 0 && !udma_modify_cmd_in_white_list(in_cmd_type)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "release version not support modify cmd type=%d.\n", + in_cmd_type); + return -EINVAL; + } + + return 0; +} + +int udma_dfx_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_query_inbuf *modify_inbuf = (struct ub_query_inbuf *)buf_in; + udma_ioctl_func_t func = NULL; + u32 cmd_type; + int ret = -EINVAL; + u32 i; + + /* check in buf size */ + if (in_size < (u32)sizeof(struct ub_query_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma dfx modify buffer in size too small\n"); + return -EINVAL; + } + + ret = udma_verify_modify_func(udma_dev, modify_inbuf->cmd_type); + if (ret != 0) { + udma_pr_err(UDMA_M_DFX, "UDMA dfx modfiy cmd verify failed.\n"); + return ret; + } + + for (i = 0; i < sizeof(g_udma_modify_cmd_funcs) / + sizeof(g_udma_modify_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type, + g_udma_modify_cmd_funcs[i].source_type); + if (cmd_type == modify_inbuf->cmd_type) { + if (udma_dfx_check_device_notmatch_for_modify( + udma_dev, + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "call modify func, cmd type(%u), sub_type(%u) and device not match, check the device name is tpf or pf.\n", + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type); + break; + } + func = g_udma_modify_cmd_funcs[i].cmd_func; + if (func != NULL) { + ret = func(udma_dev, buf_in, in_size, buf_out, + out_size); + udma_dfx_print( + "call modify func, cmd_type:%u, sub_type:%u, source_type:%u\n", + g_udma_modify_cmd_funcs[i].cmd_type, + g_udma_modify_cmd_funcs[i].sub_type, + g_udma_modify_cmd_funcs[i].source_type); + } + break; + } + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h new file mode 100644 index 000000000..0a6479487 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_modify.h @@ -0,0 +1,15 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_MODIFY_H +#define UDMA_DFX_MODIFY_H + +#include <linux/types.h> +#include "udma_common.h" + +int udma_dfx_modify_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int udma_verify_modify_func(struct udma_device *udma_dev, u32 in_cmd_type); + +#endif /* UDMA_DFX_MODIFY_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c new file mode 100644 index 000000000..d69fd1013 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.c @@ -0,0 +1,108 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> + +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "ub_mpu_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_perf_ctr.h" + +static int udma_read_perf_counter(struct udma_device *udma_dev, + ub_dfx_perf_ctr_cmd_inbuf_s *cmd_in, + struct ub_perf_ctr_outbuf *cmd_ret) +{ + u16 out_size = (u16)sizeof(struct ub_perf_ctr_outbuf); + int ret; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_READ_PERF_CTR, cmd_in, + sizeof(ub_dfx_perf_ctr_cmd_inbuf_s), + cmd_ret, &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_ret->head.status != 0) { + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "read perf counter failed, err(%d), status(0x%x), " + "out size(0x%x)\n", + ret, cmd_ret->head.status, out_size); + return -EFAULT; + } + + return 0; +} + +int udma_dfx_query_perf_ctr(struct udma_device *udma_dev, + const struct ub_perf_ctr_inbuf *inbuf, + struct ub_perf_ctr_outbuf *outbuf) + +{ + struct ub_perf_ctr_outbuf *ctr_cur = outbuf; + struct ub_dfx_perf_ctr_cmd_inbuf cmd_in = { 0 }; + int ret; + + cmd_in.gap_index = inbuf->gap_index; + ret = udma_read_perf_counter(udma_dev, &cmd_in, ctr_cur); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query perf ctr for realtime: read counter failed, ret(%d)\n", + ret); + return -EFAULT; + } + + return 0; +} + +static struct udma_dfx_perf_ctr_cmd_func g_udma_cmd_funcs[] = { + { UB_PERF_CMD_TYPE, udma_dfx_query_perf_ctr }, +}; + +int udma_dfx_perf_ctr_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_perf_ctr_inbuf *inbuf = (struct ub_perf_ctr_inbuf *)buf_in; + struct ub_perf_ctr_outbuf *outbuf = + (struct ub_perf_ctr_outbuf *)buf_out; + udma_dfx_perf_ctr_func_t func = NULL; + u32 cmd_type; + u32 i; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_perf_ctr_inbuf)) { + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx perf ctr buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(struct ub_perf_ctr_outbuf)) { + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx perf ctr buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_cmd_funcs) / sizeof(g_udma_cmd_funcs[0]); + i++) { + cmd_type = g_udma_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + *out_size = sizeof(struct ub_perf_ctr_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + return -EINVAL; +} \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h new file mode 100644 index 000000000..e5fc07e23 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_ctr.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_PERF_CTR_H +#define UDMA_DFX_PERF_CTR_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_dfx_perf_ctr_func_t)(struct udma_device *udma_dev, + const struct ub_perf_ctr_inbuf *inbuf, + struct ub_perf_ctr_outbuf *outbuf); + +struct udma_dfx_perf_ctr_cmd_func { + u32 cmd_id; + udma_dfx_perf_ctr_func_t cmd_func; +}; + +int udma_dfx_perf_ctr_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_PERF_CTR_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c new file mode 100644 index 000000000..0e69f69b8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.c @@ -0,0 +1,152 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> +#include "hinic5_hw.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_segment.h" +#include "udma_rc_table.h" +#include "udma_tpg.h" +#include "udma_vtp.h" +#include "udma_tp.h" +#include "ub_pub_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_perf_query.h" + +static int udma_check_device_ver(struct udma_device *udma_dev, + const struct ub_perf_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args) +{ + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "device type(%d) does not support the command for performance query.", + udma_dev->device_type); + return -EINVAL; + } + return 0; +} + +static void udma_dfx_fill_perf_stat_outbuf(u8 clear_flag, + struct ub_perf_query_outbuf *buf_out, + struct cmdq_rtt *rtt) +{ + buf_out->dispatch_times = (u64)atomic64_read(&rtt->times); + buf_out->cmdq_rtt_sum = (u64)atomic64_read(&rtt->cmdq_rtt_sum); + buf_out->cmdq_npu_rtt_sum = (u64)atomic64_read(&rtt->cmdq_npu_rtt_sum); + if (clear_flag != 0) { + atomic64_set(&rtt->times, 0); + atomic64_set(&rtt->cmdq_rtt_sum, 0); + atomic64_set(&rtt->cmdq_npu_rtt_sum, 0); + } +} + +static int udma_dfx_perf_query_stat(struct udma_device *udma_dev, + const struct ub_perf_query_inbuf *buf_in, + struct ub_perf_query_outbuf *buf_out) +{ + u32 offset = 0; + void *node = NULL; + switch (buf_in->cmd_type) { + case UB_QUERY_TYPE_JETTY: + node = (void *)udma_find_jetty(udma_dev, buf_in->xid); + offset = offsetof(struct udma_jetty, cmdq_rtt); + break; + case UB_QUERY_TYPE_JFS: + node = (void *)udma_find_jfs(udma_dev, buf_in->xid); + offset = offsetof(struct udma_jfs, cmdq_rtt); + break; + case UB_QUERY_TYPE_JFR: + node = (void *)udma_find_jfr(udma_dev, buf_in->xid); + offset = offsetof(struct udma_jfr, cmdq_rtt); + break; + case UB_QUERY_TYPE_RC_QUEUE: + node = (void *)udma_find_rcq(udma_dev, buf_in->xid); + offset = offsetof(struct udma_rc_queue, cmdq_rtt); + break; + case UB_QUERY_TYPE_SEG: + node = (void *)udma_seg_find(udma_dev, buf_in->xid); + offset = offsetof(struct udma_segment, cmdq_rtt); + break; + case UB_QUERY_TYPE_TPG: + node = (void *)udma_get_tpg_by_id(udma_dev, buf_in->xid); + offset = offsetof(struct udma_tpg, cmdq_rtt); + break; + case UB_QUERY_TYPE_VTP: + node = (void *)udma_get_vtp_by_id(udma_dev, buf_in->xid, + buf_in->vfid); + offset = offsetof(struct udma_vtp, cmdq_rtt); + break; + case UB_QUERY_TYPE_TP: + node = (void *)udma_find_tp(udma_dev, buf_in->xid); + offset = offsetof(struct udma_tp, cmdq_rtt); + break; + default: + udma_err(udma_dev->dev, UDMA_M_DFX, "invalid cmd type:%u\n", + buf_in->cmd_type); + return -EINVAL; + } + if (node == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "query perf stat fail, cmd_type:%u, id:%u\n", + buf_in->cmd_type, buf_in->xid); + return -EINVAL; + } + udma_dfx_fill_perf_stat_outbuf(buf_in->clear_flag, buf_out, + (struct cmdq_rtt *)(node + offset)); + udma_dfx_obj_put(udma_dev, buf_in->cmd_type, node, buf_in->xid); + return 0; +} + +static struct udma_dfx_perf_query_cmd_func g_udma_perf_query_cmd_funcs[] = { + { UB_QUERY_TYPE_JETTY, udma_check_device_ver, + udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_JFS, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_JFR, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_RC_QUEUE, udma_check_device_ver, + udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_SEG, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_TPG, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_VTP, udma_check_device_ver, udma_dfx_perf_query_stat }, + { UB_QUERY_TYPE_TP, udma_check_device_ver, udma_dfx_perf_query_stat }, +}; + +int udma_dfx_perf_query_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_perf_query_inbuf *inbuf = + (struct ub_perf_query_inbuf *)buf_in; + struct ub_perf_query_outbuf *outbuf = + (struct ub_perf_query_outbuf *)buf_out; + udma_dfx_perf_query_func_t func = NULL; + u32 cmd_type; + u32 i; + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_perf_query_cmd_funcs) / + sizeof(g_udma_perf_query_cmd_funcs[0]); + i++) { + cmd_type = g_udma_perf_query_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_perf_query_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + *out_size = sizeof(struct ub_perf_query_outbuf); + + return func(udma_dev, inbuf, outbuf); + } + } + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h new file mode 100644 index 000000000..ec1e450eb --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_perf_query.h @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_DFX_PERF_QUERY_H +#define UDMA_DFX_PERF_QUERY_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" +#include "udma_dfx.h" + +typedef int (*udma_dfx_perf_query_func_t)( + struct udma_device *udma_dev, const struct ub_perf_query_inbuf *inbuf, + struct ub_perf_query_outbuf *outbuf); +typedef int (*udma_perf_query_check_func_t)( + struct udma_device *udma_dev, const struct ub_perf_query_inbuf *buf_in, + const struct udma_dfx_check_args *check_args); + +struct udma_dfx_perf_query_cmd_func { + u32 cmd_id; + udma_perf_query_check_func_t check_func; + udma_dfx_perf_query_func_t cmd_func; +}; + +int udma_dfx_perf_query_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_PERF_QUERY_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c new file mode 100644 index 000000000..319d7472d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.c @@ -0,0 +1,182 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/highmem.h> + +#include "ub_mpu_cmd.h" +#include "comm_defs.h" +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "udma_dfx.h" +#include "udma_dfx_port_sta.h" + +static int udma_read_port_counter(struct udma_device *udma_dev, + ub_dfx_port_statistics_cmd_inbuf_s *cmd_in, + struct ub_dfx_port_statistics_cmd *cmd_ret) +{ + u16 out_size = (u16)sizeof(struct ub_dfx_port_statistics_cmd); + int ret; + + ret = hinic5_msg_to_mgmt_sync( + udma_dev->hwdev, HINIC5_MOD_UB, UB_MPU_CMD_DFX_READ_PORT_STA, + cmd_in, sizeof(ub_dfx_port_statistics_cmd_inbuf_s), cmd_ret, + &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_ret->head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "read port statistics counter failed, err(%d), status(0x%x), " + "out size(0x%x)\n", + ret, cmd_ret->head.status, out_size); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "port traffic query counter success from mpu\n"); + + return 0; +} + +static int udma_read_vf_counter(struct udma_device *udma_dev, + ub_dfx_vf_statistics_cmd_inbuf_s *cmd_in, + struct ub_dfx_port_statistics_cmd *cmd_ret) +{ + u16 out_size = (u16)sizeof(struct ub_dfx_port_statistics_cmd); + int ret; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_DFX_READ_VF_STA, cmd_in, + sizeof(ub_dfx_vf_statistics_cmd_inbuf_s), + cmd_ret, &out_size, 0, HINIC5_CHANNEL_UB); + if (ret != 0 || out_size == 0 || cmd_ret->head.status != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "read vf statistics counter failed, err(%d), status(0x%x), " + "out size(0x%x)\n", + ret, cmd_ret->head.status, out_size); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "vf traffic query counter success from mpu\n"); + + return 0; +} + +int udma_dfx_query_port_sta_realtime(struct udma_device *udma_dev, + const struct ub_port_sta_inbuf *inbuf, + union ub_port_sta_outbuf *outbuf) +{ + struct ub_dfx_port_statistics_cmd *port_traffic = &outbuf->port_traffic; + ub_dfx_port_statistics_cmd_inbuf_s cmd_in = { 0 }; + int ret; + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query port traffic port_id(%u) cos(%u)\n", inbuf->port_id, + inbuf->cos); + cmd_in.port_id = inbuf->port_id; + cmd_in.cos = inbuf->cos; + ret = udma_read_port_counter(udma_dev, &cmd_in, port_traffic); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query port traffic for realtime: read counter failed, ret(%d)\n", + ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query port traffic for realtime success\n"); + + return 0; +} + +int udma_dfx_query_vf_sta_realtime(struct udma_device *udma_dev, + const struct ub_port_sta_inbuf *inbuf, + union ub_port_sta_outbuf *outbuf) +{ + struct ub_dfx_port_statistics_cmd *port_traffic = &outbuf->port_traffic; + struct ub_dfx_vf_statistics_cmd_inbuf cmd_in = { 0 }; + int ret; + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query vf traffic vf_id(%u)\n", inbuf->vf_id); + cmd_in.vf_id = inbuf->vf_id; + ret = udma_read_vf_counter(udma_dev, &cmd_in, port_traffic); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query vf traffic for realtime: read counter failed, ret(%d)\n", + ret); + return -EFAULT; + } + + udma_info(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "query vf traffic for realtime success\n"); + + return 0; +} + +static struct udma_dfx_port_sta_cmd_func g_udma_port_cmd_funcs[] = { + { UB_PORT_STA_RTCB, udma_dfx_query_port_sta_realtime }, + { UB_VF_STA_RTCB, udma_dfx_query_vf_sta_realtime }, +}; + +int udma_dfx_port_sta_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_port_sta_inbuf *inbuf = (struct ub_port_sta_inbuf *)buf_in; + union ub_port_sta_outbuf *outbuf = (union ub_port_sta_outbuf *)buf_out; + udma_dfx_port_sta_func_t func = NULL; + u32 cmd_type; + u32 i; + int ret; + + /* check in and out buf size */ + if (in_size < (u32)sizeof(struct ub_port_sta_inbuf)) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx query port/vf sta realtime buffer in size too small\n"); + return -EINVAL; + } + if (*out_size < (u32)sizeof(union ub_port_sta_outbuf)) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "udma dfx query port/vf sta realtime buffer out size too small\n"); + return -EINVAL; + } + + *out_size = 0; + for (i = 0; i < sizeof(g_udma_port_cmd_funcs) / + sizeof(g_udma_port_cmd_funcs[0]); + i++) { + cmd_type = g_udma_port_cmd_funcs[i].cmd_id; + if (cmd_type == inbuf->cmd_type) { + func = g_udma_port_cmd_funcs[i].cmd_func; + if (func == NULL) { + udma_err( + udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: function of cmd(0x%x) is not supported\n", + __func__, cmd_type); + return -EINVAL; + } + + ret = func(udma_dev, inbuf, outbuf); + if (ret != 0) { + *out_size = 0; + return ret; + } + + *out_size = sizeof(union ub_port_sta_outbuf); + return 0; + } + } + + udma_err(udma_dev->dev, UDMA_M_PORT_STA_DFX, + "%s: cmd(0x%x) is not exist\n", __func__, cmd_type); + + return -EINVAL; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h new file mode 100644 index 000000000..d3117916a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_port_sta.h @@ -0,0 +1,24 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_PORT_STA_H +#define UDMA_DFX_PORT_STA_H + +#include <linux/types.h> +#include "ub_pub_cmd.h" +#include "udma_common.h" + +typedef int (*udma_dfx_port_sta_func_t)(struct udma_device *udma_dev, + const struct ub_port_sta_inbuf *inbuf, + union ub_port_sta_outbuf *outbuf); + +struct udma_dfx_port_sta_cmd_func { + u32 cmd_id; + udma_dfx_port_sta_func_t cmd_func; +}; + +int udma_dfx_port_sta_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); + +#endif /* UDMA_DFX_PORT_STA_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c new file mode 100644 index 000000000..3539968aa --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.c @@ -0,0 +1,861 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/types.h> +#include "hinic5_hw.h" +#include "udma_common.h" +#include "ub_pub_cmd.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_cmd_vtp.h" +#include "udma_dfx.h" +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_query.h" +#include "udma_dfx_latch.h" +#include "udma_dfx_query_tp.h" + +static int +udma_dfx_query_tpc_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_dfx_tp_ctx_query *tpc = &outbuf->tp; + struct ubcore_res_key key = { 0 }; + int ret; + + (void)memset_s(tpc, sizeof(struct ub_dfx_tp_ctx_query), 0, + sizeof(struct ub_dfx_tp_ctx_query)); + key.type = UBCORE_RES_KEY_TP; + key.key = inbuf->tpn; + ret = udma_query_ctx(&udev->ub_dev, &key, (void *)tpc); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to query tpc from cache, tpn=%u, ret=%d!", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpc_from_host_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_dfx_tp_ctx_query *tpc_ctx = &outbuf->tp; + int ret; + + ret = udma_query_tp_ctx_from_host(udev, inbuf->tpn, tpc_ctx); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get tp ctx from host, tpn=%u, ret=%d", + inbuf->tpn, ret); + return ret; + } + + return ret; +} + +static int udma_dfx_query_tp_mpt_from_list(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_mapt_ctx_query *tp_mpt = &outbuf->mpt; + int ret; + + ret = udma_query_tp_mpt_from_host(udev, inbuf->tpn, tp_mpt); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get tp mpt from host, tpn=%u, ret=%d", + inbuf->tpn, ret); + return ret; + } + + return ret; +} + +static int udma_dfx_query_tp_pcqc_from_list(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + int ret; + + ret = udma_query_tp_pcqc_from_host(udev, inbuf->tpn, pcqc); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_DFX, + "Failed to get tp pcqc from host, tpn=%u, ret=%d", + inbuf->tpn, ret); + return ret; + } + + return ret; +} + +static int +udma_dfx_query_tp_wqe_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tp_wqe *tpqe = &outbuf->tpqe; + int ret; + + (void)memset_s(tpqe, sizeof(struct ub_tp_wqe), 0, + sizeof(struct ub_tp_wqe)); + + ret = udma_cmd_tp_query_wqe(udev, inbuf->tpn, inbuf->wqe_id, tpqe, + sizeof(struct ub_tp_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_cache_exec failed! tpn(%u) ci(%u) ret(%d)", + inbuf->tpn, inbuf->wqe_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tp_wqe_from_host_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tp_wqe *tpqe = &outbuf->tpqe; + int ret; + + (void)memset_s(tpqe, sizeof(struct ub_tp_wqe), 0, + sizeof(struct ub_tp_wqe)); + + ret = udma_query_tp_wqe_from_host(udev, inbuf->tpn, inbuf->wqe_id, tpqe, + sizeof(struct ub_tp_wqe)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_host_exec failed! tpn(%u) ci(%u) ret(%d)", + inbuf->tpn, inbuf->wqe_id, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tp_pcqc_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + ub_pcqc_query_rsp_s *pcqc = &outbuf->pcqc; + int ret; + + ret = udma_cmd_tp_query_pcqc(udev, inbuf->tpn, pcqc, + sizeof(ub_pcqc_query_rsp_s)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query tp pcqc from cache, tpn = %u, ret=%d!", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpc_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpc_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tpc_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tpc_from_host(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpc_host failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tpc_from_host_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_mpt(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_mpt failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tp_mpt_from_list(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_pcqc_from_host(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_pcqc failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tp_pcqc_from_list(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_wqe_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tp_wqe_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_wqe_from_host(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_wqe_from_host failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tp_wqe_from_host_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_pcqc_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_pcqc_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tp_pcqc_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_tp_mtt_from_list(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + struct mtt *ub_mtt = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->tpn, false); + if (cqm_obj == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn = %u\n", inbuf->tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udev, inbuf->tpn); + if (ub_tp == NULL) { + udma_err(udev->dev, UDMA_M_DFX, "failed to find tp, tpn = %u\n", + inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udev->dev, UDMA_M_DFX, + "failed to get ub_tp, tpn = %u\n", inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + + ub_mtt = (struct mtt *)&ub_tp->ub_mtt; + + ret = udma_dfx_query_cmtt(udev, ub_mtt, outbuf); + +cleanup: + cqm5_object_put(cqm_obj); + return ret; +} + +static int udma_dfx_query_tp_mtt(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_mtt failed, ret=%d!\n", ret); + return ret; + } + + return udma_dfx_query_tp_mtt_from_list(udma_dev, inbuf, outbuf); +} + +static int +udma_dfx_query_tp_num_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + u32 *tp_state_num = outbuf->tp_state_num; + int ret; + + ret = udma_cmd_tp_query_cnt(udev, tp_state_num, + sizeof(outbuf->tp_state_num)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_num_from_cache_exec failed, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_num_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_num_from_cache failed.\n"); + return ret; + } + + return udma_dfx_query_tp_num_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_oor_tp_info_from_cache(struct udma_device *udma_dev, + const void *buf_in, + u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_oor_tp_info *oor_tp_info = &outbuf->oor_tp_info; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_oor_tp_info_from_cache failed.\n"); + return ret; + } + + ret = udma_cmd_tp_query_oor_info(udma_dev, inbuf->tpn, inbuf->msn, + oor_tp_info, + sizeof(struct ub_oor_tp_info)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_cmd_tp_query_oor_info failed, ret=%d.\n", ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_msn_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_msn_entry_outbuf *tp_msn_entry = &outbuf->msn_entry_info; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_oor_tp_info_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + ret = udma_cmd_tp_query_msn_entry(udma_dev, inbuf->tpn, + inbuf->entry_index, tp_msn_entry, + sizeof(struct ub_msn_entry_outbuf)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_cmd_tp_query_msn_entry failed, ret=%d.\n", ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_cc_info_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, + + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + ub_cc_dfx_info_s *cc_info = &outbuf->cc_info; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_cc_info_from_cache failed.\n"); + return ret; + } + + ret = udma_cmd_tp_query_cc_info(udma_dev, inbuf->tpn, cc_info); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_cc_info_from_cache failed, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tp_q_depth(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_q_depth failed.\n"); + return ret; + } + + outbuf->q_depth = udma_sysfs_get_tp_depth(&udma_dev->sysfs_res); + + return 0; +} + +static int +udma_dfx_query_tpg_ctx_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tpg_ctx_query *tpg_ctx = &outbuf->tpg_ctx; + int ret; + + ret = udma_cmd_tpg_query_ctx_from_cache( + udev, inbuf->tpg_id, tpg_ctx, sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_ctx_from_cache_exec failed, ret=%d.\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpg_ctx_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_ctx_from_cache failed, ret=%d.\n", + ret); + return ret; + } + + return udma_dfx_query_tpg_ctx_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int udma_dfx_query_vtp_ctx_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_vtp_ctx_query *vtp_ctx = &outbuf->vtp_ctx; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_vtp_ctx_from_cache failed, ret=%d.\n", + ret); + return ret; + } + + ret = udma_cmd_query_vtp(udma_dev, inbuf->vfid, inbuf->vtp_id, vtp_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_vtp_ctx_from_cache failed, ret=%d.\n", + ret); + return ret; + } + return ret; +} + +static int udma_dfx_query_tp_srp_bitmap_from_cache(struct udma_device *udma_dev, + const void *buf_in, + u32 in_size, void *buf_out, + u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_cmd_tp_query_srp_bitmap_outbuf *srp_bitmap = + &outbuf->srp_bitmap; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_srp_bitmap_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + ret = udma_cmd_tp_query_srp_bitmap( + udma_dev, inbuf->tpn, srp_bitmap, + sizeof(struct ub_cmd_tp_query_srp_bitmap_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma query tpn(%u) srp bitmap from_cache failed, ret=%d.\n", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tp_srp_ctr_bitmap_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf *srp_ctr_bitmap = + &outbuf->srp_ctr_bitmap; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tp_srp_ctr_bitmap_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + ret = udma_cmd_tp_query_srp_ctr_bitmap( + udma_dev, inbuf->tpn, srp_ctr_bitmap, + sizeof(struct ub_cmd_tp_query_srp_ctr_bitmap_outbuf)); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma query tpn(%u) srp ctr bitmap from_cache failed, ret=%d.\n", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int +udma_dfx_query_tpg_flow_from_cache_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tpg_flow *val = &outbuf->tpg_flow; + int ret; + + val->tpgn = inbuf->tpg_id; + ret = udma_query_tpg_flow(udma_dev, val); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_flow_from_cache_exec failed, ret=%d!\n", + ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpg_flow_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "udma_dfx_query_tpg_flow_from_cache failed, ret=%d!\n", + ret); + return ret; + } + + return udma_dfx_query_tpg_flow_from_cache_exec(udma_dev, inbuf, outbuf); +} + +static int +udma_dfx_query_tpc_ext_from_cache_exec(struct udma_device *udev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct ub_tp_ctx_ext_query *tpc_ext = &outbuf->tpc_ext; + struct ubcore_res_key key = { 0 }; + int ret; + + key.type = UBCORE_RES_KEY_TP; + key.key = inbuf->tpn; + memset_s(tpc_ext, sizeof(struct ub_tp_ctx_ext_query), 0, + sizeof(struct ub_tp_ctx_ext_query)); + + ret = udma_cmd_tp_query_ctx_ext(udev, &key, tpc_ext); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_DFX, + "Failed to query extern tpc from cache, tpn(%u), ret(%d).", + inbuf->tpn, ret); + return ret; + } + + return 0; +} + +static int udma_dfx_query_tpc_ext_from_cache(struct udma_device *udma_dev, + const void *buf_in, u32 in_size, + void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + union ub_query_outbuf *outbuf = (union ub_query_outbuf *)buf_out; + int ret; + + ret = udma_dfx_query_prepare(udma_dev, buf_out, out_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "[%s] Failed to prepare buf_out, ret(%d).\n", __func__, + ret); + return ret; + } + + ret = udma_dfx_query_tpc_ext_from_cache_exec(udma_dev, inbuf, outbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "[%s] Failed to execute querying tpc ext, ret(%d).\n", + __func__, ret); + return ret; + } + + return 0; +} + +static struct udma_dfx_query_cmd_func g_udma_query_tp_cmd_funcs[] = { + + /* query */ + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpc_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_HOST, + udma_dfx_query_tpc_from_host }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_MPT, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_mpt }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_MTT, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_mtt }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_WQE, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_wqe_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_WQE, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_wqe_from_host }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_PCQC, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_pcqc_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_PCQC, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_pcqc_from_host }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CNT, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_num_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_OOR_INFO, UB_QUERY_FROM_CACHE, + udma_dfx_query_oor_tp_info_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_SRP_BITMAP, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_srp_bitmap_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_SRP_CTR_BITMAP, + UB_QUERY_FROM_CACHE, udma_dfx_query_tp_srp_ctr_bitmap_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CC_INFO, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_cc_info_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_Q_DEPTH, UB_QUERY_FROM_HOST, + udma_dfx_query_tp_q_depth }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_CTX_EXT, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpc_ext_from_cache }, + { UB_QUERY_TYPE_TP, UB_QUERY_SUB_TYPE_MSN_ENTRY, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_msn_from_cache }, + + { UB_QUERY_TYPE_TPG, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpg_ctx_from_cache }, + { UB_QUERY_TYPE_TPG, UB_QUERY_SUB_TYPE_FLOW, UB_QUERY_FROM_CACHE, + udma_dfx_query_tpg_flow_from_cache }, + + { UB_QUERY_TYPE_VTP, UB_QUERY_SUB_TYPE_CTX, UB_QUERY_FROM_CACHE, + udma_dfx_query_vtp_ctx_from_cache }, + { UB_QUERY_TYPE_TPG, UB_QUERY_SUB_TYPE_TP_LATCH, UB_QUERY_FROM_CACHE, + udma_dfx_query_tp_latch_data }, +}; + +bool udma_dfx_check_device_notmatch_for_query( + const struct udma_device *udma_dev, u32 cmd_type, u32 sub_type) +{ + static const u32 tpf_obj_query[] = { UB_QUERY_TYPE_TP, + UB_QUERY_TYPE_TPG }; + static const u32 pf_obj_query[] = { + UB_QUERY_TYPE_RC_QUEUE, UB_QUERY_TYPE_JETTY, + UB_QUERY_TYPE_JETTY_GROUP, UB_QUERY_TYPE_JFC, + UB_QUERY_TYPE_JFR, UB_QUERY_TYPE_JFS, + UB_QUERY_TYPE_SEG + }; + bool ret = false; + u32 i = 0; + /* determine tp/tpg/utp must not be on pf device on */ + if (!udma_dev->ub_dev.attr.tp_maintainer) { + for (i = 0; i < ARRAY_SIZE(tpf_obj_query); i++) { + if (cmd_type == tpf_obj_query[i]) { + ret = true; + break; + } + if (cmd_type == UB_QUERY_TYPE_DEV && + (sub_type == UB_QUERY_SUB_TYPE_DIP || + sub_type == UB_QUERY_SUB_TYPE_SIP)) { + ret = true; + break; + } + } + } else { + for (i = 0; i < ARRAY_SIZE(pf_obj_query); i++) { + if (cmd_type == pf_obj_query[i]) { + ret = true; + break; + } + } + } + return ret; +} + +int udma_dfx_query_tp_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + struct ub_query_inbuf *query_inbuf = (struct ub_query_inbuf *)buf_in; + udma_ioctl_func_t func = NULL; + u32 cmd_type; + int ret = -EINVAL; + u32 i; + + for (i = 0; i < sizeof(g_udma_query_tp_cmd_funcs) / + sizeof(g_udma_query_tp_cmd_funcs[0]); + i++) { + cmd_type = ub_dfx_get_cmd_type( + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type, + g_udma_query_tp_cmd_funcs[i].source_type); + if (cmd_type == query_inbuf->cmd_type) { + if (udma_dfx_check_device_notmatch_for_query( + udma_dev, + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type)) { + udma_err( + udma_dev->dev, UDMA_M_DFX, + "call query func, cmd type(%u), sub_type(%u) and device not match, check the device name is tpf or pf.\n", + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type); + break; + } + /* TODO: later need to consider ICSL submission for invalid xid queue query of intercept */ + func = g_udma_query_tp_cmd_funcs[i].cmd_func; + if (func != NULL) { + ret = func(udma_dev, buf_in, in_size, buf_out, + out_size); + udma_dfx_print( + "call query tp func, cmd_type:%u, sub_type:%u, source_type:%u\n", + g_udma_query_tp_cmd_funcs[i].cmd_type, + g_udma_query_tp_cmd_funcs[i].sub_type, + g_udma_query_tp_cmd_funcs[i] + .source_type); + } + break; + } + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h new file mode 100644 index 000000000..417c5e53f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/dfx/udma_dfx_query_tp.h @@ -0,0 +1,16 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_DFX_QUERY_TP_H +#define UDMA_DFX_QUERY_TP_H + +#include <linux/types.h> +#include "udma_common.h" + +int udma_dfx_query_tp_func(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +bool udma_dfx_check_device_notmatch_for_query( + const struct udma_device *udma_dev, u32 cmd_type, u32 sub_type); + +#endif /* UDMA_DFX_QUERY_TP_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c new file mode 100644 index 000000000..60adf80ee --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.c @@ -0,0 +1,1457 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include <linux/netdevice.h> +#include <linux/if_vlan.h> + +#include "comm_defs.h" +#include "hmm_common.h" +#include "hmm_em_inner.h" +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "ub_mpu_cmd.h" +#include "ub_aeqe.h" +#include "ubcore_api.h" +#include "udma_common.h" +#include "udma_jetty.h" +#include "udma_comp.h" +#include "udma_device.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_rc_table.h" +#include "udma_jfc.h" +#include "udma_tp.h" +#include "udma_vtp.h" +#include "udma_cmd_event.h" +#include "udma_cmd_jetty.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_dfx_latch.h" +#include "udma_resource.h" + +#define UB_MTT_PAGE_SIZE_4K 4096 +#define UB_MTT_PAGE_SIZE_64K (64 * 1024) +#define UB_MTT_PAGE_SIZE_2M (2 * 1024 * 1024) +#define UB_MTT_PAGE_SIZE_4K_SHIFT 12 +#define UB_MTT_PAGE_SIZE_64K_SHIFT 16 +#define UB_MTT_PAGE_SIZE_2M_SHIFT 21 +#define UB_EM_MIN_ORDER 2 /* ub rc extend table at least 4 page */ +#define UB_DEV_STATUS_BIT_PORT_LINK 0 +#define UB_CPU_ENDIAN_MAGIC 0x12345678 +#define LATCH_DATA_LEN_MASK 0xffff +#define VTP_SWITCH_TIME_WAIT 100 // ms +#define VTP_ROLLBACK_TIME_WAIT 20 // ms +#define UBRC_TABLE_DEPTH_MIN 128 +#define UBRC_TABLE_DEPTH_MAX 256 + +u32 g_ubrc_table_depth = 0U; /* rdma_rc table depth, used for write TAMSN engine manage WQE information */ +module_param(g_ubrc_table_depth, uint, S_IRUGO); +MODULE_PARM_DESC( + g_ubrc_table_depth, + "128 - 256, The value is a power of 2, otherwise the value is upward. default: 128"); + +enum { + UDMA_MTT_PAGE_SIZE_4K = 0, + UDMA_MTT_PAGE_SIZE_64K = 1, + UDMA_MTT_PAGE_SIZE_2M = 2 +}; + +enum { + UB_CL_TYPE_CQC_SRQC = + 0x4, /* cl_start: 0x120, cl_end: 0x13f, cl_size: 0 */ +}; + +void udma_cq_completion(void *svc_hd, u32 cqn, void *cq_handler) +{ + struct udma_jfc *jfc = NULL; + + if (cq_handler == NULL) { + udma_pr_err(UDMA_M_RESOURCE, "Cq_handler is null\n"); + return; + } + + jfc = (struct udma_jfc *)cq_handler; + + ++jfc->arm_sn; + + if (jfc->base.jfce_handler == NULL) { + udma_pr_err(UDMA_M_RESOURCE, "jfce_handler is null\n"); + return; + } + jfc->base.jfce_handler(&jfc->base); +} + +u8 udma_async_event_level(void *svc_hd, u8 event_type, u8 *val) +{ + (void)svc_hd; + (void)event_type; + (void)val; + return 0; +} + +static void udma_handle_jfc_async_event(struct udma_device *udma_dev, + u32 jfc_id, u8 event_type) +{ + struct udma_jfc *jfc = NULL; + struct tag_cqm_object *cqm_obj_queue = NULL; + + cqm_obj_queue = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_RDMA_SCQ, + jfc_id, false); + if (cqm_obj_queue == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get cqm obj queue, jfc_id:%u, event_type:%u\n", + jfc_id, event_type); + return; + } + jfc = udma_find_jfc(udma_dev, jfc_id); + if (jfc == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%u) is jfc, but the context not found.\n", + jfc_id); + goto out; + } + udma_jfc_put(jfc); + + jfc = cqmobj_to_ub_jfc(cqm_obj_queue); + if (jfc != NULL) { + udma_jfc_async_event(udma_dev, jfc, event_type); + } else { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%u) is jfc, but the context not found.\n", + jfc_id); + } + +out: + cqm5_object_put(cqm_obj_queue); +} + +static void udma_handle_jetty_grp_async_event(struct udma_device *udma_dev, + u32 xid, u8 event_type) +{ + struct udma_jetty_grp *jetty_grp = NULL; + struct tag_cqm_object *cqm_obj = NULL; + u32 jetty_grp_id = xid; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_RDMA_SCQ, + jetty_grp_id, false); + if (cqm_obj == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get cqm obj queue, jetty_grp_id:%u, event_type:%u\n", + jetty_grp_id, event_type); + return; + } + jetty_grp = udma_find_jetty_group(udma_dev, jetty_grp_id); + if (jetty_grp == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%u) is jetty grp, but the context not found.\n", + jetty_grp_id); + goto out; + } + udma_jetty_group_put(jetty_grp); + jetty_grp = cqmobj_to_ub_jetty_grp(cqm_obj); + if (jetty_grp != NULL) { + udma_jetty_grp_async_event(udma_dev, jetty_grp, event_type); + } else { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event xid(%d) is jetty_grp, but the context not found.\n", + xid); + } + +out: + cqm5_object_put(cqm_obj); +} + +static bool udma_refuse_report_flag_check(struct udma_device *udma_dev, + struct udma_tp *tp) +{ + if (tp->refuse_report_flush_done == 1) { + udma_warn_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "do not report event to ubcore when refuse_report_flush_done flag set, tpn(%u)\n", + tp->tp.tpn); + return true; + } + + return false; +} + +static void udma_handle_tp_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + struct tag_cqm_object *cqm_obj_queue = NULL; + struct ubcore_event event = { NULL }; + struct ubcore_tp *ubc_tp = NULL; + struct udma_tp *tp = NULL; + + udma_dump_tp_latch_info(udma_dev); /* dump tp latch data */ + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "drop_aeqe_en(%u), drop tp aeqe\n", + drop_aeqe_en); + return; + } + + cqm_obj_queue = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + aeqe->xid, false); + if (cqm_obj_queue == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get cqm obj queue, xid:%u, sub_type:%u\n", + aeqe->xid, aeqe->sub_type); + return; + } + + tp = cqmobj_to_ub_tp(cqm_obj_queue); + if (tp->cqm_priv_type != UDMA_CQM_PRIV_TYPE_TP) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "cqm obj is not a tp, xid:%u, sub_type:%u, type:%u\n", + aeqe->xid, aeqe->sub_type, tp->cqm_priv_type); + goto out; + } + + ubc_tp = (struct ubcore_tp *)&(tp->tp); + + event.ub_dev = ubc_tp->ub_dev; + event.element.tp = ubc_tp; + + switch (aeqe->sub_type) { + case UB_AEQ_TYPE_TP_FATAL_ERR: + case UB_AEQ_TYPE_TP_RETRY_EXCEED: + event.event_type = UBCORE_EVENT_TP_ERR; + break; + case UB_AEQ_TYPE_TP_ERR_FLUSH_EMPTY: + event.event_type = UBCORE_EVENT_TP_FLUSH_DONE; + if (udma_refuse_report_flag_check(udma_dev, tp)) + goto out; + break; + case UB_AEQ_TYPE_TP_SUBHEALTH: + event.event_type = UBCORE_EVENT_TP_SUSPEND; + event.element.tp->state = UBCORE_TP_STATE_SUSPENDED; + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown tp asyn event(%u).\n", aeqe->sub_type); + goto out; + } + + udma_info_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "tp(%u) data udp(%u) ack udp(%u) report event(%u) to ubcore .\n", + aeqe->xid, event.element.tp->data_udp_start, + event.element.tp->ack_udp_start, event.event_type); + ubcore_dispatch_async_event(&event); + +out: + cqm5_object_put(cqm_obj_queue); +} + +static void udma_handle_device_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + struct ubcore_event event = { 0 }; + + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "debug_en&drop_aeqe_en enable, drop device err aeqe\n"); + return; + } + + event.ub_dev = &udma_dev->ub_dev; + event.event_type = UBCORE_EVENT_DEV_FATAL; + + udma_info_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "aeqe xid(%u) report event(%u) to ubcore.\n", aeqe->xid, + (u32)event.event_type); + ubcore_dispatch_async_event(&event); +} + +static void udma_handle_jfs_async_event(struct udma_device *udma_dev, + struct udma_jfs *jfs, u8 event_type) +{ + udma_jfs_async_event(udma_dev, jfs, event_type); + return; +} + +static void udma_handle_jetty_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + struct tag_cqm_object *cqm_obj_queue = NULL; + struct udma_jetty *jetty = NULL; + + cqm_obj_queue = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + aeqe->xid, false); + if (cqm_obj_queue == NULL) { + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get cqm obj, queue_type:%u id:%u, sub_type:%u\n", + aeqe->queue_type, aeqe->xid, aeqe->sub_type); + return; + } + + jetty = cqmobj_to_ub_jetty(cqm_obj_queue); + if (jetty == NULL) { + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Failed to get jetty from object\n"); + goto leave; + } + + switch (jetty->cqm_priv_type) { + case UDMA_JETTY_TYPE_JFR: + udma_jfr_async_event(udma_dev, (struct udma_jfr *)jetty, + aeqe->sub_type); + break; + case UDMA_JETTY_TYPE_JFS: + udma_handle_jfs_async_event(udma_dev, (struct udma_jfs *)jetty, + aeqe->sub_type); + break; + case UDMA_JETTY_TYPE_JETTY: + udma_jetty_async_event(udma_dev, jetty, aeqe->sub_type); + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "Unknown jetty type(%u)\n", + jetty->cqm_priv_type); + break; + } + +leave: + cqm5_object_put(cqm_obj_queue); +} + +static void udma_handle_ta_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + + udma_dump_ta_latch_info(udma_dev); + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "drop_aeqe_en(%u), drop ta aeqe\n", + drop_aeqe_en); + return; + } + + switch (aeqe->queue_type) { + case UB_AEQE_TYPE_JFC: + udma_handle_jfc_async_event(udma_dev, aeqe->xid, + aeqe->sub_type); + break; + case UB_AEQE_TYPE_JFR: + case UB_AEQE_TYPE_JFS: + case UB_AEQE_TYPE_JETTY: + udma_handle_jetty_async_event(udma_dev, aeqe); + break; + case UB_AEQE_TYPE_JETTY_GROUP: + udma_handle_jetty_grp_async_event(udma_dev, aeqe->xid, + aeqe->sub_type); + break; + case UB_AEQE_TYPE_TP_DFX: + case UB_AEQE_TYPE_TA_DFX: + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "fatal_dfx_event,type(%u),dfx_info(0x%x),xid(0x%x)\n", + aeqe->queue_type, aeqe->udf_dfx, aeqe->xid); + break; + case UB_AEQE_TYPE_JFRC: + udma_err_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "jfrc_dfx_event,type(%u),api_type(0x%x),api_err_code(0x%x),dfx_info(0x%x),xid(0x%x)\n", + aeqe->queue_type, aeqe->api_type, aeqe->api_err_code, + aeqe->udf_dfx, aeqe->xid); + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown queue type(%u), udf_dfx(%u)\n", + aeqe->queue_type, aeqe->udf_dfx); + break; + } +} + +static void udma_handle_vtp_async_event(struct udma_device *udma_dev, + union ub_aeqe_t *aeqe) +{ + u32 drop_aeqe_en = udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en; + u32 debug_en = udma_dev->sysfs_res.dfx_ctx.debug_en; + struct ubcore_event event = { 0 }; + struct udma_vtp *ub_vtp = NULL; + u64 cur_time = jiffies_to_msecs(jiffies); + + if (debug_en == 1 && drop_aeqe_en == 1) { + udma_warn_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "drop_aeqe_en(%u), drop vtp aeqe\n", + drop_aeqe_en); + return; + } + + ub_vtp = udma_get_vtp_by_id(udma_dev, aeqe->xid, aeqe->udf_dfx); + if (ub_vtp == NULL) { + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "vtp not found, vtpn:%u function_id:%u.\n", + aeqe->xid, aeqe->udf_dfx); + return; + } + + event.ub_dev = ub_vtp->base.ub_dev; + event.element.vtp = &ub_vtp->base; + + switch (aeqe->sub_type) { + case UB_AEQ_TYPE_VTP_MIG_ING_ERR: + event.event_type = UBCORE_EVENT_MIGRATE_VTP_SWITCH; + if (cur_time - ub_vtp->mig_switch_time < VTP_SWITCH_TIME_WAIT) { + goto OUT; + } + ub_vtp->mig_switch_time = cur_time; + break; + case UB_AEQ_TYPE_VTP_MIG_BACK_ERR: + event.event_type = UBCORE_EVENT_MIGRATE_VTP_ROLLBACK; + if (cur_time - ub_vtp->mig_rollback_time < + VTP_ROLLBACK_TIME_WAIT) { + goto OUT; + } + ub_vtp->mig_rollback_time = cur_time; + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown queue type(%u)\n", aeqe->sub_type); + goto OUT; + } + + udma_info_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "vtp(%u) report event(%u) to ubcore .\n", aeqe->xid, + event.event_type); + ubcore_dispatch_async_event(&event); +OUT: + udma_vtp_put(ub_vtp); +} + +static void udma_async_event_handle(u8 event_type, u8 *val, + struct udma_device *udma_dev) +{ + union ub_aeqe_t *aeqe = (union ub_aeqe_t *)val; + udma_info_limit( + udma_dev->dev, UDMA_M_ASYNC_EVENT, + "event_type(%d), qtype(%d), aeqe: xid(%u), credit_vld(%u), sub_type(%u), api_type(%u), dw0(0x%x), dfx_info(0x%x) .\n", + event_type, aeqe->queue_type, aeqe->xid, aeqe->credit_vld, + aeqe->sub_type, aeqe->api_type, aeqe->dw0_value, + aeqe->dw1_value); + + switch (event_type) { + case UB_CQM_AEQE_TYPE_TA_INFO: + case UB_CQM_AEQE_TYPE_TA_WARN: + case UB_CQM_AEQE_TYPE_TA_ERR: + udma_handle_ta_async_event(udma_dev, aeqe); + break; + case UB_CQM_AEQE_TYPE_TP_INFO: + case UB_CQM_AEQE_TYPE_TP_WARN: + case UB_CQM_AEQE_TYPE_TP_ERR: + udma_handle_tp_async_event(udma_dev, aeqe); + break; + case UB_CQM_AEQE_TYPE_VTP_INFO: + case UB_CQM_AEQE_TYPE_VTP_WARN: + case UB_CQM_AEQE_TYPE_VTP_ERR: + udma_dump_ta_latch_info(udma_dev); + udma_handle_vtp_async_event(udma_dev, aeqe); + break; + case UB_CQM_AEQE_TYPE_DEVICE_ERR: + udma_handle_device_async_event(udma_dev, aeqe); + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_ASYNC_EVENT, + "unknown async event type(%u)\n", event_type); + return; + } + + if (aeqe->credit_vld == UB_AEQE_CREDIT_ENABLE) { + udma_cmd_async_event_credit_return(udma_dev, 1); + } +} + +void udma_async_event(void *udma_dev, u8 event_type, u8 *val) +{ + if (udma_dev == NULL || val == NULL) { + udma_pr_err_limit(UDMA_M_ASYNC_EVENT, + "udma_dev and val both are null\n"); + return; + } + + udma_async_event_handle(event_type, val, + (struct udma_device *)udma_dev); +} + +static int udma_init_ubrc_table(struct udma_device *udma_dev, + struct udma_comp_priv *comp_priv) +{ + u32 ubrc_per_channel = 0; /* ubrc num per channel 128 + 16 */ + u32 log_ubrc_per_seg = 0; /* min num of entry, 2^log_ubrc_per_seg */ + u32 ubrc_pow_of_two = 0; + u32 channel_num = 0; + u32 ubrc_size = 0; + u32 max_order = 0; /* ubrc buddy max priority num */ + int ret = 0; + u32 i = 0; + + channel_num = udma_dev->ub_cap.dev_cap.max_jetty + + udma_dev->ub_cap.dev_cap.max_tp; + ubrc_per_channel = roundup_pow_of_two(comp_priv->ub_cap.ubrc_depth + + UBRC_EXT_ENTRY); + log_ubrc_per_seg = comp_priv->ub_cap.log_ubrc_seg; + ubrc_size = comp_priv->ub_cap.ubrc_entry_sz; + for (i = 1; i < (u64)channel_num * ubrc_per_channel; i = (i << 1)) + max_order++; + + max_order = (max_order > log_ubrc_per_seg) ? + (max_order - log_ubrc_per_seg) : + 0; + + ret = hmm_buddy_init(&comp_priv->ubrc_buddy, max_order); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize ubrc's buddy failed, ret(%d)\n", -ENOMEM); + return -ENOMEM; + } + + ubrc_pow_of_two = + (u32)(HMM_EM_ROUNDUP_POW_OF_TWO( + (u32)((u64)channel_num * ubrc_per_channel)) & + 0xffffffff); + + ret = hmm_em_init_table(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table, ubrc_size, + ubrc_pow_of_two, 0, UB_EM_MIN_ORDER); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize ubrc's em_table failed, ret(%d)\n", ret); + goto err_out; + } + + return 0; + +err_out: + hmm_buddy_cleanup(&comp_priv->ubrc_buddy); + + return -ENOMEM; +} + +static void udma_deinit_ubrc_table(struct udma_comp_priv *comp_priv) +{ + hmm_em_cleanup_table(comp_priv->hmm_priv.dev, + &comp_priv->ubrc_em_table); + + hmm_buddy_cleanup(&comp_priv->ubrc_buddy); +} + +static int udma_init_table(struct udma_device *udma_dev, + struct udma_comp_priv *comp_priv) +{ + int ret; + + ret = hmm_mtt_init(&comp_priv->hmm_priv); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize mtt table failed, ret(%d)\n", ret); + return ret; + } + + ret = udma_init_ubrc_table(udma_dev, comp_priv); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Initialize ubrc's table failed, ret(%d)\n", ret); + goto err_init_ubrc_table; + } + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "UB init table successful\n"); + return 0; + +err_init_ubrc_table: + hmm_mtt_cleanup(&comp_priv->hmm_priv); + return ret; +} + +static void udma_deinit_table(struct udma_comp_priv *comp_priv) +{ + udma_deinit_ubrc_table(comp_priv); + + hmm_mtt_cleanup(&comp_priv->hmm_priv); +} + +static void copy_value_to_comp_priv(struct udma_device *udma_dev, + struct udma_comp_priv *comp_priv) +{ + struct ub_dev_cap_udma_res *udma_res_cap = + &udma_dev->ub_cap.dev_cap_res.udma_res; + struct ub_dev_cap_sdk_res *sdk_res_cap = + &udma_dev->ub_cap.dev_cap_res.sdk_res; + + comp_priv->hmm_priv.dev_cap.mpt_entry_sz = sdk_res_cap->mpt_entry_sz; + comp_priv->hmm_priv.dev_cap.dmtt_cl_start = sdk_res_cap->dmtt_cl_start; + comp_priv->hmm_priv.dev_cap.dmtt_cl_end = sdk_res_cap->dmtt_cl_end; + comp_priv->hmm_priv.dev_cap.dmtt_cl_sz = sdk_res_cap->dmtt_cl_sz; + comp_priv->ub_cap.cmtt_cl_start = sdk_res_cap->cmtt_cl_start; + comp_priv->ub_cap.cmtt_cl_end = sdk_res_cap->cmtt_cl_end; + comp_priv->ub_cap.cmtt_cl_sz = sdk_res_cap->cmtt_cl_sz; + comp_priv->ub_cap.wqe_cl_start = sdk_res_cap->wqe_cl_start; + comp_priv->ub_cap.wqe_cl_end = sdk_res_cap->wqe_cl_end; + comp_priv->ub_cap.wqe_cl_sz = sdk_res_cap->wqe_cl_sz; + + comp_priv->hmm_priv.dev_cap.log_mtt = udma_res_cap->log_mtt; + comp_priv->hmm_priv.dev_cap.num_mtts = udma_res_cap->num_mtts; + comp_priv->hmm_priv.dev_cap.log_mtt_seg = udma_res_cap->log_mtt_seg; + comp_priv->hmm_priv.dev_cap.mtt_entry_sz = udma_res_cap->mtt_entry_sz; + comp_priv->ub_cap.ubrc_entry_sz = udma_res_cap->ubrc_entry_size; + comp_priv->ub_cap.log_ubrc_seg = udma_res_cap->log_ubrc_seg; + comp_priv->ub_cap.ubrc_depth = udma_res_cap->ubrc_depth; // 128 +} + +static int udma_init_comp_priv(struct udma_device *udma_dev) +{ + struct udma_comp_priv *comp_priv = NULL; + int ret; + + if (udma_dev->hwdev == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, "Hwdev is null\n"); + return -EINVAL; + } + + comp_priv = (struct udma_comp_priv *)devm_kzalloc( + udma_dev->dev, sizeof(struct udma_comp_priv), GFP_KERNEL); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Alloc memory for comp_priv failed, ret(%d)\n", + -ENOMEM); + return -ENOMEM; + } + + comp_priv->hmm_priv.hwdev = udma_dev->hwdev; + comp_priv->hmm_priv.dev = udma_dev->dev; + copy_value_to_comp_priv(udma_dev, comp_priv); + + switch (g_mtt_page_size) { + case UDMA_MTT_PAGE_SIZE_4K: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_4K_SHIFT; + break; + case UDMA_MTT_PAGE_SIZE_64K: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_64K; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_64K_SHIFT; + break; + case UDMA_MTT_PAGE_SIZE_2M: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_2M; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_2M_SHIFT; + break; + default: + comp_priv->hmm_priv.mtt_page_size = UB_MTT_PAGE_SIZE_4K; + comp_priv->hmm_priv.mtt_page_shift = UB_MTT_PAGE_SIZE_4K_SHIFT; + break; + } + ret = udma_init_table(udma_dev, comp_priv); + if (ret != 0) { + devm_kfree(udma_dev->dev, comp_priv); + return ret; + } + + ret = hinic5_register_service_adapter(udma_dev->hwdev, + (void *)comp_priv, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "put ub_comp_res failed, ret(%d)\n", ret); + udma_deinit_table(comp_priv); + devm_kfree(udma_dev->dev, comp_priv); + return ret; + } + + comp_priv->udma_dev = udma_dev; + return ret; +} + +static int udma_init_cqm_server(struct udma_device *udma_dev) +{ + service_register_template_s svc_template = { 0 }; + struct ub_dev_cap_sdk_res *sdk_res = + &udma_dev->ub_cap.dev_cap_res.sdk_res; + + svc_template.service_type = SERVICE_T_UB; + svc_template.scq_ctx_size = sdk_res->cqc_entry_sz; + svc_template.srq_ctx_size = sdk_res->srqc_entry_sz; + svc_template.service_handle = udma_dev; + svc_template.embedded_cq_ceq_callback = NULL; + svc_template.no_cq_ceq_callback = NULL; + svc_template.shared_cq_ceq_callback = udma_cq_completion; + svc_template.aeq_level_callback = udma_async_event_level; + svc_template.aeq_callback = udma_async_event; + + return cqm5_service_register(udma_dev->hwdev, &svc_template); +} + +static int udma_alloc_ceqs(struct udma_device *udma_dev) +{ + struct ub_dev_cap_udma_res *udma_res = + &udma_dev->ub_cap.dev_cap_res.udma_res; + int ret; + + ret = hinic5_alloc_ceqs(udma_dev->hwdev, SERVICE_T_UB, + (int)udma_res->num_ceq_vectors, udma_dev->ceqn, + &udma_dev->ceq_num); + if (ret < 0) + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Failed to alloc cqes, num_comp_vectors(%u) .\n", + udma_res->num_ceq_vectors); + + return ret; +} + +static void udma_free_ceqs(struct udma_device *udma_dev) +{ + int i; + + for (i = 0; i < udma_dev->ceq_num; ++i) + hinic5_free_ceq(udma_dev->hwdev, SERVICE_T_UB, + udma_dev->ceqn[i]); +} + +static int udma_get_cpu_endian(struct udma_device *udma_dev) +{ + int cpu_mode = 0; + union { + u32 i; + u8 s[4]; /* 4 occupy bit symbol */ + } c; + c.i = UB_CPU_ENDIAN_MAGIC; + + if (c.s[0] == 0x12) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "CPU is be\n"); + cpu_mode = 1; + } else { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "CPU is le\n"); + cpu_mode = 0; + } + + return cpu_mode; +} + +static int udma_set_func_cpu_endian(struct udma_device *udma_dev) +{ + struct ub_cmd_func_set_cpu_endian cmd = { 0 }; + u16 cmd_size = sizeof(cmd); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + cmd.cpu_endian = (u8)udma_get_cpu_endian(udma_dev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_SET_CPU_ENDIAN, &cmd, + sizeof(cmd), &cmd, &cmd_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_size == 0) || (cmd.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to set cpu endian, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, cmd.head.status, cmd_size); + return -EINVAL; + } + + return 0; +} + +void udma_fill_dev_default_cos(struct udma_device *udma_dev, + const struct ub_get_cfg_cap_resp *resp) +{ + u32 cos_value = 0; + udma_dev->default_cos.cfg_cos_en = 0; // default not support user configuration cos value + udma_dev->default_cos.jfrc_cos = resp->dev_cap.jfrc_cos; + udma_dev->default_cos.vf_default_cos = resp->dev_cap.vf_default_cos; + udma_dev->default_cos.pf_default_cos = resp->dev_cap.pf_default_cos; + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return; + } + + udma_dev->default_cos.func_cos = (u8)resp->dev_cap.func_cos; + + cos_value = udma_dev->ub_cap.dev_cap.virtualization == 1 ? + resp->dev_cap.func_cos : + resp->dev_cap.default_jetty_cos; + udma_dev->default_cos.pf_default_cos = (u16)cos_value; + udma_dev->default_cos.vf_default_cos = (u16)cos_value; + udma_info( + udma_dev->dev, UDMA_M_DEV, + "get dev_default_cos virtualization:%u, func_cos:%u, jetty_cos:%u\n", + udma_dev->ub_cap.dev_cap.virtualization, resp->dev_cap.func_cos, + resp->dev_cap.default_jetty_cos); +} + +void udma_print_dev_cap_info(const struct udma_device *udma_dev, + const struct ub_service_cap *ub_cap, + const struct ub_get_cfg_cap_resp *resp) +{ + udma_info( + udma_dev->dev, UDMA_M_DEV, + "get dev cap, max_jetty:0x%x, max_tp:0x%x, max_jfc:0x%x, max_jfr:0x%x, " + "max_jfs:0x%x, max_jfrc_depth:0x%x\n", + udma_dev->ub_cap.dev_cap.max_jetty, + udma_dev->ub_cap.dev_cap.max_tp, + udma_dev->ub_cap.dev_cap.max_jfc, + udma_dev->ub_cap.dev_cap.max_jfr, + udma_dev->ub_cap.dev_cap.max_jfs, + udma_dev->ub_cap.dev_cap_res.udma_res.jfrc_depth); + + udma_info( + udma_dev->dev, UDMA_M_DEV, + "get dev cap, slice:0x%x, virtualization:0x%x, feature 0x%x\n", + udma_dev->slice, resp->dev_cap.virtualization, + udma_dev->ub_cap.dev_cap.feature.value); +} + +static u32 udma_get_ta_resp_depth(enum udma_device_type device_type, + u32 default_depth) +{ + if (device_type != UDMA_DEV_TYPE_HI1825_V100) { + g_ubrc_table_depth = default_depth; + return default_depth; + } + + if (g_ubrc_table_depth < UBRC_TABLE_DEPTH_MIN || + g_ubrc_table_depth > UBRC_TABLE_DEPTH_MAX) { + g_ubrc_table_depth = default_depth; + return default_depth; + } + + return (u32)roundup_pow_of_two((u64)g_ubrc_table_depth); +} + +int udma_get_device_cap(struct udma_device *udma_dev, + struct ub_service_cap *ub_cap) +{ + struct ub_get_cfg_cap_cmd cmd = { 0 }; + struct ub_get_cfg_cap_resp resp = { 0 }; + u16 resp_size = sizeof(resp); + u32 is_tpf = ub_cap->net_dev_cap.is_tpf; + struct ub_dev_cap_res *dev_cap_res = &udma_dev->ub_cap.dev_cap_res; + int ret; + + if (udma_vram5_get_kexec_flag() == 1) { + return 0; + } + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CFG_CAP, &cmd, sizeof(cmd), + &resp, &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EINVAL; + } + + (void)memcpy_s(&udma_dev->ub_cap.dev_cap_res.sdk_res, + sizeof(struct ub_dev_cap_sdk_res), &ub_cap->sdk_res, + sizeof(struct ub_dev_cap_sdk_res)); + (void)memcpy_s(&udma_dev->ub_cap.net_dev_cap, + sizeof(struct ub_net_dev_cap), &ub_cap->net_dev_cap, + sizeof(struct ub_net_dev_cap)); + (void)memcpy_s(&udma_dev->ub_cap.dev_cap_res.udma_res, + sizeof(struct ub_dev_cap_udma_res), &resp.udma_res, + sizeof(struct ub_dev_cap_udma_res)); + (void)memcpy_s(&udma_dev->ub_cap.dev_cap, sizeof(struct ub_dev_cap), + &resp.dev_cap, sizeof(struct ub_dev_cap)); + + dev_cap_res->udma_res.ubrc_depth = udma_get_ta_resp_depth( + udma_dev->device_type, dev_cap_res->udma_res.ubrc_depth); + + if ((ub_cap->sdk_res.max_jetty < resp.udma_res.max_jfrc_num) && + (is_tpf == 0)) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "max_jetty[%u] is too small!.\n", + ub_cap->sdk_res.max_jetty); + return -EINVAL; + } + + udma_dev->ub_cap.dev_cap.max_jetty = + (is_tpf == 0) ? + ub_cap->sdk_res.max_jetty - resp.udma_res.max_jfrc_num : + ub_cap->sdk_res.max_jetty; + udma_dev->ub_cap.dev_cap.max_jetty_grp = ub_cap->sdk_res.max_jetty_grp; + udma_dev->ub_cap.dev_cap.max_jfc = ub_cap->sdk_res.max_jfc; + udma_dev->ub_cap.dev_cap.max_jfr = + (is_tpf == 0) ? min(udma_dev->ub_cap.dev_cap.max_jetty, + ub_cap->sdk_res.max_jfr) : + 0; + udma_dev->ub_cap.dev_cap.max_tp = ub_cap->sdk_res.max_tp; + udma_dev->ub_cap.dev_cap.max_tpg = ub_cap->sdk_res.max_tpg; + udma_dev->ub_cap.dev_cap.max_vtp = ub_cap->sdk_res.max_vtp; + udma_dev->slice = udma_dev->ub_cap.dev_cap.slice; + udma_dev->ub_cap.dev_cap.max_jfs = + (is_tpf == 0) ? + ub_cap->sdk_res.max_jetty - resp.udma_res.max_jfrc_num : + ub_cap->sdk_res.max_jetty; // jfs and jetty shared resource + + udma_fill_dev_default_cos(udma_dev, &resp); + udma_print_dev_cap_info(udma_dev, ub_cap, &resp); + + return 0; +} + +int udma_get_mtt_protect_state(struct udma_device *udma_dev, u32 *mtt_pro_en) +{ + struct ub_get_cfg_cap_cmd cmd = { 0 }; + struct ub_get_cfg_cap_resp resp = { 0 }; + u16 resp_size = sizeof(resp); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_CFG_CAP, &cmd, sizeof(cmd), + &resp, &resp_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EIO; + } + *mtt_pro_en = resp.dev_cap.mtt_pro_en; + + return 0; +} + +static void udma_get_split_bdf_info(struct udma_dev_bdf_info *bdf_info, u16 bdf) +{ + bdf_info->bus_id = (u8)(bdf >> UB_BDF_BUS_BIT); + bdf_info->device_id = (bdf >> UB_BDF_DEVICE_ID_BIT) & + UB_BDF_DEVICE_ID_MASK; + bdf_info->func_id = bdf & UB_BDF_FUNC_ID_MASK; +} + +static int udma_generate_bdf_info(struct udma_cmd_get_bdf_info *bdf_info_get, + struct udma_device *udma_dev) +{ + struct udma_dev_bdf_info *bdf_info = udma_dev->bdf_info; + u16 host_id = hinic5_pcie_itf_id(udma_dev->hwdev); + struct pci_dev *pdev = container_of(udma_dev->dev, struct pci_dev, dev); + u32 bus = pdev->bus->number + 1; + bool fake_bdf = false; + u32 vf_func_id = 0; + u16 bdf_id = 0; + u32 i = 0; + u32 j = 0; + + fake_bdf = hinic5_is_multi_bm(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_MAIN, + "generate bdf info host_id:%u, bus:%u, fake_bdf:%d", host_id, + bus, fake_bdf); + /* PF bdf info */ + for (i = 0; i < UB_PCIE_MODE_PF_NUM; i++) { + if (fake_bdf && + host_id != (u16)bdf_info_get->pf_bdf_info[i].itf_idx) + bdf_info_get->pf_bdf_info[i].bdf = + (u16)((bus << UB_BDF_BUS_BIT) + i); + + bdf_id = bdf_info_get->pf_bdf_info[i].bdf; + bdf_info[i].bdf_info_vld = + bdf_info_get->pf_bdf_info[i].pf_bdf_info_vld; + bdf_info[i].host_id = bdf_info_get->pf_bdf_info[i].itf_idx; + udma_get_split_bdf_info(&bdf_info[i], bdf_id); + } + + /* VF bdf info */ + for (i = 0; i < UB_PCIE_MODE_PF_NUM; i++) { + for (j = 0; j < bdf_info_get->vf_bdf_info[i].max_vfs; j++) { + vf_func_id = + bdf_info_get->vf_bdf_info[i].glb_pf_vf_offset + + 1 + (u16)j; + if (vf_func_id > UB_FUNCTION_MAX_NUM) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "vf_func_id(%u) bigger than UB_FUNCTION_MAX_NUM(%d)\n", + vf_func_id, UB_FUNCTION_MAX_NUM); + udma_err(udma_dev->dev, UDMA_M_MAIN, + "i:%d, j:%d, glb_pf_vf_offset:%u \n", + i, j, + bdf_info_get->vf_bdf_info[i] + .glb_pf_vf_offset); + return -EPERM; + } + + bdf_info[vf_func_id].bdf_info_vld = + bdf_info_get->pf_bdf_info[i].pf_bdf_info_vld; + bdf_info[vf_func_id].host_id = + bdf_info_get->pf_bdf_info[i].itf_idx; + if (bdf_info_get->vf_bdf_info[i].bus_num != 0 || + fake_bdf) { + bdf_id = + (u16)(bdf_info_get->pf_bdf_info[i].bdf + + bdf_info_get->vf_bdf_info[i] + .vf_offset + + bdf_info_get->vf_bdf_info[i] + .vf_stride * + j); + } else { + bdf_id = 0; + } + udma_get_split_bdf_info(&bdf_info[vf_func_id], bdf_id); + } + } + udma_dev->bdf_cnt = vf_func_id + 1; + return 0; +} + +int udma_get_pcie_bdf_info(struct udma_device *udma_dev) +{ + struct udma_cmd_get_bdf_info bdf_info_get = { 0 }; + u16 out_size = sizeof(bdf_info_get); + int ret; + + if (udma_dev->bdf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "bdf_info is null,maybe it's not tpf.\n"); + return -EINVAL; + } + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_GET_BDF_INFO, &bdf_info_get, + sizeof(bdf_info_get), &bdf_info_get, + &out_size, 0, HINIC5_CHANNEL_UB); + if ((ret != 0) || (out_size == 0) || (bdf_info_get.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_MAIN, + "get bdf_info failed, ret:%d, status:0x%x, out size:0x%x\n", + ret, bdf_info_get.head.status, out_size); + return -EINVAL; + } + mutex_lock(&udma_dev->bdf_mutex); + ret = udma_generate_bdf_info(&bdf_info_get, udma_dev); + udma_info(udma_dev->dev, UDMA_M_MAIN, "get pcie bdf info cnt:%u\n", + udma_dev->bdf_cnt); + mutex_unlock(&udma_dev->bdf_mutex); + + return ret; +} + +static int udma_get_hw_info(struct udma_device *udma_dev) +{ + u64 bdf_len = 0; + int ret = 0; + + udma_dev->hw_info.config_num_ports = + udma_dev->ub_cap.net_dev_cap.port_cnt; + udma_dev->hw_info.ep_id = hinic5_ep_id(udma_dev->hwdev); + udma_dev->hw_info.phy_port = 1; + udma_dev->hw_info.is_vf = + (hinic5_func_type(udma_dev->hwdev) == TYPE_VF); + udma_dev->hw_info.cpu_endian = (u8)udma_get_cpu_endian(udma_dev); + + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0 && + udma_dev->dev_bus_type == HINIC5_DEVICE_T_PCI) { + bdf_len = + sizeof(struct udma_dev_bdf_info) * UB_FUNCTION_MAX_NUM; + udma_dev->bdf_info = (struct udma_dev_bdf_info *)devm_kzalloc( + udma_dev->dev, bdf_len, GFP_KERNEL); + if (udma_dev->bdf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "failed to allocate memory for bdf info\n"); + return -ENOMEM; + } + mutex_init(&udma_dev->bdf_mutex); + + ret = udma_get_pcie_bdf_info(udma_dev); + if (ret != 0) { + devm_kfree(udma_dev->dev, udma_dev->bdf_info); + udma_dev->bdf_info = NULL; + } + } + + return ret; +} + +int udma_cleanup_ipsu_table(struct udma_device *udma_dev) +{ + struct ub_cmd_ipsu_ip_entry cmd = { 0 }; + struct ub_cmd_ipsu_ip_entry resp = { 0 }; + u16 resp_size = sizeof(struct ub_cmd_ipsu_ip_entry); + int ret; + + cmd.func_id = hinic5_global_func_id(udma_dev->hwdev); + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UB, + UB_MPU_CMD_CLEAR_IP_ENTRY, &cmd, + sizeof(cmd), &resp, &resp_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (resp_size == 0) || (resp.head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to get dev cap, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, resp.head.status, resp_size); + return -EINVAL; + } + return 0; +} + +static void udma_cleanup_comp_priv(struct udma_device *udma_dev) +{ + struct udma_comp_priv *comp_priv = NULL; + + comp_priv = (struct udma_comp_priv *)get_hmm_comp_priv(udma_dev->hwdev, + SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, "Comp_priv is null\n"); + return; + } + + udma_deinit_table(comp_priv); + + hinic5_unregister_service_adapter((void *)udma_dev->hwdev, + SERVICE_T_UB); + + devm_kfree(udma_dev->dev, comp_priv); + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Ub cleanup comp_priv successful\n"); +} + +static int udma_set_nic_event(struct udma_device *udma_dev, + struct hinic5_event_info *event, + enum ubcore_event_type *type) +{ + switch (event->type) { + case EVENT_NIC_LINK_UP: + if (test_and_set_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) != 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set up\n"); + return -1; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_UP\n"); + *type = UBCORE_EVENT_PORT_ACTIVE; + break; + + case EVENT_NIC_LINK_DOWN: + if (test_and_clear_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) == 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set down\n"); + return -1; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_DOWN\n"); + *type = UBCORE_EVENT_PORT_DOWN; + break; + + case EVENT_NIC_DCB_STATE_CHANGE: + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "DCB state change not supported\n"); + return -1; + + default: + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Nic event unsupported, event: %d\n", event->type); + return -1; + } + + return 0; +} + +static int udma_set_comm_event(struct udma_device *udma_dev, + struct hinic5_event_info *event, + enum ubcore_event_type *type) +{ + if (event->type == EVENT_COMM_PCIE_LINK_DOWN || + event->type == EVENT_COMM_HEART_LOST) { + *type = UBCORE_EVENT_ELR_ERR; + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "get event: UBCORE_EVENT_ELR_ERR\n"); + return 0; + } + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "comm event unsupported, event: %d\n", event->type); + + return -1; +} + +void udma_event(struct hinic5_lld_dev *lld_dev, void *adapter, + struct hinic5_event_info *nic_event) +{ + struct udma_device *udma_dev = (struct udma_device *)adapter; + struct ubcore_event ubc_event = { NULL }; + int ret = -1; + + udma_pr_info(UDMA_M_RESOURCE, "udma_event UB\n"); + + if (adapter == NULL) { + udma_pr_err(UDMA_M_RC_TABLE, + "event, input params adapter is null\n"); + return; + } + + if (nic_event == NULL) { + udma_pr_err(UDMA_M_RESOURCE, + "event, input params nic_event is null\n"); + return; + } + + /* + * nic perceive to event (such as link up/down), will via sdk inform each (count word) function on of service driver, UB only disable care this function on of event, + * for at not belong at ub device event perform intercept handle. + */ + if (!hinic5_support_ub(lld_dev->hwdev, NULL)) { + udma_pr_err(UDMA_M_RESOURCE, + "the device(func id:%u) not support UB service\n", + hinic5_global_func_id(lld_dev->hwdev)); + return; + } + + ubc_event.ub_dev = &udma_dev->ub_dev; + ubc_event.element.port_id = hinic5_physical_port_id(udma_dev->hwdev); + + switch (nic_event->service) { + case EVENT_SRV_NIC: + ret = udma_set_nic_event(udma_dev, nic_event, + &ubc_event.event_type); + break; + + case EVENT_SRV_COMM: + ret = udma_set_comm_event(udma_dev, nic_event, + &ubc_event.event_type); + break; + + default: + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Event from svc(%u) is not supported, event(%u)\n", + nic_event->service, nic_event->type); + } + + if (ret != 0) + return; + + ubcore_dispatch_async_event(&ubc_event); +} + +int udma_dev_event(struct notifier_block *netdev_nb, unsigned long event, + void *ptr) +{ + struct ubcore_event ubc_event = { NULL }; + struct udma_device *udma_dev = NULL; + struct net_device *ndev = NULL; + + if (netdev_nb == NULL || ptr == NULL) + return NOTIFY_BAD; + + udma_dev = container_of(netdev_nb, struct udma_device, netdev_nb); + ndev = netdev_notifier_info_to_dev( + (const struct netdev_notifier_info *)ptr); + if (is_vlan_dev(ndev)) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "event: %lu, vlan dev event: %s, skip\n", event, + ndev->name); + return NOTIFY_DONE; + } + + if (udma_dev->ndev != ndev) + return NOTIFY_DONE; + + ubc_event.ub_dev = &udma_dev->ub_dev; + ubc_event.element.port_id = hinic5_physical_port_id(udma_dev->hwdev); + + if (event == NETDEV_UP) { + if (test_and_set_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) != 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set up\n"); + return NOTIFY_DONE; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_UP\n"); + ubc_event.event_type = UBCORE_EVENT_PORT_ACTIVE; + } else if (event == NETDEV_DOWN) { + if (test_and_clear_bit( + UB_DEV_STATUS_BIT_PORT_LINK, + (volatile unsigned long *)&udma_dev->status) == 0) { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "device link already set down\n"); + return NOTIFY_DONE; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "Report NIC LINK_DOWN\n"); + ubc_event.event_type = UBCORE_EVENT_PORT_DOWN; + } else { + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "unsupported event: %lu\n", event); + return NOTIFY_DONE; + } + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "dispatch async event, event_type: %u, port_id: %u\n", + ubc_event.event_type, ubc_event.element.port_id); + + ubcore_dispatch_async_event(&ubc_event); + return NOTIFY_OK; +} + +int udma_alloc_fake_dma_page(struct udma_device *udma_dev, + struct ub_cmd_func_attr *func_attr) +{ + u64 fake_dma_pa = 0; + void *fake_dma_va = NULL; + if (udma_dev->ub_cap.net_dev_cap.is_tpf != 0) { + fake_dma_va = dma_alloc_coherent(udma_dev->dev, + (size_t)PAGE_SIZE_4K, + &fake_dma_pa, GFP_KERNEL); + if (fake_dma_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "failed to alloc fake dma page\n"); + return -EFAULT; + } + udma_info(udma_dev->dev, UDMA_M_RESOURCE, + "udma alloc fake dma page: va 0x%p\n", fake_dma_va); + func_attr->attr2_mask.bs.fake_dma_pa_en = 1; + func_attr->fake_dma_pa = fake_dma_pa; + udma_dev->fake_dma_addr.pa = fake_dma_pa; + udma_dev->fake_dma_addr.va = fake_dma_va; + } + return 0; +} + +int udma_init_res(struct udma_device *udma_dev) +{ + struct ub_cmd_func_attr func_attr = { 0 }; + int ret; + + ret = hinic5_stateful_init(udma_dev->hwdev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "Failed to init stateful resource\n"); + return ret; + } + + ret = udma_init_comp_priv(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RESOURCE, + "init comp_priv failed, ret(%d)\n", ret); + goto err_init_comp_priv; + } + + ret = udma_init_cqm_server(udma_dev); + if (ret != 0) + goto err_init_cqm; + + ret = udma_alloc_ceqs(udma_dev); + if (ret != 0) + goto err_alloc_ceqs; + + ret = hinic5_alloc_db_addr(udma_dev->hwdev, &udma_dev->kernel_db_map, + &udma_dev->kernel_dwqe_map); + if (ret != 0) + goto err_alloc_db; + + if (udma_vram5_get_kexec_flag() == 0) { + ret = udma_set_func_cpu_endian(udma_dev); + if (ret != 0) + goto err_set_cpu_endian; + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, func_en, + UDMA_FUNC_ENABLE); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, + drv_support_mtt_pro, + UDMA_DRV_SUP_MTT_PRO_ENABLE); + + ret = udma_alloc_fake_dma_page(udma_dev, &func_attr); + if (ret != 0) + goto err_alloc_fake_dma_page; + + ret = udma_set_func_attr(udma_dev, + hinic5_global_func_id(udma_dev->hwdev), + &func_attr); + if (ret != 0) + goto err_set_func_state; + } + + ret = udma_get_hw_info(udma_dev); + if (ret != 0) { + goto err_get_hw_info; + } + return ret; + +err_get_hw_info: +err_set_func_state: +err_alloc_fake_dma_page: +err_set_cpu_endian: + hinic5_free_db_addr(udma_dev->hwdev, udma_dev->kernel_db_map, + udma_dev->kernel_dwqe_map); + +err_alloc_db: + udma_free_ceqs(udma_dev); + +err_alloc_ceqs: + cqm5_service_unregister(udma_dev->hwdev, SERVICE_T_UB); + +err_init_cqm: + udma_cleanup_comp_priv(udma_dev); + +err_init_comp_priv: + hinic5_stateful_deinit(udma_dev->hwdev); + + return ret; +} + +void udma_free_res(struct udma_device *udma_dev) +{ + struct ub_cmd_func_attr func_attr = { 0 }; + u16 func_id = hinic5_global_func_id(udma_dev->hwdev); + + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "udma_free_res start\n"); + + if (udma_dev->bdf_info != NULL) { + devm_kfree(udma_dev->dev, udma_dev->bdf_info); + udma_dev->bdf_info = NULL; + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, func_en, + UDMA_FUNC_DISABLE); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, drv_support_mtt_pro, + UDMA_DRV_SUP_MTT_PRO_DISABLE); + udma_set_func_attr(udma_dev, func_id, &func_attr); + hinic5_free_db_addr(udma_dev->hwdev, udma_dev->kernel_db_map, + udma_dev->kernel_dwqe_map); + udma_free_ceqs(udma_dev); + cqm5_service_unregister(udma_dev->hwdev, SERVICE_T_UB); + udma_cleanup_ipsu_table(udma_dev); + udma_del_func_res(udma_dev); + udma_cleanup_comp_priv(udma_dev); + hinic5_stateful_deinit(udma_dev->hwdev); + udma_info(udma_dev->dev, UDMA_M_RESOURCE, "udma_free_res success\n"); +} + +int udma_cmd_dfx_aeqe_drop(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size) +{ + const struct ub_query_inbuf *inbuf = (struct ub_query_inbuf *)buf_in; + + udma_dev->sysfs_res.dfx_ctx.drop_aeqe_en = + (u8)(!!(inbuf->drop_aeqe_en)); + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h new file mode 100644 index 000000000..280f4afe4 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/event/udma_resource.h @@ -0,0 +1,31 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_RESOURCE_H +#define UDMA_RESOURCE_H + +#include "udma_common.h" + +enum udma_func_state { UDMA_FUNC_DISABLE = 0, UDMA_FUNC_ENABLE }; + +enum udma_drv_mtt_pro_state { + UDMA_DRV_SUP_MTT_PRO_DISABLE = 0, + UDMA_DRV_SUP_MTT_PRO_ENABLE +}; + +int udma_init_res(struct udma_device *udma_dev); +void udma_free_res(struct udma_device *udma_dev); +int udma_get_device_cap(struct udma_device *udma_dev, + struct ub_service_cap *ub_cap); +void udma_event(struct hinic5_lld_dev *lld_dev, void *adapter, + struct hinic5_event_info *nic_event); +int udma_dev_event(struct notifier_block *netdev_nb, unsigned long event, + void *ptr); +int udma_cleanup_ipsu_table(struct udma_device *udma_dev); +int udma_cmd_dfx_aeqe_drop(struct udma_device *udma_dev, const void *buf_in, + u32 in_size, void *buf_out, u32 *out_size); +int udma_get_pcie_bdf_info(struct udma_device *udma_dev); +int udma_get_mtt_protect_state(struct udma_device *udma_dev, u32 *mtt_pro_en); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c new file mode 100644 index 000000000..8f280fb38 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.c @@ -0,0 +1,2326 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/dma-mapping.h> +#include <linux/version.h> + +#include "ubcore_uapi.h" +#include "ub_aeqe.h" +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_jfc.h" +#include "udma_misc.h" +#include "udma_ubrc_comp.h" +#include "udma_ucontext.h" +#include "udma_srq.h" +#include "udma_comp.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_srq.h" +#include "udma_abi.h" +#include "udma_base_sq.h" +#include "ub_xqe_format.h" +#include "udma_jetty.h" + +static void udma_print_jetty_cfg(const struct ubcore_jetty_cfg *cfg, + const struct udma_device *udma_dev) +{ + if (cfg == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "ubcore jetty cfg is null.\n"); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "id:%u, priority:%u, trans_mode:%u, eid_index:%u\n", cfg->id, + cfg->priority, cfg->trans_mode, cfg->eid_index); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "jfr_depth:%u, jfs_depth:%u, flag.share_jfr:%u\n", + cfg->jfr_depth, cfg->jfs_depth, cfg->flag.bs.share_jfr); + udma_debug( + udma_dev->dev, UDMA_M_JETTY, + "max_send_sge:%u, max_recv_sge:%u, max_send_rsge:%u, max_inline_data:%u\n", + cfg->max_send_sge, cfg->max_recv_sge, cfg->max_send_rsge, + cfg->max_inline_data); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "rnr_retry:%u, err_timeout:%u, min_rnr_timer:%u\n", + cfg->rnr_retry, cfg->err_timeout, cfg->min_rnr_timer); + if (cfg->send_jfc == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->send_jfc is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->send_jfc->id:%u\n", + cfg->send_jfc->id); + } + if (cfg->recv_jfc == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->recv_jfc is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->recv_jfc->id:%u\n", + cfg->recv_jfc->id); + } + if (cfg->jfr == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->jfr is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, "cfg->jfr->id:%u\n", + cfg->jfr->jfr_id.id); + } + if (cfg->jetty_grp == NULL) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->jetty_grp is null.\n"); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "cfg->jetty_grp->id:%u\n", + cfg->jetty_grp->jetty_grp_id.id); + } +} + +static void udma_jetty_group_kref_release(struct kref *ref_cnt) +{ + struct udma_jetty_grp *jetty_group = + container_of(ref_cnt, struct udma_jetty_grp, ref_cnt); + + complete(&jetty_group->comp); +} + +void udma_jetty_group_put(void *obj) +{ + struct udma_jetty_grp *jetty_group = obj; + + if (jetty_group != NULL) { + kref_put(&jetty_group->ref_cnt, udma_jetty_group_kref_release); + } +} + +void udma_jetty_group_get(void *obj) +{ + struct udma_jetty_grp *jetty_group = obj; + + if (jetty_group != NULL) { + kref_get(&jetty_group->ref_cnt); + } +} + +struct udma_jetty_grp *udma_find_jetty_group(struct udma_device *udev, + u32 jetty_grp_id) +{ + struct udma_jetty_grp *udma_jetty_grp_tmp = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jetty_group_list.node_lock; + mutex_lock(lock); + list_for_each_entry(udma_jetty_grp_tmp, + &udev->device_res->jetty_group_list.node, node) { + if (udma_jetty_grp_tmp->jetty_grp_id == jetty_grp_id) { + udma_jetty_group_get(udma_jetty_grp_tmp); + mutex_unlock(lock); + return udma_jetty_grp_tmp; + } + } + mutex_unlock(lock); + + return NULL; +} + +static void udma_jetty_kref_release(struct kref *ref_cnt) +{ + struct udma_jetty *jetty = + container_of(ref_cnt, struct udma_jetty, ref_cnt); + + complete(&jetty->comp); +} + +void udma_jetty_put(void *obj) +{ + struct udma_jetty *jetty = obj; + + if (jetty != NULL) { + kref_put(&jetty->ref_cnt, udma_jetty_kref_release); + } +} + +void udma_jetty_get(void *obj) +{ + struct udma_jetty *jetty = obj; + + if (jetty != NULL) { + kref_get(&jetty->ref_cnt); + } +} + +struct udma_jetty *udma_find_jetty(struct udma_device *udev, u32 jetty_id) +{ + struct udma_jetty *ub_jetty = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jetty_list.node_lock; + mutex_lock(lock); + list_for_each_entry(ub_jetty, &udev->device_res->jetty_list.node, + node) { + if (ub_jetty->jetty_id == jetty_id) { + udma_jetty_get(ub_jetty); + mutex_unlock(lock); + return ub_jetty; + } + } + mutex_unlock(lock); + + return NULL; +} + +int udma_query_res_jetty(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_val *ret_val = + (struct ubcore_res_jetty_val *)(uintptr_t)val->addr; + struct ubcore_jetty *jetty = NULL; + struct udma_jetty *ub_jetty = NULL; + + ub_jetty = udma_find_jetty(udma_dev, key->key); + if (ub_jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to find jetty, id(%u)\n", key->key); + return -EINVAL; + } + + jetty = &ub_jetty->ubcore_jetty; + ret_val->jetty_id = ub_jetty->jetty_id; + ret_val->jfs_depth = jetty->jetty_cfg.jfs_depth; + + ret_val->priority = jetty->jetty_cfg.priority; + if (ub_jetty->ub_jfr == NULL) { + ret_val->jfr_id = U32_MAX; + } else { + ret_val->jfr_id = ub_jetty->ub_jfr->jfr_id; + } + + if (jetty->jetty_cfg.send_jfc == NULL) { + ret_val->send_jfc_id = U32_MAX; + } else { + ret_val->send_jfc_id = jetty->jetty_cfg.send_jfc->id; + } + + if ((jetty->jetty_cfg.jfr != NULL) && + (jetty->jetty_cfg.jfr->jfr_cfg.jfc != NULL)) { + ret_val->recv_jfc_id = jetty->jetty_cfg.jfr->jfr_cfg.jfc->id; + } else { + ret_val->recv_jfc_id = U32_MAX; + } + + ret_val->state = ub_jetty->state; + udma_jetty_put(ub_jetty); + return 0; +} + +int udma_query_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *ub_jetty = NULL; + struct ub_ta_context_query jetty_ctx = { 0 }; + int ret; + u32 dw_value; + + if (jetty == NULL || jetty->ub_dev == NULL || cfg == NULL || + attr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "query jetty invalid parameter. jetty(%d) or jetty->ub_dev or cfg(%d) or attr(%d) is null.\n", + !!jetty, !!cfg, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + ub_jetty = to_udma_jetty(jetty); + if (ub_jetty->ub_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "ub_jfr is null\n"); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udma_dev, jetty->jetty_id.id, + &ub_jetty->cmdq_rtt, &jetty_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "query jetty(%u) ctx cmdq failed\n", + jetty->jetty_id.id); + return -EIO; + } + + attr->rx_threshold = ub_jetty->ub_jfr->srq.container_warn_th; + + dw_value = cpu_to_be32(jetty_ctx.ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + attr->state = (dw_value >> UDMA_STATE_IN_DW_OFFSET) & + UDMA_STATE_IN_DW_MASK; + *cfg = jetty->jetty_cfg; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "udma query jetty, rx_threshold: %u, state: %u\n", + attr->rx_threshold, attr->state); + + return 0; +} + +static u32 ub_umem_page_count(const struct ubcore_umem *umem) +{ + return (u32)((ALIGN(umem->va + umem->length, PAGE_SIZE) - + ALIGN_DOWN(umem->va, PAGE_SIZE)) / + PAGE_SIZE); +} + +int udma_create_user_mtt(struct udma_device *udma_dev, struct ubcore_umem *umem, + struct mtt *ub_mtt) +{ + u32 npages; + int ret; + + npages = ub_umem_page_count(umem); + ub_mtt->mtt_type = MTT_CMTT_TYPE; + + ret = hmm_mtt_alloc(udma_dev->hwdev, npages, PAGE_SHIFT, ub_mtt, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to alloc mtt\n"); + return ret; + } + + ret = udma_umem_write_mtt(udma_dev, ub_mtt, umem); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to write mtt\n"); + hmm_mtt_free(udma_dev->hwdev, ub_mtt, SERVICE_T_UB); + return ret; + } + + return 0; +} + +void udma_destroy_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt) +{ + hmm_mtt_free(udma_dev->hwdev, ub_mtt, SERVICE_T_UB); +} + +struct ubcore_umem *udma_get_umem(struct udma_device *udma_dev, u64 buf_addr, + u32 buf_size) +{ + union ubcore_umem_flag access = { 0 }; + + access.bs.non_pin = 0; + access.bs.writable = 1; + return ubcore_umem_get(&udma_dev->ub_dev, buf_addr, buf_size, access); +} + +void udma_put_umem(struct ubcore_umem *umem) +{ + ubcore_umem_release(umem); +} + +static int udma_alloc_jetty_context(struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_device *ub_dev) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct tag_cqm_qpc_mpt *jettyc_info = NULL; + u32 low_bit_mode = CQM_XID_LOW_BIT_NONE; + u32 xid_low = 0; + u32 qpn_tmp; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "to_udma_dev failed.\n"); + return -EINVAL; + } + + if (udma_inc_jfc_used_cnt(cfg->send_jfc) == 1) { + low_bit_mode = + (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) ? + CQM_XID_LOW_BIT_1_1_1 : + CQM_XID_LOW_BIT_0_0_1; + xid_low = cfg->send_jfc->id; + } + qpn_tmp = udma_get_qpn_mod(udma_dev, low_bit_mode, xid_low); + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + jettyc_info = udma_object_qpc_create( + udma_dev, UBEP_JETTY_CONTEXT_SIZE, jetty, + &udma_dev->jettyn_record, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + /* stub get jetty information, call mpu interface use */ + udma_info( + udma_dev->dev, UDMA_M_MISC, + "get qpc object, record(%u), xid(%u), start(%u), max_jetty_num(%u)\n", + udma_dev->jettyn_record.num, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + if (jettyc_info == NULL) { + udma_dec_jfc_used_cnt(cfg->send_jfc); + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create jettyc by cqm object\n"); + return -ENOMEM; + } + + if (cfg->send_jfc != NULL) + udma_info(udma_dev->dev, UDMA_M_JETTY, + "get jetty id(%u), jfc id(%u), jfc used cnt(%d)\n", + jettyc_info->xid, cfg->send_jfc->id, + udma_read_jfc_used_cnt(cfg->send_jfc)); + + jetty->jettyc_info = jettyc_info; + + return 0; +} + +static void udma_free_jetty_context(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct ubcore_jfc *jfc) +{ + if (jetty->jettyc_info != NULL) { + cqm5_object_delete(&(jetty->jettyc_info->object)); + jetty->jettyc_info = NULL; + udma_dec_jfc_used_cnt(jfc); + } +} + +static int udma_jetty_cache_out_cmd(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)sizeof(u32), &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc wb_va\n"); + return -ENOMEM; + } + + ret = udma_cmd_jetty_cache_out_ctx(udma_dev, jetty, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +int udma_alloc_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc, bool is_fixed) +{ + u32 response_depth = (u32)roundup_pow_of_two(MAX_DEST_RM_ATOMIC); + int ret; + + response_depth = + is_fixed ? response_depth : + udma_dev->ub_cap.dev_cap_res.udma_res.ubrc_depth; + if (*rsp_depth != response_depth) { + if (*rsp_depth > 0) + udma_ubrc_free(udma_dev, ubrc); + + ret = udma_ubrc_alloc(udma_dev, response_depth, ubrc); + if (ret != 0) { + *rsp_depth = 0; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc ubrc, ret(%d)\n", ret); + return ret; + } + } + + *rsp_depth = response_depth; + + return 0; +} + +void udma_free_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc) +{ + if (*rsp_depth > 0) { + udma_ubrc_free(udma_dev, ubrc); + } + + *rsp_depth = 0; +} + +static int udma_update_jetty_group(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct ubcore_jetty_group *jetty_grp, + enum UB_JETTY_GRP_UPDATE_TYPE_E type) +{ + char *log = (type == UB_JETTY_GRP_ADD_JETTY) ? "add to" : "delete from"; + int ret = 0; + + if (jetty_grp != NULL) { + ret = udma_cmd_jetty_grp_update_ctx( + udma_dev, jetty, jetty_grp->jetty_grp_id.id, type); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "JETTY(%u) %s jetty group(%u) failed.\n", + jetty->jetty_id, log, + jetty_grp->jetty_grp_id.id); + } else { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "JETTY(%u) %s jetty group(%u) success.\n", + jetty->jetty_id, log, + jetty_grp->jetty_grp_id.id); + } + } + return ret; +} + +static void udma_create_jetty_kernel_set_attr(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct tag_cqm_queue *sq_buf_info) +{ + jetty->sq_info = sq_buf_info; + jetty->sdb.db_record = (__be32 *)(void *)(&sq_buf_info->q_header_vaddr + ->doorbell_record); + jetty->sdb.dma = sq_buf_info->q_header_paddr; + + sq_buf_info->q_header_vaddr->doorbell_record = 0; + jetty->page_shift = UB_ILOG2(sq_buf_info->q_room_buf_1.buf_size); +} + +static int udma_jetty_init_sq(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct tag_cqm_queue *sq_buf_info) +{ + u32 dev_max_inline_size = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + u32 depth = cfg->jfs_depth; + struct udma_send_queue *sq = &jetty->sq; + u32 wr_max_length; + + spin_lock_init(&sq->lock); + sq->baseblk_shift = depth_align_power2(UB_JFS_WQEBB); + sq->buf_size = udma_get_kernel_sq_size(udma_dev, depth, &wr_max_length); + sq->baseblk_cnt = sq->buf_size >> sq->baseblk_shift; + sq->max_wr_num = + (sq->baseblk_cnt / (wr_max_length >> sq->baseblk_shift)) - 1; + sq->buf = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + jetty->queue_addr.va : + sq_buf_info->q_room_buf_1.direct.va; + sq->buf_curr = sq->buf; + sq->max_inline_size = (cfg->max_inline_data > dev_max_inline_size) ? + dev_max_inline_size : + cfg->max_inline_data; + + (void)memset_s(sq->buf, sq->buf_size, 0xff, sq->buf_size); + + sq->pi = 0; + sq->head = 0; + sq->tail = 0; + + sq->wrid_size = sq->baseblk_cnt / UB_MIN_WR_WQEBB_NUM; + sq->wrid = (u64 *)kzalloc(sq->wrid_size * sizeof(u64), GFP_KERNEL); + if (sq->wrid == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create sq wrid.\n"); + return -ENOMEM; + } + + sq->is_hw_flush_done = false; + + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->baseblk_shift:%u.\n", + sq->baseblk_shift); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->buf_size:%u.\n", + sq->buf_size); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->baseblk_cnt:%u.\n", + sq->baseblk_cnt); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->max_wr_num:%u.\n", + sq->max_wr_num); + udma_info(udma_dev->dev, UDMA_M_JETTY, "sq->max_inline_size:%u.\n", + sq->max_inline_size); + + return 0; +} + +static int udma_create_jetty_context_cmd(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + int ret = 0; + + ret = udma_alloc_jetty_context(jetty, cfg, ub_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma alloc jettyc failed.\n"); + return ret; + } + + jetty->jetty_id = jetty->jettyc_info->xid; + jetty->ubcore_jetty.jetty_id.id = jetty->jetty_id; + + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) { + ret = udma_alloc_opt_ubrc(udma_dev, &jetty->rsp_depth, + &jetty->ubrc, false); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to alloc ubrc, ret(%d)\n", ret); + goto err_and_free_jetty_context; + } + } + + ret = udma_cmd_jetty_create_ctx(udma_dev, jetty, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma create jetty(%u) failed, ret %d.\n", + jetty->jetty_id, ret); + goto err_and_free_ubrc; + } + jetty->state = UBCORE_JETTY_STATE_READY; + + return 0; + +err_and_free_ubrc: + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) + udma_free_opt_ubrc(udma_dev, &jetty->rsp_depth, &jetty->ubrc); +err_and_free_jetty_context: + udma_free_jetty_context(udma_dev, jetty, cfg->send_jfc); + return ret; +} + +static int udma_trans_mode_check(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg) +{ + if (cfg->trans_mode != UBCORE_TP_RC && + cfg->trans_mode != UBCORE_TP_RM && + cfg->trans_mode != UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "invalid trans_mode:%u\n", + cfg->trans_mode); + return -EINVAL; + } + + if (cfg->trans_mode == UBCORE_TP_RC) { + if (cfg->flag.bs.order_type != UBCORE_OT) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "not support trans_mode:rc, order_type:%u\n", + cfg->flag.bs.order_type); + return -EINVAL; + } + + if (cfg->jetty_grp != NULL) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "rs static mode does not support jetty group\n"); + return -EINVAL; + } + } + return 0; +} + +static int udma_cap_param_check(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg) +{ + u32 cap_max_inline = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + u16 cap_jfs_depth = udma_dev->ub_cap.dev_cap.max_jfs_depth; + u16 cap_jfs_sge = udma_dev->ub_cap.dev_cap.max_jfs_sge; + u32 cfg_max_inline = cfg->max_inline_data; + u16 cfg_jfs_sge = cfg->max_send_sge; + u32 cfg_jfs_depth = cfg->jfs_depth; + + if ((cfg->flag.bs.share_jfr == 0) || (cfg->jfr == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "ub share jfr flag is %u, shared jfr is %s null.\n", + cfg->flag.bs.share_jfr, + (cfg->jfr == NULL) ? "" : "not"); + return -EINVAL; + } + + if (udma_trans_mode_check(udma_dev, cfg) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "not support trans mode.\n"); + return -EINVAL; + } + + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "tpf not support this function.\n"); + return -EINVAL; + } + + if (cfg_jfs_depth == 0 || cfg_jfs_depth > cap_jfs_depth || + cfg_jfs_sge > cap_jfs_sge || cfg_max_inline > cap_max_inline) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Invalid parameter: cfg_jfs_depth: %u, cap_jfs_depth: %u," + "cfg_jfs_sge: %u, cap_jfs_sge: %u cfg_max_inline: %u cap_max_inline: %u.\n", + cfg_jfs_depth, cap_jfs_depth, cfg_jfs_sge, cap_jfs_sge, + cfg_max_inline, cap_max_inline); + return -EINVAL; + } + + if (cfg->flag.bs.outorder_comp != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "error: outorder_comp: %u\n", + cfg->flag.bs.outorder_comp); + return -EINVAL; + } + + return 0; +} + +static int init_ub_jetty_param_kernel(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + u32 wr_max_length; + u32 depth = cfg->jfs_depth; + + jetty->sq_buf_len = + udma_get_kernel_sq_size(udma_dev, depth, &wr_max_length); + jetty->ubcore_jetty.jetty_cfg = *cfg; + jetty->trans_mode = cfg->trans_mode; + jetty->ub_mtt.mtt_type = MTT_CMTT_TYPE; + + return 0; +} + +int udma_alloc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, struct queue_addr_info *queue_addr) +{ + u8 mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + u32 buf_size; + u32 queue_len; + int ret; + + buf_size = mtt_pro_en == 1 ? PAGE_SIZE : buf_len; + *cqm_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, buf_size, + object_priv, true, CQM_INDEX_INVALID, 0, 0); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create queue by cqm object.\n"); + return -ENOMEM; + } + + if (mtt_pro_en == 1) { + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + } + + return 0; +} + +int udma_fill_kernel_mtt(struct mtt *ub_mtt, dma_addr_t pa) +{ + ub_mtt->mtt_type = MTT_CMTT_TYPE; + ub_mtt->mtt_paddr = pa; + ub_mtt->mtt_layers = MTT_ZERO_LAYER; + return 0; +} + +int udma_init_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + cqm_queue_s *cqm_buf_info, dma_addr_t pa) +{ + u32 page_shift; + int ret; + + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return udma_fill_kernel_mtt(ub_mtt, pa); + } + + page_shift = UB_ILOG2(cqm_buf_info->q_room_buf_1.buf_size); + ub_mtt->mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(udma_dev->hwdev, + cqm_buf_info->q_room_buf_1.buf_number, page_shift, + ub_mtt, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to alloc MTT.\n"); + return ret; + } + + ret = udma_buf_write_mtt(udma_dev, ub_mtt, &cqm_buf_info->q_room_buf_1); + if (ret != 0) { + hmm_mtt_free(udma_dev->hwdev, ub_mtt, SERVICE_T_UB); + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed to write MTT.\n"); + return ret; + } + + return 0; +} + +void udma_uninit_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt) +{ + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return; + } + + if (ub_mtt->mtt_seg != NULL) { + udma_destroy_user_mtt(udma_dev, ub_mtt); + ub_mtt->mtt_seg = NULL; + } +} + +int udma_free_queue_resource(struct udma_device *udma_dev, + cqm_queue_s *cqm_buf_info, + struct queue_addr_info *queue_addr) +{ + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + hinic5_dma_free_coherent_align(udma_dev->dev, + &queue_addr->mem_align); + } + + if (cqm_buf_info != NULL) { + cqm5_object_delete(&cqm_buf_info->object); + } + + return 0; +} + +static int udma_jetty_init_sq_kernel(struct udma_device *udma_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + struct tag_cqm_queue *sq_buf_info = NULL; + int ret; + + ret = udma_alloc_queue_resource(udma_dev, &sq_buf_info, jetty, + jetty->sq_buf_len, &jetty->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to alloc queue resource, ret = %d.\n", ret); + return ret; + } + + ret = udma_init_kernel_mtt(udma_dev, &jetty->ub_mtt, sq_buf_info, + jetty->queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_jetty_init_sq(udma_dev, jetty, cfg, sq_buf_info); + if (ret != 0) { + goto err_jetty_init_sq; + } + + udma_create_jetty_kernel_set_attr(udma_dev, jetty, sq_buf_info); + + return 0; + +err_jetty_init_sq: + udma_uninit_kernel_mtt(udma_dev, &jetty->ub_mtt); +err_init_kernel_mtt: + udma_free_queue_resource(udma_dev, sq_buf_info, &jetty->queue_addr); + jetty->sq_info = NULL; + + return ret; +} + +static void udma_jetty_deinit_sq_kernel(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + if (jetty->sq.wrid != NULL) { + kfree(jetty->sq.wrid); + jetty->sq.wrid = NULL; + } + + udma_uninit_kernel_mtt(udma_dev, &jetty->ub_mtt); + udma_free_queue_resource(udma_dev, jetty->sq_info, &jetty->queue_addr); + jetty->sq_info = NULL; +} + +static int udma_create_jetty_kernel(struct ubcore_device *ub_dev, + struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + ret = init_ub_jetty_param_kernel(udma_dev, jetty, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "init jetty param failed\n"); + return ret; + } + + jetty->ub_jfr = to_udma_jfr(cfg->jfr); + + ret = udma_jetty_init_sq_kernel(udma_dev, jetty, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "init jetty sq failed\n"); + return ret; + } + + ret = udma_create_jetty_context_cmd(udma_dev, jetty, cfg, NULL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create jetty cmd failed\n"); + goto err_jetty_cmd; + } + + return 0; + +err_jetty_cmd: + udma_jetty_deinit_sq_kernel(udma_dev, jetty); + + return ret; +} + +static void udma_jetty_init(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg, + struct ub_create_jetty_ucmd *ucmd, + struct udma_jetty *jetty) +{ + jetty->ubcore_jetty.jetty_cfg = *cfg; + jetty->trans_mode = cfg->trans_mode; + jetty->rq_buf_len = ucmd->index_q_buf_size; + + jetty->sq_buf_len = ucmd->buf_size; + jetty->sq.baseblk_cnt = ucmd->sq_wqebb_cnt; + + if (udma_dev == NULL || udma_dev->dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!udma_dev); + return; + } + + udma_debug( + udma_dev->dev, UDMA_M_JETTY, + "index_q_buf_addr: %llu, index_q_buf_size: %u, sq_buf_len: %u, baseblk_cnt: %d.\n", + ucmd->index_q_buf_addr, ucmd->index_q_buf_size, + jetty->sq_buf_len, jetty->sq.baseblk_cnt); + + jetty->ub_jfr = to_udma_jfr(cfg->jfr); +} + +static int udma_get_dma_addr(struct udma_device *udma_dev, uint64_t uva_addr, + dma_addr_t *dma_addr) +{ + struct udma_queue_buffer_info *queue_info = NULL; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma = NULL; + int ret = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) + mmap_read_lock(mm); +#else + down_read(&mm->mmap_sem); +#endif + + vma = find_vma(mm, uva_addr); + if (vma == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "find vma failed.\n"); + ret = -ENOMEM; + goto err_find_vma; + } + + queue_info = (struct udma_queue_buffer_info *)vma->vm_private_data; + *dma_addr = queue_info->dma_addr; + +err_find_vma: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) + mmap_read_unlock(mm); +#else + up_read(&mm->mmap_sem); +#endif + return ret; +} + +int udma_fill_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + u64 buf_addr) +{ + dma_addr_t dma_addr; + int ret; + + ret = udma_get_dma_addr(udma_dev, buf_addr, &dma_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma get phys addr failed, ret = %d\n", ret); + return ret; + } + + ub_mtt->mtt_type = MTT_CMTT_TYPE; + ub_mtt->mtt_paddr = dma_addr; + ub_mtt->mtt_layers = MTT_ZERO_LAYER; + return 0; +} + +int udma_init_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem **umem, u64 buf_addr, u32 buf_size) +{ + int ret; + + udma_debug(udma_dev->dev, UDMA_M_JETTY, "mtt_pro_en = %d.\n", + udma_dev->ub_cap.dev_cap.mtt_pro_en); + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return udma_fill_user_mtt(udma_dev, ub_mtt, buf_addr); + } + + *umem = udma_get_umem(udma_dev, buf_addr, buf_size); + if (IS_ERR_OR_NULL(*umem)) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to get umem, ret = %d.\n", + PTR_ERR_OR_ZERO(*umem)); + return -ENOMEM; + } + + ret = udma_create_user_mtt(udma_dev, *umem, ub_mtt); + if (ret != 0) { + ubcore_umem_release(*umem); + *umem = NULL; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma create user mtt failed, ret = %d.\n", ret); + } + + return ret; +} + +void udma_uninit_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem *umem) +{ + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + return; + } + udma_destroy_user_mtt(udma_dev, ub_mtt); + ub_mtt->mtt_seg = NULL; + ubcore_umem_release(umem); +} + +/* store queue info, just for jetty/jfr dfx function */ +void udma_fill_queue_addr_info(struct queue_addr_info *queue_addr, + dma_addr_t queue_pa, u32 len, u8 mtt_pro_en) +{ + if (mtt_pro_en == 1) { + queue_addr->pa = queue_pa; + queue_addr->va = phys_to_virt(queue_pa); + queue_addr->size = len; + } +} + +static int udma_jetty_init_sq_user(struct udma_device *udma_dev, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, + struct udma_jetty *jetty) +{ + struct ub_create_jetty_ucmd *ucmd = uinfo->ucmd; + struct udma_context *udma_ctx = uinfo->udma_ctx; + int ret; + + ret = udma_init_user_mtt(udma_dev, &jetty->ub_mtt, &jetty->umem, + ucmd->buf_addr, ucmd->buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma init user mtt failed, ret = %d.\n", ret); + return ret; + } + udma_fill_queue_addr_info(&jetty->queue_addr, jetty->ub_mtt.mtt_paddr, + jetty->sq_buf_len, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_db_map_user(udma_ctx, ucmd->sdb_addr, &jetty->sdb); + if (ret != 0) { + udma_uninit_user_mtt(udma_dev, &jetty->ub_mtt, jetty->umem); + jetty->umem = NULL; + udma_err(udma_dev->dev, UDMA_M_JETTY, + "udma db map user failed.\n"); + } + + return ret; +} + +static void udma_jetty_deinit_sq_user(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct udma_context *udma_ctx) +{ + udma_db_unmap_user(udma_ctx, &jetty->sdb); + udma_uninit_user_mtt(udma_dev, &jetty->ub_mtt, jetty->umem); + jetty->umem = NULL; +} + +static void udma_jetty_deinit_sq(struct udma_device *udma_dev, + struct udma_jetty *jetty, + struct udma_context *udma_ctx) +{ + if (udma_ctx == NULL) { + udma_jetty_deinit_sq_kernel(udma_dev, jetty); + } else { + udma_jetty_deinit_sq_user(udma_dev, jetty, udma_ctx); + } +} + +static int udma_create_jetty_user(struct ubcore_device *ub_dev, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, + struct udma_jetty *jetty) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + udma_jetty_init(udma_dev, cfg, uinfo->ucmd, jetty); + + ret = udma_jetty_init_sq_user(udma_dev, cfg, uinfo, jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "init jetty sq failed\n"); + return ret; + } + + ret = udma_create_jetty_context_cmd(udma_dev, jetty, cfg, uinfo); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create jetty cmd failed\n"); + goto err_jetty_cmd; + } + + return ret; + +err_jetty_cmd: + udma_jetty_deinit_sq_user(udma_dev, jetty, uinfo->udma_ctx); + + return ret; +} + +static int udma_destroy_jetty_one(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + struct udma_context *udma_ctx = to_udma_ctx(jetty->ubcore_jetty.uctx); + struct ubcore_jetty_group *jetty_grp = + jetty->ubcore_jetty.jetty_cfg.jetty_grp; + int ret; + + ret = udma_update_jetty_group(udma_dev, jetty, jetty_grp, + UB_JETTY_GRP_DEL_JETTY); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "delete jetty from jetty_grp failed"); + return ret; + } + + ret = udma_cmd_jetty_delete_ctx(udma_dev, jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to delete jetty(0x%06x).\n", jetty->jetty_id); + return ret; + } + + ret = udma_jetty_cache_out_cmd(udma_dev, jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty:0x%06x cache invalid.\n", jetty->jetty_id); + return ret; + } + udma_delete_list(udma_dev, &jetty->node, + &udma_dev->device_res->jetty_list); + udma_jetty_put(jetty); + wait_for_completion(&jetty->comp); + if (jetty->trans_mode == UBCORE_TP_RM || + jetty->trans_mode == UBCORE_TP_RC) + udma_free_opt_ubrc(udma_dev, &jetty->rsp_depth, &jetty->ubrc); + + udma_free_jetty_context(udma_dev, jetty, + jetty->ubcore_jetty.jetty_cfg.send_jfc); + + udma_jetty_deinit_sq(udma_dev, jetty, udma_ctx); + + jetty->state = UBCORE_JETTY_STATE_RESET; + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "JETTY destroyed successfully, id: %u.\n", jetty->jetty_id); + + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_JETTY, jetty->jetty_id, + &jetty->cmdq_rtt); + kfree(jetty); + + return 0; +} + +static struct udma_jetty * +udma_create_jetty_one(struct ubcore_device *ub_dev, + const struct ubcore_jetty_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, u8 *cos) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_jetty *jetty = NULL; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return NULL; + } + + if (udma_get_dcb_cfg_cos(udma_dev, NULL, cfg->priority, cos, 1) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to get cos from dcb info, pri:%u\n", + cfg->priority); + return NULL; + } + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "Get cos from dcb info successful, cos:%u\n", *cos); + + jetty = (struct udma_jetty *)kzalloc(sizeof(struct udma_jetty), + GFP_KERNEL); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "alloc JETTY failed.\n"); + return NULL; + } + + jetty->cqm_priv_type = UDMA_JETTY_TYPE_JETTY; + jetty->cos = *cos; + udma_cmdq_rtt_statistic_init(&jetty->cmdq_rtt); + if (uinfo->udma_ctx == NULL) { + if (udma_create_jetty_kernel(ub_dev, jetty, cfg) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create kernel jetty failed\n"); + goto err_alloc_jetty; + } + } else { + if (udma_create_jetty_user(ub_dev, cfg, uinfo, jetty) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "create user jetty failed\n"); + goto err_alloc_jetty; + } + } + kref_init(&jetty->ref_cnt); + init_completion(&jetty->comp); + udma_store_list(udma_dev, &jetty->node, + &udma_dev->device_res->jetty_list); + + if (udma_update_jetty_group(udma_dev, jetty, cfg->jetty_grp, + UB_JETTY_GRP_ADD_JETTY) != 0) { + if (uinfo->udma_ctx == NULL) { + jetty->ubcore_jetty.uctx = NULL; + } else { + jetty->ubcore_jetty.uctx = &uinfo->udma_ctx->base; + } + jetty->ubcore_jetty.ub_dev = &udma_dev->ub_dev; + jetty->ubcore_jetty.jetty_cfg.jetty_grp = NULL; + udma_destroy_jetty_one(udma_dev, jetty); + return NULL; + } + + return jetty; + +err_alloc_jetty: + kfree(jetty); + + return NULL; +} + +int udma_clear_pi_on_chip(struct udma_device *udma_dev, u32 id, u8 cos) +{ + union tag_ub_sq_db_u jetty_db; + int ret; + u8 sq_pi_on_chip = 0; + + if (udma_dev->ub_cap.dev_cap.feature.bs.pi_on_chip_en != 0) + sq_pi_on_chip = + (u8)(id < udma_dev->ub_cap.dev_cap.max_pi_on_chip_num); + + if (sq_pi_on_chip == 0) + return 0; + + udma_base_construct_sq_db(&jetty_db, id, 0, cos); + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, 0, + jetty_db.sq_db_value); + + wmb(); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "xid:0x%x, jetty_db.sq_db_value:0x%llx, ret %d\n", id, + jetty_db.sq_db_value, ret); + return ret; +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *ub_dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_create_jfs_uinfo uinfo = { 0 }; + struct udma_jetty *jetty = NULL; + int ret; + u8 cos; + + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "create jetty invalid parameter. udma_dev(%d), cfg(%d).\n", + !!udma_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + udma_print_jetty_cfg(cfg, udma_dev); + + if (udma_cap_param_check(udma_dev, cfg) != 0) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "jetty cfg invalid parameter.\n"); + return ERR_PTR(-EINVAL); + } + + ret = udma_create_jfs_uinfo_init(udma_dev, udata, &uinfo); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "init uinfo failed\n"); + return ERR_PTR(ret); + } + + jetty = udma_create_jetty_one(ub_dev, cfg, &uinfo, &cos); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Create jetty faild\n"); + goto err_and_deinit_uinfo; + } + + if (udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, jetty->round, + 1) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to copy to user driver\n"); + goto err_and_free_jetty; + } + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "JETTY create successfully, id:%u, mode:%d.\n", + jetty->ubcore_jetty.jetty_id.id, + jetty->ubcore_jetty.jetty_cfg.trans_mode); + + (void)udma_clear_pi_on_chip(udma_dev, jetty->jetty_id, jetty->cos); + + return &jetty->ubcore_jetty; + +err_and_free_jetty: + jetty->ubcore_jetty.uctx = udma_get_uctx(udata); + udma_destroy_jetty_one(udma_dev, jetty); +err_and_deinit_uinfo: + udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, 0, 0); + + return ERR_PTR(-EFAULT); +} + +int udma_destroy_jetty(struct ubcore_jetty *jetty) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + + if (jetty == NULL || jetty->ub_dev == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "destroy jetty invalid parameter. jetty(%d) or jetty->ub_dev is null.\n", + !!jetty); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + + udma_info(udma_dev->dev, UDMA_M_JETTY, "Destroy jetty, jetty_id: %u\n", + udma_jetty->jetty_id); + + return udma_destroy_jetty_one(udma_dev, udma_jetty); +} + +int udma_modify_jetty_parse_udata(struct udma_device *udma_dev, + struct udma_send_queue *sq, + struct ubcore_udata *udata) +{ + u32 ucmd_len = sizeof(struct ub_modify_jetty_uinfo); + struct ub_modify_jetty_uinfo *uinfo = NULL; + int ret = -EFAULT; + + if ((udata == NULL) || (udata->uctx == NULL) || + (udata->udrv_data == NULL)) + return 0; + + if (udata->udrv_data->in_len != ucmd_len) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "in_len(%u) != ucmd_len(%u)\n", + udata->udrv_data->in_len, ucmd_len); + return -EINVAL; + } + + if ((uinfo = kzalloc(ucmd_len, GFP_KERNEL)) == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Zalloc ucmd failed: %u\n", ucmd_len); + return -ENOMEM; + } + + if (copy_from_user(uinfo, (void *)(uintptr_t)udata->udrv_data->in_addr, + ucmd_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Copy from user failed\n"); + goto leave; + } + + sq->pi = uinfo->sq_pi; + ret = 0; + udma_info(udma_dev->dev, UDMA_M_JETTY, "Update sq pi(%u)\n", sq->pi); + +leave: + if (uinfo != NULL) + kfree(uinfo); + + return ret; +} + +int udma_modify_jetty(struct ubcore_jetty *jetty, + struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_jetty *udma_jetty = NULL; + struct udma_device *udma_dev = NULL; + int ret; + + if (jetty == NULL || jetty->ub_dev == NULL || attr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "modify jetty invalid parameter. jetty(%d) or jetty->ub_dev or attr(%d) is null\n", + !!jetty, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "Modify jetty, mask:0x%08x, rx_threshold: %u, state: %d\n", + attr->mask, attr->rx_threshold, attr->state); + + if ((attr->mask & UBCORE_JETTY_RX_THRESHOLD) != 0 || + (attr->mask & UBCORE_JETTY_STATE) == 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Wrong mask: 0x%08x\n", + attr->mask); + return -EINVAL; + } + + if (attr->state < UBCORE_JETTY_STATE_RESET || + attr->state > UBCORE_JETTY_STATE_ERROR) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Invalid jetty state: %d\n", attr->state); + return -EINVAL; + } + + if (udma_modify_jetty_parse_udata(udma_dev, &udma_jetty->sq, udata) != + 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed parse udata\n"); + return -EINVAL; + } + + ret = udma_cmd_jetty_modify_ctx(udma_dev, udma_jetty, attr->state, + udata); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to modify jetty(%u)\n", udma_jetty->jetty_id); + return ret; + } + + udma_jetty->state = attr->state; + udma_jetty->sq.is_hw_flush_done = false; + + udma_info(udma_dev->dev, UDMA_M_JETTY, "Modify jetty %u, ret %d\n", + udma_jetty->jetty_id, ret); + + return ret; +} + +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct ubcore_tjetty *tjetty = NULL; + + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "import jetty invalid parameter. udma_dev(%d), cfg(%d).\n", + !!udma_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + if (cfg->type != UBCORE_JETTY) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "remote jetty type(%d) invalid \n", cfg->type); + return ERR_PTR(-EINVAL); + } + + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "remote jetty token_policy=%u dont support\n", + cfg->flag.bs.token_policy); + return ERR_PTR(-EINVAL); + } + + if (cfg->tp_type == UBCORE_CTP) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "remote jetty tp_type %d invalid\n", cfg->tp_type); + return ERR_PTR(-EINVAL); + } + + if (cfg->trans_mode == UBCORE_TP_RC) { + if (cfg->flag.bs.order_type != UBCORE_OT) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "not support trans_mode:rc, order_type:%u\n", + cfg->flag.bs.order_type); + return ERR_PTR(-EINVAL); + } + + if (cfg->flag.bs.share_tp == 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "rs static mode must support share_tp\n"); + return ERR_PTR(-EINVAL); + } + } + + tjetty = (struct ubcore_tjetty *)kcalloc( + 1, sizeof(struct ubcore_tjetty), GFP_KERNEL); + if (tjetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "alloc tjetty failed\n"); + return ERR_PTR(-ENOMEM); + } + + return tjetty; +} + +int udma_unimport_jetty(struct ubcore_tjetty *tjetty) +{ + if (tjetty == NULL) { + udma_pr_err(UDMA_M_JETTY, + "unimport jetty invalid parameter. tjetty(%d).\n", + !!tjetty); + return -EINVAL; + } + kfree(tjetty); + + return 0; +} + +int udma_bind_jetty(struct ubcore_jetty *jetty, struct ubcore_tjetty *tjetty, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + int ret; + + if (jetty == NULL || tjetty == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "bind jetty invalid parameter. jetty(%d), tjetty(%d).\n", + !!jetty, !!tjetty); + return -EINVAL; + } + + if (jetty->ub_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "Invalid ub_dev\n"); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "udma bind jetty, jetty id:%u, remote jetty id:%u\n", + jetty->jetty_id.id, tjetty->cfg.id.id); + + if (jetty->jetty_cfg.trans_mode != UBCORE_TP_RC || + tjetty->cfg.trans_mode != UBCORE_TP_RC) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Invalid jetty type, jetty mode is %u, remote jetty mode is %u.\n", + (u32)jetty->jetty_cfg.trans_mode, + (u32)tjetty->cfg.trans_mode); + return -EINVAL; + } + + if (jetty->jetty_cfg.flag.bs.order_type != + tjetty->cfg.flag.bs.order_type || + tjetty->cfg.flag.bs.share_tp != 1) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "order_type is different or share_tp is NOT 1, local jetty flag:%u, remote jetty flag:%u.\n", + jetty->jetty_cfg.flag.bs.order_type, + tjetty->cfg.flag.bs.order_type); + return -EINVAL; + } + + if (jetty->remote_jetty == tjetty) { + udma_info(udma_dev->dev, UDMA_M_JETTY, + "reentry bind jetty(%u) with remote jetty(%u).\n", + jetty->jetty_id.id, tjetty->cfg.id.id); + return 0; + } + + if (jetty->remote_jetty != NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "The jetty has already bind a remote jetty.\n"); + return -EEXIST; + } + + udma_jetty = to_udma_jetty(jetty); + ret = udma_cmd_jetty_bind_ctx(udma_dev, udma_jetty, tjetty->cfg.id.id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to bind jetty(%u) with remote jetty(%u).\n", + jetty->jetty_id.id, tjetty->cfg.id.id); + return ret; + } + + return 0; +} + +int udma_unbind_jetty(struct ubcore_jetty *jetty) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + int ret; + + if (jetty == NULL || jetty->ub_dev == NULL || + jetty->remote_jetty == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "unbind jetty invalid parameter. jetty(%d) or ub_dev, remote_jetty is null.\n", + !!jetty); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + if (jetty->jetty_cfg.trans_mode != UBCORE_TP_RC) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Invalid jetty type(%u).\n", + (u32)jetty->jetty_cfg.trans_mode); + return -EINVAL; + } + + udma_jetty = to_udma_jetty(jetty); + ret = udma_cmd_jetty_unbind_ctx(udma_dev, udma_jetty); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to unbind jetty(%u) with remote jetty(%u).\n", + jetty->jetty_id.id, jetty->remote_jetty->cfg.id.id); + return ret; + } + + udma_info( + udma_dev->dev, UDMA_M_JETTY, + "udma unbind jetty success, jetty id:%u, remote jetty id:%u\n", + jetty->jetty_id.id, jetty->remote_jetty->cfg.id.id); + + return 0; +} + +void udma_jetty_async_event(struct udma_device *udma_dev, + struct udma_jetty *jetty, u8 type) +{ + struct ubcore_jetty *ubcore_jetty = &jetty->ubcore_jetty; + struct ubcore_event event = { 0 }; + + if (ubcore_jetty->jfae_handler != NULL) { + event.ub_dev = ubcore_jetty->ub_dev; + event.element.jetty = ubcore_jetty; + switch (type) { + case UB_AEQ_TYPE_JETTY_ERR: + event.event_type = UBCORE_EVENT_JETTY_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JETTY, + "unknown jetty asyn event(%u).\n", type); + return; + } + + ubcore_jetty->jfae_handler(&event, ubcore_jetty->uctx); + } +} + +static int udma_flush_jetty_one_normal(struct udma_device *udma_dev, + struct udma_jetty *udma_jetty, + struct ubcore_cr *cr) +{ + struct udma_send_queue *sq = &udma_jetty->sq; + + if (sq->flush_tail == sq->head) + return -EAGAIN; + + cr->user_ctx = sq->wrid[sq->flush_tail & (sq->wrid_size - 1)]; + ++sq->flush_tail; + + cr->status = UBCORE_CR_WR_UNHANDLED; + cr->flag.bs.s_r = 0; // send + cr->flag.bs.jetty = 1; // is jetty + cr->completion_len = 0; + cr->local_id = udma_jetty->jetty_id; + if (udma_jetty->ubcore_jetty.remote_jetty != NULL && + udma_jetty->ubcore_jetty.remote_jetty->tp != NULL) + cr->tpn = udma_jetty->ubcore_jetty.remote_jetty->tp->tpn; + cr->user_data = (uintptr_t)(&udma_jetty->ubcore_jetty); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "success, jetty_id(%u), tail(%u), flush_tail(%u), head(%u)\n", + udma_jetty->jetty_id, sq->tail, sq->flush_tail, sq->head); + + return 0; +} + +static int udma_flush_jetty_one(struct udma_device *udma_dev, + struct udma_jetty *udma_jetty, + struct ubcore_cr *cr) +{ + return udma_flush_jetty_one_normal(udma_dev, udma_jetty, cr); +} + +int udma_flush_jetty(struct ubcore_jetty *jetty, int cr_cnt, + struct ubcore_cr *cr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + int nflushed = 0; + ulong flags = 0; + int ret; + + if (jetty == NULL || jetty->ub_dev == NULL || cr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "invalid parameter. jetty(%d) or jetty->ub_dev or cr(%d) is null.\n", + !!jetty, !!cr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "udma_dev is null\n"); + return -EINVAL; + } + + spin_lock_irqsave(&udma_jetty->sq.lock, flags); + if (udma_jetty->sq.is_hw_flush_done == false) { + spin_unlock_irqrestore(&udma_jetty->sq.lock, flags); + udma_err(udma_dev->dev, UDMA_M_JETTY, + "%s: hardware is not flush done, jetty_id(%u)\n", + __func__, udma_jetty->jetty_id); + return -EINVAL; + } + + /* forge not execute of WQE of CQE */ + for (nflushed = 0; nflushed < cr_cnt; ++nflushed) { + ret = udma_flush_jetty_one(udma_dev, udma_jetty, &cr[nflushed]); + if (ret != 0) + break; + } + + if (udma_jetty->sq.flush_tail == udma_jetty->sq.head) + udma_jetty->sq.is_hw_flush_done = false; + + spin_unlock_irqrestore(&udma_jetty->sq.lock, flags); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "%s: success, jetty_id(%u), nflushed(%d)\n", __func__, + udma_jetty->jetty_id, nflushed); + + return nflushed; +} + +int udma_post_jetty_recv_wr(struct ubcore_jetty *jetty, + struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + struct udma_srq *srq = NULL; + int ret = 0; + + if (jetty == NULL || jetty->ub_dev == NULL || wr == NULL || + bad_wr == NULL) { + udma_pr_err( + UDMA_M_JETTY, + "Invalid parameter. jetty(%d), wr(%d), bad_wr(%d).\n", + !!jetty, !!wr, !!bad_wr); + return -EINVAL; + } + + udma_jetty = to_udma_jetty(jetty); + udma_dev = to_udma_dev(jetty->ub_dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JETTY, "udma_dev is null\n"); + *bad_wr = wr; + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "start post jetty recv, jetty id:%u, state:%d\n", + udma_jetty->jetty_id, udma_jetty->state); + + if (udma_jetty->state != UBCORE_JETTY_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty state: %d, posting not supported\n", + udma_jetty->state); + *bad_wr = wr; + return -EINVAL; + } + + if ((udma_jetty->ub_jfr == NULL) || + (udma_jetty->ub_jfr->state != UBCORE_JFR_STATE_READY)) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "ub_jfr is NULL or jfr state is not ready\n"); + *bad_wr = wr; + return -EINVAL; + } + + srq = &udma_jetty->ub_jfr->srq; + udma_debug(udma_dev->dev, UDMA_M_JETTY, "jfr: %u, max_sge: %u\n", + srq->jfr_id, srq->max_sge); + + ret = udma_post_srq_wr(udma_dev, srq, wr, bad_wr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Post jetty recv failed: %d\n", ret); + } else { + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "Post jetty recv success, jetty id: %u\n", + udma_jetty->jetty_id); + } + + return ret; +} + +bool udma_is_sq_dwqe(const struct udma_send_queue *sq, const u32 wr_cnt, + struct udma_device *udma_dev) +{ + u32 wqebb_cnt = sq->wqe_size >> sq->baseblk_shift; + u32 dwqe_dis = udma_sysfs_get_dwqe_dis(&udma_dev->sysfs_res); + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "wr_cnt 0x%x, head 0x%x, tail 0x%x, dwqe_dis 0x%x\n", wr_cnt, + sq->head, sq->tail, dwqe_dis); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "wqe_size 0x%x, dwqe_size 0x%x, baseblk_shift 0x%x\n", + sq->wqe_size, + udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size, + sq->baseblk_shift); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "wqebb_cnt 0x%x, rmd_wqebb_cnt 0x%x\n", wqebb_cnt, + sq->rmd_wqebb_cnt); + + return ((wr_cnt == 1) && (sq->head == sq->tail) && (dwqe_dis == 0) && + (udma_dev->kernel_dwqe_map != NULL) && + (sq->dwqe_curr != NULL) && + (sq->wqe_size <= + udma_dev->ub_cap.dev_cap_res.udma_res.direct_wqe_size) && + (wqebb_cnt <= sq->rmd_wqebb_cnt)); +} + +void udma_dwqe_copy(u64 *dst, u64 *src, u32 bytecnt) +{ +#define BYTE_MASK 255 + + u64 *src_tmp = src; + u32 bytecnt_tmp = bytecnt; + u64 *dst_tmp = + (u64 *)(void *)((u8 *)dst + (u32)((bytecnt_tmp & BYTE_MASK) + << 2)); // 2 is addr offset + + while ((int)bytecnt_tmp > 0) { + *dst_tmp++ = *src_tmp++; + bytecnt_tmp -= (u32)sizeof(u64); + } +} + +void udma_post_send_jetty_dwqe(struct udma_device *udma_dev, + struct udma_jetty *jetty) +{ + ulong flags = 0; + ub_sq_wqe_s *wqe = (ub_sq_wqe_s *)jetty->sq.dwqe_curr; + + union tag_ub_sq_db_u dwqe_db; + dwqe_db.sq_db_value = 0; + dwqe_db.sq_db.dw0.bs.xqn = jetty->jetty_id; + dwqe_db.sq_db.dw0.bs.type = UB_DIRECT_WQE_SRV_TYPE + << UB_DWQE_DB_TYPE_SHIFT; // high 2bit + dwqe_db.sq_db.dw1.bs.pi = jetty->sq.pi & + 0xFFFF; // complete pi, normal is pi[15:8] + dwqe_db.sq_db.dw0.bs.cntx_size = UB_CNTX_SIZE; + dwqe_db.sq_db.dw0.bs.cos = jetty->cos; + udma_debug(udma_dev->dev, UDMA_M_JETTY, "cos: %u, pi: %u\n", jetty->cos, + jetty->sq.pi); + + wqe->ctrl_sec.dw2_value = cpu_to_be32(dwqe_db.sq_db_value & 0xFFFFFFFF); + wqe->ctrl_sec.dw3_value = + cpu_to_be32(dwqe_db.sq_db_value >> ADDR_H_OFFSET); + + wmb(); + *(jetty->sdb.db_record) = cpu_to_be32(jetty->sq.pi); + wmb(); + + spin_lock_irqsave(&udma_dev->dwqe_lock, flags); + udma_dwqe_copy((u64 *)udma_dev->kernel_dwqe_map, (u64 *)(void *)wqe, + (u32)jetty->sq.wqe_size); + spin_unlock_irqrestore(&udma_dev->dwqe_lock, flags); +} + +static int udma_update_jetty_db(struct udma_device *udma_dev, + struct udma_jetty *jetty, u32 wr_cnt) +{ + union tag_ub_sq_db_u jetty_db; + int ret; + + if (udma_is_sq_dwqe(&jetty->sq, wr_cnt, udma_dev)) { + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "the wqe is jetty dwqe\n"); + udma_post_send_jetty_dwqe(udma_dev, jetty); + return 0; + } + + wmb(); + udma_base_construct_sq_db(&jetty_db, jetty->jetty_id, jetty->sq.pi, + jetty->cos); + udma_debug(udma_dev->dev, UDMA_M_JETTY, "cos: %u, pi: %u\n", jetty->cos, + jetty->sq.pi); + *(jetty->sdb.db_record) = cpu_to_be32(jetty->sq.pi); + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + jetty->sq.pi & 0xff, jetty_db.sq_db_value); + + wmb(); + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "xid:0x%x, jetty_db.sq_db_value:0x%llx\n", jetty->jetty_id, + jetty_db.sq_db_value); + return ret; +} + +bool check_k_is_fast_path(const struct ubcore_jfs_wr *wr, + const struct udma_send_queue *sq) +{ + // current wr of request is 1, switch queue is empty + if ((wr != NULL && wr->next == NULL) && sq->head == sq->tail) { + return true; + } + return false; +} + +static int udma_post_jetty_send_wr_one(struct ubcore_jetty *jetty, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + struct udma_post_wqe post_wqe = { 0 }; + u32 wr_cnt = 0; + ulong flags = 0; + int ret = 0; + + udma_dev = to_udma_dev(jetty->ub_dev); + udma_jetty = to_udma_jetty(jetty); + spin_lock_irqsave(&udma_jetty->sq.lock, flags); + post_wqe.jfc = to_udma_jfc(udma_jetty->ubcore_jetty.jetty_cfg.send_jfc); + post_wqe.tp_mode = udma_jetty->trans_mode; + post_wqe.sq = &udma_jetty->sq; + post_wqe.max_send_sge = jetty->jetty_cfg.max_send_sge; + post_wqe.is_fast_path = check_k_is_fast_path(wr, &udma_jetty->sq); + + ret = udma_construct_wqe(udma_dev, wr, bad_wr, &wr_cnt, &post_wqe); + + if (wr_cnt != 0) { + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "before: sq.head:%u, wr_cnt:%u\n", + udma_jetty->sq.head, wr_cnt); + udma_update_jetty_db(udma_dev, udma_jetty, wr_cnt); + udma_jetty->sq.head += wr_cnt; + atomic64_add( + wr_cnt, + &udma_jetty->sq.udma_driver_sq_ctr.post_send_wr_num); + atomic64_add( + wr_cnt, + &udma_dev->udma_driver_device_ctr.post_send_wr_num); + udma_debug( + udma_dev->dev, UDMA_M_JETTY, + "after: sq.head:%u, wr_cnt:%u, dev_post_send_wr_num:%llu, jetty_post_send_wr_num:%llu\n", + udma_jetty->sq.head, wr_cnt, + (u64)atomic64_read(&udma_dev->udma_driver_device_ctr + .post_send_wr_num), + (u64)atomic64_read(&udma_jetty->sq.udma_driver_sq_ctr + .post_send_wr_num)); + } + spin_unlock_irqrestore(&udma_jetty->sq.lock, flags); + + return ret; +} + +int udma_post_jetty_send_wr(struct ubcore_jetty *jetty, + struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jetty *udma_jetty = NULL; + + if ((jetty == NULL) || (jetty->ub_dev == NULL) || (wr == NULL) || + (bad_wr == NULL)) { + udma_pr_err( + UDMA_M_JFC, + "%s Invalid parameter: jetty(%d), or jetty->ub_dev, wr(%d), *bad_wr(%d).\n", + __func__, !!jetty, !!wr, !!bad_wr); + return -EINVAL; + } + + udma_jetty = to_udma_jetty(jetty); + udma_dev = to_udma_dev(jetty->ub_dev); + if (udma_dev == NULL || udma_jetty == NULL) { + udma_pr_err(UDMA_M_JFC, + "failed: %s to_udma_dev(%d), udma_jetty(%d).\n", + __func__, !!udma_dev, !!udma_jetty); + *bad_wr = wr; + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JETTY, + "start post jetty send, jetty id: %u, state: %d\n", + udma_jetty->jetty_id, udma_jetty->state); + + if (udma_jetty->state != UBCORE_JETTY_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty state: %d, posting not supported\n", + udma_jetty->state); + *bad_wr = wr; + return -EINVAL; + } + + if (jetty->jetty_cfg.trans_mode == UBCORE_TP_RC && + jetty->remote_jetty == NULL) { + udma_err( + udma_dev->dev, UDMA_M_JETTY, + "Invalid parameter, remote jetty is null, maybe it's not bind jetty.\n"); + *bad_wr = wr; + return -EINVAL; + } + + return udma_post_jetty_send_wr_one(jetty, wr, bad_wr); +} + +#ifdef UB_SUPPORT_JETTY_GRP +static int udma_alloc_jetty_grp_idx(struct udma_device *udma_dev, + struct udma_jetty_grp *jetty_grp) +{ + int ret = 0; + u32 xid_start = SCQC_JETTY_GRP_START(udma_dev); + u32 xid_end = + xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty_grp; + + jetty_grp->cqm_jetty_grp = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, 0, jetty_grp, + &udma_dev->jetty_grpn_record, false, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_NONE, + 0), + xid_start, xid_end); + if (jetty_grp->cqm_jetty_grp == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to create cqm jetty group.\n"); + ret = -ENOMEM; + } else { + jetty_grp->jetty_grp_id = jetty_grp->cqm_jetty_grp->index; + jetty_grp->base.jetty_grp_id.id = jetty_grp->jetty_grp_id; + } + + return ret; +} + +static void udma_free_jetty_grp_idx(struct udma_jetty_grp *jetty_grp) +{ + cqm5_object_delete(&jetty_grp->cqm_jetty_grp->object); + jetty_grp->cqm_jetty_grp = NULL; +} +#endif + +int udma_create_jetty_grp_pre_check(const struct ubcore_device *dev, + const struct ubcore_jetty_grp_cfg *cfg) +{ + if (dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JFC, + "create jetty grp invalid parameter. dev(%d), cfg(%d).\n", + !!dev, !!cfg); + return -EINVAL; + } + + if (cfg->policy != UBCORE_JETTY_GRP_POLICY_RR && + cfg->policy != UBCORE_JETTY_GRP_POLICY_HASH_HINT) { + udma_pr_err(UDMA_M_JFC, + "create jetty grp policy(%u) invalid.\n", + cfg->policy); + return -EINVAL; + } + + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE && + cfg->flag.bs.token_policy != UBCORE_TOKEN_PLAIN_TEXT) { + udma_pr_err(UDMA_M_JFC, + "create jetty grp token policy(%u) invalid.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + return 0; +} + +struct ubcore_jetty_group * +udma_create_jetty_grp(struct ubcore_device *dev, + struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata) +{ +#ifdef UB_SUPPORT_JETTY_GRP + struct udma_device *udma_dev = NULL; + struct udma_jetty_grp *jetty_grp = NULL; + int ret; + + ret = udma_create_jetty_grp_pre_check(dev, cfg); + if (ret != 0) { + udma_pr_err(UDMA_M_JFC, + "create jetty grp invalid parameter, ret(%d).\n", + ret); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "create jetty grp, policy: %d.\n", cfg->policy); + + jetty_grp = (struct udma_jetty_grp *)kzalloc( + sizeof(struct udma_jetty_grp), GFP_KERNEL); + if (jetty_grp == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "alloc jetty grp failed.\n"); + return ERR_PTR(-ENOMEM); + } + + jetty_grp->cqm_priv_type = UDMA_CQM_PRIV_TYPE_JETTY_GRP; + + ret = udma_alloc_jetty_grp_idx(udma_dev, jetty_grp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "alloc jetty grp idx failed\n"); + goto err_and_free_jetty_grp; + } + + ret = udma_cmd_jetty_grp_create_ctx(udma_dev, jetty_grp, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "init jetty grp ctx failed, id: %u.\n", + jetty_grp->jetty_grp_id); + goto err_and_free_jetty_grp_idx; + } + jetty_grp->base.jetty_grp_cfg = *cfg; + + kref_init(&jetty_grp->ref_cnt); + init_completion(&jetty_grp->comp); + + udma_store_list(udma_dev, &jetty_grp->node, + &udma_dev->device_res->jetty_group_list); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "jetty grp created successfully, id: %u.\n", + jetty_grp->jetty_grp_id); + return &jetty_grp->base; + +err_and_free_jetty_grp_idx: + udma_free_jetty_grp_idx(jetty_grp); +err_and_free_jetty_grp: + kfree(jetty_grp); + + return ERR_PTR(-EFAULT); +#else + udma_pr_err(UDMA_M_JFC, + "create jetty grp is not currently supported.\n"); + return ERR_PTR(-EPERM); +#endif +} + +int udma_destroy_jetty_grp(struct ubcore_jetty_group *jetty_grp) +{ +#ifdef UB_SUPPORT_JETTY_GRP + struct udma_device *udma_dev = NULL; + struct udma_jetty_grp *udma_jetty_grp = NULL; + int ret; + + if (jetty_grp == NULL || jetty_grp->ub_dev == NULL) { + udma_pr_err( + UDMA_M_JFC, + "destroy jetty grp invalid parameter. jetty_grp or(%d) jetty_grp->ub_dev is null\n", + !!jetty_grp); + return -EINVAL; + } + + udma_dev = to_udma_dev(jetty_grp->ub_dev); + udma_jetty_grp = to_udma_jetty_grp(jetty_grp); + + udma_info(udma_dev->dev, UDMA_M_JETTY, + "start to destroy jetty grp, id: %u.\n", + jetty_grp->jetty_grp_id.id); + + ret = udma_cmd_jetty_grp_delete_ctx(udma_dev, udma_jetty_grp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "failed to free jetty grp ctx, id: %u, ret: %d.\n", + jetty_grp->jetty_grp_id.id, ret); + return -EFAULT; + } + + udma_delete_list(udma_dev, &udma_jetty_grp->node, + &udma_dev->device_res->jetty_group_list); + udma_jetty_group_put(udma_jetty_grp); + wait_for_completion(&udma_jetty_grp->comp); + udma_free_jetty_grp_idx(udma_jetty_grp); + udma_info(udma_dev->dev, UDMA_M_JETTY, + "jetty grp destroyed successfully, id: %u.\n", + jetty_grp->jetty_grp_id.id); + + kfree(udma_jetty_grp); + return 0; +#else + udma_pr_err(UDMA_M_JFC, + "destroy jetty grp is not currently supported.\n"); + return -EPERM; +#endif +} + +int udma_query_res_jetty_group(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_group_val *ret_val = + (struct ubcore_res_jetty_group_val *)(uintptr_t)val->addr; + ub_jetty_grp_ctx_info_s jetty_grp_ctx; + struct udma_jetty_grp *ub_jetty_grp = NULL; + u32 jetty_num_max; + u32 index; + u32 val_index; + + (void)memset_s(&jetty_grp_ctx, sizeof(ub_jetty_grp_ctx_info_s), 0, + sizeof(ub_jetty_grp_ctx_info_s)); + + ub_jetty_grp = udma_find_jetty_group(udma_dev, key->key); + if (ub_jetty_grp == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to get ubcore_jetty_group, jetty group id(%u)", + key->key); + return -EINVAL; + } + + (void)memcpy_s((u8 *)&jetty_grp_ctx, sizeof(ub_jetty_grp_ctx_info_s), + ub_jetty_grp->cqm_jetty_grp->q_ctx_vaddr, + sizeof(ub_jetty_grp_ctx_info_s)); + udma_jetty_group_put(ub_jetty_grp); + + jetty_num_max = (u32)ARRAY_SIZE(jetty_grp_ctx.jetty); + ret_val->jetty_cnt = 0; + for (index = 0, val_index = 0; index < jetty_num_max; index++) { + jetty_grp_ctx.jetty[index].value = + be32_to_cpu(jetty_grp_ctx.jetty[index].value); + if (jetty_grp_ctx.jetty[index].vld != 0) { + ret_val->jetty_cnt++; + val_index++; + } + } + + if (ret_val->jetty_cnt == 0) { + ret_val->jetty_list = NULL; + return 0; + } + + ret_val->jetty_list = (u32 *)vmalloc(sizeof(u32) * ret_val->jetty_cnt); + if (ret_val->jetty_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "jetty_list vmalloc failed, jetty_cnt(%u)\n", + ret_val->jetty_cnt); + return -ENOMEM; + } + + for (index = 0, val_index = 0; index < jetty_num_max; index++) { + jetty_grp_ctx.jetty[index].value = + be32_to_cpu(jetty_grp_ctx.jetty[index].value); + if (jetty_grp_ctx.jetty[index].vld != 0) { + ret_val->jetty_list[val_index] = + jetty_grp_ctx.jetty[index].jetty_id; + val_index++; + } + } + + return 0; +} + +void udma_jetty_grp_async_event(struct udma_device *udma_dev, + struct udma_jetty_grp *udma_jetty_grp, u8 type) +{ + struct ubcore_jetty_group *ubcore_jetty_group = &udma_jetty_grp->base; + struct ubcore_event event = { 0 }; + + if (ubcore_jetty_group->jfae_handler != NULL) { + event.ub_dev = ubcore_jetty_group->ub_dev; + event.element.jetty_grp = ubcore_jetty_group; + switch (type) { + case UB_AEQ_TYPE_JETTY_GROUP_ERR: + event.event_type = UBCORE_EVENT_JETTY_GRP_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JETTY, + "unknown jetty group asyn event(%u).\n", + type); + return; + } + + ubcore_jetty_group->jfae_handler(&event, + ubcore_jetty_group->uctx); + } +} + +u32 udma_cal_page_size(u64 buf_size) +{ + u32 page_size = PAGE_SIZE_4K; + + if (buf_size <= PAGE_SIZE_4K) { + page_size = PAGE_SIZE_4K; + } else if (buf_size > PAGE_SIZE_4K && buf_size <= PAGE_SIZE_64K) { + page_size = PAGE_SIZE_64K; + } else if (buf_size > PAGE_SIZE_64K && buf_size <= PAGE_SIZE_2M) { + page_size = PAGE_SIZE_2M; + } + + page_size = (u32)ub_size_align((u64)page_size, PAGE_SIZE); + + return page_size; +} + +int udma_dfx_modify_jetty_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct udma_jetty *jetty = NULL; + int ret = 0; + + jetty = udma_find_jetty(udma_dev, inbuf->jettyid); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to get jetty, jetty_id = %u", inbuf->jettyid); + return -EINVAL; + } + + ret = udma_cmd_jetty_modify_dfx(udma_dev, jetty, &inbuf->jetty_attr); + + udma_jetty_put(jetty); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h new file mode 100644 index 000000000..18adcb910 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jetty.h @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_JETTY_H +#define UDMA_JETTY_H + +#include <linux/kernel.h> +#include <linux/sched.h> +#include "ubcore_api.h" +#include "hinic5_cqm.h" +#include "ub_npu_srq_cmd_defs.h" +#include "udma_common.h" +#include "udma_comp.h" +#include "udma_srq.h" +#include "udma_db.h" +#include "udma_jfc.h" +#include "ub_npu_jetty_cmd_defs.h" + +#define UBEP_JETTY_CONTEXT_SIZE 512 + +#define UB_DEV_MAX_TASK_JFS_NUM (8 * 1024) + +#define UB_MODIFY_JETTY_TIMEOUT 10 + +#define JETTY_WQE_SHIFT 6 +#define UB_JETTY_WQEBB_SIZE_SHIFT 4 +#define UB_JETTY_WQEBB_SIZE_MASK 0x7 +#define UB_JETTY_PAGE_SIZE_MASK 0xf +#define UB_JETTY_SIZE_MASK 0x1f +#define UB_RC_TABLE_SIZE_128B 0x7 +#define UB_RC_EXT_ORDER_SHIFT 3 + +#define UB_ILOG2(n) ilog2(n) + +#define UB_JFS_WQEBB 64 + +#define UB_DESTROY_FLOW_TIMEOUT 10 + +#define UB_SGE_SHIFT 4 +#define UB_WQE_CTRL_SEC_LENGTH 2 +#define UB_WQE_VA_TYPE_VIRTUAL 1 +#define UB_DEID_LENGTH 4 +#define UB_DEID_ONE_OFFSET 8 +#define UB_DEID_TWO_OFFSET 16 +#define UB_DEID_THREE_OFFSET 24 +#define UB_WQE_CMP_TASK_LEN_SHIFT 28 +#define UB_WQE_CMP_TASK_LEN1 1u +#define UB_TASK_SEG_ALIGN 8 +#define ADDR_H_OFFSET 32 +#define UB_JFS_WRITE_WQE_TSK_LEN 64 +#define UBCORE_ODR_BITMASK 0x7 +#define UB_DIRECT_WQE_SRV_TYPE 2 +#define UB_DWQE_DB_TYPE_SHIFT 3 +#define UB_JFS_SEND_WQE_TSK_LEN 64 +#define UB_DEID_BYTE_LENGTH 16 +#define UB_SEID_BYTE_LENGTH 16 +#define UB_WR_RD_TKV_DEID_LENGTH 80 +#define UB_SEND_TKV_DEID_LENGTH 80 +#define UB_INLINE_BYTE_LENGTH 4 +#define UB_JFS_WRITE_IMM_WQE_TSK_LEN 96 +#define UB_WRITE_IMME_TKV_DEID_LENGTH 112 +#define UB_SLICE_LEN_SHIFT 15 +#define UB_SLICE_LEN (1 << UB_SLICE_LEN_SHIFT) /* 32k */ +#define UB_SLICE_LEN_64K_SHIFT 16 +#define UB_SLICE_LEN_64K (1 << UB_SLICE_LEN_64K_SHIFT) +#define UB_SLICE_LEN_2G_SHIFT 31 +#define UB_SLICE_LEN_2G (1 << UB_SLICE_LEN_2G_SHIFT) +#define UB_MIN_WR_WQEBB_NUM 2 +#define UB_ATOMIC_LENGTH 8 +#define UB_MAX_WRITE_READ_LEN (2 * 1024 * 1024 * 1024L) + +#define UB_ERROR_SUSPEND 1 + +#define SET_DEID(x, tjetty) \ + cpu_to_be32( \ + ((tjetty)->cfg.id.eid.raw[(x)*4] << UB_DEID_THREE_OFFSET) | \ + ((tjetty)->cfg.id.eid.raw[(x)*4 + 1] << UB_DEID_TWO_OFFSET) | \ + ((tjetty)->cfg.id.eid.raw[(x)*4 + 2] << UB_DEID_ONE_OFFSET) | \ + ((tjetty)->cfg.id.eid.raw[(x)*4 + 3])) + +typedef struct ub_rseg { + struct ubcore_target_seg base; + union ubcore_import_seg_flag flag; + u32 token; +} ub_rseg_t; + +#pragma pack(push, 1) +typedef struct ub_cmd_sge { + u64 va; + u32 length; + union { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u32 L : 1; + u32 token_id : 31; +#else + u32 token_id : 31; + u32 L : 1; +#endif + } bs; + u32 dw3_value; + } dw3; +} ub_cmd_sge_t; +#pragma pack(pop) + +struct udma_sge_info { + u32 sge_num; + u32 max_inline_size; +}; + +typedef struct udma_driver_sq_ctr { + atomic64_t post_send_wr_num; +} udma_driver_sq_ctr_t; + +typedef struct udma_send_queue { + spinlock_t lock; + u64 *wrid; + void *buf; + void *buf_curr; + void *dwqe_curr; + u32 wrid_size; + u32 buf_size; + u32 tpn; + u32 head; + u32 tail; + u32 wqe_size; + u32 rmd_wqebb_cnt; + u32 pi; + u32 max_wr_num; + u32 max_inline_size; + u32 baseblk_cnt; + u32 baseblk_shift; + u32 flush_tail; + u8 is_hw_flush_done; // hardware flush_done complete when set true + udma_driver_sq_ctr_t udma_driver_sq_ctr; +} udma_send_queue_t; + +struct udma_jfs { + u32 cqm_priv_type; + struct ubcore_jfs ubcore_jfs; + struct tag_cqm_qpc_mpt *jfsc_info; + struct ubcore_umem *umem; + struct mtt ub_mtt; + struct udma_db sdb; + struct ubrc ubrc; + enum ubcore_transport_mode trans_mode; + enum ubcore_jetty_state state; + u32 jfs_id; + u32 sq_buf_len; + u32 rsp_depth; + cqm_queue_s *jfs_buf_info; + struct queue_addr_info queue_addr; + u8 cos; + u32 page_shift; + struct udma_send_queue sq; // in create jfs when perform initialize + struct list_head node; + u32 round; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_jfr { + u32 cqm_priv_type; + struct ubcore_jfr ubcore_jfr; + struct tag_cqm_qpc_mpt *jfrc_info; + struct udma_srq srq; + struct udma_index_queue index_queue; + struct udma_db sdb; + enum ubcore_jfr_state state; + u32 idx_queue_lth; /* JFR index queue limit threshold. */ + u32 jfr_id; + struct list_head node; + u32 round; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_jetty { + u32 cqm_priv_type; + struct ubcore_jetty ubcore_jetty; + struct tag_cqm_qpc_mpt *jettyc_info; + struct tag_cqm_queue *sq_info; + struct queue_addr_info queue_addr; + struct ubcore_umem *umem; + struct udma_jfr *ub_jfr; + struct udma_db sdb; + struct mtt ub_mtt; + struct ubrc ubrc; + enum ubcore_transport_mode trans_mode; + enum ubcore_jetty_state state; + u32 jetty_id; + u32 sq_buf_len; + u32 rq_buf_len; + u32 rsp_depth; + u32 page_shift; + struct udma_send_queue sq; + u8 cos; + struct list_head node; + u32 round; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_jetty_grp { + u32 cqm_priv_type; + struct ubcore_jetty_group base; + struct tag_cqm_queue *cqm_jetty_grp; + u32 jetty_grp_id; + struct list_head node; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +struct udma_post_wqe { + bool is_jetty; + bool is_fast_path; + union { + struct udma_jetty *jetty; + struct udma_jfs *jfs; + }; + struct udma_send_queue *sq; + struct udma_jfc *jfc; + u32 tpn; + enum ubcore_transport_mode tp_mode; + u8 max_send_sge; +}; + +struct udma_create_jfs_uinfo { + u32 resp_len; + u32 cmd_len; + struct ub_create_jetty_resp *resp; + struct ub_create_jetty_ucmd *ucmd; + struct udma_context *udma_ctx; + struct ubcore_udata *udata; +}; + +struct udma_jfr_modify_cfg { + enum ubcore_jfr_state state; + u32 idx_queue_lth; /* JFR index queue limit threshold. */ + struct { + u32 state : 1; + u32 idx_queue_lth : 1; + u32 rsvd : 30; + } mask; +}; + +struct udma_jetty_grp *udma_find_jetty_group(struct udma_device *udev, + u32 jetty_grp_id); +struct udma_jetty *udma_find_jetty(struct udma_device *udev, u32 jetty_id); +int udma_alloc_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc, bool is_fixed); +void udma_free_opt_ubrc(struct udma_device *udma_dev, u32 *rsp_depth, + struct ubrc *ubrc); +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *ub_dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfs(struct ubcore_jfs *jfs); +int udma_modify_jfs(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata); +int udma_flush_jfs(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr); + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *ub_dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); +u32 udma_get_jfr_threshold(struct udma_device *udma_dev, + ub_cmd_srq_query_rsp_s *srq_entry, + struct ub_ta_context_query *jfr_entry); +int udma_query_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr); +int udma_destroy_jfr(struct ubcore_jfr *jfr); + +int udma_query_jetty(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr); +int udma_query_res_jfr(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val); +int udma_post_jfr_wr(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *ub_dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty(struct ubcore_jetty *jetty); +int udma_modify_jetty(struct ubcore_jetty *jetty, + struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata); +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_flush_jetty(struct ubcore_jetty *jetty, int cr_cnt, + struct ubcore_cr *cr); +int udma_unimport_jfr(struct ubcore_tjetty *tjfr); +void udma_jfr_async_event(struct udma_device *udma_dev, struct udma_jfr *jfr, + u8 type); +int udma_modify_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); +void udma_jetty_async_event(struct udma_device *udma_dev, + struct udma_jetty *jetty, u8 type); +void udma_jfs_async_event(struct udma_device *udma_dev, struct udma_jfs *jfs, + u8 type); +void udma_jetty_grp_async_event(struct udma_device *udma_dev, + struct udma_jetty_grp *udma_jetty_grp, u8 type); +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_jetty(struct ubcore_tjetty *tjetty); +int udma_bind_jetty(struct ubcore_jetty *jetty, struct ubcore_tjetty *tjetty, + struct ubcore_udata *udata); +int udma_unbind_jetty(struct ubcore_jetty *jetty); +int udma_post_jetty_recv_wr(struct ubcore_jetty *jetty, + struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +struct ubcore_jetty_group * +udma_create_jetty_grp(struct ubcore_device *dev, + struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty_grp(struct ubcore_jetty_group *jetty_grp); + +struct ubcore_umem *udma_get_umem(struct udma_device *udma_dev, u64 buf_addr, + u32 buf_size); + +void udma_destroy_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt); +int udma_create_user_mtt(struct udma_device *udma_dev, struct ubcore_umem *umem, + struct mtt *ub_mtt); + +u32 udma_get_kernel_sq_size(const struct udma_device *udma_dev, u32 depth, + u32 *wr_max_length); + +int udma_query_res_jfs(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val); +int udma_query_res_jetty(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_query_res_jetty_group(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); +ub_sqwqe_optype_e ub_jfs_parse_opcode(enum ubcore_opcode opcode); +int udma_construct_wqe(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr, u32 *wr_cnt, + struct udma_post_wqe *post_wqe); +bool check_k_is_fast_path(const struct ubcore_jfs_wr *wr, + const struct udma_send_queue *sq); + +bool udma_is_sq_dwqe(const struct udma_send_queue *sq, const u32 wr_cnt, + struct udma_device *udma_dev); +int udma_post_jetty_send_wr(struct ubcore_jetty *jetty, + struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); +int udma_query_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr); +int udma_post_jfs_wr(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); + +int udma_create_jfs_uinfo_init(struct udma_device *udma_dev, + struct ubcore_udata *udata, + struct udma_create_jfs_uinfo *uinfo); +int udma_create_jfs_uinfo_deinit(struct udma_device *udma_dev, + struct udma_create_jfs_uinfo *uinfo, u8 cos, + u32 round, u8 copy); +int udma_clear_pi_on_chip(struct udma_device *udma_dev, u32 id, u8 cos); + +void udma_dwqe_copy(u64 *dst, u64 *src, u32 bytecnt); +struct udma_jfs *udma_find_jfs(struct udma_device *udev, u32 jfs_id); +struct udma_jfr *udma_find_jfr(struct udma_device *udev, u32 jfr_id); + +int udma_modify_jetty_parse_udata(struct udma_device *udma_dev, + struct udma_send_queue *sq, + struct ubcore_udata *udata); +void udma_put_umem(struct ubcore_umem *umem); +void udma_jetty_put(void *obj); +void udma_jetty_get(void *obj); +void udma_jfs_put(void *obj); +void udma_jfs_get(void *obj); +void udma_jfr_put(void *obj); +void udma_jfr_get(void *obj); +void udma_jetty_group_put(void *obj); +void udma_jetty_group_get(void *obj); +int udma_init_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem **umem, u64 buf_addr, u32 buf_size); +void udma_uninit_user_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + struct ubcore_umem *umem); +int udma_alloc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, struct queue_addr_info *queue_addr); +int udma_free_queue_resource(struct udma_device *udma_dev, + cqm_queue_s *jfs_buf_info, + struct queue_addr_info *queue_addr); +int udma_init_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt, + cqm_queue_s *cqm_buf_info, dma_addr_t pa); +void udma_uninit_kernel_mtt(struct udma_device *udma_dev, struct mtt *ub_mtt); +int udma_fill_kernel_mtt(struct mtt *ub_mtt, dma_addr_t pa); + +u32 udma_cal_page_size(u64 buf_size); +void udma_fill_queue_addr_info(struct queue_addr_info *queue_addr, + dma_addr_t queue_pa, u32 len, u8 mtt_pro_en); +int udma_dfx_modify_jetty_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf); +u32 udma_jetty_get_qpn_mod(struct udma_device *udma_dev, + struct ubcore_jfc *urma_jfc); + +static inline struct udma_jfs *to_udma_jfs(const struct ubcore_jfs *jfs) +{ + return (struct udma_jfs *)udma_container_of(jfs, struct udma_jfs, + ubcore_jfs); +} + +static inline struct udma_jfr *to_udma_jfr(const struct ubcore_jfr *jfr) +{ + return (struct udma_jfr *)udma_container_of(jfr, struct udma_jfr, + ubcore_jfr); +} + +static inline struct udma_jetty *to_udma_jetty(const struct ubcore_jetty *jetty) +{ + return (struct udma_jetty *)udma_container_of(jetty, struct udma_jetty, + ubcore_jetty); +} + +static inline struct udma_jetty_grp * +to_udma_jetty_grp(const struct ubcore_jetty_group *jetty_grp) +{ + return (struct udma_jetty_grp *)udma_container_of( + jetty_grp, struct udma_jetty_grp, base); +} + +// size to on take align of multiple of,align must is 2 of power time power +static inline u64 ub_size_align(u64 size, u64 align) +{ + return ((size + align - 1) & (~(align - 1))); +} + +static inline struct udma_jfr * +cqmobj_to_ub_jfr(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_jfr *)qpc_info->priv; +} + +static inline struct udma_jfs * +cqmobj_to_ub_jfs(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_jfs *)qpc_info->priv; +} + +static inline struct udma_jfc * +cqmobj_to_ub_jfc(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_info = NULL; + cqm_info = container_of(object, struct tag_cqm_queue, object); + return (struct udma_jfc *)cqm_info->priv; +} + +static inline struct udma_jetty * +cqmobj_to_ub_jetty(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *qpc_info = NULL; + qpc_info = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_jetty *)qpc_info->priv; +} + +static inline struct udma_jetty_grp * +cqmobj_to_ub_jetty_grp(const struct tag_cqm_object *object) +{ + struct tag_cqm_queue *cqm_info = NULL; + cqm_info = container_of(object, struct tag_cqm_queue, object); + return (struct udma_jetty_grp *)cqm_info->priv; +} + +static inline u32 depth_align_power2(u32 value) +{ + u32 shift; + + for (shift = 0; (1U << shift) < value; shift++) { + ; // do nothing + } + + return shift; +} + +static inline u32 to_ubcore_jfr_state(enum UB_JETTY_STATE_E state) +{ + switch (state) { + case UB_JETTY_STATE_RESET: + return UBCORE_JFR_STATE_RESET; + case UB_JETTY_STATE_READY: + return UBCORE_JFR_STATE_READY; + case UB_JETTY_STATE_ERROR: + return UBCORE_JFR_STATE_ERROR; + default: + udma_pr_err(UDMA_M_JETTY, + "to ubcore jfr state failed! state = %d\n", state); + return UBCORE_JFR_STATE_ERROR + 1; + } +} + +static inline int udma_inc_jfc_used_cnt(struct ubcore_jfc *urma_jfc) +{ + struct udma_jfc *jfc = to_udma_jfc(urma_jfc); + + if (jfc == NULL) + return -EINVAL; + + return atomic_inc_return(&jfc->used_by_jfs); +} + +static inline void udma_dec_jfc_used_cnt(const struct ubcore_jfc *urma_jfc) +{ + struct udma_jfc *jfc = to_udma_jfc(urma_jfc); + + if (jfc != NULL) + atomic_dec(&jfc->used_by_jfs); +} + +static inline int udma_read_jfc_used_cnt(struct ubcore_jfc *urma_jfc) +{ + struct udma_jfc *jfc = to_udma_jfc(urma_jfc); + + return (jfc == NULL) ? -1 : (int)atomic_read(&jfc->used_by_jfs); +} + +static inline struct tag_ub_sq_wqe_s * +udma_get_next_wqe(struct udma_send_queue *sq, struct tag_ub_sq_wqe_s *old_wqe) +{ + if (((sq->pi + 1) & (sq->baseblk_cnt - 1)) == 0) { + return (ub_sq_wqe_s *)((uint8_t *)sq->buf - + (1ULL << sq->baseblk_shift)); + } + + return old_wqe; +} + +#define UDMA_MAX_SLICE_SHIFT (UB_SLICE_LEN_2G_SHIFT - UB_SLICE_LEN_SHIFT) +static inline bool udma_get_jetty_slice_en(struct udma_device *udma_dev) +{ + return (udma_dev->sysfs_res.dfx_ctx.debug_en == 1) ? + (bool)udma_dev->sysfs_res.dfx_ctx.perf.jetty_slice_en : + true; +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c new file mode 100644 index 000000000..8fae4c286 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.c @@ -0,0 +1,1358 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "ubcore_uapi.h" + +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_jetty.h" +#include "udma_cqm.h" +#include "udma_mtt.h" +#include "udma_comp.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_abi.h" +#include "udma_resource.h" +#include "udma_cmd_jfc.h" +#include "udma_jfc.h" + +#include "ub_xqe_format.h" +#include "ub_aeqe.h" + +#define CQE_WQE_IDX_SHIFT 12 +#define CQE_WQE_IDX_MASK 0xfffUL +#define INVALID_CR_OPCODE 0xff + +#define UB_RESERVED_CQE 7 + +static void udma_jfc_kref_release(struct kref *ref_cnt) +{ + struct udma_jfc *jfc = container_of(ref_cnt, struct udma_jfc, ref_cnt); + + complete(&jfc->comp); +} + +void udma_jfc_put(void *obj) +{ + struct udma_jfc *jfc = obj; + + if (jfc != NULL) { + kref_put(&jfc->ref_cnt, udma_jfc_kref_release); + } +} + +void udma_jfc_get(void *obj) +{ + struct udma_jfc *jfc = obj; + + kref_get(&jfc->ref_cnt); +} + +struct udma_jfc *udma_find_jfc(struct udma_device *udev, u32 jfc_id) +{ + struct udma_jfc *jfc = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jfc_list.node_lock; + mutex_lock(lock); + list_for_each_entry(jfc, &udev->device_res->jfc_list.node, node) { + if (jfc->base.id == jfc_id) { + udma_jfc_get(jfc); + mutex_unlock(lock); + return jfc; + } + } + mutex_unlock(lock); + + return NULL; +} + +static void udma_set_jfc_inline(struct udma_device *udma_dev, + struct udma_jfc *jfc) +{ + if ((jfc->base.jfc_cfg.flag.bs.jfc_inline != 0) && + (udma_dev->ub_cap.dev_cap.feature.bs.jfc_inline == 0)) { + jfc->base.jfc_cfg.flag.bs.jfc_inline = 0; + udma_warn(udma_dev->dev, UDMA_M_JFC, + "Device does not support jfc inline\n"); + } +} + +u32 udma_get_jfc_inline(struct udma_device *udma_dev, + const struct ubcore_jfc *jfc) +{ + u32 jfc_inline = jfc->jfc_cfg.flag.bs.jfc_inline; + if ((jfc_inline != 0) && + (udma_dev->ub_cap.dev_cap.feature.bs.jfc_inline == 0)) { + udma_warn(udma_dev->dev, UDMA_M_JFC, + "Device does not support jfc inline\n"); + jfc_inline = 0; + } + return jfc_inline; +} + +static struct tag_cqm_queue *udma_get_cqm_jfc(struct udma_device *udma_dev, + u32 size, void *jfc, + bool room_header_alloc) +{ + u32 xid_start = SCQC_JFC_START(udma_dev); + u32 xid_end = xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jfc; + u32 qpn_tmp; + + qpn_tmp = udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_NONE, 0); + + return udma_object_queue_create(udma_dev, CQM_OBJECT_RDMA_SCQ, size, + jfc, &udma_dev->jfcn_record, + room_header_alloc, qpn_tmp, xid_start, + xid_end); +} + +static void udma_init_user_jfc_param(struct udma_jfc *jfc, + const struct ubcore_jfc_cfg *cfg, + u8 mtt_pro_en, u32 buf_size) +{ + uint32_t page_size; + + jfc->base.jfc_cfg = *(cfg); + jfc->base.jfc_cfg.depth += UB_RESERVED_CQE; + page_size = mtt_pro_en == 1 ? + udma_cal_page_size(jfc->base.jfc_cfg.depth * + UB_CQE_SIZE) : + PAGE_SIZE; + jfc->base.jfc_cfg.depth = + (jfc->base.jfc_cfg.depth < (page_size / UB_CQE_SIZE)) ? + (page_size / UB_CQE_SIZE) : + ((u32)roundup_pow_of_two(jfc->base.jfc_cfg.depth)); + jfc->base.id = jfc->cqm_jfc->index; + jfc->page_shift = (u32)ilog2(page_size); + jfc->cq_buf_size = buf_size; +} + +static int udma_create_user_jfc(struct udma_device *udma_dev, + const struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata, + struct udma_jfc *jfc) +{ + struct udma_context *udma_ctx = to_udma_ctx(udata->uctx); + struct ub_create_jetty_ucmd ucmd = { 0 }; + int ret; + + ret = (int)copy_from_user( + &ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (u32)sizeof(struct ub_create_jetty_ucmd))); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to copy JFC udata, ret = %d.\n", ret); + ret = -EFAULT; + goto err; + } + + jfc->cqm_jfc = udma_get_cqm_jfc(udma_dev, 0, jfc, false); + if (jfc->cqm_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to create cqm jfc.\n"); + ret = -ENOMEM; + goto err; + } + + udma_init_user_jfc_param(jfc, cfg, udma_dev->ub_cap.dev_cap.mtt_pro_en, + ucmd.buf_size); + ret = udma_init_user_mtt(udma_dev, &jfc->ub_mtt, &jfc->umem, + ucmd.buf_addr, ucmd.buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "udma init user mtt failed, ret = %d.\n", ret); + goto err_init_user_mtt; + } + + ret = udma_db_map_user(udma_ctx, ucmd.db_addr, &jfc->db); + if (ret != 0) + goto err_map_db; + + return 0; + +err_map_db: + udma_uninit_user_mtt(udma_dev, &jfc->ub_mtt, jfc->umem); + jfc->umem = NULL; +err_init_user_mtt: + cqm5_object_delete(&jfc->cqm_jfc->object); + jfc->cqm_jfc = NULL; + +err: + return ret; +} + +static void udma_init_kernel_jfc_param(struct udma_jfc *jfc, + const struct ubcore_jfc_cfg *cfg, + u32 depth, u32 buf_size, u8 mtt_pro_en) +{ + jfc->base.jfc_cfg = *(cfg); + jfc->base.jfc_cfg.depth = depth; + jfc->base.id = jfc->cqm_jfc->index; + jfc->buf = &jfc->cqm_jfc->q_room_buf_1; + + jfc->jfc_va = mtt_pro_en == 1 ? jfc->queue_addr.va : + jfc->buf->direct.va; + (void)memset_s(jfc->jfc_va, buf_size, 0xff, buf_size); + jfc->ci = 0; + jfc->db.db_record = (__be32 *)(void *)(&jfc->cqm_jfc->q_header_vaddr + ->doorbell_record); + *(jfc->db.db_record) = 0; + jfc->db.dma = jfc->cqm_jfc->q_header_paddr; + + jfc->page_shift = (u32)ilog2(jfc->buf->buf_size); + spin_lock_init(&jfc->lock); +} + +int udma_alloc_jfc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, + struct queue_addr_info *queue_addr) +{ + u8 mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + u32 queue_len; + u32 buf_size; + int ret; + + buf_size = mtt_pro_en == 1 ? PAGE_SIZE : buf_len; + *cqm_buf_info = udma_get_cqm_jfc(udma_dev, buf_size, object_priv, true); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, "Create cqm jfc failed.\n"); + return -ENOMEM; + } + + if ((*cqm_buf_info)->current_q_room != CQM_RDMA_Q_ROOM_1) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get current_q_room failed.\n"); + ret = -EINVAL; + goto err_get_qroom; + } + + if (mtt_pro_en == 1) { + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc dma addr, ret = %d\n", ret); + ret = -ENOMEM; + goto err_get_qroom; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + } + + return 0; +err_get_qroom: + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + return ret; +} + +static int udma_create_kernel_jfc(struct udma_device *udma_dev, + const struct ubcore_jfc_cfg *cfg, + struct udma_jfc *jfc) +{ + u32 buf_size; + u32 page_size; + u32 depth; + u32 mtt_pro_en = 0; + int ret; + + ret = udma_sysfs_get_mtt_pro_en(&udma_dev->sysfs_res, &mtt_pro_en); + if (ret != 0) { + return ret; + } + udma_dev->ub_cap.dev_cap.mtt_pro_en = mtt_pro_en; + + depth = cfg->depth + UB_RESERVED_CQE; + page_size = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + udma_cal_page_size(depth * UB_CQE_SIZE) : + PAGE_SIZE; + depth = (depth < (page_size / UB_CQE_SIZE)) ? + (page_size / UB_CQE_SIZE) : + ((u32)roundup_pow_of_two(depth)); + buf_size = sizeof(struct tag_ub_cqe) * depth; + + ret = udma_alloc_jfc_queue_resource(udma_dev, &jfc->cqm_jfc, jfc, + buf_size, &jfc->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "alloc jfc queue resource failed, ret = %d\n", ret); + return ret; + } + + udma_init_kernel_jfc_param(jfc, cfg, depth, buf_size, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_init_kernel_mtt(udma_dev, &jfc->ub_mtt, jfc->cqm_jfc, + jfc->queue_addr.pa); + if (ret != 0) { + udma_free_queue_resource(udma_dev, jfc->cqm_jfc, + &jfc->queue_addr); + jfc->cqm_jfc = NULL; + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to init kernel mtt, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +int udma_query_res_jfc(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jfc_val *ret_val = + (struct ubcore_res_jfc_val *)(uintptr_t)val->addr; + ub_jfc_ctx_query_info_s tmp_ctx; + ub_jfc_ctx_query_info_s *ctx = NULL; + struct ubcore_jfc *jfc = NULL; + struct udma_jfc *ub_jfc = NULL; + + ub_jfc = udma_find_jfc(udma_dev, key->key); + if (ub_jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to find ub_jfc, id(%u)\n", key->key); + return -EINVAL; + } + + jfc = &ub_jfc->base; + ctx = (ub_jfc_ctx_query_info_s *)(ub_jfc->cqm_jfc->q_ctx_vaddr); + tmp_ctx.dw2 = be32_to_cpu(ctx->dw2); + ret_val->depth = jfc->jfc_cfg.depth; + ret_val->jfc_id = jfc->id; + udma_jfc_put(ub_jfc); + if (tmp_ctx.state == UB_CQC_STATE_HW) { + ret_val->state = UBCORE_JFC_STATE_VALID; + } else if (tmp_ctx.state == UB_CQC_STATE_SW) { + ret_val->state = UBCORE_JFC_STATE_INVALID; + } else { + ret_val->state = UBCORE_JFC_STATE_ERROR; + } + + return 0; +} + +static void udma_free_jfc_buf_user(struct ubcore_ucontext *uctx, + struct udma_jfc *jfc, + struct udma_device *udma_dev) +{ + if (uctx != NULL) { + udma_db_unmap_user(to_udma_ctx(uctx), &jfc->db); + } + + udma_uninit_user_mtt(udma_dev, &jfc->ub_mtt, jfc->umem); + jfc->umem = NULL; + + if (jfc->cqm_jfc != NULL) { + cqm5_object_delete(&jfc->cqm_jfc->object); + jfc->cqm_jfc = NULL; + } +} + +static void udma_free_jfc_buf_kernel(struct udma_jfc *jfc, + struct udma_device *udma_dev) +{ + udma_uninit_kernel_mtt(udma_dev, &jfc->ub_mtt); + udma_free_queue_resource(udma_dev, jfc->cqm_jfc, &jfc->queue_addr); + jfc->cqm_jfc = NULL; +} + +static int udma_create_jfc_check_param(struct ubcore_device *ub_dev, + const struct ubcore_jfc_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_JFC, + "Invalid parameter: ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "tpf not support this function.\n"); + return -EFAULT; + } + + if (cfg->depth == 0 || + cfg->depth > udma_dev->ub_cap.dev_cap.max_jfc_depth) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "jfc cfg depth out of range, depth:%u, max_depth:%u\n", + cfg->depth, udma_dev->ub_cap.dev_cap.max_jfc_depth); + return -EINVAL; + } + + return 0; +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *ub_dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + int ret; + + ret = udma_create_jfc_check_param(ub_dev, cfg); + if (ret != 0) { + return ERR_PTR(ret); + } + + udma_dev = to_udma_dev(ub_dev); + jfc = (struct udma_jfc *)kzalloc(sizeof(struct udma_jfc), GFP_KERNEL); + if (jfc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, "alloc JFC Failed.\n"); + ret = -ENOMEM; + goto err; + } + jfc->cqm_priv_type = UDMA_CQM_PRIV_TYPE_JFC; + udma_cmdq_rtt_statistic_init(&jfc->cmdq_rtt); + + if (udma_get_uctx(udata) != NULL) + ret = udma_create_user_jfc(udma_dev, cfg, udata, jfc); + else + ret = udma_create_kernel_jfc(udma_dev, cfg, jfc); + if (ret != 0) + goto err_alloc_jfc; + + udma_set_jfc_inline(udma_dev, jfc); + ret = udma_cmd_jfc_ctx(udma_dev, jfc, NULL, UB_CMD_JFC_CREATE_CTX); + if (ret != 0) + goto err_alloc_ctx; + + jfc->arm_sn = 1; + atomic_set(&jfc->used_by_jfs, 0); + + kref_init(&jfc->ref_cnt); + init_completion(&jfc->comp); + udma_store_list(udma_dev, &jfc->node, &udma_dev->device_res->jfc_list); + + udma_info(udma_dev->dev, UDMA_M_JFC, + "JFC created successfully, id: %u.\n", jfc->base.id); + return &jfc->base; + +err_alloc_ctx: + if (udma_get_uctx(udata) != NULL) { + udma_free_jfc_buf_user(udma_get_uctx(udata), jfc, udma_dev); + } else { + udma_free_jfc_buf_kernel(jfc, udma_dev); + } + +err_alloc_jfc: + kfree(jfc); + +err: + return ERR_PTR(ret); +} + +static int udma_modify_jfc_check_param(struct udma_device *udma_dev, + struct ubcore_jfc_attr *attr) +{ + if ((attr->mask & + (UBCORE_JFC_MODERATE_COUNT | UBCORE_JFC_MODERATE_PERIOD)) == 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "JFC modify mask is not set or invalid.\n"); + return -EINVAL; + } + + if ((attr->mask & UBCORE_JFC_MODERATE_COUNT) != 0 && + (attr->moderate_count > udma_dev->ub_cap.dev_cap.max_jfc_depth)) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "modify max_cnt(%u) > max_jfc_depth(%u).\n", + attr->moderate_count, + udma_dev->ub_cap.dev_cap.max_jfc_depth); + return -EINVAL; + } + + return 0; +} + +int udma_modify_jfc(struct ubcore_jfc *ub_jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + int ret; + + if ((ub_jfc == NULL) || (ub_jfc->ub_dev == NULL) || (attr == NULL)) { + udma_pr_err(UDMA_M_JFC, + "Invalid parameter: ub_jfc(%d), attr(%d).\n", + !!ub_jfc, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_jfc->ub_dev); + jfc = to_udma_jfc(ub_jfc); + udma_info(udma_dev->dev, UDMA_M_JFC, "Modify jfc started, id: %u.\n", + ub_jfc->id); + + ret = udma_modify_jfc_check_param(udma_dev, attr); + if (ret != 0) { + return ret; + } + udma_info(udma_dev->dev, UDMA_M_JFC, + "modify type:%u, max_cnt:0x%x, timeout:0x%x\n", attr->mask, + attr->moderate_count, attr->moderate_period); + + ret = udma_cmd_jfc_ctx(udma_dev, jfc, attr, UB_CMD_JFC_MODIFY_CTX); + if (ret != 0) + return -EFAULT; + + udma_info(udma_dev->dev, UDMA_M_JFC, "Modify jfc finished, id: %u.\n", + ub_jfc->id); + return ret; +} + +static int udma_free_jfc_ctx(struct udma_device *udma_dev, struct udma_jfc *jfc) +{ + int ret; + + ret = udma_cmd_jfc_ctx(udma_dev, jfc, NULL, UB_CMD_JFC_DELETE_CTX); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Fail to delete jfc ctx.\n"); + return ret; + } + + ret = udma_cmd_jfc_ctx(udma_dev, jfc, NULL, UB_CMD_JFC_CACHE_INVLD_CTX); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Fail to cache invld jfc ctx.\n"); + return ret; + } + + return 0; +} + +int udma_destroy_jfc(struct ubcore_jfc *ub_jfc) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + int ret; + + if ((ub_jfc == NULL) || (ub_jfc->ub_dev == NULL)) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: ub_jfc(%d).\n", + !!ub_jfc); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_jfc->ub_dev); + jfc = to_udma_jfc(ub_jfc); + + udma_delete_list(udma_dev, &jfc->node, &udma_dev->device_res->jfc_list); + udma_jfc_put(jfc); + wait_for_completion(&jfc->comp); + + ret = udma_free_jfc_ctx(udma_dev, jfc); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Failed to free jfc ctx, id: %u, ret: %d.\n", + ub_jfc->id, ret); + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&jfc->ref_cnt); + return ret; + } + + if (ub_jfc->uctx == NULL) { + udma_free_jfc_buf_kernel(jfc, udma_dev); + } else { + udma_free_jfc_buf_user(ub_jfc->uctx, jfc, udma_dev); + } + + udma_info(udma_dev->dev, UDMA_M_JFC, + "JFC destroyed successfully, id: %u.\n", ub_jfc->id); + kfree(jfc); + return 0; +} + +void udma_jfc_async_event(struct udma_device *udma_dev, struct udma_jfc *jfc, + u8 type) +{ + struct ubcore_jfc *ub_jfc = &jfc->base; + struct ubcore_event event = { 0 }; + + if (ub_jfc->jfae_handler != NULL) { + event.ub_dev = ub_jfc->ub_dev; + event.element.jfc = ub_jfc; + switch (type) { + case UB_AEQ_TYPE_JFC_ERR: + event.event_type = UBCORE_EVENT_JFC_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JFC, + "unknown jfc asyn event(%u).\n", type); + return; + } + + ub_jfc->jfae_handler(&event, ub_jfc->uctx); + } +} + +static void udma_print_cqe_info(struct udma_device *udma_dev, + const struct udma_jfc *jfc, + const struct tag_ub_cqe *cqe) +{ + u32 step = sizeof(u32); + u32 i; + u32 *k_cqe = (u32 *)cqe; + + if (udma_kernel_log_level_prior(UDMA_LOG_LEVEL_INFO)) { + for (i = 0; i < sizeof(struct tag_ub_cqe) / step; i += step) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "%08x %08x %08x %08x\n", *(k_cqe + i), + *(k_cqe + i + 0x1), *(k_cqe + i + 0x2), + *(k_cqe + i + 0x3)); + } + } else { + for (i = 0; i < sizeof(struct tag_ub_cqe) / step; i += step) { + udma_debug(udma_dev->dev, UDMA_M_JFC, + "%08x %08x %08x %08x\n", *(k_cqe + i), + *(k_cqe + i + 0x1), *(k_cqe + i + 0x2), + *(k_cqe + i + 0x3)); + } + } +} + +static void udma_dfx_print_cqe(struct udma_device *udma_dev, + const struct udma_jfc *jfc, + const struct tag_ub_cqe *cqe) +{ + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: udma_dev(%d).\n", + !!udma_dev); + return; + } + + if (udma_kernel_log_level_prior(UDMA_LOG_LEVEL_INFO)) { + if (cqe->dw0.bs.status == UB_CQ_STATUS_OK) { + return; + } + udma_print_cqe_info(udma_dev, jfc, cqe); + return; + } + + udma_print_cqe_info(udma_dev, jfc, cqe); +} + +static void handle_recv_inline_cqe(struct udma_device *udma_dev, + struct tag_ub_cqe *cqe, struct udma_srq *srq, + struct ubcore_cr *cr, u32 container_idx) +{ + struct ub_wqe_srq_data_seg *sge_list = NULL; + u8 *cqe_inline_buf = NULL; + u32 data_len; + u32 wqe_idx; + u32 sge_idx; + u32 sge_len; + u64 data; + u32 size; + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "handle recv inline cqe, data_len = %u, cqe_inline_data = 0x%x-%x\n", + cqe->byte_cnt, cqe->inline_data[0], cqe->inline_data[1]); + + data = htonll(((u64)(cqe->inline_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->inline_data[1])); + + wqe_idx = (container_idx * srq->container_size) & (srq->max_depth - 1); + sge_list = (struct ub_wqe_srq_data_seg *)ub_srq_get_wqe(srq, wqe_idx); + cqe_inline_buf = (u8 *)&data; + data_len = cqe->byte_cnt; + + for (sge_idx = 0; (sge_idx < srq->max_sge) && (data_len != 0); + sge_idx++) { + sge_len = htonl(sge_list[sge_idx].dw2.length); + size = (sge_len < data_len) ? sge_len : data_len; + + (void)memcpy_s( + (void *)(uintptr_t)htonll(sge_list[sge_idx].addr), + sge_len, (void *)cqe_inline_buf, size); + + data_len -= size; + cqe_inline_buf += size; + } + + if (data_len != 0) + udma_err(udma_dev->dev, UDMA_M_JFC, + "handle_recv_inl_cqe error, remain data_len = %u\n", + data_len); +} + +static int udma_check_recv_jetty_id(struct udma_device *udma_dev, + const struct tag_ub_cqe *cqe, u32 id) +{ + if (cqe->dw3.bs.local_jfx_num != id) { + udma_err( + udma_dev->dev, UDMA_M_JFC, + "jetty/jfr id is incorrect, cqe_id:0x%x, real_id:0x%x\n", + cqe->dw3.bs.wqe_idx, id); + return -EINVAL; + } + return 0; +} + +static struct udma_srq * +udma_get_srq_by_cqm_object(struct udma_device *udma_dev, struct tag_ub_cqe *cqe, + struct tag_cqm_object *object, struct ubcore_cr *cr) +{ + struct udma_jetty *jetty = NULL; + struct udma_jfr *jfr = NULL; + struct udma_srq *srq = NULL; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: udma_dev(%d).\n", + !!udma_dev); + return NULL; + } + + jetty = cqmobj_to_ub_jetty(object); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get srq failed or invalid, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return NULL; + } + + if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) { + if (udma_check_recv_jetty_id(udma_dev, cqe, jetty->jetty_id) != + 0) + return NULL; + + cr->user_data = (uintptr_t)(void *)(&jetty->ubcore_jetty); + + if (jetty->ub_jfr != NULL) + srq = &jetty->ub_jfr->srq; + else + udma_err(udma_dev->dev, UDMA_M_JFC, "ub_jfr is NULL\n"); + } else if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JFR) { + jfr = cqmobj_to_ub_jfr(object); + if (udma_check_recv_jetty_id(udma_dev, cqe, jfr->jfr_id) != 0) + return NULL; + + cr->user_data = (uintptr_t)(void *)(&jfr->ubcore_jfr); + + srq = &jfr->srq; + } + + if ((srq == NULL) || (srq->wrid == NULL) || + (srq->container_size == 0) || (srq->max_depth == 0)) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get srq failed or invalid, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return NULL; + } + + cr->flag.bs.jetty = (jfr == NULL) ? 0 : 1; + return srq; +} + +static void udma_parse_cr_status(struct udma_device *udma_dev, + const struct tag_ub_cqe *cqe, + struct ubcore_cr *cr) +{ + u8 status = cqe->dw0.bs.status; + u32 sub_status = cqe->dw2.bs.sub_status; + + cr->status = UBCORE_CR_WR_UNHANDLED; + switch (status) { + case UB_CQ_STATUS_OK: + cr->status = UBCORE_CR_SUCCESS; + break; + case UB_CQ_STATUS_UNSUPPORTED_OPCODE: + cr->status = UBCORE_CR_UNSUPPORTED_OPCODE_ERR; + break; + case UB_CQ_STATUS_LOCAL_OPERATION_ERROR: + if (sub_status == UB_CQ_SUB_STATUS_LOCAL_LENGTH_ERROR) { + cr->status = UBCORE_CR_LOC_LEN_ERR; + } else if (sub_status == UB_CQ_SUB_STATUS_LOCAL_ACCESS_ERROR) { + cr->status = UBCORE_CR_LOC_ACCESS_ERR; + } else if (sub_status == + UB_CQ_SUB_STATUS_REMOTE_RESPONSE_LENGTH_ERROR) { + cr->status = UBCORE_CR_REM_RESP_LEN_ERR; + } else { + cr->status = UBCORE_CR_LOC_OPERATION_ERR; + } + break; + case UB_CQ_STATUS_REMOTE_OPERATION_ERROR: + if (sub_status == UB_CQ_SUB_STATUS_REMOTE_UNSUPPORTED_REQUEST) { + cr->status = UBCORE_CR_REM_UNSUPPORTED_REQ_ERR; + } else if (sub_status == UB_CQ_SUB_STATUS_REMOTE_ACCESS_ABORT) { + cr->status = UBCORE_CR_REM_ACCESS_ABORT_ERR; + } else { + cr->status = UBCORE_CR_REM_OPERATION_ERR; + } + break; + case UB_CQ_STATUS_TRANSACTION_RETRY_COUNTER_EXCEEDED: + cr->status = UBCORE_CR_RNR_RETRY_CNT_EXC_ERR; + break; + case UB_CQ_STATUS_TRANSACTION_ACK_TIMEOUT: + cr->status = UBCORE_CR_ACK_TIMEOUT_ERR; + break; + case UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED: + cr->status = UBCORE_CR_FLUSH_ERR; + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFC, + "Invalid cr status: 0x%x, sub_status: 0x%x.\n", status, + sub_status); + break; + } +} + +static void udma_trans_flush_state(struct tag_ub_cqe *cqe, struct ubcore_cr *cr, + enum ubcore_jetty_state state) +{ + if (cqe->dw1.bs.fake == 0) { + cr->flag.bs.suspend_done = + (state == UBCORE_JETTY_STATE_SUSPENDED) ? + cqe->dw1.bs.flush_done : + 0; + cr->flag.bs.flush_err_done = + (state == UBCORE_JETTY_STATE_ERROR) ? + cqe->dw1.bs.flush_done : + 0; + return; + } + + cr->flag.bs.suspend_done = 0; + cr->flag.bs.flush_err_done = 0; + + if (cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED) { + if (state == UBCORE_JETTY_STATE_SUSPENDED) { + cr->status = UBCORE_CR_WR_SUSPEND_DONE; + } else if (state == UBCORE_JETTY_STATE_ERROR) { + cr->status = UBCORE_CR_WR_FLUSH_ERR_DONE; + } + } +} + +static void udma_set_drop_flag(const struct udma_device *udma_dev, + const struct tag_ub_cqe *cqe, + struct udma_jetty *jetty, + enum ubcore_jetty_state state, bool *drop_flag) +{ + if (cqe->dw1.bs.fake == 0) + return; + + /* user not switch status is ERR, will cause fake andset flush done of CQE corresponding to cr status in expected outside (i.e. ubcore status cannot indicates cr is fake of), item before scheme is lose掉 this cr */ + if ((state != UBCORE_JETTY_STATE_SUSPENDED) && + (state != UBCORE_JETTY_STATE_ERROR)) { + udma_warn( + udma_dev->dev, UDMA_M_JFC, + "fake cqe but jetty_state(%u) is not SUSPEND or ERR, cqe status(%u), cr will be dropped\n", + state, cqe->dw0.bs.status); + *drop_flag = true; + return; + } +} + +static int udma_update_recv_jetty_tail(struct udma_jfc *jfc, + struct tag_ub_cqe *cqe, + struct ubcore_cr *cr, + struct tag_cqm_object *object, + bool *fake_flag_recv) +{ + struct udma_device *udma_dev = to_udma_dev(jfc->base.ub_dev); + u32 wqe_idx = cqe->dw5.bs.big_rqe_idx; + struct udma_srq *srq = NULL; + + srq = udma_get_srq_by_cqm_object(udma_dev, cqe, object, cr); + if (srq == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get jetty/jfr failed, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return -EINVAL; + } + + if (cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED && + cqe->dw1.bs.fake == 1) { + /* fake flag will be marked when fake flush cqe.wqe_idx == srq.head. the cqe will be skipped */ + if (wqe_idx * srq->container_size == (u32)srq->head) { + udma_err(udma_dev->dev, UDMA_M_JFC, "fake cqe(%u)\n", + wqe_idx); + jfc->ci++; + *fake_flag_recv = true; + return -EINVAL; + } + if (!srq->flush_flag) { + srq->stop_index = (u16)wqe_idx; + udma_err(udma_dev->dev, UDMA_M_JFC, + "srq flush start, wqe(%u)\n", wqe_idx); + } else { + wqe_idx = srq->stop_index; + } + + srq->flush_flag = true; + } + + cr->user_ctx = srq->wrid[(wqe_idx * srq->container_size) & + (srq->max_depth - 1)]; + udma_free_srq_wqe(srq, wqe_idx); + + if (cqe->dw1.bs.inline_flag != 0) + handle_recv_inline_cqe(udma_dev, cqe, srq, cr, wqe_idx); + + if (cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED && + cqe->dw1.bs.fake == 1) { + srq->stop_index = udma_get_next_srq_wqe(srq, srq->stop_index); + if (srq->stop_index * srq->container_size == srq->head) { + srq->flush_flag = false; + udma_err(udma_dev->dev, UDMA_M_JFC, + "srq flush done(%u)\n", srq->stop_index); + jfc->ci++; + } + } + + udma_parse_cr_status(udma_dev, cqe, cr); + + udma_debug(udma_dev->dev, UDMA_M_JFC, + "update srq tail: id:0x%x, tail:0x%x, status:0x%x-%x\n", + cqe->dw3.bs.local_jfx_num, srq->tail, cqe->dw0.bs.status, + cqe->dw2.bs.sub_status); + return 0; +} + +static void udma_handle_send_inline_cqe(struct udma_device *udma_dev, + struct udma_send_queue *sq, + struct tag_ub_cqe *cqe) +{ + struct tag_ub_sq_wqe_s *wqe = NULL; + struct ub_cmd_sge *sge = NULL; + struct ub_cmd_sge tmp_sge; + ub_sq_wqe_task_com_s tmp; + u8 *cqe_inline_buf = NULL; + u32 pi = cqe->dw1.bs.pi; + u32 wqe_off; + u32 data_len; + u32 size; + u64 data; + + wqe_off = (pi & (sq->baseblk_cnt - 1)) << sq->baseblk_shift; + wqe = (struct tag_ub_sq_wqe_s *)(sq->buf + wqe_off); + tmp.dw0_value = ntohl(wqe->tsk_sec.com.dw0_value); + if (tmp.op_type != UB_TX_READ) { + udma_warn(udma_dev->dev, UDMA_M_JFC, + "only read support inline_flag, op_type: %d\n", + tmp.op_type); + return; + } + + data = ntohll(((u64)(cqe->inline_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->inline_data[1])); + cqe_inline_buf = (u8 *)&data; + data_len = cqe->byte_cnt; + + //preventstop wrap around + wqe_off = ((pi + 1) & (sq->baseblk_cnt - 1)) << sq->baseblk_shift; + sge = (ub_cmd_sge_t *)(sq->buf + wqe_off + UB_DEID_BYTE_LENGTH); + + do { + if ((u8 *)sge >= (u8 *)(sq->buf + sq->buf_size)) { + sge = (ub_cmd_sge_t *)sq->buf; + } + tmp_sge = *sge; + tmp_sge.dw3.dw3_value = ntohl(tmp_sge.dw3.dw3_value); + tmp_sge.length = ntohl(tmp_sge.length); + tmp_sge.va = ntohll(tmp_sge.va); + size = (tmp_sge.length > data_len) ? data_len : tmp_sge.length; + memcpy_s((void *)(uintptr_t)tmp_sge.va, tmp_sge.length, + (void *)cqe_inline_buf, size); + + data_len -= size; + cqe_inline_buf += size; + sge++; + } while ((tmp_sge.dw3.bs.L == 0) && (data_len != 0)); +} + +static int +udma_update_send_jetty_tail(struct udma_device *udma_dev, struct udma_jfc *jfc, + struct tag_ub_cqe *cqe, struct ubcore_cr *cr, + struct tag_cqm_object *object, bool *drop_flag) +{ + struct udma_send_queue *sq = NULL; + struct udma_jetty *jetty = NULL; + enum ubcore_jetty_state state; + struct udma_jfs *jfs = NULL; + u8 fake = cqe->dw1.bs.fake; + u16 sq_tail_overflow; + + jetty = cqmobj_to_ub_jetty(object); + if (jetty != NULL) { + if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) { + sq = &jetty->sq; + state = jetty->state; + cr->user_data = (uintptr_t)(&jetty->ubcore_jetty); + } else if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JFS) { + jfs = cqmobj_to_ub_jfs(object); + sq = &jfs->sq; + state = jfs->state; + cr->user_data = (uintptr_t)&jfs->ubcore_jfs; + } + } + + if ((sq == NULL) || (sq->wrid == NULL) || (sq->wrid_size == 0)) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get jetty/jfs failed, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return -EINVAL; + } + + if (cqe->dw1.bs.inline_flag != 0) + udma_handle_send_inline_cqe(udma_dev, sq, cqe); + + if (fake == 0) { + sq_tail_overflow = + (cqe->dw3.bs.wqe_idx < (sq->tail & CQE_WQE_IDX_MASK)) ? + (1 << CQE_WQE_IDX_SHIFT) : + 0; + sq->tail = cqe->dw3.bs.wqe_idx + + (sq->tail & (~CQE_WQE_IDX_MASK)) + sq_tail_overflow; + cr->user_ctx = sq->wrid[sq->tail & (sq->wrid_size - 1)]; + sq->tail++; + } + + if (cqe->dw1.bs.flush_done == 1) + sq->flush_tail = sq->tail; + + cr->flag.bs.jetty = + (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) ? 1 : 0; + udma_parse_cr_status(udma_dev, cqe, cr); + udma_trans_flush_state(cqe, cr, state); + udma_set_drop_flag(udma_dev, cqe, jetty, state, drop_flag); + + sq->is_hw_flush_done = cqe->dw1.bs.flush_done; + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "update %s tail: id:0x%x, tail:0x%x, wqe_idx: %u, fake: %u, status:0x%x-%x\n", + (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) ? "send jetty" : + "jfs", + cqe->dw3.bs.local_jfx_num, sq->tail, cqe->dw3.bs.wqe_idx, fake, + cqe->dw0.bs.status, cqe->dw2.bs.sub_status); + + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "jetty_type(%u), state(%u), status(%u), flush_done(%u), suspend_done(%u), flush_err_done(%u)\n", + jetty->cqm_priv_type, state, cr->status, cqe->dw1.bs.flush_done, + cr->flag.bs.suspend_done, cr->flag.bs.flush_err_done); + + return 0; +} + +static bool udma_should_get_new_object(struct udma_device *udma_dev, + struct tag_cqm_object *object, + const struct tag_ub_cqe *cqe) +{ + struct udma_jetty *jetty = NULL; + struct udma_jfr *jfr = NULL; + u32 round; + + if (cqe->dw0.bs.s_r == UB_SQ_CQE) + return false; + + if (object == NULL) + return true; + + jetty = cqmobj_to_ub_jetty(object); + if (jetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "get jetty/jfr from object failed\n"); + return true; + } + + if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) { + round = jetty->round; + } else if (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JFR) { + jfr = cqmobj_to_ub_jfr(object); + round = jfr->round; + } else { + udma_err(udma_dev->dev, UDMA_M_JFC, + "cqm obj not match jetty/jfr\n"); + return true; + } + + if ((u8)round == cqe->dw15.bs.jetty_round) + return false; + + udma_debug(udma_dev->dev, UDMA_M_JFC, + "jetty_type(%u), jfx round(%x), cqe round(%x)\n", + jetty->cqm_priv_type, round, cqe->dw15.bs.jetty_round); + + return (jetty->cqm_priv_type == UDMA_JETTY_TYPE_JETTY) ? true : false; +} + +static int udma_update_jetty_tail(struct udma_device *udma_dev, + struct udma_jfc *jfc, struct tag_ub_cqe *cqe, + struct ubcore_cr *cr, bool *drop_flag) +{ + struct tag_cqm_object *object = + cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + cqe->dw3.bs.local_jfx_num, false); + int ret; + + if (udma_should_get_new_object(udma_dev, object, cqe) == true) { + if (object != NULL) { + cqm5_object_put(object); + } + /* for at jetty recv cqe, and jetty not match of scenario, try use jfr report up CR */ + object = cqm5_object_get(udma_dev->hwdev, + CQM_OBJECT_SERVICE_CTX, + cqe->dw15.bs.jfrn, false); + + udma_warn( + udma_dev->dev, UDMA_M_JFC, + "get cqe info, local_jfx_num(%u), jfrn(%u), status(%u)\n", + cqe->dw3.bs.local_jfx_num, cqe->dw15.bs.jfrn, + cqe->dw0.bs.status); + + if (object != NULL) { + cqe->dw3.bs.local_jfx_num = cqe->dw15.bs.jfrn; + cqe->dw0.bs.status = + UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED; + } + } + + if (object == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFC, + "Get cqm object failed, jetty_id:0x%x\n", + cqe->dw3.bs.local_jfx_num); + return -EINVAL; + } + + if (cqe->dw0.bs.s_r == UB_SQ_CQE) { + atomic64_inc(&jfc->udma_driver_jfc_ctr.poll_sq_num); + atomic64_inc(&udma_dev->udma_driver_device_ctr.poll_sq_num); + ret = udma_update_send_jetty_tail(udma_dev, jfc, cqe, cr, + object, drop_flag); + } else { + atomic64_inc(&jfc->udma_driver_jfc_ctr.poll_rq_num); + atomic64_inc(&udma_dev->udma_driver_device_ctr.poll_rq_num); + ret = udma_update_recv_jetty_tail(jfc, cqe, cr, object, + drop_flag); + } + + cqm5_object_put(object); + return ret; +} + +static void udma_fill_recv_cr(struct udma_device *udma_dev, + struct tag_ub_cqe *cqe, struct ubcore_cr *cr) +{ + udma_debug(udma_dev->dev, UDMA_M_JFC, "recv_cqe.optype(0x%x)\n", + cqe->dw1.bs.op_type); + cr->remote_id.id = cqe->dw2.bs.rmt_jfx_num; + (void)memcpy_s(&cr->remote_id.eid, sizeof(cr->remote_id.eid), + cqe->remote_eid, sizeof(cqe->remote_eid)); + cr->flag.bs.s_r = 1; //recv + switch (cqe->dw1.bs.op_type) { + case UB_CQE_RQ_OPTYPE_WRITE_IMMEDIATE: + cr->opcode = UBCORE_CR_OPC_WRITE_WITH_IMM; + cr->imm_data = + ((u64)(cqe->imme_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->imme_data[1]); + break; + + case UB_CQE_RQ_OPTYPE_SEND: + cr->opcode = UBCORE_CR_OPC_SEND; + break; + + case UB_CQE_RQ_OPTYPE_SEND_IMMEDIATE: + cr->opcode = UBCORE_CR_OPC_SEND_WITH_IMM; + cr->imm_data = + ((u64)(cqe->imme_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->imme_data[1]); + break; + + case UB_CQE_RQ_OPTYPE_SEND_INVALIDATE: + cr->opcode = UBCORE_CR_OPC_SEND_WITH_INV; + cr->imm_data = + ((u64)(cqe->imme_data[0]) << UB_LONG_DATA_OFFSET) | + (u64)(cqe->imme_data[1]); + break; + + default: + udma_err(udma_dev->dev, UDMA_M_JFC, "invalid cqe optype:%x\n", + cqe->dw1.bs.op_type); + break; + } +} + +static void udma_fill_cr(struct udma_device *udma_dev, struct tag_ub_cqe *cqe, + struct ubcore_cr *cr) +{ + cr->local_id = cqe->dw3.bs.local_jfx_num; + cr->completion_len = cqe->byte_cnt; + cr->tpn = cqe->dw0.bs.tpn; + + cr->flag.bs.s_r = 0; //send + cr->opcode = (enum ubcore_cr_opcode)INVALID_CR_OPCODE; + + if (cqe->dw0.bs.s_r == UB_RQ_CQE) + udma_fill_recv_cr(udma_dev, cqe, cr); +} + +static int udma_poll_jfc_one(struct udma_jfc *jfc, struct ubcore_cr *cr, + bool *drop_flag) +{ + struct udma_device *udma_dev = to_udma_dev(jfc->base.ub_dev); + u32 idx = jfc->ci & (jfc->base.jfc_cfg.depth - 1); + struct tag_ub_cqe *cqe = NULL; + u32 valid_owner; + int ret; + if (jfc->base.ub_dev == NULL) { + udma_pr_err(UDMA_M_JFC, "Invalid parameter: ub_dev(%d).\n", + !!jfc->base.ub_dev); + return -EINVAL; + } + + valid_owner = (((jfc->ci >> ilog2(jfc->base.jfc_cfg.depth)) & 1) == 0) ? + 1 : + 0; + cqe = (struct tag_ub_cqe *)((u8 *)jfc->jfc_va + + idx * sizeof(struct tag_ub_cqe)); + if (cqe->dw0.bs.o == valid_owner) + return -EAGAIN; + + rmb(); + + udma_dfx_print_cqe(udma_dev, jfc, cqe); + + /* fake flush cqe is a flag to flush jfr wr, can't move to next cqe until flush done */ + if (!(cqe->dw0.bs.s_r == UB_RQ_CQE && cqe->dw1.bs.fake == 1 && + cqe->dw0.bs.status == UB_CQ_STATUS_JETTY_WORK_REQUEST_FLUSHED)) { + jfc->ci++; + } + + ret = udma_update_jetty_tail(udma_dev, jfc, cqe, cr, drop_flag); + if (*drop_flag == true) + return 0; + if (ret != 0) + return ret; + + udma_fill_cr(udma_dev, cqe, cr); + return 0; +} + +int udma_poll_jfc(struct ubcore_jfc *ub_jfc, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *jfc = NULL; + bool drop_flag = false; + ulong flags = 0; + int npolled; + int ret = 0; + + if ((ub_jfc == NULL) || (ub_jfc->ub_dev == NULL) || (cr == NULL)) { + udma_pr_err(UDMA_M_JFC, + "%s Invalid parameter: ub_jfc(%d), cr(%d).\n", + __func__, !!ub_jfc, !!cr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_jfc->ub_dev); + jfc = to_udma_jfc(ub_jfc); + spin_lock_irqsave(&jfc->lock, flags); + + for (npolled = 0; npolled < cr_cnt; ++npolled) { + ret = udma_poll_jfc_one(jfc, cr + npolled, &drop_flag); + if (ret != 0) + break; + if (drop_flag) { + drop_flag = false; + --npolled; + udma_info(udma_dev->dev, UDMA_M_JFC, + "drop flag is set, drop cqe\n"); + } + } + + rmb(); + *jfc->db.db_record = cpu_to_be32(jfc->ci & UB_CQ_UPDATE_CI_MASK); + + spin_unlock_irqrestore(&jfc->lock, flags); + + if (ret == 0 || ret == -EAGAIN) { + if (npolled > 0) { + atomic64_add(npolled, + &jfc->udma_driver_jfc_ctr.poll_total_num); + atomic64_add(npolled, &udma_dev->udma_driver_device_ctr + .poll_total_num); + udma_debug( + udma_dev->dev, UDMA_M_JFC, + "jfc_id:0x%x, poll_cnt:%d, jfc_total_poll:%llu, dev_total_poll:%llu\n", + jfc->base.id, npolled, + (u64)atomic64_read(&jfc->udma_driver_jfc_ctr + .poll_total_num), + (u64)atomic64_read( + &udma_dev->udma_driver_device_ctr + .poll_total_num)); + } + return npolled; + } + return ret; +} + +static int udma_update_jfc_db(struct udma_device *udma_dev, + struct udma_jfc *jfc, u32 cmd, u32 arm_sn) +{ + union tag_ub_cq_db_u jfc_db; + int ret; + + jfc_db.cq_db_value = 0; + jfc_db.cq_db.dw0.bs.type = CQ_DB_TYPE; + jfc_db.cq_db.dw0.bs.cqn = jfc->base.id; + jfc_db.cq_db.dw0.bs.c = 1; + jfc_db.cq_db.dw0.bs.n = 1; + jfc_db.cq_db.dw0.bs.cos = 0; + jfc_db.cq_db.dw0.bs.cqc_type = 0; + *(jfc->db.db_record) = cpu_to_be32(jfc->ci & UB_CQ_UPDATE_CI_MASK); + + jfc_db.cq_db.dw1.bs.ci = jfc->ci & UB_CQ_UPDATE_CI_MASK; + jfc_db.cq_db.dw1.bs.cmd = cmd; + jfc_db.cq_db.dw1.bs.cmd_sn = arm_sn; + + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + jfc->ci & 0xff, jfc_db.cq_db_value); + + udma_debug(udma_dev->dev, UDMA_M_JFC, + "xid:0x%x, jfc_db.cq_db_value:0x%llx\n", jfc->base.id, + jfc_db.cq_db_value); + return ret; +} + +int udma_rearm_jfc(struct ubcore_jfc *jfc, bool solicited_only) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfc *ub_jfc = NULL; + u32 arm_sn; + u32 cmd; + int ret; + + if ((jfc == NULL) || (jfc->ub_dev == NULL)) { + udma_pr_err(UDMA_M_JFC, "%s Invalid parameter: jfc(%d).\n", + __func__, !!jfc); + return -EINVAL; + } + + ub_jfc = to_udma_jfc(jfc); + udma_dev = to_udma_dev(jfc->ub_dev); + + cmd = (u32)((solicited_only != 0) ? UB_ARM_JFC_SOLICITED : + UB_ARM_JFC_UN_SOLICITED); + arm_sn = (u32)ub_jfc->arm_sn & (u32)UB_CQ_ARMSN_MASK; + + ret = udma_update_jfc_db(udma_dev, ub_jfc, cmd, arm_sn); //update db + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFC, + "udma update jfc(0x%x) db failed,\n", jfc->id); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h new file mode 100644 index 000000000..52ee9a4fd --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfc.h @@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_JFC_H +#define UDMA_JFC_H + +#include "ubcore_api.h" +#include "udma_common.h" +#include "udma_db.h" +#include "hinic5_cqm.h" + +#define UB_CQE_SIZE 64 +#define UB_JFC_CQE_SIZE 2 +#define CQ_DB_TYPE 17 + +#define UB_CQ_ARMSN_MASK 3 +#define UB_CQ_UPDATE_CI_MASK 0xffffff +#define UB_LONG_DATA_OFFSET 32 + +enum ub_arm_jfc_type { UB_ARM_JFC_SOLICITED = 1, UB_ARM_JFC_UN_SOLICITED = 2 }; + +typedef struct udma_driver_jfc_ctr { + atomic64_t poll_total_num; + atomic64_t poll_sq_num; + atomic64_t poll_rq_num; +} udma_driver_jfc_ctr_t; + +typedef struct udma_jfc { + u32 cqm_priv_type; + struct ubcore_jfc base; + struct tag_cqm_queue *cqm_jfc; + struct tag_cqm_buf *buf; + struct queue_addr_info queue_addr; + void *jfc_va; + struct udma_db db; + struct mtt ub_mtt; + struct ubcore_umem *umem; + spinlock_t lock; + u32 ci; + u32 page_shift; + u32 cq_buf_size; + u32 arm_sn; + udma_driver_jfc_ctr_t udma_driver_jfc_ctr; + struct list_head node; + atomic_t used_by_jfs; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +} udma_jfc_t; + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *ub_dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); +int udma_modify_jfc(struct ubcore_jfc *ub_jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); +int udma_destroy_jfc(struct ubcore_jfc *ub_jfc); +int udma_poll_jfc(struct ubcore_jfc *ub_jfc, int cr_cnt, struct ubcore_cr *cr); +void udma_jfc_async_event(struct udma_device *udma_dev, struct udma_jfc *jfc, + u8 type); +int udma_rearm_jfc(struct ubcore_jfc *jfc, bool solicited_only); +struct udma_jfc *udma_find_jfc(struct udma_device *udev, u32 jfc_id); +int udma_query_res_jfc(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val); +void udma_jfc_put(void *obj); +void udma_jfc_get(void *obj); + +u32 udma_get_jfc_inline(struct udma_device *udma_dev, + const struct ubcore_jfc *jfc); + +static inline struct udma_jfc *to_udma_jfc(const struct ubcore_jfc *jfc) +{ + return (struct udma_jfc *)udma_container_of(jfc, struct udma_jfc, base); +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c new file mode 100644 index 000000000..3d8269f56 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfr.c @@ -0,0 +1,1042 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "ubcore_uapi.h" +#include "ub_aeqe.h" +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_jetty.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_abi.h" +#include "udma_srq.h" +#include "udma_cmd_srq.h" +#include "udma_cmd_jfr.h" +#include "udma_cmd_jetty.h" + +#define UB_QUERY_JFR_MASK 0x3 + +static void udma_jfr_kref_release(struct kref *ref_cnt) +{ + struct udma_jfr *jfr = container_of(ref_cnt, struct udma_jfr, ref_cnt); + + complete(&jfr->comp); +} + +void udma_jfr_put(void *obj) +{ + struct udma_jfr *jfr = obj; + + if (jfr != NULL) { + kref_put(&jfr->ref_cnt, udma_jfr_kref_release); + } +} + +void udma_jfr_get(void *obj) +{ + struct udma_jfr *jfr = obj; + + kref_get(&jfr->ref_cnt); +} + +struct udma_jfr *udma_find_jfr(struct udma_device *udev, u32 jfr_id) +{ + struct udma_jfr *udma_jfr = NULL; + struct mutex *lock = NULL; + + if (udev == NULL) { + udma_pr_err(UDMA_M_JFR, "Invalid parameter: udev(%d).\n", + !!udev); + return NULL; + } + lock = &udev->device_res->jfr_list.node_lock; + mutex_lock(lock); + list_for_each_entry(udma_jfr, &udev->device_res->jfr_list.node, node) { + if (udma_jfr->jfr_id == jfr_id) { + udma_jfr_get(udma_jfr); + mutex_unlock(lock); + return udma_jfr; + } + } + mutex_unlock(lock); + + return NULL; +} + +static int udma_create_jfr_check(struct ubcore_device *ub_dev, + const struct ubcore_jfr_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_JFR, + "Invalid parameter, ub_dev(%d), cfg(%d)\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "tpf not support this function.\n"); + return -EINVAL; + } + + if (cfg->depth == 0 || + cfg->depth > udma_dev->ub_cap.dev_cap.max_jfr_depth) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "invalid parameter: jfr depth:%u, dev_cap jfr_depth:%u\n", + cfg->depth, udma_dev->ub_cap.dev_cap.max_jfr_depth); + return -EINVAL; + } + + if (cfg->max_sge > udma_dev->ub_cap.dev_cap.max_jfr_sge) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "invalid parameter: jfr max_sge:%u, dev_cap max_jfr_sge:%u\n", + cfg->max_sge, udma_dev->ub_cap.dev_cap.max_jfr_sge); + return -EINVAL; + } + + if (cfg->flag.bs.token_policy != UBCORE_TOKEN_NONE && + cfg->flag.bs.token_policy != UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "token policy(%u) only support none or plain text.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + if (cfg->flag.bs.tag_matching != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr don't support tag matching.\n"); + return -EINVAL; + } + + if (cfg->trans_mode != UBCORE_TP_RC && + cfg->trans_mode != UBCORE_TP_RM && + cfg->trans_mode != UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr don't support trans_mode:%u\n", cfg->trans_mode); + return -EINVAL; + } + + if (cfg->trans_mode == UBCORE_TP_RC && + cfg->flag.bs.order_type != UBCORE_OT) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr don't support trans_mode:RC, order_type:%u.\n", + cfg->flag.bs.order_type); + return -EINVAL; + } + + return 0; +} + +static int udma_alloc_jfrc(struct udma_device *udma_dev, struct udma_jfr *jfr) +{ + struct tag_cqm_qpc_mpt *jfrc_info = NULL; + u32 qpn_tmp = udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_NONE, 0); + + jfrc_info = udma_object_qpc_create( + udma_dev, UBEP_JETTY_CONTEXT_SIZE, jfr, + &udma_dev->jettyn_record, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + if (jfrc_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create jfrc by cqm object failed\n"); + return -ENOMEM; + } + + jfr->jfrc_info = jfrc_info; + jfr->jfr_id = jfrc_info->xid; + jfr->srq.jfr_id = jfrc_info->xid; + return 0; +} + +static int udma_create_jfr_context_update(struct udma_device *udma_dev, + struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + int ret = 0; + + ret = udma_cmd_jfr_create_ctx(udma_dev, jfr, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create jfr context failed, ret:%d\n", ret); + return ret; + } + + if ((udma_get_uctx(udata) != NULL)) { + if (udata->udrv_data->out_addr == (uint64_t)NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "out_addr is NUll\n"); + return ret; + } + + if ((copy_to_user((void *)(uintptr_t)udata->udrv_data->out_addr, + &jfr->round, sizeof(u32)) != 0)) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Failed to copy to user driver\n"); + return -EIO; + } + } + + return ret; +} + +static int udma_jfr_cache_out_cmd(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "allocate wb_va failed\n"); + return -ENOMEM; + } + + ret = udma_cmd_jfr_cache_out_ctx(udma_dev, udma_jfr, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static int udma_delete_jfr_context_and_cache_out(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr) +{ + int ret; + + ret = udma_cmd_jfr_delete_ctx(udma_dev, udma_jfr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete jfrc:0x%06x failed.\n", udma_jfr->jfr_id); + return ret; + } + + ret = udma_jfr_cache_out_cmd(udma_dev, udma_jfr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr:0x%06x cache invalid.\n", udma_jfr->jfr_id); + return ret; + } + + if (udma_jfr->jfrc_info != NULL) { + cqm5_object_delete(&(udma_jfr->jfrc_info->object)); + udma_jfr->jfrc_info = NULL; + } + + return ret; +} + +static void udma_jfr_user_deinit_umem(struct udma_device *udma_dev, + struct ubcore_umem **umem, + struct mtt *ub_mtt) +{ + if (ub_mtt->mtt_seg != NULL) { + udma_destroy_user_mtt(udma_dev, ub_mtt); + ub_mtt->mtt_seg = NULL; + } + + if (!IS_ERR_OR_NULL(*umem)) { + udma_put_umem(*umem); + *umem = NULL; + } +} + +static int udma_jfr_user_init_umem(struct udma_device *udma_dev, + struct ubcore_umem **umem, + struct mtt *ub_mtt, u64 buf_addr, + u32 buf_size) +{ + int ret = 0; + *umem = udma_get_umem(udma_dev, buf_addr, buf_size); + if (IS_ERR_OR_NULL(*umem)) { + udma_err(udma_dev->dev, UDMA_M_JFR, "get jfr umem failed.\n"); + *umem = NULL; + return -ENOMEM; + } + + ret = udma_create_user_mtt(udma_dev, *umem, ub_mtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create user mtt failed, ret:%d\n", ret); + udma_put_umem(*umem); + *umem = NULL; + } + + return ret; +} + +static int udma_jfr_create_user(struct udma_device *udma_dev, + const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata, + struct udma_jfr *jfr) +{ + struct ub_create_jetty_ucmd ucmd = { 0 }; + struct udma_context *udma_ctx = NULL; + u32 ucmd_len; + int ret = 0; + + udma_ctx = to_udma_ctx(udata->uctx); + ucmd_len = min(udata->udrv_data->in_len, (u32)sizeof(ucmd)); + if (copy_from_user(&ucmd, (void *)(uintptr_t)udata->udrv_data->in_addr, + ucmd_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "copy JFR udata failed, ret:%d.\n", ret); + return -EFAULT; + } + jfr->srq.buf_size = ucmd.buf_size; + + ret = udma_init_user_mtt(udma_dev, &jfr->srq.ub_mtt, &jfr->srq.umem, + ucmd.buf_addr, ucmd.buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "udma init user mtt failed, ret = %d.\n", ret); + return ret; + } + udma_fill_queue_addr_info(&jfr->srq.queue_addr, + jfr->srq.ub_mtt.mtt_paddr, jfr->srq.buf_size, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + if (jfr->srq.index_queue_en) { + ret = udma_jfr_user_init_umem(udma_dev, &jfr->index_queue.umem, + &jfr->index_queue.ub_mtt, + ucmd.index_q_buf_addr, + ucmd.index_q_buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "init jfr index queue umem failed, ret:%d\n", + ret); + goto err_init_index_queue_umem; + } + } + + ret = udma_db_map_user(udma_ctx, ucmd.sdb_addr, &jfr->sdb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "map user db failed, ret:%d\n", ret); + goto err_map_db; + } + + ret = udma_create_srqc_update(udma_dev, &jfr->srq, &jfr->sdb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "create srqc failed, ret:%d\n", ret); + goto err_create_srqc; + } + return ret; + +err_create_srqc: + udma_db_unmap_user(udma_ctx, &jfr->sdb); +err_map_db: + udma_uninit_user_mtt(udma_dev, &jfr->srq.ub_mtt, jfr->srq.umem); + jfr->srq.umem = NULL; + if (jfr->srq.index_queue_en) { + udma_jfr_user_deinit_umem(udma_dev, &jfr->index_queue.umem, + &jfr->index_queue.ub_mtt); + } +err_init_index_queue_umem: + udma_jfr_user_deinit_umem(udma_dev, &jfr->srq.umem, &jfr->srq.ub_mtt); + return ret; +} + +int udma_alloc_srq_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, struct udma_jfr *jfr, + u32 buf_len, struct queue_addr_info *queue_addr) +{ + u8 mtt_pro_en = udma_dev->ub_cap.dev_cap.mtt_pro_en; + u32 xid_start = 0; + u32 xid_end = xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jfr; + u32 queue_len; + u32 buf_size; + int ret; + + buf_size = mtt_pro_en == 1 ? PAGE_SIZE : buf_len; + *cqm_buf_info = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SRQ, buf_size, (void *)jfr, + &udma_dev->srqn_record, true, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_0_0_1, + jfr->jfr_id), + xid_start, xid_end); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to create queue by cqm object.\n"); + return -ENOMEM; + } + + if (mtt_pro_en == 1) { + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + udma_err(udma_dev->dev, UDMA_M_DEV, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + } + + return 0; +} + +static int udma_jfr_create_kernel(struct udma_device *udma_dev, + const struct ubcore_jfr_cfg *cfg, + struct udma_jfr *jfr) +{ + struct tag_cqm_queue *srq_buf_info = NULL; + struct udma_srq *srq = &jfr->srq; + int ret = 0; + + ret = udma_alloc_srq_resource(udma_dev, &srq_buf_info, jfr, + srq->buf_size, &jfr->srq.queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, + "Failed to create srq queue, ret = %d.\n", ret); + return -ENOMEM; + } + + ret = udma_init_kernel_mtt(udma_dev, &jfr->srq.ub_mtt, srq_buf_info, + jfr->srq.queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_srq_init_res(udma_dev, srq, srq_buf_info); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "init srq buf failed, ret:%d\n", ret); + goto err_and_free_queue_mtt; + } + + srq->wrid = (u64 *)kzalloc((size_t)((u32)srq->max_depth * sizeof(u64)), + GFP_KERNEL); + + if (srq->wrid == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "alloc srq wrid fail, ret:%d\n", ret); + ret = -ENOMEM; + goto err_and_free_queue_mtt; + } + + ret = udma_cmd_srq_create_ctx(udma_dev, srq, &srq->db); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "enable srq context failed, ret:%d\n", ret); + goto err_and_free_queue_res; + } + + return 0; + +err_and_free_queue_res: + kfree(srq->wrid); + srq->wrid = NULL; +err_and_free_queue_mtt: + udma_uninit_kernel_mtt(udma_dev, &jfr->srq.ub_mtt); +err_init_kernel_mtt: + udma_free_queue_resource(udma_dev, srq_buf_info, &jfr->srq.queue_addr); + jfr->srq.srqc_info = NULL; + return ret; +} + +static int udma_jfr_delete_kernel(struct udma_device *udma_dev, + struct udma_jfr *jfr) +{ + struct udma_srq *srq = &jfr->srq; + int ret; + + ret = udma_cmd_srq_delete_ctx(udma_dev, srq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "srqc delete and cache out failed, ret:%d\n", ret); + return ret; + } + + if (srq->wrid != NULL) { + kfree(srq->wrid); + srq->wrid = NULL; + } + + udma_uninit_kernel_mtt(udma_dev, &srq->ub_mtt); + udma_free_queue_resource(udma_dev, srq->srqc_info, &srq->queue_addr); + srq->srqc_info = NULL; + + return 0; +} + +static int udma_jfr_delete_user(struct udma_device *udma_dev, + struct udma_context *udma_ctx, + struct udma_jfr *udma_jfr) +{ + int ret; + + if (udma_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "jfr is null\n"); + return -EINVAL; + } + + ret = udma_delete_srqc_and_cache_out(udma_dev, &udma_jfr->srq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete srqc:0x%06x failed, ret:%d\n", + udma_jfr->srq.srqn, ret); + return ret; + } + + udma_db_unmap_user(udma_ctx, &udma_jfr->sdb); + + if (udma_jfr->srq.index_queue_en) { + udma_jfr_user_deinit_umem(udma_dev, &udma_jfr->index_queue.umem, + &udma_jfr->index_queue.ub_mtt); + } + + udma_uninit_user_mtt(udma_dev, &udma_jfr->srq.ub_mtt, + udma_jfr->srq.umem); + udma_jfr->srq.umem = NULL; + + return ret; +} + +/* 1825v100 modify jfr index queue limit threshold: idx_wqe_cnt_lth */ +static int udma_modify_jfr_idx_queue_lth(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr, + u16 idx_queue_lth) +{ + struct udma_jfr_modify_cfg cfg = { 0 }; + + cfg.mask.idx_queue_lth = 1; + cfg.idx_queue_lth = idx_queue_lth; + + if (udma_cmd_jfr_modify_ctx(udma_dev, udma_jfr, &cfg) != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr index queue limit threshold cmd failed, jfrid:%u, old_lth:%u, new_lth:%u\n", + udma_jfr->jfr_id, udma_jfr->idx_queue_lth, + idx_queue_lth); + return -EFAULT; + } + udma_jfr->idx_queue_lth = idx_queue_lth; + udma_info( + udma_dev->dev, UDMA_M_JFR, + "modify jfr index queue limit threshold successfully, jfrid:%u\n", + udma_jfr->jfr_id); + return 0; +} + +static int udma_modify_jfr_threshold(struct udma_device *udma_dev, + struct udma_jfr *jfr, u32 threshold) +{ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + return udma_modify_jfr_idx_queue_lth(udma_dev, jfr, + (u16)threshold); + } + return udma_modify_srq_threshold(udma_dev, &jfr->srq, (u8)threshold); +} + +static int udma_modify_jfr_state(struct udma_device *udma_dev, + struct udma_jfr *udma_jfr, + enum ubcore_jfr_state state, + struct ubcore_udata *udata) +{ + struct udma_db *sdb = &udma_jfr->sdb; + struct udma_srq *srq = &udma_jfr->srq; + struct udma_jfr_modify_cfg cfg = { 0 }; + + cfg.mask.state = 1; + cfg.state = state; + + if (udma_cmd_jfr_modify_ctx(udma_dev, udma_jfr, &cfg) != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr state cmd failed, jfrid:%u, old_state:%u, new_state:%u\n", + udma_jfr->jfr_id, udma_jfr->state, state); + return -EFAULT; + } + + if (udma_jfr->state == UBCORE_JFR_STATE_RESET && + state == UBCORE_JFR_STATE_READY) { + if (udma_get_uctx(udata) == NULL) { + (void)memset_s(srq->wrid, + srq->max_depth * sizeof(*(srq->wrid)), 0, + srq->max_depth * sizeof(*(srq->wrid))); + srq->container_warn_th = 0; + udma_srq_init_res(udma_dev, srq, srq->srqc_info); + sdb = &srq->db; + } + + if (udma_cmd_srq_create_ctx(udma_dev, srq, sdb) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "RESET: Enable srq context failed.\n"); + return -EFAULT; + } + } + udma_jfr->state = state; + udma_info(udma_dev->dev, UDMA_M_JFR, + "modify jfr state successfully, jfrid:%u\n", + udma_jfr->jfr_id); + return 0; +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *ub_dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfr *jfr = NULL; + int ret = 0; + + if (udma_create_jfr_check(ub_dev, cfg) != 0) { + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + jfr = (struct udma_jfr *)kcalloc(1, sizeof(*jfr), GFP_KERNEL); + if (jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "kcalloc JFR failed.\n"); + return ERR_PTR(-ENOMEM); + } + + jfr->cqm_priv_type = UDMA_JETTY_TYPE_JFR; + udma_srq_init_attr(udma_dev, &jfr->srq, cfg->depth, cfg->max_sge); + if (jfr->srq.index_queue_en) { + udma_index_queue_init_attr(udma_dev, &jfr->index_queue, + cfg->depth); + } + jfr->ubcore_jfr.jfr_cfg = *cfg; + udma_cmdq_rtt_statistic_init(&jfr->cmdq_rtt); + + if (udma_alloc_jfrc(udma_dev, jfr) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "allocate jfrc failed, ret:%d\n", ret); + ret = -ENOMEM; + goto err_init_srq; + } + + if (udma_get_uctx(udata) == NULL) + ret = udma_jfr_create_kernel(udma_dev, cfg, jfr); + else + ret = udma_jfr_create_user(udma_dev, cfg, udata, jfr); + /* udma_jfr_create interface in has failure exception log */ + if (ret != 0) + goto err_create_jfr; + + ret = udma_create_jfr_context_update(udma_dev, jfr, cfg, udata); + /* udma_create_jfr_context_update interface in has failure exception log */ + if (ret != 0) + goto err_create_jfrc; + + jfr->ubcore_jfr.jfr_id.id = jfr->jfr_id; + kref_init(&jfr->ref_cnt); + init_completion(&jfr->comp); + udma_store_list(udma_dev, &jfr->node, &udma_dev->device_res->jfr_list); + + jfr->state = UBCORE_JFR_STATE_READY; + udma_info(udma_dev->dev, UDMA_M_JFR, + "create jfr:%u, srq:%u successfully.\n", jfr->jfr_id, + jfr->srq.srqn); + return &jfr->ubcore_jfr; + +err_create_jfrc: + if (udma_get_uctx(udata) == NULL) + udma_jfr_delete_kernel(udma_dev, jfr); + else + udma_jfr_delete_user(udma_dev, to_udma_ctx(udata->uctx), jfr); +err_create_jfr: + cqm5_object_delete(&(jfr->jfrc_info->object)); + jfr->jfrc_info = NULL; +err_init_srq: + kfree(jfr); + return ERR_PTR(ret); +} + +int udma_destroy_jfr(struct ubcore_jfr *jfr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfr *udma_jfr = NULL; + int ret; + + if (jfr == NULL || jfr->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFR, "Invalid parameter, jfr(%d).\n", !!jfr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfr->ub_dev); + udma_jfr = to_udma_jfr(jfr); + udma_jfr->state = UBCORE_JFR_STATE_RESET; + + udma_delete_list(udma_dev, &udma_jfr->node, + &udma_dev->device_res->jfr_list); + udma_jfr_put(udma_jfr); + wait_for_completion(&udma_jfr->comp); + + ret = udma_delete_jfr_context_and_cache_out(udma_dev, udma_jfr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete jfrc:0x%06x failed. ret:%d\n", + udma_jfr->jfr_id, ret); + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&udma_jfr->ref_cnt); + return ret; + } + + if (jfr->uctx == NULL) + ret = udma_jfr_delete_kernel(udma_dev, udma_jfr); + else + ret = udma_jfr_delete_user(udma_dev, to_udma_ctx(jfr->uctx), + udma_jfr); + if (ret != 0) { + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&udma_jfr->ref_cnt); + udma_err(udma_dev->dev, UDMA_M_JFR, + "delete jfr:0x%06x failed. ret:%d\n", udma_jfr->jfr_id, + ret); + return ret; + } + + udma_info(udma_dev->dev, UDMA_M_JFR, + "destroy jfr:0x%06x successfully.\n", udma_jfr->jfr_id); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_JFR, udma_jfr->jfr_id, + &udma_jfr->cmdq_rtt); + kfree(udma_jfr); + return ret; +} + +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *ub_dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct ubcore_tjetty *tjetty = NULL; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_JFR, + "Invalid parameter, ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "remote jfr token_policy:%u dont support\n", + cfg->flag.bs.token_policy); + return ERR_PTR(-EINVAL); + } + + if (cfg->trans_mode != UBCORE_TP_RM && + cfg->trans_mode != UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "remote jfr trans_mode:%u dont support\n", + cfg->trans_mode); + return ERR_PTR(-EINVAL); + } + + tjetty = (struct ubcore_tjetty *)kcalloc( + 1, sizeof(struct ubcore_tjetty), GFP_KERNEL); + if (tjetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, "alloc tjfr failed\n"); + return ERR_PTR(-ENOMEM); + } + + return tjetty; +} + +int udma_unimport_jfr(struct ubcore_tjetty *tjfr) +{ + if (tjfr == NULL) { + udma_pr_err(UDMA_M_JFR, + "%s: Invalid parameter, tjfr is null.\n", __func__); + return -EINVAL; + } + + kfree(tjfr); + + return 0; +} + +void udma_jfr_async_event(struct udma_device *udma_dev, struct udma_jfr *jfr, + u8 type) +{ + struct ubcore_jfr *ubcore_jfr = &jfr->ubcore_jfr; + struct ubcore_event event = { 0 }; + + if (ubcore_jfr->jfae_handler != NULL) { + event.ub_dev = ubcore_jfr->ub_dev; + event.element.jfr = ubcore_jfr; + switch (type) { + case UB_AEQ_TYPE_JFR_ERR: + event.event_type = UBCORE_EVENT_JFR_ERR; + break; + case UB_AEQ_TYPE_JFR_LIMIT_REACHED: + event.event_type = UBCORE_EVENT_JFR_LIMIT_REACHED; + break; + default: + udma_err_limit(udma_dev->dev, UDMA_M_JFR, + "unknown jfr asyn event:%u.\n", type); + return; + } + + ubcore_jfr->jfae_handler(&event, ubcore_jfr->uctx); + } +} + +int udma_query_res_jfr(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jfr_val *ret_val = + (struct ubcore_res_jfr_val *)(uintptr_t)val->addr; + struct ub_ta_context_query *jfr_ctx = NULL; // jfr reuse jetty context + struct ubcore_jfr *jfr = NULL; + struct udma_jfr *udma_jfr = NULL; + u32 dw_value; + + udma_jfr = udma_find_jfr(udma_dev, key->key); + if (udma_jfr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "ubcore find jfr failed, jfrid:%u\n", key->key); + return -EINVAL; + } + jfr = &udma_jfr->ubcore_jfr; + + jfr_ctx = (struct ub_ta_context_query *)udma_jfr->jfrc_info->vaddr; + + ret_val->depth = jfr->jfr_cfg.depth; + ret_val->jfc_id = jfr->jfr_cfg.jfc->id; + ret_val->jfr_id = jfr->jfr_id.id; + dw_value = be32_to_cpu(jfr_ctx->ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + ret_val->state = (u8)to_ubcore_jfr_state( + (dw_value >> UDMA_STATE_IN_DW_OFFSET) & UDMA_STATE_IN_DW_MASK); + + udma_jfr_put(udma_jfr); + udma_info(udma_dev->dev, UDMA_M_JFR, + "query jfr successfully, jfrid:%u\n", key->key); + + return 0; +} + +int udma_modify_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfr *udma_jfr = NULL; + int ret = 0; + + if (jfr == NULL || jfr->ub_dev == NULL || attr == NULL) { + udma_pr_err(UDMA_M_JFR, + "%s: Invalid parameter, jfr(%d), attr(%d).\n", + __func__, !!jfr, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfr->ub_dev); + udma_jfr = to_udma_jfr(jfr); + + udma_info(udma_dev->dev, UDMA_M_JFR, + "modify jfr, mask:0x%08x, rx_threshold:%u, state:%d\n", + attr->mask, attr->rx_threshold, attr->state); + + if ((attr->mask & (UBCORE_JFR_RX_THRESHOLD | UBCORE_JFR_STATE)) == 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "JFR modify mask is not set or invalid, mask:0x%x\n", + attr->mask); + return -EINVAL; + } + + if (attr->state < UBCORE_JFR_STATE_RESET || + attr->state > UBCORE_JFR_STATE_ERROR) { + udma_err(udma_dev->dev, UDMA_M_JFR, "invalid jfr state:%d\n", + attr->state); + return -EINVAL; + } + + if (udma_jfr->srq.container_size == 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "invalid jfr srq container_size\n"); + return -EINVAL; + } + + if ((attr->mask & UBCORE_JFR_RX_THRESHOLD) != 0) { + if (udma_modify_jfr_threshold(udma_dev, udma_jfr, + (u16)(attr->rx_threshold)) != 0) { + ret = -EFAULT; + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr threshold failed, jfrid:%u, limit_threshold:%u, srq_threshold:%u, dst_threshhold:%u\n", + udma_jfr->jfr_id, udma_jfr->idx_queue_lth, + udma_jfr->srq.container_warn_th, + attr->rx_threshold); + } + } + + if ((attr->mask & UBCORE_JFR_STATE) != 0) { + if (udma_modify_jfr_state(udma_dev, udma_jfr, attr->state, + udata) != 0) { + ret = -EFAULT; + udma_err( + udma_dev->dev, UDMA_M_JFR, + "modify jfr state failed, jfrid:%u, old_state:%u, new_state:%d\n", + udma_jfr->jfr_id, udma_jfr->state, attr->state); + } + } + + return ret; +} + +int udma_post_jfr_wr(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct udma_jfr *ub_jfr = NULL; + struct udma_device *udma_dev = NULL; + struct udma_srq *srq = NULL; + int ret = 0; + + if (jfr == NULL || jfr->ub_dev == NULL || wr == NULL || + bad_wr == NULL) { + udma_pr_err( + UDMA_M_JFR, + "%s: Invalid parameter, jfr(%d), wr(%d), bad_wr(%d).\n", + __func__, !!jfr, !!wr, !!bad_wr); + return -EINVAL; + } + + ub_jfr = to_udma_jfr(jfr); + udma_dev = to_udma_dev(jfr->ub_dev); + srq = &ub_jfr->srq; + + udma_debug(udma_dev->dev, UDMA_M_JFR, "start post jfr, jfr id:%u\n", + ub_jfr->jfr_id); + + if (ub_jfr->state != UBCORE_JFR_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "jfr state: %d, posting not supported\n", + ub_jfr->state); + *bad_wr = wr; + return -EINVAL; + } + + ret = udma_post_srq_wr(udma_dev, srq, wr, bad_wr); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_JFR, + "post jfr recv failed, ret:%d\n", ret); + else + udma_debug(udma_dev->dev, UDMA_M_JFR, + "post jfr recv successfully, jfr id:%u\n", + ub_jfr->jfr_id); + + return ret; +} + +/* 1825v100 of limit threshold in jfr context in, 1823v200 of srq threshold in srq context in */ +u32 udma_get_jfr_threshold(struct udma_device *udma_dev, + ub_cmd_srq_query_rsp_s *srq_entry, + struct ub_ta_context_query *jfr_entry) +{ + u32 warn_th; + + /* 1825v100 of limit threshold in jfr context of dw4 in of high 16 bit */ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + return be32_to_cpu(jfr_entry->ctx[UDMA_JETTY_CTX_DW4]) >> + UDMA_IDX_QUEUE_LTH_IN_DW_OFFSET; + } + + /* 1823v200 of threshold in srq context, 2 of warn_th time power is watermark value */ + srq_entry->dw2.value = be32_to_cpu(srq_entry->dw2.value); + warn_th = srq_entry->dw2.bs_warn_th.warn_th; + udma_info(udma_dev->dev, UDMA_M_JFR, "warn_th:%u\n", + srq_entry->dw2.bs_warn_th.warn_th); + return (warn_th == 0) ? 0 : (1 << warn_th); +} + +int udma_query_jfr(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr) +{ + struct ub_ta_context_query *jfr_entry = NULL; + ub_cmd_srq_query_rsp_s *srq_entry = NULL; + struct udma_device *udma_dev = NULL; + struct udma_jfr *ub_jfr = NULL; + u32 jfrid, dw_value; + int ret; + + if (jfr == NULL || jfr->ub_dev == NULL || attr == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_JFR, + "%s: Invalid parameter, jfr(%d), attr(%d), cfg(%d).\n", + __func__, !!jfr, !!attr, !!cfg); + return -EINVAL; + } + udma_dev = to_udma_dev(jfr->ub_dev); + ub_jfr = to_udma_jfr(jfr); + jfrid = ub_jfr->jfr_id; + + jfr_entry = (struct ub_ta_context_query *)kzalloc( + sizeof(struct ub_ta_context_query), GFP_KERNEL); + if (jfr_entry == NULL) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "failed to allocate memroy for ub jetty context when allocate\n"); + return -ENOMEM; + } + + srq_entry = (ub_cmd_srq_query_rsp_s *)kzalloc( + sizeof(ub_cmd_srq_query_rsp_s), GFP_KERNEL); + if (srq_entry == NULL) { + udma_err( + udma_dev->dev, UDMA_M_JFR, + "failed to allocate memroy for srq context when allocate\n"); + kfree(jfr_entry); + return -ENOMEM; + } + + ret = udma_cmd_jetty_query_ctx(udma_dev, ub_jfr->jfr_id, + &ub_jfr->cmdq_rtt, jfr_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "query jfr from cache failed, jfrid:%u, ret:%d\n", + jfrid, ret); + goto free_mem; + } + + ret = udma_cmd_srq_query_ctx(udma_dev, ub_jfr->srq.srqn, srq_entry); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "query srq from cache failed, jfrid:%u, ret:%d\n", + jfrid, ret); + goto free_mem; + } + + attr->mask = UB_QUERY_JFR_MASK; + + attr->rx_threshold = + udma_get_jfr_threshold(udma_dev, srq_entry, jfr_entry); + + dw_value = be32_to_cpu(jfr_entry->ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + attr->state = to_ubcore_jfr_state( + (dw_value >> UDMA_STATE_IN_DW_OFFSET) & UDMA_STATE_IN_DW_MASK); + *cfg = jfr->jfr_cfg; + + udma_info( + udma_dev->dev, UDMA_M_JFR, + "query jfr:%u: depth:%u, trans_mode:%u, rx_threshold:%u, state:%u\n", + jfrid, cfg->depth, cfg->trans_mode, attr->rx_threshold, + attr->state); + +free_mem: + kfree(jfr_entry); + kfree(srq_entry); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c new file mode 100644 index 000000000..8d8101a99 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_jfs.c @@ -0,0 +1,2107 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <asm/atomic.h> + +#include "ubcore_uapi.h" +#include "ubcore_types.h" + +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_jfc.h" +#include "udma_jetty.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_abi.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfs.h" +#include "udma_base_sq.h" + +#include "ub_aeqe.h" +#include "ub_xqe_format.h" + +static void udma_jfs_kref_release(struct kref *ref_cnt) +{ + struct udma_jfs *jfs = container_of(ref_cnt, struct udma_jfs, ref_cnt); + + complete(&jfs->comp); +} + +void udma_jfs_put(void *obj) +{ + struct udma_jfs *jfs = obj; + + if (jfs != NULL) { + kref_put(&jfs->ref_cnt, udma_jfs_kref_release); + } +} + +void udma_jfs_get(void *obj) +{ + struct udma_jfs *jfs = obj; + + kref_get(&jfs->ref_cnt); +} + +struct udma_jfs *udma_find_jfs(struct udma_device *udev, u32 jfs_id) +{ + struct udma_jfs *jfs = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->jfs_list.node_lock; + mutex_lock(lock); + list_for_each_entry(jfs, &udev->device_res->jfs_list.node, node) { + if (jfs->jfs_id == jfs_id) { + udma_jfs_get(jfs); + mutex_unlock(lock); + return jfs; + } + } + mutex_unlock(lock); + + return NULL; +} + +int udma_query_res_jfs(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct ubcore_res_jfs_val *ret_val = + (struct ubcore_res_jfs_val *)(uintptr_t)val->addr; + struct ubcore_jfs *jfs = NULL; + struct udma_jfs *udma_jfs = NULL; + + udma_jfs = udma_find_jfs(udma_dev, key->key); + if (udma_jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to find JFS, id(%u)\n", key->key); + return -EINVAL; + } + jfs = &udma_jfs->ubcore_jfs; + + ret_val->depth = jfs->jfs_cfg.depth; + ret_val->jfc_id = jfs->jfs_cfg.jfc->id; + ret_val->jfs_id = jfs->jfs_id.id; + ret_val->priority = jfs->jfs_cfg.priority; + ret_val->state = udma_jfs->state; + udma_jfs_put(udma_jfs); + return 0; +} + +static int udma_create_jfs_cmd(struct udma_device *udma_dev, + struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + int ret; + + if (jfs->trans_mode == UBCORE_TP_RM) { + ret = udma_alloc_opt_ubrc(udma_dev, &jfs->rsp_depth, &jfs->ubrc, + false); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc ubrc, ret(%d)\n", ret); + return ret; + } + } + + ret = udma_cmd_jfs_create_ctx(udma_dev, jfs, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to send create, jfs_id(%u), ret(%d)\n", + jfs->jfs_id, ret); + if (jfs->trans_mode == UBCORE_TP_RM) + udma_free_opt_ubrc(udma_dev, &jfs->rsp_depth, + &jfs->ubrc); + } + + return ret; +} + +static int udma_alloc_jfsc(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + struct tag_cqm_qpc_mpt *jfsc_info = NULL; + u32 low_bit_mode = CQM_XID_LOW_BIT_NONE; + u32 xid_low = 0; + u32 qpn_tmp; + + if (udma_inc_jfc_used_cnt(cfg->jfc) == 1) { + low_bit_mode = + (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) ? + CQM_XID_LOW_BIT_1_1_1 : + CQM_XID_LOW_BIT_0_0_1; + xid_low = cfg->jfc->id; + } + qpn_tmp = udma_get_qpn_mod(udma_dev, low_bit_mode, xid_low); + jfsc_info = udma_object_qpc_create( + udma_dev, UBEP_JETTY_CONTEXT_SIZE, jfs, + &udma_dev->jettyn_record, qpn_tmp, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num, + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jetty); + if (jfsc_info == NULL) { + udma_dec_jfc_used_cnt(cfg->jfc); + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to create jfsc by cqm object\n"); + return -ENOMEM; + } + + if (cfg->jfc != NULL) + udma_info(udma_dev->dev, UDMA_M_JFS, + "get jfs id(%u), jfc id(%u), jfc used cnt(%d)\n", + jfsc_info->xid, cfg->jfc->id, + udma_read_jfc_used_cnt(cfg->jfc)); + + jfs->jfsc_info = jfsc_info; + + return 0; +} + +static void udma_object_qpc_free(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + cqm5_object_delete(&(jfs->jfsc_info->object)); + jfs->jfsc_info = NULL; +} + +static void udma_free_jfsc(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfc *jfc) +{ + udma_object_qpc_free(udma_dev, jfs); + udma_dec_jfc_used_cnt(jfc); +} + +static int udma_init_sq_user(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct ub_create_jetty_ucmd *ucmd, + struct udma_context *udma_ctx, + struct udma_jfs *jfs) +{ + int ret; + + jfs->sq_buf_len = ucmd->buf_size; + jfs->sq.baseblk_cnt = ucmd->sq_wqebb_cnt; + + ret = udma_init_user_mtt(udma_dev, &jfs->ub_mtt, &jfs->umem, + ucmd->buf_addr, ucmd->buf_size); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "udma init user mtt failed, ret = %d.\n", ret); + return ret; + } + + ret = udma_db_map_user(udma_ctx, ucmd->sdb_addr, &jfs->sdb); + if (ret != 0) { + udma_uninit_user_mtt(udma_dev, &jfs->ub_mtt, jfs->umem); + jfs->umem = NULL; + udma_err(udma_dev->dev, UDMA_M_JFS, "Map user DB failed.\n"); + } + + return ret; +} + +static u32 udma_get_wqe_buf_size(const struct udma_device *udma_dev, u32 depth, + u32 wr_max_length) +{ + u64 align = PAGE_SIZE; + uint64_t size = 1L << depth_align_power2(wr_max_length * (depth + 1)); + + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + align = (u64)udma_cal_page_size(size); + } + + return (u32)ub_size_align(size, align); +} + +u32 udma_get_kernel_sq_size(const struct udma_device *udma_dev, u32 depth, + u32 *wr_max_length) +{ + u32 max_jfs_inline_size = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + u32 max_jfs_sge_size; + u32 qbuf_size; + + max_jfs_sge_size = udma_dev->ub_cap.dev_cap.max_jfs_sge * + (u32)sizeof(struct ub_cmd_sge); + *wr_max_length = sizeof(struct tag_ub_sq_wqe_s); + *wr_max_length += max_jfs_sge_size > max_jfs_inline_size ? + max_jfs_sge_size : + max_jfs_inline_size; + *wr_max_length = 1L << depth_align_power2((u32)ub_size_align( + *wr_max_length, UB_JFS_WQEBB)); + qbuf_size = udma_get_wqe_buf_size(udma_dev, depth, *wr_max_length); + return qbuf_size; +} + +static void udma_create_jfs_kernel_set_attr(struct udma_jfs *jfs, + cqm_queue_s *jfs_buf_info, + u8 mtt_pro_en) +{ + jfs->jfs_buf_info = jfs_buf_info; + jfs->sdb.db_record = (__be32 *)(void *)(&jfs_buf_info->q_header_vaddr + ->doorbell_record); + jfs->sdb.dma = jfs_buf_info->q_header_paddr; + + jfs_buf_info->q_header_vaddr->doorbell_record = 0; + jfs->page_shift = (u32)UB_ILOG2(jfs_buf_info->q_room_buf_1.buf_size); +} + +static int udma_jfs_init_sq(struct udma_device *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + u32 dev_max_inline_size = udma_dev->ub_cap.dev_cap.max_jfs_inline_size; + struct udma_send_queue *sq = &jfs->sq; + u32 wr_max_length; + + spin_lock_init(&sq->lock); + sq->baseblk_shift = depth_align_power2(UB_JFS_WQEBB); + sq->buf_size = + udma_get_kernel_sq_size(udma_dev, cfg->depth, &wr_max_length); + sq->baseblk_cnt = sq->buf_size >> sq->baseblk_shift; + sq->max_wr_num = + (sq->baseblk_cnt / (wr_max_length >> sq->baseblk_shift)) - 1; + sq->buf = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + jfs->queue_addr.va : + jfs->jfs_buf_info->q_room_buf_1.direct.va; + sq->buf_curr = sq->buf; + sq->max_inline_size = (cfg->max_inline_data > dev_max_inline_size) ? + dev_max_inline_size : + cfg->max_inline_data; + + (void)memset_s(sq->buf, sq->buf_size, 0xff, sq->buf_size); + + sq->wrid_size = sq->baseblk_cnt / UB_MIN_WR_WQEBB_NUM; + sq->wrid = (u64 *)kzalloc(sq->wrid_size * sizeof(u64), GFP_KERNEL); + if (sq->wrid == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to create sq wrid.\n"); + return -ENOMEM; + } + + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->baseblk_shift:%u.\n", + sq->baseblk_shift); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->buf_size:%u.\n", + sq->buf_size); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->baseblk_cnt:%u.\n", + sq->baseblk_cnt); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->max_wr_num:%u.\n", + sq->max_wr_num); + udma_info(udma_dev->dev, UDMA_M_JFS, "sq->max_inline_size:%u.\n", + sq->max_inline_size); + + return 0; +} + +static int udma_flush_jfs_one_normal(struct udma_device *udma_dev, + struct udma_jfs *jfs, struct ubcore_cr *cr) +{ + struct udma_send_queue *sq = &jfs->sq; + + if (sq->flush_tail == sq->head) + return -EAGAIN; + + cr->user_ctx = sq->wrid[sq->flush_tail & (sq->wrid_size - 1)]; + ++sq->flush_tail; + + cr->status = UBCORE_CR_WR_UNHANDLED; + cr->flag.bs.s_r = 0; // send + cr->flag.bs.jetty = 0; // is jfs + cr->completion_len = 0; + cr->local_id = jfs->jfs_id; + cr->user_data = (uintptr_t)(&jfs->ubcore_jfs); + + udma_info( + udma_dev->dev, UDMA_M_JFS, + "flush jfs one success, jfs_id(%u), tail(%u), flush_tail(%u), head(%u)\n", + jfs->jfs_id, sq->tail, sq->flush_tail, sq->head); + + return 0; +} + +static int udma_flush_jfs_one(struct udma_device *udma_dev, + struct udma_jfs *jfs, struct ubcore_cr *cr) +{ + return udma_flush_jfs_one_normal(udma_dev, jfs, cr); +} + +int udma_flush_jfs(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *udma_jfs = NULL; + int nflushed = 0; + int ret; + + if ((jfs == NULL) || (jfs->ub_dev == NULL) || (cr == NULL)) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: jfs(%d),cr(%d).\n", + !!jfs, !!cr); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfs->ub_dev); + udma_jfs = to_udma_jfs(jfs); + if (udma_jfs->sq.is_hw_flush_done == false) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "hardware is not flush done, jfs_id(%u)\n", + udma_jfs->jfs_id); + return -EINVAL; + } + + /* forge not execute of WQE of CQE */ + for (nflushed = 0; nflushed < cr_cnt; ++nflushed) { + ret = udma_flush_jfs_one(udma_dev, udma_jfs, &cr[nflushed]); + if (ret != 0) + break; + } + + if (udma_jfs->sq.flush_tail == udma_jfs->sq.head) { + udma_jfs->sq.head = udma_jfs->sq.tail; + udma_jfs->sq.is_hw_flush_done = false; + } + + udma_info(udma_dev->dev, UDMA_M_JFS, + "flush jfs success, jfs_id(%u), nflushed(%d)\n", + udma_jfs->jfs_id, nflushed); + + return nflushed; +} + +static void udma_deinit_sq_user(struct udma_device *udma_dev, + struct udma_context *udma_ctx, + struct udma_jfs *jfs) +{ + udma_db_unmap_user(udma_ctx, &jfs->sdb); + udma_uninit_user_mtt(udma_dev, &jfs->ub_mtt, jfs->umem); + jfs->umem = NULL; +} + +static void udma_deinit_sq_kernel(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + if (jfs->sq.wrid != NULL) { + kfree(jfs->sq.wrid); + jfs->sq.wrid = NULL; + } + udma_uninit_kernel_mtt(udma_dev, &jfs->ub_mtt); + udma_free_queue_resource(udma_dev, jfs->jfs_buf_info, &jfs->queue_addr); + jfs->jfs_buf_info = NULL; +} + +static void udma_deinit_sq(struct udma_device *udma_dev, + struct udma_context *udma_ctx, struct udma_jfs *jfs) +{ + if (udma_ctx == NULL) + udma_deinit_sq_kernel(udma_dev, jfs); + else + udma_deinit_sq_user(udma_dev, udma_ctx, jfs); +} + +static int udma_init_sq_kernel(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct udma_jfs *jfs) +{ + cqm_queue_s *jfs_buf_info = NULL; + u32 wr_max_length; + int ret; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "to_udma_dev failed.\n"); + return -EINVAL; + } + + jfs->sq_buf_len = + udma_get_kernel_sq_size(udma_dev, cfg->depth, &wr_max_length); + ret = udma_alloc_queue_resource(udma_dev, &jfs_buf_info, jfs, + jfs->sq_buf_len, &jfs->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to alloc queue resource, ret = %d.\n", ret); + return ret; + } + + udma_create_jfs_kernel_set_attr(jfs, jfs_buf_info, + udma_dev->ub_cap.dev_cap.mtt_pro_en); + + ret = udma_init_kernel_mtt(udma_dev, &jfs->ub_mtt, jfs_buf_info, + jfs->queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_jfs_init_sq(udma_dev, jfs, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Failed to init jfs sq.\n"); + goto err_jfs_init_sq; + } + + return 0; + +err_jfs_init_sq: + udma_uninit_kernel_mtt(udma_dev, &jfs->ub_mtt); +err_init_kernel_mtt: + udma_free_queue_resource(udma_dev, jfs_buf_info, &jfs->queue_addr); + jfs->jfs_buf_info = NULL; + return ret; +} + +static int udma_init_sq(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct ub_create_jetty_ucmd *ucmd, + struct udma_context *udma_ctx, struct udma_jfs *jfs) +{ + if (udma_ctx == NULL) + return udma_init_sq_kernel(udma_dev, cfg, jfs); + else + return udma_init_sq_user(udma_dev, cfg, ucmd, udma_ctx, jfs); +} + +static int udma_init_jfs(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, struct udma_jfs *jfs, + struct udma_create_jfs_uinfo *uinfo) +{ + jfs->cqm_priv_type = UDMA_JETTY_TYPE_JFS; + jfs->ubcore_jfs.jfs_cfg = *cfg; + jfs->trans_mode = cfg->trans_mode; + + if (uinfo == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: uinfo(%d).\n", + !!uinfo); + return -EINVAL; + } + + if (udma_init_sq(udma_dev, cfg, uinfo->ucmd, uinfo->udma_ctx, jfs) != + 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "init sq user failed,\n"); + return -EINVAL; + } + + if (udma_alloc_jfsc(udma_dev, jfs, cfg) != 0) + goto err_and_deinit_sq; + + jfs->jfs_id = jfs->jfsc_info->xid; + jfs->ubcore_jfs.jfs_id.id = jfs->jfs_id; + + if (udma_create_jfs_cmd(udma_dev, jfs, cfg) != 0) + goto err_and_delete_obj; + + return 0; + +err_and_delete_obj: + udma_free_jfsc(udma_dev, jfs, cfg->jfc); +err_and_deinit_sq: + udma_deinit_sq(udma_dev, uinfo->udma_ctx, jfs); + + return -EPERM; +} + +static struct udma_jfs *udma_create_jfs_one(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct udma_create_jfs_uinfo *uinfo, + u8 cos) +{ + struct udma_jfs *jfs = NULL; + struct udma_context *udma_ctx = uinfo->udma_ctx; + + udma_info(udma_dev->dev, UDMA_M_JFS, + "start to create jfs, from user: %d\n", + (udma_ctx == NULL) ? 0 : 1); + + jfs = (struct udma_jfs *)kzalloc(sizeof(struct udma_jfs), GFP_KERNEL); + if (jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Alloc JFS failed.\n"); + return NULL; + } + + jfs->cos = cos; + udma_cmdq_rtt_statistic_init(&jfs->cmdq_rtt); + if (udma_init_jfs(udma_dev, cfg, jfs, uinfo) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "init jfs failed.\n"); + goto err_and_free_jfs; + } + + jfs->state = UBCORE_JETTY_STATE_READY; + + kref_init(&jfs->ref_cnt); + init_completion(&jfs->comp); + + udma_store_list(udma_dev, &jfs->node, &udma_dev->device_res->jfs_list); + + udma_info(udma_dev->dev, UDMA_M_JFS, + "Create JFS successfully, id: %u.\n", + jfs->ubcore_jfs.jfs_id.id); + + (void)udma_clear_pi_on_chip(udma_dev, jfs->jfs_id, jfs->cos); + + return jfs; + +err_and_free_jfs: + kfree(jfs); + + return NULL; +} + +static int udma_jfs_cache_out_cmd(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Failed to alloc wb_va\n"); + return -ENOMEM; + } + + ret = udma_cmd_jfs_cache_out_ctx(udma_dev, jfs, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static int udma_destroy_jfs_one(struct udma_device *udma_dev, + struct udma_jfs *jfs) +{ + struct udma_context *udma_ctx = to_udma_ctx(jfs->ubcore_jfs.uctx); + int ret; + + udma_info(udma_dev->dev, UDMA_M_JFS, "start to destroy jfs: %u\n", + jfs->jfs_id); + + ret = udma_cmd_jfs_delete_ctx(udma_dev, jfs); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to delete JFS(0x%06x).\n", jfs->jfs_id); + return ret; + } + + ret = udma_jfs_cache_out_cmd(udma_dev, jfs); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Cache out JFS:0x%06x failed.\n", jfs->jfs_id); + return ret; + } + udma_jfs_put(jfs); + wait_for_completion(&jfs->comp); + udma_deinit_sq(udma_dev, udma_ctx, jfs); + udma_free_opt_ubrc(udma_dev, &jfs->rsp_depth, &jfs->ubrc); + udma_free_jfsc(udma_dev, jfs, jfs->ubcore_jfs.jfs_cfg.jfc); + jfs->state = UBCORE_JETTY_STATE_RESET; + + udma_delete_list(udma_dev, &jfs->node, &udma_dev->device_res->jfs_list); + + udma_info(udma_dev->dev, UDMA_M_JFS, + "Destroyed JFS successfully, id: %u.\n", jfs->jfs_id); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_JFS, jfs->jfs_id, + &jfs->cmdq_rtt); + kfree(jfs); + + return ret; +} + +static int udma_jfs_cfg_check(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg) +{ + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "tpf not support this function.\n"); + return -EINVAL; + } + + if ((cfg->trans_mode != UBCORE_TP_RM) && + (cfg->trans_mode != UBCORE_TP_UM)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: jfs tp_mode: %d.\n", + cfg->trans_mode); + return -EINVAL; + } + + if (udma_dev->jfs_current_cnt >= udma_dev->ub_cap.dev_cap.max_jfs) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: jfs current cnt(0x%x) is over max jfs cnt(0x%x).\n", + udma_dev->jfs_current_cnt, + udma_dev->ub_cap.dev_cap.max_jfs); + return -EINVAL; + } + + if ((cfg->depth == 0) || + (cfg->depth > udma_dev->ub_cap.dev_cap.max_jfs_depth) || + (cfg->max_sge > udma_dev->ub_cap.dev_cap.max_jfs_sge) || + cfg->max_inline_data > + udma_dev->ub_cap.dev_cap.max_jfs_inline_size) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: jfs depth: %u max_depth:%u sge: %u max_sge: %u inline_size: %u max_inline_size:%u\n", + cfg->depth, udma_dev->ub_cap.dev_cap.max_jfs_depth, + cfg->max_sge, udma_dev->ub_cap.dev_cap.max_jfs_sge, + cfg->max_inline_data, + udma_dev->ub_cap.dev_cap.max_jfs_inline_size); + return -EINVAL; + } + + if (cfg->flag.bs.outorder_comp != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "outorder_comp: %u\n", + cfg->flag.bs.outorder_comp); + return -EINVAL; + } + + return 0; +} + +static int udma_jfs_get_cos(struct udma_device *udma_dev, + const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata, u8 *cos) +{ + int ret; + + ret = udma_get_dcb_cfg_cos(udma_dev, NULL, cfg->priority, cos, 1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to get cos from dcb info, ret:%d, pri:%u\n", + ret, cfg->priority); + return ret; + } + udma_info(udma_dev->dev, UDMA_M_JFS, + "Get cos from dcb info successful, cos:%u\n", *cos); + + return 0; +} + +int udma_create_jfs_uinfo_init(struct udma_device *udma_dev, + struct ubcore_udata *udata, + struct udma_create_jfs_uinfo *uinfo) +{ + if ((udata == NULL) || (udata->uctx == NULL) || + (udata->udrv_data == NULL)) + return 0; + + uinfo->cmd_len = sizeof(struct ub_create_jetty_ucmd); + uinfo->resp_len = sizeof(struct ub_create_jetty_resp); + + if (udata->udrv_data->in_len < uinfo->cmd_len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "in_len(%u) < cmd_len(%u)\n", udata->udrv_data->in_len, + uinfo->cmd_len); + return -EINVAL; + } + + if (udata->udrv_data->out_len < uinfo->resp_len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "out_len(%u) < resp_len(%u)\n", + udata->udrv_data->out_len, uinfo->resp_len); + return -EINVAL; + } + + uinfo->udata = udata; + uinfo->udma_ctx = to_udma_ctx(udma_get_uctx(udata)); + if ((uinfo->ucmd = kzalloc(uinfo->cmd_len, GFP_KERNEL)) == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "zalloc cmd buf failed: %u\n", uinfo->cmd_len); + return -ENOMEM; + } + + if ((uinfo->resp = kzalloc(uinfo->resp_len, GFP_KERNEL)) == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "zalloc resp buf failed: %u\n", uinfo->resp_len); + goto err_alloc_resp; + } + + if (copy_from_user(uinfo->ucmd, + (void *)(uintptr_t)udata->udrv_data->in_addr, + uinfo->cmd_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "copy from user failed: %u\n", uinfo->cmd_len); + goto err_and_free_all; + } + + return 0; + +err_and_free_all: + kfree(uinfo->resp); + uinfo->resp = NULL; +err_alloc_resp: + kfree(uinfo->ucmd); + uinfo->ucmd = NULL; + + return -EINVAL; +} + +int udma_create_jfs_uinfo_deinit(struct udma_device *udma_dev, + struct udma_create_jfs_uinfo *uinfo, u8 cos, + u32 round, u8 copy) +{ + if (uinfo->udma_ctx == NULL) + return 0; + + if (copy != 0) { + uinfo->resp->cos = cos; + uinfo->resp->slice = udma_get_jetty_slice_en(udma_dev) ? + udma_dev->slice : + UDMA_MAX_SLICE_SHIFT; + uinfo->resp->pattern = udma_dev->ub_dev.cfg.pattern; + uinfo->resp->jetty_round = round; + + if (copy_to_user((void *)(uintptr_t) + uinfo->udata->udrv_data->out_addr, + uinfo->resp, uinfo->resp_len) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to copy to user driver\n"); + return -EINVAL; + } + } + + if (uinfo->ucmd) { + kfree(uinfo->ucmd); + uinfo->ucmd = NULL; + } + + if (uinfo->resp) { + kfree(uinfo->resp); + uinfo->resp = NULL; + } + + return 0; +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *ub_dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_create_jfs_uinfo uinfo = { 0 }; + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + int ret; + u8 cos; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_JFS, "ub_dev(%d), cfg(%d) is NULL", !!ub_dev, + !!cfg); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + udma_info(udma_dev->dev, UDMA_M_JFS, + "start create jfs, depth: %u, eid_index: %u\n", cfg->depth, + cfg->eid_index); + + if (udma_jfs_cfg_check(udma_dev, cfg) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "check cfg failed\n"); + return ERR_PTR(-EINVAL); + } + + ret = udma_create_jfs_uinfo_init(udma_dev, udata, &uinfo); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "init uinfo failed\n"); + return ERR_PTR(ret); + } + + if (udma_jfs_get_cos(udma_dev, cfg, udata, &cos) != 0) + goto err_and_deinit_uinfo; + + jfs = udma_create_jfs_one(udma_dev, cfg, &uinfo, cos); + if (jfs == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "create master jfs failed\n"); + goto err_and_deinit_uinfo; + } + + if (udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, jfs->round, + 1) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Failed to copy to user driver\n"); + goto err_and_free_jfs; + } + + return &jfs->ubcore_jfs; + +err_and_free_jfs: + jfs->ubcore_jfs.uctx = udma_get_uctx(udata); + udma_destroy_jfs_one(udma_dev, jfs); +err_and_deinit_uinfo: + udma_create_jfs_uinfo_deinit(udma_dev, &uinfo, cos, 0, 0); + + return ERR_PTR(-EFAULT); +} + +int udma_query_jfs(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *ub_jfs = NULL; + struct ub_ta_context_query jfs_ctx = { 0 }; + int ret; + u32 dw_value; + + if (jfs == NULL || jfs->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "jfs(%d), or jfs->ub_dev is NULL\n", + !!jfs); + return -EINVAL; + } + + ub_jfs = to_udma_jfs(jfs); + udma_dev = to_udma_dev(jfs->ub_dev); + if (cfg == NULL || attr == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "cfg(%d), attr(%d) is NULL\n", !!cfg, !!attr); + return -EINVAL; + } + + ret = udma_cmd_jetty_query_ctx(udma_dev, jfs->jfs_id.id, + &ub_jfs->cmdq_rtt, &jfs_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "query jfs ctx cmdq failed\n"); + return -EIO; + } + + dw_value = cpu_to_be32(jfs_ctx.ctx[UDMA_JETTY_CTX_STATE_OFFSET]); + attr->state = (dw_value >> UDMA_STATE_IN_DW_OFFSET) & + UDMA_STATE_IN_DW_MASK; + *cfg = jfs->jfs_cfg; + + udma_info(udma_dev->dev, UDMA_M_JFS, + "JFS state: %d, depth: %u, trans_mode: %d\n", attr->state, + cfg->depth, cfg->trans_mode); + return 0; +} + +int udma_destroy_jfs(struct ubcore_jfs *jfs) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *udma_jfs = NULL; + + if (jfs == NULL || jfs->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "jfs(%d), or jfs->ub_dev is NULL\n", + !!jfs); + return -EINVAL; + } + + udma_dev = to_udma_dev(jfs->ub_dev); + udma_jfs = to_udma_jfs(jfs); + + udma_info(udma_dev->dev, UDMA_M_JFS, "destroy jfs, id(%u), depth(%u)\n", + jfs->jfs_id.id, jfs->jfs_cfg.depth); + + return udma_destroy_jfs_one(udma_dev, udma_jfs); +} + +void udma_jfs_async_event(struct udma_device *udma_dev, struct udma_jfs *jfs, + u8 type) +{ + struct ubcore_jfs *ubcore_jfs = &jfs->ubcore_jfs; + struct ubcore_event event = { 0 }; + + if (ubcore_jfs->jfae_handler != NULL) { + event.ub_dev = ubcore_jfs->ub_dev; + event.element.jfs = ubcore_jfs; + switch (type) { + case UB_AEQ_TYPE_JFS_ERR: + event.event_type = UBCORE_EVENT_JFS_ERR; + break; + + default: + udma_err_limit(udma_dev->dev, UDMA_M_JFS, + "unknown jfs asyn event(%u).\n", type); + return; + } + + ubcore_jfs->jfae_handler(&event, ubcore_jfs->uctx); + } +} + +static void udma_fill_sge(struct ub_cmd_sge *sge, struct ubcore_sge *sgl, + u32 *len) +{ + sge->dw3.dw3_value = 0; + sge->length = cpu_to_be32(sgl->len); + *len += sgl->len; + sge->va = cpu_to_be64(sgl->addr); + sge->dw3.bs.token_id = sgl->tseg->seg.token_id; + sge->dw3.dw3_value = cpu_to_be32(sge->dw3.dw3_value); +} + +ub_sqwqe_optype_e ub_jfs_parse_opcode(enum ubcore_opcode opcode) +{ + switch (opcode) { + case UBCORE_OPC_READ: + return UB_TX_READ; + case UBCORE_OPC_WRITE: + return UB_TX_WRITE; + case UBCORE_OPC_WRITE_IMM: + return UB_TX_WRITE_IMMEDIATE; + case UBCORE_OPC_SEND: + return UB_TX_SEND; + case UBCORE_OPC_SEND_IMM: + return UB_TX_SEND_IMMEDIATE; + case UBCORE_OPC_SEND_INVALIDATE: + return UB_TX_SEND_INVALIDATE; + case UBCORE_OPC_CAS: + return UB_TX_ATOMIC_COMPARE_SWAP; + case UBCORE_OPC_FADD: + return UB_TX_ATOMIC_FETCH_ADD; + default: + return UB_ERR_TYPE; + } +} + +static u32 udma_get_inline_sg_len(struct udma_device *udma_dev, + const struct ubcore_sg *sg) +{ + struct ubcore_sge *tmp_sge = NULL; + u32 total_len = 0; + u32 i; + + tmp_sge = sg->sge; + for (i = 0; i < sg->num_sge; i++) { + if (tmp_sge == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge Is NULL\n"); + return 0; + } + + if (total_len > UINT_MAX - tmp_sge->len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. sge len is error!\n"); + return 0; + } + + total_len += tmp_sge->len; + tmp_sge++; + } + + return total_len; +} + +static u32 udma_wqebb_cnt(struct udma_device *udma_dev, + struct udma_send_queue *sq, + enum UB_SQWQE_OPTYPE_E opcode, + const struct ubcore_jfs_wr *wr) +{ + u32 wqe_size = (u32)(sizeof(struct tag_ub_wqe_ctrl_sec) + + sizeof(struct tag_ub_sq_wqe_com_task)); + + switch (opcode) { + case UB_TX_READ: + wqe_size += (u32)(wr->rw.dst.num_sge << UB_SGE_SHIFT); + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_read) - + UB_SEID_BYTE_LENGTH - UB_INLINE_BYTE_LENGTH; + break; + case UB_TX_WRITE: + if (wr->flag.bs.inline_flag == 0) { + wqe_size += (u32)(wr->rw.src.num_sge << UB_SGE_SHIFT); + } + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_read) - + UB_SEID_BYTE_LENGTH - UB_INLINE_BYTE_LENGTH; + if (wr->flag.bs.inline_flag != 0) { + wqe_size += + udma_get_inline_sg_len(udma_dev, &wr->rw.src); + } + break; + case UB_TX_WRITE_IMMEDIATE: + if (wr->flag.bs.inline_flag == 0) { + wqe_size += (u32)(wr->rw.src.num_sge << UB_SGE_SHIFT); + } + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_write) - + UB_SEID_BYTE_LENGTH; + if (wr->flag.bs.inline_flag != 0) { + wqe_size += + udma_get_inline_sg_len(udma_dev, &wr->rw.src); + } + break; + case UB_TX_SEND: + case UB_TX_SEND_IMMEDIATE: + case UB_TX_SEND_INVALIDATE: + if (wr->flag.bs.inline_flag == 0) { + wqe_size += (u32)(wr->send.src.num_sge << UB_SGE_SHIFT); + } + wqe_size += (u32)sizeof(struct tag_ub_sq_wqe_task_send) - + UB_SEID_BYTE_LENGTH; + if (wr->flag.bs.inline_flag != 0) { + wqe_size += + udma_get_inline_sg_len(udma_dev, &wr->rw.src); + } + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFS, + "Unsupported ubcore opcode:%u\n", (u32)wr->opcode); + break; + } + + wqe_size = (u32)ub_size_align(wqe_size, 1ULL << sq->baseblk_shift); + sq->wqe_size = wqe_size; + + return wqe_size >> sq->baseblk_shift; +} + +static void udma_fill_ctrl_and_task_com(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_post_wqe *post_wqe, + enum UB_SQWQE_OPTYPE_E opcode, + u8 pattern) +{ + wqe->ctrl_sec.dw0_value = 0; + wqe->ctrl_sec.dw1_value = 0; + wqe->ctrl_sec.cr = wr->flag.bs.complete_enable; + wqe->ctrl_sec.va = UB_WQE_VA_TYPE_VIRTUAL; + wqe->ctrl_sec.ctrlsl = UB_WQE_CTRL_SEC_LENGTH; + /* fill wqe ctrl head section:cl bit */ + wqe->ctrl_sec.dw1_value = UB_WQE_CMP_TASK_LEN1 + << UB_WQE_CMP_TASK_LEN_SHIFT; + // fast path indicate + wqe->ctrl_sec.fast_dwqe_en = post_wqe->is_fast_path & wqe->ctrl_sec.cr; + + wqe->tsk_sec.com.dw0_value = 0; + wqe->tsk_sec.com.dw2_value = 0; + wqe->tsk_sec.com.dw3_value = 0; + wqe->tsk_sec.com.dw4_value = 0; + wqe->tsk_sec.com.data_len = 0; + wqe->tsk_sec.com.f = wr->flag.bs.fence; + wqe->tsk_sec.com.op_type = (u8)opcode; + wqe->tsk_sec.com.se = wr->flag.bs.solicited_enable; + wqe->tsk_sec.com.c = wqe->ctrl_sec.cr; + wqe->tsk_sec.com.vtpn = post_wqe->tpn; + wqe->tsk_sec.com.odr = wr->flag.value & UBCORE_ODR_BITMASK; + wqe->tsk_sec.com.upi = pattern; + if (wr->tjetty->cfg.flag.bs.token_policy == UBCORE_TOKEN_NONE) { + wqe->tsk_sec.com.token_vld = 0; + } else if (wr->tjetty->cfg.flag.bs.token_policy == + UBCORE_TOKEN_PLAIN_TEXT) { + wqe->tsk_sec.com.token_vld = 1; + } else { + udma_pr_err(UDMA_M_JFS, + "Bug : Invalid token_policy=%u, please check!\n", + wr->tjetty->cfg.flag.bs.token_policy); + } +} + +static void udma_fill_inline_wqe(struct udma_send_queue *sq, + struct tag_ub_sq_wqe_s *wqe, u8 *sge_addr, + struct ubcore_sge *sgl, + const struct udma_sge_info *sge_info) +{ + u32 curr_ofst = 0; + u32 total_len = 0; + u32 sq_rmd_size; + u32 rmd_size; + u32 i; + + for (i = 0; i < sge_info->sge_num; i++) { + sq_rmd_size = + (u32)((sq->buf + sq->buf_size) - (void *)sge_addr) - + curr_ofst; + if (sq_rmd_size < sgl[i].len) { + rmd_size = sgl[i].len - sq_rmd_size; + (void)memcpy_s((void *)(sge_addr + curr_ofst), + sq_rmd_size, + (void *)(uintptr_t)sgl[i].addr, + sq_rmd_size); + sge_addr = sq->buf; + curr_ofst = 0; + (void)memcpy_s( + (void *)(sge_addr + curr_ofst), rmd_size, + (void *)(uintptr_t)(sgl[i].addr + sq_rmd_size), + rmd_size); + total_len += sgl[i].len; + curr_ofst += rmd_size; + } else { + (void)memcpy_s((void *)(sge_addr + curr_ofst), + sgl[i].len, + (void *)(uintptr_t)sgl[i].addr, + sgl[i].len); + total_len += sgl[i].len; + curr_ofst += sgl[i].len; + } + } + + wqe->ctrl_sec.df = 0x1; + wqe->ctrl_sec.bdsl = (u32)align_size(total_len, UB_TASK_SEG_ALIGN) / + UB_TASK_SEG_ALIGN; + wqe->tsk_sec.com.data_len = cpu_to_be32(total_len); + if (wqe->tsk_sec.com.op_type == UB_TX_WRITE) { + wqe->tsk_sec.wr_rd.length = cpu_to_be32(total_len); + } else if (wqe->tsk_sec.com.op_type == UB_TX_WRITE_IMMEDIATE) { + wqe->tsk_sec.write_immt.length = cpu_to_be32(total_len); + /* total_data_len is in second wqebb, need to handle wrap-around logic */ + wqe = udma_get_next_wqe(sq, wqe); + wqe->tsk_sec.write_immt.total_data_len = cpu_to_be32(total_len); + } +} + +static void udma_fill_token_vld(ub_sq_wqe_s *wqe, const struct ubcore_sge *sge) +{ + if (sge->tseg->seg.attr.bs.token_policy == UBCORE_TOKEN_NONE) { + wqe->tsk_sec.com.token_vld = 0; + } else if (sge->tseg->seg.attr.bs.token_policy == + UBCORE_TOKEN_PLAIN_TEXT) { + wqe->tsk_sec.com.token_vld = 1; + } else { + udma_pr_err(UDMA_M_JFS, + "Bug : Invalid token_policy=%u, please check!\n", + sge->tseg->seg.attr.bs.token_policy); + } +} + +static void udma_fill_rw_task_sec(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, u8 **sge_addr, + ub_sqwqe_optype_e opcode) +{ + struct ubcore_sge *sge = NULL; + struct ub_rseg *rseg = NULL; + + sge = (opcode == UB_TX_READ) ? wr->rw.src.sge : wr->rw.dst.sge; + + rseg = container_of(sge->tseg, ub_rseg_t, base); + wqe->ctrl_sec.tsl = UB_JFS_WRITE_WQE_TSK_LEN / UB_TASK_SEG_ALIGN; + wqe->ctrl_sec.bdsl = + wr->rw.src.num_sge * (sizeof(ub_cmd_sge_t) / UB_TASK_SEG_ALIGN); + wqe->tsk_sec.wr_rd.upi = 0; + wqe->tsk_sec.wr_rd.remote_token_value = cpu_to_be32(rseg->token); + wqe->tsk_sec.wr_rd.remote_va_l32 = cpu_to_be32(sge->addr & 0xFFFFFFFF); + wqe->tsk_sec.wr_rd.remote_va_h32 = + cpu_to_be32(sge->addr >> ADDR_H_OFFSET); + wqe->tsk_sec.wr_rd.remote_token_id = + cpu_to_be32(sge->tseg->seg.token_id); + udma_fill_token_vld(wqe, sge); + + /* write flow in wqe fill to deid before exactly fill complete a wqebb, need re new determine next a wqebb of place */ + wqe = udma_get_next_wqe(sq, wqe); + memcpy_s(wqe->tsk_sec.wr_rd.deid, sizeof(wqe->tsk_sec.wr_rd.deid), + wr->tjetty->cfg.id.eid.raw, + sizeof(wr->tjetty->cfg.id.eid.raw)); + *sge_addr = (u8 *)wqe + UB_WR_RD_TKV_DEID_LENGTH; +} + +static void udma_fill_rw_imme_task_sec(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, + u8 **sge_addr) +{ + struct ub_rseg *rseg = + container_of(wr->rw.dst.sge->tseg, ub_rseg_t, base); + + wqe->ctrl_sec.tsl = UB_JFS_WRITE_IMM_WQE_TSK_LEN / UB_TASK_SEG_ALIGN; + wqe->ctrl_sec.bdsl = + wr->rw.src.num_sge * (sizeof(ub_cmd_sge_t) / UB_TASK_SEG_ALIGN); + + wqe->tsk_sec.write_immt.remote_va_l32 = + cpu_to_be32(wr->rw.dst.sge->addr & 0xFFFFFFFF); + wqe->tsk_sec.write_immt.remote_va_h32 = + cpu_to_be32(wr->rw.dst.sge->addr >> ADDR_H_OFFSET); + wqe->tsk_sec.write_immt.remote_token_id = + cpu_to_be32(wr->rw.dst.sge->tseg->seg.token_id); + + wqe->tsk_sec.write_immt.imme_data_l = + cpu_to_be32(wr->rw.notify_data & 0xFFFFFFFF); + wqe->tsk_sec.write_immt.imme_data_h = + cpu_to_be32(wr->rw.notify_data >> ADDR_H_OFFSET); + udma_fill_token_vld(wqe, wr->rw.dst.sge); + + // before plane of is a complete of WQEBB, next plane th second WQEBB may need wrap around + wqe = udma_get_next_wqe(sq, wqe); + wqe->tsk_sec.write_immt.total_data_len = 0; + wqe->tsk_sec.write_immt.upi = 0; + wqe->tsk_sec.write_immt.dw3_value = 0; + wqe->tsk_sec.write_immt.djt = wr->tjetty->cfg.type; + wqe->tsk_sec.write_immt.djtn = wr->tjetty->cfg.id.id; + wqe->tsk_sec.write_immt.dw3_value = + cpu_to_be32(wqe->tsk_sec.write_immt.dw3_value); + wqe->tsk_sec.write_immt.remote_seg_token_value = + cpu_to_be32(rseg->token); + wqe->tsk_sec.write_immt.remote_jfr_token_value = + cpu_to_be32(wr->tjetty->cfg.token_value.token); + + memcpy_s(wqe->tsk_sec.write_immt.deid, + sizeof(wqe->tsk_sec.write_immt.deid), + wr->tjetty->cfg.id.eid.raw, + sizeof(wr->tjetty->cfg.id.eid.raw)); + + *sge_addr = (u8 *)wqe + UB_WRITE_IMME_TKV_DEID_LENGTH; +} + +static void udma_fill_send_task_sec(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, u8 **sge_addr) +{ + if (wqe->tsk_sec.com.op_type == UB_TX_SEND_INVALIDATE) { + wqe->tsk_sec.send.imme_data_l = 0; + wqe->tsk_sec.send.invalid_token_id = + cpu_to_be32(wr->send.tseg->seg.token_id); + } else { + wqe->tsk_sec.send.imme_data_l = + cpu_to_be32(wr->send.imm_data & 0xFFFFFFFF); + wqe->tsk_sec.send.imme_data_h = + cpu_to_be32(wr->send.imm_data >> ADDR_H_OFFSET); + } + + wqe->ctrl_sec.tsl = UB_JFS_SEND_WQE_TSK_LEN / UB_TASK_SEG_ALIGN; + wqe->ctrl_sec.bdsl = wr->send.src.num_sge * + (sizeof(ub_cmd_sge_t) / UB_TASK_SEG_ALIGN); + wqe->tsk_sec.send.upi = 0; + wqe->tsk_sec.send.dw2_value = 0; + wqe->tsk_sec.send.hint = wr->send.target_hint; + wqe->tsk_sec.send.djt = wr->tjetty->cfg.type; + wqe->tsk_sec.send.djtn = wr->tjetty->cfg.id.id; + wqe->tsk_sec.send.dw2_value = cpu_to_be32(wqe->tsk_sec.send.dw2_value); + wqe->tsk_sec.send.remote_token_value = + cpu_to_be32(wr->tjetty->cfg.token_value.token); + if (wr->tjetty->cfg.flag.bs.token_policy == UBCORE_TOKEN_NONE) { + wqe->tsk_sec.com.token_vld = 0; + } else if (wr->tjetty->cfg.flag.bs.token_policy == + UBCORE_TOKEN_PLAIN_TEXT) { + wqe->tsk_sec.com.token_vld = 1; + } else { + udma_pr_err(UDMA_M_JFS, + "Bug : Invalid token_policy=%u, please check!\n", + wr->tjetty->cfg.flag.bs.token_policy); + } + /* send flow in wqe fill to deid before exactly fill complete a wqebb, need re new determine next a wqebb of place */ + wqe = udma_get_next_wqe(sq, wqe); + + memcpy_s(wqe->tsk_sec.send.deid, sizeof(wqe->tsk_sec.send.deid), + wr->tjetty->cfg.id.eid.raw, + sizeof(wr->tjetty->cfg.id.eid.raw)); + *sge_addr = (u8 *)wqe + UB_SEND_TKV_DEID_LENGTH; +} + +static void udma_fill_slice_bit(struct udma_device *udma_dev, + enum ubcore_transport_mode trans_mode, + struct tag_ub_sq_wqe_s *wqe) +{ + if (trans_mode == UBCORE_TP_RM || trans_mode == UBCORE_TP_RC) { + if (cpu_to_be32(wqe->tsk_sec.com.data_len) > + ((1 << udma_dev->slice) * UB_SLICE_LEN)) { + udma_debug(udma_dev->dev, UDMA_M_JFS, + "data_len is %u, over slice len\n", + cpu_to_be32(wqe->tsk_sec.com.data_len)); + wqe->tsk_sec.com.s = 1; + } + } +} + +static void udma_update_wqe_fast_dwqe_en(ub_sqwqe_optype_e opcode, + ub_sq_wqe_s *wqe) +{ + u32 fast_opc = (opcode == UB_TX_READ) || (opcode == UB_TX_WRITE) || + (opcode == UB_TX_WRITE_IMMEDIATE) || + (opcode == UB_TX_SEND); + u32 fast_para = (cpu_to_be32(wqe->tsk_sec.com.data_len) <= + UB_FAST_DATA_LEN_MAX) && + (wqe->tsk_sec.com.token_vld == 0); + wqe->ctrl_sec.fast_dwqe_en &= (fast_opc & fast_para); +} + +static void udma_fill_rw_or_send_sge(u8 *sge_addr, struct udma_send_queue *sq, + struct ubcore_sge *sgl, + const struct udma_sge_info *sge_info, + u32 *len) +{ + struct ub_cmd_sge *sge = NULL; + u32 sq_rmd_size; + u32 i; + + sge = (struct ub_cmd_sge *)sge_addr; + for (i = 0; i < sge_info->sge_num; i++) { + sq_rmd_size = (u32)((sq->buf + sq->buf_size) - (void *)sge); + if (sq_rmd_size > sizeof(struct ub_cmd_sge)) { + udma_fill_sge(sge, sgl, len); + sge++; + } else if (sq_rmd_size == sizeof(struct ub_cmd_sge)) { + udma_fill_sge(sge, sgl, len); + sge = sq->buf; + } else { + sge = sq->buf; + udma_fill_sge(sge, sgl, len); + sge++; + } + sgl++; + } + + if (sge_info->sge_num != 0) { + if (sge == sq->buf) { + sge = (struct ub_cmd_sge *)(sq->buf + sq->buf_size - + sizeof(struct ub_cmd_sge)); + } else { + sge--; + } + sge->dw3.dw3_value = cpu_to_be32(sge->dw3.dw3_value); + sge->dw3.bs.L = 1; + sge->dw3.dw3_value = cpu_to_be32(sge->dw3.dw3_value); + } +} + +static void udma_fill_rw_cmd(struct udma_device *udma_dev, u8 *sge_addr, + struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq) +{ + struct udma_sge_info sge_info = {}; + struct ubcore_sge *sgl = NULL; + u32 len = 0; + + sgl = (wr->opcode == UBCORE_OPC_READ) ? wr->rw.dst.sge : wr->rw.src.sge; + sge_info.sge_num = (wr->opcode == UBCORE_OPC_READ) ? + wr->rw.dst.num_sge : + wr->rw.src.num_sge; + sge_info.max_inline_size = sq->max_inline_size; + + if (wr->flag.bs.inline_flag != 0) { + udma_fill_inline_wqe(sq, wqe, sge_addr, sgl, &sge_info); + return; + } + + udma_fill_rw_or_send_sge(sge_addr, sq, sgl, &sge_info, &len); + + wqe->tsk_sec.com.data_len = cpu_to_be32(len); + wqe->tsk_sec.wr_rd.length = cpu_to_be32(len); + if (wqe->tsk_sec.com.op_type == UB_TX_WRITE_IMMEDIATE) { + /* total_data_len is in second wqebb, need to handle wrap-around logic */ + wqe = udma_get_next_wqe(sq, wqe); + wqe->tsk_sec.write_immt.total_data_len = cpu_to_be32(len); + } +} + +static void udma_fill_send_cmd(struct udma_device *udma_dev, u8 *sge_addr, + struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq) +{ + struct udma_sge_info sge_info = {}; + u32 len = 0; + + sge_info.sge_num = wr->send.src.num_sge; + sge_info.max_inline_size = sq->max_inline_size; + + if (wr->flag.bs.inline_flag != 0) { + udma_fill_inline_wqe(sq, wqe, sge_addr, wr->send.src.sge, + &sge_info); + return; + } + + udma_fill_rw_or_send_sge(sge_addr, sq, wr->send.src.sge, &sge_info, + &len); + wqe->tsk_sec.com.data_len = cpu_to_be32(len); +} + +static void udma_update_pi_and_fill_o_bit(struct tag_ub_sq_wqe_s *wqe, + const struct ubcore_jfs_wr *wr, + struct udma_send_queue *sq, + u32 wqebb_cnt) +{ + struct tag_ub_sq_wqe_s *next_wqe = NULL; + u32 offset = 0; + + sq->pi = sq->pi + wqebb_cnt; + /* fill wqe ctrl head section:mask_pi bit */ + wqe->ctrl_sec.dw1_value |= (sq->pi & (sq->baseblk_cnt - 1)) & 0xFFFF; + offset = (sq->pi & (sq->baseblk_cnt - 1)) << sq->baseblk_shift; + sq->buf_curr = sq->buf + offset; + wqe->ctrl_sec.o = (((sq->pi - wqebb_cnt) & sq->baseblk_cnt) == 0) ? 0 : + 1; + wqe->ctrl_sec.dw0_value = cpu_to_be32(wqe->ctrl_sec.dw0_value); + wqe->ctrl_sec.dw1_value = cpu_to_be32(wqe->ctrl_sec.dw1_value); + wqe->tsk_sec.com.dw0_value = cpu_to_be32(wqe->tsk_sec.com.dw0_value); + wqe->tsk_sec.com.dw4_value = cpu_to_be32(wqe->tsk_sec.com.dw4_value); + // set next wqebb o bit invalid + next_wqe = (struct tag_ub_sq_wqe_s *)sq->buf_curr; + next_wqe->ctrl_sec.o = !((((sq->pi) & sq->baseblk_cnt) == 0) ? 0 : 1); + next_wqe->ctrl_sec.dw0_value = + cpu_to_be32(next_wqe->ctrl_sec.dw0_value); +} + +int udma_fill_jfs_cmd(struct udma_device *udma_dev, struct tag_ub_sq_wqe_s *wqe, + enum UB_SQWQE_OPTYPE_E opcode, + const struct ubcore_jfs_wr *wr, + struct udma_post_wqe *post_wqe) +{ + struct udma_send_queue *sq = post_wqe->sq; + u8 *sge_addr = NULL; + u32 wqebb_cnt; + + wqebb_cnt = udma_wqebb_cnt(udma_dev, sq, opcode, wr); + udma_fill_ctrl_and_task_com(wqe, wr, post_wqe, opcode, + udma_dev->ub_dev.cfg.pattern); + + switch (opcode) { + case UB_TX_READ: + case UB_TX_WRITE: + udma_fill_rw_task_sec(wqe, wr, sq, &sge_addr, opcode); + udma_fill_rw_cmd(udma_dev, sge_addr, wqe, wr, sq); + break; + case UB_TX_WRITE_IMMEDIATE: + udma_fill_rw_imme_task_sec(wqe, wr, sq, &sge_addr); + udma_fill_rw_cmd(udma_dev, sge_addr, wqe, wr, sq); + break; + case UB_TX_SEND: + case UB_TX_SEND_IMMEDIATE: + case UB_TX_SEND_INVALIDATE: + udma_fill_send_task_sec(wqe, wr, sq, &sge_addr); + udma_fill_send_cmd(udma_dev, sge_addr, wqe, wr, sq); + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid opcode\n"); + return -EINVAL; + } + + udma_update_wqe_fast_dwqe_en(opcode, wqe); + wqe = sq->buf_curr; + udma_fill_slice_bit(udma_dev, post_wqe->tp_mode, wqe); + udma_update_pi_and_fill_o_bit(wqe, wr, sq, wqebb_cnt); + + return 0; +} + +void udma_post_send_jfs_dwqe(struct udma_device *udma_dev, struct udma_jfs *jfs) +{ + ulong flags = 0; + ub_sq_wqe_s *wqe = (ub_sq_wqe_s *)jfs->sq.dwqe_curr; + + union tag_ub_sq_db_u dwqe_db; + dwqe_db.sq_db_value = 0; + dwqe_db.sq_db.dw0.bs.xqn = jfs->jfs_id; + dwqe_db.sq_db.dw0.bs.type = UB_DIRECT_WQE_SRV_TYPE + << UB_DWQE_DB_TYPE_SHIFT; // high 2bit + dwqe_db.sq_db.dw1.bs.pi = jfs->sq.pi & + 0xFFFF; // complete pi, normal is pi[15:8] + dwqe_db.sq_db.dw0.bs.cntx_size = UB_CNTX_SIZE; + dwqe_db.sq_db.dw0.bs.cos = jfs->cos; + udma_debug(udma_dev->dev, UDMA_M_JFS, "cos: %u, pi: %u\n", jfs->cos, + jfs->sq.pi); + + wqe->ctrl_sec.dw2_value = cpu_to_be32(dwqe_db.sq_db_value & 0xFFFFFFFF); + wqe->ctrl_sec.dw3_value = + cpu_to_be32(dwqe_db.sq_db_value >> ADDR_H_OFFSET); + wmb(); + *(jfs->sdb.db_record) = cpu_to_be32(jfs->sq.pi); + wmb(); + + spin_lock_irqsave(&udma_dev->dwqe_lock, flags); + udma_dwqe_copy((u64 *)udma_dev->kernel_dwqe_map, (u64 *)(uintptr_t)wqe, + (u32)jfs->sq.wqe_size); + spin_unlock_irqrestore(&udma_dev->dwqe_lock, flags); +} + +static int udma_update_jfs_db(struct udma_device *udma_dev, + struct udma_jfs *jfs, u32 wr_cnt) +{ + union tag_ub_sq_db_u jfs_db; + int ret; + + if (udma_is_sq_dwqe(&jfs->sq, wr_cnt, udma_dev)) { + udma_debug(udma_dev->dev, UDMA_M_JFS, "the wqe is jfs dwqe\n"); + udma_post_send_jfs_dwqe(udma_dev, jfs); + return 0; + } + + wmb(); + udma_base_construct_sq_db(&jfs_db, jfs->jfs_id, jfs->sq.pi, jfs->cos); + udma_debug(udma_dev->dev, UDMA_M_JFS, "cos : %u, pi : %u\n", jfs->cos, + jfs->sq.pi); + *(jfs->sdb.db_record) = cpu_to_be32(jfs->sq.pi); + + wmb(); + ret = cqm5_ring_hardware_db(udma_dev->hwdev, SERVICE_T_UB, + jfs->sq.pi & 0xff, jfs_db.sq_db_value); + if (ret != CQM_SUCCESS) { + udma_err(udma_dev->dev, UDMA_M_JFS, "DB write field ret: %d.\n", + ret); + ret = -EIO; + } + + wmb(); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "xid:0x%x, jfs_db.sq_db_value:0x%llx\n", jfs->jfs_id, + jfs_db.sq_db_value); + return ret; +} + +static bool udma_get_sg_len(struct udma_device *udma_dev, + const struct ubcore_sg *sg, u64 *total_len) +{ + struct ubcore_sge *tmp_sge = NULL; + u32 i; + + tmp_sge = sg->sge; + for (i = 0; i < sg->num_sge; i++) { + if (tmp_sge == NULL || tmp_sge->tseg == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge Or Tseg Is NULL\n"); + return false; + } + + if (*total_len > UINT_MAX - tmp_sge->len) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. sge len is error!\n"); + return false; + } + + *total_len += tmp_sge->len; + + tmp_sge++; + } + if (*total_len == 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "total_len cannot be 0.\n"); + return false; + } + return true; +} + +static int udma_wr_check_inline(struct udma_device *udma_dev, + const struct ubcore_sg *sg, u32 max_inline_size) +{ + struct ubcore_sge *tmp_sge = NULL; + u32 total_len = 0; + u32 i; + + tmp_sge = sg->sge; + for (i = 0; i < sg->num_sge; i++) { + if (tmp_sge == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge Or Tseg Is NULL\n"); + return -EINVAL; + } + + if (tmp_sge->addr == 0 && tmp_sge->len != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. Sge->add is 0 and sge->len is not 0\n"); + return -EINVAL; + } + total_len += tmp_sge->len; + if (total_len > max_inline_size) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Inline src_len:%u is over max_inline_size:%u\n", + total_len, max_inline_size); + return -EINVAL; + } + tmp_sge++; + } + + return 0; +} + +static int udma_wr_check_read(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, u8 max_send_sge) +{ + u64 dst_len = 0; + + if (wr->flag.bs.inline_flag != 0) + return 0; + + if (wr->rw.src.num_sge > max_send_sge) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid parameter.\n"); + return -EINVAL; + } + + if ((wr->rw.src.sge == NULL) || (wr->rw.src.sge->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. sge or tseg is NULL\n"); + return -EINVAL; + } + + if (!udma_get_sg_len(udma_dev, &wr->rw.dst, &dst_len)) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. dst_sge is NULL or sge len is error\n"); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JFS, + "read num_sge is %u, dst_len is %llu\n", wr->rw.src.num_sge, + dst_len); + if (dst_len > UB_MAX_WRITE_READ_LEN) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. read dst_len is %llu over 2GB\n", + dst_len); + return -EINVAL; + } + + return 0; +} + +static int udma_wr_check_write_common(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + u8 max_send_sge, + const struct udma_post_wqe *post_wqe) +{ + u64 src_len = 0; + + if (wr->flag.bs.inline_flag != 0) { + return udma_wr_check_inline(udma_dev, &wr->rw.src, + post_wqe->sq->max_inline_size); + } + + if (wr->rw.src.num_sge > max_send_sge) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid parameter.\n"); + return -EINVAL; + } + + if (!udma_get_sg_len(udma_dev, &wr->rw.src, &src_len)) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. src_sge is NULL or sge len is error\n"); + return -EINVAL; + } + + if ((wr->rw.dst.sge == NULL) || (wr->rw.dst.sge->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. dst_sge is NULL\n"); + return -EINVAL; + } + + udma_debug(udma_dev->dev, UDMA_M_JFS, + "write num_sge is %u, src_len is %llu\n", wr->rw.src.num_sge, + src_len); + if (src_len > UB_MAX_WRITE_READ_LEN) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. write src_len is %llu over 2G\n", + src_len); + return -EINVAL; + } + + return 0; +} + +static int udma_wr_check_send(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + const struct udma_post_wqe *post_wqe) +{ + u64 src_len = 0; + bool ret = true; + + if ((wr->opcode == UBCORE_OPC_SEND_INVALIDATE) && + (wr->send.tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. tseg is NULL\n"); + return -EINVAL; + } + + if (wr->flag.bs.inline_flag != 0) { + return udma_wr_check_inline(udma_dev, &wr->rw.src, + post_wqe->sq->max_inline_size); + } + + ret = udma_get_sg_len(udma_dev, &wr->send.src, &src_len); + if (!ret) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. src_sge is NULL or sge len is error\n"); + return -EINVAL; + } + + if (src_len > ((1 << udma_dev->slice) * UB_SLICE_LEN) || + src_len > UB_SLICE_LEN_64K) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter. src_len %llu is over %d\n", + src_len, (1 << udma_dev->slice) * UB_SLICE_LEN); + return -EINVAL; + } + udma_debug(udma_dev->dev, UDMA_M_JFS, "UB_SLICE_LEN: %d\n", + (1 << udma_dev->slice) * UB_SLICE_LEN); + return 0; +} + +static int udma_wr_check_cas(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr) +{ + if ((wr->cas.dst == NULL) || (wr->cas.dst->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, dst or tseg is NULL\n"); + return -EINVAL; + } + + if ((wr->cas.src == NULL) || (wr->cas.src->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, src or tseg is NULL\n"); + return -EINVAL; + } + + return 0; +} + +static int udma_wr_check_fadd(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr) +{ + if ((wr->faa.dst == NULL) || (wr->faa.dst->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, dst or tseg is NULL\n"); + return -EINVAL; + } + + if ((wr->faa.src == NULL) || (wr->faa.src->tseg == NULL)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, dst or tseg is NULL\n"); + return -EINVAL; + } + + return 0; +} + +int udma_wr_check(const struct ubcore_jfs_wr *wr, u8 max_send_sge, + struct udma_device *udma_dev, + const struct udma_post_wqe *post_wqe) +{ + int ret = 0; + + if (post_wqe->tp_mode == UBCORE_TP_UM && + (wr->opcode != UBCORE_OPC_SEND && + wr->opcode != UBCORE_OPC_SEND_IMM)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "The um operation type does not support opcode %u\n", + (u32)wr->opcode); + return -EINVAL; + } + + if (wr->tjetty == NULL || wr->tjetty->vtpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "send.tjetty or send.tjetty->tp is null!\n"); + return -EINVAL; + } + + if (wr->tjetty->cfg.flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "token_policy=%u dont support\n", + wr->tjetty->cfg.flag.bs.token_policy); + return -EINVAL; + } + + if ((wr->flag.bs.inline_flag != 0) && + (wr->opcode != UBCORE_OPC_WRITE) && + (wr->opcode != UBCORE_OPC_SEND) && + (wr->opcode != UBCORE_OPC_WRITE_IMM)) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: opcode=%u dont support inline\n", + wr->opcode); + return -EINVAL; + } + + switch (wr->opcode) { + case UBCORE_OPC_READ: + ret = udma_wr_check_read(udma_dev, wr, max_send_sge); + break; + case UBCORE_OPC_WRITE: + case UBCORE_OPC_WRITE_IMM: + ret = udma_wr_check_write_common(udma_dev, wr, max_send_sge, + post_wqe); + break; +#ifdef UB_SUPPORT_SEND_IMM_OR_INV + case UBCORE_OPC_SEND_IMM: + case UBCORE_OPC_SEND_INVALIDATE: +#endif + case UBCORE_OPC_SEND: + ret = udma_wr_check_send(udma_dev, wr, post_wqe); + break; + case UBCORE_OPC_CAS: + ret = udma_wr_check_cas(udma_dev, wr); + break; + case UBCORE_OPC_FADD: + ret = udma_wr_check_fadd(udma_dev, wr); + break; + default: + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid opcode :%u\n", + (u32)wr->opcode); + return -EINVAL; + } + + return ret; +} + +int udma_handle_jfs_wr(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, u32 wr_cnt, + struct udma_post_wqe *post_wqe) +{ + enum UB_SQWQE_OPTYPE_E opcode = ub_jfs_parse_opcode(wr->opcode); + struct tag_ub_sq_wqe_s *wqe = post_wqe->sq->buf_curr; + struct udma_send_queue *sq = post_wqe->sq; + u32 wqe_len; + int ret = 0; + + if (opcode >= UB_ERR_TYPE) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Invalid opcode :%u\n", + (u32)wr->opcode); + return -EINVAL; + } + + post_wqe->sq->dwqe_curr = post_wqe->sq->buf_curr; + sq->wrid[(sq->head + wr_cnt) & (sq->wrid_size - 1)] = wr->user_ctx; + + ret = udma_fill_jfs_cmd(udma_dev, wqe, opcode, wr, post_wqe); + if (ret != 0) { + return ret; + } + + if (opcode == UB_TX_ATOMIC_COMPARE_SWAP || + opcode == UB_TX_ATOMIC_FETCH_ADD) { + wqe_len = sizeof(struct tag_ub_sq_wqe_s) + + sizeof(struct ubcore_sge); + } else if (wr->flag.bs.inline_flag == 0) { + wqe_len = sizeof(struct tag_ub_sq_wqe_s) + + sizeof(struct ubcore_sge) * wr->rw.src.num_sge; + } else { + wqe_len = sizeof(struct tag_ub_sq_wqe_s) + + be32_to_cpu(wqe->tsk_sec.com.data_len); + } + udma_dump_dfx(udma_dev, (u64)(uintptr_t)wqe, wqe_len, + (u64)(uintptr_t)sq->buf, sq->buf_size); + + return 0; +} + +int udma_sq_overflow(struct udma_send_queue *sq, u32 wr_num, + struct udma_jfc *jfc) +{ + ulong flags = 0; + + u32 cur = sq->head - sq->tail; + if ((cur + wr_num) < sq->max_wr_num) { + return 0; + } + if (jfc == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: jfc(%d).\n", !!jfc); + return -EINVAL; + } + + if (jfc == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter: jfc(%d).\n", !!jfc); + return 0; + } + + spin_lock_irqsave(&jfc->lock, flags); + cur = sq->head - sq->tail; + spin_unlock_irqrestore(&jfc->lock, flags); + + return ((cur + wr_num) >= sq->max_wr_num); +} + +int udma_construct_wqe(struct udma_device *udma_dev, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr, u32 *wr_cnt, + struct udma_post_wqe *post_wqe) +{ + struct ubcore_jfs_wr *iter = NULL; + int ret = 0; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + *bad_wr = (struct ubcore_jfs_wr *)wr; + return -EINVAL; + } + + for (iter = (struct ubcore_jfs_wr *)wr; iter != NULL; + iter = (struct ubcore_jfs_wr *)(void *)iter->next) { + if (udma_sq_overflow(post_wqe->sq, *wr_cnt, post_wqe->jfc) != + 0) { + udma_info(udma_dev->dev, UDMA_M_JFS, "sq overflow!\n"); + *bad_wr = iter; + return -EAGAIN; + } + + if (udma_wr_check(iter, post_wqe->max_send_sge, udma_dev, + post_wqe) != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "wr check false!\n"); + *bad_wr = iter; + return -EINVAL; + } + + if (iter->tjetty == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid parameter: iter->tjetty is null.\n"); + *bad_wr = iter; + return -EINVAL; + } + + if (post_wqe->tp_mode != iter->tjetty->cfg.trans_mode) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "Invalid parameter, local jetty trans_mode is %u, target jetty trans_mode is %u.\n", + post_wqe->tp_mode, + iter->tjetty->cfg.trans_mode); + *bad_wr = iter; + return -EINVAL; + } + + if ((post_wqe->tp_mode != UBCORE_TP_RM) && + (iter->tjetty->cfg.type == UBCORE_JETTY_GROUP)) { + udma_err( + udma_dev->dev, UDMA_M_JFS, + "ERR: tp_mode(%d) and tjetty type(%d) not match\n", + post_wqe->tp_mode, iter->tjetty->cfg.type); + *bad_wr = iter; + return -EINVAL; + } + + post_wqe->tpn = iter->tjetty->vtpn->vtpn; + post_wqe->sq->wqe_size = 0; + post_wqe->sq->rmd_wqebb_cnt = + post_wqe->sq->baseblk_cnt - + (post_wqe->sq->pi & (post_wqe->sq->baseblk_cnt - 1)); + + udma_debug( + udma_dev->dev, UDMA_M_JFS, + "handle jfs START.trans_mode:%u,tjetty_type:%d,tpn:%u,ubcore opcode:%u\n", + post_wqe->tp_mode, iter->tjetty->cfg.type, + post_wqe->tpn, wr->opcode); + ret = udma_handle_jfs_wr(udma_dev, iter, *wr_cnt, post_wqe); + if (ret != 0) { + *bad_wr = (struct ubcore_jfs_wr *)iter; + return ret; + } + (*wr_cnt)++; + + if ((*wr_cnt) > post_wqe->sq->max_wr_num) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "ERR: wr_cnt[%u] big than max_wr_num[%u]\n", + (*wr_cnt), post_wqe->sq->max_wr_num); + *bad_wr = iter; + return -EINVAL; + } + } + + return ret; +} + +int udma_post_jfs_wr_one(struct ubcore_jfs *ubcore_jfs, + const struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + struct udma_post_wqe post_wqe = { 0 }; + ulong flags = 0; + u32 wr_cnt = 0; + int ret = 0; + + if (ubcore_jfs == NULL || ubcore_jfs->ub_dev == NULL) { + udma_pr_err(UDMA_M_JFS, "Invalid parameter. ubcore_jfs(%d) .\n", + !!ubcore_jfs); + *bad_wr = (struct ubcore_jfs_wr *)wr; + return -EINVAL; + } + + jfs = to_udma_jfs(ubcore_jfs); + udma_dev = to_udma_dev(ubcore_jfs->ub_dev); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "Start to post jfs send wr, jfs_id:%u\n", jfs->jfs_id); + spin_lock_irqsave(&jfs->sq.lock, flags); + post_wqe.jfc = to_udma_jfc(jfs->ubcore_jfs.jfs_cfg.jfc); + post_wqe.tp_mode = jfs->trans_mode; + post_wqe.sq = &jfs->sq; + post_wqe.max_send_sge = jfs->ubcore_jfs.jfs_cfg.max_sge; + post_wqe.is_fast_path = check_k_is_fast_path(wr, post_wqe.sq); + + ret = udma_construct_wqe(udma_dev, wr, bad_wr, &wr_cnt, &post_wqe); + + if (wr_cnt != 0) { + udma_update_jfs_db(udma_dev, jfs, wr_cnt); + jfs->sq.head += wr_cnt; + atomic64_add(wr_cnt, + &jfs->sq.udma_driver_sq_ctr.post_send_wr_num); + atomic64_add( + wr_cnt, + &udma_dev->udma_driver_device_ctr.post_send_wr_num); + udma_debug( + udma_dev->dev, UDMA_M_JFS, + "dev_post_send_wr_num:%llu, post_jfs_wr_num:%llu\n", + (u64)atomic64_read(&udma_dev->udma_driver_device_ctr + .post_send_wr_num), + (u64)atomic64_read( + &jfs->sq.udma_driver_sq_ctr.post_send_wr_num)); + } + spin_unlock_irqrestore(&jfs->sq.lock, flags); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "Finished to post jfs send wr, wr_cnt:%u\n", wr_cnt); + return ret; +} + +int udma_post_jfs_wr(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + + if ((ubcore_jfs == NULL) || (ubcore_jfs->ub_dev == NULL) || + (wr == NULL) || (bad_wr == NULL)) { + udma_pr_err( + UDMA_M_JFC, + "%s Invalid parameter: ubcore_jfs(%d), wr(%d), bad_wr(%d).\n", + __func__, !!ubcore_jfs, !!wr, !!bad_wr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ubcore_jfs->ub_dev); + jfs = to_udma_jfs(ubcore_jfs); + udma_debug(udma_dev->dev, UDMA_M_JFS, + "start post jfs send, jfs id: %u, state: %d\n", jfs->jfs_id, + jfs->state); + + if (jfs->state != UBCORE_JETTY_STATE_READY) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "jfs state: %d, posting not supported\n", jfs->state); + *bad_wr = wr; + return -EINVAL; + } + + return udma_post_jfs_wr_one(ubcore_jfs, wr, bad_wr); +} + +int udma_modify_jfs(struct ubcore_jfs *ubcore_jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_jfs *jfs = NULL; + int ret; + + if (ubcore_jfs == NULL || ubcore_jfs->ub_dev == NULL || attr == NULL) { + udma_pr_err( + UDMA_M_JFC, + "ubcore_jfs(%d), attr(%d) or ubcore_jfs->ub_dev is NULL\n", + !!ubcore_jfs, !!attr); + return -EINVAL; + } + + udma_dev = to_udma_dev(ubcore_jfs->ub_dev); + jfs = to_udma_jfs(ubcore_jfs); + udma_info(udma_dev->dev, UDMA_M_JFS, "Modify jfs(%u), state(%u)\n", + jfs->jfs_id, attr->state); + + if ((attr->mask & UBCORE_JFS_STATE) == 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "JFS modify mask is not set or invalid.\n"); + return -EINVAL; + } + + if (attr->state < UBCORE_JETTY_STATE_RESET || + attr->state > UBCORE_JETTY_STATE_ERROR) { + udma_err(udma_dev->dev, UDMA_M_JFS, + "Invalid jfs state: 0x%08x\n", attr->state); + return -EINVAL; + } + + if (udma_modify_jetty_parse_udata(udma_dev, &jfs->sq, udata) != 0) { + udma_err(udma_dev->dev, UDMA_M_JETTY, "Failed parse udata\n"); + return -EINVAL; + } + + ret = udma_cmd_jfs_modify_ctx(udma_dev, jfs, attr->state, udata); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFS, "Failed to modify jfs\n"); + } else { + jfs->state = attr->state; + jfs->sq.is_hw_flush_done = false; + } + + udma_info(udma_dev->dev, UDMA_M_JFS, "Modify jfs %u, ret %d\n", + jfs->jfs_id, ret); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c new file mode 100644 index 000000000..fd71b0f56 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_cmd_srq.h" +#include "udma_jetty.h" +#include "ub_ta_wqe_format.h" +#include "udma_srq.h" + +/* Containers are managed using a linked list, with one sentinel node added to the linked list */ +#define UDMA_CONTAINER_GUARD_NODE 1 + +static u8 ub_get_container_sz(u32 container_mode) +{ + switch (container_mode) { + case UB_SRQ_MODE_1: + return UB_SRQ_CONT_NUM_MODE1; + case UB_SRQ_MODE_2: + return UB_SRQ_CONT_NUM_MODE2; + case UB_SRQ_MODE_3: + return UB_SRQ_CONT_NUM_MODE3; + case UB_SRQ_MODE_4: + return UB_SRQ_CONT_NUM_MODE4; + default: + return UB_SRQ_CONT_NUM_MODE0; + } +} + +static void udma_set_srq_depth(struct udma_srq *srq, u32 depth) +{ + /* container in non- link wqe of rqe count */ + uint32_t no_link_wqe_num = srq->container_size - 1; + /* get this depth at least need how many container, round up */ + uint32_t container_num = + ((depth + 1) + (no_link_wqe_num - 1)) / + no_link_wqe_num; /* Note: this at adaptation no_link_wqe scenario and container_size equal to 2 of scenario, other scenario need analyze */ + + srq->max_depth = container_num * srq->container_size; + srq->max_depth = roundup_pow_of_two(srq->max_depth) & 0xffffffff; +} + +static uint32_t udma_get_rqe_buf_size(struct udma_device *udma_dev, + struct udma_srq *srq, u32 wqe_size) +{ + uint32_t buf_size = 0; + uint32_t page_sz = PAGE_SIZE; + + buf_size = srq->max_depth * wqe_size; + if (udma_dev->ub_cap.dev_cap.mtt_pro_en == 1) { + page_sz = udma_cal_page_size(buf_size); + } + + if (buf_size < page_sz) { + buf_size = page_sz; + srq->max_depth = buf_size >> srq->wqe_shift; + } + + return ALIGN(buf_size, page_sz); +} + +int udma_srq_init_attr(struct udma_device *udma_dev, struct udma_srq *srq, + u32 depth, u8 max_sge) +{ + u32 wqe_size = 0; + u32 temp_size = 0; + + spin_lock_init(&srq->lock); + srq->index_queue_en = udma_get_index_queue_en(udma_dev); + srq->container_flag = UB_SRQ_CONTAINER_EN; + srq->container_mode = srq->index_queue_en ? UB_SRQ_MODE_4 : + UB_SRQ_MODE_1; + srq->container_size = ub_get_container_sz(srq->container_mode); + srq->container_warn_th = 0; + + udma_set_srq_depth(srq, depth); + srq->max_sge = srq->index_queue_en ? UB_SRQ_MAX_SGE : + SRQ_MAX_SGE(max_sge); + temp_size = (u32)roundup_pow_of_two( + (u32)sizeof(struct ub_wqe_srq_next_seg) + + (srq->max_sge) * (u32)sizeof(struct ub_wqe_srq_data_seg)); + wqe_size = SRQ_WQE_SIZE(temp_size); + srq->wqe_shift = (u32)ilog2(wqe_size); + srq->buf_size = udma_get_rqe_buf_size(udma_dev, srq, wqe_size); + + return 0; +} + +int udma_index_queue_init_attr(struct udma_device *udma_dev, + struct udma_index_queue *index_queue, u32 depth) +{ + u32 buf_size = 0; + + index_queue->wqe_shift = (u32)ilog2(sizeof(ubg_jfr_index_wqe_s)); + index_queue->max_depth = roundup_pow_of_two(depth) & 0xffffffff; + buf_size = index_queue->max_depth << index_queue->wqe_shift; + if (buf_size < PAGE_SIZE) + index_queue->max_depth = PAGE_SIZE >> + (u16)index_queue->wqe_shift; + index_queue->buf_size = ALIGN(buf_size, PAGE_SIZE); + + return 0; +} + +int udma_create_srqc_update(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb) +{ + struct tag_cqm_queue *srqc_info = NULL; + int ret = 0; + + u32 xid_start = 0; + u32 xid_end = xid_start + udma_dev->ub_cap.dev_cap_res.sdk_res.max_jfr; + u32 qpn_tmp = + udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_1_1_1, srq->jfr_id); + + srq->cqm_priv_type = UDMA_CQM_PRIV_TYPE_SRQ; + srqc_info = udma_object_queue_create(udma_dev, CQM_OBJECT_RDMA_SRQ, 0, + srq, &udma_dev->srqn_record, false, + qpn_tmp, xid_start, xid_end); + if (srqc_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Create srqc by cqm object failed.\n"); + return -ENOMEM; + } + + srq->srqc_info = srqc_info; + srq->srqn = srq->srqc_info->index; + + ret = udma_cmd_srq_create_ctx(udma_dev, srq, sdb); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "Enable srq context failed. ret: %d\n", ret); + cqm5_object_delete(&(srqc_info->object)); + srq->srqc_info = NULL; + return ret; + } + + return ret; +} + +int udma_delete_srqc_and_cache_out(struct udma_device *udma_dev, + struct udma_srq *srq) +{ + int ret; + + ret = udma_cmd_srq_delete_ctx(udma_dev, srq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "Delete srq and cache out failed. ret: %d\n", ret); + return ret; + } + + if (srq->srqc_info != NULL) { + cqm5_object_delete(&(srq->srqc_info->object)); + srq->srqc_info = NULL; + } + return ret; +} + +int udma_srq_init_res(struct udma_device *udma_dev, struct udma_srq *srq, + struct tag_cqm_queue *buf_info) +{ + struct ub_wqe_srq_data_seg *sge = NULL; + struct ub_container_srq_wqe_head *next = NULL; + u32 container_cap = srq->container_size; + u32 wr_cnt; + + srq->srqc_info = buf_info; + srq->srqn = buf_info->index; + srq->buf = &buf_info->q_room_buf_1; + srq->srq_va = udma_dev->ub_cap.dev_cap.mtt_pro_en == 1 ? + srq->queue_addr.va : + srq->buf->direct.va; + + srq->db.db_record = + (__be32 *)(void *)(&buf_info->q_header_vaddr->doorbell_record); + srq->db.dma = buf_info->q_header_paddr; + *srq->db.db_record = 0; + srq->counter_left = 0; + srq->container_counter = 0; + if (srq->flush_flag == true) { + udma_err(udma_dev->dev, UDMA_M_JFR, + "init srq, now flush_flag is true.\n"); + srq->flush_flag = false; + } + + srq->head = 0; + srq->tail = (s32)(srq->max_depth - container_cap); + + for (wr_cnt = 0; wr_cnt < srq->max_depth; ++wr_cnt) { + next = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, wr_cnt); + + if ((wr_cnt + 1U) % container_cap == 0) { + next->dw3.bs.link_flag = 1; + next->dw2.bs.next_idx = + (u16)((wr_cnt + 1) & (srq->max_depth - 1)) / + container_cap; + next->dw2.bs.cur_idx = + (u16)(wr_cnt & (srq->max_depth - 1)) / + container_cap; + + next->dw4.value = (u32)htonl(next->dw4.value); + next->dw3.value = (u32)htonl(next->dw3.value); + next->dw2.value = (u32)htonl(next->dw2.value); + } else { + /* we just need to init the first sge section; */ + sge = (struct ub_wqe_srq_data_seg *)(void *)next; + sge[0].dw3.lkey = + (u32)htonl((u32)UB_WQE_NEXT_SGE_INVALID); + sge[0].dw2.length = 0; + } + } + + return 0; +} + +void udma_free_srq_wqe(struct udma_srq *srq, u32 container_idx) +{ + u32 last_link_wqe_idx = + ((u32)container_idx + 1) * srq->container_size - 1; + struct ub_container_srq_wqe_head *tail_cont_link_wqe = NULL; + struct ub_container_srq_wqe_head *last_cont_link_wqe = NULL; + u32 tail_link_wqe_idx = 0; + + spin_lock(&srq->lock); + + last_cont_link_wqe = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, last_link_wqe_idx); + last_cont_link_wqe->dw3.value = + (u32)ntohl(last_cont_link_wqe->dw3.value); + if (last_cont_link_wqe->dw3.bs.link_flag != 0) { + tail_link_wqe_idx = (u32)srq->tail + (srq->container_size - 1); + + tail_cont_link_wqe = + (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, tail_link_wqe_idx); + tail_cont_link_wqe->dw2.value = + (u32)ntohl(tail_cont_link_wqe->dw2.value); + tail_cont_link_wqe->dw2.bs.next_idx = container_idx; + tail_cont_link_wqe->dw2.value = + (u32)htonl(tail_cont_link_wqe->dw2.value); + + srq->tail = (s32)(container_idx * srq->container_size); + } + last_cont_link_wqe->dw3.value = + (u32)htonl(last_cont_link_wqe->dw3.value); + + spin_unlock(&srq->lock); +} + +u16 udma_get_next_srq_wqe(struct udma_srq *srq, u16 cur_idx) +{ + struct ub_container_srq_wqe_head *cur_link_wqe = NULL; + u16 cur_link_wqe_idx = (u16)(cur_idx * srq->container_size + + (srq->container_size - 1)); + u16 next_idx; + + spin_lock(&srq->lock); + cur_link_wqe = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, cur_link_wqe_idx); + cur_link_wqe->dw2.value = (u32)ntohl(cur_link_wqe->dw2.value); + next_idx = cur_link_wqe->dw2.bs.next_idx; + cur_link_wqe->dw2.value = (u32)htonl(cur_link_wqe->dw2.value); + spin_unlock(&srq->lock); + + return next_idx; +} + +static u32 udma_srq_get_max_avail_wr(struct udma_srq *srq) +{ + u32 link_wqe_count = 0; + + if (srq->container_flag != 0 && srq->container_size != 0) { + link_wqe_count = srq->max_depth / srq->container_size; + return (srq->max_depth - link_wqe_count) - + (srq->container_size - 1); + } else { + return srq->max_depth - 1; + } +} + +int udma_modify_srq_threshold(struct udma_device *udma_dev, + struct udma_srq *srq, u8 container_warn_th) +{ + u32 max_avail_wr = 0; + int ret = 0; + + max_avail_wr = udma_srq_get_max_avail_wr(srq); + if (container_warn_th > max_avail_wr) { + udma_err( + udma_dev->dev, UDMA_M_SRQ, + "rx_threshold > max_wr, rx_threshold(%u), max_avail_wr(%u)\n", + container_warn_th, max_avail_wr); + return -EINVAL; + } + + mutex_lock(&srq->mutex); + ret = udma_cmd_srq_modify_ctx(udma_dev, srq, container_warn_th); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "modify srq threshold cmdq failed, srqn(%u).\n", + srq->srqn); + mutex_unlock(&srq->mutex); + return -EFAULT; + } + mutex_unlock(&srq->mutex); + + srq->container_warn_th = container_warn_th; + udma_info(udma_dev->dev, UDMA_M_SRQ, + "modify srq threshold success, srqn(%u)\n", srq->srqn); + return ret; +} + +static int udma_sge_check_vld(struct udma_device *udma_dev, + const struct ubcore_sge *sge, u32 data_size, + u64 max_msg_sz) +{ + if ((data_size + sge->len > max_msg_sz) || + (data_size + sge->len < data_size) || + (sge->len > UB_SGE_LEN_LIMIT)) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "data_size: %u, wr sge len: %u, max_msg_sz: %llu\n", + data_size, sge->len, max_msg_sz); + return -EINVAL; + } + + if (sge->tseg == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, "sge tseg is null\n"); + return -EINVAL; + } + + return 0; +} + +int udma_srq_pre_post_wqe_check(struct udma_device *udma_dev, + const struct udma_srq *srq, + const struct ubcore_jfr_wr *wr) +{ + u32 data_size = 0; + u32 sge_num = 0; + + if ((u32)wr->src.num_sge > srq->max_sge) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "wr sge num(%u) > max sge(%u)\n", wr->src.num_sge, + srq->max_sge); + return -EINVAL; + } + + if (srq->head == srq->tail) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "srq is full, head: %d, tail: %d\n", srq->head, + srq->tail); + return -EINVAL; + } + + if (wr->src.num_sge != 0 && wr->src.sge == NULL) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "sge num is %u, wr sge is null\n", wr->src.num_sge); + return -EINVAL; + } + + for (sge_num = 0, data_size = 0; sge_num < wr->src.num_sge; ++sge_num) { + if (udma_sge_check_vld(udma_dev, &wr->src.sge[sge_num], + data_size, UB_MAX_WRITE_READ_LEN) != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "wr sge(%u) is invalid\n", sge_num); + return -EINVAL; + } + data_size += wr->src.sge[sge_num].len; + } + return 0; +} + +static void udma_get_container_head(struct udma_srq *srq, + struct ub_container_srq_wqe_head **next) +{ + struct ub_container_srq_wqe_head tmp; + + *next = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, (u32)srq->head); + tmp.dw3.value = (u32)ntohl((*next)->dw3.value); + if (tmp.dw3.bs.link_flag != 0) { + tmp.dw2.value = (u32)ntohl((*next)->dw2.value); + srq->head = + (s32)((u32)(tmp.dw2.bs.next_idx) * srq->container_size); + (*next) = (struct ub_container_srq_wqe_head *)ub_srq_get_wqe( + srq, (u32)srq->head); + } +} + +static int udma_init_srq_wqe_header(struct udma_device *udma_dev, + struct udma_srq *srq, + const struct ubcore_jfr_wr *wr, + struct ub_wqe_srq_data_seg **sge, + u16 wr_num) +{ + struct ub_container_srq_wqe_head *next = NULL; + + if (srq->container_flag == 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "only support container mode\n"); + return -EINVAL; + } + + udma_get_container_head(srq, &next); + if (srq->head == srq->tail) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "srq is full, head: %d, tail: %d\n", srq->head, + srq->tail); + return -EINVAL; + } + + *sge = (struct ub_wqe_srq_data_seg *)(void *)next; + srq->wrid[srq->head] = wr->user_ctx; + srq->head += 1; + + udma_get_container_head(srq, &next); + + return 0; +} + +static int udma_construct_sge(struct udma_device *udma_dev, + const struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr, + struct ub_wqe_srq_data_seg *sge) +{ + u32 sge_num = 0; + + for (sge_num = 0; sge_num < wr->src.num_sge; ++sge_num) { + sge[sge_num].addr = htonll(wr->src.sge[sge_num].addr); + sge[sge_num].dw2.length = htonl(wr->src.sge[sge_num].len); + sge[sge_num].dw3.lkey = + htonl((wr->src.sge[sge_num].tseg->seg.token_id) & + (~UB_WQE_NEXT_SGE_INVALID)); + } + + /* + * Set the last sge L bit. To avoid case sge = 0, need to judge sge number; + * When wr.sge_num == 0,set sge[0] L flags + */ + if (wr->src.num_sge != 0) { + sge[sge_num - 1].dw3.lkey = + htonl((wr->src.sge[sge_num - 1].tseg->seg.token_id) | + UB_WQE_NEXT_SGE_INVALID); + } else { + sge[0].dw3.lkey = htonl(UB_WQE_NEXT_SGE_INVALID); + sge[0].dw2.length = 0; + } + + return 0; +} + +int udma_post_srq_wr(struct udma_device *udma_dev, struct udma_srq *srq, + const struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr) +{ + struct ub_wqe_srq_data_seg *sge = NULL; + u16 wr_num; + ulong flags = 0; + int ret = 0; + + spin_lock_irqsave(&srq->lock, flags); + for (wr_num = 0; wr != NULL; ++wr_num, wr = wr->next) { + ret = udma_srq_pre_post_wqe_check(udma_dev, srq, wr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "check post wqe failed, wr num: %u\n", wr_num); + *bad_wr = (struct ubcore_jfr_wr *)wr; + goto out; + } + + ret = udma_init_srq_wqe_header(udma_dev, srq, wr, &sge, wr_num); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "init srq wqe header failed, wr num: %u\n", + wr_num); + *bad_wr = (struct ubcore_jfr_wr *)wr; + goto out; + } + + ret = udma_construct_sge(udma_dev, wr, bad_wr, sge); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SRQ, + "construct sge failed, wr num: %u\n", wr_num); + *bad_wr = (struct ubcore_jfr_wr *)wr; + goto out; + } + udma_dump_dfx(udma_dev, (u64)(uintptr_t)sge, + (u32)(1 << (unsigned)srq->wqe_shift), + (u64)(uintptr_t)srq->srq_va, srq->buf_size); + } + +out: + if (wr_num != 0) { + srq->container_counter += (u16)(srq->counter_left + wr_num) / + (srq->container_size - 1); + srq->counter_left = (u8)((srq->counter_left + wr_num) % + (srq->container_size - 1)); + atomic64_add(wr_num, &srq->udma_driver_rq_ctr.post_recv_wr_num); + atomic64_add( + wr_num, + &udma_dev->udma_driver_device_ctr.post_recv_wr_num); + udma_debug(udma_dev->dev, UDMA_M_SRQ, + "post_jfr_wr_num:%llu, dev_post_recv_wr_num:%llu\n", + (u64)atomic64_read( + &srq->udma_driver_rq_ctr.post_recv_wr_num), + (u64)atomic64_read(&udma_dev->udma_driver_device_ctr + .post_recv_wr_num)); + /* + * Make sure that descriptors are written before + * we write doorbell record. + */ + wmb(); + + *(srq->db.db_record) = (u32)htonl((u32)srq->container_counter); + } + spin_unlock_irqrestore(&srq->lock, flags); + + udma_debug( + udma_dev->dev, UDMA_M_SRQ, + "post srq recv, wr_num: %u, container_counter: %u, conter_left: %u\n", + wr_num, srq->container_counter, srq->counter_left); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h new file mode 100644 index 000000000..094b43160 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/jetty/udma_srq.h @@ -0,0 +1,205 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_SRQ_H +#define UDMA_SRQ_H + +#include "udma_common.h" +#include "udma_db.h" +#include "hinic5_cqm.h" + +#define UB_SRQ_MAX_SGE 15 +#define UB_SRQ_MIN_SGE 7 +#define UB_WQEBB_SIZE_MIN 128 +#define UB_SRQ_CONTAINER_EN 1 + +#define UB_SGE_LEN_LIMIT ((u32)(1U << 31)) +#define SRQ_WQEBB_SIZE_CAL_SECTTOR \ + 4 //(2^rq_wqebb_size)*16B => divide 16 means shift need to minus 4 +#define UB_SRQ_CONTAINER_WARTH_MASK 0xF + +#define UB_WQE_NEXT_SGE_INVALID (1UL << 31) + +#define SRQ_MAX_SGE(x) \ + (((x) <= UB_SRQ_MIN_SGE) ? UB_SRQ_MIN_SGE : UB_SRQ_MAX_SGE) +#define SRQ_WQE_SIZE(x) (max((u32)UB_WQEBB_SIZE_MIN, x)) + +#ifndef htonll +#if BYTE_ORDER == LITTLE_ENDIAN +#define htonll(x) \ + ((((x)&0x00000000000000ffULL) << 56) | \ + (((x)&0x000000000000ff00ULL) << 40) | \ + (((x)&0x0000000000ff0000ULL) << 24) | \ + (((x)&0x00000000ff000000ULL) << 8) | \ + (((x)&0x000000ff00000000ULL) >> 8) | \ + (((x)&0x0000ff0000000000ULL) >> 24) | \ + (((x)&0x00ff000000000000ULL) >> 40) | \ + (((x)&0xff00000000000000ULL) >> 56)) + +#elif BYTE_ORDER == BIG_ENDIAN +#define htonll(x) (x) +#else +#error BYTE_ORDER is neither LITTLE_ENDIAN nor BIG_ENDIAN +#endif +#endif + +#ifndef ntohll +#define ntohll(x) htonll(x) +#endif + +enum srq_state { + UB_SRQ_STATE_INVALID = 0x0, + UB_SRQ_STATE_ERR = 0x1, + UB_SRQ_STATE_VALID = 0xf, + UB_SRQ_STATE_MEM_INIT = 0xa +}; + +/** + * srq container_mode: + * mode: 0 -> Not container + * mode: 1 -> container_size: 2 + * mode: 2 -> container_size: 4 + * mode: 3 -> container_size: 8 + * mode: 4 -> container_size: 16 + */ +enum ub_srq_mode { + UB_SRQ_MODE_0 = 0, + UB_SRQ_MODE_1, + UB_SRQ_MODE_2, + UB_SRQ_MODE_3, + UB_SRQ_MODE_4 +}; + +enum ub_srq_cont_num_mode { + UB_SRQ_CONT_NUM_MODE0 = 0, + UB_SRQ_CONT_NUM_MODE1 = 2, + UB_SRQ_CONT_NUM_MODE2 = 4, + UB_SRQ_CONT_NUM_MODE3 = 8, + UB_SRQ_CONT_NUM_MODE4 = 16 +}; +struct ub_wqe_srq_next_seg { + u16 reserved1; + u16 pcnt; // indicate the pi + u8 signature; + u8 reserved2; + u16 next_wqe_index; + u32 reserved3[2]; +}; + +struct ub_container_srq_wqe_head { + u32 next_gpa_h; + + union { + struct { + u32 rsvd : 11; + u32 next_gpa_vd : 1; + u32 next_gpa_l : 20; /* indicate the pi */ + } bs; + + u32 value; + } dw1; + + union { + struct { + u32 next_idx : 16; + u32 cur_idx : 16; + } bs; + u32 value; + } dw2; + + union { + struct { + u32 rsvd2 : 30; + u32 link_flag : 1; + u32 rsvd : 1; + } bs; + u32 value; + } dw3; + + union { + struct { + u32 osd_next_idx : 16; + u32 osd_cur_idx : 16; + } bs; + u32 value; + } dw4; +}; + +typedef struct udma_driver_rq_ctr { + atomic64_t post_recv_wr_num; +} udma_driver_rq_ctr_t; + +struct udma_srq { + u32 cqm_priv_type; + struct tag_cqm_queue *srqc_info; + struct queue_addr_info queue_addr; + struct ubcore_umem *umem; + struct mtt ub_mtt; + struct mutex mutex; + spinlock_t lock; + u32 srqn; + u32 jfr_id; + cqm_buf_s *buf; + void *srq_va; + u64 *wrid; + s32 head; + s32 tail; + u32 max_depth; + u32 max_sge; + u32 wqe_shift; + u32 buf_size; + struct udma_db db; + u8 container_flag; + u8 container_size; + u8 container_mode; + u8 counter_left; + u8 container_warn_th; + bool flush_flag; + bool index_queue_en; + u16 stop_index; + u16 container_counter; + udma_driver_rq_ctr_t udma_driver_rq_ctr; +}; + +struct udma_index_queue { + struct ubcore_umem *umem; + struct mtt ub_mtt; + u32 buf_size; + u32 max_depth; + u32 wqe_shift; +}; + +static inline void *ub_srq_buf_offset(void *va, int offset) +{ + return (void *)((char *)va + offset); +} + +static inline void *ub_srq_get_wqe(struct udma_srq *srq, u32 n) +{ + return ub_srq_buf_offset(srq->srq_va, + (int)(n << (unsigned)srq->wqe_shift)); +} + +int udma_srq_init_attr(struct udma_device *udma_dev, struct udma_srq *srq, + u32 depth, u8 max_sge); +int udma_index_queue_init_attr(struct udma_device *udma_dev, + struct udma_index_queue *index_queue, u32 depth); +int udma_create_srqc_update(struct udma_device *udma_dev, struct udma_srq *srq, + struct udma_db *sdb); +int udma_delete_srqc_and_cache_out(struct udma_device *udma_dev, + struct udma_srq *srq); +int udma_srq_init_res(struct udma_device *udma_dev, struct udma_srq *srq, + struct tag_cqm_queue *buf_info); +void udma_free_srq_wqe(struct udma_srq *srq, u32 container_idx); +int udma_modify_srq_threshold(struct udma_device *udma_dev, + struct udma_srq *srq, u8 container_warn_th); +int udma_post_srq_wr(struct udma_device *udma_dev, struct udma_srq *srq, + const struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); +u16 udma_get_next_srq_wqe(struct udma_srq *srq, u16 cur_idx); + +int udma_srq_pre_post_wqe_check(struct udma_device *udma_dev, + const struct udma_srq *srq, + const struct ubcore_jfr_wr *wr); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c new file mode 100644 index 000000000..d9ababf90 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.c @@ -0,0 +1,1049 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <asm/io.h> +#include <linux/vmalloc.h> + +#include "ubcore_types.h" +#include "hinic5_cqm.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_mtt.h" +#include "udma_jetty.h" +#include "udma_cmd_com.h" +#include "udma_cmd_jfrc.h" +#include "udma_xid_cfg.h" +#include "udma_resource.h" +#include "udma_fast_msg.h" +#include "udma_tpg.h" +#include "udma_vtp.h" +#include "udma_rc_table.h" + +#define UB_DMA_JETTY 0 //reserved xid 0 +#define UB_RC_QUEUE_CONTEXT_SIZE (512) + +//current temporarily not support dynamic configuration +#define UB_MIN_RC_NUM 16 +#define UB_MID_RC_NUM 32 +#define MIN_SLICE_SIZE (32 * 1024) +#define SLICE_EN_FAIL 0 +#define SLICE_EN_OK 1 + +#define UB_MAX_CNT_SIZE_SHIFT 10 //1k granularity align + +/* compatible A5 use of urma tag */ +union udma_device_cfg_feature { + struct { + uint32_t taack_flush : 1; /* enable sending TAACK after data written into memory */ + uint32_t inline_reduce : 1; /* enable inline reduce */ + uint32_t npu_direct_urma : 1; /* enable NPU-direct URMA */ + uint32_t bus_multipath : 1; /* enable multipath transport in bus */ + uint32_t reserved : 28; + } bs; + uint32_t value; +}; + +static void +udma_config_device_init_attr(struct udma_device *udma_dev, + const struct ub_cmd_func_attr *func_attr); + +static int udma_create_rc_queue_ctx(struct udma_rc_queue *udma_rcq, + struct udma_device *udma_dev, u32 rc_cnt) +{ + udma_rcq->cqm_priv_type = UDMA_JETTY_TYPE_RC; + udma_rcq->rc_ctx = cqm5_object_qpc_mpt_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_SERVICE_CTX, + UB_RC_QUEUE_CONTEXT_SIZE, udma_rcq, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_NONE, + 0), + 0, rc_cnt); + if (udma_rcq->rc_ctx == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc context memory allocation failed\n"); + return -ENOMEM; + } + + return 0; +} + +int udma_alloc_rcq_resource(struct udma_device *udma_dev, + cqm_queue_s **cqm_buf_info, void *object_priv, + u32 buf_len, struct queue_addr_info *queue_addr) +{ + u32 queue_len; + int ret; + + *cqm_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, PAGE_SIZE, + object_priv, true, CQM_INDEX_INVALID, 0, 0); + if (*cqm_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "Failed to create queue by cqm object.\n"); + return -ENOMEM; + } + + queue_len = udma_cal_page_size(buf_len); + ret = hinic5_dma_zalloc_coherent_align(udma_dev->dev, queue_len, + queue_len, GFP_KERNEL, + &queue_addr->mem_align); + if (ret != 0) { + cqm5_object_delete(&(*cqm_buf_info)->object); + *cqm_buf_info = NULL; + udma_err(udma_dev->dev, UDMA_M_JFRC, + "Failed to alloc dma addr, ret = %d\n", ret); + return -ENOMEM; + } + udma_fill_kernel_queue_addr_info(queue_len, queue_addr); + + return 0; +} + +static int udma_create_rc_queue_buf_info(struct udma_rc_queue *udma_rcq, + struct udma_device *udma_dev, + u32 total_depth) +{ + u32 page_size; + u32 rcqe_size; + int ret = 0; + + rcqe_size = UB_RC_QUEUE_WQEBB_SIZE * UB_PER_RCQE_WQEBB_NUM; + page_size = udma_cal_page_size(total_depth * rcqe_size); + + udma_rcq->total_depth = total_depth < (page_size / rcqe_size) ? + (page_size / rcqe_size) : + ((u32)roundup_pow_of_two(total_depth)); + udma_rcq->total_size = udma_rcq->total_depth * rcqe_size; + ret = udma_alloc_rcq_resource(udma_dev, &udma_rcq->rc_queue, udma_rcq, + udma_rcq->total_size, + &udma_rcq->queue_addr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "alloc queue resource failed, ret = %d\n", ret); + return ret; + } + + udma_rcq->rcq_va = udma_rcq->queue_addr.va; + return ret; +} + +int udma_free_rc_queue_resource(struct udma_device *udma_dev, + cqm_queue_s *cqm_buf_info, + struct queue_addr_info *queue_addr) +{ + hinic5_dma_free_coherent_align(udma_dev->dev, &queue_addr->mem_align); + if (cqm_buf_info != NULL) { + cqm5_object_delete(&cqm_buf_info->object); + } + + return 0; +} + +static int udma_create_rc_queue_pcqc_info(struct udma_rc_queue *udma_rcq, + struct udma_device *udma_dev) +{ + udma_rcq->pcqc = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_SCQ, 0, udma_rcq, + false, udma_rcq->rc_ctx->xid, 0, 0); + if (udma_rcq->pcqc == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc pcqc memory allocation failed\n"); + return -ENOMEM; + } + + return 0; +} + +static int udma_create_jfrc_context(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + int ret; + + ret = udma_alloc_opt_ubrc(udma_dev, &udma_rcq->rsp_depth, + &udma_rcq->ubrc, true); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to alloc ubrc, ret(%d)\n", ret); + return ret; + } + + ret = udma_cmd_jfrc_create_ctx(udma_dev, udma_rcq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "cmd jfrc create failed, ret(%d)\n", ret); + udma_free_opt_ubrc(udma_dev, &udma_rcq->rsp_depth, + &udma_rcq->ubrc); + } + + return ret; +} + +/** + * input @total_depth: RC queue of total depth + * input @index: struct udma_device in rc_queue member address of index + * input @rc_cnt: need created rc queue of count + */ +static int udma_create_one_rc_queue(struct udma_device *udma_dev, + u32 total_depth, int index, u32 rc_cnt) +{ + struct udma_rc_queue *udma_rcq = &udma_dev->rc_queue[index]; + int ret; + + /* udma_rcq in caller allocate memory when already ensure non- NULL, not again repeat check */ + ret = udma_create_rc_queue_ctx(udma_rcq, udma_dev, rc_cnt); + if (ret != 0) { + return ret; + } + + ret = udma_create_rc_queue_buf_info(udma_rcq, udma_dev, total_depth); + if (ret != 0) { + goto err_queue_create; + } + + (void)memset_s(udma_rcq->rcq_va, + (size_t)((unsigned)udma_rcq->total_size), 0, + (size_t)((unsigned)udma_rcq->total_size)); + udma_rcq->rc_queue->q_header_vaddr->doorbell_record = 0; + udma_cmdq_rtt_statistic_init(&udma_rcq->cmdq_rtt); + + ret = udma_fill_kernel_mtt(&udma_rcq->ub_mtt, udma_rcq->queue_addr.pa); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "Failed to init kernel mtt, ret = %d.\n", ret); + goto err_init_kernel_mtt; + } + + ret = udma_create_rc_queue_pcqc_info(udma_rcq, udma_dev); + if (ret != 0) { + goto err_init_kernel_mtt; + } + + ret = udma_create_jfrc_context(udma_dev, udma_rcq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "udma jfrc create failed, ret(%d)\n", ret); + goto err_create_cmdq; + } + + udma_store_list(udma_dev, &udma_rcq->node, + &udma_dev->device_res->rc_list); + udma_info(udma_dev->dev, UDMA_M_JFRC, + "index(%d) depth(%u) rc_id(%u) pcqc_id(%u) success\n", index, + udma_rcq->total_depth, udma_rcq->rc_ctx->xid, + udma_rcq->pcqc->index); + + return ret; + +err_create_cmdq: + cqm5_object_delete(&udma_rcq->pcqc->object); +err_init_kernel_mtt: + udma_free_rc_queue_resource(udma_dev, udma_rcq->rc_queue, + &udma_rcq->queue_addr); + udma_rcq->rc_queue = NULL; +err_queue_create: + cqm5_object_delete(&udma_rcq->rc_ctx->object); + + return ret; +} + +struct udma_rc_queue *udma_find_rcq(struct udma_device *udma_dev, u32 rcq_id) +{ + struct udma_rc_queue *rcq = NULL; + struct mutex *lock = NULL; + + lock = &udma_dev->device_res->rc_list.node_lock; + mutex_lock(lock); + list_for_each_entry(rcq, &udma_dev->device_res->rc_list.node, node) { + if (rcq->rc_ctx->xid == rcq_id) { + mutex_unlock(lock); + return rcq; + } + } + mutex_unlock(lock); + + return NULL; +} + +static int udma_create_rcq_pre_check(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + /* if already create RC queue, then not again repeat create */ + if (udma_dev->is_rc_queue_configured != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc queue has already configured\n"); + return -EALREADY; + } + + if (cfg->rc_cfg.rc_cnt > + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "ubcore rc_cnt(%u) excceeds max_rc(%u)\n", + cfg->rc_cfg.rc_cnt, + udma_dev->ub_cap.dev_cap_res.udma_res.max_jfrc_num); + return -EINVAL; + } + + if (cfg->rc_cfg.depth > + udma_dev->ub_cap.dev_cap_res.udma_res.jfrc_depth) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "ubcore rc_depth(%u) excceeds max rc depth(%u)\n", + cfg->rc_cfg.depth, + udma_dev->ub_cap.dev_cap_res.udma_res.jfrc_depth); + return -EINVAL; + } + + return 0; +} + +static int udma_jfrc_queue_free(struct udma_device *udma_dev, + struct udma_rc_queue *udma_rcq) +{ + int ret; + + ret = udma_cmd_jfrc_delete_ctx(udma_dev, udma_rcq); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "rc queue delete cmd failed, rcq_id(%u), ret(%d)\n", + udma_rcq->rc_ctx->xid, ret); + return ret; + } + + ret = udma_cmd_jfrc_cache_invalidate(udma_dev, udma_rcq); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFRC, + "rc queue cache invalidate cmd failed, rcq_id(%u), ret(%d)\n", + udma_rcq->rc_ctx->xid, ret); + return ret; + } + + udma_free_opt_ubrc(udma_dev, &udma_rcq->rsp_depth, &udma_rcq->ubrc); + cqm5_object_delete(&udma_rcq->pcqc->object); + udma_free_rc_queue_resource(udma_dev, udma_rcq->rc_queue, + &udma_rcq->queue_addr); + udma_rcq->rc_queue = NULL; + cqm5_object_delete(&udma_rcq->rc_ctx->object); + + return ret; +} + +/** + * input @index: struct udma_device in rc_queue member address of index +*/ +static int udma_delete_one_rc_queue(struct udma_device *udma_dev, int index) +{ + struct udma_rc_queue *udma_rcq = &udma_dev->rc_queue[index]; + + udma_delete_list(udma_dev, &udma_rcq->node, + &udma_dev->device_res->rc_list); + + return udma_jfrc_queue_free(udma_dev, udma_rcq); +} + +static int udma_set_rc_cnt(struct udma_device *udma_dev) +{ + u16 func_id = hinic5_global_func_id(udma_dev->hwdev); + struct ub_cmd_func_attr func_attr = { 0 }; + + if (udma_dev->ub_dev.attr.tp_maintainer == 1) { + udma_info(udma_dev->dev, UDMA_M_JFRC, + "tpf not support set rc cnt\n"); + return 0; + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr1, rcq_cnt, + (u32)ilog2(udma_dev->rc_queue_cnt)); + udma_info(udma_dev->dev, UDMA_M_JFRC, + "func_id:%u mask 0x%x,rcq_cnt:%u=ilog2(%u)", func_id, + func_attr.attr1_mask.bs.rcq_cnt, func_attr.attr1.rcq_cnt, + udma_dev->rc_queue_cnt); + return udma_set_func_attr(udma_dev, func_id, &func_attr); +} + +static u32 udma_get_rc_cnt(u32 rc_cnt) +{ + return UB_MAX_RC_NUM; +} + +int udma_create_rc_table(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + u32 cur_idx = 0, i; + int ret; + + /* 1.mask check 2.tpf not reserved 0 1 and create rc */ + if (cfg->mask.bs.rc_cnt == 0 || cfg->mask.bs.rc_depth == 0 || + udma_dev->ub_dev.attr.tp_maintainer || + cfg->rc_cfg.rc_cnt < UB_MIN_RC_NUM) { + /* not create RC, normal return, not affect outside layer function */ + udma_info( + udma_dev->dev, UDMA_M_JFRC, + "mask rc_cnt(%u) or rc_depth(%u) = 0, or is tpf(%u), or rc_cnt(%u) < 4\n", + cfg->mask.bs.rc_cnt, cfg->mask.bs.rc_depth, + udma_dev->ub_dev.attr.tp_maintainer, + cfg->rc_cfg.rc_cnt); + return 0; + } + mutex_lock(&udma_dev->dev_mutex); + ret = udma_create_rcq_pre_check(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "create rc table pre check failed\n"); + goto end; + } + + udma_dev->rc_queue_cnt = udma_get_rc_cnt(cfg->rc_cfg.rc_cnt); + udma_dev->rc_queue = (struct udma_rc_queue *)devm_kzalloc( + udma_dev->dev, + sizeof(struct udma_rc_queue) * udma_dev->rc_queue_cnt, + GFP_KERNEL); + if (udma_dev->rc_queue == NULL) { + ret = -ENOMEM; + udma_err(udma_dev->dev, UDMA_M_JFRC, + "create rc table: memory allocation failed\n"); + goto end; + } + + for (cur_idx = 0; cur_idx < udma_dev->rc_queue_cnt; cur_idx++) { + ret = udma_create_one_rc_queue(udma_dev, cfg->rc_cfg.depth, + (int)cur_idx, + udma_dev->rc_queue_cnt); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_JFRC, + "udma_create_one_rc_table iteration(%u) failed\n", + cur_idx); + goto err_udma_create_one_rc_queue; + } + } + ret = udma_set_rc_cnt(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_JFRC, "set rc cnt failed\n"); + goto err_udma_create_one_rc_queue; + } + udma_dev->is_rc_queue_configured = true; + udma_info(udma_dev->dev, UDMA_M_JFRC, "create rc table success\n"); + goto end; + +err_udma_create_one_rc_queue: + /* free already allocate of rc resource */ + for (i = 0; i < cur_idx; ++i) { + if (udma_delete_one_rc_queue(udma_dev, (int)i) != 0) + udma_err(udma_dev->dev, UDMA_M_JFRC, + "delete one rc queue failed\n"); + } + + udma_dev->rc_queue_cnt = 0; + udma_dev->is_rc_queue_configured = false; + + devm_kfree(udma_dev->dev, udma_dev->rc_queue); + udma_dev->rc_queue = NULL; + +end: + mutex_unlock(&udma_dev->dev_mutex); + + return ret; +} + +static int udma_set_suspend_para(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + udma_info(udma_dev->dev, UDMA_M_TP, + "cfg_suspend_period %x, cfg_suspend_cnt %x, cfg_mask %x.\n", + cfg->suspend_period, cfg->suspend_cnt, cfg->mask.value); + + if (cfg->mask.bs.suspend_period == 0 || cfg->mask.bs.suspend_cnt == 0) { + udma_info(udma_dev->dev, UDMA_M_TP, + "mask %x, suspend_period or suspend_cnt is zero.\n", + cfg->mask.value); + return 0; + } + + if (cfg->suspend_period == 0 || cfg->suspend_cnt == 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "suspend_period %x, suspend_cnt %x.\n", + cfg->suspend_period, cfg->suspend_cnt); + return -EINVAL; + } + + udma_info(udma_dev->dev, UDMA_M_TP, + "suspend_period %x, suspend_cnt %x.\n", cfg->suspend_period, + cfg->suspend_cnt); + + udma_dev->suspend_period = cfg->suspend_period; + udma_dev->suspend_cnt = cfg->suspend_cnt; + + return 0; +} + +static int udma_set_plug_attach_en(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg, + bool res_clear) +{ + union ubcore_device_cfg_mask mask = cfg->mask; + void *hwdev = udma_dev->hwdev; + u8 attach_en; + int ret; + + udma_debug( + udma_dev->dev, UDMA_M_MAIN, + "start to set attach_en flag, func_id:%u, mask:%u, clear:%u\n", + adapted_ue_idx(cfg), mask.value, (u8)res_clear); + + if (!hinic5_is_master_host(hwdev)) { + udma_debug(udma_dev->dev, UDMA_M_MAIN, + "no need to set attach_en flag\n"); + return 0; + } + + if (res_clear) { + attach_en = 0; + if (UDMA_IS_MIG_VFID(adapted_ue_idx(cfg))) { + udma_vf_migrate_end_proc( + udma_dev, + adapted_ue_idx( + cfg)); //vport delete scenario, force cleanup live migration vtp buffer + } + } else if ((mask.bs.pattern | mask.bs.virtualization | + mask.bs.min_jetty_cnt | mask.bs.max_jetty_cnt | + mask.bs.min_jfr_cnt | mask.bs.max_jfr_cnt) == 1) { + attach_en = 1; + } else { + udma_debug(udma_dev->dev, UDMA_M_MAIN, + "no need to set attach_en flag\n"); + return 0; + } + + ret = hisdk5_set_plug_srv_bitmap(hwdev, COMM_PLUG_SRV_UB, + adapted_ue_idx(cfg), attach_en); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "set plug service bitmap failed, ret:%d\n", ret); + return ret; + } + + udma_info(udma_dev->dev, UDMA_M_MAIN, + "set plug service bitmap success, attach_en:%u\n", attach_en); + return 0; +} + +static int udma_fill_func_attr(struct udma_device *udma_dev, + struct ubcore_device_cfg *cfg, + struct ub_cmd_func_attr *func_attr, + bool res_clear) +{ + int ret = 0; + u8 cos = 0; + u32 slice = UB_SLICE_NUM64K; + u32 pattern = (cfg->pattern == UBCORE_PATTERN_3) ? PATTERN_TYPE_3 : + PATTERN_TYPE_1; + u32 fast_path_timer_disable = (adapted_ue_idx(cfg) < PF_MAX_NUM) ? + g_udma_fast_path_timer_disable : + 0; + union udma_device_cfg_feature feature = { 0 }; + + if (cfg->mask.bs.slice == 1) { + if (udma_dev->slice_cfg_en == SLICE_EN_OK && + cfg->slice == UB_SLICE_NUM128K) { + slice = cfg->slice; + } + slice = (u32)ilog2(slice / MIN_SLICE_SIZE); + if (res_clear == false) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr1, + slice, slice); + udma_debug(udma_dev->dev, UDMA_M_RC_TABLE, + "func_attr->slice:%u\n", + func_attr->attr1.slice); + } + } + if (cfg->mask.bs.virtualization == 1) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR( + func_attr, attr1, virtualization, cfg->virtualization); + } + if (cfg->mask.bs.pattern == 1) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr1, + pattern_type, pattern); + } + if (cfg->mask.bs.ta2tp_sched_alg == 1) { + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR( + func_attr, attr2, ta2tp_type, cfg->ta2tp_sched_alg); + } + + if (cfg->mask.bs.feature == 1) { + feature.value = cfg->feature.value; + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, + taack_flush_en, + feature.bs.taack_flush); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, reduce_en, + feature.bs.inline_reduce); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, + bus_multi_path_en, + feature.bs.bus_multipath); + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, hbm_en, + feature.bs.npu_direct_urma); + } + + if (cfg->mask.bs.dscp == 1) { + ret = udma_get_cos_by_pri(udma_dev, (u8)cfg->dscp, &cos); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "Failed to get cos from dcb info, ret=%d, dscp:%u\n", + ret, cfg->dscp); + return ret; + } + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr2, func_cos, + cos); + udma_debug(udma_dev->dev, UDMA_M_RC_TABLE, + "func_attr->func_cos:%d\n", + func_attr->attr2.func_cos); + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK_PTR(func_attr, attr1, + fast_path_timer_disable, + fast_path_timer_disable); + + return 0; +} + +static int udma_set_func_table(struct udma_device *udma_dev, + struct ubcore_device_cfg *cfg, bool res_clear) +{ + struct udma_device *udma_dev_update = NULL; + struct ub_cmd_func_attr func_attr = { 0 }; + int ret = 0; + + ret = udma_fill_func_attr(udma_dev, cfg, &func_attr, res_clear); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "fill func attr failed\n"); + return ret; + } + + ret = udma_set_func_attr(udma_dev, adapted_ue_idx(cfg), &func_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "mpu set func table failed\n"); + return ret; + } + + udma_dev_update = + udma_find_dev_by_fe_idx(udma_dev, adapted_ue_idx(cfg)); + if (udma_dev_update != NULL) { + /* config dev configured fe and tpf is in same a host on, update this fe dev attribute */ + udma_config_device_init_attr(udma_dev_update, &func_attr); + } + + return 0; +} + +static int udma_check_nic_plug_attach_en(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + u8 attach_en = 0; + int ret; + + u16 ue_idx = adapted_ue_idx(cfg); + if ((ue_idx < PF_MAX_NUM) || (ue_idx >= UB_VF_NUM_MAX)) { + udma_debug(udma_dev->dev, UDMA_M_MAIN, "func %d is not ub vf\n", + ue_idx); + return 0; + } + + if (!hinic5_is_master_host(udma_dev->hwdev)) { + udma_debug(udma_dev->dev, UDMA_M_MAIN, + "no need to get attach_en flag\n"); + return 0; + } + + /* prevent呆, check nic whether be ovs enable, use nic of srv type */ + ret = hisdk5_get_plug_srv_bitmap(udma_dev->hwdev, COMM_PLUG_SRV_NIC, + ue_idx, &attach_en); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "get plug srv bitmap failed, ret:%d\n", ret); + return ret; + } + + if (attach_en == 0) { + udma_err(udma_dev->dev, UDMA_M_MAIN, + "attach_en: %d flag is invalid.\n", attach_en); + return -EFAULT; + } + + return 0; +} + +static int udma_set_func_cfg(struct udma_device *udma_dev, + struct ubcore_device_cfg *cfg, bool res_clear) +{ + int ret = 0; + + /* same step 1823 DTS2024122616167 problem single */ + ret = udma_check_nic_plug_attach_en(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "get plug srv bitmap failed, ret: %d, func id:%d\n", + ret, adapted_ue_idx(cfg)); + return ret; + } + + ret = udma_set_func_table(udma_dev, cfg, res_clear); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "set func table failed\n"); + return ret; + } + + ret = udma_vm_glb_xid_cfg_set(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "vm global xid cfg set failed\n"); + return ret; + } + + ret = udma_set_plug_attach_en(udma_dev, cfg, res_clear); + if (ret != 0) { + return ret; + } + + return 0; +} + +static void +udma_config_device_init_attr(struct udma_device *udma_dev, + const struct ub_cmd_func_attr *func_attr) +{ + if (func_attr->attr1_mask.bs.virtualization != 0) + udma_dev->ub_cap.dev_cap.virtualization = + func_attr->attr1.virtualization; + + if (func_attr->attr1_mask.bs.slice != 0) { + udma_dev->slice = func_attr->attr1.slice; + } + + if (func_attr->attr1_mask.bs.pattern_type != 0) { + udma_dev->ub_cap.dev_cap.pattern_type = + func_attr->attr1.pattern_type; + udma_dev->ub_dev.cfg.pattern = + (udma_dev->ub_cap.dev_cap.pattern_type == + PATTERN_TYPE_3) ? + UBCORE_PATTERN_3 : + UBCORE_PATTERN_1; + udma_dev->ub_dev.attr.pattern = udma_dev->ub_dev.cfg.pattern; + } + + if (func_attr->attr2_mask.bs.func_cos != 0) { + udma_dev->default_cos.func_cos = func_attr->attr2.func_cos; + } + + if (func_attr->attr2_mask.bs.bus_multi_path_en != 0) { + udma_dev->ub_cap.dev_cap.dcs_en = + func_attr->attr2.bus_multi_path_en; + } + + if (func_attr->attr2_mask.bs.hbm_en != 0 && + udma_dev->ub_cap.dev_cap.hbm_en != func_attr->attr2.hbm_en) { + udma_dev->ub_cap.dev_cap.hbm_en = func_attr->attr2.hbm_en; + udma_dev->ub_cap.dev_cap.dcs_en = func_attr->attr2.hbm_en; + } +} + +static int udma_cfg_max_cnt_check(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg) +{ + /* passed in of max_jetty_cnt need is 1k of multiple data */ + if (cfg->mask.bs.max_jetty_cnt == 1 && + ((cfg->max_jetty_cnt & ((1U << UB_MAX_CNT_SIZE_SHIFT) - 1)) != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "max_jetty_cnt should be a multiple of 1k. cur_value: %d.\n", + cfg->max_jetty_cnt); + return -EINVAL; + } + if (cfg->mask.bs.max_jfr_cnt == 1 && + ((cfg->max_jfr_cnt & ((1U << UB_MAX_CNT_SIZE_SHIFT) - 1)) != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RC_TABLE, + "max_jfr_cnt should be a multiple of 1k. cur_value: %d.\n", + cfg->max_jfr_cnt); + return -EINVAL; + } + return EOK; +} + +int udma_config_device(struct ubcore_device *ub_dev, + struct ubcore_device_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + bool res_clear = false; + int ret; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err( + UDMA_M_RC_TABLE, + "%s: parameters is incorrect, ub_dev(%d), cfg(%d).\n", + __func__, !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_cfg_max_cnt_check(udma_dev, cfg) != EOK) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "input cfg, max_cnt check failed!\n"); + return -EINVAL; + } + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "slice: %u virtualization:%u pattern:%u mask:%u\n", + cfg->slice, cfg->virtualization, cfg->pattern, + cfg->mask.value); + + ret = udma_create_rc_table(udma_dev, cfg); + if (ret != 0) { + return ret; + } + + ret = udma_set_suspend_para(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "tp suspend para set failed\n"); + goto err_xid_cfg_set; + } + + if (udma_dev->ub_dev.attr.virtualization) { + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "virtualization is 1, return 0\n"); + return 0; + } + + if (cfg->mask.bs.max_jetty_cnt == 1 && cfg->max_jetty_cnt == 0) { + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, + "virtual clear filter resource\n"); + res_clear = true; + } + + /* inline_reduce of enable depend npu_direct_urma of enable */ + if (cfg->feature.bs.inline_reduce == 1 && + cfg->feature.bs.npu_direct_urma == 0) { + udma_warn( + udma_dev->dev, UDMA_M_RC_TABLE, + "inline reduce enable invalid, as npu_direct_urma disable!\n"); + } + + ret = udma_set_func_cfg(udma_dev, cfg, res_clear); + if (ret != 0) { + goto err_xid_cfg_set; + } + + /* create reserved TP lock store of TPG */ + ret = udma_create_tp_latch_tpg(udma_dev); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "TP latch create fail ret=%d.\n", ret); + } + + return ret; + +err_xid_cfg_set: + if (cfg->mask.bs.rc_cnt != 0 && cfg->mask.bs.rc_depth != 0) + udma_delete_rc_table(udma_dev); + + return ret; +} + +void udma_delete_rc_table(struct udma_device *udma_dev) +{ + u32 i; + + /* tpf did not create RC */ + if (udma_dev->ub_dev.attr.tp_maintainer) { + return; + } + + if (udma_dev->is_rc_queue_configured == 0 || + udma_dev->rc_queue_cnt == 0 || udma_dev->rc_queue == NULL) { + udma_warn(udma_dev->dev, UDMA_M_RC_TABLE, + "no rc queue to delete\n"); + return; + } + + for (i = 0; i < udma_dev->rc_queue_cnt; ++i) { + if (udma_delete_one_rc_queue(udma_dev, (int)i) != 0) + udma_err(udma_dev->dev, UDMA_M_JFRC, + "delete one rc queue failed\n"); + } + + udma_dev->rc_queue_cnt = 0; + udma_dev->is_rc_queue_configured = false; + + devm_kfree(udma_dev->dev, udma_dev->rc_queue); + udma_dev->rc_queue = NULL; + + udma_info(udma_dev->dev, UDMA_M_RC_TABLE, "delete rc table success\n"); +} + +/** + * func: dfx tool from cache query target rc_id of rc queue of rc context + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc context content +*/ +int udma_query_rc_queue_ctx_from_cache(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size) +{ + return udma_cmd_jfrc_query_qpc_ctx(udma_dev, rc_id, outbuf); +} + +/** + * func: dfx tool from host memory query target rc_id of rc queue of rc context + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc context content +*/ +int udma_query_rc_queue_ctx_from_host(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + struct tag_cqm_object *cqm_obj = NULL; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + rc_id, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + rcq_ctx = container_of(cqm_obj, struct tag_cqm_qpc_mpt, object); + + memcpy_fromio((void *)outbuf, (void *)rcq_ctx->vaddr, outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_JFRC, "success, rc_id(%u)\n", rc_id); + + cqm5_object_put(cqm_obj); + + return 0; +} + +/** + * func: dfx tool from cache query target rc_id of rc queue of pcqc content + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc queue pcqc content +*/ +int udma_query_rc_queue_pcqc_from_cache(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size) +{ + return udma_cmd_jfrc_query_pcqc_ctx(udma_dev, rc_id, outbuf); +} + +/** + * func: dfx tool from host memory query target rc_id of rc queue of pcqc content + * input @rc_id: NA + * input @outbuf_size: NA + * output @outbuf: stores queried rc queue pcqc content +*/ +int udma_query_rc_queue_pcqc_from_host(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_queue *pcqc = NULL; + struct tag_cqm_object *cqm_obj = NULL; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_RDMA_SCQ, rc_id, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + pcqc = container_of(cqm_obj, struct tag_cqm_queue, object); + + memcpy_fromio((void *)outbuf, pcqc->q_ctx_vaddr, outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_JFRC, "success, rc_id(%u)\n", rc_id); + + cqm5_object_put(cqm_obj); + + return 0; +} + +/** + * func: dfx tool from cache query target rc_id of rc table ci index of command content + * input @rc_id: NA + * input @ci: target consumer index + * input @outbuf_size: NA + * output @outbuf: NA +*/ +int udma_query_rc_queue_wqe_from_cache(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size) +{ + return udma_cmd_jfrc_query_wqe(udma_dev, rc_id, ci, outbuf); +} + +static inline struct udma_rc_queue * +cqmobj_to_ub_rc_queue(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *rcq_ctx = NULL; + rcq_ctx = container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_rc_queue *)rcq_ctx->priv; +} + +/** + * func: dfx tool from host query target rc_id of rc table ci index of command content + * input @rc_id: NA + * input @ci: target consumer index + * input @outbuf_size: NA + * output @outbuf: NA +*/ +int udma_query_rc_queue_wqe_from_host(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_rc_queue *udma_rcq = NULL; + u64 rcqe_offset; // target ci of address offset + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + rc_id, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "failed to get cqm object, rc_id(%u)\n", rc_id); + return -EINVAL; + } + + udma_rcq = cqmobj_to_ub_rc_queue(cqm_obj); + if (udma_rcq->cqm_priv_type != UDMA_JETTY_TYPE_RC) { + udma_err(udma_dev->dev, UDMA_M_JFRC, + "cqmobj is not rc, rc_id(%u), type(%u)\n", rc_id, + udma_rcq->cqm_priv_type); + ret = -EINVAL; + goto cleanup; + } + + if (ci >= udma_rcq->total_depth) { + udma_err( + udma_dev->dev, UDMA_M_JFRC, + "ci exceeds the maximum depth, rc_id(%u) ci(%u) total_depth(%u)\n", + rc_id, ci, udma_rcq->total_depth); + ret = -EINVAL; + goto cleanup; + } + + rcqe_offset = ci * outbuf_size; + memcpy_fromio((void *)outbuf, udma_rcq->rcq_va + rcqe_offset, + outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_JFRC, "success, rc_id(%u) ci(%u)\n", + rc_id, ci); +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h new file mode 100644 index 000000000..9551b8d90 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/rc_table/udma_rc_table.h @@ -0,0 +1,46 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef _UDMA_RC_TABLE_H +#define _UDMA_RC_TABLE_H + +#include <linux/mutex.h> +#include "ubcore_api.h" +#include "ub_npu_jfrc_cmd_defs.h" +#include "ub_npu_jetty_cmd_defs.h" +#include "hmm_common.h" +#include "hinic5_cqm.h" +#include "udma_common.h" + +#define UB_SLICE_NUM64K 65536 +#define UB_SLICE_NUM128K 131072 +enum { PATTERN_TYPE_1 = 0x0, PATTERN_TYPE_2 = 0x1, PATTERN_TYPE_3 = 0x2 }; + +extern u32 g_udma_fast_path_timer_disable; + +int udma_config_device(struct ubcore_device *ub_dev, + struct ubcore_device_cfg *cfg); +int udma_create_rc_table(struct udma_device *udma_dev, + const struct ubcore_device_cfg *cfg); +void udma_delete_rc_table(struct udma_device *udma_dev); +int udma_query_rc_queue_ctx_from_cache(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_ctx_from_host(struct udma_device *udma_dev, u32 rc_id, + struct ub_ta_context_query *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_pcqc_from_cache(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_pcqc_from_host(struct udma_device *udma_dev, u32 rc_id, + ub_pcqc_query_rsp_s *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_wqe_from_cache(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size); +int udma_query_rc_queue_wqe_from_host(struct udma_device *udma_dev, u32 rc_id, + u32 ci, struct ub_rc_queue_wqe *outbuf, + u32 outbuf_size); +struct udma_rc_queue *udma_find_rcq(struct udma_device *udma_dev, u32 rcq_id); +#endif /* _UDMA_RC_TABLE_H */ \ No newline at end of file diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c new file mode 100644 index 000000000..8e9b63ace --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.c @@ -0,0 +1,73 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "udma_key.h" + +struct ubcore_token_id *udma_alloc_key_id(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_keyid *udma_key = NULL; + int ret; + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_KEY, "Invalid parameter: ub_dev is null.\n"); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + udma_key = (struct udma_keyid *)kzalloc(sizeof(struct udma_keyid), + GFP_KERNEL); + if (udma_key == NULL) { + udma_err(udma_dev->dev, UDMA_M_KEY, + "allocate key id memory failed.\n"); + return ERR_PTR(-ENOMEM); + } + mutex_init(&udma_key->mpt_mutex); + + ret = hmm_mpt_alloc(udma_dev->hwdev, &udma_key->mpt, SERVICE_T_UB); + if (ret != 0) { + kfree(udma_key); + udma_err(udma_dev->dev, UDMA_M_KEY, + "create key id mpt failed, ret:%d\n", ret); + return ERR_PTR(-ENOMEM); + } + + udma_key->base.ub_dev = ub_dev; + udma_key->base.uctx = udma_get_uctx(udata); + udma_key->base.token_id = udma_key->mpt.mpt_index; + + udma_debug(udma_dev->dev, UDMA_M_KEY, "allocate key id:%u success.\n", + udma_key->base.token_id); + + return &udma_key->base; +} + +int udma_free_key_id(struct ubcore_token_id *token_id) +{ + struct udma_device *udma_dev = NULL; + struct udma_keyid *udma_key = NULL; + u32 id; + + if ((token_id == NULL) || (token_id->ub_dev == NULL)) { + udma_pr_err(UDMA_M_KEY, "Invalid parameter: token_id(%d).\n", + !!token_id); + return -EINVAL; + } + + id = token_id->token_id; + udma_dev = to_udma_dev(token_id->ub_dev); + udma_key = to_udma_keyid(token_id); + mutex_lock(&udma_key->mpt_mutex); + hmm_mpt_free(udma_dev->hwdev, &udma_key->mpt); + mutex_unlock(&udma_key->mpt_mutex); + kfree(udma_key); + + udma_info(udma_dev->dev, UDMA_M_KEY, "free key id:%u success.\n", id); + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h new file mode 100644 index 000000000..fcdb3cd18 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_key.h @@ -0,0 +1,29 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_KEY_H +#define UDMA_KEY_H + +#include "ubcore_types.h" +#include "udma_common.h" + +typedef struct udma_keyid { + struct ubcore_token_id base; + struct mpt mpt; + struct mutex mpt_mutex; +} udma_keyid_t; + +struct ubcore_token_id *udma_alloc_key_id(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata); +int udma_free_key_id(struct ubcore_token_id *token_id); + +static inline struct udma_keyid * +to_udma_keyid(const struct ubcore_token_id *key_id) +{ + return (struct udma_keyid *)udma_container_of(key_id, struct udma_keyid, + base); +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c new file mode 100644 index 000000000..23d7719ec --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.c @@ -0,0 +1,788 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#include <linux/device.h> +#include <linux/iommu.h> +#include <linux/version.h> +#include <linux/vmstat.h> +#include <linux/scatterlist.h> +#include <linux/dma-mapping.h> +#include <linux/mm.h> + +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_mtt.h" +#include "udma_key.h" +#include "udma_misc.h" +#include "udma_ucontext.h" +#include "comm_defs.h" +#include "udma_jetty.h" +#include "udma_shm_manage.h" +#include "udma_cmd_segment.h" +#include "ubcnet_mpu_cmd.h" +#include "ubcnet_mpu_cmd_defs.h" +#include "udma_segment.h" + +static inline int udma_get_seg_umem(struct ubcore_device *ub_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + union ubcore_umem_flag umem_flag = {}; + + umem_flag.bs.non_pin = cfg->flag.bs.non_pin; + umem_flag.bs.writable = 1; + udma_seg->umem = ubcore_umem_get(ub_dev, cfg->va, cfg->len, umem_flag); + + if (IS_ERR_OR_NULL(udma_seg->umem)) { + udma_pr_err( + UDMA_M_SEG, + "Failed to get umem, get umem ptr err or zero ret:%d\n", + (int)PTR_ERR_OR_ZERO(udma_seg->umem)); + return -EINVAL; + } + return 0; +} + +static inline u32 udma_get_umem_page_count(const struct ubcore_umem *umem, + u32 os_page_size) +{ + return (u32)((ALIGN(umem->va + umem->length, os_page_size) - + ALIGN_DOWN(umem->va, os_page_size)) / + os_page_size); +} + +static inline u32 udma_get_key_from_index(u32 mpt_index) +{ + return (mpt_index >> SEG_KEY_RIGHT_SHIFT_OFS) | + (mpt_index << SEG_KEY_LEFT_SHIFT_OFS); +} + +static void udma_fill_udma_seg(struct udma_segment *udma_seg, + const struct ubcore_seg_cfg *cfg, + struct udma_keyid *udma_key) +{ + udma_seg->va = cfg->va; + udma_seg->size = cfg->len; + udma_seg->access = cfg->flag.bs.access; + udma_seg->key = udma_get_key_from_index( + udma_key->mpt.mpt_index); /* by mpt index convert to key */ + udma_seg->key_type = cfg->flag.bs.token_policy; + udma_seg->token_value = cfg->token_value.token; + udma_seg->mpt = &udma_key->mpt; +} + +static void udma_seg_kref_release(struct kref *ref_cnt) +{ + struct udma_segment *seg = + container_of(ref_cnt, struct udma_segment, ref_cnt); + + complete(&seg->comp); +} + +void udma_seg_put(void *obj) +{ + struct udma_segment *seg = obj; + + if (seg != NULL) { + kref_put(&seg->ref_cnt, udma_seg_kref_release); + } +} + +void udma_seg_get(void *obj) +{ + struct udma_segment *seg = obj; + + kref_get(&seg->ref_cnt); +} + +struct udma_segment *udma_seg_find(struct udma_device *udma_dev, u32 key_id) +{ + struct udma_segment *udma_seg = NULL; + + mutex_lock(&udma_dev->device_res->seg_list.node_lock); + list_for_each_entry(udma_seg, &udma_dev->device_res->seg_list.node, + node) { + if (udma_seg->mpt->mpt_index == key_id) { + udma_seg_get(udma_seg); + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + return udma_seg; + } + } + mutex_unlock(&udma_dev->device_res->seg_list.node_lock); + + return NULL; +} + +int udma_alloc_david_fe(struct udma_device *udma_dev, u32 eid_d, u32 tid_d) +{ + ubcnet_mpu_cmd_tid_fe_op_s cmd = { 0 }; + u16 cmd_size = sizeof(cmd); + int ret; + + cmd.op_type = UBCNET_TBL_ADD; + cmd.ctrl_fe = hinic5_global_func_id(udma_dev->hwdev); + cmd.deid = eid_d; + cmd.tokenid = tid_d; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UBCNET, + UBCNET_MPU_CMD_RDMA_TID_FE_OP, &cmd, + sizeof(cmd), &cmd, &cmd_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_size == 0) || (cmd.msg_head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to alloc david fe, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, cmd.msg_head.status, cmd_size); + return -EINVAL; + } + + return 0; +} + +int udma_free_david_fe(struct udma_device *udma_dev, u32 eid_d, u32 tid_d) +{ + ubcnet_mpu_cmd_tid_fe_op_s cmd = { 0 }; + u16 cmd_size = sizeof(cmd); + int ret; + + cmd.op_type = UBCNET_TBL_DEL; + cmd.ctrl_fe = hinic5_global_func_id(udma_dev->hwdev); + cmd.deid = eid_d; + cmd.tokenid = tid_d; + + ret = hinic5_msg_to_mgmt_sync(udma_dev->hwdev, HINIC5_MOD_UBCNET, + UBCNET_MPU_CMD_RDMA_TID_FE_OP, &cmd, + sizeof(cmd), &cmd, &cmd_size, 0, + HINIC5_CHANNEL_UB); + if ((ret != 0) || (cmd_size == 0) || (cmd.msg_head.status != 0)) { + udma_err( + udma_dev->dev, UDMA_M_RESOURCE, + "Failed to free david fe, err(%d), status(0x%x), out_size(0x%x) .\n", + ret, cmd.msg_head.status, cmd_size); + return -EINVAL; + } + + return 0; +} + +int udma_david_info_init(struct udma_device *udma_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + struct mem_uva uva = { 0 }; + union acquire_attr attr = { 0 }; + int ret; + + uva.va = cfg->va; + uva.size = cfg->len; + attr.bs.require_pin = 1; + + //debug debug, remove later + if (udma_dev->sysfs_res.dfx_ctx.hbm_en == 1) { + udma_seg->uba.eid.eid = udma_dev->sysfs_res.dfx_ctx.eid; + udma_seg->uba.token_id = udma_dev->sysfs_res.dfx_ctx.tid; + goto david_fe_alloc; + } + + ret = udma_ubdevshm_acquire_uba(&udma_seg->ctx, &uva, &attr, NULL, + &udma_seg->uba); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to acquire uba, ret:%d\n", ret); + return ret; + } + +david_fe_alloc: + ret = udma_alloc_david_fe(udma_dev, udma_seg->uba.eid.eid, + udma_seg->uba.token_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to alloc david fe, ret:%d\n", ret); + goto david_fe_alloc_fail; + } + + udma_seg->david_hbm_en = true; + return 0; + +david_fe_alloc_fail: + //debug debug, remove later + if (udma_dev->sysfs_res.dfx_ctx.hbm_en == 1) { + return 0; + } + + if (udma_ubdevshm_release_uba(&udma_seg->ctx, &udma_seg->uba) != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, "failed to release uba\n"); + } + + return ret; +} + +int udma_david_info_uninit(struct udma_device *udma_dev, + struct udma_segment *udma_seg) +{ + int ret; + + ret = udma_free_david_fe(udma_dev, udma_seg->uba.eid.eid, + udma_seg->uba.token_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to release fe, ret:%d\n", ret); + return ret; + } + + //debug debug, remove later + if (udma_dev->sysfs_res.dfx_ctx.hbm_en == 1) { + return 0; + } + + ret = udma_ubdevshm_release_uba(&udma_seg->ctx, &udma_seg->uba); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to release uba, ret:%d\n", ret); + return ret; + } + + return 0; +} + +int udma_register_seg_user(struct ubcore_device *ub_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + struct udma_device *udma_dev = NULL; + u32 os_page_size = PAGE_SIZE; + u32 os_page_shift = PAGE_SHIFT; + u32 npages; + int ret; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_SEG, + "Invalid parameter: ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + udma_dev = to_udma_dev(ub_dev); + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "to_udma_dev failed.\n"); + return -EINVAL; + } + + ret = udma_get_seg_umem(ub_dev, cfg, udma_seg); + if (ret != 0) { + udma_warn(udma_dev->dev, UDMA_M_SEG, + "failed to get seg umem, ret:%d\n", ret); + return udma_david_info_init(udma_dev, cfg, udma_seg); + } + + if (udma_sysfs_get_huge_page_en(&udma_dev->sysfs_res) == 1) { + os_page_size = udma_cal_page_size(udma_seg->umem->length); + os_page_shift = (u32)ilog2(os_page_size); + } + + npages = udma_get_umem_page_count(udma_seg->umem, os_page_size); + udma_seg->mtt.mtt_type = MTT_DMTT_TYPE; + + ret = hmm_mtt_alloc(udma_dev->hwdev, npages, os_page_shift, + &udma_seg->mtt, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate mtt, ret:%d\n", ret); + goto err_alloc_seg_mem; + } + + ret = udma_umem_write_mtt(udma_dev, &udma_seg->mtt, udma_seg->umem); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d\n", ret); + goto err_write_mtt; + } + + return 0; + +err_write_mtt: + hmm_mtt_free(udma_dev->hwdev, &udma_seg->mtt, SERVICE_T_UB); + udma_seg->mtt.mtt_seg = NULL; + +err_alloc_seg_mem: + ubcore_umem_release(udma_seg->umem); + udma_seg->umem = NULL; + + return ret; +} + +static int udma_sg_set_pages(struct sg_table *sgt, u64 len, u64 npages, + void *buf, bool is_vmalloc) +{ + struct scatterlist *sg = NULL; + struct page *vm_page = NULL; + u64 sg_len = len; + size_t min; + u64 i; + + sg = &sgt->sgl[0]; + for (i = 0; i < npages; i++) { + if (is_vmalloc) { + min = min_t(size_t, sg_len, + PAGE_SIZE - offset_in_page((uintptr_t)buf)); + vm_page = vmalloc_to_page(buf); + if (!vm_page) { + return -ENOMEM; + } + + sg_set_page(sg, vm_page, (u32)min, + offset_in_page((uintptr_t)buf)); + } else { + min = min_t(size_t, sg_len, PAGE_SIZE); + sg_set_buf(sg, buf, (u32)min); + } + + buf += min; + sg_len -= min; + sg = sg_next(sg); + } + + return 0; +} + +static int udma_get_kmem(struct ubcore_device *ub_dev, + struct udma_seg_kmem *kmem, + const struct ubcore_seg_cfg *cfg) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct sg_table *sgt = &kmem->sgt; + void *buf = (void *)(uintptr_t)cfg->va; + u64 len = cfg->len; + bool is_vmalloc; + u64 npages; + int ret; + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return -EINVAL; + } + + if (udma_dev == NULL || udma_dev->dev == NULL) { + udma_pr_err( + UDMA_M_SEG, + "Invalid parameter: ub_dev(%d), udma_dev->dev(%d).\n", + !!udma_dev, !!udma_dev->dev); + return -EINVAL; + } + + is_vmalloc = is_vmalloc_addr(buf); + if (is_vmalloc) { + npages = DIV_ROUND_UP(len + offset_in_page((uintptr_t)buf), + PAGE_SIZE); + } else if (virt_addr_valid(buf) != 0) { + npages = DIV_ROUND_UP(len, PAGE_SIZE); + } else { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Error: the va buf is invalid\n"); + return -EINVAL; + } + + kmem->npages = (u32)npages; + ret = sg_alloc_table(sgt, (u32)npages, GFP_KERNEL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate table, ret:%d\n", ret); + return ret; + } + + ret = udma_sg_set_pages(sgt, len, npages, buf, is_vmalloc); + if (ret != 0) { + sg_free_table(sgt); + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to set sg pages, vm_page is NULL\n"); + return ret; + } + +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 19, 90)) + ret = dma_map_sg(ub_dev->dma_dev, sgt->sgl, sgt->nents, + DMA_BIDIRECTIONAL); +#else + ret = dma_map_sgtable(ub_dev->dma_dev, sgt, DMA_BIDIRECTIONAL, 0); +#endif + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to map sgtable, ret:%d\n", ret); + sg_free_table(sgt); + return ret; + } + + return 0; +} + +static void udma_kmem_release(struct ubcore_device *ub_dev, + struct udma_seg_kmem *kmem) +{ + if (kmem->sgt.orig_nents != 0) { +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 19, 90)) + dma_unmap_sg(ub_dev->dma_dev, kmem->sgt.sgl, kmem->sgt.nents, + DMA_BIDIRECTIONAL); +#else + dma_unmap_sgtable(ub_dev->dma_dev, &kmem->sgt, + DMA_BIDIRECTIONAL, 0); +#endif + sg_free_table(&kmem->sgt); + kmem->sgt.orig_nents = 0; + kmem->sgt.nents = 0; + } + + return; +} + +int udma_register_seg_kernel(struct ubcore_device *ub_dev, + const struct ubcore_seg_cfg *cfg, + struct udma_segment *udma_seg) +{ + struct udma_device *udma_dev = NULL; + udma_seg_kmem_t *kmem = &udma_seg->kmem; + struct mtt *mtt = &udma_seg->mtt; + int ret; + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "Invalid parameter. ub_dev(%d) .\n", + !!ub_dev); + return -EINVAL; + } + udma_dev = to_udma_dev(ub_dev); + + ret = udma_get_kmem(ub_dev, kmem, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to get kmem, ret:%d\n", ret); + return ret; + } + + mtt->mtt_type = MTT_DMTT_TYPE; + ret = hmm_mtt_alloc(udma_dev->hwdev, kmem->npages, PAGE_SHIFT, mtt, + SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate mtt, ret:%d\n", ret); + goto alloc_mtt_err; + } + + ret = udma_mem_write_mtt_update(udma_dev, mtt, kmem->sgt.sgl, + kmem->sgt.nents); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to write mtt, ret:%d\n", ret); + goto write_mtt_err; + } + + return 0; + +write_mtt_err: + hmm_mtt_free(udma_dev->hwdev, mtt, SERVICE_T_UB); + mtt->mtt_seg = NULL; +alloc_mtt_err: + udma_kmem_release(ub_dev, kmem); + return ret; +} + +static int udma_check_seg_cfg_param(struct udma_device *udma_dev, + const struct ubcore_seg_cfg *cfg) +{ + u64 free_pages; + + if (cfg->flag.bs.token_policy > UBCORE_TOKEN_PLAIN_TEXT) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Invalid segment token_policy:%u.\n", + cfg->flag.bs.token_policy); + return -EINVAL; + } + + if (cfg->flag.bs.access >= UB_MAX_ACCESS_VALUE) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Invalid segment access(%u).\n", cfg->flag.bs.access); + return -EINVAL; + } + + if (cfg->len > MAX_SEGMENT_LEN) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "invalid segment len:0x%llx\n", cfg->len); + return -EINVAL; + } + + free_pages = global_zone_page_state(NR_FREE_PAGES); + if ((cfg->len >> PAGE_SHIFT) >= free_pages) { + udma_err( + udma_dev->dev, UDMA_M_SEG, + "free pages are not enough, len:0x%llx, free pages:0x%llx, page_shift:0x%x\n", + cfg->len, free_pages, (u32)PAGE_SHIFT); + return -EINVAL; + } + + if (cfg->token_id == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, "invalid token_id\n"); + return -EINVAL; + } + + return 0; +} + +static void udma_mem_relese(struct ubcore_device *ub_dev, + struct udma_segment *udma_seg, + struct ubcore_ucontext *uctx) +{ + struct udma_device *udma_dev = to_udma_dev(ub_dev); + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_SEG, "Invalid parameter: ub_dev(%d).\n", + !!ub_dev); + return; + } + + if (udma_seg->mtt.mtt_seg != NULL) { + hmm_mtt_free(udma_dev->hwdev, &udma_seg->mtt, SERVICE_T_UB); + udma_seg->mtt.mtt_seg = NULL; + } + + if (uctx != NULL) { + /* user mode segment */ + if (udma_seg->umem != NULL) { + ubcore_umem_release(udma_seg->umem); + udma_seg->umem = NULL; + } + } else { + /* kernel mode segment */ + udma_kmem_release(ub_dev, &udma_seg->kmem); + } +} + +static int udma_register_seg_check_param(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_SEG, + "Invalid parameter: ub_dev(%d), cfg(%d).\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + if (udma_dev_is_tpf(udma_dev) == 1) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "tpf not support this function.\n"); + return -EINVAL; + } + + return udma_check_seg_cfg_param(udma_dev, cfg); +} + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_segment *udma_seg = NULL; + struct udma_keyid *udma_key = NULL; + int ret; + + ret = udma_register_seg_check_param(ub_dev, cfg); + if (ret != 0) { + return ERR_PTR(ret); + } + + udma_dev = to_udma_dev(ub_dev); + udma_seg = udma_seg_find(udma_dev, cfg->token_id->token_id); + if (udma_seg != NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "Error: token id:%u has been registered\n", + cfg->token_id->token_id); + udma_seg_put(udma_seg); + return ERR_PTR(-EINVAL); + } + + udma_seg = (struct udma_segment *)kzalloc(sizeof(struct udma_segment), + GFP_KERNEL); + if (udma_seg == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to allocate segment\n"); + return ERR_PTR(-ENOMEM); + } + + udma_key = to_udma_keyid(cfg->token_id); + udma_fill_udma_seg(udma_seg, cfg, udma_key); + udma_cmdq_rtt_statistic_init(&udma_seg->cmdq_rtt); + + if (udma_get_uctx(udata) != NULL) + ret = udma_register_seg_user(ub_dev, cfg, udma_seg); + else + ret = udma_register_seg_kernel(ub_dev, cfg, udma_seg); + + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to register segment, ret:%d\n", ret); + goto err_register_seg; + } + + ret = udma_cmd_seg_create_ctx(udma_dev, udma_seg, udma_key); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to create mpt context, ret:%d\n", ret); + goto err_create_mpt_ctx; + } + udma_seg->tseg.seg.token_id = udma_seg->key; + + kref_init(&udma_seg->ref_cnt); + init_completion(&udma_seg->comp); + + udma_store_list(udma_dev, &udma_seg->node, + &udma_dev->device_res->seg_list); + + udma_info( + udma_dev->dev, UDMA_M_SEG, + "register segment success, token_id:0x%x,tseg token_id:0x%x,len:0x%llx,va:0x%llx,mpt_index:0x%x\n", + udma_key->base.token_id, udma_seg->tseg.seg.token_id, cfg->len, + (u64)cfg->va, udma_key->mpt.mpt_index); + + return &udma_seg->tseg; + +err_create_mpt_ctx: + udma_mem_relese(ub_dev, udma_seg, udma_get_uctx(udata)); +err_register_seg: + kfree(udma_seg); + + return ERR_PTR(-EFAULT); +} + +int udma_unregister_seg(struct ubcore_target_seg *tseg) +{ + struct udma_device *udma_dev = NULL; + struct udma_keyid *udma_key = NULL; + struct udma_segment *udma_seg = NULL; + int ret; + + if (tseg == NULL) { + udma_pr_err(UDMA_M_SEG, "%s Invalid parameter: tseg is null.\n", + __func__); + return -EINVAL; + } + + if ((tseg->ub_dev == NULL) || (tseg->token_id == NULL)) { + udma_pr_err( + UDMA_M_SEG, + "%s Invalid parameter: tseg->ub_dev(%d), tseg->token_id(%d).\n", + __func__, !!tseg->ub_dev, !!tseg->token_id); + return -EINVAL; + } + + udma_dev = to_udma_dev(tseg->ub_dev); + udma_key = to_udma_keyid(tseg->token_id); + udma_seg = to_ub_seg(tseg); + + udma_delete_list(udma_dev, &udma_seg->node, + &udma_dev->device_res->seg_list); + udma_seg_put(udma_seg); + wait_for_completion(&udma_seg->comp); + + mutex_lock(&udma_key->mpt_mutex); + ret = udma_cmd_seg_delete_ctx(udma_dev, udma_key, udma_seg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to disable seg's mpt, ret:%d\n", ret); + mutex_unlock(&udma_key->mpt_mutex); + /* delete failed, reference count reinitialized, ensure reentrant destory interface when reference count can normal put */ + kref_init(&udma_seg->ref_cnt); + return ret; + } + + if (udma_seg->david_hbm_en) { + udma_david_info_uninit(udma_dev, udma_seg); + } else { + udma_mem_relese(tseg->ub_dev, udma_seg, tseg->uctx); + } + + udma_info(udma_dev->dev, UDMA_M_SEG, "unregister segment:%u success\n", + udma_key->base.token_id); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_SEG, udma_key->base.token_id, + &udma_seg->cmdq_rtt); + mutex_unlock(&udma_key->mpt_mutex); + kfree(udma_seg); + + return ret; +} + +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *ub_dev, + struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ub_rseg *rseg = NULL; + + if ((ub_dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_SEG, + "%s Invalid parameter: ub_dev(%d), cfg(%d).\n", + __func__, !!ub_dev, !!cfg); + return ERR_PTR(-EINVAL); + } + + rseg = (struct ub_rseg *)kzalloc(sizeof(struct ub_rseg), GFP_KERNEL); + if (rseg == NULL) { + udma_err(&ub_dev->dev, UDMA_M_SEG, "failed to allocate rseg\n"); + return ERR_PTR(-ENOMEM); + } + + rseg->token = cfg->token_value.token; + + return &rseg->base; +} + +int udma_unimport_seg(struct ubcore_target_seg *tseg) +{ + struct ub_rseg *rseg = NULL; + + if (tseg == NULL) { + udma_pr_err(UDMA_M_JFC, "%s Invalid parameter: tseg is null.\n", + __func__); + return -EINVAL; + } + + rseg = container_of(tseg, struct ub_rseg, base); + kfree(rseg); + + return 0; +} + +int udma_query_res_segment(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_seg_val *res_seg = + (struct ubcore_res_seg_val *)(uintptr_t)val->addr; + struct ubcore_seg_info *seg_list = NULL; + struct udma_segment *udma_seg = NULL; + int ret = 0; + + udma_seg = udma_seg_find(udma_dev, key->key); //key=token_id + if (udma_seg == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "failed to find segment, id:%u\n", key->key); + return -EINVAL; + } + + res_seg->seg_cnt = TOKEN_ID_SEG_CNT; + seg_list = (struct ubcore_seg_info *)vmalloc( + sizeof(struct ubcore_seg_info)); //only supports 1(count word) + if (seg_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_SEG, + "segment list vmalloc fail.\n"); + ret = -ENOMEM; + goto OUT; + } + + res_seg->seg_list = seg_list; + (void)memcpy_s(seg_list->ubva.eid.raw, UBCORE_EID_SIZE, + udma_dev->eid.raw, UBCORE_EID_SIZE); + seg_list->ubva.va = udma_seg->va; + seg_list->len = udma_seg->size; + seg_list->token_id = udma_seg->mpt->mpt_index; +OUT: + udma_seg_put(udma_seg); + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h new file mode 100644 index 000000000..013113d0e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_segment.h @@ -0,0 +1,78 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. + + +#ifndef UDMA_SEGMENT_H +#define UDMA_SEGMENT_H + +#include "ubdevshm.h" +#include "udma_common.h" + +#define TOKEN_ID_SEG_CNT 1 + +typedef struct udma_seg_kmem { + u32 npages; + struct sg_table sgt; +} udma_seg_kmem_t; + +typedef struct udma_segment { + struct ubcore_target_seg tseg; + struct ubcore_umem *umem; + struct udma_seg_kmem kmem; + struct mtt mtt; + struct mpt *mpt; + struct access_ctx ctx; + struct mem_uba uba; + u64 va; + u64 size; + u32 key; + u32 access; + int enabled; + u32 block_size; + u32 token_value; + u8 key_type; + bool david_hbm_en; + struct list_head node; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +} udma_segment_t; + +#define UB_ACCESS_LOCAL_WRITE (0x1 << 0) +#define UB_ACCESS_REMOTE_READ (0x1 << 1) +#define UB_ACCESS_REMOTE_WRITE (0x1 << 2) +#define UB_ACCESS_REMOTE_ATOMIC (0x1 << 3) +#define UB_ACCESS_REMOTE_INVALIDATE (0x1 << 4) + +#define SEG_KEY_RIGHT_SHIFT_OFS 24 +#define SEG_KEY_LEFT_SHIFT_OFS 8 +#define MAX_SEGMENT_LEN 0x8000000000ULL //max support 512GB +#define UB_MAX_ACCESS_VALUE 32 + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *ub_dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unregister_seg(struct ubcore_target_seg *tseg); +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *ub_dev, + struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_seg(struct ubcore_target_seg *tseg); +int udma_query_res_segment(struct udma_device *udma_dev, + const struct ubcore_device *dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val); + +struct ubcore_token_id *udma_alloc_key_id(struct ubcore_device *ub_dev, + union ubcore_token_id_flag flag, + struct ubcore_udata *udata); +struct udma_segment *udma_seg_find(struct udma_device *udma_dev, u32 key_id); +void udma_seg_put(void *obj); +void udma_seg_get(void *obj); + +static inline struct udma_segment * +to_ub_seg(const struct ubcore_target_seg *tseg) +{ + return container_of(tseg, struct udma_segment, tseg); +} + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c new file mode 100644 index 000000000..aa4a6587e --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.c @@ -0,0 +1,67 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include <linux/module.h> +#include <linux/kallsyms.h> + +#include "ossl_knl.h" +#include "udma_log.h" +#include "udma_shm_manage.h" + +static ubdevshm_acquire_uba_t _udma_ubdevshm_acquire_uba = NULL; +static ubdevshm_release_uba_t _udma_ubdevshm_release_uba = NULL; + +int udma_ubdevshm_acquire_uba(struct access_ctx *ctx, struct mem_uva *va, + union acquire_attr *attr, invalidate func, + struct mem_uba *uba) +{ + if (_udma_ubdevshm_acquire_uba != NULL) { + return _udma_ubdevshm_acquire_uba(ctx, va, attr, func, uba); + } + + return -EINVAL; +} + +int udma_ubdevshm_release_uba(struct access_ctx *ctx, struct mem_uba *uba) +{ + if (_udma_ubdevshm_release_uba != NULL) { + return _udma_ubdevshm_release_uba(ctx, uba); + } + + return -EINVAL; +} + +int udma_init_shm_related_symbols(void) +{ + _udma_ubdevshm_acquire_uba = + (ubdevshm_acquire_uba_t)symbol_get(ubdevshm_acquire_uba); + if (_udma_ubdevshm_acquire_uba == NULL) { + /* if find not to shm related of symbol, description shm driver not load, directly exit i.e. can, not support david passthrough */ + udma_pr_info(UDMA_M_SHM, "ubdev_shm driver is unloaded.\n"); + return 0; + } + + _udma_ubdevshm_release_uba = + (ubdevshm_release_uba_t)symbol_get(ubdevshm_release_uba); + if (_udma_ubdevshm_release_uba == NULL) { + symbol_put(ubdevshm_acquire_uba); + udma_pr_info(UDMA_M_SHM, "ubdev_shm driver is unloaded.\n"); + return 0; + } + + return 0; +} + +void udma_uninit_shm_related_symbols(void) +{ + if (_udma_ubdevshm_acquire_uba != NULL) { + symbol_put(ubdevshm_acquire_uba); + _udma_ubdevshm_acquire_uba = NULL; + } + + if (_udma_ubdevshm_release_uba != NULL) { + symbol_put(ubdevshm_release_uba); + _udma_ubdevshm_release_uba = NULL; + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h new file mode 100644 index 000000000..d6f3daae3 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/segment/udma_shm_manage.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SHM_MANAGE_H +#define UDMA_SHM_MANAGE_H + +#include "ubdevshm.h" + +typedef int __iomem (*ubdevshm_acquire_uba_t)(struct access_ctx *ctx, + struct mem_uva *va, + union acquire_attr *attr, + invalidate func, + struct mem_uba *uba); +typedef int __iomem (*ubdevshm_release_uba_t)(struct access_ctx *ctx, + struct mem_uba *uba); + +int udma_ubdevshm_acquire_uba(struct access_ctx *ctx, struct mem_uva *va, + union acquire_attr *attr, invalidate func, + struct mem_uba *uba); +int udma_ubdevshm_release_uba(struct access_ctx *ctx, struct mem_uba *uba); +int udma_init_shm_related_symbols(void); +void udma_uninit_shm_related_symbols(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c new file mode 100644 index 000000000..9a4a7621a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.c @@ -0,0 +1,476 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. + + +#include <linux/slab.h> +#include <linux/mm_types.h> +#include <linux/uaccess.h> +#include <linux/poll.h> +#include <linux/list.h> +#include <linux/device.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/netdevice.h> +#include <linux/types.h> +#include <linux/stat.h> + +#include "udma_misc.h" +#include "udma_sysfs_common.h" +#include "udma_sysfs_config.h" +#include "udma_sysfs.h" + +/* and above on is old code, pending refactor; and above next is refactor after of code */ +void *udma_sysfs_get_child_value(void *parent, u32 offset) +{ + return (void *)((uintptr_t)parent + offset); +} +/** + * @brief configuration initialize check + * @param[in] udma_dev: device information + * @param[in] node: node configuration information + * @details used for determine this configuration whether need initialize + * @return + * - true: check success can initialize + * - false: check failure, cannot initialize + */ +static bool +udma_sysfs_func_type_and_version_check(struct udma_device *udma_dev, + struct udma_sysfs_support_info *support) +{ + enum udma_device_type sysfs_ver = udma_dev->device_type; + enum udma_sysfs_func_type device_func_type = + (udma_dev_is_tpf(udma_dev) == 1) ? UDMA_SYSFS_TPF : + UDMA_SYSFS_TAF; + + if ((support->func_ver & (u32)sysfs_ver) == 0) { + return false; + } + + if (support->func_type != UDMA_SYSFS_ALL && + support->func_type != device_func_type) { + return false; + } + + return true; +} + +struct udma_sysfs_node *udma_sysfs_get_node_from_child_list_by_name( + struct udma_sysfs_child_node_info *child_info, const char *name) +{ + struct udma_sysfs_node *child_list = child_info->node_list; + int length = child_info->length; + int i = 0; + + for (i = 0; i < length; ++i) { + if (strcmp(child_list[i].attr.name, name) == 0) { + return &child_list[i]; + } + } + + return NULL; +} + +/* common show function, all directory all will register this function, traverse this directory next all file, found name same of file after call this file register of show function, to get data */ +static ssize_t udma_sysfs_common_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct udma_sysfs_head *head = to_sysfs_head_by_kobject(kobj); + struct udma_device *udma_dev = to_udma_dev_by_sysfs(head->base); + struct udma_sysfs_child_node_info *child_info = &head->node->child_info; + struct udma_sysfs_node *child_node = NULL; + ssize_t ret = -EIO; + + child_node = udma_sysfs_get_node_from_child_list_by_name(child_info, + attr->name); + if (child_node == NULL) { + return ret; + } + + if (child_node->ops.show == NULL) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "%s Unregistered 'show' function.\n", + child_node->attr.name); + return ret; + } + + mutex_lock(&head->mutex); + ret = child_node->ops.show(udma_dev, head, child_node, buf); + mutex_unlock(&head->mutex); + + return ret; +} + +/* common store function, all directory all will register this function, traverse this directory next all file, found name same of file after call this file register of store function, to update data */ +ssize_t udma_sysfs_common_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct udma_sysfs_head *head = to_sysfs_head_by_kobject(kobj); + struct udma_device *udma_dev = to_udma_dev_by_sysfs(head->base); + struct udma_sysfs_child_node_info *child_info = &head->node->child_info; + struct udma_sysfs_node *child_node = NULL; + ssize_t ret = -EIO; + + child_node = udma_sysfs_get_node_from_child_list_by_name(child_info, + attr->name); + if (child_node == NULL) { + return ret; + } + + if (child_node->ops.store == NULL) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "%s Unregistered 'store' function.\n", + child_node->attr.name); + return ret; + } + + mutex_lock(&head->mutex); + ret = child_node->ops.store(udma_dev, head, child_node, buf); + mutex_unlock(&head->mutex); + + return (ret == (ssize_t)0) ? (ssize_t)count : ret; +} + +/* register common callback, as directory context file of callback inport */ +struct sysfs_ops udma_sysfs_common_ops = { .show = udma_sysfs_common_show, + .store = udma_sysfs_common_store }; +static void udma_sysfs_default_release(struct kobject *kobj) +{ +} +struct kobj_type udma_sysfs_common_ktype = { + .release = udma_sysfs_default_release, + .sysfs_ops = &udma_sysfs_common_ops, +}; + +static int udma_sysfs_init_sysfs_nodes(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_res *base, + struct udma_sysfs_node *child_nodes, + int child_nodes_num); +static void udma_sysfs_deinit_sysfs_nodes(struct udma_device *udma_dev, + void *parent_addr, + struct kobject *parent_kobj, + struct udma_sysfs_node *child_nodes, + int child_nodes_num); + +/** + * @brief recursively initialize kobj create file or subdirectory + * @param[in] udma_dev: device information + * @param[in] parent_kobj: parent directory of kobj + * @param[in] parent_addr: base address of parent data structure + * @param[in] base: filesystem object + * @param[in] node: configuration node corresponding to current file + * @details recursively initialize kobj create file + * @return + * - zero: success + * - not zero: failure + */ +static int udma_sysfs_init_dir_node(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_res *base, + struct udma_sysfs_node *node) +{ + void *start_address = NULL; + struct udma_sysfs_head *head = NULL; + char name[UDMA_SYSFS_NAME_MAX_LEN] = ""; + u32 i = 0; + int ret = 0; + + start_address = udma_sysfs_get_child_value(parent_addr, + node->position_info.offset); + + for (i = 0; i < node->position_info.array_num; ++i) { + head = (struct udma_sysfs_head + *)(start_address + + i * node->position_info.element_size); + + head->node = node; + head->base = base; + head->parent_addr = parent_addr; + ret = node->name_ext.get_name_func(node->attr.name, (int)i, + node->name_ext.ext_value, + name); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "kobject(%s, %d), get name failed, ret: %d\n", + node->attr.name, i, ret); + return ret; + } + mutex_init(&head->mutex); + ret = kobject_init_and_add(&head->kobj, + &udma_sysfs_common_ktype, + parent_kobj, name); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to add kobject(%s, %d), name: %s, ret: %d\n", + node->attr.name, i, name, ret); + kobject_put(&head->kobj); + mutex_destroy(&head->mutex); + return ret; + } + head->kobj_flag = UDMA_SYSFS_INITIALIZED; + ret = udma_sysfs_init_sysfs_nodes(udma_dev, &head->kobj, head, + head->base, + node->child_info.node_list, + node->child_info.length); + if (ret != 0) { + break; + } + if (node->ext_update == NULL) { + continue; + } + ret = node->ext_update(udma_dev, head); + if (ret != 0) { + break; + } + } + + return ret; +} + +/** + * @brief free node corresponding directory next resource and corresponding kobj + * @param[in] udma_dev: device information + * @param[in] parent_addr: base address of parent data structure + * @param[in] node: configuration node corresponding to current file + * @details if node corresponding directory already initialize, free directory next resource and kobj, directory lock + * @return void + */ +static void udma_sysfs_deinit_dir_node(struct udma_device *udma_dev, + void *parent_addr, + struct udma_sysfs_node *node) +{ + void *start_address = NULL; + struct udma_sysfs_head *head = NULL; + u32 i = 0; + + start_address = udma_sysfs_get_child_value(parent_addr, + node->position_info.offset); + + for (i = 0; i < node->position_info.array_num; ++i) { + head = (struct udma_sysfs_head + *)(start_address + + i * node->position_info.element_size); + if (head->kobj_flag == UDMA_SYSFS_UNINITIALIZED) { + continue; + } + udma_sysfs_deinit_sysfs_nodes(udma_dev, head, &head->kobj, + node->child_info.node_list, + node->child_info.length); + kobject_put(&head->kobj); + mutex_destroy(&head->mutex); + head->kobj_flag = UDMA_SYSFS_UNINITIALIZED; + } +} + +/** + * @brief create node corresponding file + * @param[in] udma_dev: device information + * @param[in] parent_kobj: parent directory of kobj + * @param[in] parent_addr: base address of parent data structure + * @param[in] node: configuration node corresponding to current file + * @param[in] index: index of current file in parent directory + * @details create node corresponding file and initialize corresponding parameters + * @return + * - zero: success + * - not zero: failure + */ +static int udma_sysfs_init_file_node(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_node *node, int index) +{ + void *value = NULL; + struct udma_sysfs_head *parent_head = + (struct udma_sysfs_head *)parent_addr; + int ret = 0; + + value = udma_sysfs_get_child_value(parent_addr, + node->position_info.offset); + if (node->value.set_def_value != NULL) { + node->value.set_def_value(value, node->value.def, + node->position_info.array_num); + } + + ret = sysfs_create_file(parent_kobj, &node->attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to create the file(%s), ret: %d\n", + node->attr.name, ret); + return ret; + } + parent_head->file_flag[index] = UDMA_SYSFS_INITIALIZED; + + if (node->ext_update == NULL) { + return 0; + } + + return node->ext_update(udma_dev, parent_addr); +} + +/** + * @brief free node corresponding file + * @param[in] parent_addr: base address of parent data structure + * @param[in] parent_kobj: parent directory of kobj + * @param[in] node: configuration node corresponding to current file + * @param[in] index: index of current file in parent directory + * @details if node corresponding file already initialize, free this file + * @return void + */ +static void udma_sysfs_deinit_file_node(void *parent_addr, + struct kobject *parent_kobj, + struct udma_sysfs_node *node, int index) +{ + struct udma_sysfs_head *parent_head = + (struct udma_sysfs_head *)parent_addr; + + if (parent_head->file_flag[index] == UDMA_SYSFS_UNINITIALIZED) { + return; + } + + sysfs_remove_file(parent_kobj, &node->attr); + parent_head->file_flag[index] = UDMA_SYSFS_UNINITIALIZED; +} + +/** + * @brief recursively initialize kobj create file or subdirectory + * @param[in] udma_dev: device information + * @param[in] parent_kobj: parent directory of kobj + * @param[in] parent_addr: base address of parent data structure + * @param[in] base: filesystem object + * @param[in] child_nodes: subdirectory files of parent directory + * @param[in] child_nodes_num: count of subdirectory files of parent directory + * @details recursively initialize kobj create file + * @return + * - zero: success + * - not zero: failure + */ +static int udma_sysfs_init_sysfs_nodes(struct udma_device *udma_dev, + struct kobject *parent_kobj, + void *parent_addr, + struct udma_sysfs_res *base, + struct udma_sysfs_node *child_nodes, + int child_nodes_num) +{ + int ret = 0; + int i = 0; + + /* cannot exceed file flag array of size */ + if (child_nodes_num > UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "sub-items in the dir num(%d) exceeded max num(%d).\n", + child_nodes_num, UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM); + return -EINVAL; + } + + for (i = 0; i < child_nodes_num; ++i) { + /* check this node whether need in this version this func load */ + if (udma_sysfs_func_type_and_version_check( + udma_dev, &(child_nodes[i].support_info)) == + false) { + udma_debug(udma_dev->dev, UDMA_M_SYSFS, + "kobject %s Not loaded.\n", + child_nodes[i].attr.name); + continue; + } + + if (child_nodes[i].type == UDMA_SYSFS_NODE_DIR) { + ret = udma_sysfs_init_dir_node(udma_dev, parent_kobj, + parent_addr, base, + &child_nodes[i]); + } else { + ret = udma_sysfs_init_file_node(udma_dev, parent_kobj, + parent_addr, + &child_nodes[i], i); + } + + if (ret != 0) { + return ret; + } + } + + return ret; +} + +/** + * @brief recursive free kobj delete file or subdirectory + * @param[in] udma_dev: device information + * @param[in] parent_addr: base address of parent data structure + * @param[in] parent_kobj: parent directory of kobj + * @param[in] child_nodes: subdirectory files of parent directory + * @param[in] child_nodes_num: count of subdirectory files of parent directory + * @details recursive free kobj delete file + * @return void + */ +static void udma_sysfs_deinit_sysfs_nodes(struct udma_device *udma_dev, + void *parent_addr, + struct kobject *parent_kobj, + struct udma_sysfs_node *child_nodes, + int child_nodes_num) +{ + int i = 0; + + /* cannot exceed file flag array of size, and init for symmetry */ + if (child_nodes_num > UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM) { + return; + } + + for (i = 0; i < child_nodes_num; ++i) { + /* check this node whether in this version this func load (done), and init for symmetry */ + if (udma_sysfs_func_type_and_version_check( + udma_dev, &(child_nodes[i].support_info)) == + false) { + continue; + } + + if (child_nodes[i].type == UDMA_SYSFS_NODE_DIR) { + udma_sysfs_deinit_dir_node(udma_dev, parent_addr, + &child_nodes[i]); + } else { + udma_sysfs_deinit_file_node(parent_addr, parent_kobj, + &child_nodes[i], i); + } + } +} + +int udma_init_sysfs(struct udma_sysfs_res *sysfs_res) +{ + int ret = 0; + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + struct kobject *base_kobj = &udma_dev->ndev->dev.kobj; + struct udma_sysfs_node *config_info = udma_sysfs_get_config_info(); + int config_len = udma_sysfs_get_config_length(); + + if (udma_dev->device_type == UDMA_DEV_TYPE_UNKNOWN) { + return -EINVAL; + } + + ret = udma_sysfs_init_sysfs_nodes(udma_dev, base_kobj, sysfs_res, + sysfs_res, config_info, config_len); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to init sysfs, ret: %d\n", ret); + udma_sysfs_deinit_sysfs_nodes(udma_dev, sysfs_res, base_kobj, + config_info, config_len); + return ret; + } + + return 0; +} + +void udma_remove_sysfs(struct udma_sysfs_res *sysfs_res) +{ + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + struct kobject *base_kobj = &udma_dev->ndev->dev.kobj; + struct udma_sysfs_node *config_info = udma_sysfs_get_config_info(); + int config_len = udma_sysfs_get_config_length(); + + if (udma_dev->device_type == UDMA_DEV_TYPE_UNKNOWN) { + return; + } + + udma_sysfs_deinit_sysfs_nodes(udma_dev, sysfs_res, base_kobj, + config_info, config_len); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h new file mode 100644 index 000000000..b93a14a2d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs.h @@ -0,0 +1,158 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_H +#define UDMA_SYSFS_H + +#include "udma_sysfs_api.h" +#include "udma_common.h" + +#define DSCP_ARRAY_LEN 8 +#define LDCP_PATTERN_ARRAY_LEN 8 +#define CAQM_PATTERN_ARRAY_LEN 3 +#define USER_DEFINED_PATTERN_ARRAY_LEN 8 +#define USER_DEFINED_TEMPLATE_NUM 2 + +#define UDMA_SYSFS_NAME_MAX_LEN 32 /* file or directory name maximum length */ +#define UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM 20 /* max number of sub-items in directory */ + +//LDPC +#define UDMA_DEFAULT_INIT_WND 1 +#define UDMA_DEFAULT_ALPHA 4 +#define UDMA_DEFAULT_BETA 6 +#define UDMA_DEFAULT_GAMMA 45 +#define UDMA_DEFAULT_ETA 31 +#define UDMA_DEFAULT_WND_MIN 31 +#define UDMA_DEFAULT_CC_UNIT 6 + +#define UDMA_MIN_LDCP_PARA 0 +#define UDMA_MAX_LDCP_PARA 127 +#define UDMA_MIN_CAQM_PARA 0 +#define UDMA_MAX_CAQM_PARA 127 + +//RX_RATE_LIMITE ACC PARAM DEFAULT +#define UDMA_ACC_DEFAULT_DOWN_PERIOD 32 +#define UDMA_ACC_DEFAULT_UP_PERIOD 160 +#define UDMA_ACC_DEFAULT_ALPHA1 1 //8Kb/s +#define UDMA_ACC_DEFAULT_N1 10 +#define UDMA_ACC_DEFAULT_ALPHA2 128 //1024Kb/s +#define UDMA_ACC_DEFAULT_N2 15 +#define UDMA_ACC_DEFAULT_ALPHA3 500 //4000Kb/s +#define UDMA_ACC_DEFAULT_BETA 13 +#define UDMA_ACC_DEFAULT_MIN_RATE 128 //1024Kb/s +#define UDMA_ACC_DEFAULT_INIT_RATE 10 //80Kb/s + +//CAQM TODO, may be modified +#define UDMA_MIN_INIT_RATE 0 +#define UDMA_MAX_INIT_RATE 65536 + +//RX_RATE_LIMITE ACC PARAM RANGE +#define UDMA_ACC_MIN_DOWN_PERIOD 0 +#define UDMA_ACC_MAX_DOWN_PERIOD 1024 +#define UDMA_ACC_MIN_BETA 0 +#define UDMA_ACC_MAX_BETA 64 +#define UDMA_ACC_MIN_UP_PERIOD 0 +#define UDMA_ACC_MAX_UP_PERIOD 1024 +#define UDMA_ACC_MIN_ALPHA1 0 +#define UDMA_ACC_MAX_ALPHA1 1023 +#define UDMA_ACC_MIN_ALPHA2 0 +#define UDMA_ACC_MAX_ALPHA2 1023 +#define UDMA_ACC_MIN_ALPHA3 0 +#define UDMA_ACC_MAX_ALPHA3 4095 +#define UDMA_ACC_MIN_N1 0 +#define UDMA_ACC_MAX_N1 32 +#define UDMA_ACC_MIN_N2 0 +#define UDMA_ACC_MAX_N2 32 +#define UDMA_ACC_MIN_MIN_RATE 0 +#define UDMA_ACC_MAX_MIN_RATE 65536 +#define UDMA_ACC_MIN_INIT_RATE 0 +#define UDMA_ACC_MAX_INIT_RATE 65536 +#define UDMA_ACC_MAX_WRED_IDX 127 +#define UDMA_ACC_MAX_WRED_VAL 65535 + +#define UDMA_MAX_TOKEN_ID_PARA 1048575 +#define UDMA_MAX_EID_PARA 1048575 + +enum udma_sysfs_node_type { + UDMA_SYSFS_NODE_DIR, /* directory */ + UDMA_SYSFS_NODE_FILE, /* file */ +}; + +enum udma_sysfs_func_type { + UDMA_SYSFS_TPF, /* tpf */ + UDMA_SYSFS_TAF, /* include pf and vf */ + UDMA_SYSFS_ALL /* tpf and taf */ +}; + +typedef void (*udma_sysfs_set_def_value)(void *value, u32 def, u32 array_num); + +struct udma_sysfs_attr_value { + u32 min; /* file parameters maximum less value */ + u32 def; /* file parameters of default value */ + u32 max; /* file parameters maximum value */ + udma_sysfs_set_def_value set_def_value; /* used for update default value */ +}; + +typedef int (*udma_sysfs_get_name_func)(const char *name, int index, + void *ext_value, char *out_name); + +struct udma_sysfs_name_ext { + udma_sysfs_get_name_func get_name_func; + void *ext_value; +}; + +struct udma_sysfs_position_info { + u32 offset; /* indicates this configuration corresponding data which count data structure in of 那 field */ + u32 array_num; /* indicates in same level exist 几 should use this configured same subdirectory, or indicates file corresponding array of array size */ + u32 element_size; /* element size, used for not perceive element data structure traverse array use */ +}; + +struct udma_sysfs_child_node_info { + struct udma_sysfs_node + *node_list; /* directory valid, record this directory context file or subdirectory of configuration */ + int length; /* directory valid, record this directory next node of total data */ +}; + +struct udma_sysfs_support_info { + enum udma_sysfs_func_type + func_type; /* indicates this configuration will in which func type load, @see > enum udma_sysfs_func_type */ + u32 func_ver; /* indicates this configuration will in which version load, @see > enum udma_device_type */ +}; + +struct udma_sysfs_ops { + ssize_t (*show)(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + char *buf); /* file corresponding to show function */ + ssize_t (*store)(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); /* file corresponding to store function */ +}; + +struct udma_sysfs_node { + struct attribute attr; /* file name or directory name */ + struct udma_sysfs_name_ext name_ext; /* used for custom file name */ + enum udma_sysfs_node_type + type; /* configuration type, @see > enum udma_sysfs_node_type */ + struct udma_sysfs_support_info + support_info; /* used for record node in which scenario generatevalid */ + struct udma_sysfs_position_info + position_info; /* used for record this node corresponding parameters of offset, element count, element size */ + struct udma_sysfs_attr_value + value; /* file valid, used for file parameters of initialize and modify check */ + struct udma_sysfs_ops ops; /* file valid, according to parameters need specifies corresponding to callback function */ + int (*ext_update)( + struct udma_device *udma_dev, + void *parent_addr); /* extension update function, when parameters or parameters group initialize after need data update to other module when use */ + struct udma_sysfs_child_node_info + child_info; /* directory valid, record this directory context file or subdirectory of configuration */ +}; + +/* field in data structure in of offset, based on this data structure object of base address 拿 to this field of address */ +void *udma_sysfs_get_child_value(void *parent, u32 offset); +ssize_t udma_sysfs_common_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h new file mode 100644 index 000000000..bf239f56b --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_api.h @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_API_H +#define UDMA_SYSFS_API_H + +#include <linux/device.h> +#include <linux/mutex.h> + +#define UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM 20 /* max number of sub-items in directory */ +#define DSCP_ARRAY_LEN 8 +#define LDCP_PATTERN_ARRAY_LEN 8 +#define CAQM_PATTERN_ARRAY_LEN 3 +#define USER_DEFINED_PATTERN_ARRAY_LEN 8 +#define USER_DEFINED_TEMPLATE_NUM 2 + +typedef struct udma_sysfs_node udma_sysfs_node_t; + +enum udma_sysfs_init_type { + UDMA_SYSFS_UNINITIALIZED, /* default value, indicates not initialize */ + UDMA_SYSFS_INITIALIZED /* initialize, free when according to this free */ +}; + +/* only in directory layer level use, constraint: as data structure of start */ +struct udma_sysfs_head { + struct kobject kobj; /* current directory of filesystem kobj */ + struct mutex mutex; /* current directory context file of read write lock */ + udma_sysfs_node_t *node; /* current directory corresponding sysfs related configuration */ + struct udma_sysfs_res *base; /* sysfs module of base object */ + void *parent_addr; /* parent directory of base address */ + enum udma_sysfs_init_type kobj_flag; /* current directory whether complete (done) initialize */ + enum udma_sysfs_init_type file_flag + [UDMA_SYSFS_ITEM_IN_DIR_MAX_NUM]; /* initialize when allocate of array, use to record directory context file initialize of status, directory free when cleanup */ +}; + +struct udma_dfx_perf_pattern { + struct udma_sysfs_head head; + u32 huge_page_en; + u32 pi_on_chip_dis; + u32 dwqe_dis; + u32 vf_start_xid; + u32 xid_low_3_bit; + u32 jetty_slice_en; + u32 ta2tp_type; + u32 tp_depth_shift; + u32 fis_en; /* enable fast inline send flag switch, only used for fast path scenario */ + u32 cfg_dscp_en; + u32 cfg_dscp[DSCP_ARRAY_LEN]; +}; + +struct udma_dfx_ctx { + struct udma_sysfs_head head; + u32 debug_en; + u32 drop_aeqe_en; + u32 dump_tp_en; + u32 tid; + u32 eid; + u32 hbm_en; + struct udma_dfx_perf_pattern perf; +}; + +struct udma_ldcp_pattern_ctx { + struct udma_sysfs_head head; + u32 init_wnd; + u32 alpha; + u32 beta; + u32 gamma; + u32 eta; + u32 wnd_min; +}; + +struct udma_cc_ldcp_ctx { + struct udma_sysfs_head head; + struct udma_ldcp_pattern_ctx pattern[LDCP_PATTERN_ARRAY_LEN]; +}; + +struct udma_caqm_pattern_ctx { + struct udma_sysfs_head head; + u32 alpha; + u32 beta; + u32 eta; + u32 wnd_min; + u32 init_wnd; + u32 cc_unit; +}; + +struct udma_cc_caqm_ctx { + struct udma_sysfs_head head; + struct udma_caqm_pattern_ctx pattern[CAQM_PATTERN_ARRAY_LEN]; +}; + +struct udma_user_defined_pattern_ctx { + struct udma_sysfs_head head; + u32 dw0; + u32 dw1; + u32 dw2; + u32 dw3; +}; + +struct udma_cc_user_defined_ctx { + struct udma_sysfs_head head; + struct udma_user_defined_pattern_ctx + pattern[USER_DEFINED_PATTERN_ARRAY_LEN]; +}; + +struct udma_cc_ctx { + struct udma_sysfs_head head; + struct udma_cc_ldcp_ctx ldcp; + struct udma_cc_caqm_ctx caqm; + struct udma_cc_user_defined_ctx user_defined[USER_DEFINED_TEMPLATE_NUM]; +}; + +struct udma_rx_rate_limit_wred { + u32 cmd; + u32 wred_idx; + u32 start_val; + u32 end_val; +}; + +struct udma_rx_rate_limit_ctx { + struct udma_sysfs_head head; + u16 down_period; + u16 up_period; + u16 alpha1; + u16 n1; + u16 alpha2; + u16 alpha3; + u16 min_rate; + u16 init_rate; + u8 n2; + u8 beta; + struct udma_rx_rate_limit_wred wred; +}; + +struct udma_sysfs_res { + struct udma_dfx_ctx dfx_ctx; + struct udma_cc_ctx cc_ctx; + struct udma_rx_rate_limit_ctx rx_rate_limit_ctx; +}; + +int udma_init_sysfs(struct udma_sysfs_res *sysfs_res); +void udma_remove_sysfs(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get driver func level debug_en + * @param[in] sysfs_res: current func of filesystem + * @details use to get driver func level debug_en + */ +bool udma_sysfs_get_debug_en(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get allocate jetty xid low bit match value + * @param[in] sysfs_res: current func of filesystem + * @param[out] out_value: get to of low bit match value (not enable debug_en or configuration value invalid when will initialize is 0) + * @details get allocate jetty xid low bit match value + * @return true: out_value is valid value + * false: out_value is invalid value + */ +bool udma_sysfs_get_xid_low_3_bit(struct udma_sysfs_res *sysfs_res, + u32 *out_value); + +/** + * @brief get dwqe_dis + * @param[in] sysfs_res: current func of filesystem + * @details get dwqe_dis, set when disable dwqe path + * @return dwqe_dis value + */ +u32 udma_sysfs_get_dwqe_dis(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get tp_q_depth + * @param[in] sysfs_res: current func of filesystem + * @details get tp_q_depth, used for allocate tp queue buf + * @return tp_q_depth value + */ +u32 udma_sysfs_get_tp_depth(struct udma_sysfs_res *sysfs_res); + +/** + * @brief set tp_q_depth + * @param[in] sysfs_res: current func of filesystem + * @param[in] new_value: to set new value + * @details set tp_q_depth, need seek value is 2 of power + * @return max at 0: set success + * less at 0: set failure + */ +int udma_sysfs_set_tp_depth(struct udma_sysfs_res *sysfs_res, u32 new_value); + +/** + * @brief get mtt_pro_en + * @param[in] sysfs_res: current func of filesystem + * @param[out] mtt_pro_en: get to of mtt_pro_en value + * @details get mtt_pro_en, if filesystem in not enable, from mpu get + * @return 0: get success + * non- 0: get fails + */ +int udma_sysfs_get_mtt_pro_en(struct udma_sysfs_res *sysfs_res, + u32 *mtt_pro_en); + +/** + * @brief get huge_page_en + * @param[in] sysfs_res: current func of filesystem + * @details get huge_page_en + * @return huge_page_en value + */ +u32 udma_sysfs_get_huge_page_en(struct udma_sysfs_res *sysfs_res); + +/** + * @brief get fis_en + * @param[in] sysfs_res: current func of filesystem + * @details get fis_en + * @return fis_en value + */ +bool udma_sysfs_get_fis_en(struct udma_sysfs_res *sysfs_res); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c new file mode 100644 index 000000000..29baa00cc --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.c @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include "ub_dw_index.h" +#include "udma_sysfs.h" +#include "udma_sysfs_api.h" +#include "udma_misc.h" +#include "udma_cmd.h" +#include "ub_pub_cmd.h" +#include "udma_resource.h" +#include "udma_sysfs_common.h" + +#define TP_Q_DEF_DEPTH 128 +#define UDMA_SYSFS_VALUE_TYPE_SIZE_U8 sizeof(u8) +#define UDMA_SYSFS_VALUE_TYPE_SIZE_U16 sizeof(u16) +#define UDMA_SYSFS_VALUE_TYPE_SIZE_U32 sizeof(u32) + +int udma_sysfs_set_rx_rate_limit(struct udma_device *udma_dev, + void *parent_addr) +{ + struct udma_rx_rate_limit_ctx *parent = + (struct udma_rx_rate_limit_ctx *)parent_addr; + struct ub_rx_rate_limit_params rx_rate_limit_param = { 0 }; + int ret; + + rx_rate_limit_param.down_period = parent->down_period; + rx_rate_limit_param.up_period = parent->up_period; + rx_rate_limit_param.alpha1 = parent->alpha1; + rx_rate_limit_param.n1 = parent->n1; + rx_rate_limit_param.alpha2 = parent->alpha2; + rx_rate_limit_param.n2 = parent->n2; + rx_rate_limit_param.alpha3 = parent->alpha3; + rx_rate_limit_param.beta = parent->beta; + rx_rate_limit_param.min_rate = parent->min_rate; + rx_rate_limit_param.init_rate = parent->init_rate; + + ret = udma_rx_rate_limit_set_para(udma_dev, &rx_rate_limit_param); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to update rx rate limit pattern, ret: %d\n", + ret); + return ret; + } + return 0; +} + +static u32 +udma_sysfs_get_index_in_parent_array(struct udma_sysfs_head *node_head) +{ + uintptr_t start_addr = (uintptr_t)udma_sysfs_get_child_value( + node_head->parent_addr, node_head->node->position_info.offset); + u32 element_size = node_head->node->position_info.element_size; + u32 array_num = node_head->node->position_info.array_num; + u32 i = 0; + + for (i = 0; i < array_num; ++i) { + if ((start_addr + i * element_size) == (uintptr_t)node_head) { + break; + } + } + + return i; +} + +static int udma_sysfs_set_ta2tp_type(struct udma_device *udma_dev, + void *parent_addr) +{ + struct ub_cmd_func_attr func_attr = { 0 }; + struct udma_dfx_perf_pattern *parent = + (struct udma_dfx_perf_pattern *)parent_addr; + bool debug_en = false; + + debug_en = udma_sysfs_get_debug_en(&udma_dev->sysfs_res); + /* debug_en not generatevalid, returns directly failure */ + if (!debug_en) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "The debug_en(%d) is not taking effect.\n", debug_en); + return -EINVAL; + } + + SET_FUNC_TBL_ATTR_VAL_WITH_MASK(func_attr, attr2, ta2tp_type, + (u8)parent->ta2tp_type); + + return udma_set_func_attr( + udma_dev, hinic5_global_func_id(udma_dev->hwdev), &func_attr); +} + +int udma_sysfs_set_ldcp(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_ldcp_pattern_ctx *parent = + (struct udma_ldcp_pattern_ctx *)parent_addr; + struct ub_cc_params cc_param = { 0 }; + int ret = 0; + + cc_param.ub_cc_algo = UB_CC_ALGO_LDCP; + cc_param.index = + (u8)udma_sysfs_get_index_in_parent_array(&(parent->head)); + cc_param.init_wnd = (u8)parent->init_wnd; + cc_param.alpha = (u8)parent->alpha; + cc_param.beta = (u8)parent->beta; + cc_param.gamma = (u8)parent->gamma; + cc_param.eta = (u8)parent->eta; + cc_param.wnd_min = (u8)parent->wnd_min; + + ret = udma_cc_set_para(udma_dev, &cc_param); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to update cc pattern, algo: %d, index: %d, ret: %d\n", + cc_param.ub_cc_algo, cc_param.index, ret); + return ret; + } + + return 0; +} + +int udma_sysfs_set_caqm(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_caqm_pattern_ctx *parent = + (struct udma_caqm_pattern_ctx *)parent_addr; + struct ub_cc_params cc_param = { 0 }; + int ret = 0; + + cc_param.ub_cc_algo = UB_CC_ALGO_CAQM; + cc_param.index = + (u8)udma_sysfs_get_index_in_parent_array(&(parent->head)); + cc_param.alpha = (u8)parent->alpha; + cc_param.beta = (u8)parent->beta; + cc_param.eta = (u8)parent->eta; + cc_param.wnd_min = (u8)parent->wnd_min; + cc_param.init_wnd = (u8)parent->init_wnd; + cc_param.cc_unit = (u8)parent->cc_unit; + + ret = udma_cc_set_para(udma_dev, &cc_param); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to update cc pattern, algo: %d, index: %d, ret: %d\n", + cc_param.ub_cc_algo, cc_param.index, ret); + return ret; + } + + return 0; +} + +int udma_sysfs_set_user_defined(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_user_defined_pattern_ctx *parent = + (struct udma_user_defined_pattern_ctx *)parent_addr; + struct ub_cc_params cc_param = { 0 }; + int ret = 0; + u32 template_index = 0; + u8 algo[USER_DEFINED_TEMPLATE_NUM] = { UB_CC_ALGO_USER_1, + UB_CC_ALGO_USER_2 }; + + template_index = udma_sysfs_get_index_in_parent_array( + (struct udma_sysfs_head *)parent->head.parent_addr); + if (template_index >= USER_DEFINED_TEMPLATE_NUM) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to get user_defined cc, index: %d not in [0, %d).\n", + template_index, USER_DEFINED_TEMPLATE_NUM); + return -EINVAL; + } + cc_param.ub_cc_algo = algo[template_index]; + cc_param.index = + (u8)udma_sysfs_get_index_in_parent_array(&(parent->head)); + cc_param.dw[DW_IDX0] = parent->dw0; + cc_param.dw[DW_IDX1] = parent->dw1; + cc_param.dw[DW_IDX2] = parent->dw2; + cc_param.dw[DW_IDX3] = parent->dw3; + + ret = udma_cc_set_para(udma_dev, &cc_param); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "Failed to update cc pattern, algo: %d, index: %d, ret: %d\n", + cc_param.ub_cc_algo, cc_param.index, ret); + return ret; + } + + return 0; +} + +void udma_sysfs_set_def_value_u8(void *value_addr, u32 def, u32 array_num) +{ + u8 *value = (u8 *)value_addr; + + for (u32 i = 0; i < array_num; ++i) { + value[i] = (u8)def; + } +} + +void udma_sysfs_set_def_value_u16(void *value_addr, u32 def, u32 array_num) +{ + u16 *value = (u16 *)value_addr; + + for (u32 i = 0; i < array_num; ++i) { + value[i] = (u16)def; + } +} + +void udma_sysfs_set_def_value_u32(void *value_addr, u32 def, u32 array_num) +{ + u32 *value = (u32 *)value_addr; + + for (u32 i = 0; i < array_num; ++i) { + value[i] = def; + } +} + +void udma_sysfs_set_def_value_wred(void *value_addr, u32 def, u32 array_num) +{ + struct udma_rx_rate_limit_wred *wred = + (struct udma_rx_rate_limit_wred *)value_addr; + + wred->cmd = def; + wred->wred_idx = def; + wred->start_val = def; + wred->end_val = def; +} + +/** + * @brief u8 class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle u8 type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_u8(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + u8 *value = (u8 *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, "%u\n", *value); +} + +/** + * @brief u16 class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle u16 type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_u16(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + u16 *value = (u16 *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, "%u\n", *value); +} + +/** + * @brief u32 class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle u32 type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + u32 *value = (u32 *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, "%u\n", *value); +} +/** + * @brief struct udma_rx_rate_limit_wred class parameters show processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @details use to handle struct udma_rx_rate_limit_wred type parameters of show dynamic as + */ +ssize_t udma_sysfs_show_wred(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf) +{ + struct udma_rx_rate_limit_wred *wred = + (struct udma_rx_rate_limit_wred *)udma_sysfs_get_child_value( + head, node->position_info.offset); + + return sprintf_s(buf, PAGE_SIZE - 1, + "wred_idx=%u start_val=%u end_val=%u\n", + wred->wred_idx, wred->start_val, wred->end_val); +} + +u32 udma_sysfs_read_u32(const void *value, u32 type_size) +{ + switch (type_size) { + case UDMA_SYSFS_VALUE_TYPE_SIZE_U8: + return (u32)(*(u8 *)value); + case UDMA_SYSFS_VALUE_TYPE_SIZE_U16: + return (u32)(*(u16 *)value); + case UDMA_SYSFS_VALUE_TYPE_SIZE_U32: + return *(u32 *)value; + default: + return 0; + } +} + +void udma_sysfs_write_u32(const void *value, u32 type_size, u32 new_value) +{ + switch (type_size) { + case UDMA_SYSFS_VALUE_TYPE_SIZE_U8: + *(u8 *)value = (u8)new_value; + break; + case UDMA_SYSFS_VALUE_TYPE_SIZE_U16: + *(u16 *)value = (u16)new_value; + break; + case UDMA_SYSFS_VALUE_TYPE_SIZE_U32: + *(u32 *)value = new_value; + break; + default: + break; + } +} + +typedef int (*udma_sysfs_u32_post_proc_func)(struct udma_device *udma_dev, + void *parent_addr); +/** + * @brief store handle base function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @param[in] type_size: handle when data of type + * @details use to handle parameters of store common dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static int udma_sysfs_store_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func, + u32 type_size) +{ + void *value = + udma_sysfs_get_child_value(head, node->position_info.offset); + u32 new_value = 0, old_value = udma_sysfs_read_u32(value, type_size); + int ret = 0; + + ret = sscanf_s(buf, "%u", &new_value); + if (ret != 1) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Failed to get %s value, ret: %d\n", node->attr.name, + ret); + return -EIO; + } + + udma_debug(udma_dev->dev, UDMA_M_SYSFS, + "%s old value:%u, new value:%u\n", node->attr.name, + old_value, new_value); + if (new_value == old_value) { + return 0; + } + + if (new_value < node->value.min || new_value > node->value.max) { + udma_err(udma_dev->dev, UDMA_M_SYSFS, + "Invalid param: %s, new value(%u) not in [%u, %u]\n", + node->attr.name, new_value, node->value.min, + node->value.max); + return -EINVAL; + } + + udma_sysfs_write_u32(value, type_size, new_value); + if (post_proc_func != NULL) { + ret = post_proc_func(udma_dev, (void *)head); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_SYSFS, + "%s, post_proc_func execution Failed, ret = %d\n", + node->attr.name, ret); + udma_sysfs_write_u32(value, type_size, old_value); + return ret; + } + } + udma_debug(udma_dev->dev, UDMA_M_SYSFS, "%s now value:%u\n", + node->attr.name, new_value); + + return 0; +} + +/** + * @brief u8 class parameters store processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @details use to handle u8 type parameters of store dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static ssize_t +udma_sysfs_store_u8_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func) +{ + return (ssize_t)udma_sysfs_store_base(udma_dev, head, node, buf, + post_proc_func, sizeof(u8)); +} + +/** + * @brief u16 class parameters store processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @details use to handle u16 type parameters of store dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static ssize_t +udma_sysfs_store_u16_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func) +{ + return udma_sysfs_store_base(udma_dev, head, node, buf, post_proc_func, + sizeof(u16)); +} + +/** + * @brief u32 class parameters store processing function + * @param[in] udma_dev: udma device information + * @param[in] head: data information of the directory where file is located + * @param[in] node: configuration information of file + * @param[in] buf: user modify of information + * @param[in] post_proc_func: post-processing function + * @details use to handle u32 type parameters of store dynamic as, its in will according to configuration information check value range, and call call handle custom update + */ +static ssize_t +udma_sysfs_store_u32_base(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf, + udma_sysfs_u32_post_proc_func post_proc_func) +{ + return udma_sysfs_store_base(udma_dev, head, node, buf, post_proc_func, + sizeof(u32)); +} + +ssize_t udma_sysfs_store_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, NULL); +} + +ssize_t udma_sysfs_store_u32_set_ta2tp_type(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_ta2tp_type); +} + +ssize_t udma_sysfs_store_u32_set_ldcp(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_ldcp); +} + +ssize_t udma_sysfs_store_u32_set_caqm(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_caqm); +} + +ssize_t udma_sysfs_store_u32_set_user_defined(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u32_base(udma_dev, head, node, buf, + udma_sysfs_set_user_defined); +} +ssize_t udma_sysfs_store_u8_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u8_base(udma_dev, head, node, buf, + udma_sysfs_set_rx_rate_limit); +} +ssize_t udma_sysfs_store_u16_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + return udma_sysfs_store_u16_base(udma_dev, head, node, buf, + udma_sysfs_set_rx_rate_limit); +} + +/* sysfs wred config usage: + * echo wr id start end > /sysfs/.../wred + * echo rd id > /sysfs/.../wred + * cat /sysfs/.../wred */ +#define UDMA_MAX_WRED_CMD_STR_LEN 3 +#define UDMA_WRED_WR_ARGC 3 +#define UDMA_WRED_RD_ARGC 1 + +static int udma_get_wred_cmd(struct udma_device *udma_dev, + struct udma_rx_rate_limit_wred *wred, + const char *buf) +{ + int ret; + u32 cmd = 0, idx = 0, start = 0, end = 0; + const char *argv_str = NULL; + + if (strlen(buf) <= UDMA_MAX_WRED_CMD_STR_LEN) { + goto wred_str_err; + } + + argv_str = buf + UDMA_MAX_WRED_CMD_STR_LEN; + if (strncmp(buf, "wr ", UDMA_MAX_WRED_CMD_STR_LEN) == 0) { + ret = sscanf_s(argv_str, "%u %u %u", &idx, &start, &end); + if (ret != UDMA_WRED_WR_ARGC) { + goto wred_str_err; + } + } else if (strncmp(buf, "rd ", UDMA_MAX_WRED_CMD_STR_LEN) == 0) { + ret = sscanf_s(argv_str, "%u", &idx); + if (ret != UDMA_WRED_RD_ARGC) { + goto wred_str_err; + } + cmd = 1; + } else { + goto wred_str_err; + } + + if (idx > UDMA_ACC_MAX_WRED_IDX || end > UDMA_ACC_MAX_WRED_VAL || + start > end) { + udma_err(udma_dev->dev, UDMA_M_RX_RATE_LIMITE, + "Invalid wred args: %u %u %u\n", idx, start, end); + goto wred_str_err; + } + + wred->cmd = cmd; + wred->wred_idx = idx; + wred->start_val = start; + wred->end_val = end; + + return 0; +wred_str_err: + udma_err(udma_dev->dev, UDMA_M_RX_RATE_LIMITE, "Invalid wred cmd %s\n", + buf); + return -EINVAL; +} + +ssize_t udma_sysfs_store_wred_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf) +{ + struct udma_rx_rate_limit_wred *wred = + (struct udma_rx_rate_limit_wred *)udma_sysfs_get_child_value( + head, node->position_info.offset); + int ret = 0; + + ret = udma_get_wred_cmd(udma_dev, wred, buf); + if (ret != 0) { + return (ssize_t)ret; + } + + ret = udma_rx_rate_limit_cfg_wred(udma_dev, wred->cmd, wred->wred_idx, + &wred->start_val, &wred->end_val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_RX_RATE_LIMITE, + "Config wred value failed: %d\n", ret); + } + + return (ssize_t)ret; +} + +int udma_sysfs_name_oneself(const char *name, int index, void *ext_value, + char *out_name) +{ + return sprintf_s(out_name, UDMA_SYSFS_NAME_MAX_LEN, "%s", name); +} + +int udma_sysfs_name_index(const char *name, int index, void *ext_value, + char *out_name) +{ + return sprintf_s(out_name, UDMA_SYSFS_NAME_MAX_LEN, "%d", index); +} + +int udma_sysfs_name_oneself_index_plus_one(const char *name, int index, + void *ext_value, char *out_name) +{ + return sprintf_s(out_name, UDMA_SYSFS_NAME_MAX_LEN, "%s_%d", name, + index + 1); +} + +int udma_sysfs_get_mtt_pro_en(struct udma_sysfs_res *sysfs_res, u32 *mtt_pro_en) +{ + struct udma_dfx_perf_pattern *perf = &sysfs_res->dfx_ctx.perf; + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + int ret = 0; + + *mtt_pro_en = perf->huge_page_en; + if (*mtt_pro_en == 1) { + return 0; + } + + ret = udma_get_mtt_protect_state(udma_dev, mtt_pro_en); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "update mtt protect state failed, ret = %d\n", ret); + } + + return ret; +} + +u32 udma_sysfs_get_huge_page_en(struct udma_sysfs_res *sysfs_res) +{ + struct udma_dfx_perf_pattern *perf = &sysfs_res->dfx_ctx.perf; + + return perf->huge_page_en; +} + +bool udma_sysfs_get_debug_en(struct udma_sysfs_res *sysfs_res) +{ + return (sysfs_res->dfx_ctx.debug_en == 1); +} + +bool udma_sysfs_get_fis_en(struct udma_sysfs_res *sysfs_res) +{ + return (sysfs_res->dfx_ctx.perf.fis_en == 1); +} + +#define UDMA_SYSFS_XID_LOW_3_BIT_VALID_MASK 0x1 +#define UDMA_SYSFS_XID_LOW_3_BIT_VALID_SHIFT 0x1 +bool udma_sysfs_get_xid_low_3_bit(struct udma_sysfs_res *sysfs_res, + u32 *out_value) +{ + u32 num = 0; + + num = sysfs_res->dfx_ctx.perf.xid_low_3_bit; + + if ((num & UDMA_SYSFS_XID_LOW_3_BIT_VALID_MASK) == 0) { + return false; + } + + *out_value = num >> UDMA_SYSFS_XID_LOW_3_BIT_VALID_SHIFT; + + return true; +} + +int udma_sysfs_init_dwqe_dis(struct udma_device *udma_dev, void *parent_addr) +{ + struct udma_dfx_perf_pattern *parent = + (struct udma_dfx_perf_pattern *)parent_addr; + + parent->dwqe_dis = udma_dev->ub_cap.dev_cap.dwqe_dis; + + return 0; +} + +u32 udma_sysfs_get_dwqe_dis(struct udma_sysfs_res *sysfs_res) +{ + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + + return udma_dev->ub_cap.dev_cap.dwqe_dis; +} + +u32 udma_sysfs_get_tp_depth(struct udma_sysfs_res *sysfs_res) +{ + struct udma_device *udma_dev = to_udma_dev_by_sysfs(sysfs_res); + + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + return TP_Q_DEF_DEPTH; + } + + return (1 << sysfs_res->dfx_ctx.perf.tp_depth_shift); +} + +int udma_sysfs_set_tp_depth(struct udma_sysfs_res *sysfs_res, u32 new_value) +{ +#define DWORD_BITS 32 + struct udma_dfx_perf_pattern *perf = &sysfs_res->dfx_ctx.perf; + struct attribute attr = { .name = "tp_depth_shift" }; + char buf[UDMA_SYSFS_NAME_MAX_LEN]; + u32 new_value_shift = 0; + int count = 0; + + new_value_shift = (u32)ilog2(new_value); + if (new_value_shift >= DWORD_BITS) { + return -EINVAL; + } + if (new_value != (1 << new_value_shift)) { + return -EINVAL; + } + + count = sprintf_s(buf, UDMA_SYSFS_NAME_MAX_LEN, "%u", new_value_shift); + if (count < 0) { + return count; + } + + return (int)udma_sysfs_common_store(&perf->head.kobj, &attr, buf, + (size_t)count); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h new file mode 100644 index 000000000..44e50b784 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_common.h @@ -0,0 +1,88 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_COMMON_H +#define UDMA_SYSFS_COMMON_H + +#include "udma_sysfs.h" +#include "udma_common.h" + +static inline struct udma_device * +to_udma_dev_by_sysfs(const struct udma_sysfs_res *sysfs) +{ + return (struct udma_device *)container_of(sysfs, struct udma_device, + sysfs_res); +} + +static inline struct udma_sysfs_head * +to_sysfs_head_by_kobject(const struct kobject *kobject) +{ + return (struct udma_sysfs_head *)container_of( + kobject, struct udma_sysfs_head, kobj); +} + +void udma_sysfs_set_def_value_u8(void *value_addr, u32 def, u32 array_num); +void udma_sysfs_set_def_value_u16(void *value_addr, u32 def, u32 array_num); +void udma_sysfs_set_def_value_u32(void *value_addr, u32 def, u32 array_num); +void udma_sysfs_set_def_value_wred(void *value_addr, u32 def, u32 array_num); +ssize_t udma_sysfs_show_u8(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_show_u16(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_show_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_show_wred(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, char *buf); +ssize_t udma_sysfs_store_u32(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, const char *buf); +ssize_t udma_sysfs_store_u32_set_ta2tp_type(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u32_set_ldcp(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u32_set_caqm(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u32_set_user_defined(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u8_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_u16_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); +ssize_t udma_sysfs_store_wred_set_rx_rate_limit(struct udma_device *udma_dev, + struct udma_sysfs_head *head, + struct udma_sysfs_node *node, + const char *buf); + +int udma_sysfs_name_oneself(const char *name, int index, void *ext_value, + char *out_name); +int udma_sysfs_name_index(const char *name, int index, void *ext_value, + char *out_name); +int udma_sysfs_name_oneself_index_plus_one(const char *name, int index, + void *ext_value, char *out_name); + +int udma_sysfs_set_rx_rate_limit(struct udma_device *udma_dev, + void *parent_addr); +int udma_sysfs_set_ldcp(struct udma_device *udma_dev, void *parent_addr); +int udma_sysfs_set_caqm(struct udma_device *udma_dev, void *parent_addr); +int udma_sysfs_set_user_defined(struct udma_device *udma_dev, + void *parent_addr); +int udma_sysfs_init_dwqe_dis(struct udma_device *udma_dev, void *parent_addr); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c new file mode 100644 index 000000000..9d0820001 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#include <linux/stat.h> + +#include "udma_sysfs.h" +#include "udma_sysfs_common.h" +#include "udma_sysfs_config.h" + +#define UDMA_SYSFS_FILE_MODE (S_IWUGO | S_IRUGO) +#define UDMA_SYSFS_NAME_INIT(_name) \ + { \ + .name = (_name), .mode = UDMA_SYSFS_FILE_MODE \ + } +#define UDMA_SYSFS_OPS_INIT(_show, _store) \ + { \ + .show = (_show), .store = (_store) \ + } +#define UDMA_SYSFS_NODE_VALUE_INIT(_min, _def, _max, _set_def_value) \ + { \ + .min = (_min), .def = (_def), .max = (_max), \ + .set_def_value = (_set_def_value) \ + } +#define UDMA_SYSFS_NODE_VALUE_INIT_U8(_min, _def, _max) \ + UDMA_SYSFS_NODE_VALUE_INIT((_min), (_def), (_max), \ + (udma_sysfs_set_def_value_u8)) +#define UDMA_SYSFS_NODE_VALUE_INIT_U16(_min, _def, _max) \ + UDMA_SYSFS_NODE_VALUE_INIT((_min), (_def), (_max), \ + (udma_sysfs_set_def_value_u16)) +#define UDMA_SYSFS_NODE_VALUE_INIT_U32(_min, _def, _max) \ + UDMA_SYSFS_NODE_VALUE_INIT((_min), (_def), (_max), \ + udma_sysfs_set_def_value_u32) +#define UDMA_SYSFS_NAME_EXT_INIT(_func, _ext_value) \ + { \ + .get_name_func = (_func), .ext_value = (_ext_value) \ + } +#define UDMA_SYSFS_SUPPORT_INIT(_type, _version) \ + { \ + .func_type = (_type), .func_ver = (_version) \ + } +#define UDMA_SYSFS_CHILD_NODE_INIT(child_node_list) \ + { \ + .node_list = child_node_list, \ + .length = ARRAY_SIZE(child_node_list) \ + } +#define UDMA_SYSFS_POSITION_INFO_INIT(_parent_type, _member, _oneself_type, \ + _array_num) \ + { \ + .offset = offsetof(_parent_type, _member), \ + .array_num = (_array_num), \ + .element_size = sizeof(_oneself_type) \ + } + +/* corresponding dfx->perf directory content, see @ > struct udma_dfx_perf_pattern */ +static struct udma_sysfs_node g_sysfs_dir_dfx_perf[] = { + { .attr = UDMA_SYSFS_NAME_INIT("jetty_slice_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, jetty_slice_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 1, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("ta2tp_type"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, ta2tp_type, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 3), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ta2tp_type) }, + { .attr = UDMA_SYSFS_NAME_INIT("xid_low_3_bit"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, xid_low_3_bit, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 15), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { + .attr = UDMA_SYSFS_NAME_INIT("dwqe_dis"), + .name_ext = + UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT( + UDMA_SYSFS_TAF, UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, dwqe_dis, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32), + .ext_update = udma_sysfs_init_dwqe_dis, + }, + { .attr = UDMA_SYSFS_NAME_INIT("tp_depth_shift"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, tp_depth_shift, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(6, 7, 11), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("huge_page_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, huge_page_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("fis_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_perf_pattern, fis_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, +}; + +/* corresponding dfx directory content, see @ > struct udma_dfx_ctx */ +static struct udma_sysfs_node g_sysfs_dir_dfx[] = { + { .attr = UDMA_SYSFS_NAME_INIT("perf"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_dfx_ctx, perf, struct udma_dfx_perf_pattern, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_dfx_perf) }, + { .attr = UDMA_SYSFS_NAME_INIT("debug_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + debug_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("drop_aeqe_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + drop_aeqe_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("dump_tp_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + dump_tp_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 1, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("tid"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + tid, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UDMA_MAX_TOKEN_ID_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("eid"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + eid, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UDMA_MAX_EID_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, + { .attr = UDMA_SYSFS_NAME_INIT("hbm_en"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TAF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT(struct udma_dfx_ctx, + hbm_en, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, 1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32) }, +}; + +/* corresponding cc->ldcp->pattern directory content, see @ > struct udma_ldcp_pattern_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_ldcp_pattern[] = { + { .attr = UDMA_SYSFS_NAME_INIT("init_wnd"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, init_wnd, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(UDMA_MIN_LDCP_PARA, + UDMA_DEFAULT_INIT_WND, + UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, alpha, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_ALPHA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("beta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, beta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_BETA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("gamma"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, gamma, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_GAMMA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("eta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, eta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_ETA, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("wnd_min"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_ldcp_pattern_ctx, wnd_min, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_LDCP_PARA, UDMA_DEFAULT_WND_MIN, UDMA_MAX_LDCP_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_ldcp) }, +}; + +/* corresponding cc->ldcp directory content, see @ > struct udma_cc_ldcp_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_ldcp[] = { + { .attr = UDMA_SYSFS_NAME_INIT("pattern"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_index, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ldcp_ctx, pattern, + struct udma_ldcp_pattern_ctx, LDCP_PATTERN_ARRAY_LEN), + .ext_update = udma_sysfs_set_ldcp, + .child_info = + UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_ldcp_pattern) }, +}; + +/* corresponding cc->caqm->pattern directory content, see @ > struct udma_caqm_pattern_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_caqm_pattern[] = { + { .attr = UDMA_SYSFS_NAME_INIT("caqm_alpha"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, alpha, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_ALPHA, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_beta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, beta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_BETA, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_eta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, eta, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_ETA, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_wnd_min"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, wnd_min, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_WND_MIN, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_init_wnd"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, init_wnd, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(UDMA_MIN_CAQM_PARA, + UDMA_DEFAULT_INIT_WND, + UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm_cc_unit"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_caqm_pattern_ctx, cc_unit, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32( + UDMA_MIN_CAQM_PARA, UDMA_DEFAULT_CC_UNIT, UDMA_MAX_CAQM_PARA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_caqm) }, +}; + +/* corresponding cc->caqm directory content, see @ > struct udma_cc_caqm_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_caqm[] = { + { .attr = UDMA_SYSFS_NAME_INIT("pattern"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_index, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_caqm_ctx, pattern, + struct udma_caqm_pattern_ctx, CAQM_PATTERN_ARRAY_LEN), + .ext_update = udma_sysfs_set_caqm, + .child_info = + UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_caqm_pattern) }, +}; + +/* corresponding cc->user_defined->pattern directory content, see @ > struct udma_user_defined_pattern_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_user_defined_pattern[] = { + { .attr = UDMA_SYSFS_NAME_INIT("dw0"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw0, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, + { .attr = UDMA_SYSFS_NAME_INIT("dw1"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw1, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, + { .attr = UDMA_SYSFS_NAME_INIT("dw2"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw2, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, + { .attr = UDMA_SYSFS_NAME_INIT("dw3"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_user_defined_pattern_ctx, dw3, u32, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U32(0, 0, UINT_MAX), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u32, + udma_sysfs_store_u32_set_user_defined) }, +}; + +/* corresponding cc->user_defined directory of content, see @ > struct udma_cc_user_defined_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc_user_defined[] = { + { .attr = UDMA_SYSFS_NAME_INIT("pattern"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_index, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_user_defined_ctx, pattern, + struct udma_user_defined_pattern_ctx, + USER_DEFINED_PATTERN_ARRAY_LEN), + //.ext_update = udma_sysfs_set_user_defined, // todo mpu side not adaptation, pending mpu side adaptation after place enable + .child_info = UDMA_SYSFS_CHILD_NODE_INIT( + g_sysfs_dir_cc_user_defined_pattern) }, +}; + +/* corresponding cc directory content, see @ > struct udma_cc_ctx */ +static struct udma_sysfs_node g_sysfs_dir_cc[] = { + { .attr = UDMA_SYSFS_NAME_INIT("ldcp"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ctx, ldcp, struct udma_cc_ldcp_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_ldcp) }, + { .attr = UDMA_SYSFS_NAME_INIT("caqm"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ctx, caqm, struct udma_cc_caqm_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_caqm) }, + { .attr = UDMA_SYSFS_NAME_INIT("user_defined"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT( + udma_sysfs_name_oneself_index_plus_one, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_cc_ctx, user_defined, + struct udma_cc_user_defined_ctx, USER_DEFINED_TEMPLATE_NUM), + .child_info = + UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc_user_defined) }, +}; + +/* corresponding cc directory content, see @ > struct udma_cc_ctx */ +static struct udma_sysfs_node g_sysfs_dir_rx_rate_limit[] = { + { .attr = UDMA_SYSFS_NAME_INIT("down_period"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, down_period, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_DOWN_PERIOD, + UDMA_ACC_DEFAULT_DOWN_PERIOD, + UDMA_ACC_MAX_DOWN_PERIOD), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("up_period"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, up_period, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_UP_PERIOD, + UDMA_ACC_DEFAULT_UP_PERIOD, + UDMA_ACC_MAX_UP_PERIOD), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha1"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, alpha1, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_ALPHA1, + UDMA_ACC_DEFAULT_ALPHA1, + UDMA_ACC_MAX_ALPHA1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("n1"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, n1, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16( + UDMA_ACC_MIN_N1, UDMA_ACC_DEFAULT_N1, UDMA_ACC_MAX_N1), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha2"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, alpha2, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_ALPHA2, + UDMA_ACC_DEFAULT_ALPHA2, + UDMA_ACC_MAX_ALPHA2), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("alpha3"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, alpha3, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_ALPHA3, + UDMA_ACC_DEFAULT_ALPHA3, + UDMA_ACC_MAX_ALPHA3), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("min_rate"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, min_rate, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16(UDMA_ACC_MIN_MIN_RATE, + UDMA_ACC_DEFAULT_MIN_RATE, + UDMA_ACC_MAX_MIN_RATE), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("init_rate"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, init_rate, u8, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U8(UDMA_MIN_INIT_RATE, + UDMA_ACC_DEFAULT_INIT_RATE, + UDMA_MAX_INIT_RATE), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u16, + udma_sysfs_store_u16_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("n2"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, n2, u8, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U8( + UDMA_ACC_MIN_N2, UDMA_ACC_DEFAULT_N2, UDMA_ACC_MAX_N2), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u8, + udma_sysfs_store_u8_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("beta"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, beta, u16, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT_U16( + UDMA_ACC_MIN_BETA, UDMA_ACC_DEFAULT_BETA, UDMA_ACC_MIN_BETA), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_u8, + udma_sysfs_store_u8_set_rx_rate_limit) }, + { .attr = UDMA_SYSFS_NAME_INIT("wred"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_FILE, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_rx_rate_limit_ctx, wred, + struct udma_rx_rate_limit_wred, 1), + .value = UDMA_SYSFS_NODE_VALUE_INIT(0, 0, UINT_MAX, + udma_sysfs_set_def_value_u32), + .ops = UDMA_SYSFS_OPS_INIT(udma_sysfs_show_wred, + udma_sysfs_store_wred_set_rx_rate_limit) }, +}; + +/* directory structure: + * ├── dfx (see @ > g_sysfs_dir_dfx) + * │ ├── perf (see @ > g_sysfs_dir_dfx_perf) + * │ │ ├── jetty_slice_en + * │ │ ├── xid_low_3_bit + * │ │ ├── dwqe_dis + * │ │ ├── tp_depth_shift + * │ │ ├── huge_page_en + * │ │ └── ta2tp_type + * │ ├── debug_en + * │ ├── drop_aeqe_en + * │ └── dump_tp_en + * ├── rx_rate_limit (see @ > g_sysfs_dir_rx_rate_limit) + * │ ├── down_period + * │ ├── up_period + * │ ├── alpha1 + * │ ├── n1 + * │ ├── alpha2 + * │ ├── alpha3 + * │ ├── min_rate + * │ ├── init_rate + * │ ├── n2 + * │ ├── beta + * │ └── wred + * └── cc (see @ > g_sysfs_dir_cc) + * ├── ldcp (see @ > g_sysfs_dir_cc_ldcp) + * │ └── pattern (see @ > g_sysfs_dir_cc_ldcp_pattern) // actual directory name is 0-7, total 8(count word) directory + * │ ├── init_wnd + * │ ├── alpha + * │ ├── beta + * │ ├── gamma + * │ ├── eta + * │ └── wnd_min + * ├── caqm (see @ > g_sysfs_dir_cc_caqm) + * │ └── pattern (see @ > g_sysfs_dir_cc_caqm_pattern) // actual directory name is 0-2, total 3(count word) directory + * │ ├── caqm_alpha + * │ ├── caqm_beta + * │ ├── caqm_eta + * │ ├── caqm_wnd_min + * │ ├── caqm_init_wnd + * │ └── caqm_cc_unit + * └── user_defined (see @ > g_sysfs_dir_cc_user_defined) // actual directory name is user_defined_1-2, total 2(count word) directory + * └── pattern (see @ > g_sysfs_dir_cc_user_defined_pattern) // actual directory name is 0-7, total 8(count word) directory + * ├── dw0 + * ├── dw1 + * ├── dw2 + * └── dw3 + * */ + +/* corresponding sysfs root directory content, see @ > struct udma_sysfs_res */ +static struct udma_sysfs_node g_sysfs_config_info[] = { + { .attr = UDMA_SYSFS_NAME_INIT("dfx"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_ALL, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_sysfs_res, dfx_ctx, struct udma_dfx_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_dfx) }, + { .attr = UDMA_SYSFS_NAME_INIT("cc"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_sysfs_res, cc_ctx, struct udma_cc_ctx, 1), + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_cc) }, + { .attr = UDMA_SYSFS_NAME_INIT("rx_rate_limit"), + .name_ext = UDMA_SYSFS_NAME_EXT_INIT(udma_sysfs_name_oneself, NULL), + .type = UDMA_SYSFS_NODE_DIR, + .support_info = UDMA_SYSFS_SUPPORT_INIT(UDMA_SYSFS_TPF, + UDMA_DEV_TYPE_HI1825_V100), + .position_info = UDMA_SYSFS_POSITION_INFO_INIT( + struct udma_sysfs_res, rx_rate_limit_ctx, + struct udma_rx_rate_limit_ctx, 1), + .ext_update = udma_sysfs_set_rx_rate_limit, + .child_info = UDMA_SYSFS_CHILD_NODE_INIT(g_sysfs_dir_rx_rate_limit) }, +}; + +struct udma_sysfs_node *udma_sysfs_get_config_info(void) +{ + return g_sysfs_config_info; +} + +int udma_sysfs_get_config_length(void) +{ + return ARRAY_SIZE(g_sysfs_config_info); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h new file mode 100644 index 000000000..6d7d1ae77 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/sysfs/udma_sysfs_config.h @@ -0,0 +1,13 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + + +#ifndef UDMA_SYSFS_CONFIG_H +#define UDMA_SYSFS_CONFIG_H + +#include "udma_sysfs.h" + +struct udma_sysfs_node *udma_sysfs_get_config_info(void); +int udma_sysfs_get_config_length(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c new file mode 100644 index 000000000..1543c35ec --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.c @@ -0,0 +1,330 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <net/ipv6.h> +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/iommu.h> +#include <linux/uaccess.h> +#include <linux/types.h> +#include <linux/semaphore.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> +#include "ubcore_api.h" +/* + * Generally, a driver should not include ubcore_uapi.h for kernel users, + * the udma_pci driver for simluation calls ubcore_find_jfs/jfrin the ubcore_uapi.h to reduce complexity + */ +#include "hinic5_crm.h" +#include "hinic5_profile.h" +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_cmd.h" +#include "udma_cmd_sip.h" +#include "udma_cmd_eid.h" +#include "udma_net.h" + +static void udma_fill_sip_update_cmd(struct ub_sip_attr *sip_attr, + const struct ubcore_net_addr *net_addr) +{ + vlan_u vlan = { 0 }; + + if (net_addr->type == UBCORE_NET_ADDR_TYPE_IPV4) { + sip_attr->sip3 = net_addr->net_addr.in4.addr; + sip_attr->dw6.sip_type = 0; + } else { + sip_attr->sip0 = net_addr->net_addr.in6.subnet_prefix & + 0xffffffff; + sip_attr->sip1 = + (net_addr->net_addr.in6.subnet_prefix >> 0x20) & + 0xffffffff; + sip_attr->sip2 = net_addr->net_addr.in6.interface_id & + 0xffffffff; + sip_attr->sip3 = (net_addr->net_addr.in6.interface_id >> 0x20) & + 0xffffffff; + sip_attr->dw6.sip_type = 1; + } + sip_attr->dw6.smac_h16 = + UDMA_GET_MAC_HI(net_addr->mac[0], net_addr->mac[1]); + sip_attr->smac_l32 = + UDMA_GET_MAC_LO(net_addr->mac[0x2], net_addr->mac[0x3], + net_addr->mac[0x4], net_addr->mac[0x5]); + + vlan.value = (u16)net_addr->vlan; + sip_attr->dw4.cvlan = vlan.bs.vlan; + sip_attr->dw4.vlan_cfi = vlan.bs.cfi; + sip_attr->dw4.vlan_pri = vlan.bs.pri; + sip_attr->dw6.tag = (vlan.bs.vlan != 0) ? VLAN_VALID : VLAN_INVALID; + + sip_attr->sip0 = be32_to_cpu(sip_attr->sip0); + sip_attr->sip1 = be32_to_cpu(sip_attr->sip1); + sip_attr->sip2 = be32_to_cpu(sip_attr->sip2); + sip_attr->sip3 = be32_to_cpu(sip_attr->sip3); +} + +static void udma_show_sip_info(struct udma_device *udma_dev, + const struct ub_sip_attr *sip_attr) +{ + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip0:0x%x\n", sip_attr->sip0); + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip1:0x%x\n", sip_attr->sip1); + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip2:0x%x\n", sip_attr->sip2); + udma_debug(udma_dev->dev, UDMA_M_DEV, "sip3:0x%x\n", sip_attr->sip3); + udma_debug(udma_dev->dev, UDMA_M_DEV, "dw4:0x%x\n", + sip_attr->dw4.dw4_value); + udma_debug(udma_dev->dev, UDMA_M_DEV, "dw5:0x%x\n", + sip_attr->dw5.dw5_value); + udma_debug(udma_dev->dev, UDMA_M_DEV, "dw6:0x%x\n", + sip_attr->dw6.dw6_value); + udma_debug(udma_dev->dev, UDMA_M_DEV, "smac_l32:0x%x\n", + sip_attr->smac_l32); +} + +int udma_query_sip(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr) +{ + int ret; + + if (sip_idx >= UDMA_SIP_NUM_MAX) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "sip index in valied, sip_index:%u, maxnum:%d\n", + sip_idx, UDMA_SIP_NUM_MAX); + return -EINVAL; + } + + ret = udma_cmd_sip_query(udma_dev, sip_idx, sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "query sip failed, sip_index:%u\n", sip_idx); + return ret; + } + udma_show_sip_info(udma_dev, sip_attr); + + return ret; +} + +int udma_add_netaddr(struct ubcore_device *dev, + struct ubcore_net_addr *net_addr, u32 index) +{ + struct udma_device *udma_dev = NULL; + struct cmd_mpu_addr_info addr_info = { 0 }; + struct ub_sip_attr sip_attr = { 0 }; + int ret; + + if (dev == NULL || net_addr == NULL) { + udma_pr_err(UDMA_M_DEV, + "parameters is incorrect, dev(%d), net_addr(%d).\n", + !!dev, !!net_addr); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + + udma_info(udma_dev->dev, UDMA_M_DEV, "start\n"); + + /* issue to MPU */ + if (net_addr->type == UBCORE_NET_ADDR_TYPE_IPV4) { + addr_info.ip_addr.ipv4.val = net_addr->net_addr.in4.addr; + addr_info.ip_ver = 0; + } else { + addr_info.ip_addr.ipv6.high = + net_addr->net_addr.in6.subnet_prefix; + addr_info.ip_addr.ipv6.low = + net_addr->net_addr.in6.interface_id; + addr_info.ip_ver = 1; + } + ret = udma_cfg_ipsu_tbl_ipaddr(udma_dev, &addr_info, TYPE_MPU_IPSU_ADD); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "cfg tbl ip addr failed\n"); + return ret; + } + + /* issue to NPU */ + udma_fill_sip_update_cmd(&sip_attr, net_addr); + ret = udma_cmd_sip_add(udma_dev, index, &sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "add sip failed, ret:%d, index:%u\n", ret, index); + return ret; + } + + ret = udma_query_sip(udma_dev, index, &sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "query sip failed, ret:%d\n", ret); + } + + return ret; +} + +int udma_del_netaddr(struct ubcore_device *dev, const u32 index) +{ + struct udma_device *udma_dev = NULL; + struct ub_sip_attr sip_attr = { 0 }; + struct cmd_mpu_addr_info addr_info = { 0 }; + int ret; + + if (dev == NULL) { + udma_pr_err(UDMA_M_DEV, "parameters is incorrect, dev(%d).\n", + !!dev); + return -EINVAL; + } + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_DEV, "delete net addr\n"); + + ret = udma_query_sip(udma_dev, index, &sip_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "query sip faild\n"); + return ret; + } + if (sip_attr.dw6.sip_type == UBCORE_NET_ADDR_TYPE_IPV4) { + addr_info.ip_ver = 0; + addr_info.ip_addr.ipv4.val = sip_attr.sip3; + } else { + addr_info.ip_ver = 1; + addr_info.ip_addr.ipv6.high = sip_attr.sip1; + addr_info.ip_addr.ipv6.high = + (addr_info.ip_addr.ipv6.high << 0x20) | sip_attr.sip0; + addr_info.ip_addr.ipv6.low = sip_attr.sip3; + addr_info.ip_addr.ipv6.low = + (addr_info.ip_addr.ipv6.low << 0x20) | sip_attr.sip2; + } + ret = udma_cfg_ipsu_tbl_ipaddr(udma_dev, &addr_info, TYPE_MPU_IPSU_DEL); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "unset ip addr failed\n"); + return ret; + } + + ret = udma_cmd_sip_del(udma_dev, index); + if (ret != 0) + udma_err(udma_dev->dev, UDMA_M_DEV, + "del sip failed, ret:%d, index:%u\n", ret, index); + + return ret; +} + +void udma_fill_ueid_info(struct udma_device *udma_dev, + struct ubcore_ueid_cfg *cfg, + struct cmd_mpu_ueid_addr_info *addr_info) +{ + if (udma_dev->ub_dev.cfg.pattern == UBCORE_PATTERN_1) { + addr_info->nlp = 0; + if (udma_dev->ub_dev.sip_table.entry->sip_info.addr.type == + UBCORE_NET_ADDR_TYPE_IPV4) { + addr_info->ip.ipv4.val = cfg->eid.in4.addr; + addr_info->ip_ver = UBCORE_NET_ADDR_TYPE_IPV4; + } else { + addr_info->ip.ipv6.high = cfg->eid.in6.subnet_prefix; + addr_info->ip.ipv6.low = cfg->eid.in6.interface_id; + addr_info->ip_ver = UBCORE_NET_ADDR_TYPE_IPV6; + } + } else { + addr_info->nlp = 1; + addr_info->upi = cfg->upi; + if ((memcpy_s(addr_info->ip.eid.id, + sizeof(addr_info->ip.eid.id), cfg->eid.raw, + sizeof(cfg->eid.raw))) != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "Copy ip eid fail.\n"); + } + } +} + +int udma_cmd_ueid_add(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct cmd_mpu_ueid_addr_info addr_info = { 0 }; + + int ret; + int roll_ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_DEV, + "Invalid parameter: dev(%d), cfg(%d).\n", !!dev, + !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_DEV, + "add ueid, vfid: %u, upi: 0x%08x, idx: %u\n", vf_id, cfg->upi, + cfg->eid_index); + + if (udma_dev_is_tpf(udma_dev) == 1 && + (vf_id == EID_ERR_VFID || + vf_id == hinic5_global_func_id(udma_dev->hwdev))) { + udma_warn(udma_dev->dev, UDMA_M_DEV, + "tpf not support add ueid\n"); + return 0; + } + /* issue to MPU */ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + udma_fill_ueid_info(udma_dev, cfg, &addr_info); + ret = udma_ipsu_ta_vf_ipaddr(udma_dev, &addr_info, + TYPE_MPU_IPSU_ADD, vf_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "ta vf add mpu failed, ret: %d \n", ret); + return ret; + } + } + /* issue to NPU */ + ret = udma_ueid_add_npu_cmdq(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "npu add ueid failed, ret:%d\n", ret); + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + goto err_add_ueid_to_npu; + } + return ret; + } + return 0; + +err_add_ueid_to_npu: + roll_ret = udma_ipsu_ta_vf_ipaddr(udma_dev, &addr_info, + TYPE_MPU_IPSU_DEL, vf_id); + if (roll_ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "ta vf ip delete failed ret: %d \n", roll_ret); + } + return ret; +} + +int udma_cmd_ueid_delete(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + struct cmd_mpu_ueid_addr_info addr_info = { 0 }; + int ret; + + if ((dev == NULL) || (cfg == NULL)) { + udma_pr_err(UDMA_M_DEV, + "Invalid parameter: dev(%d), cfg(%d).\n", !!dev, + !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + udma_info(udma_dev->dev, UDMA_M_DEV, + "delete ueid, vfid: %u, upi: 0x%08x, idx: %u\n", vf_id, + cfg->upi, cfg->eid_index); + + ret = delete_ueid_cmd(dev, vf_id, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, "Delete ueid failed: %d\n", + ret); + return ret; + } + + /* issue to mpu */ + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100) { + udma_fill_ueid_info(udma_dev, cfg, &addr_info); + ret = udma_ipsu_ta_vf_ipaddr(udma_dev, &addr_info, + TYPE_MPU_IPSU_DEL, vf_id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_DEV, + "ta vf ip delete failed ret:%d \n", ret); + } + } + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h new file mode 100644 index 000000000..eca88d7a2 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/table_manage/udma_net.h @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_NET_H +#define UDMA_NET_H + +#include "udma_common.h" +#include "udma_cmd_sip.h" + +#define UDMA_SIP_NUM_MAX (16) // need and sdk in specification corresponding + +#define MTU_VAL_256 256 +#define MTU_VAL_512 512 +#define MTU_VAL_1024 1024 +#define MTU_VAL_2048 2048 +#define MTU_VAL_4096 4096 +#define MTU_VAL_8192 8192 + +#define UDMA_GET_MAC_HI(a, b) (((a) << 8) | (b)) +#define UDMA_GET_MAC_LO(a, b, c, d) \ + (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) + +#define VLAN_INVALID 0 +#define VLAN_VALID 1 + +typedef union tag_vlan { + struct { +#if (BYTE_ORDER == BIG_ENDIAN) + u16 pri : 3; + u16 cfi : 1; + u16 vlan : 12; +#else + u16 vlan : 12; + u16 cfi : 1; + u16 pri : 3; +#endif + } bs; + u16 value; +} vlan_u; + +static inline enum ubcore_mtu udma_mtu_int_to_enum(u32 mtu) +{ + if (mtu >= MTU_VAL_8192) + return UBCORE_MTU_8192; + else if (mtu >= MTU_VAL_4096) + return UBCORE_MTU_4096; + else if (mtu >= MTU_VAL_2048) + return UBCORE_MTU_2048; + else if (mtu >= MTU_VAL_1024) + return UBCORE_MTU_1024; + else if (mtu >= MTU_VAL_512) + return UBCORE_MTU_512; + else + return UBCORE_MTU_256; +} + +static inline enum ubcore_mtu udma_get_support_mtu(u32 mtu) +{ + if (mtu >= UBCORE_MTU_8192) + return UBCORE_MTU_8192; + else if (mtu >= UBCORE_MTU_4096) + return UBCORE_MTU_4096; + else + return UBCORE_MTU_1024; +} + +int udma_add_netaddr(struct ubcore_device *dev, + struct ubcore_net_addr *net_addr, u32 index); +int udma_del_netaddr(struct ubcore_device *dev, const u32 index); +int udma_query_sip(struct udma_device *udma_dev, u32 sip_idx, + struct ub_sip_attr *sip_attr); +int udma_cmd_ueid_add(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); +int udma_cmd_ueid_delete(struct ubcore_device *dev, u16 vf_id, + struct ubcore_ueid_cfg *cfg); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c new file mode 100644 index 000000000..22410060a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.c @@ -0,0 +1,1809 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_cmd.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_com.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_tpg.h" +#include "udma_net.h" +#include "udma_misc.h" +#include "udma_jetty.h" +#include "ubcore_uapi.h" +#include "udma_ubrc_comp.h" +#include "udma_cmd_tp.h" +#include "udma_vram.h" +#include "udma_tp.h" + +#define UB_TP_DIP_MASK 0xffffffff +#define UB_TP_DIP_OFFSET 0x20 +#define RIGHT_SHIFT_NUM 2 + +enum { UDMA_MULTI_MTU_DISABLE = 0, UDMA_MULTI_MTU_ENABLE }; + +u32 g_multi_mtu_enable = UDMA_MULTI_MTU_DISABLE; +module_param(g_multi_mtu_enable, uint, 0664); +MODULE_PARM_DESC(g_multi_mtu_enable, + "0 for only support 4K, 1 for support 1K/4K/8K"); + +static void udma_tp_kref_release(struct kref *ref_cnt) +{ + struct udma_tp *tp = container_of(ref_cnt, struct udma_tp, ref_cnt); + + complete(&tp->comp); +} + +void udma_tp_put(void *obj) +{ + struct udma_tp *tp = (struct udma_tp *)obj; + + if (tp != NULL) { + kref_put(&tp->ref_cnt, udma_tp_kref_release); + } +} + +void udma_tp_get(void *obj) +{ + struct udma_tp *tp = (struct udma_tp *)obj; + + kref_get(&tp->ref_cnt); +} + +struct udma_tp *udma_find_tp(struct udma_device *udev, u32 tpn) +{ + struct udma_tp *udma_tp = NULL; + struct mutex *lock = NULL; + + lock = &udev->device_res->tpn_list.node_lock; + mutex_lock(lock); + list_for_each_entry(udma_tp, &udev->device_res->tpn_list.node, node) { + if (udma_tp->tp_info->xid == tpn) { + udma_tp_get(udma_tp); + mutex_unlock(lock); + return udma_tp; + } + } + mutex_unlock(lock); + + return NULL; +} + +static int udma_tpn_alloc(struct udma_device *udma_dev, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp) +{ + struct udma_tpg *ub_tpg = NULL; + struct ubcore_tp *tp = &ub_tp->tp; + u32 tp_all_num; + u32 i; + + if (cfg->tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, "Not support no tpg\n"); + return -EINVAL; + } + + ub_tpg = to_udma_tpg(cfg->tpg); + tp->tpg = cfg->tpg; + tp_all_num = ub_tpg->tp_num; + for (i = 0; i < tp_all_num; i++) { + if (ub_tpg->tp_used[i] == 0) { + ub_tp->tp_info = ub_tpg->tpinfo_list[i]; + tp->tpn = ub_tpg->tpinfo_list[i]->xid; + ub_tpg->tp_used[i] = 1; + ub_tp->tp_info->priv = (void *)ub_tp; + ub_tp->entry_cnt = ub_tpg->tp_q_depth; + udma_debug(udma_dev->dev, UDMA_M_TP, + "index(%u), tpn(%u)\n", i, tp->tpn); + break; + } + } + if (ub_tp->tp_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "The tp is not found in tpg.\n"); + return -EINVAL; + } + + return 0; +} + +static struct tag_cqm_queue *udma_create_pcqc(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + return cqm5_object_rdma_queue_create(udma_dev->hwdev, SERVICE_T_UB, + CQM_OBJECT_RDMA_SCQ, TP_PCQC_SIZE, + NULL, false, ub_tp->tp_info->xid, + 0, 0); +} + +static void udma_tpn_free(struct udma_device *udma_dev, struct udma_tp *ub_tp) +{ + struct ubcore_tp *tp = &ub_tp->tp; + struct udma_tpg *ub_tpg = NULL; + u32 i; + + if (tp->tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, "Not support no tpg\n"); + tp->tpn = TPN_INVALID_NUM; + return; + } + + ub_tpg = to_udma_tpg(tp->tpg); + for (i = 0; i < ub_tpg->tp_num; i++) { + if (ub_tpg->tpinfo_list[i] != NULL && + ub_tpg->tpinfo_list[i]->xid == tp->tpn) { + ub_tpg->tp_used[i] = 0; + } + } + + tp->tpn = TPN_INVALID_NUM; +} + +static void udma_create_qp_kernel_set_attr(struct udma_tp *ub_tp) +{ + ub_tp->buf.direct.buf = ub_tp->tp_buf_info->q_room_buf_1.direct.va; + ub_tp->buf.direct.map = ub_tp->tp_buf_info->q_room_buf_1.direct.pa; + ub_tp->buf.page_shift = + ilog2(ub_tp->tp_buf_info->q_room_buf_1.buf_size); + + ub_tp->kdb->db_record = + (__be32 *)(void *)(&ub_tp->tp_buf_info->q_header_vaddr + ->doorbell_record); + ub_tp->kdb->dma = ub_tp->tp_buf_info->q_header_paddr; + + ub_tp->tp_buf_info->q_header_vaddr->doorbell_record = 0; + ub_tp->tp_buf_info_xid = ub_tp->tp_buf_info->index; +} + +static int udma_create_queue_kernel_normal(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + struct tag_cqm_queue *tp_buf_info = NULL; + int ret; + + ub_tp->kdb = + (struct udma_kdb *)kzalloc(sizeof(struct udma_kdb), GFP_KERNEL); + if (ub_tp->kdb == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to alloc mem for tp(%u) queue db.\n", + ub_tp->tp.tpn); + return -ENOMEM; + } + + tp_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, + (u32)ub_tp->buf_size, ub_tp, true, CQM_INDEX_INVALID, 0, 0); + if (tp_buf_info == NULL) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "failed to create udma queue by cqm object, func_id:%u\n", + (u32)hinic5_global_func_id(udma_dev->hwdev)); + ret = -ENOMEM; + goto err_alloc_queue; + } + + (void)memset_s(tp_buf_info->q_room_buf_1.direct.va, + (u32)ub_tp->buf_size, 0xff, (u32)ub_tp->buf_size); + + ub_tp->ub_mtt.mtt_type = MTT_CMTT_TYPE; + ret = hmm_mtt_alloc(udma_dev->hwdev, + tp_buf_info->q_room_buf_1.buf_number, + (u32)ilog2(tp_buf_info->q_room_buf_1.buf_size), + &ub_tp->ub_mtt, SERVICE_T_UB); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to allocate udma mtt, func_id:%u\n", + (u32)hinic5_global_func_id(udma_dev->hwdev)); + goto err_alloc_mtt; + } + + ret = udma_buf_write_mtt(udma_dev, &ub_tp->ub_mtt, + &tp_buf_info->q_room_buf_1); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to write udma mtt, func_id:%u\n", + (u32)hinic5_global_func_id(udma_dev->hwdev)); + goto err_write_mtt; + } + ub_tp->tp_buf_info = tp_buf_info; + udma_create_qp_kernel_set_attr(ub_tp); + + return 0; + +err_write_mtt: + hmm_mtt_free(udma_dev->hwdev, &ub_tp->ub_mtt, SERVICE_T_UB); +err_alloc_mtt: + cqm5_object_delete(&tp_buf_info->object); +err_alloc_queue: + kfree(ub_tp->kdb); + ub_tp->kdb = NULL; + return ret; +} + +static void udma_destroy_queue_kernel_normal(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + kfree(ub_tp->kdb); + ub_tp->kdb = NULL; + + hmm_mtt_free(udma_dev->hwdev, &ub_tp->ub_mtt, SERVICE_T_UB); + + cqm5_object_delete(&ub_tp->tp_buf_info->object); + ub_tp->tp_buf_info = NULL; +} + +int udma_tp_create_ubrc_table(struct udma_device *udma_dev, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask, + struct udma_tp *ub_tp) +{ + struct udma_comp_priv *comp_priv = NULL; + int ret = 0; + + if (udma_dev == NULL || attr == NULL) { + udma_pr_err(UDMA_M_TP, + "Invalid parameter: udma_dev(%d),attr(%d).\n", + !!udma_dev, !!attr); + return -EINVAL; + } + + if (mask.bs.flag == 0) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "mask not enable, oor_en:%u\n", + attr->flag.bs.oor_en); + return 0; + } + + udma_debug(udma_dev->dev, UDMA_M_TP, "oor enable:%u\n", + attr->flag.bs.oor_en); + /* same tp channel only allocate once */ + if (attr->flag.bs.oor_en != 1 || ub_tp->oor_en != 0) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "oor has already configured\n"); + return ret; + } + + comp_priv = (struct udma_comp_priv *)(void *)get_hmm_comp_priv( + udma_dev->hwdev, SERVICE_T_UB); + if (comp_priv == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get hmm comp priv.\n"); + return -EINVAL; + } + ub_tp->rc_table_num = comp_priv->ub_cap.ubrc_depth; + ub_tp->rc_log_ubrc_seg = comp_priv->ub_cap.log_ubrc_seg; + ret = udma_ubrc_alloc(udma_dev, ub_tp->rc_table_num, &ub_tp->rc_table); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "failed alloc ub rc.\n"); + return ret; + } + + ret = udma_cmd_tp_create_ubrc(udma_dev, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed send create_ubrc cmd.\n"); + udma_ubrc_free(udma_dev, &ub_tp->rc_table); + return ret; + } + ub_tp->oor_en = 1; + + return ret; +} + +static void udma_tp_free_ubrc_table(struct udma_device *udma_dev, + struct udma_tp *ub_tp) +{ + if (ub_tp->rc_table.vaddr != NULL) + udma_ubrc_free(udma_dev, &ub_tp->rc_table); +} + +static int udma_queue_alloc(struct udma_device *udma_dev, + const struct ubcore_tp_cfg *cfg, + struct udma_tp *ub_tp) +{ + int ret; + + ret = udma_tpn_alloc(udma_dev, cfg, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate tpn\n"); + return -ENOMEM; + } + + ub_tp->pcqc_info = udma_create_pcqc(udma_dev, ub_tp); + if (ub_tp->pcqc_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to create pcqc, xid: %u\n", + ub_tp->tp_info->xid); + goto err_create_pcqc; + } + + ub_tp->entry_size = TP_Q_WQEBB; + ub_tp->buf_size = ub_tp->entry_cnt * ub_tp->entry_size; + + ret = udma_create_queue_kernel_normal(udma_dev, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to create tp queue, ret:%d\n", ret); + goto err_alloc_queue; + } + + return 0; + +err_alloc_queue: + cqm5_object_delete(&ub_tp->pcqc_info->object); + ub_tp->pcqc_info = NULL; +err_create_pcqc: + udma_tpn_free(udma_dev, ub_tp); + ub_tp->tp_info = NULL; + return -ENOMEM; +} + +struct ubcore_tp *udma_create_tp(struct ubcore_device *ub_dev, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata) +{ + uint64_t cycle_start = 0; + struct ubcore_tp *tp[1] = { 0 }; + struct udma_device *udma_dev = to_udma_dev(ub_dev); + int ret; + + if (udma_dev == NULL) { + pr_err("Invalid parameter. udma_dev(%d) .\n", !!udma_dev); + return (struct ubcore_tp *)ERR_PTR(-EINVAL); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + ret = udma_create_multi_tp(ub_dev, 1, cfg, udata, tp); + if (ret == 1) { + return tp[0]; + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(to_udma_dev(ub_dev)->dev, UDMA_M_TP, + "[PERF]udma_create_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return (struct ubcore_tp *)ERR_PTR(-EFAULT); +} + +static int udma_modify_tp_mtu_check(struct udma_device *udma_dev, + const struct ubcore_tp_attr *attr) +{ + udma_debug(udma_dev->dev, UDMA_M_TP, + "modify tp mt_1k_en:%u, g_multi_mtu_enable:%u\n", + udma_dev->mtu_1k_en, g_multi_mtu_enable); + if (udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + /* DFX tool supports 1K mtu configuration */ + if (((udma_dev->mtu_1k_en == 0) || + (attr->mtu != UBCORE_MTU_1024)) && + (attr->mtu != UBCORE_MTU_4096) && + (attr->mtu != UBCORE_MTU_8192)) { + udma_err(udma_dev->dev, UDMA_M_TP, + " modify tp: don't support mtu:%u\n", + attr->mtu); + return -EINVAL; + } + } else { + if ((g_multi_mtu_enable == UDMA_MULTI_MTU_DISABLE && + attr->mtu != UBCORE_MTU_4096) || + (g_multi_mtu_enable == UDMA_MULTI_MTU_ENABLE && + (attr->mtu != UBCORE_MTU_1024 && + attr->mtu != UBCORE_MTU_4096 && + attr->mtu != UBCORE_MTU_8192))) { + udma_err( + udma_dev->dev, UDMA_M_TP, + " modify tp mtu check failed: don't support mtu:%u\n", + attr->mtu); + return -EINVAL; + } + } + return 0; +} + +static int udma_modify_attr_check(struct udma_device *udma_dev, + struct ubcore_tp *tp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + udma_debug(udma_dev->dev, UDMA_M_TP, + "attr.data_udp_start:%u attr.ack_udp_start:%u\n", + attr->data_udp_start, attr->ack_udp_start); + udma_debug(udma_dev->dev, UDMA_M_TP, + "mask.data_udp_start:%u mask.ack_udp_start:%u\n", + mask.bs.data_udp_start, mask.bs.ack_udp_start); + udma_debug(udma_dev->dev, UDMA_M_TP, "mask:%u\n", mask.value); + + if (mask.bs.flag != 0) { + if ((attr->flag.bs.sr_en == 1) && + (udma_dev->ub_cap.dev_cap.feature.bs.selective_retrans == + 0)) { + udma_err(udma_dev->dev, UDMA_M_TP, + "modify tp: don't support selective retrans"); + return -EINVAL; + } + if ((attr->flag.bs.oor_en == 1) && + (udma_dev->ub_cap.dev_cap.feature.bs.oor == 0)) { + udma_err(udma_dev->dev, UDMA_M_TP, + "modify tp: don't support oor"); + return -EINVAL; + } + } + if (mask.bs.mtu == 1) { + return udma_modify_tp_mtu_check(udma_dev, attr); + } + return 0; +} + +void udma_modify_tp_for_host(struct ubcore_tp *tp, + const struct udma_tp_modify_dfx *tp_dfx) +{ + if (tp_dfx->mask.peer_tpn == 1) + tp->peer_tpn = tp_dfx->attr.peer_tpn; + + if (tp_dfx->mask.state == 1) + tp->state = (enum ubcore_tp_state)tp_dfx->attr.state; + + if (tp_dfx->mask.tx_psn == 1) + tp->tx_psn = tp_dfx->attr.tx_psn; + + if (tp_dfx->mask.rx_psn == 1) + tp->rx_psn = tp_dfx->attr.rx_psn; + + if (tp_dfx->mask.mtu == 1) + tp->mtu = (enum ubcore_mtu)tp_dfx->attr.mtu; + + if (tp_dfx->mask.cc_pattern_idx == 1) + tp->cc_pattern_idx = tp_dfx->attr.cc_pattern_idx; + + if (tp_dfx->mask.flag == 1) { + tp->flag.bs.oor_en = tp_dfx->attr.oor_en; + tp->flag.bs.sr_en = tp_dfx->attr.sr_en; + tp->flag.bs.cc_en = tp_dfx->attr.cc_en; + tp->flag.bs.cc_alg = tp_dfx->attr.cc_alg; + tp->flag.bs.spray_en = tp_dfx->attr.spray_en; + } +} + +int udma_modify_tp(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct ubcore_tp *fail_tp[1] = { NULL }; + int modify_num; + + if (tp == NULL || tp->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter. tp(%d) .\n", !!tp); + return -EINVAL; + } + + udma_dev = to_udma_dev(tp->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + modify_num = udma_modify_multi_tp(1, &tp, attr, &mask, fail_tp); + if (modify_num != 1) { + udma_err(udma_dev->dev, UDMA_M_TP, + "modify tp failed, tpn(0x%x)\n", tp->tpn); + return -EINVAL; + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_modify_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, "success, tpn(0x%x)\n", tp->tpn); + + return 0; +} + +int udma_query_tpc(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *val) +{ + int ret; + + ret = udma_cmd_tp_query_ctx(udma_dev, key, val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp, key:0x%x\n", + key->key); + return ret; + } + + return 0; +} + +int udma_query_tpc_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *val) +{ + int ret; + + ret = udma_cmd_tp_query_ctx_info(udma_dev, key, val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp info, key:0x%x\n", + key->key); + return ret; + } + + return 0; +} + +static void +udma_fill_res_ret_val_info(const struct udma_device *udma_dev, + struct ubcore_res_val *val, u32 buf_len, + struct tag_ub_tp_query_info_outbuf *outbuf, + struct ubcore_res_tp_val *ret_val) +{ + val->len = buf_len; + + ret_val->tpn = outbuf->srctpn; + ret_val->oor_en = outbuf->dw0.oor_en; + ret_val->selective_retrans_en = outbuf->dw0.srp_en; + ret_val->state = outbuf->dw0.state; + ret_val->tx_psn = outbuf->next_epsn; + ret_val->rx_psn = outbuf->next_send_psn; + ret_val->dscp = outbuf->dw0.tclass >> RIGHT_SHIFT_NUM; + ret_val->data_udp_start = outbuf->dw1.data_udp_srcport; + ret_val->ack_udp_start = outbuf->dw1.ack_udp_srcport; + + ret_val->spray_en = 0; // not support + ret_val->udp_range = 0; // not support +} + +int udma_query_res_tp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_tp_val *ret_val = + (struct ubcore_res_tp_val *)(uintptr_t)val->addr; + struct tag_ub_tp_query_info_outbuf *outbuf = NULL; + int ret; + + ret = check_tp_query_ret_val(ret_val, val->len, + sizeof(struct ubcore_res_tp_val), + udma_dev->dev, key); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tp res: failed to check return vaule, key:0x%x\n", + key->key); + return ret; + } + + outbuf = (struct tag_ub_tp_query_info_outbuf *)kzalloc( + sizeof(struct tag_ub_tp_query_info_outbuf), GFP_KERNEL); + if (outbuf == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to alloc\n"); + return -ENOMEM; + } + ret = udma_cmd_tp_query_ctx_info(udma_dev, key, outbuf); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query tp res: failed to query tp, key:0x%x\n", + key->key); + kfree(outbuf); + return ret; + } + udma_fill_res_ret_val_info(udma_dev, val, + sizeof(struct ubcore_res_tp_val), outbuf, + ret_val); + kfree(outbuf); + + return 0; +} + +static int udma_tp_cache_out_cmd(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u64 wb_dma = 0; + void *wb_va = NULL; + int ret; + + wb_va = dma_alloc_coherent(udma_dev->dev, (size_t)4UL, &wb_dma, + GFP_KERNEL); + if (wb_va == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to alloc dma wb_va\n"); + return -ENOMEM; + } + + ret = udma_cmd_tp_send_cache_out(udma_dev, cnt, tp, wb_dma); + + dma_free_coherent(udma_dev->dev, sizeof(u32), wb_va, wb_dma); + + return ret; +} + +static int udma_change_tp_to_err(struct ubcore_tp *tp) +{ + union ubcore_tp_attr_mask mask; + struct ubcore_tp_attr attr; + + mask.value = 0; + mask.bs.state = 1; + + attr.state = UBCORE_TP_STATE_ERR; + + if (udma_modify_tp(tp, &attr, mask) != 0) + return -1; + + tp->state = UBCORE_TP_STATE_ERR; + return 0; +} + +int udma_destroy_tp(struct ubcore_tp *tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tp *ub_tp = NULL; + int destroy_num; + u32 tpn; + int ret; + + if (tp == NULL || tp->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter, tp(%d)\n", !!tp); + return -EINVAL; + } + + tpn = tp->tpn; + udma_dev = to_udma_dev(tp->ub_dev); + ub_tp = to_udma_tp(tp); + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + udma_info(udma_dev->dev, UDMA_M_TP, + "destroy tp start, tpn(%u), peer_tpn(%u)\n", tpn, + tp->peer_tpn); + + if (tp->state != UBCORE_TP_STATE_ERR && + tp->state != UBCORE_TP_STATE_RESET) { + ub_tp->refuse_report_flush_done = 1; + + ret = udma_change_tp_to_err(tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "change tp to err failed, tp->state(%d)\n", + tp->state); + return -EFAULT; + } + } + + destroy_num = udma_destroy_multi_tp(1, &tp); + if (destroy_num != 1) { + udma_err(udma_dev->dev, UDMA_M_TP, + "destroy tp failed, tpn(%u)\n", tpn); + return -EFAULT; + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_info(udma_dev->dev, UDMA_M_TP, "destroy tp success, tpn(%u)\n", + tpn); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_TP, tpn, &ub_tp->cmdq_rtt); + return 0; +} + +int udma_query_tp_wqe_from_host(struct udma_device *udma_dev, u32 tpn, + u32 wqe_id, struct ub_tp_wqe *outbuf, + u32 outbuf_size) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + u64 tpqe_offset = 0; // target ci of address offset + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL || ub_tp->tp_buf_info == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "tp addr is null, check tpn(%u) is right\n", tpn); + ret = -EFAULT; + goto cleanup; + } + + if (wqe_id >= (ub_tp->entry_cnt / UB_TP_WQE_NUM_WQEBB)) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "wqe_id exceeds the max wqe num, tpn(%u) wqe_id(%u) entry_cnt(%u)\n", + tpn, wqe_id, ub_tp->entry_cnt); + ret = -EINVAL; + goto cleanup; + } + + tpqe_offset = wqe_id * outbuf_size; + memcpy_fromio( + (void *)outbuf, + (void *)((uintptr_t)ub_tp->tp_buf_info->q_room_buf_1.direct.va + + tpqe_offset), + outbuf_size); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp wqe from host success, tpn:%u, wqe_id:%u\n", tpn, + wqe_id); +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +static int udma_create_tp_host(struct udma_device *udma_dev, + struct udma_tp **ub_tp, struct ubcore_tp **tp, + const struct ubcore_tp_cfg *cfg) +{ + int ret; + + *ub_tp = (struct udma_tp *)udma_vram_kzalloc(udma_dev->dev, + UDMA_VRAM_TYPE_TP, + sizeof(struct udma_tp), + GFP_KERNEL); + if (*ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate tp memory\n"); + return -ENOMEM; + } + (*ub_tp)->pri = cfg->dscp; + *tp = &((*ub_tp)->tp); + (*tp)->tpg = NULL; + (*ub_tp)->cqm_priv_type = UDMA_CQM_PRIV_TYPE_TP; + udma_cmdq_rtt_statistic_init(&(*ub_tp)->cmdq_rtt); + (void)memcpy_s(&(*ub_tp)->cfg, sizeof(struct ubcore_tp_cfg), cfg, + sizeof(struct ubcore_tp_cfg)); + + ret = udma_queue_alloc(udma_dev, cfg, (*ub_tp)); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp: failed to allocate info\n"); + udma_vram_kfree(udma_dev->dev, *ub_tp, UDMA_VRAM_TYPE_TP); + return -ENOMEM; + } + + kref_init(&(*ub_tp)->ref_cnt); + init_completion(&(*ub_tp)->comp); + + udma_store_list(udma_dev, &(*ub_tp)->node, + &udma_dev->device_res->tpn_list); + + udma_info(udma_dev->dev, UDMA_M_TP, "success, tpn:%u\n", + (*ub_tp)->tp.tpn); + + return 0; +} + +static void udma_destroy_tp_host(struct udma_device *udma_dev, + struct udma_tp *ub_tp, bool del_object) +{ + u32 tpn = 0; + + if ((udma_dev == NULL) || (ub_tp == NULL)) { + udma_pr_err(UDMA_M_TP, + "Invalid parameter: udma_dev(%d), ub_tp(%d).\n", + !!udma_dev, !!ub_tp); + return; + } + tpn = ub_tp->tp.tpn; // udma_tpn_free will set tpn is invalid, because this here first record tpn used for log print + + udma_delete_list(udma_dev, &ub_tp->node, + &udma_dev->device_res->tpn_list); + udma_tp_put(ub_tp); + wait_for_completion(&ub_tp->comp); + + udma_tpn_free(udma_dev, ub_tp); + if (del_object) { + cqm5_object_delete(&ub_tp->tp_info->object); + } + ub_tp->tp_info = NULL; + + udma_tp_free_ubrc_table(udma_dev, ub_tp); // modify tp flow created in + udma_destroy_queue_kernel_normal(udma_dev, ub_tp); + + cqm5_object_delete(&ub_tp->pcqc_info->object); + ub_tp->pcqc_info = NULL; + + udma_info(udma_dev->dev, UDMA_M_TP, "destroy tp success, tpn(%u)\n", + tpn); + udma_vram_kfree(udma_dev->dev, ub_tp, UDMA_VRAM_TYPE_TP); +} + +static void udma_print_tp_cfg(const struct ubcore_tp_cfg *cfg, + struct udma_device *udma_dev, u32 cnt) +{ + u32 i; + + for (i = 0; i < cnt; i++) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "start print ubcore tp cfg[%u]\n", i); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "flag:%u, local_jetty_id:%u, fe_idx:%u, peer_jetty_id:%u\n", + cfg[i].flag.bs.target, cfg[i].local_jetty.id, + adapted_ue_idx(&cfg[i]), cfg[i].peer_jetty.id); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "trans_mode:%u, retry_num:%u, retry_factor:%u, ack_timeout:%u\n", + cfg[i].trans_mode, cfg[i].retry_num, + cfg[i].retry_factor, cfg[i].ack_timeout); + udma_debug(udma_dev->dev, UDMA_M_TP, "dscp:%u, oor_cnt:%u\n", + cfg[i].dscp, cfg[i].oor_cnt); + if (cfg[i].tpg == NULL) + udma_err(udma_dev->dev, UDMA_M_TP, + "cfg->tpg is null.\n"); + else + udma_debug(udma_dev->dev, UDMA_M_TP, + "cfg->tpg->tpgn:%u\n", cfg[i].tpg->tpgn); + udma_debug(udma_dev->dev, UDMA_M_TP, + "end print ubcore tp cfg[%u]\n", i); + } +} + +/* function description: create multi tp, at most same when create UB_CMD_TP_CNT_MAX + * return value: success created count + */ +int udma_create_multi_tp(struct ubcore_device *dev, u32 cnt, + struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata, + struct ubcore_tp **tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tp *ub_tp[UB_CMD_TP_CNT_MAX] = { NULL }; + u32 host_create_num = 0; + u32 cmd_create_num = 0; + u32 i; + int ret; + + if (dev == NULL || cfg == NULL || tp == NULL) { + udma_pr_err(UDMA_M_TP, + "Invalid parameter: dev(%d), cfg(%d), tp(%d)\n", + !!dev, !!cfg, !!tp); + return 0; + } + + udma_dev = to_udma_dev(dev); + udma_print_tp_cfg(cfg, udma_dev, cnt); + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + if (cnt > UB_CMD_TP_CNT_MAX) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "max number is %d, cnt %d is larger than UB_CMD_TP_CNT_MAX.\n", + UB_CMD_TP_CNT_MAX, cnt); + return 0; + } + + for (i = 0; i < cnt; i++) { + ret = udma_create_tp_host(udma_dev, &ub_tp[i], &tp[i], cfg + i); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "fail to create tp no.%u\n", i); + host_create_num = i; + goto err_alloc_tp_host; // free already allocated memory, return 0 + } + } + + cmd_create_num = udma_cmd_tp_create_ctx(dev, cnt, cfg, + ub_tp); // return create success of count + if (cmd_create_num != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, %u tp success\n", + cmd_create_num); + goto err_create_tp_cmd; // free no success send_cmd of tp + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_create_multi_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, "create %u tp succeed\n", + cmd_create_num); + + return (int)cmd_create_num; + +err_create_tp_cmd: + for (i = cmd_create_num; i < cnt; i++) + udma_destroy_tp_host(udma_dev, ub_tp[i], false); + return (int)cmd_create_num; + +err_alloc_tp_host: + for (i = 0; i < host_create_num; i++) + udma_destroy_tp_host(udma_dev, ub_tp[i], false); + return 0; +} + +/* return value: destroy cmd destroy of tp count */ +static u32 udma_send_destroy_multi_tp_cmd(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u32 delete_destroy_num; + int ret; + + delete_destroy_num = udma_cmd_tp_destroy_ctx(udma_dev, cnt, tp); + if (delete_destroy_num != cnt) + udma_err(udma_dev->dev, UDMA_M_TP, + "delete_destroy_num(%u) not equal to cnt(%u)\n", + delete_destroy_num, cnt); + + /* mtt of cacheinvalidate only needs execute once */ + ret = udma_tp_cache_out_cmd(udma_dev, delete_destroy_num, tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, "tp:0x%06x cache invalid\n", + tp[0]->tpn); + return 0; // return cacheout success destroy of count + } + + udma_debug(udma_dev->dev, UDMA_M_TP, "delete_destroy_num(%u)\n", + delete_destroy_num); + + return delete_destroy_num; +} + +/* function description: destroy multi tp, at most same when destroy UB_CMD_TP_CNT_MAX + * return value: success destroy of count + */ +int udma_destroy_multi_tp(u32 cnt, struct ubcore_tp **tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tp *ub_tp = NULL; + u32 destroy_num = 0; // success destroy of tp count + u32 i; + + if (tp == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter, tp is null\n"); + return 0; + } + + if (cnt == 0 || cnt > UB_CMD_TP_CNT_MAX) { + udma_pr_err( + UDMA_M_TP, + "cnt(%d) is zero or larger than UB_CMD_TP_CNT_MAX(%d)\n", + cnt, UB_CMD_TP_CNT_MAX); + return 0; + } + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!tp[0]->ub_dev); + return -EINVAL; + } + + udma_dev = to_udma_dev(tp[0]->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + destroy_num = udma_send_destroy_multi_tp_cmd(udma_dev, cnt, tp); + if (destroy_num != cnt) + udma_err(udma_dev->dev, UDMA_M_TP, + "destroy cmd failed, %u tp destroyed, cnt(%u)\n", + destroy_num, cnt); + + for (i = 0; i < destroy_num; + i++) { // destroy host in cmdq destroy success of basebase on execute + ub_tp = to_udma_tp(tp[i]); + udma_destroy_tp_host(udma_dev, ub_tp, true); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_multi_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: destroy_num(%u) tp success, cnt(%u)\n", __func__, + destroy_num, cnt); + return (int)destroy_num; +} + +/* return value: via attr check of tp count */ +static u32 udma_tp_get_valid_attr_num_for_modify( + struct udma_device *udma_dev, u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, union ubcore_tp_attr_mask *mask) +{ + int ret; + u32 i; + + for (i = 0; i < cnt; ++i) { + if (tp[i] == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: tp[%u] is NULL, cnt(%u)\n", __func__, i, + cnt); + return i; + } + + udma_debug( + udma_dev->dev, UDMA_M_TP, + "%s: index(%u), tpn(%u), cc_en(%u), cc_alg(%u), cc_pat_idx(%u)\n", + __func__, i, tp[i]->tpn, attr[i].flag.bs.cc_en, + attr[i].flag.bs.cc_alg, attr[i].cc_pattern_idx); + ret = udma_modify_attr_check(udma_dev, tp[i], &attr[i], + mask[i]); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: invalid attr, tpn(%u), index(%u)\n", + __func__, tp[i]->tpn, i); + return i; + } + } + + return cnt; +} + +/* return value: success create ubrc table of tp count */ +static u32 udma_multi_tp_create_ubrc_table(struct udma_device *udma_dev, + u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + struct udma_tp *ub_tp = NULL; + int ret; + u32 i; + + for (i = 0; i < cnt; ++i) { + ub_tp = to_udma_tp(tp[i]); + ret = udma_tp_create_ubrc_table(udma_dev, &attr[i], mask[i], + ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "%s: create ubrc failed, tpn(%u), index(%u)\n", + __func__, tp[i]->tpn, i); + return i; + } + } + + return cnt; +} + +/* return value: success get cos of tp count */ +static u32 udma_modify_multi_get_cos(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask) +{ + struct ub_sip_attr sip = { 0 }; + struct udma_tp *ub_tp = NULL; + int ret; + u32 i; + + for (i = 0; i < cnt; ++i) { + if (mask[i].bs.local_net_addr_idx == 0) + continue; + + ub_tp = to_udma_tp(tp[i]); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "to_udma_tp failed, tp index:%d.\n", i); + return 0; + } + ret = udma_query_sip(udma_dev, attr[i].local_net_addr_idx, + &sip); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query sip failed, index(%u), ret(%d), sip_idx(%u)\n", + i, ret, attr[i].local_net_addr_idx); + return i; + } + + ret = udma_get_dcb_cfg_cos(udma_dev, &sip, ub_tp->pri, + &ub_tp->cos, 0); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "get cos failed, index(%u), tpn(%u), ret(%d), sip_idx(%u), pri(%u)\n", + i, tp[i]->tpn, ret, attr[i].local_net_addr_idx, + ub_tp->pri); + return i; + } + udma_debug( + udma_dev->dev, UDMA_M_TP, + "Get cos from dcb info successful, index(%u), cos(%u)\n", + i, ub_tp->cos); + } + + return cnt; +} + +/* func: calculate modify need send down of cmdq count + * input @cmd_modify_tp_cnt: need send down cmdq of tp count + */ +static inline u32 udma_modify_cmd_times_calculate(u32 cmd_modify_tp_cnt) +{ + if (cmd_modify_tp_cnt == 0) + return 0; + else if (cmd_modify_tp_cnt > 0 && + cmd_modify_tp_cnt <= UB_CMD_TP_MODIFY_CNT_MAX) + return 1; + else + return 2; // need send down 2 time modify cmd +} + +static u32 udma_modify_multi_tp_cmd_entry(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + u32 success_num = 0, modify_total_success_num = 0; + u32 modify_cmd_times; // modify cmdq send down count + u32 left_cmd_cnt; // need send down cmdq of remaining tp count + u32 cmd_cnt; // once cmdq send down of tp count + u32 begin_idx = 0; // input parameter tp, attr, mask of start index + u32 i; + + modify_cmd_times = udma_modify_cmd_times_calculate(cnt); + + left_cmd_cnt = cnt; + for (i = 0; i < modify_cmd_times; ++i) { + cmd_cnt = (left_cmd_cnt > UB_CMD_TP_MODIFY_CNT_MAX) ? + UB_CMD_TP_MODIFY_CNT_MAX : + left_cmd_cnt; + left_cmd_cnt -= cmd_cnt; + udma_debug(udma_dev->dev, UDMA_M_TP, + "%s: cmd_cnt(%u), left_cmd_cnt(%u)\n", __func__, + cmd_cnt, left_cmd_cnt); + + success_num = udma_cmd_tp_modify_ctx(cmd_cnt, &tp[begin_idx], + &attr[begin_idx], + &mask[begin_idx]); + if (success_num != cmd_cnt) { + modify_total_success_num += success_num; + udma_err( + udma_dev->dev, UDMA_M_TP, + "%s: modify cmd failed, success_num(%u), cmd_cnt(%u), " + "modify_total_success_num(%u)\n", + __func__, success_num, cmd_cnt, + modify_total_success_num); + return modify_total_success_num; + } + begin_idx += cmd_cnt; + modify_total_success_num += cmd_cnt; + } + + return modify_total_success_num; +} + +static int udma_tp_cfg_info_alloc(struct udma_device *udma_dev, u32 cnt, + struct udma_tp ***ub_tp, + struct ubcore_tp_cfg **cfg, + struct ubcore_tp **tp) +{ + u32 i; + + *ub_tp = (struct udma_tp **)kcalloc(cnt, sizeof(struct udma_tp *), + GFP_KERNEL); + if (*ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, "ub tp alloc failed\n"); + return -ENOMEM; + } + + *cfg = (struct ubcore_tp_cfg *)kcalloc( + cnt, sizeof(struct ubcore_tp_cfg), GFP_KERNEL); + if (*cfg == NULL) { + kfree(*ub_tp); + udma_err(udma_dev->dev, UDMA_M_TP, "tp cfg alloc failed\n"); + return -ENOMEM; + } + + for (i = 0; i < cnt; i++) { + (*ub_tp)[i] = to_udma_tp(tp[i]); + if ((*ub_tp)[i]->tp_buf_info != NULL) { + (void)memset_s( + (*ub_tp)[i]->tp_buf_info->q_room_buf_1.direct.va, + (u32)(*ub_tp)[i]->buf_size, 0xff, + (u32)(*ub_tp)[i]->buf_size); + (*ub_tp)[i] + ->tp_buf_info->q_header_vaddr->doorbell_record = + 0; + } + + (void)memcpy_s(&(*cfg)[i], sizeof(struct ubcore_tp_cfg), + &(*ub_tp)[i]->cfg, sizeof(struct ubcore_tp_cfg)); + } + + return 0; +} + +static void udma_tp_cfg_info_free(struct udma_tp **ub_tp, + struct ubcore_tp_cfg *cfg) +{ + kfree(ub_tp); + kfree(cfg); +} + +static u32 udma_handle_tp_create_ctx(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + u32 cmd_create_num; + struct udma_tp **ub_tp = NULL; + struct ubcore_tp_cfg *cfg = NULL; + int ret = 0; + + ret = udma_tp_cfg_info_alloc(udma_dev, cnt, &ub_tp, &cfg, tp); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "tp cfg info alloc failed, ret = %d\n", ret); + return (u32)ret; + } + + cmd_create_num = + udma_cmd_tp_create_ctx(&udma_dev->ub_dev, cnt, cfg, ub_tp); + if (cmd_create_num != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to send cqm cmd, %u tp success\n", + cmd_create_num); + } + udma_tp_cfg_info_free(ub_tp, cfg); + + return cmd_create_num; +} + +static bool udma_check_reuse_state(struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + return (tp[0]->state == UBCORE_TP_STATE_ERR && mask->bs.state == 1 && + attr[0].state == UBCORE_TP_STATE_RESET); +} + +static u32 udma_handle_tp_create_ubrc(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp) +{ + struct udma_tp *ub_tp = NULL; + int ret; + u32 i; + + for (i = 0; i < cnt; i++) { + ub_tp = to_udma_tp(tp[i]); + /* oor_en no has enabled, description did not create ubrc, then tp restore when not need initialize ubrc */ + if (ub_tp->oor_en == 0) { + continue; + } + + ret = udma_cmd_tp_create_ubrc(udma_dev, ub_tp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp ubrc failed, i(%u)\n", i); + return i; + } + } + + return cnt; +} + +static u32 udma_handle_tp_reuse(struct udma_device *udma_dev, u32 cnt, + struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask) +{ + u32 destroy_num = 0; + u32 cmd_create_ctx_num = 0; + u32 cmd_create_urbc_num = 0; + + if (udma_dev->device_type == UDMA_DEV_TYPE_HI1825_V100 || + !udma_check_reuse_state(tp, attr, mask)) { + return cnt; + } + + destroy_num = udma_send_destroy_multi_tp_cmd(udma_dev, cnt, tp); + if (destroy_num != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "destroy tp cmd failed, %u tp destroyed, cnt(%u)\n", + destroy_num, cnt); + } + + cmd_create_ctx_num = + udma_handle_tp_create_ctx(udma_dev, destroy_num, tp); + if (cmd_create_ctx_num != destroy_num) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp cmd fail, %u tp ctx created, cnt(%u)\n", + cmd_create_ctx_num, destroy_num); + } + + cmd_create_urbc_num = + udma_handle_tp_create_ubrc(udma_dev, cmd_create_ctx_num, tp); + if (cmd_create_urbc_num != cmd_create_ctx_num) { + udma_err(udma_dev->dev, UDMA_M_TP, + "create tp ubrc fail, %u tp ubrc created, cnt(%u)\n", + cmd_create_urbc_num, cmd_create_ctx_num); + } + + udma_debug(udma_dev->dev, UDMA_M_TP, + "tp reuse success, tp reuse num = %u\n", + cmd_create_urbc_num); + return cmd_create_urbc_num; +} + +static void udma_print_multi_tp_attr(struct ubcore_tp_attr *attr, + const union ubcore_tp_attr_mask *mask, + struct udma_device *udma_dev, u32 cnt) +{ + u32 dip0 = 0; + u32 dip1 = 0; + u32 dip2 = 0; + u32 dip3 = 0; + u32 i; + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_TP, "%s Invalid parameter: udma_dev(%d).\n", + __func__, !!udma_dev); + return; + } + + if (attr == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "ubcore vtp attr is null.\n"); + return; + } + + for (i = 0; i < cnt; i++) { + udma_debug(udma_dev->dev, UDMA_M_TP, + "start print ubcore tp attr[%u]\n", i); + udma_debug(udma_dev->dev, UDMA_M_TP, + "flag:%u, peer_tpn:%u, state:%u, tx_psn:%u\n", + attr[i].flag.value, attr[i].peer_tpn, attr[i].state, + attr[i].tx_psn); + + udma_debug( + udma_dev->dev, UDMA_M_TP, + "rx_psn:%u, mtu:%u, cc_pattern_idx:%u, peer_ext_addr:%llu\n", + attr[i].rx_psn, attr[i].mtu, attr[i].cc_pattern_idx, + attr[i].peer_ext.addr); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "oos_cnt:%u, local_net_addr_idx:%u, data_udp_start:%u, ack_udp_start:%u\n", + attr[i].oos_cnt, attr[i].local_net_addr_idx, + attr[i].data_udp_start, attr[i].ack_udp_start); + udma_debug( + udma_dev->dev, UDMA_M_TP, + "udp_range:%u, hop_limit:%u, flow_label:%u, port_id:%u\n", + attr[i].udp_range, attr[i].hop_limit, + attr[i].flow_label, attr[i].port_id); + udma_debug(udma_dev->dev, UDMA_M_TP, "mn:%u, mask:%u\n", + attr[i].mn, mask[i].value); + + if (attr[i].peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + dip3 = attr[i].peer_net_addr.net_addr.in4.addr; + } else { + /* dip0 - dip3 */ + dip0 = attr[i].peer_net_addr.net_addr.in6.subnet_prefix & + UB_TP_DIP_MASK; + dip1 = (attr[i].peer_net_addr.net_addr.in6 + .subnet_prefix >> + UB_TP_DIP_OFFSET) & + UB_TP_DIP_MASK; + dip2 = attr[i].peer_net_addr.net_addr.in6.interface_id & + UB_TP_DIP_MASK; + dip3 = (attr[i].peer_net_addr.net_addr.in6 + .interface_id >> + UB_TP_DIP_OFFSET) & + UB_TP_DIP_MASK; + } + udma_debug(udma_dev->dev, UDMA_M_TP, + "dsip0:0x%x, dsip1:0x%x, dsip2:0x%x, dsip3:0x%x\n", + dip0, dip1, dip2, dip3); + + udma_debug(udma_dev->dev, UDMA_M_TP, + "end print ubcore tp attr[%u]\n", i); + } +} + +int udma_modify_multi_tp(u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask, + struct ubcore_tp **fail_tp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + u32 ubrc_success_num = 0, cos_success_num = 0, modify_success_num = 0, + reuse_success_num = 0; + u32 k; + + if (tp == NULL || fail_tp == NULL || attr == NULL || mask == NULL) { + udma_pr_err( + UDMA_M_TP, + "%s: Invalid parameter, tp(%d), fail_tp(%d), attr(%d), mask(%d)\n", + __func__, !!tp, !!fail_tp, !!attr, !!mask); + return 0; + } + + if (cnt == 0 || cnt > UB_CMD_TP_CNT_MAX) { + udma_pr_err( + UDMA_M_TP, + "%s: cnt(%u) is zero or larger than UB_CMD_TP_CNT_MAX(%d)\n", + __func__, cnt, UB_CMD_TP_CNT_MAX); + return 0; + } + if (tp[0]->ub_dev == NULL) { + udma_pr_err(UDMA_M_TP, "Invalid parameter: ub_dev(%d).\n", + !!tp[0]->ub_dev); + return -EINVAL; + } + + udma_dev = to_udma_dev(tp[0]->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + + udma_print_multi_tp_attr(attr, mask, udma_dev, cnt); + + if (udma_tp_get_valid_attr_num_for_modify(udma_dev, cnt, tp, attr, + mask) != cnt) { + udma_err(udma_dev->dev, UDMA_M_TP, + "attr check failed, cnt(%u)\n", cnt); + return 0; + } + + ubrc_success_num = + udma_multi_tp_create_ubrc_table(udma_dev, cnt, tp, attr, mask); + + cos_success_num = udma_modify_multi_get_cos(udma_dev, ubrc_success_num, + tp, attr, mask); + + modify_success_num = udma_modify_multi_tp_cmd_entry( + udma_dev, cos_success_num, tp, attr, mask); + + reuse_success_num = udma_handle_tp_reuse(udma_dev, modify_success_num, + tp, attr, mask); + + for (k = reuse_success_num; k < cnt; ++k) + fail_tp[k - reuse_success_num] = tp[k]; + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_modify_multi_tp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TP, "modify %u tp succeed, cnt(%u)\n", + reuse_success_num, cnt); + + return (int)reuse_success_num; +} + +int udma_query_tp_ctx_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_dfx_tp_ctx_query *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query ub_tp fail for tp ctx, tpn:%u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + + memcpy_fromio((void *)outbuf, (void *)ub_tp->tp_info->vaddr, + sizeof(struct ub_dfx_tp_ctx_query)); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp ctx from host success, tpn:%u\n", tpn); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +int udma_query_tp_mpt_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_mapt_ctx_query *outbuf) +{ + struct udma_tp *ub_tp = NULL; + struct tag_cqm_object *cqm_obj = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query ub_tp fail for tp mpt, tpn:%u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + + (void)memcpy_s((void *)outbuf, sizeof(struct ub_mapt_ctx_query), + (void *)ub_tp->tp_info->vaddr, + sizeof(struct ub_mapt_ctx_query)); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp mpt from host success, tpn:%u\n", tpn); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +int udma_query_tp_pcqc_from_host(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, tpn, + false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to get cqm object, tpn:%u\n", tpn); + return -EINVAL; + } + ub_tp = udma_find_tp(udma_dev, tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_DFX, + "failed to find tp, tpn = %u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + udma_tp_put(ub_tp); + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query ub_tp fail for tp pcqc, tpn:%u\n", tpn); + ret = -EINVAL; + goto cleanup; + } + + memcpy_fromio((void *)outbuf, (void *)ub_tp->pcqc_info->q_ctx_vaddr, + sizeof(ub_pcqc_query_rsp_s)); + + udma_info(udma_dev->dev, UDMA_M_TP, + "query tp pcqc from host success, tpn:%u\n", tpn); + +cleanup: + cqm5_object_put(cqm_obj); + + return ret; +} + +static int udma_check_dscp_vl_params(struct ubcore_device *dev, const u8 *dscp, + const u8 *vl, u8 num) +{ + struct udma_device *udma_dev = NULL; + + if (dev == NULL || dscp == NULL || vl == NULL) { + udma_pr_err( + UDMA_M_TP, + "%s: Invalid parameter, dev(%d), dscp(%d), vl(%d)\n", + __func__, !!dev, !!dscp, !!vl); + return -EINVAL; + } + + udma_dev = to_udma_dev(dev); + if (num > UDMA_DSCP_NUM) { + udma_err(udma_dev->dev, UDMA_M_TP, + "num is larger than UDMA_DSCP_NUM\n"); + return -EINVAL; + } + + return 0; +} + +int udma_config_dscp_vl(struct ubcore_device *dev, u8 *dscp, u8 *vl, u8 num) +{ + struct udma_device *udma_dev = NULL; + u8 dscp2cos[UDMA_DSCP_NUM] = { 0 }; + int ret; + u8 i; + + ret = udma_check_dscp_vl_params(dev, dscp, vl, num); + if (ret != 0) { + return ret; + } + + udma_dev = to_udma_dev(dev); + ret = udma_dscp_cos_map(udma_dev, dscp2cos, sizeof(dscp2cos), + MGMT_MSG_CMD_OP_GET); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "get origin cos failed, err:%d\n", ret); + return ret; + } + + for (i = 0; i < num; i++) { + if (dscp[i] > UDMA_DSCP_MAX || vl[i] > UDMA_COS_MAX) { + udma_err(udma_dev->dev, UDMA_M_TP, + "err dscp:%u cos:%u\n", dscp[i], vl[i]); + return -EINVAL; + } + dscp2cos[dscp[i]] = vl[i]; + } + + ret = udma_dscp_cos_map(udma_dev, dscp2cos, sizeof(dscp2cos), + MGMT_MSG_CMD_OP_SET); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "set port cos failed, err:%d\n", ret); + return ret; + } + + for (i = 0; i < num; i++) { + udma_dev->dscp2cos[dscp[i]] = vl[i]; + udma_dev->dscp_bitmap |= BIT((u64)dscp[i]); + } + + return 0; +} + +int udma_query_dscp_vl(struct ubcore_device *dev, uint8_t *dscp, uint8_t num, + uint8_t *vl) +{ + struct udma_device *udma_dev = NULL; + int ret; + u8 i; + + ret = udma_check_dscp_vl_params(dev, dscp, vl, num); + if (ret != 0) { + return ret; + } + + udma_dev = to_udma_dev(dev); + for (i = 0; i < num; i++) { + if (dscp[i] >= UDMA_DSCP_MAX) { + udma_err(udma_dev->dev, UDMA_M_TP, + "query dscp cos err dscp(%u)\n", dscp[i]); + return -EINVAL; + } + vl[i] = udma_dev->dscp2cos[dscp[i]]; + } + + return 0; +} + +int udma_get_dscp_cos(struct udma_device *udma_dev, u8 *dscp, u32 size, + u8 is_dev) +{ + u8 dscp2cos[UDMA_DSCP_MAX_NUM] = { 0 }; + int ret = 0; + u8 i; + + if (is_dev == 0) { + ret = udma_dscp_cos_map(udma_dev, dscp2cos, sizeof(dscp2cos), + MGMT_MSG_CMD_OP_GET); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "get port cos failed, err:%d\n", ret); + return ret; + } + goto result; + } + + for (i = 0; i < UDMA_DSCP_MAX_NUM; i++) { + if ((udma_dev->dscp_bitmap & BIT((u64)i)) == 0) { + ret = hinic5_get_cos_by_pri(udma_dev->hwdev, i, + &dscp2cos[i]); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "get nic cos failed, pri:%d, err:%d\n", + i, ret); + return ret; + } + } else { + dscp2cos[i] = udma_dev->dscp2cos[i]; + } + } + +result: + (void)memcpy_s(dscp, size, dscp2cos, sizeof(dscp2cos)); + + return 0; +} + +int udma_dfx_modify_tp_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf) +{ + struct tag_cqm_object *cqm_obj = NULL; + struct udma_tp *ub_tp = NULL; + int ret = 0; + + cqm_obj = cqm5_object_get(udma_dev->hwdev, CQM_OBJECT_SERVICE_CTX, + inbuf->tpn, false); + if (cqm_obj == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get cqm object, tpn = %u\n", inbuf->tpn); + return -EINVAL; + } + + ub_tp = udma_find_tp(udma_dev, inbuf->tpn); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to find tp, tpn = %u\n", inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + + ub_tp = cqmobj_to_ub_tp(cqm_obj); + if (ub_tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to get ub_tp for object, tpn = %u\n", + inbuf->tpn); + ret = -EINVAL; + goto cleanup; + } + + ret = udma_cmd_tp_modify_dfx(udma_dev, ub_tp, &inbuf->tp_attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TP, + "failed to modify tp, tpn = %u ret = %d\n", inbuf->tpn, + ret); + goto cleanup; + } + + udma_modify_tp_for_host(&ub_tp->tp, &inbuf->tp_attr); + + udma_info(udma_dev->dev, UDMA_M_TP, "query object success, id:%u\n", + inbuf->tpn); + +cleanup: + udma_tp_put(ub_tp); + cqm5_object_put(cqm_obj); + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h new file mode 100644 index 000000000..7cbfdd5b7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tp.h @@ -0,0 +1,184 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef CPU_UDMA_TP_H +#define CPU_UDMA_TP_H + +#include <linux/mutex.h> +#include "ubcore_api.h" +#include "udma_common.h" +#include "hinic5_cqm.h" +#include "udma_db.h" + +#define CPU_UBEP_MAX_TP 512 +#define MAX_HOP_LIMIT 255 + +#define UBEP_PAGE_SHIFT 12 +#define UBEP_TP_CONTEXT_SIZE 1024 +#define UBEP_TPG_CONTEXT_SIZE 1024 + +#define TPGN_INVALID 0xffffffff +#define TPN_INVALID_NUM 0xffffffff +#define TP_DEFAULT_RETRY_EXP 3 + +#define TP_WQE_SHIFT 6 +#define TP_WQEBB_SIZE_SHIFT 4 +#define TP_WQEBB_SIZE_MASK 0x7 +#define TP_MTT_GPA_VALID 0x1 +#define TP_WQE_PREFETCH_MAXNUM 4 +#define TP_SQ_WQEBB_SIZE_SHIFT 4 +#define TP_SQ_WQEBB_SIZE_MASK 0x7 +#define TP_SQ_PD_MASK 0x3ffff +#define TP_SQ_PAGE_SIZE_MASK 0xf +#define TP_SQ_WQECNT_LTH 4 +#define TP_SQ_PREFETCH_MINNUM 2 +#define TP_SQ_PREFETCH_MAXNUM 4 +#define TP_SQ_SIZE_MASK 0x1f +#define TP_SQ_CQN_MASK 0xfffff +#define TP_RQ_WQECNT_LTH 0xe +#define TP_RQ_MTT_PREFETCH_MAXLEN0 3 +#define TP_RQ_MTT_PREFETCH_MAXWQE 7 +#define TP_RQ_WQE_PREFETCH_MAXNUM 7 +#define TP_SRQ_INIT_STATE 0xf +#define TP_MAX_PI_ON_CHIP_QPN 1024 +#define TP_QP_MAX_PFETCH_MTT_LAYER 3 +#define TP_QP_MAX_TIMER_NUM 5 + +#define TP_PCQC_SIZE 128 +#define BIT_CNT_PER_BYTE 8 +#define DB_SIZE 4 + +#define TP_GET_MAC_HI(a, b) (((a) << 8) | (b)) +#define TP_GET_MAC_LO(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d)) + +#define TP_Q_WQEBB 64 +#define TP_RC_GPA_H_SHIFT 40 +#define TP_RC_GPA_L_SHIFT 8 + +#define UDMA_TP_OOR_64 64 +#define UDMA_TP_OOR_128 128 +#define UDMA_TP_OOR_256 256 +#define UDMA_TP_OOR_512 512 +#define UDMA_TP_OOR_1024 1024 + +struct udma_buf_list { + void *buf; + dma_addr_t map; +}; + +struct udma_buf { + struct udma_buf_list direct; + int nbufs; + + int npages; + int page_shift; +}; + +struct udma_tp { + u32 cqm_priv_type; + struct ubcore_tp tp; + struct ubcore_tp_cfg cfg; + struct udma_kdb *kdb; + struct tag_cqm_qpc_mpt *tp_info; + u32 entry_size; + u32 entry_cnt; + u8 pri; + u8 cos; + struct udma_buf buf; + u32 buf_size; + struct mtt ub_mtt; + struct tag_cqm_queue *tp_buf_info; + struct tag_cqm_queue *pcqc_info; + u32 tp_buf_info_xid; + u8 oor_en; + u8 refuse_report_flush_done; + u8 rsv[2]; + struct ubrc rc_table; + u32 rc_table_num; + u32 rc_log_ubrc_seg; + struct list_head node; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +static inline struct udma_tp *to_udma_tp(const struct ubcore_tp *tp) +{ + return (struct udma_tp *)udma_container_of(tp, struct udma_tp, tp); +} + +static inline struct udma_tp * +cqmobj_to_ub_tp(const struct tag_cqm_object *object) +{ + struct tag_cqm_qpc_mpt *tp_info = + container_of(object, struct tag_cqm_qpc_mpt, object); + return (struct udma_tp *)tp_info->priv; +} + +static inline int check_tp_query_ret_val(const void *ret_val, u32 val_len, + u32 ret_len, struct device *dev, + const struct ubcore_res_key *key) +{ + if (ret_val == NULL) { + udma_err(dev, UDMA_M_TP, + "failed to check ret_val, ret_val invalid (0x%x).\n", + key->key); + return -EDESTADDRREQ; + } + if (val_len < ret_len) { + udma_err(dev, UDMA_M_TP, + "Failed to check len, type: %u, val->len: %u.\n", + (u32)(key->type), val_len); + return (int)(ret_len); + } + return 0; +} + +struct udma_tp *udma_find_tp(struct udma_device *udev, u32 tpn); +struct ubcore_tp *udma_create_tp(struct ubcore_device *ub_dev, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata); +int udma_dfx_modify_tp_exec(struct udma_device *udma_dev, + const struct ub_query_inbuf *inbuf, + union ub_query_outbuf *outbuf); +int udma_modify_tp(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); +int udma_destroy_tp(struct ubcore_tp *tp); +int udma_query_tpc(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ub_dfx_tp_ctx_query *val); +int udma_query_tpc_info(struct udma_device *udma_dev, + struct ubcore_res_key *key, + struct tag_ub_tp_query_info_outbuf *val); +int udma_query_res_tp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_query_tp_wqe_from_host(struct udma_device *udma_dev, u32 tpn, + u32 wqe_id, struct ub_tp_wqe *outbuf, + u32 outbuf_size); + +int udma_create_multi_tp(struct ubcore_device *dev, u32 cnt, + struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata, + struct ubcore_tp **tp); +int udma_modify_multi_tp(u32 cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask, + struct ubcore_tp **fail_tp); +int udma_destroy_multi_tp(u32 cnt, struct ubcore_tp **tp); +int udma_tp_create_ubrc_table(struct udma_device *udma_dev, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask, + struct udma_tp *ub_tp); +int udma_query_tp_ctx_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_dfx_tp_ctx_query *outbuf); +int udma_query_tp_mpt_from_host(struct udma_device *udma_dev, u32 tpn, + struct ub_mapt_ctx_query *outbuf); +int udma_query_tp_pcqc_from_host(struct udma_device *udma_dev, u32 tpn, + ub_pcqc_query_rsp_s *outbuf); +int udma_config_dscp_vl(struct ubcore_device *dev, u8 *dscp, u8 *vl, u8 num); +int udma_get_dscp_cos(struct udma_device *udma_dev, u8 *dscp, u32 size, + u8 is_dev); +int udma_query_dscp_vl(struct ubcore_device *dev, uint8_t *dscp, uint8_t num, + uint8_t *vl); +void udma_tp_put(void *obj); +void udma_tp_get(void *obj); +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c new file mode 100644 index 000000000..6657f271a --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.c @@ -0,0 +1,677 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_mtt.h" +#include "udma_misc.h" +#include "ubcore_uapi.h" +#include "udma_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_vram.h" +#include "udma_tpg.h" + +#define UB_TP_CNT_16 16 +#define UB_TP_CNT_32 32 + +static int udma_fill_tpg_res_ret_val(const struct udma_device *udma_dev, + struct ubcore_res_val *val, u32 buf_len, + struct udma_tpg *ub_tpg, + struct ubcore_res_tpg_val *ret_val) +{ + u32 i; + + val->len = buf_len; + if (ub_tpg->tp_num > MAX_TP_CNT_IN_TPG) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "query tpg res: tp num large than tp num max, tp_num:0x%x\n", + ub_tpg->tp_num); + return -EINVAL; + } + + ret_val->tp_cnt = ub_tpg->tp_num; + ret_val->tp_list = (u32 *)vmalloc(sizeof(u32) * ret_val->tp_cnt); + if (ret_val->tp_list == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "tp_list vmalloc failed.\n"); + return -ENOMEM; + } + ret_val->dscp = ub_tpg->sl; + for (i = 0; i < ub_tpg->tp_num; i++) { + if (ub_tpg->tp_used[i] == 0) + continue; + ret_val->tp_list[i] = ub_tpg->tpinfo_list[i]->xid; + } + + return 0; +} + +static inline int check_tpg_query_ret_val(const void *ret_val, u32 val_len, + u32 ret_len, struct device *dev, + const struct ubcore_res_key *key) +{ + if (ret_val == NULL) { + udma_err(dev, UDMA_M_TPG, + "failed to check ret_val, ret_val invalid (0x%x).\n", + key->key); + return -EDESTADDRREQ; + } + if (val_len < ret_len) { + udma_err(dev, UDMA_M_TPG, + "Failed to check len, type: %u, val->len: %u.\n", + (u32)key->type, val_len); + return (int)ret_len; + } + return 0; +} + +int udma_query_res_tpg(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_tpg_val *ret_val = + (struct ubcore_res_tpg_val *)(uintptr_t)val->addr; + struct udma_tpg *ub_tpg = NULL; + int ret; + + ret = check_tpg_query_ret_val(ret_val, val->len, + sizeof(struct ubcore_res_tpg_val), + udma_dev->dev, key); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "query tpg res: failed to check return vaule, key:0x%x\n", + key->key); + return ret; + } + + ub_tpg = udma_get_tpg_by_id(udma_dev, key->key); + if (ub_tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to find tpg, id(%u)\n", key->key); + return -EINVAL; + } + + ret = udma_fill_tpg_res_ret_val(udma_dev, val, + sizeof(struct ubcore_res_tp_val), + ub_tpg, ret_val); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "query tpg res: failed to fill tpg res, key:0x%x\n", + key->key); + } + + udma_tpg_put(ub_tpg); + return ret; +} + +int udma_query_tpg_flow(struct udma_device *udma_dev, struct ub_tpg_flow *val) +{ + struct tag_ub_tp_query_info_outbuf *tpcval = NULL; + struct ubcore_res_key key = { 0 }; + struct udma_tpg *ub_tpg = NULL; + int ret = 0; + u32 i; + + tpcval = (struct tag_ub_tp_query_info_outbuf *)kzalloc( + sizeof(struct tag_ub_tp_query_info_outbuf), GFP_KERNEL); + if (tpcval == NULL) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "query tpg flow:failed to allocate memroy for tp ctx\n"); + return -ENOMEM; + } + + ub_tpg = udma_get_tpg_by_id(udma_dev, val->tpgn); + if (ub_tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to find tpg, id(%u)\n", val->tpgn); + kfree(tpcval); + return -EINVAL; + } + + val->tp_num = ub_tpg->tp_num; + + for (i = 0; i < val->tp_num; i++) { + if (ub_tpg->tp_used[i] == 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "tp %u not in use.\n", + ub_tpg->tpinfo_list[i]->xid); + continue; + } + key.key = ub_tpg->tpinfo_list[i]->xid; + ret = udma_query_tpc_info(udma_dev, &key, tpcval); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_TP, + "query tpg flow: failed to query tp context, key:0x%x\n", + key.key); + continue; + } + + val->tp_flow_info[i].tpn = tpcval->srctpn; + val->tp_flow_info[i].send_cnt = tpcval->tp_msn; + val->tp_flow_info[i].infly = tpcval->infly_wr; + val->tp_flow_info[i].wr = tpcval->pending_wr; + } + udma_tpg_put(ub_tpg); + kfree(tpcval); + + return ret; +} + +static int udma_tpg_alloc_tpn(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + u32 qpc_num = 0; //success created qpc count + u32 qpn_tmp; + u32 i; + u32 end; + + ub_tpg->tp_num = cfg->tp_cnt; + mutex_lock( + &udma_dev->tp_mutex); //solve uvs multi thread and send scenario even现 tpg in tp xid not continuous problem (need ensure 16 or 32 tpn continuous) + qpn_tmp = CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, + CQM_XID_LOW_BIT_NONE, 0); + for (i = 0; i < ub_tpg->tp_num; i++) { + end = udma_dev->ub_cap.dev_cap.max_tp; + ub_tpg->tpinfo_list[i] = udma_object_qpc_create( + udma_dev, UBEP_TP_CONTEXT_SIZE, NULL, + &udma_dev->tpn_record, qpn_tmp, 0, end); + if (ub_tpg->tpinfo_list[i] == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "tpn allocation failed, index(%u)\n", i); + qpc_num = i; + goto err_create_qpc; + } + } + mutex_unlock(&udma_dev->tp_mutex); + return 0; + +err_create_qpc: + for (i = 0; i < qpc_num; ++i) { + cqm5_object_delete(&ub_tpg->tpinfo_list[i]->object); + ub_tpg->tpinfo_list[i] = NULL; + } + mutex_unlock(&udma_dev->tp_mutex); + return -ENOMEM; +} + +static void udma_tpg_free_tpn(struct udma_device *udma_dev, + struct udma_tpg *ub_tpg, bool del_object) +{ + u32 i; + + mutex_lock(&udma_dev->tp_mutex); + for (i = 0; i < ub_tpg->tp_num; i++) { + if (del_object) { + cqm5_object_delete(&ub_tpg->tpinfo_list[i]->object); + } + ub_tpg->tpinfo_list[i] = NULL; + } + mutex_unlock(&udma_dev->tp_mutex); +} + +static int udma_tpg_rm_alloc(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + struct ubcore_tpg *tpg = &ub_tpg->tpg; + int ret = 0; + + tpg->tpgn = udma_bitmap_alloc_id(udma_dev->bitmap); //via bitmap allocate + if (tpg->tpgn >= udma_dev->bitmap->size) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode, tpgn allocation failed\n"); + tpg->tpgn = 0; + return -EBUSY; + } + + ret = udma_tpg_alloc_tpn(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode, tpn allocation failed\n"); + goto err_alloc_tpn; + } + + ret = udma_cmd_tpg_create(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, "send cmd failed.\n"); + goto err_cmd_tpg_create; + } + + return 0; + +err_cmd_tpg_create: + udma_tpg_free_tpn(udma_dev, ub_tpg, true); +err_alloc_tpn: + udma_bitmap_free_id(udma_dev->bitmap, tpg->tpgn); + return ret; +} + +static int udma_tpg_rm_free(struct udma_device *udma_dev, + struct udma_tpg *ub_tpg) +{ + int ret; + + ret = udma_cmd_tpg_delete(udma_dev, ub_tpg->tpg.tpgn, + &ub_tpg->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode. Failed to delete tp.\n"); + return ret; + } + + udma_tpg_free_tpn(udma_dev, ub_tpg, false); + + udma_bitmap_free_id(udma_dev->bitmap, ub_tpg->tpg.tpgn); + + return 0; +} + +static int udma_tpg_rc_alloc(struct udma_device *udma_dev, + const struct ubcore_tpg_cfg *cfg, + struct udma_tpg *ub_tpg) +{ + struct ubcore_tpg *tpg = &ub_tpg->tpg; + int ret = 0; + + tpg->tpgn = udma_bitmap_alloc_id(udma_dev->bitmap); //via bitmap allocate + if (tpg->tpgn >= udma_dev->bitmap->size) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tpgn allocation failed\n"); + tpg->tpgn = 0; + return -EBUSY; + } + + ret = udma_tpg_alloc_tpn(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tpn allocation failed\n"); + goto err_alloc_tpn; + } + + return 0; + +err_alloc_tpn: + udma_bitmap_free_id(udma_dev->bitmap, tpg->tpgn); + return ret; +} + +static void udma_tpg_rc_free(struct udma_device *udma_dev, + struct udma_tpg *ub_tpg) +{ + udma_bitmap_free_id(udma_dev->bitmap, ub_tpg->tpg.tpgn); + udma_tpg_free_tpn(udma_dev, ub_tpg, false); +} + +static void udma_tpg_kref_release(struct kref *ref_cnt) +{ + struct udma_tpg *tpg = container_of(ref_cnt, struct udma_tpg, ref_cnt); + + complete(&tpg->comp); +} + +void udma_tpg_put(void *obj) +{ + struct udma_tpg *tpg = obj; + + if (tpg != NULL) { + kref_put(&tpg->ref_cnt, udma_tpg_kref_release); + } +} + +void udma_tpg_get(void *obj) +{ + struct udma_tpg *tpg = obj; + + kref_get(&tpg->ref_cnt); +} + +struct udma_tpg *udma_get_tpg_by_id(struct udma_device *udma_dev, u32 tpgn) +{ + struct udma_tpg *tpg_now = NULL; + struct mutex *lock = NULL; + + udma_info(udma_dev->dev, UDMA_M_TPG, "query tpg id: %u\n", tpgn); + + lock = &udma_dev->device_res->tpg_list.node_lock; + mutex_lock(lock); + list_for_each_entry(tpg_now, &udma_dev->device_res->tpg_list.node, + node) { + if (tpg_now->tpg.tpgn == tpgn) { + udma_tpg_get(tpg_now); + mutex_unlock(lock); + return tpg_now; + } + } + mutex_unlock(lock); + + return NULL; +} + +static void udma_print_tpg_cfg(const struct ubcore_tpg_cfg *cfg, + struct udma_device *udma_dev) +{ + u32 dip0 = 0; + u32 dip1 = 0; + u32 dip2 = 0; + u32 dip3 = 0; + + udma_debug(udma_dev->dev, UDMA_M_TPG, "start print ubcore tpg cfg.\n"); + + dip0 = cfg->local_eid.in6.subnet_prefix & 0xffffffff; + dip1 = (cfg->local_eid.in6.subnet_prefix >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + dip2 = cfg->local_eid.in6.interface_id & 0xffffffff; + dip3 = (cfg->local_eid.in6.interface_id >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + + udma_debug(udma_dev->dev, UDMA_M_TPG, "local_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + dip0 = cfg->peer_eid.in6.subnet_prefix & 0xffffffff; + dip1 = (cfg->peer_eid.in6.subnet_prefix >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + dip2 = cfg->peer_eid.in6.interface_id & 0xffffffff; + dip3 = (cfg->peer_eid.in6.interface_id >> 0x20) & + 0xffffffff; //after assembling 0x20 bit + + udma_debug(udma_dev->dev, UDMA_M_TPG, "peer_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + udma_debug(udma_dev->dev, UDMA_M_TPG, + "trans_mode:%u, dscp:%u, cc_alg:%u, cc_pattern_idx:%u\n", + cfg->trans_mode, cfg->dscp, cfg->cc_alg, + cfg->cc_pattern_idx); + udma_debug(udma_dev->dev, UDMA_M_TPG, "tp_cnt:%u\n", cfg->tp_cnt); + + udma_debug(udma_dev->dev, UDMA_M_TPG, "end print ubcore tpg cfg.\n"); +} + +static int udma_check_create_tpg_params(struct ubcore_device *ub_dev, + struct ubcore_tpg_cfg *cfg) +{ + struct udma_device *udma_dev = NULL; + + if (ub_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_TPG, + "Invalid parameter, ub_dev(%d), cfg(%d)\n", + !!ub_dev, !!cfg); + return -EINVAL; + } + + udma_dev = to_udma_dev(ub_dev); + udma_print_tpg_cfg(cfg, udma_dev); + + if ((cfg->trans_mode == UBCORE_TP_RC) && + (cfg->tp_cnt != 2)) { //RC mode 1(count word) TPG has 2(count word) TP + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tp num(%u) is wrong.\n", cfg->tp_cnt); + return -EINVAL; + } + + if (cfg->trans_mode == UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_TPG, "wrong trans mode: UM.\n"); + return -EINVAL; + } + + if ((cfg->trans_mode != UBCORE_TP_RC) && + (cfg->tp_cnt != UB_TP_CNT_16) && (cfg->tp_cnt != UB_TP_CNT_32)) { + udma_err(udma_dev->dev, UDMA_M_RC_TABLE, + "tp_cnt should be 16 or 32. cur_value: %d.\n", + cfg->tp_cnt); + return -EINVAL; + } + + return 0; +} + +struct ubcore_tpg *udma_create_tpg(struct ubcore_device *ub_dev, + struct ubcore_tpg_cfg *cfg, + struct ubcore_udata *udata) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct ubcore_tpg *tpg = NULL; + struct udma_tpg *ub_tpg = NULL; + int ret; + + ret = udma_check_create_tpg_params(ub_dev, cfg); + if (ret != 0) { + return ERR_PTR(ret); + } + + udma_dev = to_udma_dev(ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + ub_tpg = (struct udma_tpg *)udma_vram_kzalloc(udma_dev->dev, + UDMA_VRAM_TYPE_TPG, + sizeof(struct udma_tpg), + GFP_KERNEL); + if (ub_tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "memory allocation failed.\n"); + return ERR_PTR(-ENOMEM); + } + tpg = &ub_tpg->tpg; + tpg->ub_dev = ub_dev; + + tpg->tpg_cfg.trans_mode = cfg->trans_mode; + ub_tpg->sl = cfg->dscp; + + if (tpg->tpg_cfg.trans_mode == UBCORE_TP_RC) { + ret = udma_tpg_rc_alloc(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RC mode, tpg failed, ret(%d)\n", ret); + goto err_tpg_alloc; + } + } else { + ret = udma_tpg_rm_alloc(udma_dev, cfg, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode, tpg failed, ret(%d)\n", ret); + goto err_tpg_alloc; + } + } + + kref_init(&ub_tpg->ref_cnt); + init_completion(&ub_tpg->comp); + udma_store_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_create_tpg cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_TPG, "create tpg %u succeed\n", + tpg->tpgn); + + return tpg; + +err_tpg_alloc: + udma_vram_kfree(udma_dev->dev, ub_tpg, UDMA_VRAM_TYPE_TPG); + return ERR_PTR(ret); +} + +int udma_destroy_tpg(struct ubcore_tpg *tpg) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_tpg *ub_tpg = NULL; + int ret; + u32 i; + + if (tpg == NULL || tpg->ub_dev == NULL) { + udma_pr_err(UDMA_M_TPG, "Invalid parameter, tpg(%d)\n", !!tpg); + return -EINVAL; + } + + udma_dev = to_udma_dev(tpg->ub_dev); + ub_tpg = to_udma_tpg(tpg); + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + udma_info(udma_dev->dev, UDMA_M_TPG, "start to destroy tpg, id: %u.\n", + tpg->tpgn); + + if (tpg->tpg_cfg.trans_mode == UBCORE_TP_UM) { + udma_err(udma_dev->dev, UDMA_M_TPG, "wrong trans mode: UM.\n"); + return -EINVAL; + } + + for (i = 0; i < ub_tpg->tp_num; i++) { + if (ub_tpg->tp_used[i] == 1) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to destroy tpg, tp %u still in use.\n", + ub_tpg->tpinfo_list[i]->xid); + return -EBUSY; + } + } + udma_delete_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + udma_tpg_put(ub_tpg); + wait_for_completion(&ub_tpg->comp); + if (tpg->tpg_cfg.trans_mode == UBCORE_TP_RC) { + udma_tpg_rc_free(udma_dev, ub_tpg); + } else { //RM mode + ret = udma_tpg_rm_free(udma_dev, ub_tpg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "RM mode. Failed to delete tp.\n"); + goto ERR_DESTORY_OUT; + } + } + + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_TPG, tpg->tpgn, + &ub_tpg->cmdq_rtt); + udma_vram_kfree(udma_dev->dev, ub_tpg, UDMA_VRAM_TYPE_TPG); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_tpg cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + return 0; +ERR_DESTORY_OUT: + kref_init(&ub_tpg->ref_cnt); + init_completion(&ub_tpg->comp); + udma_store_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + return ret; +} + +static int udma_tp_latch_tpg_alloc(struct udma_device *udma_dev, + u32 expect_tpgn) +{ + u32 tpgn; + int ret = 0; + + tpgn = udma_bitmap_alloc_id(udma_dev->bitmap); //via bitmap allocate + if (tpgn >= udma_dev->bitmap->size || tpgn != expect_tpgn) { + udma_err( + udma_dev->dev, UDMA_M_TPG, + "TP latch, tpgn allocation failed, tpgn:%u, expect_tpgn:%u.\n", + tpgn, expect_tpgn); + udma_bitmap_free_id(udma_dev->bitmap, tpgn); + return -EBUSY; + } + + ret = udma_cmd_tp_latch_tpg_create(udma_dev, tpgn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Send tp latch create tpg cmd failed, ret = %d.\n", + ret); + udma_bitmap_free_id(udma_dev->bitmap, tpgn); + return ret; + } + + return 0; +} + +void udma_tp_latch_tpg_free(struct udma_device *udma_dev, u32 tpgn) +{ + if (udma_cmd_tpg_delete(udma_dev, tpgn, NULL) != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Failed to delete tp latch tpg(%u).\n", tpgn); + } + udma_bitmap_free_id(udma_dev->bitmap, tpgn); + udma_info(udma_dev->dev, UDMA_M_TPG, + "Delete tp latch tpg(%u) successful.\n", tpgn); + return; +} + +int udma_create_tp_latch_tpg(struct udma_device *udma_dev) +{ + int ret; + u32 cur_idx; + + /* non- TPF not reserved 0 ~ 15 of TPG */ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_info( + udma_dev->dev, UDMA_M_TPG, + "Non-TPF(%u) or non-HI1825(%u) not create tp latch tpg.\n", + udma_dev->ub_dev.attr.tp_maintainer, + udma_dev->device_type); + return 0; + } + /* if already create TP Latch TPG, then not again repeat create */ + if (udma_dev->latch_tpg_configured == 1) { + return 0; + } + + for (cur_idx = 0; cur_idx < UB_TP_LATCH_RSV_TPG_NUM; cur_idx++) { + ret = udma_tp_latch_tpg_alloc(udma_dev, cur_idx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_TPG, + "Create tp latch tpg(%u) ret:%d.\n", cur_idx, + ret); + goto ERR_TPG_ALLOC; + } + udma_info(udma_dev->dev, UDMA_M_TPG, + "Create tp latch tpg(%u) successful.\n", cur_idx); + } + udma_dev->latch_tpg_configured = 1; + udma_info(udma_dev->dev, UDMA_M_TPG, + "Create tp latch tpg successful.\n"); + return 0; + +ERR_TPG_ALLOC: + /* free already allocate of TPG resource */ + for (u32 i = 0; i < cur_idx; i++) { + udma_tp_latch_tpg_free(udma_dev, i); + } + udma_dev->latch_tpg_configured = 0; + return -EFAULT; +} + +void udma_destroy_tp_latch_tpg(struct udma_device *udma_dev) +{ + /* non- TPF did not create TP latch tpg */ + if (!udma_dev->ub_dev.attr.tp_maintainer || + udma_dev->device_type != UDMA_DEV_TYPE_HI1825_V100) { + udma_info(udma_dev->dev, UDMA_M_TPG, + "Non-TPF not create tp latch tpg, no delete.\n"); + return; + } + + if (udma_dev->latch_tpg_configured == 0) { + udma_info(udma_dev->dev, UDMA_M_TPG, + "Not create tp latch tpg, no delete.\n"); + return; + } + + for (u32 i = 0; i < UB_TP_LATCH_RSV_TPG_NUM; i++) { + udma_tp_latch_tpg_free(udma_dev, i); + } + udma_dev->latch_tpg_configured = 0; + return; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h new file mode 100644 index 000000000..d003cee33 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_tpg.h @@ -0,0 +1,54 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef CPU_UDMA_TPG_H +#define CPU_UDMA_TPG_H + +#include <linux/mutex.h> +#include "ubcore_api.h" +#include "udma_common.h" +#include "ub_npu_tp_cmd_defs.h" +#include "ub_npu_tpg_cmd_defs.h" +#include "hinic5_cqm.h" +#include "udma_comp.h" + +#define TPG_BITMAP_SIZE 16384 //16K=256K/16 +#define UB_TP_LATCH_RSV_TPG_NUM 16 + +struct udma_tpg { + struct ubcore_tpg tpg; + struct tag_cqm_queue *tpg_info; + enum ubcore_transport_mode trans_mode; + u32 tp_num; + u32 tp_q_depth; + struct tag_cqm_qpc_mpt *tpinfo_list[MAX_TP_CNT_IN_TPG]; + u8 tp_used[MAX_TP_CNT_IN_TPG]; //indicates this tp whether already create + u8 sl; + struct list_head node; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +}; + +static inline struct udma_tpg *to_udma_tpg(const struct ubcore_tpg *tpg) +{ + return (struct udma_tpg *)udma_container_of(tpg, struct udma_tpg, tpg); +} + +int udma_query_res_tpg(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); + +int udma_query_tpg_flow(struct udma_device *udma_dev, struct ub_tpg_flow *val); + +struct ubcore_tpg *udma_create_tpg(struct ubcore_device *ub_dev, + struct ubcore_tpg_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_tpg(struct ubcore_tpg *tpg); +struct udma_tpg *udma_get_tpg_by_id(struct udma_device *udma_dev, u32 tpgn); +void udma_tpg_put(void *obj); +void udma_tpg_get(void *obj); +int udma_create_tp_latch_tpg(struct udma_device *udma_dev); +void udma_destroy_tp_latch_tpg(struct udma_device *udma_dev); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c new file mode 100644 index 000000000..d8d1ed0a0 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.c @@ -0,0 +1,209 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_comp.h" +#include "udma_tp.h" +#include "udma_net.h" +#include "udma_misc.h" +#include "ubcore_uapi.h" +#include "udma_cmd_utp.h" +#include "udma_utp.h" + +#define UB_UTP_DIP_OFFSET 0x20 + +static void udma_free_utp(struct udma_device *udma_dev, struct udma_utp *utp) +{ + cqm5_object_delete(&utp->cqm_utpn->object); + kfree(utp); +} + +static struct udma_utp *udma_alloc_utp(struct udma_device *udma_dev) +{ + struct udma_utp *utp = NULL; + u32 xid_end; + + utp = (struct udma_utp *)kzalloc(sizeof(struct udma_utp), GFP_KERNEL); + if (utp == NULL) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Alloc utpn: alloc utpn failed.\n"); + return NULL; + } + utp->cqm_priv_type = UDMA_CQM_PRIV_TYPE_UTP; + xid_end = SCQC_UTP_START(udma_dev) + + udma_dev->ub_cap.dev_cap_res.sdk_res.max_utp; + utp->cqm_utpn = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, 0, utp, &udma_dev->utpn_record, + false, + CQM_DYNAMIC_XID_MOD(CQM_XID_SEARCH_RANGE, CQM_XID_LOW_BIT_NONE, + 0), + SCQC_UTP_START(udma_dev), xid_end); + if (utp->cqm_utpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Alloc utpn: create cqm utpn failed.\n"); + goto err_alloc_cqm; + } + + udma_info(udma_dev->dev, UDMA_M_UTP, "alloc utpn success: %u\n", + utp->cqm_utpn->index); + + return utp; + +err_alloc_cqm: + kfree(utp); + + return NULL; +} + +static int udma_utp_cfg_check(struct udma_device *udma_dev, + const struct ubcore_utp_cfg *cfg) +{ + u64 dip; + + udma_info(udma_dev->dev, UDMA_M_UTP, "udp_start:%u\n", cfg->udp_start); + udma_info(udma_dev->dev, UDMA_M_UTP, "udp_range:%u\n", cfg->udp_range); + udma_info(udma_dev->dev, UDMA_M_UTP, "local_net_addr_idx:%u\n", + cfg->local_net_addr_idx); + udma_info(udma_dev->dev, UDMA_M_UTP, "desp:%u\n", cfg->dscp); + udma_info(udma_dev->dev, UDMA_M_UTP, "flow_label:%u\n", + cfg->flow_label); + udma_info(udma_dev->dev, UDMA_M_UTP, "hop_limit:%u\n", cfg->hop_limit); + udma_info(udma_dev->dev, UDMA_M_UTP, "port id:%u\n", cfg->port_id); + udma_info(udma_dev->dev, UDMA_M_UTP, "mtu:%u\n", cfg->mtu); + + if (cfg->peer_net_addr.type == UBCORE_NET_ADDR_TYPE_IPV4) { + udma_info(udma_dev->dev, UDMA_M_UTP, "peer is ipv4\n"); + udma_info(udma_dev->dev, UDMA_M_UTP, "dip3:0x%x\n", + cfg->peer_net_addr.net_addr.in4.addr); + } else { + udma_info(udma_dev->dev, UDMA_M_UTP, "peer is ipv6\n"); + /* dip0 - dip3 */ + dip = cfg->peer_net_addr.net_addr.in6.subnet_prefix; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip0:0x%llx\n", dip); + dip = cfg->peer_net_addr.net_addr.in6.subnet_prefix >> + UB_UTP_DIP_OFFSET; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip1:0x%llx\n", dip); + dip = cfg->peer_net_addr.net_addr.in6.interface_id; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip2:0x%llx\n", dip); + dip = cfg->peer_net_addr.net_addr.in6.interface_id >> + UB_UTP_DIP_OFFSET; + udma_info(udma_dev->dev, UDMA_M_UTP, "dip3:0x%llx\n", dip); + } + + udma_info(udma_dev->dev, UDMA_M_UTP, "vlan:%u\n", + (u32)cfg->peer_net_addr.vlan); + udma_info(udma_dev->dev, UDMA_M_UTP, "dmac_h16:%u\n", + (u32)UDMA_GET_MAC_HI(cfg->peer_net_addr.mac[0], + cfg->peer_net_addr.mac[1])); + udma_info(udma_dev->dev, UDMA_M_UTP, "dmac_l32:%u\n", + (u32)UDMA_GET_MAC_LO(cfg->peer_net_addr.mac[0x2], + cfg->peer_net_addr.mac[0x3], + cfg->peer_net_addr.mac[0x4], + cfg->peer_net_addr.mac[0x5])); + if ((cfg->mtu != UBCORE_MTU_1024) && (cfg->mtu != UBCORE_MTU_4096) && + (cfg->mtu != UBCORE_MTU_8192)) { + udma_err(udma_dev->dev, UDMA_M_UTP, "mtu:%u not support\n", + cfg->mtu); + return -EINVAL; + } + + return 0; +} + +struct ubcore_utp *udma_create_utp(struct ubcore_device *dev, + struct ubcore_utp_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_device *udma_dev = NULL; + struct udma_utp *utp = NULL; + int ret; + + if (dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_UTP, "Invalid parameter, dev(%d), cfg(%d)\n", + !!dev, !!cfg); + return NULL; + } + + udma_dev = to_udma_dev(dev); + ret = udma_utp_cfg_check(udma_dev, cfg); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "create utp: utp cfg err\n"); + return NULL; + } + + utp = udma_alloc_utp(udma_dev); + if (utp == NULL) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "create utp: failed to allocate info\n"); + return NULL; + } + + ret = udma_cmd_create_utp(udma_dev, cfg, utp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "create utp: failed to send cqm cmd\n"); + goto err_create_utp_cmd; + } + + utp->base.utpn = utp->cqm_utpn->index; + + udma_store_list(udma_dev, &utp->node, &udma_dev->device_res->utp_list); + + return &utp->base; + +err_create_utp_cmd: + udma_free_utp(udma_dev, utp); + + return NULL; +} + +int udma_destroy_utp(struct ubcore_utp *utp) +{ + struct udma_device *udma_dev = NULL; + struct udma_utp *udma_utp = NULL; + int ret; + + if (utp == NULL || utp->ub_dev == NULL) { + udma_pr_err(UDMA_M_UTP, "Invalid parameter, utp(%d)\n", !!utp); + return -EINVAL; + } + + udma_utp = to_udma_utp(utp); + udma_dev = to_udma_dev(utp->ub_dev); + udma_info(udma_dev->dev, UDMA_M_UTP, "Destroy utp, utpn: %u", + utp->utpn); + + ret = udma_cmd_destroy_utp(udma_dev, udma_utp); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Destroy utp: msg cmd failed\n"); + return ret; + } + + udma_delete_list(udma_dev, &udma_utp->node, + &udma_dev->device_res->utp_list); + udma_free_utp(udma_dev, udma_utp); + udma_info(udma_dev->dev, UDMA_M_UTP, "Destroy utp success\n"); + + return 0; +} + +int udma_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx) +{ + int ret; + + ret = udma_cmd_query_utp(udma_dev, utpn, utp_ctx); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_UTP, + "Faield query utp ctx, utpn:%u\n", utpn); + return ret; + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h new file mode 100644 index 000000000..62c305c1f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_utp.h @@ -0,0 +1,32 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +#ifndef UDMA_UTP_H +#define UDMA_UTP_H + +#include "ubcore_types.h" +#include "ubcore_api.h" +#include "hinic5_cqm.h" +#include "udma_common.h" +#include "ub_npu_utp_cmd_defs.h" + +typedef struct udma_utp { + u32 cqm_priv_type; + struct ubcore_utp base; + struct tag_cqm_queue *cqm_utpn; + struct list_head node; +} udma_utp_t; + +static inline struct udma_utp *to_udma_utp(const struct ubcore_utp *utp) +{ + return (struct udma_utp *)udma_container_of(utp, struct udma_utp, base); +} + +struct ubcore_utp *udma_create_utp(struct ubcore_device *dev, + struct ubcore_utp_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_utp(struct ubcore_utp *utp); +int udma_query_utp(struct udma_device *udma_dev, u32 utpn, + struct ub_dip_ctx_query *utp_ctx); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c new file mode 100644 index 000000000..ac762c678 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.c @@ -0,0 +1,1011 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#include <linux/vmalloc.h> +#include "ubcore_types.h" + +#include "udma_common.h" +#include "udma_cqm.h" +#include "udma_misc.h" +#include "udma_cmd.h" +#include "udma_cmd_vtp.h" +#include "udma_vram.h" +#include "udma_vtp.h" + +#define UB_VTP_DIP_MASK 0xffffffff +#define UB_VTP_DIP_OFFSET 0x20 + +static int udma_fill_vtp_buf_node(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + cfg->vtpn * sizeof(struct ub_cmd_restore_vtp_node)); + if (node->vld == UDMA_MIG_BUF_USED) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "vtpn node already exist, vtpn:%u vfid:%u\n", + cfg->vtpn, buf->func_id); + return -EINVAL; + } + + if (udma_fill_vtp_node(udma_dev, cfg, node) != 0) { + return -EINVAL; + } + + node->vld = UDMA_MIG_BUF_USED; + buf->total++; + return 0; +} + +static int udma_destroy_vtp_node(struct udma_device *udma_dev, + const struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + vtp->cfg.vtpn * + sizeof(struct ub_cmd_restore_vtp_node)); + if (node->vld == UDMA_MIG_BUF_IDLE) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to destroy vtpn in buf, node is idle, vtpn:%u, vfid:%u\n", + vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + return -EINVAL; + } + node->vld = UDMA_MIG_BUF_IDLE; + buf->total--; + return 0; +} + +int udma_restore_all_vtp_cmd(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + int ret = 0; + u32 num; + + while (buf->total > 0) { + num = (buf->total < UB_VTP_RESTORE_NUM_MAX) ? + buf->total : + UB_VTP_RESTORE_NUM_MAX; + ret = udma_cmd_restore_vtp_create_multi_vld(udma_dev, buf, num); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to send all restore_vtp cmd, ret:%d, cur_idx:%u\n", + ret, buf->cur_idx); + return -EFAULT; + } + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Send vtp cmd msg, total:%u cur_idx:%u\n", + buf->total, buf->cur_idx); + } + return ret; +} + +// by before plane of flow ensure restore_vtp in has at least 1(count word) not be use of BUF +int udma_create_vtp_buf(struct udma_device *udma_dev, struct udma_vtp_buf *buf) +{ + u64 buf_size = sizeof(struct ub_cmd_restore_vtp_node) * UB_MAX_VTP_NUM; + struct tag_cqm_cmd_buf *cqm_cmd_buf = NULL; + struct ub_cmd_vtp_restore *inbuf = NULL; + int ret = -ENOMEM; + + mutex_lock(&buf->vf_mutex); + buf->addr = vmalloc(buf_size); + if (buf->addr == NULL) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "create_vtp_buf: failed to alloc buffer for restore vtp\n"); + goto out; + } + (void)memset_s(buf->addr, buf_size, 0, buf_size); + buf->total = 0; + buf->cur_idx = 0; + buf->mig_sts = UDMA_MIG_FULL_VOLUME; + + buf->cqm_cmd_buf = + udma_cmd_restore_vtp_get_cqm_cmd_buf(udma_dev, buf->func_id); + if (buf->cqm_cmd_buf == NULL) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "create_vtp_buf: failed to alloc buffer for cqm_cmd_buf\n"); + buf->mig_sts = UDMA_MIG_IDLE; + vfree(buf->addr); + buf->addr = NULL; + goto out; + } + cqm_cmd_buf = (struct tag_cqm_cmd_buf *)buf->cqm_cmd_buf; + inbuf = (struct ub_cmd_vtp_restore *)cqm_cmd_buf->buf; + inbuf->body.num = 0; + ret = 0; +out: + mutex_unlock(&buf->vf_mutex); + return ret; +} + +void udma_free_vtp_buf(struct udma_device *udma_dev, struct udma_vtp_buf *buf) +{ + if (buf->addr != NULL) { + vfree(buf->addr); + buf->addr = NULL; + } + + if (buf->cqm_cmd_buf != NULL) { + udma_cqm5_cmd_free_inoutbuf(udma_dev->hwdev, buf->cqm_cmd_buf, + NULL); + buf->cqm_cmd_buf = NULL; + } +} + +int udma_restore_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt) +{ + struct udma_vtp_buf *buf = NULL; + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Recv restore vtp: vtpn:%u vf:%u\n", cfg->vtpn, + adapted_ue_idx(cfg)); + + buf = udma_mig_get_vf_buf(udma_dev, adapted_ue_idx(cfg)); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Create VTP: vf:%u status idle.\n", + adapted_ue_idx(cfg)); + return udma_cmd_create_vtp(udma_dev, cfg, rtt); + } + + mutex_lock(&buf->vf_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Restore vtp: vtpn:%u vf:%u sts:%u\n", cfg->vtpn, + adapted_ue_idx(cfg), buf->mig_sts); + if (buf->mig_sts == UDMA_MIG_IDLE) { + ret = udma_cmd_create_vtp(udma_dev, cfg, rtt); + } else if (buf->mig_sts == UDMA_MIG_FULL_VOLUME) { + ret = udma_fill_vtp_buf_node(udma_dev, cfg, buf); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL) { + ret = udma_cmd_restore_vtp_create_vtpn(udma_dev, cfg, buf); + } else { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig create VTP: faild, vf:%u sts:%u.\n", + adapted_ue_idx(cfg), buf->mig_sts); + ret = -EPERM; + } + mutex_unlock(&buf->vf_mutex); + + return ret; +} + +static void udma_print_vtp_cfg(const struct ubcore_vtp_cfg *cfg, + const struct udma_device *udma_dev) +{ + u32 dip0 = 0; + u32 dip1 = 0; + u32 dip2 = 0; + u32 dip3 = 0; + + udma_debug(udma_dev->dev, UDMA_M_VTP, "start print ubcore vtp cfg.\n"); + udma_debug(udma_dev->dev, UDMA_M_VTP, + "fe_idx:%u, vtpn:%u, local_jetty:%u, peer_jetty:%u\n", + adapted_ue_idx(cfg), cfg->vtpn, cfg->local_jetty, + cfg->peer_jetty); + udma_debug(udma_dev->dev, UDMA_M_VTP, "flag:%u\n", cfg->flag.value); + + dip0 = cfg->local_eid.in6.subnet_prefix & UB_VTP_DIP_MASK; + dip1 = (cfg->local_eid.in6.subnet_prefix >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + dip2 = cfg->local_eid.in6.interface_id & UB_VTP_DIP_MASK; + dip3 = (cfg->local_eid.in6.interface_id >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + + udma_debug(udma_dev->dev, UDMA_M_VTP, "local_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + dip0 = cfg->peer_eid.in6.subnet_prefix & UB_VTP_DIP_MASK; + dip1 = (cfg->peer_eid.in6.subnet_prefix >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + dip2 = cfg->peer_eid.in6.interface_id & UB_VTP_DIP_MASK; + dip3 = (cfg->peer_eid.in6.interface_id >> UB_VTP_DIP_OFFSET) & + UB_VTP_DIP_MASK; + + udma_debug(udma_dev->dev, UDMA_M_VTP, "peer_eid:0x%x %x %x %x\n", dip0, + dip1, dip2, dip3); + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", cfg->trans_mode, + UB_VTP_TYPE_TPG, cfg->tpg->tpgn); + break; + case UBCORE_TP_RC: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", cfg->trans_mode, + UB_VTP_TYPE_TP, cfg->tp->tpn); + break; + case UBCORE_TP_UM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", cfg->trans_mode, + UB_VTP_TYPE_DIP, cfg->utp->utpn); + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, "Unknown trans_mode: %u\n", + cfg->trans_mode); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, "end print ubcore vtp cfg.\n"); +} + +static int udma_check_vtp_cfg(const struct ubcore_vtp_cfg *cfg, + const struct udma_device *udma_dev) +{ + if (udma_dev == NULL || cfg == NULL) { + udma_pr_err(UDMA_M_VTP, + "Invalid parameter: udma_dev(%d), cfg(%d).\n", + !!udma_dev, !!cfg); + return -EINVAL; + } + + if (cfg->vtpn >= UB_MAX_VTP_NUM) { // VTPN take value [64, 128k-1] + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, vtpn:%u invalid\n", cfg->vtpn); + return -EINVAL; + } + + switch (cfg->trans_mode) { + case UBCORE_TP_RM: + if (cfg->tpg == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, tpg is null\n"); + return -EINVAL; + } + return 0; + case UBCORE_TP_RC: + if (cfg->tp == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, tp is null\n"); + return -EINVAL; + } + return 0; + case UBCORE_TP_UM: + if (cfg->utp == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, utp is null\n"); + return -EINVAL; + } + return 0; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, + "Invalid parameter, trans_mode:%u\n", cfg->trans_mode); + return -EINVAL; + } +} + +struct ubcore_vtp *udma_create_vtp(struct ubcore_device *ub_dev, + struct ubcore_vtp_cfg *cfg, + struct ubcore_udata *udata) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = to_udma_dev(ub_dev); + struct udma_vtp *vtp = NULL; + int ret; + + if (udma_dev == NULL) { + pr_err("Invalid parameter. udma_dev(%d) .\n", !!udma_dev); + return ERR_PTR(-EINVAL); + } + + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + ret = udma_check_vtp_cfg(cfg, udma_dev); + if (ret != 0) { + return ERR_PTR(-EINVAL); + } + + udma_print_vtp_cfg(cfg, udma_dev); + + udma_debug( + udma_dev->dev, UDMA_M_VTP, + "Create vtp, vtpn: %u, vfid: %u, trans_mode: %u, migrate: %u\n", + cfg->vtpn, adapted_ue_idx(cfg), cfg->trans_mode, + cfg->flag.bs.migrate); + + vtp = (struct udma_vtp *)udma_vram_kzalloc(udma_dev->dev, + UDMA_VRAM_TYPE_VTP, + sizeof(struct udma_vtp), + GFP_KERNEL); + if (vtp == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Create vtp: alloc vtp failed, vtpn: %u, vfid: %u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + return ERR_PTR(-ENOMEM); + } + + udma_cmdq_rtt_statistic_init(&vtp->cmdq_rtt); + ret = (!UDMA_IS_MIG_VFID(adapted_ue_idx(cfg))) ? + udma_cmd_create_vtp(udma_dev, cfg, &vtp->cmdq_rtt) : + udma_restore_vtp(udma_dev, cfg, &vtp->cmdq_rtt); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Create vtp: send cmd failed, vtpn: %u, vfid: %u\n", + cfg->vtpn, adapted_ue_idx(cfg)); + goto err_and_free_vtp; + } + (void)memcpy_s((void *)&vtp->base.cfg, sizeof(struct ubcore_vtp_cfg), + cfg, sizeof(struct ubcore_vtp_cfg)); + udma_store_list(udma_dev, &vtp->node, &udma_dev->device_res->vtp_list); + + kref_init(&vtp->ref_cnt); + init_completion(&vtp->comp); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_create_vtp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return &vtp->base; + +err_and_free_vtp: + udma_vram_kfree(udma_dev->dev, vtp, UDMA_VRAM_TYPE_VTP); + + return ERR_PTR(-EFAULT); +} + +struct ubcore_vtpn *udma_alloc_vtpn(struct ubcore_device *ub_dev) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_vtpn *vtpn = NULL; + u32 qpn_tmp; + + if (ub_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "Invalid parameter, ub_dev is null\n"); + return ERR_PTR(-EINVAL); + } + + udma_dev = to_udma_dev(ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + vtpn = (struct udma_vtpn *)kzalloc(sizeof(struct udma_vtpn), + GFP_KERNEL); + if (vtpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Alloc vtpn: alloc vtpn failed.\n"); + return ERR_PTR(-ENOMEM); + } + qpn_tmp = udma_get_qpn_mod(udma_dev, CQM_XID_LOW_BIT_NONE, 0); + vtpn->cqm_priv_type = UDMA_CQM_PRIV_TYPE_VTPN; + vtpn->cqm_vtpn = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, UB_VTPC_SIZE, vtpn, + &udma_dev->vtpn_record, false, qpn_tmp, + SCQC_VTP_START(udma_dev), + udma_dev->ub_cap.dev_cap_res.sdk_res.max_vtp); + if (vtpn->cqm_vtpn == NULL) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Alloc vtpn: create cqm vtpn failed.\n"); + goto err_alloc_cqm; + } + + vtpn->base.vtpn = vtpn->cqm_vtpn->index; + udma_store_list(udma_dev, &vtpn->node, &udma_dev->device_res->vtp_list); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_alloc_vtpn cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_debug(udma_dev->dev, UDMA_M_VTP, "alloc vtpn success: %u\n", + vtpn->base.vtpn); + + return &vtpn->base; + +err_alloc_cqm: + kfree(vtpn); + + return ERR_PTR(-EFAULT); +} + +void udma_set_vtp_buf_idle(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + u8 origin_mig_sts; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Start to set vtp buf idle : vfid:%u\n", buf->func_id); + + mutex_lock(&mig_mgt->mig_vf_mgt_mutex); + + mutex_lock(&buf->vf_mutex); + origin_mig_sts = buf->mig_sts; + buf->total = 0; + buf->cur_idx = 0; + buf->mig_sts = UDMA_MIG_IDLE; + udma_free_vtp_buf(udma_dev, buf); + mutex_unlock(&buf->vf_mutex); + + mig_mgt->migrating_num = (origin_mig_sts == UDMA_MIG_IDLE) ? + mig_mgt->migrating_num : + mig_mgt->migrating_num - 1; + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Set vtp buf idle done: vfid:%u migrating_num:%u\n", + buf->func_id, mig_mgt->migrating_num); +} + +/* + * need consider call this function of scenario contain: + * 1. live migration the once full same step VTP before: from 128K buf in delete corresponding node; + * 2. live migration the once full same step VTP complete ~ destination end restore during: destroy operation cache in size as 32 of inoutbuf in, full 32 then flush down; + * 3. live migration complete after: receive to before live migration during created VTP request corresponding to destroy, directly send to microcode, this when ub_en already is 1 + */ +static int udma_mig_destroy_vtp(struct ubcore_vtp *vtp, + struct udma_device *udma_dev, u32 vfid) +{ + struct udma_vtp_buf *buf = NULL; + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Recv mig destroy VTP: vfid:%u vtpn:%u\n", vfid, + vtp->cfg.vtpn); + + buf = udma_mig_get_vf_buf(udma_dev, vfid); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + return udma_cmd_destroy_vtp(udma_dev, vtp); + } + + mutex_lock(&buf->vf_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig destroy VTP: vf:%u vtpn:%u sts:%u\n", vfid, + vtp->cfg.vtpn, buf->mig_sts); + if (buf->mig_sts == UDMA_MIG_IDLE) { + ret = udma_cmd_destroy_vtp(udma_dev, vtp); + } else if (buf->mig_sts == UDMA_MIG_FULL_VOLUME) { + ret = udma_destroy_vtp_node(udma_dev, vtp, buf); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL) { + ret = udma_cmd_restore_vtp_destroy_vtpn(udma_dev, vtp, buf); + } else { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig destroy VTP: vf:%u recv unexpected req.\n", vfid); + ret = -EPERM; + } + mutex_unlock(&buf->vf_mutex); + + return ret; +} + +int udma_destroy_vtp(struct ubcore_vtp *vtp) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_vtp *ub_vtp = NULL; + int ret; + u32 vfid; + + if (vtp == NULL || vtp->ub_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "Invalid parameter, vtp(%d)\n", !!vtp); + return -EINVAL; + } + + ub_vtp = to_udma_vtp(vtp); + udma_dev = to_udma_dev(vtp->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + vfid = adapted_ue_idx(&vtp->cfg); + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp, vtpn:%u vfid:%u mig:%u\n", vtp->cfg.vtpn, vfid, + vtp->cfg.flag.bs.migrate); + + if (vtp->cfg.vtpn >= UB_MAX_VTP_NUM) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp, vtpn:%u invalid. vfid:%u\n", + vtp->cfg.vtpn, vfid); + return -EINVAL; + } + + udma_delete_list(udma_dev, &ub_vtp->node, + &udma_dev->device_res->vtp_list); + udma_vtp_put(ub_vtp); + wait_for_completion(&ub_vtp->comp); + + ret = (!UDMA_IS_MIG_VFID(vfid)) ? + udma_cmd_destroy_vtp(udma_dev, vtp) : + udma_mig_destroy_vtp(vtp, udma_dev, vfid); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp: msg cmd failed\n"); + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Destroy vtp success, vtpn: %u, vfid: %u\n", vtp->cfg.vtpn, + vfid); + udma_print_cmdq_rtt_stat(udma_dev, UDMA_M_VTP, vtp->cfg.vtpn, + &ub_vtp->cmdq_rtt); + udma_vram_kfree(udma_dev->dev, ub_vtp, UDMA_VRAM_TYPE_VTP); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_destroy_vtp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return ret; +} + +int udma_free_vtpn(struct ubcore_vtpn *vtpn) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + struct udma_vtpn *ub_vtpn = NULL; + u32 id; + + if (vtpn == NULL || vtpn->ub_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "Invalid parameter, vtpn(%d)\n", + !!vtpn); + return -EINVAL; + } + + id = vtpn->vtpn; + ub_vtpn = to_udma_vtpn(vtpn); + udma_dev = to_udma_dev(vtpn->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + udma_info(udma_dev->dev, UDMA_M_VTP, "Free vtpn: %u\n", id); + + udma_delete_list(udma_dev, &ub_vtpn->node, + &udma_dev->device_res->vtp_list); + + if (ub_vtpn->cqm_vtpn != NULL) { + cqm5_object_delete(&ub_vtpn->cqm_vtpn->object); + } + + kfree(ub_vtpn); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_free_vtpn cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + udma_info(udma_dev->dev, UDMA_M_VTP, "Free vtpn %u success\n", id); + + return 0; +} + +static void udma_vtp_kref_release(struct kref *ref_cnt) +{ + struct udma_vtp *vtp = container_of(ref_cnt, struct udma_vtp, ref_cnt); + + complete(&vtp->comp); +} + +void udma_vtp_put(void *obj) +{ + struct udma_vtp *vtp = obj; + + if (vtp != NULL) { + kref_put(&vtp->ref_cnt, udma_vtp_kref_release); + } +} + +void udma_vtp_get(void *obj) +{ + struct udma_vtp *vtp = obj; + + kref_get(&vtp->ref_cnt); +} + +struct udma_vtp *udma_get_vtp_by_id(struct udma_device *udma_dev, u32 vtpn, + u32 function_id) +{ + struct udma_vtp *vtp_now = NULL; + struct mutex *lock = NULL; + + udma_info_limit(udma_dev->dev, UDMA_M_VTP, + "query vtp id: %u, function_id:%u\n", vtpn, + function_id); + + lock = &udma_dev->device_res->vtp_list.node_lock; + mutex_lock(lock); + list_for_each_entry(vtp_now, &udma_dev->device_res->vtp_list.node, + node) { + if (vtp_now->base.cfg.vtpn == vtpn && + adapted_ue_idx(&vtp_now->base.cfg) == function_id) { + udma_vtp_get(vtp_now); + goto found; + } + } + + vtp_now = NULL; +found: + mutex_unlock(lock); + return vtp_now; +} + +static int udma_get_vtp_trans_mode(const ub_vtp_ctx_info_s *vtp_ctx) +{ + switch (vtp_ctx->type) { + case UB_VTP_TYPE_TPG: + return UBCORE_TP_RM; + case UB_VTP_TYPE_TP: + return UBCORE_TP_RC; + case UB_VTP_TYPE_DIP: + return UBCORE_TP_UM; + default: + return -EINVAL; + } +} + +int udma_query_res_vtp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_vtp_val *vtp = + (struct ubcore_res_vtp_val *)(uintptr_t)(val->addr); + u32 vfid = key->key_ext; + struct ub_vtp_ctx_query *vtp_ctx = NULL; + ub_vtp_ctx_info_s *vtp_ctx_info = NULL; + u32 vtp_id = key->key; + int ret; + + if (val->len < sizeof(struct ubcore_res_vtp_val)) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Failed to check len, type: %u, val->len: %u.\n", + (u32)key->type, val->len); + return (int)sizeof(struct ubcore_res_vtp_val); + } + + vtp_ctx = (struct ub_vtp_ctx_query *)kzalloc( + sizeof(struct ub_vtp_ctx_query), GFP_KERNEL); + if (vtp_ctx == NULL) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "query vtp res:failed to allocate memroy for vtp ctx\n"); + return -ENOMEM; + } + + ret = udma_cmd_query_vtp(udma_dev, vfid, vtp_id, vtp_ctx); + if (ret != 0) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "query vtp res: failed to query vtp context, key:0x%x\n", + key->key); + kfree(vtp_ctx); + return ret; + } + vtp_ctx_info = (ub_vtp_ctx_info_s *)(void *)vtp_ctx; + vtp_ctx_info->dw0 = be32_to_cpu(vtp_ctx_info->dw0); + adapted_ue_idx(vtp) = (u16)vfid; + vtp->vtpn = key->key; + (void)memcpy_s(&vtp->local_eid, UBCORE_EID_SIZE, vtp_ctx_info->seid, + UBCORE_EID_SIZE); + (void)memcpy_s(&vtp->peer_eid, UBCORE_EID_SIZE, vtp_ctx_info->deid, + UBCORE_EID_SIZE); + ret = udma_get_vtp_trans_mode(vtp_ctx_info); + if (ret < 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "query vtp res:failed to get vtp trans_mode:%d\n", + ret); + kfree(vtp_ctx); + return ret; + } + vtp->trans_mode = (u32)ret; + vtp->tpgn = vtp_ctx_info->index; + + kfree(vtp_ctx); + return 0; +} + +static void udma_print_vtp_attr(const struct ubcore_vtp *vtp, + const struct ubcore_vtp_attr *attr, + const union ubcore_vtp_attr_mask *mask) +{ + struct udma_device *udma_dev = to_udma_dev(vtp->ub_dev); + + if (udma_dev == NULL) { + udma_pr_err(UDMA_M_VTP, "to_udma_dev failed.\n"); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, "start print ubcore vtp attr.\n"); + switch (vtp->cfg.trans_mode) { + case UBCORE_TP_RM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", + vtp->cfg.trans_mode, UB_VTP_TYPE_TPG, + attr->tp.tpg->tpgn); + break; + case UBCORE_TP_RC: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", + vtp->cfg.trans_mode, UB_VTP_TYPE_TP, + attr->tp.tp->tpn); + break; + case UBCORE_TP_UM: + udma_debug(udma_dev->dev, UDMA_M_VTP, + "trans_mode:%u, type:%u, id:%u\n", + vtp->cfg.trans_mode, UB_VTP_TYPE_DIP, + attr->tp.utp->utpn); + break; + default: + udma_err(udma_dev->dev, UDMA_M_VTP, "Unknown trans_mode: %u\n", + vtp->cfg.trans_mode); + return; + } + + udma_debug(udma_dev->dev, UDMA_M_VTP, "mask:%u\n", mask->value); + udma_debug(udma_dev->dev, UDMA_M_VTP, "end print ubcore vtp attr.\n"); +} + +static int udma_modify_vtp_node(struct udma_device *udma_dev, + struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + u32 new_tpn; + int ret; + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + vtp->cfg.vtpn * + sizeof(struct ub_cmd_restore_vtp_node)); + if (node->vld == UDMA_MIG_BUF_IDLE) { + udma_err( + udma_dev->dev, UDMA_M_VTP, + "Failed to modify vtpn in buf, node is idle, vtpn:%u, vfid:%u\n", + vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + return -EINVAL; + } + + ret = udma_cmd_modify_get_new_tpn(udma_dev, &vtp->cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get tpn failed, vtpn:%u vfid:%u\n", vtp->cfg.vtpn, + adapted_ue_idx(&vtp->cfg)); + return ret; + } + node->id = new_tpn; + udma_info(udma_dev->dev, UDMA_M_VTP, "Modify vtpn:%u vf:%u tpn:%u\n", + vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg), new_tpn); + + return 0; +} + +int udma_flush_remain_modify(struct udma_device *udma_dev, + struct udma_vtp_buf *buf) +{ + struct ub_cmd_restore_vtp_node *node = + (struct ub_cmd_restore_vtp_node *)((u8 *)buf->addr); + int vtpn, ret; + + for (vtpn = 0; vtpn < UB_MAX_VTP_NUM; vtpn++) { + if (buf->total == 0) { + break; + } + if (node->vld == UDMA_MIG_BUF_IDLE) { + node = (vtpn == UB_MAX_VTP_NUM - 1) ? node : node + 1; + continue; + } + ret = udma_cmd_send_modify(udma_dev, buf->func_id, vtpn, + node->id); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Send modify failed, vtpn:%u vfid:%u\n", vtpn, + buf->func_id); + return ret; + } + node->vld = UDMA_MIG_BUF_IDLE; + node++; + buf->total--; + } + return 0; +} + +static int udma_modify_during_fls(struct udma_device *udma_dev, + struct ubcore_vtp *vtp, + struct udma_vtp_buf *buf, + struct ubcore_vtp_attr *attr) +{ + struct ub_cmd_restore_vtp_node *node = NULL; + u32 new_tpn; + int ret; + + ret = udma_cmd_modify_get_new_tpn(udma_dev, &vtp->cfg, attr, &new_tpn); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Get tpn failed, vtpn:%u vfid:%u\n", vtp->cfg.vtpn, + adapted_ue_idx(&vtp->cfg)); + return ret; + } + + node = (struct ub_cmd_restore_vtp_node + *)((u8 *)buf->addr + + vtp->cfg.vtpn * + sizeof(struct ub_cmd_restore_vtp_node)); + node->vld = UDMA_MIG_BUF_USED; + node->id = new_tpn; + buf->total++; + return 0; +} + +static int udma_mig_modify_vtp(struct udma_device *udma_dev, + struct ubcore_vtp *vtp, + struct ubcore_vtp_attr *attr) +{ + struct udma_vtp_buf *buf = NULL; + u16 vfid = adapted_ue_idx(&vtp->cfg); + int ret = 0; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig Modify VTP: vtpn:%u vfid:%u\n", vtp->cfg.vtpn, vfid); + + buf = udma_mig_get_vf_buf(udma_dev, vfid); + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_debug(udma_dev->dev, UDMA_M_VTP, + "Mig Modify VTP: vtpn:%u vfid:%u\n", vtp->cfg.vtpn, + vfid); + return udma_cmd_modify_vtp(udma_dev, vtp, attr); + } + + mutex_lock(&buf->vf_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig Modify VTP: vtpn:%u vf:%u sts:%u\n", vtp->cfg.vtpn, vfid, + buf->mig_sts); + if (buf->mig_sts == UDMA_MIG_FULL_VOLUME) { + ret = udma_modify_vtp_node(udma_dev, vtp, attr, buf); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL) { + ret = udma_cmd_restore_vtp_modify_vtpn(udma_dev, &vtp->cfg, buf, + attr); + } else if (buf->mig_sts == UDMA_MIG_INCREMENTAL_MODIFY) { + ret = udma_modify_during_fls(udma_dev, vtp, buf, attr); + } else if (buf->mig_sts == UDMA_MIG_NORMAL) { + ret = udma_cmd_modify_vtp(udma_dev, vtp, attr); + } + mutex_unlock(&buf->vf_mutex); + + return ret; +} + +int udma_modify_vtp(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + union ubcore_vtp_attr_mask *mask) +{ + uint64_t cycle_start = 0; + struct udma_device *udma_dev = NULL; + int ret; + + if (vtp == NULL || vtp->ub_dev == NULL || attr == NULL || + mask == NULL) { + udma_pr_err(UDMA_M_VTP, + "Invalid parameter, vtp(%d), attr(%d), mask(%d)\n", + !!vtp, !!attr, !!mask); + return -EINVAL; + } + udma_print_vtp_attr(vtp, attr, mask); + + udma_dev = to_udma_dev(vtp->ub_dev); + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + cycle_start = udma_get_cycles_dfx(); + } + if (mask->bs.tp == 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify VTP: Invalid mask:0x%x vtpn:%u vfid:%u\n", + mask->value, vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + return -EINVAL; + } + + ret = (!UDMA_IS_MIG_VFID(adapted_ue_idx(&vtp->cfg))) ? + udma_cmd_modify_vtp(udma_dev, vtp, attr) : + udma_mig_modify_vtp(udma_dev, vtp, attr); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Modify VTP: proc failed ret:%d vtpn:%u vfid:%u\n", + ret, vtp->cfg.vtpn, adapted_ue_idx(&vtp->cfg)); + } + if (unlikely(udma_dev->sysfs_res.dfx_ctx.debug_en == 1)) { + udma_warn(udma_dev->dev, UDMA_M_TP, + "[PERF]udma_modify_vtp cost cycles:%llu\n", + udma_get_cycles_dfx() - cycle_start); + } + + return ret; +} + +static int udma_vf_migrate_start_proc(struct udma_device *udma_dev, u16 vfid) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = + &mig_mgt->buf_array[UDMA_GET_MIG_VF_IDX(vfid)]; + + if (UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Duplicate proc, vf:%u is migrating, sts:%u\n", vfid, + buf->mig_sts); + return -EPERM; + } + + mutex_lock(&mig_mgt->mig_vf_mgt_mutex); + if (mig_mgt->migrating_num == UDMA_MAX_MIG_VF_NUM) { + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_err(udma_dev->dev, UDMA_M_VTP, + "Migrating vf num has reached the upper limit.\n"); + return -EPERM; + } + + if (udma_create_vtp_buf(udma_dev, buf) != 0) { + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_err(udma_dev->dev, UDMA_M_VTP, + "Create vtp buf failed, vf:%u.\n", vfid); + return -ENOMEM; + } + mig_mgt->migrating_num += 1; + mutex_unlock(&mig_mgt->mig_vf_mgt_mutex); + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig start done: vf:%u, migrating:%u\n", vfid, + mig_mgt->migrating_num); + + return 0; +} + +int udma_vf_migrate_end_proc(struct udma_device *udma_dev, u16 vfid) +{ + struct udma_mig_vf_mgt *mig_mgt = &udma_dev->mig_vf_mgt; + struct udma_vtp_buf *buf = + &mig_mgt->buf_array[UDMA_GET_MIG_VF_IDX(vfid)]; + + if (!UDMA_IS_VF_MIGRATING(buf->mig_sts)) { + udma_info(udma_dev->dev, UDMA_M_VTP, "Migrate: vf:%u is idle\n", + vfid); + return 0; + } + + udma_set_vtp_buf_idle(udma_dev, + buf); // UVS rollback scenario next, buf in may still exist node + udma_info(udma_dev->dev, UDMA_M_VTP, + "Mig end Done: vf:%u, migrating:%u\n", vfid, + mig_mgt->migrating_num); + + return 0; +} + +// used for receive UVS notify of migration state change +int udma_mig_status_change_proc(struct udma_device *udma_dev, void *para) +{ + struct udma_vf_mig_status_in *mig_status = + (struct udma_vf_mig_status_in *)para; + int ret; + + udma_info(udma_dev->dev, UDMA_M_VTP, + "recv Mig change: vf:%u sts:%u migrating_num:%u\n", + mig_status->vfid, mig_status->sts, + udma_dev->mig_vf_mgt.migrating_num); + + if (!udma_dev->ub_dev.attr.tp_maintainer) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig change: global func id:%u not tp maintainer.\n", + hinic5_global_func_id(udma_dev->hwdev)); + return -EINVAL; + } + + if (!UDMA_IS_MIG_VFID(mig_status->vfid)) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig change: vfid:%u invalid\n", mig_status->vfid); + return -EINVAL; + } + + ret = (mig_status->sts == UDMA_VF_MIG_START) ? + udma_vf_migrate_start_proc(udma_dev, mig_status->vfid) : + udma_vf_migrate_end_proc(udma_dev, mig_status->vfid); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_VTP, + "Mig change: failed, vf:%u sts:%u\n", mig_status->vfid, + mig_status->sts); + } + + return ret; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h new file mode 100644 index 000000000..3407d7a64 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/tp/udma_vtp.h @@ -0,0 +1,66 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + + +#ifndef UDMA_VTP_H +#define UDMA_VTP_H + +#include "udma_cmd_vtp.h" + +#define UB_VTPC_SIZE 128 + +typedef struct udma_vtp { + struct ubcore_vtp base; + struct list_head node; + u64 mig_switch_time; + u64 mig_rollback_time; + struct cmdq_rtt cmdq_rtt; + struct kref ref_cnt; /** reference count */ + struct completion comp; /** reference count */ +} udma_vtp_t; + +typedef struct udma_vtpn { + u32 cqm_priv_type; + struct ubcore_vtpn base; + struct list_head node; + struct tag_cqm_queue *cqm_vtpn; +} udma_vtpn_t; + +static inline struct udma_vtp *to_udma_vtp(const struct ubcore_vtp *vtp) +{ + return (struct udma_vtp *)udma_container_of(vtp, struct udma_vtp, base); +} + +static inline struct udma_vtpn *to_udma_vtpn(const struct ubcore_vtpn *vtpn) +{ + return (struct udma_vtpn *)udma_container_of(vtpn, struct udma_vtpn, + base); +} + +int udma_mig_status_change_proc(struct udma_device *udma_dev, void *para); +struct ubcore_vtp *udma_create_vtp(struct ubcore_device *ub_dev, + struct ubcore_vtp_cfg *cfg, + struct ubcore_udata *udata); +struct ubcore_vtpn *udma_alloc_vtpn(struct ubcore_device *ub_dev); +int udma_destroy_vtp(struct ubcore_vtp *vtp); +int udma_free_vtpn(struct ubcore_vtpn *vtpn); +int udma_query_res_vtp(struct udma_device *udma_dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); +int udma_restore_vtp(struct udma_device *udma_dev, + const struct ubcore_vtp_cfg *cfg, struct cmdq_rtt *rtt); +struct udma_vtp *udma_get_vtp_by_id(struct udma_device *udma_dev, u32 vtpn, + u32 function_id); +int udma_modify_vtp(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + union ubcore_vtp_attr_mask *mask); +int udma_restore_all_vtp_cmd(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +void udma_free_vtp_buf(struct udma_device *udma_dev, struct udma_vtp_buf *buf); +void udma_set_vtp_buf_idle(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +int udma_flush_remain_modify(struct udma_device *udma_dev, + struct udma_vtp_buf *buf); +int udma_vf_migrate_end_proc(struct udma_device *udma_dev, u16 vfid); +void udma_vtp_put(void *obj); +void udma_vtp_get(void *obj); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c new file mode 100644 index 000000000..5701864c3 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.c @@ -0,0 +1,96 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include "udma_log.h" +#include "udma_userctl_uvs.h" +#include "udma_userctl_dfx.h" +#include "udma_userctl.h" + +enum udma_user_ctl_op_feat { + UDMA_USER_CTL_OP_FEAT_UVS, + UDMA_USER_CTL_OP_FEAT_DFX, +}; + +static struct udma_user_ctl_op_entry g_udma_user_ctl_op_feats[] = { + { UDMA_USER_CTL_OP_FEAT_UVS, udma_userctl_uvs_dispatch }, + { UDMA_USER_CTL_OP_FEAT_DFX, udma_userctl_dfx_dispatch }, +}; + +int udma_user_ctl(struct ubcore_device *dev, struct ubcore_user_ctl *user_ctl) +{ + struct udma_device *udev = NULL; + udma_user_ctl_op_func op_func = NULL; + int ret; + + if (dev == NULL || user_ctl == NULL) { + udma_pr_err( + UDMA_M_USER_CTL, + "User_ctl invalid parameter. dev(%d), user_ctl(%d).\n", + !!dev, !!user_ctl); + return -EINVAL; + } + + udev = to_udma_dev(dev); + + udma_info(udev->dev, UDMA_M_USER_CTL, "Udma user_ctl, opcode(%u).\n", + user_ctl->in.opcode); + + op_func = UDMA_USER_CTL_GET_OP_ENTRY_FUNC( + g_udma_user_ctl_op_feats, + UDMA_USER_CTL_GET_OP_FEAT(user_ctl->in.opcode)); + if (op_func == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get feature callback, opcode(%u).\n", + user_ctl->in.opcode); + return -EINVAL; + } + + ret = op_func(udev, user_ctl); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_USER_CTL, + "Failed to execute the function of opcode(%u), ret(%d).\n", + user_ctl->in.opcode, ret); + return ret; + } + + return 0; +} + +int32_t udma_check_cmd_out_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size) +{ + if (user_ctl->out.len < valid_size) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->out.len(%u).\n", + user_ctl->out.len); + return -EINVAL; + } + if (user_ctl->out.addr == 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->out.addr(0x%llx).\n", + user_ctl->out.addr); + return -EINVAL; + } + return 0; +} + +int32_t udma_check_cmd_in_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size) +{ + if (user_ctl->in.len != valid_size) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->in.len(%u).\n", + user_ctl->in.len); + return -EINVAL; + } + if (user_ctl->in.addr == 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! user_ctl->in.addr(0x%llx).\n", + user_ctl->in.addr); + return -EINVAL; + } + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h new file mode 100644 index 000000000..a2d82cc3f --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl.h @@ -0,0 +1,51 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef UDMA_USERCTL_H +#define UDMA_USERCTL_H + +#include "ubcore_types.h" +#include "udma_common.h" + +/* The size of opcode is 32 bits. The format of opcode is as follows: + * [31:16]-reserved, [15:8]-feature, [7:0]-function */ +#define UDMA_USER_CTL_OP_FEAT_SHIFT 8 +#define UDMA_USER_CTL_OP_FEAT_MASK 0xff +#define UDMA_USER_CTL_OP_FUNC_MASK 0xff + +#define UDMA_USER_CTL_GET_OP_FEAT(opcode) \ + (((opcode) >> UDMA_USER_CTL_OP_FEAT_SHIFT) & UDMA_USER_CTL_OP_FEAT_MASK) +#define UDMA_USER_CTL_GET_OP_FUNC(opcode) ((opcode)&UDMA_USER_CTL_OP_FUNC_MASK) + +typedef int (*udma_user_ctl_op_func)(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); + +struct udma_user_ctl_op_entry { + u32 id; + udma_user_ctl_op_func func; +}; + +static inline udma_user_ctl_op_func +udma_user_ctl_get_op_entry_func(struct udma_user_ctl_op_entry *op_list, + u32 list_size, u32 id) +{ + u32 index; + for (index = 0; index < list_size; ++index) { + if (op_list[index].id == id) { + return op_list[index].func; + } + } + return NULL; +} + +#define UDMA_USER_CTL_GET_OP_ENTRY_FUNC(op_list, id) \ + ({ udma_user_ctl_get_op_entry_func(op_list, ARRAY_SIZE(op_list), id); }) + +int32_t udma_check_cmd_in_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size); +int32_t udma_check_cmd_out_param(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl, + uint64_t valid_size); + +#endif /* UDMA_USERCTL_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c new file mode 100644 index 000000000..a478bae88 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/build_bug.h> +#include "ubcore_types.h" +#include "udma_log.h" +#include "udma_userctl.h" +#include "udma_resource.h" +#include "udma_dfx_latch.h" +#include "udma_userctl_intf.h" +#include "udma_jetty.h" +#include "udma_cmd_jetty.h" +#include "udma_cmd_jfc.h" +#include "udma_cmd_tp.h" +#include "udma_cmd_tpg.h" +#include "udma_userctl_dfx.h" + +enum udma_user_ctl_opcode_dfx_func { + UDMA_USER_CTL_OP_DFX_PRINT_RC_TABLE, + UDMA_USER_CTL_OP_DFX_TA_STAUS, + UDMA_USER_CTL_OP_DFX_JETTY_DUMP, + UDMA_USER_CTL_OP_DFX_TP_DUMP +}; + +static int32_t udma_user_ctl_dfx_jetty_dump(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + struct ub_user_ctl_dfx_dump_in in_data = { 0 }; + struct ub_user_ctl_dfx_jetty_dump_out out_data = { 0 }; + struct ub_ta_context_query *jetty_entry = + (struct ub_ta_context_query *)&out_data; + struct ub_jfc_ctx_query *jfc_entry = + (struct ub_jfc_ctx_query *)&out_data; + uint32_t obj_type = 0; + int32_t ret = 0; + + BUILD_BUG_ON_MSG(sizeof(jetty_entry->ctx) != sizeof(out_data.ctx), + "The size of ctx needs to be consistent."); + + ret = udma_check_cmd_in_param(udev, user_ctl, + sizeof(struct ub_user_ctl_dfx_dump_in)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(struct ub_user_ctl_dfx_jetty_dump_out)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = (int32_t)copy_from_user(&in_data, + (void *)(uintptr_t)(user_ctl->in.addr), + sizeof(in_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from user failed to copy: %d\n", + ret); + return -EFAULT; + } + + obj_type = in_data.flag.obj_type; + if (obj_type == UB_USER_CTL_JFC) { + ret = udma_cmd_jfc_query_ctx(udev, in_data.xid, jfc_entry); + } else if (obj_type == UB_USER_CTL_JETTY || + obj_type == UB_USER_CTL_JFS || obj_type == UB_USER_CTL_JFR) { + ret = udma_cmd_jetty_query_ctx(udev, in_data.xid, NULL, + jetty_entry); + } else { + udma_err(udev->hwdev, UDMA_M_USER_CTL, + "unsupported obj type(%d).\n", obj_type); + return -EINVAL; + } + + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to query obj_type(%d) from cache! obj_id(%u)", + in_data.flag.obj_type, in_data.xid); + return ret; + } + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + return -EFAULT; + } + + return 0; +} + +static int32_t udma_user_ctl_dfx_tp_dump(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + struct ub_user_ctl_dfx_dump_in in_data = { 0 }; + struct ub_user_ctl_dfx_tp_dump_out out_data = { 0 }; + struct ub_dfx_tp_ctx_query *tp_entry = + (struct ub_dfx_tp_ctx_query *)out_data.tp_ctx; + struct ub_tpg_ctx_query *tpg_entry = + (struct ub_tpg_ctx_query *)out_data.tpg_ctx_req; + struct udma_tp *ub_tp = NULL; + struct udma_tpg *ub_tpg = NULL; + int32_t ret = 0; + struct ubcore_res_key key = { 0 }; + + if (udev->sysfs_res.dfx_ctx.dump_tp_en == 0) { + udma_warn(udev->dev, UDMA_M_USER_CTL, + "Dump TP is not enabled.\n"); + return 0; + } + + BUILD_BUG_ON_MSG(sizeof(tp_entry->ctx) != sizeof(out_data.tp_ctx), + "The size of ctx needs to be consistent."); + BUILD_BUG_ON_MSG(sizeof(tpg_entry->ctx_req) != + sizeof(out_data.tpg_ctx_req), + "The size of ctx needs to be consistent."); + + ret = udma_check_cmd_in_param(udev, user_ctl, + sizeof(struct ub_user_ctl_dfx_dump_in)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(struct ub_user_ctl_dfx_tp_dump_out)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = (int32_t)copy_from_user(&in_data, + (void *)(uintptr_t)(user_ctl->in.addr), + sizeof(in_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from user failed to copy: %d\n", + ret); + return -EFAULT; + } + + ub_tp = udma_find_tp(udev, in_data.xid); + if (ub_tp == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get tp, tp_id(%u)\n", in_data.xid); + return -EINVAL; + } + + key.type = UBCORE_RES_KEY_TP; + key.key = in_data.xid; + ret = udma_cmd_tp_query_ctx(udev, &key, tp_entry); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to query tp from cache! tp_id(%u)", + in_data.xid); + udma_tp_put(ub_tp); + return ret; + } + + ub_tpg = to_udma_tpg(ub_tp->cfg.tpg); + + ret = udma_cmd_tpg_query_ctx_from_cache( + udev, ub_tpg->tpg.tpgn, tpg_entry, + sizeof(struct ub_tpg_ctx_query)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to query tpg from cache! tpg_id(%u)", + ub_tpg->tpg.tpgn); + udma_tp_put(ub_tp); + return ret; + } + + udma_tp_put(ub_tp); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dfx_print_rc_table(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + udma_info(udev->dev, UDMA_M_USER_CTL, + "udma print dfx data:rc table.\n"); + udma_dump_ta_latch_info( + udev); /* item before only has 1825 supports exception cqe do dump,@see ub_1825v100_poll_jfc_one */ + + return 0; +} + +static struct udma_user_ctl_op_entry g_udma_user_ctl_dfx_funcs[] = { + { UDMA_USER_CTL_OP_DFX_PRINT_RC_TABLE, + udma_user_ctl_dfx_print_rc_table }, + { UDMA_USER_CTL_OP_DFX_JETTY_DUMP, udma_user_ctl_dfx_jetty_dump }, + { UDMA_USER_CTL_OP_DFX_TP_DUMP, udma_user_ctl_dfx_tp_dump }, +}; + +int udma_userctl_dfx_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + udma_user_ctl_op_func op_func = NULL; + int ret; + + op_func = UDMA_USER_CTL_GET_OP_ENTRY_FUNC( + g_udma_user_ctl_dfx_funcs, + UDMA_USER_CTL_GET_OP_FUNC(user_ctl->in.opcode)); + if (op_func == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get DFX callback, opcode(%u).\n", + user_ctl->in.opcode); + return -EINVAL; + } + + ret = op_func(udev, user_ctl); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_USER_CTL, + "Failed to execute the function of DFX, opcode(%u), ret(%d).\n", + user_ctl->in.opcode, ret); + return ret; + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h new file mode 100644 index 000000000..a21e104d1 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_dfx.h @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef UDMA_USERCTL_DFX_H +#define UDMA_USERCTL_DFX_H + +#include "ubcore_types.h" +#include "udma_common.h" + +int udma_userctl_dfx_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +int32_t udma_user_ctl_dfx_print_rc_table(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +#endif /* UDMA_USERCTL_DFX_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c new file mode 100644 index 000000000..488fe0463 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.c @@ -0,0 +1,364 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#include <linux/uaccess.h> +#include "ubcore_types.h" +#include "ub_npu_base_cmd.h" +#include "ub_npu_userctl_defs.h" +#include "udma_log.h" +#include "udma_userctl_intf.h" +#include "udma_userctl.h" +#include "udma_common.h" +#include "udma_cmd.h" +#include "udma_cqm.h" +#include "udma_vtp.h" +#include "udma_cmd_com.h" +#include "udma_userctl_uvs.h" + +void udma_fill_cmd_userctl_ctx(struct ub_cmd_userctl *cmd_buf, u8 cmd_type, + u8 cmd_subtype) +{ + u16 cmd_len = sizeof(*cmd_buf); + udma_fill_cmd_header(&cmd_buf->header, cmd_type, cmd_subtype, cmd_len, + 0); +} + +static int32_t udma_cqm_send(struct udma_device *dev, + struct tag_cqm_cmd_buf *in, + struct tag_cqm_cmd_buf *out, u8 cmd_subtype, + u64 func_id) +{ + int32_t ret = 0; + struct ub_cmd_userctl *cmd_buf = NULL; + cmd_buf = (struct ub_cmd_userctl *)in->buf; + cmd_buf->body.func_id = func_id; + + udma_fill_cmd_userctl_ctx(cmd_buf, UB_CMD_TYPE_USERCTL, cmd_subtype); + udma_cpu_to_be32(cmd_buf, sizeof(*cmd_buf)); + ret = udma_send_cmd_box(dev, cmd_subtype, in, out, NULL, NULL); + return ret; +} + +int32_t udma_user_ctl_uvs_trunk_pfc_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + ub_user_ctl_uvs_trunk_pfc_status_out_s out_data = { 0 }; + struct mag_cmd_port_stats stats = { 0 }; + int32_t ret = 0; + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(ub_user_ctl_uvs_trunk_pfc_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + return ret; + } + + ret = hinic5_get_phy_port_stats(udev->hwdev, &stats); + if (ret != 0) { + udma_err(udev->hwdev, UDMA_M_USER_CTL, + "Failed to get PFC stats from hinic5, ret: %d.\n", + ret); + return ret; + } + + out_data.rx_pfc_packets = stats.mac_rx_pfc_pkt_num; + (void)memcpy_s(&out_data.rx_pri_pfc_packets, sizeof(u64) * PFC_PRIORITY, + (void *)&stats.mac_rx_pfc_pri0_pkt_num, + sizeof(u64) * PFC_PRIORITY); + out_data.tx_pfc_packets = stats.mac_tx_pfc_pkt_num; + (void)memcpy_s(&out_data.tx_pri_pfc_packets, sizeof(u64) * PFC_PRIORITY, + (void *)&stats.mac_tx_pfc_pri0_pkt_num, + sizeof(u64) * PFC_PRIORITY); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + ret = -EFAULT; + } + + return ret; +} + +static int32_t udma_user_ctl_uvs_trunk_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + ub_user_ctl_uvs_card_status_out_s out_data = { 0 }; + int32_t ret = 0; + uint64_t func_id = 0; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + + ret = udma_check_cmd_out_param( + udev, user_ctl, sizeof(ub_user_ctl_uvs_card_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + goto err_out; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udev->hwdev, &cqm_cmd_inbuf, (u16)sizeof(struct ub_cmd_userctl), + &cqm_cmd_outbuf, sizeof(ub_user_ctl_uvs_card_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_out; + } + + ret = udma_cqm_send(udev, cqm_cmd_inbuf, cqm_cmd_outbuf, + UB_CMD_TRUNK_STATS, func_id); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_out; + } + udma_be32_to_cpu((void *)cqm_cmd_outbuf->buf, + UVS_TRUNK_STATUS_FISRT_FOUR_FIELDS); + udma_be64_to_cpu((void *)cqm_cmd_outbuf->buf + + UVS_TRUNK_STATUS_FISRT_FOUR_FIELDS, + UVS_TRUNK_STATUS_LAST_FOUR_FIELDS); + + (void)memcpy_s(&out_data, sizeof(ub_user_ctl_uvs_card_status_out_s), + (void *)cqm_cmd_outbuf->buf, + sizeof(ub_user_ctl_uvs_card_status_out_s)); + + udma_debug( + udev->hwdev, UDMA_M_USER_CTL, + "rx_err_packets(%u) rx_loss_packets(%u) rx_ce_packets(%u)" + "tx_port_err_packets(%u) tx_port_pkts(%llu) tx_port_bytes(%llu) rx_port_pkts(%llu) rx_port_bytes(%llu)\n", + out_data.rx_err_packets, out_data.rx_loss_packets, + out_data.rx_ce_packets, out_data.tx_port_err_packets, + out_data.tx_port_pkts, out_data.tx_port_bytes, + out_data.rx_port_pkts, out_data.rx_port_bytes); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + ret = -EFAULT; + goto err_out; + } + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; + +err_out: + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +static int32_t udma_user_ctl_uvs_vf_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + ub_user_ctl_uvs_vf_status_out_s out_data = { 0 }; + struct ub_user_ctl_uvs_vf_status_in in_data = { 0 }; + int32_t ret = 0; + struct tag_cqm_cmd_buf *cqm_cmd_outbuf = NULL; + struct tag_cqm_cmd_buf *cqm_cmd_inbuf = NULL; + ret = udma_check_cmd_out_param(udev, user_ctl, + sizeof(ub_user_ctl_uvs_vf_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + goto err_out; + } + ret = udma_check_cmd_in_param( + udev, user_ctl, sizeof(struct ub_user_ctl_uvs_vf_status_in)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Invalid parameter! ret: %d.\n", ret); + goto err_out; + } + + ret = udma_cqm_cmd_zalloc_inoutbuf( + udev->hwdev, &cqm_cmd_inbuf, (u16)sizeof(struct ub_cmd_userctl), + &cqm_cmd_outbuf, sizeof(ub_user_ctl_uvs_vf_status_out_s)); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to alloc cqm cmd inoutbuf, ret(%d)\n", ret); + goto err_out; + } + + ret = (int32_t)copy_from_user(&in_data, + (void *)(uintptr_t)(user_ctl->in.addr), + user_ctl->in.len); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from user failed to copy: %d\n", + ret); + ret = -EFAULT; + goto err_out; + } + ret = udma_cqm_send(udev, cqm_cmd_inbuf, cqm_cmd_outbuf, + UB_CMD_VF_STATS, in_data.func_id); + if (ret != 0) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "failed to send cqm cmd, ret(%d)\n", ret); + goto err_out; + } + udma_be64_to_cpu((void *)cqm_cmd_outbuf->buf, + sizeof(ub_user_ctl_uvs_vf_status_out_s)); + + (void)memcpy_s(&out_data, sizeof(ub_user_ctl_uvs_vf_status_out_s), + (void *)cqm_cmd_outbuf->buf, + sizeof(ub_user_ctl_uvs_vf_status_out_s)); + + udma_debug( + udev->hwdev, UDMA_M_USER_CTL, + "func_id(%d) rx_packets(%lld) rx_bytes(%lld) tx_packets(%lld) tx_bytes(%lld)" + "rx_err_pkt_cnt(%lld) rx_ce_pkt_cnt(%lld) rx_not_ready_cnt(%lld) tx_timeout_cnt(%lld) tx_no_vld_tp_cnt(%lld)\n", + in_data.func_id, out_data.rx_packets, out_data.rx_bytes, + out_data.tx_packets, out_data.tx_bytes, out_data.rx_err_pkt_cnt, + out_data.rx_ce_pkt_cnt, out_data.rx_not_ready_cnt, + out_data.tx_timeout_cnt, out_data.tx_no_vld_tp_cnt); + + ret = (int32_t)copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), + &out_data, sizeof(out_data)); + if (ret != 0) { + udma_err( + udev->hwdev, UDMA_M_USER_CTL, + "The number of data bytes from kernel failed to copy: %d\n", + ret); + ret = -EFAULT; + goto err_out; + } + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return 0; +err_out: + udma_cqm5_cmd_free_inoutbuf(udev->hwdev, cqm_cmd_inbuf, cqm_cmd_outbuf); + return ret; +} + +int udma_user_ctl_uvs_mig_status_change(struct udma_device *udma_dev, + struct ubcore_user_ctl *user_ctl) +{ + struct udma_vf_mig_status_out mig_rst = { 0 }; + struct udma_vf_mig_status_in mig_sts = { 0 }; + int ret; + + if (user_ctl->in.len != sizeof(struct udma_vf_mig_status_in) || + user_ctl->out.len != sizeof(struct udma_vf_mig_status_out)) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] len invalid, in:%u out:%u\n", + user_ctl->in.len, user_ctl->out.len); + return -EINVAL; + } + + if (copy_from_user(&mig_sts, (void *)(uintptr_t)(user_ctl->in.addr), + user_ctl->in.len) != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] copy from user failed\n"); + return -EINVAL; + } + udma_info(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] vf:%u sts:%u.\n", mig_sts.vfid, + mig_sts.sts); + + ret = udma_mig_status_change_proc(udma_dev, (void *)&mig_sts); + if (ret != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] failed, vf:%u sts:%u rst:%d.\n", + mig_sts.vfid, mig_sts.sts, ret); + } + mig_rst.result = ret; + if (copy_to_user((void *)(uintptr_t)(user_ctl->out.addr), &mig_rst, + user_ctl->out.len) != 0) { + udma_err( + udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig sts change] copy to user failed, vf:%u sts:%u.\n", + mig_sts.vfid, mig_sts.sts); + return -EINVAL; + } + return 0; +} + +int udma_user_ctl_uvs_mig_vf_status(struct udma_device *udma_dev, + struct ubcore_user_ctl *user_ctl) +{ + struct udma_mig_vf_status_in data_in = { 0 }; + struct udma_mig_vf_status_out data_out = { 0 }; + int ret; + + if (user_ctl->in.len != sizeof(struct udma_mig_vf_status_in) || + user_ctl->out.len != sizeof(struct udma_mig_vf_status_out)) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] len invalid, in:%u out:%u\n", + user_ctl->in.len, user_ctl->out.len); + return -EINVAL; + } + + if (copy_from_user(&data_in, (void *)user_ctl->in.addr, + user_ctl->in.len) != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] copy from user failed\n"); + return -EINVAL; + } + udma_info(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] vf:%u \n", data_in.vfid); + + ret = udma_query_mig_vf_status(udma_dev, data_in.vfid); + data_out.result = ret; + udma_info(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] vm status is %d \n", ret); + if (copy_to_user((void *)user_ctl->out.addr, &data_out, + user_ctl->out.len) != 0) { + udma_err(udma_dev->dev, UDMA_M_USER_CTL, + "[UVS][Mig vf status] copy to user failed\n"); + return -EINVAL; + } + return 0; +} + +enum udma_user_ctl_opcode_uvs_func { + UDMA_USER_CTL_OP_UVS_TRUNK_PFC_STATUS, + UDMA_USER_CTL_OP_UVS_TRUNK_STATUS, + UDMA_USER_CTL_OP_UVS_VF_STATUS, + UDMA_USER_CTL_OP_UVS_MIG_STATUS_CHANGE, + UDMA_USER_CTL_OP_UVS_MIG_VF_STATUS +}; + +static struct udma_user_ctl_op_entry g_udma_user_ctl_uvs_funcs[] = { + { UDMA_USER_CTL_OP_UVS_TRUNK_PFC_STATUS, + udma_user_ctl_uvs_trunk_pfc_status }, + { UDMA_USER_CTL_OP_UVS_TRUNK_STATUS, udma_user_ctl_uvs_trunk_status }, + { UDMA_USER_CTL_OP_UVS_VF_STATUS, udma_user_ctl_uvs_vf_status }, + { UDMA_USER_CTL_OP_UVS_MIG_STATUS_CHANGE, + udma_user_ctl_uvs_mig_status_change }, + { UDMA_USER_CTL_OP_UVS_MIG_VF_STATUS, udma_user_ctl_uvs_mig_vf_status }, +}; + +int udma_userctl_uvs_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl) +{ + udma_user_ctl_op_func op_func = NULL; + int ret; + + op_func = UDMA_USER_CTL_GET_OP_ENTRY_FUNC( + g_udma_user_ctl_uvs_funcs, + UDMA_USER_CTL_GET_OP_FUNC(user_ctl->in.opcode)); + if (op_func == NULL) { + udma_err(udev->dev, UDMA_M_USER_CTL, + "Failed to get UVS callback, opcode(%u).\n", + user_ctl->in.opcode); + return -EINVAL; + } + + ret = op_func(udev, user_ctl); + if (ret != 0) { + udma_err( + udev->dev, UDMA_M_USER_CTL, + "Failed to execute the function of UVS, opcode(%u), ret(%d).\n", + user_ctl->in.opcode, ret); + return ret; + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h new file mode 100644 index 000000000..b38a74433 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/userctl/udma_userctl_uvs.h @@ -0,0 +1,23 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +#ifndef UDMA_USERCTL_UVS_H +#define UDMA_USERCTL_UVS_H + +#include "ubcore_types.h" +#include "udma_common.h" +#include "ub_npu_base_cmd.h" +#include "ub_npu_userctl_defs.h" + +#define UVS_TRUNK_STATUS_FISRT_FOUR_FIELDS (4 * sizeof(u32)) +#define UVS_TRUNK_STATUS_LAST_FOUR_FIELDS (4 * sizeof(u64)) +typedef struct ub_cmd_userctl { + ub_cmd_com_header_s header; + ub_cmd_body_userctl_s body; +} ub_cmd_userctl_s; + +int udma_userctl_uvs_dispatch(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +int32_t udma_user_ctl_uvs_trunk_pfc_status(struct udma_device *udev, + struct ubcore_user_ctl *user_ctl); +#endif /* UDMA_USERCTL_UVS_H */ diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c new file mode 100644 index 000000000..a26053996 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.c @@ -0,0 +1,225 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/vmalloc.h> + +#include "udma_vram_base.h" +#include "udma_vram_mem.h" +#include "udma_vram_cfg.h" +#include "udma_vram_type_dev.h" +#include "udma_vram_type_tpg.h" +#include "udma_vram_type_tp.h" +#include "udma_vram_type_vtp.h" +#include "udma_vram.h" + +typedef int (*udma_vram_type_register)(void); + +static const udma_vram_type_register g_vram_type_reg_arr[] = { + udma_vram_device_register, udma_vram_tpg_register, + udma_vram_tp_register, udma_vram_vtp_register +}; + +int udma_get5_use_vram_flag(void) +{ + return udma_get_use_vram_base_flag(); +} + +int udma_vram5_get_kexec_flag(void) +{ + return udma_vram_get_kexec_base_flag(); +} + +void *udma_vram_kzalloc(void *dev, u32 type, size_t size, gfp_t flag) +{ + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "type(%d) is out of range(%d).\n", + type, UDMA_VRAM_TYPE_MAX); + return NULL; + } + + if (udma_get5_use_vram_flag() == 0) { + return type == UDMA_VRAM_TYPE_DEV ? + devm_kzalloc((struct device *)dev, size, flag) : + kzalloc(size, flag); + } + + return udma_vram_kzalloc_from_block(type, size); +} + +void udma_vram_kfree(void *dev, void *ptr, u32 type) +{ + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "type(%d) is out of range(%d).\n", + type, UDMA_VRAM_TYPE_MAX); + return; + } + + if (udma_get5_use_vram_flag() == 0) { + return type == UDMA_VRAM_TYPE_DEV ? + devm_kfree((struct device *)dev, ptr) : + kfree(ptr); + } + + return udma_vram_kfree_from_block(ptr, type); +} + +int udma_vram_mem_manager_recover(void) +{ + if (udma_get5_use_vram_flag() == 0) { + return 0; + } + + return udma_vram_block_recover(); +} + +static int udma_vram_type_register_all(void) +{ + int ret; + int i; + + for (i = 0; i < ARRAY_SIZE(g_vram_type_reg_arr); i++) { + ret = g_vram_type_reg_arr[i](); + if (ret < 0) { + udma_pr_err(UDMA_M_VRAM, + "vram type(id: %d) register failed.\n", i); + return ret; + } + } + + return 0; +} + +static int udma_vram_block_node(u32 type, struct udma_vram_block *block) +{ + udma_vram_recover_func callback = NULL; + u32 max_num = block->node_num; + void *node = NULL; + u32 index; + int ret; + + for (index = 0; index < max_num; index++) { + if (test_bit(index, (const unsigned long *)block->bitmap) == + 1) { + node = (void *)((u8 *)block->buf + + index * block->node_size); + callback = udma_vram_cfg_get_recover(type); + ret = callback(node); + if (ret < 0) { + udma_pr_err( + UDMA_M_VRAM, + "vram block node(index: %u) recover failed.\n", + index); + return ret; + } + } + } + + return 0; +} + +static void udma_store_block_addr(struct udma_vram_block_header *header) +{ + header->block->last_block_addr = (uintptr_t)(header->block); +} + +static int udma_vram_mem_block_node_recover(void) +{ + struct udma_vram_header *vram_header = NULL; + u32 i; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "vram header get failed.\n"); + return -EINVAL; + } + + udma_pr_err( + UDMA_M_VRAM, + "vram header recover success. vram_header->vram_name is %s\n", + vram_header->vram_name); + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + + if (udma_vram_block_node(i, vram_header->header[i].block) != + 0) { + udma_pr_err( + UDMA_M_VRAM, + "vram block node(vram type: %d) recover failed.\n", + i); + return -EINVAL; + } + udma_store_block_addr(&vram_header->header[i]); + } + + return 0; +} + +int udma_vram_init(void) +{ + int ret; + + if (udma_get5_use_vram_flag() == 0) { + udma_pr_warn(UDMA_M_VRAM, + "%s: This system does not support VRAM!\n", + __func__); + return 0; + } + + ret = udma_vram_type_register_all(); + if (ret < 0) { + udma_pr_err(UDMA_M_VRAM, "%s: vram type register failed.\n", + __func__); + return ret; + } + + return udma_vram_mem_init(); +} + +void udma_vram_uninit(void) +{ + if (udma_get5_use_vram_flag() == 0) { + udma_pr_warn(UDMA_M_VRAM, + "%s: This system does not support VRAM!\n", + __func__); + return; + } + + udma_vram_mem_deinit(); +} + +int udma_vram_mem_manager_init(void) +{ + int ret; + + udma_vram_base_init(); + if (udma_vram5_get_kexec_flag() == 0) { + return udma_vram_init(); + } + + ret = udma_vram_type_register_all(); + if (ret < 0) { + udma_pr_err(UDMA_M_VRAM, "%s: vram type register failed.\n", + __func__); + return ret; + } + + return udma_vram_mem_manager_recover(); +} + +void udma_vram_mem_manager_uninit(void) +{ + udma_vram_uninit(); + udma_vram_base_uninit(); +} + +int udma_vram_mem_obj_recover(void) +{ + if (udma_vram5_get_kexec_flag() == 1) { + return udma_vram_mem_block_node_recover(); + } + + return 0; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h new file mode 100644 index 000000000..a6c32d820 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram.h @@ -0,0 +1,95 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_H +#define UDMA_VRAM_H + +#include <linux/types.h> + +#include "udma_common.h" + +enum { + UDMA_VRAM_TYPE_DEV = 0, + UDMA_VRAM_TYPE_TPG, + UDMA_VRAM_TYPE_TP, + UDMA_VRAM_TYPE_VTP, + UDMA_VRAM_TYPE_MAX, +}; + +/** +* get kexec flag. kexec flag set means OS is in hot replace state. +* @param[in] [Required] void +* Return: 1 means kexec flag is set, 0 means not. +*/ +int udma_vram5_get_kexec_flag(void); + +/** +* get vram use flag. vram use flag set means OS support vram management. +* @param[in] [Required] void +* Return: 1 means vram use flag is set, 0 means not. +*/ +int udma_get5_use_vram_flag(void); + +/** +* vram memroy allocate interface. +* @param[in] [Required] dev: dev related. +* @param[in] [Required] type: vram type, see UDMA_VRAM_TYPE. +* @param[in] [Required] size: size to allocate vram memory. +* @param[in] [Required] flag: Kernel get free pages flag. +* Return: void*. +*/ +void *udma_vram_kzalloc(void *dev, u32 type, size_t size, gfp_t flag); + +/** +* vram memroy free interface. +* @param[in] [Required] dev: dev related. +* @param[in] [Required] ptr: vram memory to free. +* @param[in] [Required] type: vram type, see UDMA_VRAM_TYPE. +* Return: void. +*/ +void udma_vram_kfree(void *dev, void *ptr, u32 type); + +/** +* vram memroy manager recover, include vram header and block recover. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_mem_manager_recover(void); + +/** +* vram memroy obj recover, include block node recover. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_mem_obj_recover(void); + +/** +* vram memroy unint, include vram header and block uninit. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +void udma_vram_uninit(void); + +/** +* vram memroy int, include vram header and block init. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_init(void); + +/** +* vram memroy manager int. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +int udma_vram_mem_manager_init(void); + +/** +* vram memroy manager unint. +* @param[in] [Required] void. +* Return: 0 on success, other value on error. +*/ +void udma_vram_mem_manager_uninit(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c new file mode 100644 index 000000000..8f969de98 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.c @@ -0,0 +1,104 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/kallsyms.h> + +#include "ossl_knl.h" + +#include "udma_vram_base.h" + +static vram_kfree_t _udma_vram_kfree = NULL; +static vram_kalloc_t _udma_vram_kalloc = NULL; +static get5_use_vram_flag_t _udma_get5_use_vram_flag = NULL; +static vram5_get_kexec_flag_t _udma_vram5_get_kexec_flag = NULL; + +void *udma_vram_base_kalloc(char *name, u64 size) +{ + if (_udma_vram_kalloc != NULL && name != NULL) { + return _udma_vram_kalloc(name, size); + } + + return NULL; +} + +void udma_vram_base_kfree(void *vaddr, char *name, u64 size) +{ + if (_udma_vram_kfree != NULL && vaddr != NULL && name != NULL) { + _udma_vram_kfree(vaddr, name, size); + } +} + +int udma_get_use_vram_base_flag(void) +{ + if (_udma_get5_use_vram_flag != NULL) { + return _udma_get5_use_vram_flag(); + } + + return 0; +} + +int udma_vram_get_kexec_base_flag(void) +{ + if (_udma_vram5_get_kexec_flag != NULL) { + return _udma_vram5_get_kexec_flag(); + } + + return 0; +} + +void udma_vram_base_init(void) +{ + _udma_vram_kalloc = + (vram_kalloc_t)(uintptr_t)symbol_get(hi5_vram_kalloc); + if (_udma_vram_kalloc == NULL) { + udma_pr_info(UDMA_M_VRAM, "_udma_vram_kalloc is NULL.\n"); + return; + } + + _udma_vram_kfree = (vram_kfree_t)(uintptr_t)symbol_get(hi5_vram_kfree); + if (_udma_vram_kfree == NULL) { + udma_pr_info(UDMA_M_VRAM, "_udma_vram_kfree is NULL.\n"); + return; + } + + _udma_get5_use_vram_flag = + (get5_use_vram_flag_t)(uintptr_t)symbol_get(get5_use_vram_flag); + if (_udma_get5_use_vram_flag == NULL) { + udma_pr_info(UDMA_M_VRAM, + "_udma_get5_use_vram_flag is NULL.\n"); + return; + } + + _udma_vram5_get_kexec_flag = + (vram5_get_kexec_flag_t)(uintptr_t)symbol_get( + vram5_get_kexec_flag); + if (_udma_vram5_get_kexec_flag == NULL) { + udma_pr_info(UDMA_M_VRAM, + "_udma_vram5_get_kexec_flag is NULL.\n"); + return; + } +} + +void udma_vram_base_uninit(void) +{ + if (_udma_vram5_get_kexec_flag != NULL) { + symbol_put(vram5_get_kexec_flag); + _udma_vram5_get_kexec_flag = NULL; + } + + if (_udma_get5_use_vram_flag != NULL) { + symbol_put(get5_use_vram_flag); + _udma_get5_use_vram_flag = NULL; + } + + if (_udma_vram_kfree != NULL) { + symbol_put(hi5_vram_kfree); + _udma_vram_kfree = NULL; + } + + if (_udma_vram_kalloc != NULL) { + symbol_put(hi5_vram_kalloc); + _udma_vram_kalloc = NULL; + } +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h new file mode 100644 index 000000000..dae0651bd --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_base.h @@ -0,0 +1,22 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_BASE_H +#define UDMA_VRAM_BASE_H + +#include "udma_common.h" + +typedef void __iomem *(*vram_kalloc_t)(char *name, u64 size); +typedef void (*vram_kfree_t)(void __iomem *vaddr, char *name, u64 size); +typedef int (*get5_use_vram_flag_t)(void); +typedef int (*vram5_get_kexec_flag_t)(void); + +void *udma_vram_base_kalloc(char *name, u64 size); +void udma_vram_base_kfree(void *vaddr, char *name, u64 size); +int udma_get_use_vram_base_flag(void); +int udma_vram_get_kexec_base_flag(void); +void udma_vram_base_init(void); +void udma_vram_base_uninit(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c new file mode 100644 index 000000000..a8d1441ec --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.c @@ -0,0 +1,108 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_cfg.h" + +struct udma_vram_cfg_table { + char name[UDMA_VRAM_NAME_SIZE]; + bool enable; /* support ECS or storage service */ + u32 node_num; + u32 node_size; + udma_vram_recover_func recover; +}; + +static struct udma_vram_cfg_table g_cfg_table[UDMA_VRAM_TYPE_MAX] = { 0 }; + +int udma_vram_cfg_register(const char *name, u32 type, u32 node_num, + u32 node_size, udma_vram_recover_func callback) +{ + int ret; + + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "type(%d) is out of range(%d).\n", + type, UDMA_VRAM_TYPE_MAX); + return -ERANGE; + } + + g_cfg_table[type].enable = true; + g_cfg_table[type].node_num = node_num; + g_cfg_table[type].node_size = node_size; + g_cfg_table[type].recover = callback; + ret = strncpy_s(g_cfg_table[type].name, sizeof(g_cfg_table[type].name), + (const char *)name, sizeof(g_cfg_table[type].name) - 1); + if (ret != 0) { + udma_pr_err(UDMA_M_VRAM, + "%s: type(%d) name strncpy_s failed, ret:%d.\n", + __func__, type, ret); + return -EIO; + } + + return 0; +} + +bool udma_vram_cfg_enable(u32 type) +{ + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "%s: type(%d) is out of range(%d).\n", + __func__, type, UDMA_VRAM_TYPE_MAX); + return 0; + } + + return g_cfg_table[type].enable; +} + +u32 udma_vram_cfg_get_num(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err(UDMA_M_VRAM, "%s: type(%d) is out of range(%d).\n", + __func__, type, UDMA_VRAM_TYPE_MAX); + return 0; + } + + return g_cfg_table[type].node_num; +} + +u32 udma_vram_cfg_get_size(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err( + UDMA_M_VRAM, + "%s: type(%d) is out of range(%d), enable flag is %d.\n", + __func__, type, UDMA_VRAM_TYPE_MAX, + udma_vram_cfg_enable(type)); + return 0; + } + + return g_cfg_table[type].node_size; +} + +char *udma_vram_cfg_get_name(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err( + UDMA_M_VRAM, + "%s: type(%d) is out of range(%d), enable flag is %d.\n", + __func__, type, UDMA_VRAM_TYPE_MAX, + udma_vram_cfg_enable(type)); + return NULL; + } + + return g_cfg_table[type].name; +} + +udma_vram_recover_func udma_vram_cfg_get_recover(u32 type) +{ + if ((type >= UDMA_VRAM_TYPE_MAX) || !udma_vram_cfg_enable(type)) { + udma_pr_err( + UDMA_M_VRAM, + "%s: type(%d) is out of range(%d), enable flag is %d.\n", + __func__, type, UDMA_VRAM_TYPE_MAX, + udma_vram_cfg_enable(type)); + return NULL; + } + + return g_cfg_table[type].recover; +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h new file mode 100644 index 000000000..b3c4b25db --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_cfg.h @@ -0,0 +1,20 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_CFG_H +#define UDMA_VRAM_CFG_H + +#include "udma_vram.h" + +typedef int (*udma_vram_recover_func)(void *node); + +int udma_vram_cfg_register(const char *name, u32 type, u32 node_num, + u32 node_size, udma_vram_recover_func callback); +bool udma_vram_cfg_enable(u32 type); +u32 udma_vram_cfg_get_num(u32 type); +u32 udma_vram_cfg_get_size(u32 type); +char *udma_vram_cfg_get_name(u32 type); +udma_vram_recover_func udma_vram_cfg_get_recover(u32 type); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h new file mode 100644 index 000000000..4f2a1faa8 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_common.h @@ -0,0 +1,25 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_COMMON_H +#define UDMA_VRAM_COMMON_H + +#define UDMA_VRAM_HEADER_NAME "UB_HEADER" +#define UDMA_DEV_NAME "UB_DEVICE" +#define UDMA_TP_NAME "UB_TP" +#define UDMA_TPG_NAME "UB_TPG" +#define UDMA_VTP_NAME "UB_VTP" +#define UDMA_TP_MTT_NAME "UB_TP_MTT" + +#define UDMA_VRAM_NAME_SIZE 15 + +#define UDMA_DEVICE_NODE 2 +#define UDMA_TP_NODE 1024 +#define UDMA_TPG_NODE 1024 +#define UDMA_VTP_NODE 1024 + +#define UDMA_NUM_BIT_DW 64 +#define UDMA_INVALID_ID 0xFFFFFFFF + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c new file mode 100644 index 000000000..c66d92d4c --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.c @@ -0,0 +1,412 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include <linux/vmalloc.h> +#include <linux/spinlock.h> + +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_base.h" +#include "udma_vram_common.h" +#include "udma_vram_mem.h" + +struct udma_vram_header *udma_get_vram_header(void) +{ + struct udma_vram_header *header = NULL; + + header = (struct udma_vram_header *)udma_vram_base_kalloc( + (char *)UDMA_VRAM_HEADER_NAME, sizeof(struct udma_vram_header)); + if (header == NULL) { + udma_pr_err(UDMA_M_VRAM, "header alloc failed.\n"); + return NULL; + } + + return header; +} + +static void udma_vram_bitmap_free_id(struct udma_vram_block *block, u32 id) +{ + spin_lock(&block->bitmap_lock); + clear_bit(id, (unsigned long *)block->bitmap); + block->used_node_cnt--; + spin_unlock(&block->bitmap_lock); +} + +static u32 udma_vram_get_bitmap_id(const void *ptr, + const struct udma_vram_block *block) +{ + u32 index; + + for (index = 0; index < block->node_num; index++) { + if (ptr == + (void *)((u8 *)block->buf + index * block->node_size)) { + return index; + } + } + + return 0; +} + +void udma_vram_kfree_from_block(void *ptr, u32 type) +{ + struct udma_vram_header *vram_header = NULL; + struct udma_vram_block *block = NULL; + u32 index; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "vram header get failed.\n"); + return; + } + + block = vram_header->header[type].block; + index = udma_vram_get_bitmap_id((const void *)ptr, + (const struct udma_vram_block *)block); + udma_vram_bitmap_free_id(block, index); +} + +static u32 udma_vram_bitmap_alloc_id(struct udma_vram_block *block) +{ + unsigned long *table = (unsigned long *)block->bitmap; + u32 offset = block->bitmap_item_last; + u32 max_num = block->node_num; + u32 index; + + spin_lock(&block->bitmap_lock); + + if (block->used_node_cnt >= max_num) { + udma_pr_err(UDMA_M_VRAM, + "used node cnt(%u) execeed max node num(%u).\n", + block->used_node_cnt, block->node_num); + spin_unlock(&block->bitmap_lock); + return UDMA_INVALID_ID; + } + + /* Search for an idle bit from the offset position. */ + index = (u32)find_next_zero_bit(table, max_num, offset); + /* The preceding search fails. Search for an idle bit + * from the beginning. + */ + if (index >= max_num) { + offset = 0; + index = (u32)find_next_zero_bit(table, max_num, offset); + } + + /* Set the found bit to 1 and reset offset. */ + if (index < max_num) { + set_bit(index, table); + block->bitmap_item_last = index; + } + + block->used_node_cnt++; + spin_unlock(&block->bitmap_lock); + return index; +} + +static void *udma_vram_get_node(struct udma_vram_block *block) +{ + u32 idx; + + idx = udma_vram_bitmap_alloc_id(block); + if (idx == UDMA_INVALID_ID) { + udma_pr_err(UDMA_M_VRAM, "vram bitmap alloc invalid idx.\n"); + return NULL; + } + + return (void *)((u8 *)block->buf + block->node_size * idx); +} + +void *udma_vram_kzalloc_from_block(u32 type, size_t size) +{ + struct udma_vram_header *vram_header = NULL; + struct udma_vram_block *block = NULL; + void *addr = NULL; + int ret; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return NULL; + } + + block = vram_header->header[type].block; + addr = udma_vram_get_node(block); + if (addr == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: udma vram get node failed.\n", + __func__); + return NULL; + } + + ret = memset_s(addr, size, 0, size); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: vram object memset_s failed.\n", + __func__); + return NULL; + } + + return addr; +} + +static int udma_vram_block_header_init(u32 type, + struct udma_vram_block_header *header) +{ + int ret; + + header->node_num = udma_vram_cfg_get_num(type); + header->node_size = udma_vram_cfg_get_size(type); + ret = strncpy_s(header->name, sizeof(header->name), + udma_vram_cfg_get_name(type), sizeof(header->name) - 1); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: header name copy failed.\n", + __func__); + } + udma_pr_warn(UDMA_M_VRAM, + "%s: header_init success. type: %u, header->name is %s\n", + __func__, type, header->name); + + return ret; +} + +static int udma_vram_block_obj_init(struct udma_vram_block_header *header) +{ + u32 vram_block_head_size; + u32 bitmap_size; + int ret; + + vram_block_head_size = sizeof(struct udma_vram_block); + bitmap_size = ALIGN(header->node_num, UDMA_NUM_BIT_DW) >> + UDMA_BYTE_BIT_SHIFT; + header->block_size = vram_block_head_size + bitmap_size + + header->node_num * header->node_size; + + header->block = (struct udma_vram_block *)udma_vram_base_kalloc( + (char *)header->name, header->block_size); + if (header->block == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block alloc failed.\n", + __func__); + return -EINVAL; + } + + ret = memset_s(header->block, header->block_size, 0, + header->block_size); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block memset_s failed.\n", + __func__); + udma_vram_base_kfree(header->block, (char *)header->name, + header->block_size); + header->block = NULL; + return -EINVAL; + } + + header->block->last_block_addr = (uintptr_t)header->block; + header->block->node_num = header->node_num; + header->block->node_size = header->node_size; + header->block->buf = (void *)((u8 *)header->block + + vram_block_head_size + bitmap_size); + header->block->used_node_cnt = 0; + spin_lock_init(&header->block->bitmap_lock); + + udma_pr_warn( + UDMA_M_VRAM, + "%s: block_init success. header->block_size is %llu, header->block->node_num:%u\n", + __func__, header->block_size, header->block->node_num); + + return 0; +} + +static int udma_vram_block_init(struct udma_vram_header *vram_header) +{ + u32 i; + int ret; + + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + ret = udma_vram_block_header_init(i, &vram_header->header[i]); + if (ret != EOK) { + udma_pr_err( + UDMA_M_VRAM, + "%s: vram header block header init failed, type:%d\n", + __func__, i); + return ret; + } + + ret = udma_vram_block_obj_init(&vram_header->header[i]); + if (ret < 0) { + udma_pr_err( + UDMA_M_VRAM, + "%s: vram block obj init failed, type:%d.\n", + __func__, i); + return ret; + } + } + + return 0; +} + +int udma_vram_mem_init(void) +{ + struct udma_vram_header *vram_header = NULL; + int ret; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return -EINVAL; + } + + ret = strncpy_s((char *)vram_header->vram_name, + sizeof(vram_header->vram_name), UDMA_VRAM_HEADER_NAME, + sizeof(vram_header->vram_name) - 1); + if (ret != EOK) { + udma_pr_err(UDMA_M_VRAM, "%s: vram_header name copy failed.\n", + __func__); + udma_vram_base_kfree(vram_header, (char *)UDMA_VRAM_HEADER_NAME, + sizeof(struct udma_vram_header)); + return -EINVAL; + } + udma_pr_warn( + UDMA_M_VRAM, + "%s: vram header malloc success. vram_header->vram_name is %s\n", + __func__, vram_header->vram_name); + + return udma_vram_block_init(vram_header); +} + +static void udma_vram_old_addr_to_new(void **addr, + struct udma_vram_block *block) +{ + uintptr_t old_addr = (uintptr_t)(*addr); + + *addr = (void *)((u8 *)(block) + old_addr - block->last_block_addr); +} + +void udma_vram_update_addr(void **addr, u32 type) +{ + struct udma_vram_header *vram_header = NULL; + + if (type >= UDMA_VRAM_TYPE_MAX) { + udma_pr_err(UDMA_M_VRAM, "%s: type(%d) is out of range(%d).\n", + __func__, type, UDMA_VRAM_TYPE_MAX); + return; + } + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return; + } + + udma_vram_old_addr_to_new(addr, vram_header->header[type].block); +} + +static struct udma_vram_block * +udma_vram_recover_block(struct udma_vram_block_header *header) +{ + struct udma_vram_block *block = NULL; + + block = (struct udma_vram_block *)udma_vram_base_kalloc( + (char *)header->name, header->block_size); + if (block == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block alloc failed.\n", + __func__); + return NULL; + } + + return block; +} +static int udma_vram_recover_block_addr(struct udma_vram_block_header *header) +{ + header->block = udma_vram_recover_block(header); + if (header->block == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block recover failed.\n", + __func__); + return -EINVAL; + } + udma_pr_err( + UDMA_M_VRAM, + "%s: block recover success. header->block->node_size is %u\n", + __func__, header->block->node_size); + udma_vram_old_addr_to_new(&header->block->buf, header->block); + udma_vram_old_addr_to_new((void **)&header->block->bitmap, + header->block); + + return 0; +} + +static int udma_vram_block_recover_cfg(struct udma_vram_header *header) +{ + u32 i; + + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + udma_pr_err( + UDMA_M_VRAM, + "%s: vram header recover success. type:%u, header_name:%s\n", + __func__, i, header->header[i].name); + if (udma_vram_recover_block_addr(&header->header[i]) != 0) { + udma_pr_err( + UDMA_M_VRAM, + "%s: vram block addr recover failed, type index:%d.\n", + __func__, i); + return -EINVAL; + } + } + + return 0; +} + +int udma_vram_block_recover(void) +{ + struct udma_vram_header *vram_header = NULL; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: vram header get failed.\n", + __func__); + return -EINVAL; + } + + udma_pr_err( + UDMA_M_VRAM, + "%s: vram header recover success. vram_header->vram_name:%s,\n", + __func__, vram_header->vram_name); + if (udma_vram_block_recover_cfg(vram_header) != 0) { + udma_pr_err(UDMA_M_VRAM, "%s: vram block recover cfg failed.\n", + __func__); + return -EINVAL; + } + + return 0; +} + +void udma_vram_mem_deinit(void) +{ + struct udma_vram_block_header *header = NULL; + struct udma_vram_header *vram_header = NULL; + u32 i; + + vram_header = udma_get_vram_header(); + if (vram_header == NULL) { + return; + } + + for (i = 0; i < UDMA_VRAM_TYPE_MAX; i++) { + if (!udma_vram_cfg_enable(i)) { + continue; + } + header = &vram_header->header[i]; + udma_vram_base_kfree((void *)header->block, header->name, + header->block_size); + } + + udma_vram_base_kfree((void *)vram_header, (char *)UDMA_VRAM_HEADER_NAME, + sizeof(struct udma_vram_header)); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h new file mode 100644 index 000000000..b6414e02d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_mem.h @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_BLOCK_H +#define UDMA_VRAM_BLOCK_H + +#include "udma_vram.h" +#include "udma_common.h" +#include "udma_vram_common.h" + +struct udma_vram_block { + u32 node_num; /* block in contain maximum max node count */ + u32 node_size; /* node of size */ + u32 used_node_cnt; /* already allocate node count */ + u32 bitmap_item_last; + spinlock_t bitmap_lock; + uintptr_t last_block_addr; /* record on once allocate of block address */ + void *buf; /* node first address */ + u64 bitmap[0]; /* node allocate used bitmap */ +}; + +struct udma_vram_block_header { + struct udma_vram_block *block; + char name[UDMA_VRAM_NAME_SIZE]; /* block name, use to vram allocate memory */ + u32 node_num; /* block in contain maximum max node count */ + u32 node_size; /* node of size */ + u64 block_size; /* block of size */ +}; + +struct udma_vram_header { + const char vram_name[UDMA_VRAM_NAME_SIZE]; + struct udma_vram_block_header header[UDMA_VRAM_TYPE_MAX]; +}; + +void *udma_vram_kzalloc_from_block(u32 type, size_t size); +void udma_vram_kfree_from_block(void *ptr, u32 type); +int udma_vram_mem_init(void); +int udma_vram_block_recover(void); +void udma_vram_mem_deinit(void); +struct udma_vram_header *udma_get_vram_header(void); +void udma_vram_update_addr(void **addr, u32 type); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c new file mode 100644 index 000000000..170eb1efa --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.c @@ -0,0 +1,37 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_dev.h" + +static int udma_vram_recover_device(void *node) +{ + struct udma_device *udma_dev = (struct udma_device *)node; + + (void)udma_dev; + + /* legacy restore */ + //rc_queue; + + udma_pr_err( + UDMA_M_VRAM, + "%s: udma_dev recover success, udma_dev->dev_name is %s .\n", + __func__, udma_dev->dev_name); + return 0; +} + +int udma_vram_device_register(void) +{ + return udma_vram_cfg_register(UDMA_DEV_NAME, UDMA_VRAM_TYPE_DEV, + UDMA_DEVICE_NODE, + sizeof(struct udma_device), + udma_vram_recover_device); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h new file mode 100644 index 000000000..d442b28ed --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_dev.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_DEV_H +#define UDMA_VRAM_TYPE_DEV_H + +int udma_vram_device_register(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c new file mode 100644 index 000000000..8323034fe --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.c @@ -0,0 +1,89 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_tp.h" +#include "udma_tpg.h" +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_tp.h" + +static struct tag_cqm_qpc_mpt *udma_get_tp_info(struct udma_tpg *ub_tpg, + uint32_t tpn) +{ + int i; + + for (i = 0; i < MAX_TP_CNT_IN_TPG; i++) { + if (tpn == ub_tpg->tpinfo_list[i]->xid) { + return ub_tpg->tpinfo_list[i]; + } + } + + return NULL; +} + +static int udma_vram_recover_tp(void *node) +{ + struct udma_tp *ub_tp = (struct udma_tp *)node; + struct udma_device *udma_dev = NULL; + struct udma_tpg *ub_tpg = NULL; + +#ifdef UBCORE_EN + int ret; + + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_TP, &ub_tp->tp); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "%s: ubcore recover failed failed. type:%d\n", + __func__, UDMA_VRAM_TYPE_TP); + return -EINVAL; + } +#endif + udma_vram_update_addr((void **)&ub_tp->tp.ub_dev, UDMA_VRAM_TYPE_DEV); + udma_vram_update_addr((void **)&ub_tp->tp.tpg, UDMA_VRAM_TYPE_TPG); + ub_tpg = to_udma_tpg(ub_tp->tp.tpg); + ub_tp->tp_info = udma_get_tp_info(ub_tpg, ub_tp->tp.tpn); + + udma_dev = to_udma_dev(ub_tp->tp.ub_dev); + ub_tp->tp_buf_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_QP, + (u32)ub_tp->buf_size, ub_tp, true, ub_tp->tp_buf_info_xid, 0, + 0); + if (ub_tp->tp_buf_info == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: tp buf info recover failed.\n", + __func__); + return -EINVAL; + } + + ub_tp->pcqc_info = cqm5_object_rdma_queue_create( + udma_dev->hwdev, SERVICE_T_UB, CQM_OBJECT_RDMA_SCQ, + TP_PCQC_SIZE, NULL, false, ub_tp->tp_info->xid, 0, 0); + if (ub_tp->pcqc_info == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: tp pcqc info recover failed.\n", + __func__); + cqm5_object_delete(&ub_tp->tp_buf_info->object); + return -EINVAL; + } + + ub_tp->buf.direct.buf = ub_tp->tp_buf_info->q_room_buf_1.direct.va; + + //to do: ub_mtt, rc_table + + udma_store_list(udma_dev, &ub_tp->node, + &udma_dev->device_res->tpn_list); + + return 0; +} + +int udma_vram_tp_register(void) +{ + return udma_vram_cfg_register(UDMA_TP_NAME, UDMA_VRAM_TYPE_TP, + UDMA_TP_NODE, sizeof(struct udma_tp), + udma_vram_recover_tp); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h new file mode 100644 index 000000000..de1e214c7 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tp.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_TP_H +#define UDMA_VRAM_TYPE_TP_H + +int udma_vram_tp_register(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c new file mode 100644 index 000000000..3eccb9197 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.c @@ -0,0 +1,69 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_tpg.h" +#include "udma_tp.h" +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_tpg.h" + +static int udma_vram_recover_tpg(void *node) +{ + struct udma_tpg *ub_tpg = (struct udma_tpg *)node; + struct udma_device *udma_dev = NULL; + u32 i; +#ifdef UBCORE_EN + int ret; + + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_TPG, &ub_tpg->tpg); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "%s: ubcore recover failed failed. type:%d\n", + __func__, UDMA_VRAM_TYPE_TPG); + return -EINVAL; + } +#endif + + udma_vram_update_addr((void **)&ub_tpg->tpg.ub_dev, UDMA_VRAM_TYPE_DEV); + + udma_dev = to_udma_dev(ub_tpg->tpg.ub_dev); + ub_tpg->tpg_info = udma_object_queue_create( + udma_dev, CQM_OBJECT_RDMA_SCQ, UBEP_TPG_CONTEXT_SIZE, ub_tpg, + NULL, false, ub_tpg->tpg.tpgn, ub_tpg->tpg.tpgn, 0); + if (ub_tpg->tpg_info == NULL) { + udma_pr_err(UDMA_M_VRAM, "%s: tpg info recover failed.\n", + __func__); + return -EINVAL; + } + + for (i = 0; i < ub_tpg->tp_num; i++) { + ub_tpg->tpinfo_list[i] = udma_object_qpc_create( + udma_dev, UBEP_TP_CONTEXT_SIZE, NULL, + &udma_dev->tpn_record, ub_tpg->tpg.tpgn, + udma_dev->ub_cap.dev_cap.max_tp, 0); + if (ub_tpg->tpinfo_list[i] == NULL) { + udma_pr_err(UDMA_M_VRAM, + "%s: tpg tpinfo list recover failed.\n", + __func__); + return -EINVAL; + } + } + + udma_store_list(udma_dev, &ub_tpg->node, + &udma_dev->device_res->tpg_list); + return 0; +} + +int udma_vram_tpg_register(void) +{ + return udma_vram_cfg_register(UDMA_TPG_NAME, UDMA_VRAM_TYPE_TPG, + UDMA_TPG_NODE, sizeof(struct udma_tpg), + udma_vram_recover_tpg); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h new file mode 100644 index 000000000..d7493200d --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_tpg.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_TPG_H +#define UDMA_VRAM_TYPE_TPG_H + +int udma_vram_tpg_register(void); + +#endif diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c new file mode 100644 index 000000000..49bae0f62 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.c @@ -0,0 +1,45 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#include "ubcore_api.h" + +#include "udma_vtp.h" +#include "udma_common.h" +#include "udma_misc.h" +#include "udma_vram.h" +#include "udma_vram_cfg.h" +#include "udma_vram_mem.h" +#include "udma_vram_common.h" +#include "udma_vram_type_vtp.h" + +static int udma_vram_recover_vtp(void *node) +{ + struct udma_vtp *ub_vtp = (struct udma_vtp *)node; + struct udma_device *udma_dev = NULL; +#ifdef UBCORE_EN + int ret; + + ret = ubcore_recover_from_udma(UDMA_VRAM_TYPE_VTP, &ub_vtp->base); + if (ret) { + udma_pr_err(UDMA_M_VRAM, + "%s: ubcore recover failed failed. type:%d\n", + __func__, UDMA_VRAM_TYPE_VTP); + return -EINVAL; + } +#endif + udma_vram_update_addr((void **)&ub_vtp->base.ub_dev, + UDMA_VRAM_TYPE_VTP); + udma_dev = to_udma_dev(ub_vtp->base.ub_dev); + udma_store_list(udma_dev, &ub_vtp->node, + &udma_dev->device_res->vtp_list); + + return 0; +} + +int udma_vram_vtp_register(void) +{ + return udma_vram_cfg_register(UDMA_VTP_NAME, UDMA_VRAM_TYPE_VTP, + UDMA_VTP_NODE, sizeof(struct udma_vtp), + udma_vram_recover_vtp); +} diff --git a/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h new file mode 100644 index 000000000..47c672978 --- /dev/null +++ b/drivers/ub/urma/hw/hiudma5/kernel/module/vram/udma_vram_type_vtp.h @@ -0,0 +1,10 @@ +//SPDX-License-Identifier: GPL-2.0 +//Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + + +#ifndef UDMA_VRAM_TYPE_VTP_H +#define UDMA_VRAM_TYPE_VTP_H + +int udma_vram_vtp_register(void); + +#endif -- 2.37.7

2 1

[PATCH OLK-6.6] Bluetooth: L2CAP: Fix use-after-free in l2cap_sock_new_connection_cb()
by Chen Jinghuang 03 Aug '26

03 Aug '26

From: Siwei Zhang <oss(a)fourdim.xyz> stable inclusion from stable-v6.6.145 commit 8c37e4338c801ebb8cee52436c01c41e009f6e87 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16859 CVE: CVE-2026-64557 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id… -------------------------------- [ Upstream commit 6fef032af0092ed5ccb767239a9ac1bc38c08a40 ] l2cap_sock_new_connection_cb() returned l2cap_pi(sk)->chan after release_sock(parent). Once the parent lock is dropped the newly enqueued child socket sk is reachable via the accept queue, so another task can accept and free it before the callback dereferences sk, resulting in a use-after-free. Rework the ->new_connection() op so the core, rather than the callback, owns the child channel's lifetime. The op now receives a pre-allocated new_chan and returns an errno instead of allocating and returning a channel. l2cap_new_connection() allocates the child channel and links it into the conn list via __l2cap_chan_add() before invoking the callback, so the conn-list reference keeps the channel alive once release_sock(parent) exposes the socket to other tasks. Channel configuration that was duplicated in l2cap_sock_init() and the various new_connection callbacks is consolidated into l2cap_chan_set_defaults(), which now inherits from the parent channel when one is supplied. Fixes: 8ffb929098a5 ("Bluetooth: Remove parent socket usage from l2cap_core.c") Cc: stable(a)kernel.org Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Siwei Zhang <oss(a)fourdim.xyz> Signed-off-by: Luiz Augusto von Dentz <luiz.von.dentz(a)intel.com> Signed-off-by: Sasha Levin <sashal(a)kernel.org> Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org> Conflicts: net/bluetooth/6lowpan.c [a trival context conflict] Signed-off-by: Chen Jinghuang <chenjinghuang2(a)huawei.com> --- include/net/bluetooth/l2cap.h | 10 ++-- net/bluetooth/6lowpan.c | 18 +----- net/bluetooth/l2cap_core.c | 78 ++++++++++++++++++++----- net/bluetooth/l2cap_sock.c | 103 ++++++++++++++++------------------ net/bluetooth/smp.c | 27 ++------- 5 files changed, 127 insertions(+), 109 deletions(-) diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 2c0356e7b571..136036b48e6d 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -614,7 +614,8 @@ struct l2cap_chan { struct l2cap_ops { char *name; - struct l2cap_chan *(*new_connection) (struct l2cap_chan *chan); + int (*new_connection)(struct l2cap_chan *chan, + struct l2cap_chan *new_chan); int (*recv) (struct l2cap_chan * chan, struct sk_buff *skb); void (*teardown) (struct l2cap_chan *chan, int err); @@ -879,9 +880,10 @@ static inline __u16 __next_seq(struct l2cap_chan *chan, __u16 seq) return (seq + 1) % (chan->tx_win_max + 1); } -static inline struct l2cap_chan *l2cap_chan_no_new_connection(struct l2cap_chan *chan) +static inline int l2cap_chan_no_new_connection(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - return NULL; + return -EOPNOTSUPP; } static inline int l2cap_chan_no_recv(struct l2cap_chan *chan, struct sk_buff *skb) @@ -957,7 +959,7 @@ int l2cap_chan_send(struct l2cap_chan *chan, struct msghdr *msg, size_t len); void l2cap_chan_busy(struct l2cap_chan *chan, int busy); void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail); int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator); -void l2cap_chan_set_defaults(struct l2cap_chan *chan); +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan); int l2cap_ertm_init(struct l2cap_chan *chan); void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index b3132079573d..34b9599f2cde 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -631,7 +631,7 @@ static struct l2cap_chan *chan_create(void) if (!chan) return NULL; - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); chan->chan_type = L2CAP_CHAN_CONN_ORIENTED; chan->mode = L2CAP_MODE_LE_FLOWCTL; @@ -744,21 +744,6 @@ static inline void chan_ready_cb(struct l2cap_chan *chan) ifup(dev->netdev); } -static inline struct l2cap_chan *chan_new_conn_cb(struct l2cap_chan *pchan) -{ - struct l2cap_chan *chan; - - chan = chan_create(); - if (!chan) - return NULL; - - chan->ops = pchan->ops; - - BT_DBG("chan %p pchan %p", chan, pchan); - - return chan; -} - static void delete_netdev(struct work_struct *work) { struct lowpan_btle_dev *entry = container_of(work, @@ -880,7 +865,6 @@ static long chan_get_sndtimeo_cb(struct l2cap_chan *chan) static const struct l2cap_ops bt_6lowpan_chan_ops = { .name = "L2CAP 6LoWPAN channel", - .new_connection = chan_new_conn_cb, .recv = chan_recv_cb, .close = chan_close_cb, .state_change = chan_state_change_cb, diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 8181174f75d9..1da65bc37779 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -525,7 +525,10 @@ void l2cap_chan_put(struct l2cap_chan *c) } EXPORT_SYMBOL_GPL(l2cap_chan_put); -void l2cap_chan_set_defaults(struct l2cap_chan *chan) +/* Initialise @chan with default values, inheriting from the parent channel + * @pchan when it is given. + */ +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan) { chan->fcs = L2CAP_FCS_CRC16; chan->max_tx = L2CAP_DEFAULT_MAX_TX; @@ -539,6 +542,31 @@ void l2cap_chan_set_defaults(struct l2cap_chan *chan) chan->retrans_timeout = L2CAP_DEFAULT_RETRANS_TO; chan->monitor_timeout = L2CAP_DEFAULT_MONITOR_TO; + if (pchan) { + BT_DBG("chan %p pchan %p", chan, pchan); + + chan->chan_type = pchan->chan_type; + chan->imtu = pchan->imtu; + chan->omtu = pchan->omtu; + chan->mode = pchan->mode; + chan->fcs = pchan->fcs; + chan->max_tx = pchan->max_tx; + chan->tx_win = pchan->tx_win; + chan->tx_win_max = pchan->tx_win_max; + chan->sec_level = pchan->sec_level; + chan->conf_state = pchan->conf_state; + chan->flags = pchan->flags; + chan->tx_credits = pchan->tx_credits; + chan->rx_credits = pchan->rx_credits; + + if (chan->chan_type == L2CAP_CHAN_FIXED) { + chan->scid = pchan->scid; + chan->dcid = pchan->scid; + } + + return; + } + chan->conf_state = 0; set_bit(CONF_NOT_COMPLETE, &chan->conf_state); @@ -3959,6 +3987,38 @@ static inline int l2cap_command_rej(struct l2cap_conn *conn, return 0; } +/* Allocate and initialise a channel for an incoming connection. + * + * The channel inherits its configuration from @pchan and is linked into @conn + * before ->new_connection() runs, so the conn list reference keeps it alive if + * the callback exposes it (e.g. via the socket accept queue) before this + * returns. The l2cap_chan_create() reference is taken over by the subsystem on + * success and dropped here on failure. + */ +static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn, + struct l2cap_chan *pchan) +{ + struct l2cap_chan *chan; + + chan = l2cap_chan_create(); + if (!chan) + return NULL; + + l2cap_chan_set_defaults(chan, pchan); + chan->ops = pchan->ops; + + __l2cap_chan_add(conn, chan); + + if (pchan->ops->new_connection && + pchan->ops->new_connection(pchan, chan) < 0) { + l2cap_chan_del(chan, 0); + l2cap_chan_put(chan); + return NULL; + } + + return chan; +} + static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u8 *data, u8 rsp_code) { @@ -4005,7 +4065,7 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, goto response; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) goto response; @@ -4023,8 +4083,6 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, chan->psm = psm; chan->dcid = scid; - __l2cap_chan_add(conn, chan); - dcid = chan->scid; __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); @@ -4893,7 +4951,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; goto response_unlock; @@ -4908,8 +4966,6 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_le_flowctl_init(chan, __le16_to_cpu(req->credits)); dcid = chan->scid; @@ -5118,7 +5174,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, continue; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; continue; @@ -5133,8 +5189,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_ecred_init(chan, __le16_to_cpu(req->credits)); /* Init response */ @@ -7400,14 +7454,12 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status) goto next; l2cap_chan_lock(pchan); - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (chan) { bacpy(&chan->src, &hcon->src); bacpy(&chan->dst, &hcon->dst); chan->src_type = bdaddr_src_type(hcon); chan->dst_type = dst_type; - - __l2cap_chan_add(conn, chan); } l2cap_chan_unlock(pchan); diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 3e9b27d3a610..47f0d5dcbb7f 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -45,7 +45,8 @@ static struct bt_sock_list l2cap_sk_list = { static const struct proto_ops l2cap_sock_ops; static void l2cap_sock_init(struct sock *sk, struct sock *parent); static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern); + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan); static void l2cap_sock_cleanup_listen(struct sock *parent); bool l2cap_is_socket(struct socket *sock) @@ -1240,6 +1241,23 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg, return err; } +/* Release the sock's ref on chan and clear the pointer so that the ref is + * dropped exactly once even if both l2cap_sock_kill() and + * l2cap_sock_destruct() run. Setting chan->data to NULL first stops any other + * task from dereferencing the now-dead sock pointer. + */ +static void l2cap_sock_put_chan(struct sock *sk) +{ + struct l2cap_chan *chan = l2cap_pi(sk)->chan; + + if (!chan) + return; + + chan->data = NULL; + l2cap_pi(sk)->chan = NULL; + l2cap_chan_put(chan); +} + /* Kill socket (only if zapped and orphan) * Must be called on unlocked socket, with l2cap channel lock. */ @@ -1250,13 +1268,9 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); - /* Sock is dead, so set chan data to NULL, avoid other task use invalid - * sock pointer. - */ - l2cap_pi(sk)->chan->data = NULL; - /* Kill poor orphan */ + l2cap_sock_put_chan(sk); - l2cap_chan_put(l2cap_pi(sk)->chan); + /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); sock_put(sk); } @@ -1499,12 +1513,13 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) } } -static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) +static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { struct sock *sk, *parent = chan->data; if (!parent) - return NULL; + return -EINVAL; lock_sock(parent); @@ -1512,25 +1527,28 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); release_sock(parent); - return NULL; + return -ENOBUFS; } sk = l2cap_sock_alloc(sock_net(parent), NULL, BTPROTO_L2CAP, - GFP_ATOMIC, 0); + GFP_ATOMIC, 0, new_chan); if (!sk) { release_sock(parent); - return NULL; - } + return -ENOMEM; + } bt_sock_reclassify_lock(sk, BTPROTO_L2CAP); l2cap_sock_init(sk, parent); + /* The conn list reference taken by l2cap_new_connection() keeps new_chan + * alive once release_sock() lets another task free this socket. + */ bt_accept_enqueue(parent, sk, false); release_sock(parent); - return l2cap_pi(sk)->chan; + return 0; } static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) @@ -1828,10 +1846,7 @@ static void l2cap_sock_destruct(struct sock *sk) BT_DBG("sk %p", sk); - if (l2cap_pi(sk)->chan) { - l2cap_pi(sk)->chan->data = NULL; - l2cap_chan_put(l2cap_pi(sk)->chan); - } + l2cap_sock_put_chan(sk); list_for_each_entry_safe(rx_busy, next, &l2cap_pi(sk)->rx_busy, list) { kfree_skb(rx_busy->skb); @@ -1863,30 +1878,12 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) BT_DBG("sk %p", sk); if (parent) { - struct l2cap_chan *pchan = l2cap_pi(parent)->chan; - sk->sk_type = parent->sk_type; bt_sk(sk)->flags = bt_sk(parent)->flags; - chan->chan_type = pchan->chan_type; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->conf_state = pchan->conf_state; - chan->mode = pchan->mode; - chan->fcs = pchan->fcs; - chan->max_tx = pchan->max_tx; - chan->tx_win = pchan->tx_win; - chan->tx_win_max = pchan->tx_win_max; - chan->sec_level = pchan->sec_level; - chan->flags = pchan->flags; - chan->tx_credits = pchan->tx_credits; - chan->rx_credits = pchan->rx_credits; - - if (chan->chan_type == L2CAP_CHAN_FIXED) { - chan->scid = pchan->scid; - chan->dcid = pchan->scid; - } - + /* Channel configuration is inherited from the parent by + * l2cap_new_connection(). + */ security_sk_clone(parent, sk); } else { switch (sk->sk_type) { @@ -1912,7 +1909,7 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) chan->mode = L2CAP_MODE_BASIC; } - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); } /* Default config options */ @@ -1931,10 +1928,10 @@ static struct proto l2cap_proto = { }; static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern) + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan) { struct sock *sk; - struct l2cap_chan *chan; sk = bt_sock_alloc(net, sock, &l2cap_proto, proto, prio, kern); if (!sk) @@ -1945,16 +1942,7 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy); - chan = l2cap_chan_create(); - if (!chan) { - sk_free(sk); - if (sock) - sock->sk = NULL; - return NULL; - } - - l2cap_chan_hold(chan); - + /* The sock takes ownership of the caller's reference on chan. */ l2cap_pi(sk)->chan = chan; return sk; @@ -1964,6 +1952,7 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, int kern) { struct sock *sk; + struct l2cap_chan *chan; BT_DBG("sock %p", sock); @@ -1978,10 +1967,16 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, sock->ops = &l2cap_sock_ops; - sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern); - if (!sk) + chan = l2cap_chan_create(); + if (!chan) return -ENOMEM; + sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern, chan); + if (!sk) { + l2cap_chan_put(chan); + return -ENOMEM; + } + l2cap_sock_init(sk, NULL); bt_sock_link(&l2cap_sk_list, sk); return 0; diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index de34bdf1802d..df192797eda9 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -3235,34 +3235,19 @@ static const struct l2cap_ops smp_chan_ops = { .get_sndtimeo = l2cap_chan_no_get_sndtimeo, }; -static inline struct l2cap_chan *smp_new_conn_cb(struct l2cap_chan *pchan) +static inline int smp_new_conn_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - struct l2cap_chan *chan; - - BT_DBG("pchan %p", pchan); - - chan = l2cap_chan_create(); - if (!chan) - return NULL; - - chan->chan_type = pchan->chan_type; - chan->ops = &smp_chan_ops; - chan->scid = pchan->scid; - chan->dcid = chan->scid; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->mode = pchan->mode; + new_chan->ops = &smp_chan_ops; /* Other L2CAP channels may request SMP routines in order to * change the security level. This means that the SMP channel * lock must be considered in its own category to avoid lockdep * warnings. */ - atomic_set(&chan->nesting, L2CAP_NESTING_SMP); - - BT_DBG("created chan %p", chan); + atomic_set(&new_chan->nesting, L2CAP_NESTING_SMP); - return chan; + return 0; } static const struct l2cap_ops smp_root_chan_ops = { @@ -3333,7 +3318,7 @@ static struct l2cap_chan *smp_add_cid(struct hci_dev *hdev, u16 cid) l2cap_add_scid(chan, cid); - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); if (cid == L2CAP_CID_SMP) { u8 bdaddr_type; -- 2.34.1

2 1

[PATCH OLK-6.6 0/3] mainline bpf
by Pu Lehui 03 Aug '26

03 Aug '26

Bradley Morgan (1): bpf: Disable xfrm_decode_session hook attachment Jann Horn (1): bpf,fork: wipe ->bpf_storage before bailouts that access it Matt Bobrowski (1): bpf: Fix UAF in sock clone early bailouts kernel/bpf/bpf_lsm.c | 3 +++ kernel/fork.c | 9 +++++---- net/core/bpf_sk_storage.c | 2 -- net/core/sock.c | 3 +++ 4 files changed, 11 insertions(+), 6 deletions(-) -- 2.34.1

2 4

[PATCH OLK-6.6] bpf,fork: wipe ->bpf_storage before bailouts that access it
by Pu Lehui 03 Aug '26

03 Aug '26

From: Jann Horn <jannh(a)google.com> stable inclusion from stable-v6.6.145 commit c3fd6f28c7ce1142a3b23dbb840eaa4777de1d74 category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/9693 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id… -------------------------------- commit 9b51a6155d14389876916726430da30eabb1d4ed upstream. Currently, copy_process() can bail out to free_task() before p->bpf_storage has been initialized, with this call graph (shown here for the !CONFIG_MEMCG case): copy_process dup_task_struct arch_dup_task_struct [copies the entire task_struct, including ->bpf_storage member] [RLIMIT_NPROC check fails] delayed_free_task free_task bpf_task_storage_free rcu_dereference(task->bpf_storage) bpf_local_storage_destroy In this case, the nascent task's ->bpf_storage member that bpf_local_storage_destroy() operates on is a plain copy of the parent's ->bpf_storage pointer, not a real initialized pointer. This leads to badness (kernel hangs, UAF). This is reachable as long as the process calling fork() has been inserted into a task storage map. Cc: stable(a)kernel.org Fixes: a10787e6d58c ("bpf: Enable task local storage for tracing programs") Signed-off-by: Jann Horn <jannh(a)google.com> Signed-off-by: Andrii Nakryiko <andrii(a)kernel.org> Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org> Conflicts: kernel/fork.c [ctx conflicts] Signed-off-by: Pu Lehui <pulehui(a)huawei.com> --- kernel/fork.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 04d934eabbc7..e92ddc0a0c66 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1290,6 +1290,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->migrate_from_cpu = -1; #endif +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(tsk->bpf_storage, NULL); + tsk->bpf_ctx = NULL; +#endif + #ifdef CONFIG_FAST_SYSCALL tsk->xinfo = NULL; #endif @@ -2608,10 +2613,6 @@ __latent_entropy struct task_struct *copy_process( p->sequential_io = 0; p->sequential_io_avg = 0; #endif -#ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); - p->bpf_ctx = NULL; -#endif /* Perform scheduler related setup. Assign this task to a CPU. */ retval = sched_fork(clone_flags, p); -- 2.34.1

2 1

[PATCH OLK-6.6] i2c: i801: fix hardware state machine corruption in error path
by Zhang Yuwei 03 Aug '26

03 Aug '26

From: Mingyu Wang <25181214217(a)stu.xidian.edu.cn> mainline inclusion from mainline-v7.2-rc1 commit 10dd1a736d557e310a77117832874729a0175d57 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16512 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?… -------------------------------- A severe livelock and subsequent Hung Task panic were observed in the i2c-i801 driver during concurrent Fuzzing. The crash is caused by an unconditional hardware register cleanup in the error handling path of i801_access(). When i801_check_pre() fails (e.g., returning -EBUSY because the SMBus controller is actively used by BIOS/ACPI), the kernel does not actually acquire the hardware ownership. However, the code jumps to the 'out' label and executes: iowrite8(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv)); This forcefully clears the INUSE_STS lock and resets the hardware status flags without owning the controller. Doing so interrupts ongoing BIOS/ACPI transactions and totally corrupts the SMBus hardware state machine. Consequently, all subsequent i801_access() calls fail at the pre-check stage, triggering an endless stream of "SMBus is busy, can't use it!" error logs. Over a slow serial console, this printk flood monopolizes the CPU (Console Livelock), starving other processes trying to acquire the mmap_lock down_read semaphore, ultimately triggering the hung task watchdog. Fix this by moving the 'out' label below the hardware register cleanup. If i801_check_pre() fails, we safely bypass the iowrite8() and only release the software locks (pm_runtime and mutex), strictly adhering to the rule of not releasing resources that were never acquired. Fixes: 1f760b87e54c ("i2c: i801: Call i801_check_pre() from i801_access()") Signed-off-by: Mingyu Wang <25181214217(a)stu.xidian.edu.cn> Cc: <stable(a)vger.kernel.org> # v6.3+ Signed-off-by: Andi Shyti <andi.shyti(a)kernel.org> Link: https://lore.kernel.org/r/20260512093534.348655-1-w15303746062@163.com Conflicts: drivers/i2c/busses/i2c-i801.c [contxt conflict] Signed-off-by: Zhang Yuwei <zhangyuwei20(a)huawei.com> --- drivers/i2c/busses/i2c-i801.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 89fdc75cdcfa..b0429fa3f573 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -918,13 +918,13 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, */ if (hwpec) outb_p(inb_p(SMBAUXCTL(priv)) & ~SMBAUXCTL_CRC, SMBAUXCTL(priv)); -out: /* * Unlock the SMBus device for use by BIOS/ACPI, * and clear status flags if not done already. */ outb_p(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv)); +out: pm_runtime_mark_last_busy(&priv->pci_dev->dev); pm_runtime_put_autosuspend(&priv->pci_dev->dev); mutex_unlock(&priv->acpi_lock); -- 2.22.0

2 1

[PATCH OLK-5.10] bpf, skmsg: fix verdict sk_data_ready racing with ktls rx
by superdcc97＠163.com 03 Aug '26

03 Aug '26

From: Xingwang Xiang <v3rdant.xiang(a)gmail.com> mainline inclusion from mainline-v7.1-rc5 commit ddf8029623a1af20e984c040e89ff918158397ab category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16350 CVE: CVE-2026-64025 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?… -------------------------------- sk_psock_strp_data_ready() already checks tls_sw_has_ctx_rx() and defers to psock->saved_data_ready when a TLS RX context is present, avoiding a conflict with the TLS strparser's ownership of the receive queue (commit e91de6afa81c ("bpf: Fix running sk_skb program types with ktls")). sk_psock_verdict_data_ready() has no equivalent guard. When a socket is inserted into a sockmap (BPF_SK_SKB_VERDICT) before TLS RX is configured, tls_sw_strparser_arm() saves sk_psock_verdict_data_ready as rx_ctx->saved_data_ready. On data arrival: tls_data_ready -> tls_strp_data_ready -> tls_rx_msg_ready -> saved_data_ready() = sk_psock_verdict_data_ready() -> tcp_read_skb() drains sk_receive_queue via __skb_unlink() without calling tcp_eat_skb(), so copied_seq is not advanced. tls_strp_msg_load() then finds tcp_inq() >= full_len (stale), calls tcp_recv_skb() on the now-empty queue, hits WARN_ON_ONCE(!first), and returns with rx_ctx->strp.anchor.frag_list pointing at a psock-owned (potentially freed) skb. tls_decrypt_sg() subsequently walks that frag_list: use-after-free. Apply the same fix as sk_psock_strp_data_ready(): if a TLS RX context is present, call psock->saved_data_ready (sock_def_readable) to wake recv() waiters and return immediately, leaving the receive queue untouched. TLS retains sole ownership of the queue and decrypts the record normally through tls_sw_recvmsg(). Fixes: ef5659280eb1 ("bpf, sockmap: Allow skipping sk_skb parser program") Signed-off-by: Xingwang Xiang <v3rdant.xiang(a)gmail.com> Link: https://patch.msgid.link/20260517145630.20521-2-v3rdant.xiang@gmail.com Signed-off-by: Jakub Kicinski <kuba(a)kernel.org> Conflicts: net/core/skmsg.c [Adaptation notes: - The upstream mainline function uses rcu_read_lock(), sk_psock(), READ_ONCE(sk->sk_socket) and ops->read_skb. The OLK-5.10 version of sk_psock_verdict_data_ready() uses the legacy sock->ops->read_sock path without RCU protection. - Transplanted the TLS RX guard by taking rcu_read_lock(), obtaining struct sk_psock *psock, and calling psock->parser.saved_data_ready(sk) (in 5.10 saved_data_ready lives inside struct sk_psock_parser) before returning, leaving the receive queue untouched for ktls. - This preserves the same logic as sk_psock_strp_data_ready() which already performs tls_sw_has_ctx_rx() detection in this tree.] Signed-off-by: Dong Chenchen <dongchenchen2(a)huawei.com> --- net/core/skmsg.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 5150fc93dd9b..b979a4cba4d2 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1056,11 +1056,21 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb, static void sk_psock_verdict_data_ready(struct sock *sk) { struct socket *sock = sk->sk_socket; + struct sk_psock *psock; read_descriptor_t desc; if (unlikely(!sock || !sock->ops || !sock->ops->read_sock)) return; + rcu_read_lock(); + psock = sk_psock(sk); + if (psock && tls_sw_has_ctx_rx(sk)) { + psock->parser.saved_data_ready(sk); + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + desc.arg.data = sk; desc.error = 0; desc.count = 1; -- 2.43.0

2 1

[PATCH OLK-5.10] netfilter: nf_tables: fix dst corruption in same register operation
by superdcc97＠163.com 03 Aug '26

03 Aug '26

From: Fernando Fernandez Mancera <fmancera(a)suse.de> mainline inclusion from mainline-v7.1-rc6 commit 18014147d3ee7831dce53fe65d7fc8d428b02552 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16331 CVE: CVE-2026-64006 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?… -------------------------------- For lshift and rshift, the shift operations are performed in a loop over 32-bit words. The loop calculates the shifted value and write it to dst, and then immediately reads from src to calculate the carry for the next iteration. Because src and dst could point to the same memory location, the carry is incorrectly calculated using the newly modified dst value instead of the original src value. Adding a temporary local variable to cache the original value before writing to dst and using it for the carry calculation solves the problem. In addition, partial overlap is rejected from control plane for all kind of operations including byteorder. This was tested with the following bytecode: table test_table ip flags 0 use 1 handle 1 ip test_table test_chain use 3 type filter hook input prio 0 policy accept packets 0 bytes 0 flags 1 ip test_table test_chain 2 [ immediate reg 1 0x44332211 0x88776655 ] [ bitwise reg 1 = ( reg 1 << 0x08000000 ) ] [ cmp eq reg 1 0x66443322 0x00887766 ] [ counter pkts 0 bytes 0 ] ip test_table test_chain 4 3 [ immediate reg 1 0x44332211 0x88776655 ] [ bitwise reg 1 = ( reg 1 << 0x08000000 ) ] [ cmp eq reg 1 0x55443322 0x00887766 ] [ counter pkts 21794 bytes 1917798 ] Fixes: 567d746b55bc ("netfilter: bitwise: add support for shifts.") Acked-by: Jeremy Sowden <jeremy(a)azazel.net> Signed-off-by: Fernando Fernandez Mancera <fmancera(a)suse.de> Signed-off-by: Florian Westphal <fw(a)strlen.de> Conflicts: include/net/netfilter/nf_tables.h net/netfilter/nft_bitwise.c net/netfilter/nft_byteorder.c [nft_bitwise_init_bool() lacks ctx/sreg2 support in 5.10; the sreg2-dreg overlap check was omitted because that control path does not exist on this branch.] Signed-off-by: Dong Chenchen <dongchenchen2(a)huawei.com> --- include/net/netfilter/nf_tables.h | 7 +++++++ net/netfilter/nft_bitwise.c | 15 +++++++++++---- net/netfilter/nft_byteorder.c | 13 ++++++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index da6ce70eb1ba..9168a6a8934c 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -153,6 +153,13 @@ static inline u64 nft_reg_load64(const u32 *sreg) return get_unaligned((u64 *)sreg); } +static inline bool nft_reg_overlap(u8 src, u8 dst, u32 len) +{ + unsigned int n = DIV_ROUND_UP(len, sizeof(u32)); + + return src != dst && src < dst + n && dst < src + n; +} + static inline void nft_data_copy(u32 *dst, const struct nft_data *src, unsigned int len) { diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c index 47129b8220e5..9b01140d5ce4 100644 --- a/net/netfilter/nft_bitwise.c +++ b/net/netfilter/nft_bitwise.c @@ -42,8 +42,10 @@ static void nft_bitwise_eval_lshift(u32 *dst, const u32 *src, u32 carry = 0; for (i = DIV_ROUND_UP(priv->len, sizeof(u32)); i > 0; i--) { - dst[i - 1] = (src[i - 1] << shift) | carry; - carry = src[i - 1] >> (BITS_PER_TYPE(u32) - shift); + u32 tmp_src = src[i - 1]; + + dst[i - 1] = (tmp_src << shift) | carry; + carry = tmp_src >> (BITS_PER_TYPE(u32) - shift); } } @@ -55,8 +57,10 @@ static void nft_bitwise_eval_rshift(u32 *dst, const u32 *src, u32 carry = 0; for (i = 0; i < DIV_ROUND_UP(priv->len, sizeof(u32)); i++) { - dst[i] = carry | (src[i] >> shift); - carry = src[i] << (BITS_PER_TYPE(u32) - shift); + u32 tmp_src = src[i]; + + dst[i] = carry | (tmp_src >> shift); + carry = tmp_src << (BITS_PER_TYPE(u32) - shift); } } @@ -183,6 +187,9 @@ static int nft_bitwise_init(const struct nft_ctx *ctx, if (err < 0) return err; + if (nft_reg_overlap(priv->sreg, priv->dreg, priv->len)) + return -EINVAL; + if (tb[NFTA_BITWISE_OP]) { priv->op = ntohl(nla_get_be32(tb[NFTA_BITWISE_OP])); switch (priv->op) { diff --git a/net/netfilter/nft_byteorder.c b/net/netfilter/nft_byteorder.c index 9d250bd60bb8..b70d955685ea 100644 --- a/net/netfilter/nft_byteorder.c +++ b/net/netfilter/nft_byteorder.c @@ -143,9 +143,16 @@ static int nft_byteorder_init(const struct nft_ctx *ctx, if (err < 0) return err; - return nft_parse_register_store(ctx, tb[NFTA_BYTEORDER_DREG], - &priv->dreg, NULL, NFT_DATA_VALUE, - priv->len); + err = nft_parse_register_store(ctx, tb[NFTA_BYTEORDER_DREG], + &priv->dreg, NULL, NFT_DATA_VALUE, + priv->len); + if (err < 0) + return err; + + if (nft_reg_overlap(priv->sreg, priv->dreg, priv->len)) + return -EINVAL; + + return 0; } static int nft_byteorder_dump(struct sk_buff *skb, const struct nft_expr *expr) -- 2.43.0

2 1

[PATCH OLK-6.6] proc: protect ptrace_may_access() with exec_update_lock (FD links)
by Lin Ruifeng 03 Aug '26

03 Aug '26

From: Jann Horn <jannh(a)google.com> stable inclusion from stable-v6.6.145 commit 138c692d2b2d63d26f2eb957d0e4fcc5d61f9ff2 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16670 CVE: CVE-2026-64375 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id… -------------------------------- [ Upstream commit 6255da28d4bb5349fe18e84cb043ccd394eba75d ] proc_pid_get_link() and proc_pid_readlink() currently look up the task from the pid once, then do the ptrace access check on that task, then look up the task from the pid a second time to do the actual access. That's racy in several ways. To fix it, pass the task to the ->proc_get_link() handler, and instead of proc_fd_access_allowed(), introduce a new helper call_proc_get_link() that looks up and locks the task, does the access check, and calls ->proc_get_link(). Fixes: 778c1144771f ("[PATCH] proc: Use sane permission checks on the /proc/<pid>/fd/ symlinks") Cc: stable(a)vger.kernel.org Signed-off-by: Jann Horn <jannh(a)google.com> Link: https://patch.msgid.link/20260518-procfs-lockfix-part1-v1-2-5c3d20e0ac33@go… Signed-off-by: Christian Brauner (Amutable) <brauner(a)kernel.org> Signed-off-by: Sasha Levin <sashal(a)kernel.org> Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org> Conflicts: fs/proc/base.c [Context Conflicts] Signed-off-by: Lin Ruifeng <linruifeng4(a)huawei.com> --- fs/proc/base.c | 119 ++++++++++++++++++--------------------------- fs/proc/fd.c | 27 +++++----- fs/proc/internal.h | 2 +- 3 files changed, 59 insertions(+), 89 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 698db152926e..aed3022beb42 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -222,33 +222,24 @@ static int get_task_root(struct task_struct *task, struct path *root) return result; } -static int proc_cwd_link(struct dentry *dentry, struct path *path) +static int proc_cwd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task = get_proc_task(d_inode(dentry)); int result = -ENOENT; - if (task) { - task_lock(task); - if (task->fs) { - get_fs_pwd(task->fs, path); - result = 0; - } - task_unlock(task); - put_task_struct(task); + task_lock(task); + if (task->fs) { + get_fs_pwd(task->fs, path); + result = 0; } + task_unlock(task); return result; } -static int proc_root_link(struct dentry *dentry, struct path *path) +static int proc_root_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task = get_proc_task(d_inode(dentry)); - int result = -ENOENT; - - if (task) { - result = get_task_root(task, path); - put_task_struct(task); - } - return result; + return get_task_root(task, path); } /* @@ -708,23 +699,6 @@ static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns, /* Here the fs part begins */ /************************************************************************/ -/* permission checks */ -static bool proc_fd_access_allowed(struct inode *inode) -{ - struct task_struct *task; - bool allowed = false; - /* Allow access to a task's file descriptors if it is us or we - * may use ptrace attach to the process and find out that - * information. - */ - task = get_proc_task(inode); - if (task) { - allowed = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS); - put_task_struct(task); - } - return allowed; -} - int proc_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -1782,16 +1756,12 @@ static const struct file_operations proc_pid_set_comm_operations = { .release = single_release, }; -static int proc_exe_link(struct dentry *dentry, struct path *exe_path) +static int proc_exe_link(struct dentry *dentry, struct path *exe_path, + struct task_struct *task) { - struct task_struct *task; struct file *exe_file; - task = get_proc_task(d_inode(dentry)); - if (!task) - return -ENOENT; exe_file = get_task_exe_file(task); - put_task_struct(task); if (exe_file) { *exe_path = exe_file->f_path; path_get(&exe_file->f_path); @@ -1801,26 +1771,42 @@ static int proc_exe_link(struct dentry *dentry, struct path *exe_path) return -ENOENT; } +static int call_proc_get_link(struct dentry *dentry, struct inode *inode, struct path *path_out) +{ + struct task_struct *task; + int ret; + + task = get_proc_task(inode); + if (!task) + return -ENOENT; + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + goto out_put_task; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + ret = -EACCES; + goto out; + } + ret = PROC_I(inode)->op.proc_get_link(dentry, path_out, task); + +out: + up_read(&task->signal->exec_update_lock); +out_put_task: + put_task_struct(task); + return ret; +} + static const char *proc_pid_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { struct path path; - int error = -EACCES; + int error; if (!dentry) return ERR_PTR(-ECHILD); - - /* Are we allowed to snoop on the tasks file descriptors? */ - if (!proc_fd_access_allowed(inode)) - goto out; - - error = PROC_I(inode)->op.proc_get_link(dentry, &path); - if (error) - goto out; - - error = nd_jump_link(&path); -out: + error = call_proc_get_link(dentry, inode, &path); + if (!error) + error = nd_jump_link(&path); return ERR_PTR(error); } @@ -1854,17 +1840,11 @@ static int proc_pid_readlink(struct dentry * dentry, char __user * buffer, int b struct inode *inode = d_inode(dentry); struct path path; - /* Are we allowed to snoop on the tasks file descriptors? */ - if (!proc_fd_access_allowed(inode)) - goto out; - - error = PROC_I(inode)->op.proc_get_link(dentry, &path); - if (error) - goto out; - - error = do_proc_readlink(&path, buffer, buflen); - path_put(&path); -out: + error = call_proc_get_link(dentry, inode, &path); + if (!error) { + error = do_proc_readlink(&path, buffer, buflen); + path_put(&path); + } return error; } @@ -2252,21 +2232,16 @@ static const struct dentry_operations tid_map_files_dentry_operations = { .d_delete = pid_delete_dentry, }; -static int map_files_get_link(struct dentry *dentry, struct path *path) +static int map_files_get_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { unsigned long vm_start, vm_end; struct vm_area_struct *vma; - struct task_struct *task; struct mm_struct *mm; int rc; rc = -ENOENT; - task = get_proc_task(d_inode(dentry)); - if (!task) - goto out; - mm = get_task_mm(task); - put_task_struct(task); if (!mm) goto out; diff --git a/fs/proc/fd.c b/fs/proc/fd.c index 4297287f6ca0..ab9157d2621d 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -172,24 +172,19 @@ static const struct dentry_operations tid_fd_dentry_operations = { .d_delete = pid_delete_dentry, }; -static int proc_fd_link(struct dentry *dentry, struct path *path) +static int proc_fd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) { - struct task_struct *task; int ret = -ENOENT; - - task = get_proc_task(d_inode(dentry)); - if (task) { - unsigned int fd = proc_fd(d_inode(dentry)); - struct file *fd_file; - - fd_file = fget_task(task, fd); - if (fd_file) { - *path = fd_file->f_path; - path_get(&fd_file->f_path); - ret = 0; - fput(fd_file); - } - put_task_struct(task); + unsigned int fd = proc_fd(d_inode(dentry)); + struct file *fd_file; + + fd_file = fget_task(task, fd); + if (fd_file) { + *path = fd_file->f_path; + path_get(&fd_file->f_path); + ret = 0; + fput(fd_file); } return ret; diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 464d729ab0c8..10b3da6d4544 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -107,7 +107,7 @@ extern struct kmem_cache *proc_dir_entry_cache; void pde_free(struct proc_dir_entry *pde); union proc_op { - int (*proc_get_link)(struct dentry *, struct path *); + int (*proc_get_link)(struct dentry *, struct path *, struct task_struct *); int (*proc_show)(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task); -- 2.34.1

2 1