[v2,3/3] event/octeontx2: add crypto adapter datapath
Checks
Commit Message
In the op new mode of crypto adapter, the completed crypto operation
is submitted to the event device by the OCTEON TX2 crypto PMD.
During event device dequeue the result of crypto operation is checked.
Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
---
drivers/common/cpt/cpt_common.h | 1 +
drivers/crypto/octeontx2/otx2_ca_helper.h | 75 +++++++++++++++++++
drivers/crypto/octeontx2/otx2_cryptodev_ops.c | 47 ++++++++++++
.../rte_pmd_octeontx2_crypto_version.map | 1 +
drivers/event/octeontx2/meson.build | 1 +
drivers/event/octeontx2/otx2_worker.h | 10 ++-
drivers/event/octeontx2/otx2_worker_dual.h | 11 ++-
7 files changed, 142 insertions(+), 4 deletions(-)
create mode 100644 drivers/crypto/octeontx2/otx2_ca_helper.h
Comments
On Thu, Sep 17, 2020 at 10:47 AM Ankur Dwivedi <adwivedi@marvell.com> wrote:
>
> In the op new mode of crypto adapter, the completed crypto operation
> is submitted to the event device by the OCTEON TX2 crypto PMD.
> During event device dequeue the result of crypto operation is checked.
>
> Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
> @@ -66,8 +67,10 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
> ws->cur_tt = event.sched_type;
> ws->cur_grp = event.queue_id;
>
> - if (event.sched_type != SSO_TT_EMPTY &&
> - event.event_type == RTE_EVENT_TYPE_ETHDEV) {
> + if (event.sched_type == SSO_TT_EMPTY)
> + goto setev_and_return;
Please move the security updation under the security flag template to
avoid performance impact
on nonsecurity case.
> +
> + if (event.event_type == RTE_EVENT_TYPE_ETHDEV) {
> otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
> (uint32_t) event.get_work0, flags, lookup_mem);
> /* Extracting tstamp, if PTP enabled*/
> @@ -76,8 +79,11 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
> otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
> flags, (uint64_t *)tstamp_ptr);
> get_work1 = mbuf;
> + } else if (event.event_type == RTE_EVENT_TYPE_CRYPTODEV) {
> + get_work1 = otx2_handle_crypto_event(get_work1);
> }
>
@@ -72,6 +72,7 @@ struct cpt_request_info {
uint64_t ei3;
} ist;
uint8_t *rptr;
+ const struct otx2_cpt_qp *qp;
/** Control path fields */
uint64_t time_out;
new file mode 100644
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 2020 Marvell International Ltd.
+ */
+
+#ifndef _OTX2_CA_HELPER_H_
+#define _OTX2_CA_HELPER_H_
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_eventdev.h>
+
+#include "cpt_pmd_logs.h"
+#include "cpt_ucode.h"
+
+#include "otx2_cryptodev.h"
+#include "otx2_cryptodev_hw_access.h"
+#include "otx2_cryptodev_ops_helper.h"
+#include "otx2_cryptodev_qp.h"
+
+static inline void
+otx2_ca_deq_post_process(const struct otx2_cpt_qp *qp,
+ struct rte_crypto_op *cop, uintptr_t *rsp,
+ uint8_t cc)
+{
+ if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+ if (likely(cc == NO_ERR)) {
+ /* Verify authentication data if required */
+ if (unlikely(rsp[2]))
+ compl_auth_verify(cop, (uint8_t *)rsp[2],
+ rsp[3]);
+ else
+ cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ } else {
+ if (cc == ERR_GC_ICV_MISCOMPARE)
+ cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ else
+ cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+
+ if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+ sym_session_clear(otx2_cryptodev_driver_id,
+ cop->sym->session);
+ memset(cop->sym->session, 0,
+ rte_cryptodev_sym_get_existing_header_session_size(
+ cop->sym->session));
+ rte_mempool_put(qp->sess_mp, cop->sym->session);
+ cop->sym->session = NULL;
+ }
+ }
+
+}
+
+static __rte_always_inline uint64_t
+otx2_handle_crypto_event(uint64_t get_work1)
+{
+ struct cpt_request_info *req;
+ struct rte_crypto_op *cop;
+ uintptr_t *rsp;
+ void *metabuf;
+ uint8_t cc;
+
+ req = (struct cpt_request_info *)(get_work1);
+ cc = otx2_cpt_compcode_get(req);
+
+ rsp = req->op;
+ metabuf = (void *)rsp[0];
+ cop = (void *)rsp[1];
+
+ otx2_ca_deq_post_process(req->qp, cop, rsp, cc);
+
+ rte_mempool_put(req->qp->meta_info.pool, metabuf);
+
+ return (uint64_t)(cop);
+}
+#endif /* _OTX2_CA_HELPER_H_ */
@@ -417,6 +417,48 @@ sym_session_configure(int driver_id, struct rte_crypto_sym_xform *xform,
return -ENOTSUP;
}
+static __rte_always_inline void __rte_hot
+otx2_ca_enqueue_req(const struct otx2_cpt_qp *qp,
+ struct cpt_request_info *req,
+ void *lmtline)
+{
+ union cpt_inst_s inst;
+ uint64_t lmt_status;
+
+ inst.u[0] = 0;
+ inst.s9x.res_addr = req->comp_baddr;
+ inst.u[2] = 0;
+ inst.u[3] = 0;
+
+ inst.s9x.ei0 = req->ist.ei0;
+ inst.s9x.ei1 = req->ist.ei1;
+ inst.s9x.ei2 = req->ist.ei2;
+ inst.s9x.ei3 = req->ist.ei3;
+
+ inst.s9x.qord = 1;
+ inst.s9x.grp = qp->ev.queue_id;
+ inst.s9x.tt = qp->ev.sched_type;
+ inst.s9x.tag = (RTE_EVENT_TYPE_CRYPTODEV << 28) |
+ qp->ev.flow_id;
+ inst.s9x.wq_ptr = (uint64_t)req >> 3;
+ req->qp = qp;
+
+ do {
+ /* Copy CPT command to LMTLINE */
+ memcpy(lmtline, &inst, sizeof(inst));
+
+ /*
+ * Make sure compiler does not reorder memcpy and ldeor.
+ * LMTST transactions are always flushed from the write
+ * buffer immediately, a DMB is not required to push out
+ * LMTSTs.
+ */
+ rte_cio_wmb();
+ lmt_status = otx2_lmt_submit(qp->lf_nq_reg);
+ } while (lmt_status == 0);
+
+}
+
static __rte_always_inline int32_t __rte_hot
otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp,
struct pending_queue *pend_q,
@@ -426,6 +468,11 @@ otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp,
union cpt_inst_s inst;
uint64_t lmt_status;
+ if (qp->ca_enable) {
+ otx2_ca_enqueue_req(qp, req, lmtline);
+ return 0;
+ }
+
if (unlikely(pend_q->pending_count >= OTX2_CPT_DEFAULT_CMD_QLEN))
return -EAGAIN;
@@ -5,6 +5,7 @@ DPDK_21 {
INTERNAL {
global:
+ otx2_cryptodev_driver_id;
otx2_ca_caps_get;
otx2_ca_qp_add;
otx2_ca_qp_del;
@@ -27,3 +27,4 @@ endforeach
deps += ['bus_pci', 'common_octeontx2', 'mempool_octeontx2', 'pmd_octeontx2', 'pmd_octeontx2_crypto']
includes += include_directories('../../crypto/octeontx2')
+includes += include_directories('../../common/cpt')
@@ -9,6 +9,7 @@
#include <rte_branch_prediction.h>
#include <otx2_common.h>
+#include "otx2_ca_helper.h"
#include "otx2_evdev.h"
#include "otx2_ethdev_sec_tx.h"
@@ -66,8 +67,10 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
ws->cur_tt = event.sched_type;
ws->cur_grp = event.queue_id;
- if (event.sched_type != SSO_TT_EMPTY &&
- event.event_type == RTE_EVENT_TYPE_ETHDEV) {
+ if (event.sched_type == SSO_TT_EMPTY)
+ goto setev_and_return;
+
+ if (event.event_type == RTE_EVENT_TYPE_ETHDEV) {
otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
(uint32_t) event.get_work0, flags, lookup_mem);
/* Extracting tstamp, if PTP enabled*/
@@ -76,8 +79,11 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
flags, (uint64_t *)tstamp_ptr);
get_work1 = mbuf;
+ } else if (event.event_type == RTE_EVENT_TYPE_CRYPTODEV) {
+ get_work1 = otx2_handle_crypto_event(get_work1);
}
+setev_and_return:
ev->event = event.get_work0;
ev->u64 = get_work1;
@@ -9,6 +9,8 @@
#include <rte_common.h>
#include <otx2_common.h>
+
+#include "otx2_ca_helper.h"
#include "otx2_evdev.h"
/* SSO Operations */
@@ -63,8 +65,10 @@ otx2_ssogws_dual_get_work(struct otx2_ssogws_state *ws,
ws->cur_tt = event.sched_type;
ws->cur_grp = event.queue_id;
- if (event.sched_type != SSO_TT_EMPTY &&
- event.event_type == RTE_EVENT_TYPE_ETHDEV) {
+ if (event.sched_type == SSO_TT_EMPTY)
+ goto setev_and_return;
+
+ if (event.event_type == RTE_EVENT_TYPE_ETHDEV) {
uint8_t port = event.sub_event_type;
event.sub_event_type = 0;
@@ -82,8 +86,11 @@ otx2_ssogws_dual_get_work(struct otx2_ssogws_state *ws,
otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, tstamp, flags,
(uint64_t *)tstamp_ptr);
get_work1 = mbuf;
+ } else if (event.event_type == RTE_EVENT_TYPE_CRYPTODEV) {
+ get_work1 = otx2_handle_crypto_event(get_work1);
}
+setev_and_return:
ev->event = event.get_work0;
ev->u64 = get_work1;