@@ -72,6 +72,7 @@ struct cpt_request_info {
uint64_t ei3;
} ist;
uint8_t *rptr;
+ const struct otx2_cpt_qp *qp;
/** Control path fields */
uint64_t time_out;
new file mode 100644
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 2020 Marvell International Ltd.
+ */
+
+#ifndef _OTX2_CA_HELPER_H_
+#define _OTX2_CA_HELPER_H_
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_eventdev.h>
+
+#include "cpt_pmd_logs.h"
+#include "cpt_ucode.h"
+
+#include "otx2_cryptodev.h"
+#include "otx2_cryptodev_hw_access.h"
+#include "otx2_cryptodev_ops_helper.h"
+#include "otx2_cryptodev_qp.h"
+
+static inline void
+otx2_ca_deq_post_process(const struct otx2_cpt_qp *qp,
+ struct rte_crypto_op *cop, uintptr_t *rsp,
+ uint8_t cc)
+{
+ if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+ if (likely(cc == NO_ERR)) {
+ /* Verify authentication data if required */
+ if (unlikely(rsp[2]))
+ compl_auth_verify(cop, (uint8_t *)rsp[2],
+ rsp[3]);
+ else
+ cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ } else {
+ if (cc == ERR_GC_ICV_MISCOMPARE)
+ cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ else
+ cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+
+ if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+ sym_session_clear(otx2_cryptodev_driver_id,
+ cop->sym->session);
+ memset(cop->sym->session, 0,
+ rte_cryptodev_sym_get_existing_header_session_size(
+ cop->sym->session));
+ rte_mempool_put(qp->sess_mp, cop->sym->session);
+ cop->sym->session = NULL;
+ }
+ }
+
+}
+
+static __rte_always_inline uint64_t
+otx2_handle_crypto_event(uint64_t get_work1)
+{
+ struct cpt_request_info *req;
+ struct rte_crypto_op *cop;
+ uintptr_t *rsp;
+ void *metabuf;
+ uint8_t cc;
+
+ req = (struct cpt_request_info *)(get_work1 >> 3);
+ cc = otx2_cpt_compcode_get(req);
+
+ rsp = req->op;
+ metabuf = (void *)rsp[0];
+ cop = (void *)rsp[1];
+
+ otx2_ca_deq_post_process(req->qp, cop, rsp, cc);
+
+ rte_mempool_put(req->qp->meta_info.pool, metabuf);
+
+ return (uint64_t)(cop);
+}
+#endif /* _OTX2_CA_HELPER_H_ */
@@ -417,6 +417,48 @@ sym_session_configure(int driver_id, struct rte_crypto_sym_xform *xform,
return -ENOTSUP;
}
+static __rte_always_inline void __rte_hot
+otx2_ca_enqueue_req(const struct otx2_cpt_qp *qp,
+ struct cpt_request_info *req,
+ void *lmtline)
+{
+ union cpt_inst_s inst;
+ uint64_t lmt_status;
+
+ inst.u[0] = 0;
+ inst.s9x.res_addr = req->comp_baddr;
+ inst.u[2] = 0;
+ inst.u[3] = 0;
+
+ inst.s9x.ei0 = req->ist.ei0;
+ inst.s9x.ei1 = req->ist.ei1;
+ inst.s9x.ei2 = req->ist.ei2;
+ inst.s9x.ei3 = req->ist.ei3;
+
+ inst.s9x.qord = 1;
+ inst.s9x.grp = qp->ev.queue_id;
+ inst.s9x.tt = qp->ev.sched_type;
+ inst.s9x.tag = (RTE_EVENT_TYPE_CRYPTODEV << 28) |
+ qp->ev.flow_id;
+ inst.s9x.wq_ptr = (uint64_t)req;
+ req->qp = qp;
+
+ do {
+ /* Copy CPT command to LMTLINE */
+ memcpy(lmtline, &inst, sizeof(inst));
+
+ /*
+ * Make sure compiler does not reorder memcpy and ldeor.
+ * LMTST transactions are always flushed from the write
+ * buffer immediately, a DMB is not required to push out
+ * LMTSTs.
+ */
+ rte_cio_wmb();
+ lmt_status = otx2_lmt_submit(qp->lf_nq_reg);
+ } while (lmt_status == 0);
+
+}
+
static __rte_always_inline int32_t __rte_hot
otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp,
struct pending_queue *pend_q,
@@ -426,6 +468,11 @@ otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp,
union cpt_inst_s inst;
uint64_t lmt_status;
+ if (qp->ca_enable) {
+ otx2_ca_enqueue_req(qp, req, lmtline);
+ return 0;
+ }
+
if (unlikely(pend_q->pending_count >= OTX2_CPT_DEFAULT_CMD_QLEN))
return -EAGAIN;
@@ -5,6 +5,7 @@ DPDK_21 {
INTERNAL {
global:
+ otx2_cryptodev_driver_id;
otx2_ca_caps_get;
otx2_ca_qp_add;
otx2_ca_qp_del;
@@ -27,3 +27,4 @@ endforeach
deps += ['bus_pci', 'common_octeontx2', 'mempool_octeontx2', 'pmd_octeontx2', 'pmd_octeontx2_crypto']
includes += include_directories('../../crypto/octeontx2')
+includes += include_directories('../../common/cpt')
@@ -9,6 +9,7 @@
#include <rte_branch_prediction.h>
#include <otx2_common.h>
+#include "otx2_ca_helper.h"
#include "otx2_evdev.h"
#include "otx2_ethdev_sec_tx.h"
@@ -66,8 +67,10 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
ws->cur_tt = event.sched_type;
ws->cur_grp = event.queue_id;
- if (event.sched_type != SSO_TT_EMPTY &&
- event.event_type == RTE_EVENT_TYPE_ETHDEV) {
+ if (event.sched_type == SSO_TT_EMPTY)
+ goto setev_and_return;
+
+ if (event.event_type == RTE_EVENT_TYPE_ETHDEV) {
otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
(uint32_t) event.get_work0, flags, lookup_mem);
/* Extracting tstamp, if PTP enabled*/
@@ -76,8 +79,11 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
flags, (uint64_t *)tstamp_ptr);
get_work1 = mbuf;
+ } else if (event.event_type == RTE_EVENT_TYPE_CRYPTODEV) {
+ get_work1 = otx2_handle_crypto_event(get_work1);
}
+setev_and_return:
ev->event = event.get_work0;
ev->u64 = get_work1;
@@ -9,6 +9,8 @@
#include <rte_common.h>
#include <otx2_common.h>
+
+#include "otx2_ca_helper.h"
#include "otx2_evdev.h"
/* SSO Operations */
@@ -63,8 +65,10 @@ otx2_ssogws_dual_get_work(struct otx2_ssogws_state *ws,
ws->cur_tt = event.sched_type;
ws->cur_grp = event.queue_id;
- if (event.sched_type != SSO_TT_EMPTY &&
- event.event_type == RTE_EVENT_TYPE_ETHDEV) {
+ if (event.sched_type == SSO_TT_EMPTY)
+ goto setev_and_return;
+
+ if (event.event_type == RTE_EVENT_TYPE_ETHDEV) {
uint8_t port = event.sub_event_type;
event.sub_event_type = 0;
@@ -82,8 +86,11 @@ otx2_ssogws_dual_get_work(struct otx2_ssogws_state *ws,
otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, tstamp, flags,
(uint64_t *)tstamp_ptr);
get_work1 = mbuf;
+ } else if (event.event_type == RTE_EVENT_TYPE_CRYPTODEV) {
+ get_work1 = otx2_handle_crypto_event(get_work1);
}
+setev_and_return:
ev->event = event.get_work0;
ev->u64 = get_work1;