[dpdk-dev] [PATCH 30/30] crypto/qat: add performance improvement into qat crypto dev.

Fiona Trahe fiona.trahe at intel.com
Fri Apr 6 20:52:12 CEST 2018


From: Tomasz Jozwiak <tomaszx.jozwiak at intel.com>

      - removed unused args. from process_response function
      - changed adf_modulo function
      - changed qat_sym_process_response into static inline version


Signed-off-by: Tomasz Jozwiak <tomaszx.jozwiak at intel.com>
Signed-off-by: Fiona Trahe <fiona.trahe at intel.com>
---
 drivers/crypto/qat/qat_qp.c      |  29 +++++----
 drivers/crypto/qat/qat_qp.h      |  10 +--
 drivers/crypto/qat/qat_sym.c     | 129 --------------------------------------
 drivers/crypto/qat/qat_sym.h     | 131 +++++++++++++++++++++++++++++++++++++--
 drivers/crypto/qat/qat_sym_pmd.c |   1 -
 5 files changed, 143 insertions(+), 157 deletions(-)

diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c
index 06ffac47b..44a24926b 100644
--- a/drivers/crypto/qat/qat_qp.c
+++ b/drivers/crypto/qat/qat_qp.c
@@ -14,6 +14,7 @@
 #include "qat_logs.h"
 #include "qat_device.h"
 #include "qat_qp.h"
+#include "qat_sym.h"
 #include "adf_transport_access_macros.h"
 
 
@@ -256,7 +257,7 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
 
 	qp->qat_dev_gen = qat_dev->qat_dev_gen;
 	qp->build_request = qat_qp_conf->build_request;
-	qp->process_response = qat_qp_conf->process_response;
+	qp->service_type = qat_qp_conf->hw->service_type;
 	qp->qat_dev = qat_dev;
 
 	PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
@@ -394,7 +395,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 
 	queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
 					ADF_BYTES_TO_MSG_SIZE(desc_size));
-	queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size);
+	queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 
 	if (queue->max_inflights < 2) {
 		PMD_DRV_LOG(ERR, "Invalid num inflights");
@@ -419,11 +420,11 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 			queue->hw_queue_number, queue_base);
 
 	PMD_DRV_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
-			" nb msgs %u, msg_size %u, max_inflights %u modulo %u",
+		" nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
 			queue->memz_name,
 			queue->queue_size, queue_size_bytes,
 			qp_conf->nb_descriptors, desc_size,
-			queue->max_inflights, queue->modulo);
+			queue->max_inflights, queue->modulo_mask);
 
 	return 0;
 
@@ -512,13 +513,9 @@ static void adf_configure_queues(struct qat_qp *qp)
 			queue->hw_queue_number, queue_config);
 }
 
-
-static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
+static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 {
-	uint32_t div = data >> shift;
-	uint32_t mult = div << shift;
-
-	return data - mult;
+	return data & modulo_mask;
 }
 
 static inline void
@@ -602,7 +599,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 			goto kick_tail;
 		}
 
-		tail = adf_modulo(tail + queue->msg_size, queue->modulo);
+		tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 		ops++;
 		nb_ops_sent++;
 	}
@@ -634,11 +631,13 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 			resp_counter != nb_ops) {
 
-		tmp_qp->process_response(ops, resp_msg,
-			tmp_qp->op_cookies[head / rx_queue->msg_size],
-			tmp_qp->qat_dev_gen);
+		if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
+			qat_sym_process_response(ops, resp_msg);
+		/* add qat_asym_process_response here */
+		/* add qat_comp_process_response here */
 
-		head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
+		head = adf_modulo(head + rx_queue->msg_size,
+				  rx_queue->modulo_mask);
 
 		resp_msg = (uint8_t *)rx_queue->base_addr + head;
 		ops++;
diff --git a/drivers/crypto/qat/qat_qp.h b/drivers/crypto/qat/qat_qp.h
index 6f07bd67c..59db945e7 100644
--- a/drivers/crypto/qat/qat_qp.h
+++ b/drivers/crypto/qat/qat_qp.h
@@ -21,11 +21,6 @@ typedef int (*build_request_t)(void *op,
 		enum qat_device_gen qat_dev_gen);
 /**< Build a request from an op. */
 
-typedef int (*process_response_t)(void **ops,
-		uint8_t *resp, void *op_cookie,
-		enum qat_device_gen qat_dev_gen);
-/**< Process a response descriptor and return the associated op. */
-
 /**
  * Structure with data needed for creation of queue pair.
  */
@@ -46,7 +41,6 @@ struct qat_qp_config {
 	uint32_t cookie_size;
 	int socket_id;
 	build_request_t build_request;
-	process_response_t process_response;
 	const char *service_str;
 };
 
@@ -59,7 +53,7 @@ struct qat_queue {
 	rte_iova_t	base_phys_addr;		/* Queue physical address */
 	uint32_t	head;			/* Shadow copy of the head */
 	uint32_t	tail;			/* Shadow copy of the tail */
-	uint32_t	modulo;
+	uint32_t	modulo_mask;
 	uint32_t	msg_size;
 	uint16_t	max_inflights;
 	uint32_t	queue_size;
@@ -85,7 +79,7 @@ struct qat_qp {
 	uint32_t nb_descriptors;
 	enum qat_device_gen qat_dev_gen;
 	build_request_t build_request;
-	process_response_t process_response;
+	enum qat_service_type service_type;
 	struct qat_pci_device *qat_dev;
 	/**< qat device this qp is on */
 } __rte_cache_aligned;
diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c
index 15244d113..887d4ebcd 100644
--- a/drivers/crypto/qat/qat_sym.c
+++ b/drivers/crypto/qat/qat_sym.c
@@ -12,44 +12,7 @@
 #include <rte_byteorder.h>
 
 #include "qat_logs.h"
-#include "qat_sym_session.h"
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
-
-#define BYTE_LENGTH    8
-/* bpi is only used for partial blocks of DES and AES
- * so AES block len can be assumed as max len for iv, src and dst
- */
-#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
-
-/** Encrypt a single partial block
- *  Depends on openssl libcrypto
- *  Uses ECB+XOR to do CFB encryption, same result, more performant
- */
-static inline int
-bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
-		uint8_t *iv, int ivlen, int srclen,
-		void *bpi_ctx)
-{
-	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
-	int encrypted_ivlen;
-	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
-	uint8_t *encr = encrypted_iv;
-
-	/* ECB method: encrypt the IV, then XOR this with plaintext */
-	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
-								<= 0)
-		goto cipher_encrypt_err;
-
-	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
-		*dst = *src ^ *encr;
-
-	return 0;
-
-cipher_encrypt_err:
-	PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
-	return -EINVAL;
-}
 
 /** Decrypt a single partial block
  *  Depends on openssl libcrypto
@@ -136,62 +99,6 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx,
 	return sym_op->cipher.data.length - last_block_len;
 }
 
-static inline uint32_t
-qat_bpicipher_postprocess(struct qat_sym_session *ctx,
-				struct rte_crypto_op *op)
-{
-	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
-	struct rte_crypto_sym_op *sym_op = op->sym;
-	uint8_t last_block_len = block_len > 0 ?
-			sym_op->cipher.data.length % block_len : 0;
-
-	if (last_block_len > 0 &&
-			ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
-
-		/* Encrypt last block */
-		uint8_t *last_block, *dst, *iv;
-		uint32_t last_block_offset;
-
-		last_block_offset = sym_op->cipher.data.offset +
-				sym_op->cipher.data.length - last_block_len;
-		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
-				uint8_t *, last_block_offset);
-
-		if (unlikely(sym_op->m_dst != NULL))
-			/* out-of-place operation (OOP) */
-			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
-						uint8_t *, last_block_offset);
-		else
-			dst = last_block;
-
-		if (last_block_len < sym_op->cipher.data.length)
-			/* use previous block ciphertext as IV */
-			iv = dst - block_len;
-		else
-			/* runt block, i.e. less than one full block */
-			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
-					ctx->cipher_iv.offset);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "BPI: src before post-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst before post-process:",
-					dst, last_block_len);
-#endif
-		bpi_cipher_encrypt(last_block, dst, iv, block_len,
-				last_block_len, ctx->bpi_ctx);
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "BPI: src after post-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst after post-process:", dst,
-				last_block_len);
-#endif
-	}
-	return sym_op->cipher.data.length - last_block_len;
-}
-
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,
@@ -659,39 +566,3 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 #endif
 	return 0;
 }
-
-int
-qat_sym_process_response(void **op, uint8_t *resp,
-		__rte_unused void *op_cookie,
-		__rte_unused enum qat_device_gen qat_dev_gen)
-{
-
-	struct icp_qat_fw_comn_resp *resp_msg =
-			(struct icp_qat_fw_comn_resp *)resp;
-	struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
-			(resp_msg->opaque_data);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-	rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
-			sizeof(struct icp_qat_fw_comn_resp));
-#endif
-
-	if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
-			ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
-			resp_msg->comn_hdr.comn_status)) {
-
-		rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
-	} else {
-		struct qat_sym_session *sess = (struct qat_sym_session *)
-						get_session_private_data(
-						rx_op->sym->session,
-						cryptodev_qat_driver_id);
-
-		if (sess->bpi_ctx)
-			qat_bpicipher_postprocess(sess, rx_op);
-		rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
-	}
-	*op = (void *)rx_op;
-
-	return 0;
-}
diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h
index bdb672be4..2eaba931e 100644
--- a/drivers/crypto/qat/qat_sym.h
+++ b/drivers/crypto/qat/qat_sym.h
@@ -6,8 +6,17 @@
 #define _QAT_SYM_H_
 
 #include <rte_cryptodev_pmd.h>
+#include <openssl/evp.h>
 
 #include "qat_common.h"
+#include "qat_sym_session.h"
+#include "qat_sym_pmd.h"
+
+#define BYTE_LENGTH    8
+/* bpi is only used for partial blocks of DES and AES
+ * so AES block len can be assumed as max len for iv, src and dst
+ */
+#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
 
 struct qat_sym_session;
 
@@ -21,9 +30,123 @@ struct qat_sym_op_cookie {
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		void *op_cookie, enum qat_device_gen qat_dev_gen);
-int
-qat_sym_process_response(void **op, uint8_t *resp,
-		__rte_unused void *op_cookie,
-		__rte_unused enum qat_device_gen qat_dev_gen);
 
+
+/** Encrypt a single partial block
+ *  Depends on openssl libcrypto
+ *  Uses ECB+XOR to do CFB encryption, same result, more performant
+ */
+static inline int
+bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt the IV, then XOR this with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_encrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_encrypt_err:
+	PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
+	return -EINVAL;
+}
+
+static inline uint32_t
+qat_bpicipher_postprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len > 0 &&
+			ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
+
+		/* Encrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset;
+
+		last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely(sym_op->m_dst != NULL))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = dst - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+		rte_hexdump(stdout, "BPI: src before post-process:", last_block,
+			last_block_len);
+		if (sym_op->m_dst != NULL)
+			rte_hexdump(stdout, "BPI: dst before post-process:",
+					dst, last_block_len);
+#endif
+		bpi_cipher_encrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+		rte_hexdump(stdout, "BPI: src after post-process:", last_block,
+			last_block_len);
+		if (sym_op->m_dst != NULL)
+			rte_hexdump(stdout, "BPI: dst after post-process:", dst,
+				last_block_len);
+#endif
+	}
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static inline void
+qat_sym_process_response(void **op, uint8_t *resp)
+{
+
+	struct icp_qat_fw_comn_resp *resp_msg =
+			(struct icp_qat_fw_comn_resp *)resp;
+	struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
+			(resp_msg->opaque_data);
+
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+	rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
+			sizeof(struct icp_qat_fw_comn_resp));
+#endif
+
+	if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
+			ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
+			resp_msg->comn_hdr.comn_status)) {
+
+		rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+	} else {
+		struct qat_sym_session *sess = (struct qat_sym_session *)
+						get_session_private_data(
+						rx_op->sym->session,
+						cryptodev_qat_driver_id);
+
+
+		if (sess->bpi_ctx)
+			qat_bpicipher_postprocess(sess, rx_op);
+		rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	}
+	*op = (void *)rx_op;
+}
 #endif /* _QAT_SYM_H_ */
diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
index 28e579b77..6b39b32f8 100644
--- a/drivers/crypto/qat/qat_sym_pmd.c
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -160,7 +160,6 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 	qat_qp_conf.hw = qp_hw_data;
 	qat_qp_conf.build_request = qat_sym_build_request;
-	qat_qp_conf.process_response = qat_sym_process_response;
 	qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
 	qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
 	qat_qp_conf.socket_id = socket_id;
-- 
2.13.6



More information about the dev mailing list