deleted file mode 100644
@@ -1,302 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015 Intel Corporation
- */
-
-#ifndef _AESNI_MB_OPS_H_
-#define _AESNI_MB_OPS_H_
-
-#ifndef LINUX
-#define LINUX
-#endif
-
-#include <intel-ipsec-mb.h>
-
-/*
- * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
- * so if macro is not defined, it means that the version is 0.49.
- */
-#if !defined(IMB_VERSION_NUM)
-#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
-#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
-#endif
-
-enum aesni_mb_vector_mode {
- RTE_AESNI_MB_NOT_SUPPORTED = 0,
- RTE_AESNI_MB_SSE,
- RTE_AESNI_MB_AVX,
- RTE_AESNI_MB_AVX2,
- RTE_AESNI_MB_AVX512
-};
-
-typedef void (*md5_one_block_t)(const void *data, void *digest);
-
-typedef void (*sha1_one_block_t)(const void *data, void *digest);
-typedef void (*sha224_one_block_t)(const void *data, void *digest);
-typedef void (*sha256_one_block_t)(const void *data, void *digest);
-typedef void (*sha384_one_block_t)(const void *data, void *digest);
-typedef void (*sha512_one_block_t)(const void *data, void *digest);
-
-typedef void (*aes_keyexp_128_t)
- (const void *key, void *enc_exp_keys, void *dec_exp_keys);
-typedef void (*aes_keyexp_192_t)
- (const void *key, void *enc_exp_keys, void *dec_exp_keys);
-typedef void (*aes_keyexp_256_t)
- (const void *key, void *enc_exp_keys, void *dec_exp_keys);
-typedef void (*aes_xcbc_expand_key_t)
- (const void *key, void *exp_k1, void *k2, void *k3);
-typedef void (*aes_cmac_sub_key_gen_t)
- (const void *exp_key, void *k2, void *k3);
-typedef void (*aes_cmac_keyexp_t)
- (const void *key, void *keyexp);
-typedef void (*aes_gcm_keyexp_t)
- (const void *key, struct gcm_key_data *keyexp);
-
-/** Multi-buffer library function pointer table */
-struct aesni_mb_op_fns {
- struct {
- init_mb_mgr_t init_mgr;
- /**< Initialise scheduler */
- get_next_job_t get_next;
- /**< Get next free job structure */
- submit_job_t submit;
- /**< Submit job to scheduler */
- get_completed_job_t get_completed_job;
- /**< Get completed job */
- flush_job_t flush_job;
- /**< flush jobs from manager */
- } job;
- /**< multi buffer manager functions */
-
- struct {
- struct {
- md5_one_block_t md5;
- /**< MD5 one block hash */
- sha1_one_block_t sha1;
- /**< SHA1 one block hash */
- sha224_one_block_t sha224;
- /**< SHA224 one block hash */
- sha256_one_block_t sha256;
- /**< SHA256 one block hash */
- sha384_one_block_t sha384;
- /**< SHA384 one block hash */
- sha512_one_block_t sha512;
- /**< SHA512 one block hash */
- } one_block;
- /**< one block hash functions */
-
- struct {
- aes_keyexp_128_t aes128;
- /**< AES128 key expansions */
- aes_keyexp_192_t aes192;
- /**< AES192 key expansions */
- aes_keyexp_256_t aes256;
- /**< AES256 key expansions */
- aes_xcbc_expand_key_t aes_xcbc;
- /**< AES XCBC key epansions */
- aes_cmac_sub_key_gen_t aes_cmac_subkey;
- /**< AES CMAC subkey expansions */
- aes_cmac_keyexp_t aes_cmac_expkey;
- /**< AES CMAC key expansions */
- aes_gcm_keyexp_t aes_gcm_128;
- /**< AES GCM 128 key expansions */
- aes_gcm_keyexp_t aes_gcm_192;
- /**< AES GCM 192 key expansions */
- aes_gcm_keyexp_t aes_gcm_256;
- /**< AES GCM 256 key expansions */
- } keyexp;
- /**< Key expansion functions */
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
- struct {
- hash_fn_t sha1;
- hash_fn_t sha224;
- hash_fn_t sha256;
- hash_fn_t sha384;
- hash_fn_t sha512;
- } multi_block;
- /** multi block hash functions */
-#endif
- } aux;
- /**< Auxiliary functions */
-};
-
-
-static const struct aesni_mb_op_fns job_ops[] = {
- [RTE_AESNI_MB_NOT_SUPPORTED] = {
- .job = {
- NULL
- },
- .aux = {
- .one_block = {
- NULL
- },
- .keyexp = {
- NULL
- },
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
- .multi_block = {
- NULL
- }
-#endif
-
- }
- },
- [RTE_AESNI_MB_SSE] = {
- .job = {
- init_mb_mgr_sse,
- get_next_job_sse,
- submit_job_sse,
- get_completed_job_sse,
- flush_job_sse
- },
- .aux = {
- .one_block = {
- md5_one_block_sse,
- sha1_one_block_sse,
- sha224_one_block_sse,
- sha256_one_block_sse,
- sha384_one_block_sse,
- sha512_one_block_sse
- },
- .keyexp = {
- aes_keyexp_128_sse,
- aes_keyexp_192_sse,
- aes_keyexp_256_sse,
- aes_xcbc_expand_key_sse,
- aes_cmac_subkey_gen_sse,
- aes_keyexp_128_enc_sse,
- aes_gcm_pre_128_sse,
- aes_gcm_pre_192_sse,
- aes_gcm_pre_256_sse
- },
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
- .multi_block = {
- sha1_sse,
- sha224_sse,
- sha256_sse,
- sha384_sse,
- sha512_sse
- }
-#endif
- }
- },
- [RTE_AESNI_MB_AVX] = {
- .job = {
- init_mb_mgr_avx,
- get_next_job_avx,
- submit_job_avx,
- get_completed_job_avx,
- flush_job_avx
- },
- .aux = {
- .one_block = {
- md5_one_block_avx,
- sha1_one_block_avx,
- sha224_one_block_avx,
- sha256_one_block_avx,
- sha384_one_block_avx,
- sha512_one_block_avx
- },
- .keyexp = {
- aes_keyexp_128_avx,
- aes_keyexp_192_avx,
- aes_keyexp_256_avx,
- aes_xcbc_expand_key_avx,
- aes_cmac_subkey_gen_avx,
- aes_keyexp_128_enc_avx,
- aes_gcm_pre_128_avx_gen2,
- aes_gcm_pre_192_avx_gen2,
- aes_gcm_pre_256_avx_gen2
- },
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
- .multi_block = {
- sha1_avx,
- sha224_avx,
- sha256_avx,
- sha384_avx,
- sha512_avx
- }
-#endif
- }
- },
- [RTE_AESNI_MB_AVX2] = {
- .job = {
- init_mb_mgr_avx2,
- get_next_job_avx2,
- submit_job_avx2,
- get_completed_job_avx2,
- flush_job_avx2
- },
- .aux = {
- .one_block = {
- md5_one_block_avx2,
- sha1_one_block_avx2,
- sha224_one_block_avx2,
- sha256_one_block_avx2,
- sha384_one_block_avx2,
- sha512_one_block_avx2
- },
- .keyexp = {
- aes_keyexp_128_avx2,
- aes_keyexp_192_avx2,
- aes_keyexp_256_avx2,
- aes_xcbc_expand_key_avx2,
- aes_cmac_subkey_gen_avx2,
- aes_keyexp_128_enc_avx2,
- aes_gcm_pre_128_avx_gen4,
- aes_gcm_pre_192_avx_gen4,
- aes_gcm_pre_256_avx_gen4
- },
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
- .multi_block = {
- sha1_avx2,
- sha224_avx2,
- sha256_avx2,
- sha384_avx2,
- sha512_avx2
- }
-#endif
- }
- },
- [RTE_AESNI_MB_AVX512] = {
- .job = {
- init_mb_mgr_avx512,
- get_next_job_avx512,
- submit_job_avx512,
- get_completed_job_avx512,
- flush_job_avx512
- },
- .aux = {
- .one_block = {
- md5_one_block_avx512,
- sha1_one_block_avx512,
- sha224_one_block_avx512,
- sha256_one_block_avx512,
- sha384_one_block_avx512,
- sha512_one_block_avx512
- },
- .keyexp = {
- aes_keyexp_128_avx512,
- aes_keyexp_192_avx512,
- aes_keyexp_256_avx512,
- aes_xcbc_expand_key_avx512,
- aes_cmac_subkey_gen_avx512,
- aes_keyexp_128_enc_avx512,
- aes_gcm_pre_128_avx_gen4,
- aes_gcm_pre_192_avx_gen4,
- aes_gcm_pre_256_avx_gen4
- },
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
- .multi_block = {
- sha1_avx512,
- sha224_avx512,
- sha256_avx512,
- sha384_avx512,
- sha512_avx512
- }
-#endif
- }
- }
-};
-
-
-#endif /* _AESNI_MB_OPS_H_ */
@@ -100,7 +100,7 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
/** Set session authentication parameters */
static int
-aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
@@ -135,13 +135,16 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
- (*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data,
+
+ IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
sess->auth.xcbc.k1_expanded,
sess->auth.xcbc.k2, sess->auth.xcbc.k3);
return 0;
}
if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
+ uint32_t dust[4*15];
+
sess->auth.algo = AES_CMAC;
uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
@@ -169,10 +172,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
sess->auth.gen_digest_len = cmac_digest_len;
else
sess->auth.gen_digest_len = sess->auth.req_digest_len;
- (*mb_ops->aux.keyexp.aes_cmac_expkey)(xform->auth.key.data,
- sess->auth.cmac.expkey);
- (*mb_ops->aux.keyexp.aes_cmac_subkey)(sess->auth.cmac.expkey,
+ IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
+ sess->auth.cmac.expkey, dust);
+ IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
sess->auth.cmac.skey1, sess->auth.cmac.skey2);
return 0;
}
@@ -180,14 +183,14 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
switch (xform->auth.algo) {
case RTE_CRYPTO_AUTH_MD5_HMAC:
sess->auth.algo = MD5;
- hash_oneblock_fn = mb_ops->aux.one_block.md5;
+ hash_oneblock_fn = mb_mgr->md5_one_block;
break;
case RTE_CRYPTO_AUTH_SHA1_HMAC:
sess->auth.algo = SHA1;
- hash_oneblock_fn = mb_ops->aux.one_block.sha1;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+ hash_oneblock_fn = mb_mgr->sha1_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
- mb_ops->aux.multi_block.sha1(
+ IMB_SHA1(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
hashed_key);
@@ -197,10 +200,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
break;
case RTE_CRYPTO_AUTH_SHA224_HMAC:
sess->auth.algo = SHA_224;
- hash_oneblock_fn = mb_ops->aux.one_block.sha224;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+ hash_oneblock_fn = mb_mgr->sha224_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
- mb_ops->aux.multi_block.sha224(
+ IMB_SHA224(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
hashed_key);
@@ -210,10 +213,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
break;
case RTE_CRYPTO_AUTH_SHA256_HMAC:
sess->auth.algo = SHA_256;
- hash_oneblock_fn = mb_ops->aux.one_block.sha256;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+ hash_oneblock_fn = mb_mgr->sha256_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
- mb_ops->aux.multi_block.sha256(
+ IMB_SHA256(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
hashed_key);
@@ -223,10 +226,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
break;
case RTE_CRYPTO_AUTH_SHA384_HMAC:
sess->auth.algo = SHA_384;
- hash_oneblock_fn = mb_ops->aux.one_block.sha384;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+ hash_oneblock_fn = mb_mgr->sha384_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
- mb_ops->aux.multi_block.sha384(
+ IMB_SHA384(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
hashed_key);
@@ -236,10 +239,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
break;
case RTE_CRYPTO_AUTH_SHA512_HMAC:
sess->auth.algo = SHA_512;
- hash_oneblock_fn = mb_ops->aux.one_block.sha512;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+ hash_oneblock_fn = mb_mgr->sha512_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
- mb_ops->aux.multi_block.sha512(
+ IMB_SHA512(mb_mgr,
xform->auth.key.data,
xform->auth.key.length,
hashed_key);
@@ -292,13 +295,12 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
/** Set session cipher parameters */
static int
-aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
uint8_t is_aes = 0;
uint8_t is_3DES = 0;
- aes_keyexp_t aes_keyexp_fn;
if (xform == NULL) {
sess->cipher.mode = NULL_CIPHER;
@@ -361,26 +363,26 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
switch (xform->cipher.key.length) {
case AES_128_BYTES:
sess->cipher.key_length_in_bytes = AES_128_BYTES;
- aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
+ IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
+ sess->cipher.expanded_aes_keys.encode,
+ sess->cipher.expanded_aes_keys.decode);
break;
case AES_192_BYTES:
sess->cipher.key_length_in_bytes = AES_192_BYTES;
- aes_keyexp_fn = mb_ops->aux.keyexp.aes192;
+ IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
+ sess->cipher.expanded_aes_keys.encode,
+ sess->cipher.expanded_aes_keys.decode);
break;
case AES_256_BYTES:
sess->cipher.key_length_in_bytes = AES_256_BYTES;
- aes_keyexp_fn = mb_ops->aux.keyexp.aes256;
+ IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
+ sess->cipher.expanded_aes_keys.encode,
+ sess->cipher.expanded_aes_keys.decode);
break;
default:
AESNI_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
-
- /* Expanded cipher keys */
- (*aes_keyexp_fn)(xform->cipher.key.data,
- sess->cipher.expanded_aes_keys.encode,
- sess->cipher.expanded_aes_keys.decode);
-
} else if (is_3DES) {
uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
sess->cipher.exp_3des_keys.key[1],
@@ -388,9 +390,12 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
switch (xform->cipher.key.length) {
case 24:
- des_key_schedule(keys[0], xform->cipher.key.data);
- des_key_schedule(keys[1], xform->cipher.key.data+8);
- des_key_schedule(keys[2], xform->cipher.key.data+16);
+ IMB_DES_KEYSCHED(mb_mgr, keys[0],
+ xform->cipher.key.data);
+ IMB_DES_KEYSCHED(mb_mgr, keys[1],
+ xform->cipher.key.data + 8);
+ IMB_DES_KEYSCHED(mb_mgr, keys[2],
+ xform->cipher.key.data + 16);
/* Initialize keys - 24 bytes: [K1-K2-K3] */
sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
@@ -398,16 +403,18 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
break;
case 16:
- des_key_schedule(keys[0], xform->cipher.key.data);
- des_key_schedule(keys[1], xform->cipher.key.data+8);
-
+ IMB_DES_KEYSCHED(mb_mgr, keys[0],
+ xform->cipher.key.data);
+ IMB_DES_KEYSCHED(mb_mgr, keys[1],
+ xform->cipher.key.data + 8);
/* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
break;
case 8:
- des_key_schedule(keys[0], xform->cipher.key.data);
+ IMB_DES_KEYSCHED(mb_mgr, keys[0],
+ xform->cipher.key.data);
/* Initialize keys - 8 bytes: [K1 = K2 = K3] */
sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
@@ -431,9 +438,11 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
}
sess->cipher.key_length_in_bytes = 8;
- des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.encode,
+ IMB_DES_KEYSCHED(mb_mgr,
+ (uint64_t *)sess->cipher.expanded_aes_keys.encode,
xform->cipher.key.data);
- des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.decode,
+ IMB_DES_KEYSCHED(mb_mgr,
+ (uint64_t *)sess->cipher.expanded_aes_keys.decode,
xform->cipher.key.data);
}
@@ -441,15 +450,10 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
}
static int
-aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
- union {
- aes_keyexp_t aes_keyexp_fn;
- aes_gcm_keyexp_t aes_gcm_keyexp_fn;
- } keyexp;
-
switch (xform->aead.op) {
case RTE_CRYPTO_AEAD_OP_ENCRYPT:
sess->cipher.direction = ENCRYPT;
@@ -473,17 +477,15 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
switch (xform->aead.key.length) {
case AES_128_BYTES:
sess->cipher.key_length_in_bytes = AES_128_BYTES;
- keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
+ IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
+ sess->cipher.expanded_aes_keys.encode,
+ sess->cipher.expanded_aes_keys.decode);
break;
default:
AESNI_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
- /* Expanded cipher keys */
- (*keyexp.aes_keyexp_fn)(xform->aead.key.data,
- sess->cipher.expanded_aes_keys.encode,
- sess->cipher.expanded_aes_keys.decode);
break;
case RTE_CRYPTO_AEAD_AES_GCM:
@@ -493,26 +495,24 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
switch (xform->aead.key.length) {
case AES_128_BYTES:
sess->cipher.key_length_in_bytes = AES_128_BYTES;
- keyexp.aes_gcm_keyexp_fn =
- mb_ops->aux.keyexp.aes_gcm_128;
+ IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
+ &sess->cipher.gcm_key);
break;
case AES_192_BYTES:
sess->cipher.key_length_in_bytes = AES_192_BYTES;
- keyexp.aes_gcm_keyexp_fn =
- mb_ops->aux.keyexp.aes_gcm_192;
+ IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
+ &sess->cipher.gcm_key);
break;
case AES_256_BYTES:
sess->cipher.key_length_in_bytes = AES_256_BYTES;
- keyexp.aes_gcm_keyexp_fn =
- mb_ops->aux.keyexp.aes_gcm_256;
+ IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
+ &sess->cipher.gcm_key);
break;
default:
AESNI_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
- (keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data,
- &sess->cipher.gcm_key);
break;
default:
@@ -539,7 +539,7 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
/** Parse crypto xform chain and set private session parameters */
int
-aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform)
{
@@ -598,13 +598,13 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
/* Default IV length = 0 */
sess->iv.length = 0;
- ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform);
+ ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
if (ret != 0) {
AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
return ret;
}
- ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess,
+ ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
cipher_xform);
if (ret != 0) {
AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
@@ -612,7 +612,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
}
if (aead_xform) {
- ret = aesni_mb_set_session_aead_parameters(mb_ops, sess,
+ ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
aead_xform);
if (ret != 0) {
AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
@@ -673,7 +673,7 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
sess = (struct aesni_mb_session *)_sess_private_data;
- if (unlikely(aesni_mb_set_session_parameters(qp->op_fns,
+ if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
sess, op->sym->xform) != 0)) {
rte_mempool_put(qp->sess_mp, _sess);
rte_mempool_put(qp->sess_mp, _sess_private_data);
@@ -989,7 +989,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
if (processed_jobs == nb_ops)
break;
- job = (*qp->op_fns->job.get_completed_job)(qp->mb_mgr);
+ job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
}
return processed_jobs;
@@ -1002,7 +1002,7 @@ flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
int processed_ops = 0;
/* Flush the remaining jobs */
- JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(qp->mb_mgr);
+ JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
if (job)
processed_ops += handle_completed_jobs(qp, job,
@@ -1042,7 +1042,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
uint8_t digest_idx = qp->digest_idx;
do {
/* Get next free mb job struct from mb manager */
- job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
+ job = IMB_GET_NEXT_JOB(qp->mb_mgr);
if (unlikely(job == NULL)) {
/* if no free mb job structs we need to flush mb_mgr */
processed_jobs += flush_mb_mgr(qp,
@@ -1052,7 +1052,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
if (nb_ops == processed_jobs)
break;
- job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
+ job = IMB_GET_NEXT_JOB(qp->mb_mgr);
}
/*
@@ -1072,8 +1072,11 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
}
/* Submit job to multi-buffer for processing */
- job = (*qp->op_fns->job.submit)(qp->mb_mgr);
-
+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
+ job = IMB_SUBMIT_JOB(qp->mb_mgr);
+#else
+ job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
+#endif
/*
* If submit returns a processed job then handle it,
* before submitting subsequent jobs
@@ -1105,6 +1108,7 @@ cryptodev_aesni_mb_create(const char *name,
struct rte_cryptodev *dev;
struct aesni_mb_private *internals;
enum aesni_mb_vector_mode vector_mode;
+ MB_MGR *mb_mgr;
/* Check CPU for support for AES instruction set */
if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
@@ -1139,18 +1143,26 @@ cryptodev_aesni_mb_create(const char *name,
RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
RTE_CRYPTODEV_FF_CPU_AESNI;
+ mb_mgr = alloc_mb_mgr(0);
+ if (mb_mgr == NULL)
+ return -ENOMEM;
+
switch (vector_mode) {
case RTE_AESNI_MB_SSE:
dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
+ init_mb_mgr_sse(mb_mgr);
break;
case RTE_AESNI_MB_AVX:
dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
+ init_mb_mgr_avx(mb_mgr);
break;
case RTE_AESNI_MB_AVX2:
dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
+ init_mb_mgr_avx2(mb_mgr);
break;
case RTE_AESNI_MB_AVX512:
dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
+ init_mb_mgr_avx512(mb_mgr);
break;
default:
break;
@@ -1161,6 +1173,7 @@ cryptodev_aesni_mb_create(const char *name,
internals->vector_mode = vector_mode;
internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
+ internals->mb_mgr = mb_mgr;
#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
@@ -1204,6 +1217,7 @@ static int
cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
{
struct rte_cryptodev *cryptodev;
+ struct aesni_mb_private *internals;
const char *name;
name = rte_vdev_device_name(vdev);
@@ -1214,6 +1228,10 @@ cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
if (cryptodev == NULL)
return -ENODEV;
+ internals = cryptodev->data->dev_private;
+
+ free_mb_mgr(internals->mb_mgr);
+
return rte_cryptodev_pmd_destroy(cryptodev);
}
@@ -509,8 +509,6 @@ aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
r = rte_ring_lookup(qp->name);
if (r)
rte_ring_free(r);
- if (qp->mb_mgr)
- free_mb_mgr(qp->mb_mgr);
rte_free(qp);
dev->data->queue_pairs[qp_id] = NULL;
}
@@ -589,13 +587,7 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
goto qp_setup_cleanup;
- qp->mb_mgr = alloc_mb_mgr(0);
- if (qp->mb_mgr == NULL) {
- ret = -ENOMEM;
- goto qp_setup_cleanup;
- }
-
- qp->op_fns = &job_ops[internals->vector_mode];
+ qp->mb_mgr = internals->mb_mgr;
qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
qp_conf->nb_descriptors, socket_id);
@@ -612,9 +604,6 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
"digest_mp_%u_%u", dev->data->dev_id, qp_id);
-
- /* Initialise multi-buffer manager */
- (*qp->op_fns->job.init_mgr)(qp->mb_mgr);
return 0;
qp_setup_cleanup:
@@ -663,7 +652,7 @@ aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
return -ENOMEM;
}
- ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode],
+ ret = aesni_mb_set_session_parameters(internals->mb_mgr,
sess_private_data, xform);
if (ret != 0) {
AESNI_MB_LOG(ERR, "failed configure session parameters");
@@ -5,7 +5,24 @@
#ifndef _RTE_AESNI_MB_PMD_PRIVATE_H_
#define _RTE_AESNI_MB_PMD_PRIVATE_H_
-#include "aesni_mb_ops.h"
+#include <intel-ipsec-mb.h>
+
+/*
+ * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
+ * so if macro is not defined, it means that the version is 0.49.
+ */
+#if !defined(IMB_VERSION_NUM)
+#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
+#endif
+
+enum aesni_mb_vector_mode {
+ RTE_AESNI_MB_NOT_SUPPORTED = 0,
+ RTE_AESNI_MB_SSE,
+ RTE_AESNI_MB_AVX,
+ RTE_AESNI_MB_AVX2,
+ RTE_AESNI_MB_AVX512
+};
#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
/**< AES-NI Multi buffer PMD device name */
@@ -83,7 +100,9 @@ static const unsigned auth_digest_byte_lengths[] = {
[AES_XCBC] = 16,
[AES_CMAC] = 16,
[AES_GMAC] = 12,
- [NULL_HASH] = 0
+ [NULL_HASH] = 0,
+ /**< Vector mode dependent pointer table of the multi-buffer APIs */
+
};
/**
@@ -115,6 +134,8 @@ struct aesni_mb_private {
/**< CPU vector instruction set mode */
unsigned max_nb_queue_pairs;
/**< Max number of queue pairs supported by device */
+ MB_MGR *mb_mgr;
+ /**< Multi-buffer instance */
};
/** AESNI Multi buffer queue pair */
@@ -123,8 +144,6 @@ struct aesni_mb_qp {
/**< Queue Pair Identifier */
char name[RTE_CRYPTODEV_NAME_MAX_LEN];
/**< Unique Queue Pair Name */
- const struct aesni_mb_op_fns *op_fns;
- /**< Vector mode dependent pointer table of the multi-buffer APIs */
MB_MGR *mb_mgr;
/**< Multi-buffer instance */
struct rte_ring *ingress_queue;
@@ -153,7 +172,9 @@ struct aesni_mb_session {
} iv;
/**< IV parameters */
- /** Cipher Parameters */
+ /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
+ /**< Vector mode dependent pointer table of the multi-buffer APIs */
+
struct {
/** Cipher direction - encrypt / decrypt */
JOB_CIPHER_DIRECTION direction;
@@ -238,7 +259,7 @@ struct aesni_mb_session {
*
*/
extern int
-aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
struct aesni_mb_session *sess,
const struct rte_crypto_sym_xform *xform);