@@ -34,48 +34,56 @@
#define __INCLUDE_HASH_FUNC_H__
static inline uint64_t
-hash_xor_key8(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key8(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0;
- xor0 = seed ^ k[0];
+ xor0 = seed ^ (k[0] & m[0]);
return (xor0 >> 32) ^ xor0;
}
static inline uint64_t
-hash_xor_key16(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key16(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0;
- xor0 = (k[0] ^ seed) ^ k[1];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
return (xor0 >> 32) ^ xor0;
}
static inline uint64_t
-hash_xor_key24(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key24(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0;
- xor0 = (k[0] ^ seed) ^ k[1];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
- xor0 ^= k[2];
+ xor0 ^= k[2] & m[2];
return (xor0 >> 32) ^ xor0;
}
static inline uint64_t
-hash_xor_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key32(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0, xor1;
- xor0 = (k[0] ^ seed) ^ k[1];
- xor1 = k[2] ^ k[3];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+ xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
xor0 ^= xor1;
@@ -83,30 +91,34 @@ hash_xor_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_xor_key40(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key40(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0, xor1;
- xor0 = (k[0] ^ seed) ^ k[1];
- xor1 = k[2] ^ k[3];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+ xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
xor0 ^= xor1;
- xor0 ^= k[4];
+ xor0 ^= k[4] & m[4];
return (xor0 >> 32) ^ xor0;
}
static inline uint64_t
-hash_xor_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key48(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0, xor1, xor2;
- xor0 = (k[0] ^ seed) ^ k[1];
- xor1 = k[2] ^ k[3];
- xor2 = k[4] ^ k[5];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+ xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+ xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
xor0 ^= xor1;
@@ -116,17 +128,19 @@ hash_xor_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_xor_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key56(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0, xor1, xor2;
- xor0 = (k[0] ^ seed) ^ k[1];
- xor1 = k[2] ^ k[3];
- xor2 = k[4] ^ k[5];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+ xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+ xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
xor0 ^= xor1;
- xor2 ^= k[6];
+ xor2 ^= k[6] & m[6];
xor0 ^= xor2;
@@ -134,15 +148,17 @@ hash_xor_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_xor_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_xor_key64(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t xor0, xor1, xor2, xor3;
- xor0 = (k[0] ^ seed) ^ k[1];
- xor1 = k[2] ^ k[3];
- xor2 = k[4] ^ k[5];
- xor3 = k[6] ^ k[7];
+ xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
+ xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
+ xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
+ xor3 = (k[6] & m[6]) ^ (k[7] & m[7]);
xor0 ^= xor1;
xor2 ^= xor3;
@@ -152,31 +168,35 @@ hash_xor_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed)
return (xor0 >> 32) ^ xor0;
}
-#if defined(RTE_ARCH_X86_64)
+#if defined(RTE_ARCH_X86_64) && defined(RTE_MACHINE_CPUFLAG_SSE4_2)
#include <x86intrin.h>
static inline uint64_t
-hash_crc_key8(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t crc0;
- crc0 = _mm_crc32_u64(seed, k[0]);
+ crc0 = _mm_crc32_u64(seed, k[0] & m[0]);
return crc0;
}
static inline uint64_t
-hash_crc_key16(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, crc0, crc1;
- k0 = k[0];
+ k0 = k[0] & m[0];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
crc0 ^= crc1;
@@ -184,16 +204,18 @@ hash_crc_key16(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_crc_key24(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, k2, crc0, crc1;
- k0 = k[0];
- k2 = k[2];
+ k0 = k[0] & m[0];
+ k2 = k[2] & m[2];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
crc0 = _mm_crc32_u64(crc0, k2);
@@ -203,18 +225,20 @@ hash_crc_key24(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_crc_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, k2, crc0, crc1, crc2, crc3;
- k0 = k[0];
- k2 = k[2];
+ k0 = k[0] & m[0];
+ k2 = k[2] & m[2];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
- crc2 = _mm_crc32_u64(k2, k[3]);
+ crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
crc3 = k2 >> 32;
crc0 = _mm_crc32_u64(crc0, crc1);
@@ -226,19 +250,21 @@ hash_crc_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_crc_key40(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, k2, crc0, crc1, crc2, crc3;
- k0 = k[0];
- k2 = k[2];
+ k0 = k[0] & m[0];
+ k2 = k[2] & m[2];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
- crc2 = _mm_crc32_u64(k2, k[3]);
- crc3 = _mm_crc32_u64(k2 >> 32, k[4]);
+ crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+ crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
crc0 = _mm_crc32_u64(crc0, crc1);
crc1 = _mm_crc32_u64(crc2, crc3);
@@ -249,20 +275,22 @@ hash_crc_key40(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_crc_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, k2, k5, crc0, crc1, crc2, crc3;
- k0 = k[0];
- k2 = k[2];
- k5 = k[5];
+ k0 = k[0] & m[0];
+ k2 = k[2] & m[2];
+ k5 = k[5] & m[5];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
- crc2 = _mm_crc32_u64(k2, k[3]);
- crc3 = _mm_crc32_u64(k2 >> 32, k[4]);
+ crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+ crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
crc1 = _mm_crc32_u64(crc3, k5);
@@ -273,22 +301,24 @@ hash_crc_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_crc_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
- k0 = k[0];
- k2 = k[2];
- k5 = k[5];
+ k0 = k[0] & m[0];
+ k2 = k[2] & m[2];
+ k5 = k[5] & m[5];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
- crc2 = _mm_crc32_u64(k2, k[3]);
- crc3 = _mm_crc32_u64(k2 >> 32, k[4]);
+ crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+ crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
- crc4 = _mm_crc32_u64(k5, k[6]);
+ crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
crc5 = k5 >> 32;
crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
@@ -300,23 +330,25 @@ hash_crc_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed)
}
static inline uint64_t
-hash_crc_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed)
+hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
+ uint64_t seed)
{
uint64_t *k = key;
+ uint64_t *m = mask;
uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
- k0 = k[0];
- k2 = k[2];
- k5 = k[5];
+ k0 = k[0] & m[0];
+ k2 = k[2] & m[2];
+ k5 = k[5] & m[5];
crc0 = _mm_crc32_u64(k0, seed);
- crc1 = _mm_crc32_u64(k0 >> 32, k[1]);
+ crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
- crc2 = _mm_crc32_u64(k2, k[3]);
- crc3 = _mm_crc32_u64(k2 >> 32, k[4]);
+ crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
+ crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
- crc4 = _mm_crc32_u64(k5, k[6]);
- crc5 = _mm_crc32_u64(k5 >> 32, k[7]);
+ crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
+ crc5 = _mm_crc32_u64(k5 >> 32, k[7] & m[7]);
crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
@@ -88,8 +88,11 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in,
uint32_t *signature)
{
uint8_t buffer[PIPELINE_FC_FLOW_KEY_MAX_SIZE];
+ uint8_t m[PIPELINE_FC_FLOW_KEY_MAX_SIZE]; /* key mask */
void *key_buffer = (key_out) ? key_out : buffer;
+ memset(m, 0xFF, sizeof(m));
+
switch (key_in->type) {
case FLOW_KEY_QINQ:
{
@@ -101,7 +104,7 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in,
qinq->cvlan = rte_cpu_to_be_16(key_in->key.qinq.cvlan);
if (signature)
- *signature = (uint32_t) hash_default_key8(qinq, 8, 0);
+ *signature = (uint32_t) hash_default_key8(qinq, m, 8, 0);
return 0;
}
@@ -118,7 +121,7 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in,
ipv4->port_dst = rte_cpu_to_be_16(key_in->key.ipv4_5tuple.port_dst);
if (signature)
- *signature = (uint32_t) hash_default_key16(ipv4, 16, 0);
+ *signature = (uint32_t) hash_default_key16(ipv4, m, 16, 0);
return 0;
}
@@ -136,7 +139,7 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in,
ipv6->port_dst = rte_cpu_to_be_16(key_in->key.ipv6_5tuple.port_dst);
if (signature)
- *signature = (uint32_t) hash_default_key64(ipv6, 64, 0);
+ *signature = (uint32_t) hash_default_key64(ipv6, m, 64, 0);
return 0;
}
@@ -832,12 +835,11 @@ app_pipeline_fc_add_bulk(struct app_params *app,
}
/* Free resources */
- app_msg_free(app, rsp);
-
for (i = rsp->n_keys; i < n_keys; i++)
if (new_flow[i])
rte_free(flow[i]);
+ app_msg_free(app, rsp);
rte_free(flow_rsp);
rte_free(flow_req);
rte_free(new_flow);
@@ -492,40 +492,16 @@ static void *pipeline_fc_init(struct pipeline_params *params,
/* Tables */
p->n_tables = 1;
{
- struct rte_table_hash_key8_ext_params
- table_hash_key8_params = {
- .n_entries = p_fc->n_flows,
- .n_entries_ext = p_fc->n_flows,
- .signature_offset = p_fc->hash_offset,
+ struct rte_table_hash_params table_hash_params = {
+ .name = p->name,
+ .key_size = p_fc->key_size,
.key_offset = p_fc->key_offset,
- .f_hash = hash_func[(p_fc->key_size / 8) - 1],
.key_mask = (p_fc->key_mask_present) ?
p_fc->key_mask : NULL,
- .seed = 0,
- };
-
- struct rte_table_hash_key16_ext_params
- table_hash_key16_params = {
- .n_entries = p_fc->n_flows,
- .n_entries_ext = p_fc->n_flows,
- .signature_offset = p_fc->hash_offset,
- .key_offset = p_fc->key_offset,
- .f_hash = hash_func[(p_fc->key_size / 8) - 1],
- .key_mask = (p_fc->key_mask_present) ?
- p_fc->key_mask : NULL,
- .seed = 0,
- };
-
- struct rte_table_hash_ext_params
- table_hash_params = {
- .key_size = p_fc->key_size,
.n_keys = p_fc->n_flows,
.n_buckets = p_fc->n_flows / 4,
- .n_buckets_ext = p_fc->n_flows / 4,
.f_hash = hash_func[(p_fc->key_size / 8) - 1],
.seed = 0,
- .signature_offset = p_fc->hash_offset,
- .key_offset = p_fc->key_offset,
};
struct rte_pipeline_table_params table_params = {
@@ -542,32 +518,19 @@ static void *pipeline_fc_init(struct pipeline_params *params,
switch (p_fc->key_size) {
case 8:
- if (p_fc->hash_offset != 0) {
- table_params.ops =
- &rte_table_hash_key8_ext_ops;
- } else {
- table_params.ops =
- &rte_table_hash_key8_ext_dosig_ops;
- }
- table_params.arg_create = &table_hash_key8_params;
+ table_params.ops = &rte_table_hash_key8_ext_ops;
break;
case 16:
- if (p_fc->hash_offset != 0) {
- table_params.ops =
- &rte_table_hash_key16_ext_ops;
- } else {
- table_params.ops =
- &rte_table_hash_key16_ext_dosig_ops;
- }
- table_params.arg_create = &table_hash_key16_params;
+ table_params.ops = &rte_table_hash_key16_ext_ops;
break;
default:
table_params.ops = &rte_table_hash_ext_ops;
- table_params.arg_create = &table_hash_params;
}
+ table_params.arg_create = &table_hash_params;
+
status = rte_pipeline_table_create(p->p,
&table_params,
&p->table_id[0]);
@@ -76,7 +76,7 @@ static pipeline_msg_req_handler handlers[] = {
pipeline_msg_req_invalid_handler,
};
-static __rte_always_inline void
+static inline __attribute__((always_inline)) void
pkt_work_dma(
struct rte_mbuf *pkt,
void *arg,
@@ -102,7 +102,7 @@ pkt_work_dma(
/* Read (dma_dst), compute (hash), write (hash) */
if (hash_enabled) {
- uint32_t hash = p->f_hash(dma_dst, dma_size, 0);
+ uint32_t hash = p->f_hash(dma_src, dma_mask, dma_size, 0);
*dma_hash = hash;
if (lb_hash) {
@@ -121,7 +121,7 @@ pkt_work_dma(
}
}
-static __rte_always_inline void
+static inline __attribute__((always_inline)) void
pkt4_work_dma(
struct rte_mbuf **pkts,
void *arg,
@@ -173,10 +173,10 @@ pkt4_work_dma(
/* Read (dma_dst), compute (hash), write (hash) */
if (hash_enabled) {
- uint32_t hash0 = p->f_hash(dma_dst0, dma_size, 0);
- uint32_t hash1 = p->f_hash(dma_dst1, dma_size, 0);
- uint32_t hash2 = p->f_hash(dma_dst2, dma_size, 0);
- uint32_t hash3 = p->f_hash(dma_dst3, dma_size, 0);
+ uint32_t hash0 = p->f_hash(dma_src0, dma_mask, dma_size, 0);
+ uint32_t hash1 = p->f_hash(dma_src1, dma_mask, dma_size, 0);
+ uint32_t hash2 = p->f_hash(dma_src2, dma_mask, dma_size, 0);
+ uint32_t hash3 = p->f_hash(dma_src3, dma_mask, dma_size, 0);
*dma_hash0 = hash0;
*dma_hash1 = hash1;
@@ -217,7 +217,7 @@ pkt4_work_dma(
}
}
-static __rte_always_inline void
+static inline __attribute__((always_inline)) void
pkt_work_swap(
struct rte_mbuf *pkt,
void *arg)
@@ -241,7 +241,7 @@ pkt_work_swap(
}
}
-static __rte_always_inline void
+static inline __attribute__((always_inline)) void
pkt4_work_swap(
struct rte_mbuf **pkts,
void *arg)
@@ -191,7 +191,7 @@ struct layout {
dst->c = src->c; \
}
-static __rte_always_inline void
+static inline __attribute__((always_inline)) void
pkt_work_routing(
struct rte_mbuf *pkt,
struct rte_pipeline_table_entry *table_entry,
@@ -317,7 +317,7 @@ pkt_work_routing(
}
}
-static __rte_always_inline void
+static inline __attribute__((always_inline)) void
pkt4_work_routing(
struct rte_mbuf **pkts,
struct rte_pipeline_table_entry **table_entries,
@@ -1349,17 +1349,19 @@ pipeline_routing_init(struct pipeline_params *params,
/* ARP table configuration */
if (p_rt->params.n_arp_entries) {
- struct rte_table_hash_key8_ext_params table_arp_params = {
- .n_entries = p_rt->params.n_arp_entries,
- .n_entries_ext = p_rt->params.n_arp_entries,
+ struct rte_table_hash_params table_arp_params = {
+ .name = p->name,
+ .key_size = 8,
+ .key_offset = p_rt->params.arp_key_offset,
+ .key_mask = NULL,
+ .n_keys = p_rt->params.n_arp_entries,
+ .n_buckets = p_rt->params.n_arp_entries / 4,
.f_hash = hash_default_key8,
.seed = 0,
- .signature_offset = 0, /* Unused */
- .key_offset = p_rt->params.arp_key_offset,
};
struct rte_pipeline_table_params table_params = {
- .ops = &rte_table_hash_key8_ext_dosig_ops,
+ .ops = &rte_table_hash_key8_ext_ops,
.arg_create = &table_arp_params,
.f_action_hit = get_arp_table_ah_hit(p_rt),
.f_action_miss = NULL,