@@ -492,18 +492,6 @@ static void *pipeline_fc_init(struct pipeline_params *params,
/* Tables */
p->n_tables = 1;
{
- struct rte_table_hash_key8_ext_params
- table_hash_key8_params = {
- .n_entries = p_fc->n_flows,
- .n_entries_ext = p_fc->n_flows,
- .signature_offset = p_fc->hash_offset,
- .key_offset = p_fc->key_offset,
- .f_hash = hash_func[(p_fc->key_size / 8) - 1],
- .key_mask = (p_fc->key_mask_present) ?
- p_fc->key_mask : NULL,
- .seed = 0,
- };
-
struct rte_table_hash_key16_ext_params
table_hash_key16_params = {
.n_entries = p_fc->n_flows,
@@ -543,7 +531,7 @@ static void *pipeline_fc_init(struct pipeline_params *params,
switch (p_fc->key_size) {
case 8:
table_params.ops = &rte_table_hash_key8_ext_ops;
- table_params.arg_create = &table_hash_key8_params;
+ table_params.arg_create = &table_hash_params;
break;
case 16:
old mode 100644
new mode 100755
@@ -1349,13 +1349,15 @@ pipeline_routing_init(struct pipeline_params *params,
/* ARP table configuration */
if (p_rt->params.n_arp_entries) {
- struct rte_table_hash_key8_ext_params table_arp_params = {
- .n_entries = p_rt->params.n_arp_entries,
- .n_entries_ext = p_rt->params.n_arp_entries,
- .f_hash = hash_default_key8,
- .seed = 0,
- .signature_offset = 0, /* Unused */
+ struct rte_table_hash_params table_arp_params = {
+ .name = p->name,
+ .key_size = 8,
.key_offset = p_rt->params.arp_key_offset,
+ .key_mask = NULL,
+ .n_keys = p_rt->params.n_arp_entries,
+ .n_buckets = p_rt->params.n_arp_entries / 4,
+ .f_hash = (rte_table_hash_op_hash)hash_default_key8,
+ .seed = 0,
};
struct rte_pipeline_table_params table_params = {
@@ -139,62 +139,8 @@ extern struct rte_table_ops rte_table_hash_ext_ops;
extern struct rte_table_ops rte_table_hash_lru_ops;
-/**
- * 8-byte key hash tables
- *
- */
-/** LRU hash table parameters */
-struct rte_table_hash_key8_lru_params {
- /** Maximum number of entries (and keys) in the table */
- uint32_t n_entries;
-
- /** Hash function */
- rte_table_hash_op_hash_nomask f_hash;
-
- /** Seed for the hash function */
- uint64_t seed;
-
- /** Byte offset within packet meta-data where the 4-byte key signature
- is located. Valid for pre-computed key signature tables, ignored for
- do-sig tables. */
- uint32_t signature_offset;
-
- /** Byte offset within packet meta-data where the key is located */
- uint32_t key_offset;
-
- /** Bit-mask to be AND-ed to the key on lookup */
- uint8_t *key_mask;
-};
-
extern struct rte_table_ops rte_table_hash_key8_lru_ops;
-/** Extendible bucket hash table parameters */
-struct rte_table_hash_key8_ext_params {
- /** Maximum number of entries (and keys) in the table */
- uint32_t n_entries;
-
- /** Number of entries (and keys) for hash table bucket extensions. Each
- bucket is extended in increments of 4 keys. */
- uint32_t n_entries_ext;
-
- /** Hash function */
- rte_table_hash_op_hash_nomask f_hash;
-
- /** Seed for the hash function */
- uint64_t seed;
-
- /** Byte offset within packet meta-data where the 4-byte key signature
- is located. Valid for pre-computed key signature tables, ignored for
- do-sig tables. */
- uint32_t signature_offset;
-
- /** Byte offset within packet meta-data where the key is located */
- uint32_t key_offset;
-
- /** Bit-mask to be AND-ed to the key on lookup */
- uint8_t *key_mask;
-};
-
extern struct rte_table_ops rte_table_hash_key8_ext_ops;
/**
@@ -42,7 +42,9 @@
#include "rte_table_hash.h"
#include "rte_lru.h"
-#define RTE_TABLE_HASH_KEY_SIZE 8
+#define KEY_SIZE 8
+
+#define KEYS_PER_BUCKET 4
#ifdef RTE_TABLE_STATS_COLLECT
@@ -76,14 +78,12 @@ struct rte_table_hash {
/* Input parameters */
uint32_t n_buckets;
- uint32_t n_entries_per_bucket;
uint32_t key_size;
uint32_t entry_size;
uint32_t bucket_size;
- uint32_t signature_offset;
uint32_t key_offset;
uint64_t key_mask;
- rte_table_hash_op_hash_nomask f_hash;
+ rte_table_hash_op_hash f_hash;
uint64_t seed;
/* Extendible buckets */
@@ -96,10 +96,46 @@ struct rte_table_hash {
};
static int
-check_params_create_lru(struct rte_table_hash_key8_lru_params *params) {
- /* n_entries */
- if (params->n_entries == 0) {
- RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__);
+keycmp(void *a, void *b, void *b_mask)
+{
+ uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask;
+
+ return a64[0] != (b64[0] & b_mask64[0]);
+}
+
+static void
+keycpy(void *dst, void *src, void *src_mask)
+{
+ uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask;
+
+ dst64[0] = src64[0] & src_mask64[0];
+}
+
+static int
+check_params_create(struct rte_table_hash_params *params)
+{
+ /* name */
+ if (params->name == NULL) {
+ RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__);
+ return -EINVAL;
+ }
+
+ /* key_size */
+ if (params->key_size != KEY_SIZE) {
+ RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__);
+ return -EINVAL;
+ }
+
+ /* n_keys */
+ if (params->n_keys == 0) {
+ RTE_LOG(ERR, TABLE, "%s: n_keys is zero\n", __func__);
+ return -EINVAL;
+ }
+
+ /* n_buckets */
+ if ((params->n_buckets == 0) ||
+ (!rte_is_power_of_2(params->n_buckets))) {
+ RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__);
return -EINVAL;
}
@@ -116,47 +152,68 @@ check_params_create_lru(struct rte_table_hash_key8_lru_params *params) {
static void *
rte_table_hash_create_key8_lru(void *params, int socket_id, uint32_t entry_size)
{
- struct rte_table_hash_key8_lru_params *p =
- (struct rte_table_hash_key8_lru_params *) params;
+ struct rte_table_hash_params *p = params;
struct rte_table_hash *f;
- uint32_t n_buckets, n_entries_per_bucket, key_size, bucket_size_cl;
- uint32_t total_size, i;
+ uint64_t bucket_size, total_size;
+ uint32_t n_buckets, i;
/* Check input parameters */
- if ((check_params_create_lru(p) != 0) ||
+ if ((check_params_create(p) != 0) ||
((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
- ((sizeof(struct rte_bucket_4_8) % 64) != 0)) {
+ ((sizeof(struct rte_bucket_4_8) % 64) != 0))
return NULL;
- }
- n_entries_per_bucket = 4;
- key_size = 8;
+
+ /*
+ * Table dimensioning
+ *
+ * Objective: Pick the number of buckets (n_buckets) so that there a chance
+ * to store n_keys keys in the table.
+ *
+ * Note: Since the buckets do not get extended, it is not possible to
+ * guarantee that n_keys keys can be stored in the table at any time. In the
+ * worst case scenario when all the n_keys fall into the same bucket, only
+ * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case
+ * defeats the purpose of the hash table. It indicates unsuitable f_hash or
+ * n_keys to n_buckets ratio.
+ *
+ * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET
+ */
+ n_buckets = rte_align32pow2(
+ (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET);
+ n_buckets = RTE_MAX(n_buckets, p->n_buckets);
/* Memory allocation */
- n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) /
- n_entries_per_bucket);
- bucket_size_cl = (sizeof(struct rte_bucket_4_8) + n_entries_per_bucket *
- entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE;
- total_size = sizeof(struct rte_table_hash) + n_buckets *
- bucket_size_cl * RTE_CACHE_LINE_SIZE;
-
- f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
+ bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_8) +
+ KEYS_PER_BUCKET * entry_size);
+ total_size = sizeof(struct rte_table_hash) + n_buckets * bucket_size;
+
+ if (total_size > SIZE_MAX) {
+ RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes"
+ " for hash table %s\n",
+ __func__, total_size, p->name);
+ return NULL;
+ }
+
+ f = rte_zmalloc_socket(p->name,
+ (size_t)total_size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
if (f == NULL) {
- RTE_LOG(ERR, TABLE,
- "%s: Cannot allocate %u bytes for hash table\n",
- __func__, total_size);
+ RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes"
+ " for hash table %s\n",
+ __func__, total_size, p->name);
return NULL;
}
- RTE_LOG(INFO, TABLE,
- "%s: Hash table memory footprint is %u bytes\n",
- __func__, total_size);
+
+ RTE_LOG(INFO, TABLE, "%s: Hash table %s memory footprint "
+ "is %" PRIu64 " bytes\n",
+ __func__, p->name, total_size);
/* Memory initialization */
f->n_buckets = n_buckets;
- f->n_entries_per_bucket = n_entries_per_bucket;
- f->key_size = key_size;
+ f->key_size = KEY_SIZE;
f->entry_size = entry_size;
- f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE;
- f->signature_offset = p->signature_offset;
+ f->bucket_size = bucket_size;
f->key_offset = p->key_offset;
f->f_hash = p->f_hash;
f->seed = p->seed;
@@ -205,7 +262,7 @@ rte_table_hash_entry_add_key8_lru(
uint64_t signature, mask, pos;
uint32_t bucket_index, i;
- signature = f->f_hash(key, f->key_size, f->seed);
+ signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed);
bucket_index = signature & (f->n_buckets - 1);
bucket = (struct rte_bucket_4_8 *)
&f->memory[bucket_index * f->bucket_size];
@@ -213,10 +270,10 @@ rte_table_hash_entry_add_key8_lru(
/* Key is present in the bucket */
for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) {
uint64_t bucket_signature = bucket->signature;
- uint64_t bucket_key = bucket->key[i];
+ uint64_t *bucket_key = &bucket->key[i];
if ((bucket_signature & mask) &&
- (*((uint64_t *) key) == bucket_key)) {
+ (keycmp(bucket_key, key, &f->key_mask) == 0)) {
uint8_t *bucket_data = &bucket->data[i * f->entry_size];
memcpy(bucket_data, entry, f->entry_size);
@@ -235,7 +292,7 @@ rte_table_hash_entry_add_key8_lru(
uint8_t *bucket_data = &bucket->data[i * f->entry_size];
bucket->signature |= mask;
- bucket->key[i] = *((uint64_t *) key);
+ keycpy(&bucket->key[i], key, &f->key_mask);
memcpy(bucket_data, entry, f->entry_size);
lru_update(bucket, i);
*key_found = 0;
@@ -247,10 +304,10 @@ rte_table_hash_entry_add_key8_lru(
/* Bucket full: replace LRU entry */
pos = lru_pos(bucket);
- bucket->key[pos] = *((uint64_t *) key);
+ keycpy(&bucket->key[pos], key, &f->key_mask);
memcpy(&bucket->data[pos * f->entry_size], entry, f->entry_size);
lru_update(bucket, pos);
- *key_found = 0;
+ *key_found = 0;
*entry_ptr = (void *) &bucket->data[pos * f->entry_size];
return 0;
@@ -268,7 +325,7 @@ rte_table_hash_entry_delete_key8_lru(
uint64_t signature, mask;
uint32_t bucket_index, i;
- signature = f->f_hash(key, f->key_size, f->seed);
+ signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed);
bucket_index = signature & (f->n_buckets - 1);
bucket = (struct rte_bucket_4_8 *)
&f->memory[bucket_index * f->bucket_size];
@@ -276,10 +333,10 @@ rte_table_hash_entry_delete_key8_lru(
/* Key is present in the bucket */
for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) {
uint64_t bucket_signature = bucket->signature;
- uint64_t bucket_key = bucket->key[i];
+ uint64_t *bucket_key = &bucket->key[i];
if ((bucket_signature & mask) &&
- (*((uint64_t *) key) == bucket_key)) {
+ (keycmp(bucket_key, key, &f->key_mask) == 0)) {
uint8_t *bucket_data = &bucket->data[i * f->entry_size];
bucket->signature &= ~mask;
@@ -296,79 +353,71 @@ rte_table_hash_entry_delete_key8_lru(
return 0;
}
-static int
-check_params_create_ext(struct rte_table_hash_key8_ext_params *params) {
- /* n_entries */
- if (params->n_entries == 0) {
- RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__);
- return -EINVAL;
- }
-
- /* n_entries_ext */
- if (params->n_entries_ext == 0) {
- RTE_LOG(ERR, TABLE, "%s: n_entries_ext is zero\n", __func__);
- return -EINVAL;
- }
-
- /* f_hash */
- if (params->f_hash == NULL) {
- RTE_LOG(ERR, TABLE, "%s: f_hash function pointer is NULL\n",
- __func__);
- return -EINVAL;
- }
-
- return 0;
-}
-
static void *
rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size)
{
- struct rte_table_hash_key8_ext_params *p =
- (struct rte_table_hash_key8_ext_params *) params;
+ struct rte_table_hash_params *p = params;
struct rte_table_hash *f;
- uint32_t n_buckets, n_buckets_ext, n_entries_per_bucket, key_size;
- uint32_t bucket_size_cl, stack_size_cl, total_size, i;
+ uint64_t bucket_size, stack_size, total_size;
+ uint32_t n_buckets_ext, i;
/* Check input parameters */
- if ((check_params_create_ext(p) != 0) ||
+ if ((check_params_create(p) != 0) ||
((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) ||
((sizeof(struct rte_bucket_4_8) % 64) != 0))
return NULL;
- n_entries_per_bucket = 4;
- key_size = 8;
+ /*
+ * Table dimensioning
+ *
+ * Objective: Pick the number of bucket extensions (n_buckets_ext) so that
+ * it is guaranteed that n_keys keys can be stored in the table at any time.
+ *
+ * The worst case scenario takes place when all the n_keys keys fall into
+ * the same bucket. Actually, due to the KEYS_PER_BUCKET scheme, the worst
+ * case takes place when (n_keys - KEYS_PER_BUCKET + 1) keys fall into the
+ * same bucket, while the remaining (KEYS_PER_BUCKET - 1) keys each fall
+ * into a different bucket. This case defeats the purpose of the hash table.
+ * It indicates unsuitable f_hash or n_keys to n_buckets ratio.
+ *
+ * n_buckets_ext = n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1
+ */
+ n_buckets_ext = p->n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1;
/* Memory allocation */
- n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) /
- n_entries_per_bucket);
- n_buckets_ext = (p->n_entries_ext + n_entries_per_bucket - 1) /
- n_entries_per_bucket;
- bucket_size_cl = (sizeof(struct rte_bucket_4_8) + n_entries_per_bucket *
- entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE;
- stack_size_cl = (n_buckets_ext * sizeof(uint32_t) + RTE_CACHE_LINE_SIZE - 1)
- / RTE_CACHE_LINE_SIZE;
- total_size = sizeof(struct rte_table_hash) + ((n_buckets +
- n_buckets_ext) * bucket_size_cl + stack_size_cl) *
- RTE_CACHE_LINE_SIZE;
-
- f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
+ bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_8) +
+ KEYS_PER_BUCKET * entry_size);
+ stack_size = RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(uint32_t));
+ total_size = sizeof(struct rte_table_hash) +
+ (p->n_buckets + n_buckets_ext) * bucket_size + stack_size;
+
+ if (total_size > SIZE_MAX) {
+ RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes "
+ "for hash table %s\n",
+ __func__, total_size, p->name);
+ return NULL;
+ }
+
+ f = rte_zmalloc_socket(p->name,
+ (size_t)total_size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
if (f == NULL) {
RTE_LOG(ERR, TABLE,
- "%s: Cannot allocate %u bytes for hash table\n",
- __func__, total_size);
+ "%s: Cannot allocate %" PRIu64 " bytes "
+ "for hash table %s\n",
+ __func__, total_size, p->name);
return NULL;
}
- RTE_LOG(INFO, TABLE,
- "%s: Hash table memory footprint is %u bytes\n",
- __func__, total_size);
+ RTE_LOG(INFO, TABLE, "%s: Hash table %s memory footprint "
+ "is %" PRIu64 " bytes\n",
+ __func__, p->name, total_size);
/* Memory initialization */
- f->n_buckets = n_buckets;
- f->n_entries_per_bucket = n_entries_per_bucket;
- f->key_size = key_size;
+ f->n_buckets = p->n_buckets;
+ f->key_size = KEY_SIZE;
f->entry_size = entry_size;
- f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE;
- f->signature_offset = p->signature_offset;
+ f->bucket_size = bucket_size;
f->key_offset = p->key_offset;
f->f_hash = p->f_hash;
f->seed = p->seed;
@@ -376,7 +425,7 @@ rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size)
f->n_buckets_ext = n_buckets_ext;
f->stack_pos = n_buckets_ext;
f->stack = (uint32_t *)
- &f->memory[(n_buckets + n_buckets_ext) * f->bucket_size];
+ &f->memory[(p->n_buckets + n_buckets_ext) * f->bucket_size];
if (p->key_mask != NULL)
f->key_mask = ((uint64_t *)p->key_mask)[0];
@@ -417,7 +466,7 @@ rte_table_hash_entry_add_key8_ext(
uint64_t signature;
uint32_t bucket_index, i;
- signature = f->f_hash(key, f->key_size, f->seed);
+ signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed);
bucket_index = signature & (f->n_buckets - 1);
bucket0 = (struct rte_bucket_4_8 *)
&f->memory[bucket_index * f->bucket_size];
@@ -428,10 +477,10 @@ rte_table_hash_entry_add_key8_ext(
for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) {
uint64_t bucket_signature = bucket->signature;
- uint64_t bucket_key = bucket->key[i];
+ uint64_t *bucket_key = &bucket->key[i];
if ((bucket_signature & mask) &&
- (*((uint64_t *) key) == bucket_key)) {
+ (keycmp(bucket_key, key, &f->key_mask) == 0)) {
uint8_t *bucket_data = &bucket->data[i *
f->entry_size];
@@ -456,7 +505,7 @@ rte_table_hash_entry_add_key8_ext(
f->entry_size];
bucket->signature |= mask;
- bucket->key[i] = *((uint64_t *) key);
+ keycpy(&bucket->key[i], key, &f->key_mask);
memcpy(bucket_data, entry, f->entry_size);
*key_found = 0;
*entry_ptr = (void *) bucket_data;
@@ -476,7 +525,7 @@ rte_table_hash_entry_add_key8_ext(
bucket_prev->next_valid = 1;
bucket->signature = 1;
- bucket->key[0] = *((uint64_t *) key);
+ keycpy(&bucket->key[0], key, &f->key_mask);
memcpy(&bucket->data[0], entry, f->entry_size);
*key_found = 0;
*entry_ptr = (void *) &bucket->data[0];
@@ -498,7 +547,7 @@ rte_table_hash_entry_delete_key8_ext(
uint64_t signature;
uint32_t bucket_index, i;
- signature = f->f_hash(key, f->key_size, f->seed);
+ signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed);
bucket_index = signature & (f->n_buckets - 1);
bucket0 = (struct rte_bucket_4_8 *)
&f->memory[bucket_index * f->bucket_size];
@@ -510,10 +559,10 @@ rte_table_hash_entry_delete_key8_ext(
for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) {
uint64_t bucket_signature = bucket->signature;
- uint64_t bucket_key = bucket->key[i];
+ uint64_t *bucket_key = &bucket->key[i];
if ((bucket_signature & mask) &&
- (*((uint64_t *) key) == bucket_key)) {
+ (keycmp(bucket_key, key, &f->key_mask) == 0)) {
uint8_t *bucket_data = &bucket->data[i *
f->entry_size];
@@ -546,16 +595,17 @@ rte_table_hash_entry_delete_key8_ext(
return 0;
}
-#define lookup_key8_cmp(key_in, bucket, pos) \
+#define lookup_key8_cmp(key_in, bucket, pos, f) \
{ \
- uint64_t xor[4], signature; \
+ uint64_t xor[4], signature, k; \
\
signature = ~bucket->signature; \
\
- xor[0] = (key_in[0] ^ bucket->key[0]) | (signature & 1);\
- xor[1] = (key_in[0] ^ bucket->key[1]) | (signature & 2);\
- xor[2] = (key_in[0] ^ bucket->key[2]) | (signature & 4);\
- xor[3] = (key_in[0] ^ bucket->key[3]) | (signature & 8);\
+ k = key_in[0] & f->key_mask; \
+ xor[0] = (k ^ bucket->key[0]) | (signature & 1); \
+ xor[1] = (k ^ bucket->key[1]) | (signature & 2); \
+ xor[2] = (k ^ bucket->key[2]) | (signature & 4); \
+ xor[3] = (k ^ bucket->key[3]) | (signature & 8); \
\
pos = 4; \
if (xor[0] == 0) \
@@ -586,12 +636,9 @@ rte_table_hash_entry_delete_key8_ext(
uint64_t *key; \
uint64_t signature; \
uint32_t bucket_index; \
- uint64_t hash_key_buffer; \
\
key = RTE_MBUF_METADATA_UINT64_PTR(mbuf1, f->key_offset);\
- hash_key_buffer = *key & f->key_mask; \
- signature = f->f_hash(&hash_key_buffer, \
- RTE_TABLE_HASH_KEY_SIZE, f->seed); \
+ signature = f->f_hash(key, &f->key_mask, KEY_SIZE, f->seed); \
bucket_index = signature & (f->n_buckets - 1); \
bucket1 = (struct rte_bucket_4_8 *) \
&f->memory[bucket_index * f->bucket_size]; \
@@ -605,12 +652,9 @@ rte_table_hash_entry_delete_key8_ext(
uint64_t pkt_mask; \
uint64_t *key; \
uint32_t pos; \
- uint64_t hash_key_buffer; \
\
key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\
- hash_key_buffer = key[0] & f->key_mask; \
- \
- lookup_key8_cmp((&hash_key_buffer), bucket2, pos); \
+ lookup_key8_cmp(key, bucket2, pos, f); \
\
pkt_mask = ((bucket2->signature >> pos) & 1LLU) << pkt2_index;\
pkts_mask_out |= pkt_mask; \
@@ -629,12 +673,9 @@ rte_table_hash_entry_delete_key8_ext(
uint64_t pkt_mask, bucket_mask; \
uint64_t *key; \
uint32_t pos; \
- uint64_t hash_key_buffer; \
\
key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\
- hash_key_buffer = *key & f->key_mask; \
- \
- lookup_key8_cmp((&hash_key_buffer), bucket2, pos); \
+ lookup_key8_cmp(key, bucket2, pos, f); \
\
pkt_mask = ((bucket2->signature >> pos) & 1LLU) << pkt2_index;\
pkts_mask_out |= pkt_mask; \
@@ -658,13 +699,10 @@ rte_table_hash_entry_delete_key8_ext(
uint64_t pkt_mask, bucket_mask; \
uint64_t *key; \
uint32_t pos; \
- uint64_t hash_key_buffer; \
\
bucket = buckets[pkt_index]; \
key = keys[pkt_index]; \
- hash_key_buffer = (*key) & f->key_mask; \
- \
- lookup_key8_cmp((&hash_key_buffer), bucket, pos); \
+ lookup_key8_cmp(key, bucket, pos, f); \
\
pkt_mask = ((bucket->signature >> pos) & 1LLU) << pkt_index;\
pkts_mask_out |= pkt_mask; \
@@ -729,28 +767,22 @@ rte_table_hash_entry_delete_key8_ext(
#define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f)\
{ \
uint64_t *key10, *key11; \
- uint64_t hash_offset_buffer10; \
- uint64_t hash_offset_buffer11; \
uint64_t signature10, signature11; \
uint32_t bucket10_index, bucket11_index; \
- rte_table_hash_op_hash_nomask f_hash = f->f_hash; \
+ rte_table_hash_op_hash f_hash = f->f_hash; \
uint64_t seed = f->seed; \
uint32_t key_offset = f->key_offset; \
\
key10 = RTE_MBUF_METADATA_UINT64_PTR(mbuf10, key_offset);\
key11 = RTE_MBUF_METADATA_UINT64_PTR(mbuf11, key_offset);\
- hash_offset_buffer10 = *key10 & f->key_mask; \
- hash_offset_buffer11 = *key11 & f->key_mask; \
\
- signature10 = f_hash(&hash_offset_buffer10, \
- RTE_TABLE_HASH_KEY_SIZE, seed); \
+ signature10 = f_hash(key10, &f->key_mask, KEY_SIZE, seed); \
bucket10_index = signature10 & (f->n_buckets - 1); \
bucket10 = (struct rte_bucket_4_8 *) \
&f->memory[bucket10_index * f->bucket_size]; \
rte_prefetch0(bucket10); \
\
- signature11 = f_hash(&hash_offset_buffer11, \
- RTE_TABLE_HASH_KEY_SIZE, seed); \
+ signature11 = f_hash(key11, &f->key_mask, KEY_SIZE, seed); \
bucket11_index = signature11 & (f->n_buckets - 1); \
bucket11 = (struct rte_bucket_4_8 *) \
&f->memory[bucket11_index * f->bucket_size]; \
@@ -763,17 +795,13 @@ rte_table_hash_entry_delete_key8_ext(
void *a20, *a21; \
uint64_t pkt20_mask, pkt21_mask; \
uint64_t *key20, *key21; \
- uint64_t hash_offset_buffer20; \
- uint64_t hash_offset_buffer21; \
uint32_t pos20, pos21; \
\
key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\
key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\
- hash_offset_buffer20 = *key20 & f->key_mask; \
- hash_offset_buffer21 = *key21 & f->key_mask; \
\
- lookup_key8_cmp((&hash_offset_buffer20), bucket20, pos20);\
- lookup_key8_cmp((&hash_offset_buffer21), bucket21, pos21);\
+ lookup_key8_cmp(key20, bucket20, pos20, f); \
+ lookup_key8_cmp(key21, bucket21, pos21, f); \
\
pkt20_mask = ((bucket20->signature >> pos20) & 1LLU) << pkt20_index;\
pkt21_mask = ((bucket21->signature >> pos21) & 1LLU) << pkt21_index;\
@@ -796,17 +824,13 @@ rte_table_hash_entry_delete_key8_ext(
void *a20, *a21; \
uint64_t pkt20_mask, pkt21_mask, bucket20_mask, bucket21_mask;\
uint64_t *key20, *key21; \
- uint64_t hash_offset_buffer20; \
- uint64_t hash_offset_buffer21; \
uint32_t pos20, pos21; \
\
key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\
key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\
- hash_offset_buffer20 = *key20 & f->key_mask; \
- hash_offset_buffer21 = *key21 & f->key_mask; \
\
- lookup_key8_cmp((&hash_offset_buffer20), bucket20, pos20);\
- lookup_key8_cmp((&hash_offset_buffer21), bucket21, pos21);\
+ lookup_key8_cmp(key20, bucket20, pos20, f); \
+ lookup_key8_cmp(key21, bucket21, pos21, f); \
\
pkt20_mask = ((bucket20->signature >> pos20) & 1LLU) << pkt20_index;\
pkt21_mask = ((bucket21->signature >> pos21) & 1LLU) << pkt21_index;\
@@ -954,7 +978,7 @@ rte_table_hash_lookup_key8_lru(
*lookup_hit_mask = pkts_mask_out;
RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return 0;
-} /* rte_table_hash_lookup_key8_lru() */
+} /* lookup LRU */
static int
rte_table_hash_lookup_key8_ext(
@@ -1104,7 +1128,7 @@ rte_table_hash_lookup_key8_ext(
*lookup_hit_mask = pkts_mask_out;
RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out));
return 0;
-} /* rte_table_hash_lookup_key8_ext() */
+} /* lookup EXT */
static int
rte_table_hash_key8_stats_read(void *table, struct rte_table_stats *stats, int clear)
@@ -231,13 +231,14 @@ app_main_loop_worker_pipeline_hash(void) {
case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT:
{
- struct rte_table_hash_key8_ext_params table_hash_params = {
- .n_entries = 1 << 24,
- .n_entries_ext = 1 << 23,
- .signature_offset = APP_METADATA_OFFSET(0),
+ struct rte_table_hash_params table_hash_params = {
+ .name = "TABLE",
+ .key_size = key_size,
.key_offset = APP_METADATA_OFFSET(32),
.key_mask = NULL,
- .f_hash = test_hash,
+ .n_keys = 1 << 24,
+ .n_buckets = 1 << 22,
+ .f_hash = (rte_table_hash_op_hash)test_hash,
.seed = 0,
};
@@ -257,12 +258,14 @@ app_main_loop_worker_pipeline_hash(void) {
case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU:
{
- struct rte_table_hash_key8_lru_params table_hash_params = {
- .n_entries = 1 << 24,
- .signature_offset = APP_METADATA_OFFSET(0),
+ struct rte_table_hash_params table_hash_params = {
+ .name = "TABLE",
+ .key_size = key_size,
.key_offset = APP_METADATA_OFFSET(32),
.key_mask = NULL,
- .f_hash = test_hash,
+ .n_keys = 1 << 24,
+ .n_buckets = 1 << 22,
+ .f_hash = (rte_table_hash_op_hash)test_hash,
.seed = 0,
};
old mode 100644
new mode 100755
@@ -441,12 +441,15 @@ test_table_hash8lru(void)
int status, i;
/* Traffic flow */
- struct rte_table_hash_key8_lru_params key8lru_params = {
- .n_entries = 1<<24,
- .f_hash = pipeline_test_hash,
- .signature_offset = APP_METADATA_OFFSET(0),
+ struct rte_table_hash_params key8lru_params = {
+ .name = "TABLE",
+ .key_size = 8,
.key_offset = APP_METADATA_OFFSET(32),
.key_mask = NULL,
+ .n_keys = 1 << 16,
+ .n_buckets = 1 << 16,
+ .f_hash = (rte_table_hash_op_hash)pipeline_test_hash,
+ .seed = 0,
};
uint8_t key8lru[8];
@@ -475,14 +478,14 @@ test_table_hash8lru(void)
VERIFY(status, CHECK_TABLE_OK);
/* Invalid parameters */
- key8lru_params.n_entries = 0;
+ key8lru_params.n_keys = 0;
status = test_table_type(&rte_table_hash_key8_lru_ops,
(void *)&key8lru_params, (void *)key8lru, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
- key8lru_params.n_entries = 1<<16;
+ key8lru_params.n_keys = 1<<16;
key8lru_params.f_hash = NULL;
status = test_table_type(&rte_table_hash_key8_lru_ops,
@@ -616,14 +619,15 @@ test_table_hash8ext(void)
int status, i;
/* Traffic flow */
- struct rte_table_hash_key8_ext_params key8ext_params = {
- .n_entries = 1<<16,
- .n_entries_ext = 1<<15,
- .f_hash = pipeline_test_hash,
- .seed = 0,
- .signature_offset = APP_METADATA_OFFSET(0),
+ struct rte_table_hash_params key8ext_params = {
+ .name = "TABLE",
+ .key_size = 8,
.key_offset = APP_METADATA_OFFSET(32),
.key_mask = NULL,
+ .n_keys = 1 << 16,
+ .n_buckets = 1 << 16,
+ .f_hash = (rte_table_hash_op_hash)pipeline_test_hash,
+ .seed = 0,
};
uint8_t key8ext[8];
@@ -652,14 +656,14 @@ test_table_hash8ext(void)
VERIFY(status, CHECK_TABLE_OK);
/* Invalid parameters */
- key8ext_params.n_entries = 0;
+ key8ext_params.n_keys = 0;
status = test_table_type(&rte_table_hash_key8_ext_ops,
(void *)&key8ext_params, (void *)key8ext, &table_packets,
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
- key8ext_params.n_entries = 1<<16;
+ key8ext_params.n_keys = 1<<16;
key8ext_params.f_hash = NULL;
status = test_table_type(&rte_table_hash_key8_ext_ops,
@@ -667,13 +671,6 @@ test_table_hash8ext(void)
NULL, 0);
VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
- key8ext_params.f_hash = pipeline_test_hash;
- key8ext_params.n_entries_ext = 0;
-
- status = test_table_type(&rte_table_hash_key8_ext_ops,
- (void *)&key8ext_params, (void *)key8ext, &table_packets, NULL, 0);
- VERIFY(status, CHECK_TABLE_TABLE_CONFIG);
-
return 0;
}
old mode 100644
new mode 100755
@@ -67,9 +67,9 @@ unsigned n_table_tests = RTE_DIM(table_tests);
/* Function prototypes */
static int
-test_table_hash_lru_generic(struct rte_table_ops *ops);
+test_table_hash_lru_generic(struct rte_table_ops *ops, uint32_t key_size);
static int
-test_table_hash_ext_generic(struct rte_table_ops *ops);
+test_table_hash_ext_generic(struct rte_table_ops *ops, uint32_t key_size);
struct rte_bucket_4_8 {
/* Cache line 0 */
@@ -655,7 +655,7 @@ test_table_lpm_ipv6(void)
}
static int
-test_table_hash_lru_generic(struct rte_table_ops *ops)
+test_table_hash_lru_generic(struct rte_table_ops *ops, uint32_t key_size)
{
int status, i;
uint64_t expected_mask = 0, result_mask;
@@ -667,43 +667,31 @@ test_table_hash_lru_generic(struct rte_table_ops *ops)
int key_found;
/* Initialize params and create tables */
- struct rte_table_hash_key8_lru_params hash_params = {
- .n_entries = 1 << 10,
- .f_hash = pipeline_test_hash,
- .seed = 0,
- .signature_offset = APP_METADATA_OFFSET(1),
+ struct rte_table_hash_params hash_params = {
+ .name = "TABLE",
+ .key_size = key_size,
.key_offset = APP_METADATA_OFFSET(32),
.key_mask = NULL,
+ .n_keys = 1 << 10,
+ .n_buckets = 1 << 10,
+ .f_hash = (rte_table_hash_op_hash)pipeline_test_hash,
+ .seed = 0,
};
- hash_params.n_entries = 0;
+ hash_params.n_keys = 0;
table = ops->f_create(&hash_params, 0, 1);
if (table != NULL)
return -1;
- hash_params.n_entries = 1 << 10;
- hash_params.signature_offset = APP_METADATA_OFFSET(1);
-
- table = ops->f_create(&hash_params, 0, 1);
- if (table == NULL)
- return -2;
-
- hash_params.signature_offset = APP_METADATA_OFFSET(0);
- hash_params.key_offset = APP_METADATA_OFFSET(1);
-
- table = ops->f_create(&hash_params, 0, 1);
- if (table == NULL)
- return -3;
-
- hash_params.key_offset = APP_METADATA_OFFSET(32);
+ hash_params.n_keys = 1 << 10;
hash_params.f_hash = NULL;
table = ops->f_create(&hash_params, 0, 1);
if (table != NULL)
return -4;
- hash_params.f_hash = pipeline_test_hash;
+ hash_params.f_hash = (rte_table_hash_op_hash)pipeline_test_hash;
table = ops->f_create(&hash_params, 0, 1);
if (table == NULL)
@@ -770,7 +758,7 @@ test_table_hash_lru_generic(struct rte_table_ops *ops)
}
static int
-test_table_hash_ext_generic(struct rte_table_ops *ops)
+test_table_hash_ext_generic(struct rte_table_ops *ops, uint32_t key_size)
{
int status, i;
uint64_t expected_mask = 0, result_mask;
@@ -782,35 +770,24 @@ test_table_hash_ext_generic(struct rte_table_ops *ops)
void *entry_ptr;
/* Initialize params and create tables */
- struct rte_table_hash_key8_ext_params hash_params = {
- .n_entries = 1 << 10,
- .n_entries_ext = 1 << 4,
- .f_hash = pipeline_test_hash,
- .seed = 0,
- .signature_offset = APP_METADATA_OFFSET(1),
+ struct rte_table_hash_params hash_params = {
+ .name = "TABLE",
+ .key_size = key_size,
.key_offset = APP_METADATA_OFFSET(32),
.key_mask = NULL,
+ .n_keys = 1 << 10,
+ .n_buckets = 1 << 10,
+ .f_hash = (rte_table_hash_op_hash)pipeline_test_hash,
+ .seed = 0,
};
- hash_params.n_entries = 0;
+ hash_params.n_keys = 0;
table = ops->f_create(&hash_params, 0, 1);
if (table != NULL)
return -1;
- hash_params.n_entries = 1 << 10;
- hash_params.n_entries_ext = 0;
- table = ops->f_create(&hash_params, 0, 1);
- if (table != NULL)
- return -2;
-
- hash_params.n_entries_ext = 1 << 4;
- hash_params.signature_offset = APP_METADATA_OFFSET(1);
- table = ops->f_create(&hash_params, 0, 1);
- if (table == NULL)
- return -2;
-
- hash_params.signature_offset = APP_METADATA_OFFSET(0);
+ hash_params.n_keys = 1 << 10;
hash_params.key_offset = APP_METADATA_OFFSET(1);
table = ops->f_create(&hash_params, 0, 1);
@@ -824,7 +801,7 @@ test_table_hash_ext_generic(struct rte_table_ops *ops)
if (table != NULL)
return -4;
- hash_params.f_hash = pipeline_test_hash;
+ hash_params.f_hash = (rte_table_hash_op_hash)pipeline_test_hash;
table = ops->f_create(&hash_params, 0, 1);
if (table == NULL)
@@ -895,20 +872,9 @@ test_table_hash_lru(void)
{
int status;
- status = test_table_hash_lru_generic(&rte_table_hash_key8_lru_ops);
- if (status < 0)
- return status;
-
status = test_table_hash_lru_generic(
- &rte_table_hash_key8_lru_ops);
- if (status < 0)
- return status;
-
- status = test_table_hash_lru_generic(&rte_table_hash_key16_lru_ops);
- if (status < 0)
- return status;
-
- status = test_table_hash_lru_generic(&rte_table_hash_key32_lru_ops);
+ &rte_table_hash_key8_lru_ops,
+ 8);
if (status < 0)
return status;
@@ -924,20 +890,7 @@ test_table_hash_ext(void)
{
int status;
- status = test_table_hash_ext_generic(&rte_table_hash_key8_ext_ops);
- if (status < 0)
- return status;
-
- status = test_table_hash_ext_generic(
- &rte_table_hash_key8_ext_ops);
- if (status < 0)
- return status;
-
- status = test_table_hash_ext_generic(&rte_table_hash_key16_ext_ops);
- if (status < 0)
- return status;
-
- status = test_table_hash_ext_generic(&rte_table_hash_key32_ext_ops);
+ status = test_table_hash_ext_generic(&rte_table_hash_key8_ext_ops, 8);
if (status < 0)
return status;