[dpdk-dev] [PATCH v3 09/18] librte_table: rework variable size key lru hash table

Cristian Dumitrescu cristian.dumitrescu at intel.com
Wed Oct 18 17:03:45 CEST 2017


Rework for the variable size key LRU hash table to use the
mask-based hash function and the unified parameter structure.

Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu at intel.com>
---
 lib/librte_table/rte_table_hash.h     |  26 ----
 lib/librte_table/rte_table_hash_lru.c | 277 +++++++++++++++++++++-------------
 test/test-pipeline/pipeline_hash.c    |   9 +-
 3 files changed, 177 insertions(+), 135 deletions(-)

diff --git a/lib/librte_table/rte_table_hash.h b/lib/librte_table/rte_table_hash.h
index 0024b99..fb5f4d7 100644
--- a/lib/librte_table/rte_table_hash.h
+++ b/lib/librte_table/rte_table_hash.h
@@ -137,32 +137,6 @@ typedef uint64_t (*rte_table_hash_op_hash_nomask)(
 
 extern struct rte_table_ops rte_table_hash_ext_ops;
 
-/** LRU hash table parameters */
-struct rte_table_hash_lru_params {
-	/** Key size (number of bytes) */
-	uint32_t key_size;
-
-	/** Maximum number of keys */
-	uint32_t n_keys;
-
-	/** Number of hash table buckets. Each bucket stores up to 4 keys. */
-	uint32_t n_buckets;
-
-	/** Hash function */
-	rte_table_hash_op_hash_nomask f_hash;
-
-	/** Seed value for the hash function */
-	uint64_t seed;
-
-	/** Byte offset within packet meta-data where the 4-byte key signature
-	is located. Valid for pre-computed key signature tables, ignored for
-	do-sig tables. */
-	uint32_t signature_offset;
-
-	/** Byte offset within packet meta-data where the key is located */
-	uint32_t key_offset;
-};
-
 extern struct rte_table_ops rte_table_hash_lru_ops;
 
 /**
diff --git a/lib/librte_table/rte_table_hash_lru.c b/lib/librte_table/rte_table_hash_lru.c
index 61050f3..eebc1cd 100644
--- a/lib/librte_table/rte_table_hash_lru.c
+++ b/lib/librte_table/rte_table_hash_lru.c
@@ -84,9 +84,8 @@ struct rte_table_hash {
 	uint32_t entry_size;
 	uint32_t n_keys;
 	uint32_t n_buckets;
-	rte_table_hash_op_hash_nomask f_hash;
+	rte_table_hash_op_hash f_hash;
 	uint64_t seed;
-	uint32_t signature_offset;
 	uint32_t key_offset;
 
 	/* Internal */
@@ -99,6 +98,7 @@ struct rte_table_hash {
 	struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX];
 
 	/* Tables */
+	uint64_t *key_mask;
 	struct bucket *buckets;
 	uint8_t *key_mem;
 	uint8_t *data_mem;
@@ -109,12 +109,39 @@ struct rte_table_hash {
 };
 
 static int
-check_params_create(struct rte_table_hash_lru_params *params)
+keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes)
 {
-	uint32_t n_buckets_min;
+	uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask;
+	uint32_t i;
+
+	for (i = 0; i < n_bytes / sizeof(uint64_t); i++)
+		if (a64[i] != (b64[i] & b_mask64[i]))
+			return 1;
+
+	return 0;
+}
+
+static void
+keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes)
+{
+	uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask;
+	uint32_t i;
+
+	for (i = 0; i < n_bytes / sizeof(uint64_t); i++)
+		dst64[i] = src64[i] & src_mask64[i];
+}
+
+static int
+check_params_create(struct rte_table_hash_params *params)
+{
+	/* name */
+	if (params->name == NULL) {
+		RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__);
+		return -EINVAL;
+	}
 
 	/* key_size */
-	if ((params->key_size == 0) ||
+	if ((params->key_size < sizeof(uint64_t)) ||
 		(!rte_is_power_of_2(params->key_size))) {
 		RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__);
 		return -EINVAL;
@@ -128,10 +155,8 @@ check_params_create(struct rte_table_hash_lru_params *params)
 	}
 
 	/* n_buckets */
-	n_buckets_min = (params->n_keys + KEYS_PER_BUCKET - 1) / params->n_keys;
 	if ((params->n_buckets == 0) ||
-		(!rte_is_power_of_2(params->n_keys)) ||
-		(params->n_buckets < n_buckets_min)) {
+		(!rte_is_power_of_2(params->n_keys))) {
 		RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__);
 		return -EINVAL;
 	}
@@ -148,13 +173,13 @@ check_params_create(struct rte_table_hash_lru_params *params)
 static void *
 rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size)
 {
-	struct rte_table_hash_lru_params *p =
-		params;
+	struct rte_table_hash_params *p = params;
 	struct rte_table_hash *t;
-	uint32_t total_size, table_meta_sz;
-	uint32_t bucket_sz, key_sz, key_stack_sz, data_sz;
-	uint32_t bucket_offset, key_offset, key_stack_offset, data_offset;
-	uint32_t i;
+	uint64_t table_meta_sz, key_mask_sz, bucket_sz, key_sz, key_stack_sz;
+	uint64_t data_sz, total_size;
+	uint64_t key_mask_offset, bucket_offset, key_offset, key_stack_offset;
+	uint64_t data_offset;
+	uint32_t n_buckets, i;
 
 	/* Check input parameters */
 	if ((check_params_create(p) != 0) ||
@@ -164,33 +189,65 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size)
 		return NULL;
 	}
 
+	/*
+	 * Table dimensioning
+	 *
+	 * Objective: Pick the number of buckets (n_buckets) so that there a chance
+	 * to store n_keys keys in the table.
+	 *
+	 * Note: Since the buckets do not get extended, it is not possible to
+	 * guarantee that n_keys keys can be stored in the table at any time. In the
+	 * worst case scenario when all the n_keys fall into the same bucket, only
+	 * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case
+	 * defeats the purpose of the hash table. It indicates unsuitable f_hash or
+	 * n_keys to n_buckets ratio.
+	 *
+	 * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET
+	 */
+	n_buckets = rte_align32pow2(
+		(p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET);
+	n_buckets = RTE_MAX(n_buckets, p->n_buckets);
+
 	/* Memory allocation */
 	table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash));
-	bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket));
+	key_mask_sz = RTE_CACHE_LINE_ROUNDUP(p->key_size);
+	bucket_sz = RTE_CACHE_LINE_ROUNDUP(n_buckets * sizeof(struct bucket));
 	key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size);
 	key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t));
 	data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size);
-	total_size = table_meta_sz + bucket_sz + key_sz + key_stack_sz +
-		data_sz;
+	total_size = table_meta_sz + key_mask_sz + bucket_sz + key_sz +
+		key_stack_sz + data_sz;
+
+	if (total_size > SIZE_MAX) {
+		RTE_LOG(ERR, TABLE,
+			"%s: Cannot allocate %" PRIu64 " bytes for hash "
+			"table %s\n",
+			__func__, total_size, p->name);
+		return NULL;
+	}
 
-	t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
+	t = rte_zmalloc_socket(p->name,
+		(size_t)total_size,
+		RTE_CACHE_LINE_SIZE,
+		socket_id);
 	if (t == NULL) {
 		RTE_LOG(ERR, TABLE,
-			"%s: Cannot allocate %u bytes for hash table\n",
-			__func__, total_size);
+			"%s: Cannot allocate %" PRIu64 " bytes for hash "
+			"table %s\n",
+			__func__, total_size, p->name);
 		return NULL;
 	}
-	RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table memory footprint is "
-		"%u bytes\n", __func__, p->key_size, total_size);
+	RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table %s memory footprint"
+		" is %" PRIu64 " bytes\n",
+		__func__, p->key_size, p->name, total_size);
 
 	/* Memory initialization */
 	t->key_size = p->key_size;
 	t->entry_size = entry_size;
 	t->n_keys = p->n_keys;
-	t->n_buckets = p->n_buckets;
+	t->n_buckets = n_buckets;
 	t->f_hash = p->f_hash;
 	t->seed = p->seed;
-	t->signature_offset = p->signature_offset;
 	t->key_offset = p->key_offset;
 
 	/* Internal */
@@ -199,16 +256,24 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size)
 	t->data_size_shl = __builtin_ctzl(entry_size);
 
 	/* Tables */
-	bucket_offset = 0;
+	key_mask_offset = 0;
+	bucket_offset = key_mask_offset + key_mask_sz;
 	key_offset = bucket_offset + bucket_sz;
 	key_stack_offset = key_offset + key_sz;
 	data_offset = key_stack_offset + key_stack_sz;
 
+	t->key_mask = (uint64_t *) &t->memory[key_mask_offset];
 	t->buckets = (struct bucket *) &t->memory[bucket_offset];
 	t->key_mem = &t->memory[key_offset];
 	t->key_stack = (uint32_t *) &t->memory[key_stack_offset];
 	t->data_mem = &t->memory[data_offset];
 
+	/* Key mask */
+	if (p->key_mask == NULL)
+		memset(t->key_mask, 0xFF, p->key_size);
+	else
+		memcpy(t->key_mask, p->key_mask, p->key_size);
+
 	/* Key stack */
 	for (i = 0; i < t->n_keys; i++)
 		t->key_stack[i] = t->n_keys - 1 - i;
@@ -246,7 +311,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry,
 	uint64_t sig;
 	uint32_t bkt_index, i;
 
-	sig = t->f_hash(key, t->key_size, t->seed);
+	sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);
 	bkt_index = sig & t->bucket_mask;
 	bkt = &t->buckets[bkt_index];
 	sig = (sig >> 16) | 1LLU;
@@ -258,8 +323,8 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry,
 		uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 			t->key_size_shl];
 
-		if ((sig == bkt_sig) && (memcmp(key, bkt_key, t->key_size)
-			== 0)) {
+		if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,
+			t->key_size) == 0)) {
 			uint8_t *data = &t->data_mem[bkt_key_index <<
 				t->data_size_shl];
 
@@ -292,7 +357,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry,
 
 			bkt->sig[i] = (uint16_t) sig;
 			bkt->key_pos[i] = bkt_key_index;
-			memcpy(bkt_key, key, t->key_size);
+			keycpy(bkt_key, key, t->key_mask, t->key_size);
 			memcpy(data, entry, t->entry_size);
 			lru_update(bkt, i);
 
@@ -311,7 +376,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry,
 		uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl];
 
 		bkt->sig[pos] = (uint16_t) sig;
-		memcpy(bkt_key, key, t->key_size);
+		keycpy(bkt_key, key, t->key_mask, t->key_size);
 		memcpy(data, entry, t->entry_size);
 		lru_update(bkt, pos);
 
@@ -330,7 +395,7 @@ rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found,
 	uint64_t sig;
 	uint32_t bkt_index, i;
 
-	sig = t->f_hash(key, t->key_size, t->seed);
+	sig = t->f_hash(key, t->key_mask, t->key_size, t->seed);
 	bkt_index = sig & t->bucket_mask;
 	bkt = &t->buckets[bkt_index];
 	sig = (sig >> 16) | 1LLU;
@@ -343,14 +408,15 @@ rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found,
 			t->key_size_shl];
 
 		if ((sig == bkt_sig) &&
-			(memcmp(key, bkt_key, t->key_size) == 0)) {
+			(keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) {
 			uint8_t *data = &t->data_mem[bkt_key_index <<
 				t->data_size_shl];
 
 			bkt->sig[i] = 0;
 			t->key_stack[t->key_stack_tos++] = bkt_key_index;
 			*key_found = 1;
-			memcpy(entry, data, t->entry_size);
+			if (entry)
+				memcpy(entry, data, t->entry_size);
 			return 0;
 		}
 	}
@@ -386,7 +452,7 @@ static int rte_table_hash_lru_lookup_unoptimized(
 
 		pkt = pkts[pkt_index];
 		key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset);
-		sig = (uint64_t) t->f_hash(key, t->key_size, t->seed);
+		sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed);
 
 		bkt_index = sig & t->bucket_mask;
 		bkt = &t->buckets[bkt_index];
@@ -399,7 +465,7 @@ static int rte_table_hash_lru_lookup_unoptimized(
 			uint8_t *bkt_key = &t->key_mem[bkt_key_index <<
 				t->key_size_shl];
 
-			if ((sig == bkt_sig) && (memcmp(key, bkt_key,
+			if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask,
 				t->key_size) == 0)) {
 				uint8_t *data = &t->data_mem[bkt_key_index <<
 					t->data_size_shl];
@@ -497,74 +563,75 @@ static int rte_table_hash_lru_lookup_unoptimized(
 	match_pos = (LUT_MATCH_POS >> (mask_all << 1)) & 3;	\
 }
 
-#define lookup_cmp_key(mbuf, key, match_key, f)			\
-{								\
+#define lookup_cmp_key(mbuf, key, match_key, f)				\
+{									\
 	uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\
-	uint64_t *bkt_key = (uint64_t *) key;			\
-								\
-	switch (f->key_size) {					\
-	case 8:							\
-	{							\
-		uint64_t xor = pkt_key[0] ^ bkt_key[0];		\
-		match_key = 0;					\
-		if (xor == 0)					\
-			match_key = 1;				\
-	}							\
-	break;							\
-								\
-	case 16:						\
-	{							\
-		uint64_t xor[2], or;				\
-								\
-		xor[0] = pkt_key[0] ^ bkt_key[0];		\
-		xor[1] = pkt_key[1] ^ bkt_key[1];		\
-		or = xor[0] | xor[1];				\
-		match_key = 0;					\
-		if (or == 0)					\
-			match_key = 1;				\
-	}							\
-	break;							\
-								\
-	case 32:						\
-	{							\
-		uint64_t xor[4], or;				\
-								\
-		xor[0] = pkt_key[0] ^ bkt_key[0];		\
-		xor[1] = pkt_key[1] ^ bkt_key[1];		\
-		xor[2] = pkt_key[2] ^ bkt_key[2];		\
-		xor[3] = pkt_key[3] ^ bkt_key[3];		\
-		or = xor[0] | xor[1] | xor[2] | xor[3];		\
-		match_key = 0;					\
-		if (or == 0)					\
-			match_key = 1;				\
-	}							\
-	break;							\
-								\
-	case 64:						\
-	{							\
-		uint64_t xor[8], or;				\
-								\
-		xor[0] = pkt_key[0] ^ bkt_key[0];		\
-		xor[1] = pkt_key[1] ^ bkt_key[1];		\
-		xor[2] = pkt_key[2] ^ bkt_key[2];		\
-		xor[3] = pkt_key[3] ^ bkt_key[3];		\
-		xor[4] = pkt_key[4] ^ bkt_key[4];		\
-		xor[5] = pkt_key[5] ^ bkt_key[5];		\
-		xor[6] = pkt_key[6] ^ bkt_key[6];		\
-		xor[7] = pkt_key[7] ^ bkt_key[7];		\
-		or = xor[0] | xor[1] | xor[2] | xor[3] |	\
-			xor[4] | xor[5] | xor[6] | xor[7];	\
-		match_key = 0;					\
-		if (or == 0)					\
-			match_key = 1;				\
-	}							\
-	break;							\
-								\
-	default:						\
-		match_key = 0;					\
-		if (memcmp(pkt_key, bkt_key, f->key_size) == 0)	\
-			match_key = 1;				\
-	}							\
+	uint64_t *bkt_key = (uint64_t *) key;				\
+	uint64_t *key_mask = f->key_mask;					\
+									\
+	switch (f->key_size) {						\
+	case 8:								\
+	{								\
+		uint64_t xor = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];	\
+		match_key = 0;						\
+		if (xor == 0)						\
+			match_key = 1;					\
+	}								\
+	break;								\
+									\
+	case 16:							\
+	{								\
+		uint64_t xor[2], or;					\
+									\
+		xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];		\
+		xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1];		\
+		or = xor[0] | xor[1];					\
+		match_key = 0;						\
+		if (or == 0)						\
+			match_key = 1;					\
+	}								\
+	break;								\
+									\
+	case 32:							\
+	{								\
+		uint64_t xor[4], or;					\
+									\
+		xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];		\
+		xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1];		\
+		xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2];		\
+		xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3];		\
+		or = xor[0] | xor[1] | xor[2] | xor[3];			\
+		match_key = 0;						\
+		if (or == 0)						\
+			match_key = 1;					\
+	}								\
+	break;								\
+									\
+	case 64:							\
+	{								\
+		uint64_t xor[8], or;					\
+									\
+		xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0];		\
+		xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1];		\
+		xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2];		\
+		xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3];		\
+		xor[4] = (pkt_key[4] & key_mask[4]) ^ bkt_key[4];		\
+		xor[5] = (pkt_key[5] & key_mask[5]) ^ bkt_key[5];		\
+		xor[6] = (pkt_key[6] & key_mask[6]) ^ bkt_key[6];		\
+		xor[7] = (pkt_key[7] & key_mask[7]) ^ bkt_key[7];		\
+		or = xor[0] | xor[1] | xor[2] | xor[3] |		\
+			xor[4] | xor[5] | xor[6] | xor[7];		\
+		match_key = 0;						\
+		if (or == 0)						\
+			match_key = 1;					\
+	}								\
+	break;								\
+									\
+	default:							\
+		match_key = 0;						\
+		if (keycmp(bkt_key, pkt_key, key_mask, f->key_size) == 0)	\
+			match_key = 1;					\
+	}								\
 }
 
 #define lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index)\
@@ -619,20 +686,20 @@ static int rte_table_hash_lru_lookup_unoptimized(
 	struct bucket *bkt10, *bkt11, *buckets = t->buckets;	\
 	uint8_t *key10, *key11;					\
 	uint64_t bucket_mask = t->bucket_mask;			\
-	rte_table_hash_op_hash_nomask f_hash = t->f_hash;		\
+	rte_table_hash_op_hash f_hash = t->f_hash;		\
 	uint64_t seed = t->seed;				\
 	uint32_t key_size = t->key_size;			\
 	uint32_t key_offset = t->key_offset;			\
 								\
 	mbuf10 = pkts[pkt10_index];				\
 	key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset);\
-	sig10 = (uint64_t) f_hash(key10, key_size, seed);	\
+	sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed);\
 	bkt10_index = sig10 & bucket_mask;			\
 	bkt10 = &buckets[bkt10_index];				\
 								\
 	mbuf11 = pkts[pkt11_index];				\
 	key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset);\
-	sig11 = (uint64_t) f_hash(key11, key_size, seed);	\
+	sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed);\
 	bkt11_index = sig11 & bucket_mask;			\
 	bkt11 = &buckets[bkt11_index];				\
 								\
diff --git a/test/test-pipeline/pipeline_hash.c b/test/test-pipeline/pipeline_hash.c
index 13d309d..3c48640 100644
--- a/test/test-pipeline/pipeline_hash.c
+++ b/test/test-pipeline/pipeline_hash.c
@@ -204,14 +204,15 @@ app_main_loop_worker_pipeline_hash(void) {
 	case e_APP_PIPELINE_HASH_KEY16_LRU:
 	case e_APP_PIPELINE_HASH_KEY32_LRU:
 	{
-		struct rte_table_hash_lru_params table_hash_params = {
+		struct rte_table_hash_params table_hash_params = {
+			.name = "TABLE",
 			.key_size = key_size,
+			.key_offset = APP_METADATA_OFFSET(32),
+			.key_mask = NULL,
 			.n_keys = 1 << 24,
 			.n_buckets = 1 << 22,
-			.f_hash = test_hash,
+			.f_hash = (rte_table_hash_op_hash)test_hash,
 			.seed = 0,
-			.signature_offset = APP_METADATA_OFFSET(0),
-			.key_offset = APP_METADATA_OFFSET(32),
 		};
 
 		struct rte_pipeline_table_params table_params = {
-- 
2.7.4



More information about the dev mailing list