You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
203 lines
6.5 KiB
203 lines
6.5 KiB
From 82116044164f1f78e4eec9f31231adc6976b928d Mon Sep 17 00:00:00 2001
|
|
From: Alaa Hleihel <ahleihel@redhat.com>
|
|
Date: Sun, 10 May 2020 15:04:08 -0400
|
|
Subject: [PATCH 090/312] [netdrv] net/mlx5: Support lockless FTE read lookups
|
|
|
|
Message-id: <20200510150452.10307-44-ahleihel@redhat.com>
|
|
Patchwork-id: 306667
|
|
Patchwork-instance: patchwork
|
|
O-Subject: [RHEL8.3 BZ 1789380 v2 43/87] net/mlx5: Support lockless FTE read lookups
|
|
Bugzilla: 1789380
|
|
RH-Acked-by: Kamal Heib <kheib@redhat.com>
|
|
RH-Acked-by: Jarod Wilson <jarod@redhat.com>
|
|
RH-Acked-by: Tony Camuso <tcamuso@redhat.com>
|
|
RH-Acked-by: Jonathan Toppins <jtoppins@redhat.com>
|
|
|
|
Bugzilla: http://bugzilla.redhat.com/1789380
|
|
Upstream: v5.5-rc1
|
|
|
|
commit 7dee607ed0e04500459db53001d8e02f8831f084
|
|
Author: Parav Pandit <parav@mellanox.com>
|
|
Date: Wed Sep 18 18:50:32 2019 -0500
|
|
|
|
net/mlx5: Support lockless FTE read lookups
|
|
|
|
During connection tracking offloads with high number of connections,
|
|
(40K connections per second), flow table group lock contention is
|
|
observed.
|
|
To improve the performance by reducing lock contention, lockless
|
|
FTE read lookup is performed as described below.
|
|
|
|
Each flow table entry is refcounted.
|
|
Flow table entry is removed when refcount drops to zero.
|
|
rhash table allows rcu protected lookup.
|
|
Each hash table entry insertion and removal is write lock protected.
|
|
|
|
Hence, it is possible to perform lockless lookup in rhash table using
|
|
following scheme.
|
|
|
|
(a) Guard FTE entry lookup per group using rcu read lock.
|
|
(b) Before freeing the FTE entry, wait for all readers to finish
|
|
accessing the FTE.
|
|
|
|
Below example of one reader and write in parallel racing, shows
|
|
protection in effect with rcu lock.
|
|
|
|
lookup_fte_locked()
|
|
rcu_read_lock();
|
|
search_hash_table()
|
|
existing_flow_group_write_lock();
|
|
tree_put_node(fte)
|
|
drop_ref_cnt(fte)
|
|
del_sw_fte(fte)
|
|
del_hash_table_entry();
|
|
call_rcu();
|
|
existing_flow_group_write_unlock();
|
|
get_ref_cnt(fte) fails
|
|
rcu_read_unlock();
|
|
rcu grace period();
|
|
[..]
|
|
kmem_cache_free(fte);
|
|
|
|
Signed-off-by: Parav Pandit <parav@mellanox.com>
|
|
Reviewed-by: Mark Bloch <markb@mellanox.com>
|
|
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
|
|
|
|
Signed-off-by: Alaa Hleihel <ahleihel@redhat.com>
|
|
Signed-off-by: Frantisek Hrbata <fhrbata@redhat.com>
|
|
---
|
|
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 70 ++++++++++++++++++-----
|
|
drivers/net/ethernet/mellanox/mlx5/core/fs_core.h | 1 +
|
|
2 files changed, 56 insertions(+), 15 deletions(-)
|
|
|
|
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
|
|
index e8064bd87aad..6e1ef05becce 100644
|
|
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
|
|
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
|
|
@@ -531,9 +531,16 @@ static void del_hw_fte(struct fs_node *node)
|
|
}
|
|
}
|
|
|
|
+static void del_sw_fte_rcu(struct rcu_head *head)
|
|
+{
|
|
+ struct fs_fte *fte = container_of(head, struct fs_fte, rcu);
|
|
+ struct mlx5_flow_steering *steering = get_steering(&fte->node);
|
|
+
|
|
+ kmem_cache_free(steering->ftes_cache, fte);
|
|
+}
|
|
+
|
|
static void del_sw_fte(struct fs_node *node)
|
|
{
|
|
- struct mlx5_flow_steering *steering = get_steering(node);
|
|
struct mlx5_flow_group *fg;
|
|
struct fs_fte *fte;
|
|
int err;
|
|
@@ -546,7 +553,8 @@ static void del_sw_fte(struct fs_node *node)
|
|
rhash_fte);
|
|
WARN_ON(err);
|
|
ida_simple_remove(&fg->fte_allocator, fte->index - fg->start_index);
|
|
- kmem_cache_free(steering->ftes_cache, fte);
|
|
+
|
|
+ call_rcu(&fte->rcu, del_sw_fte_rcu);
|
|
}
|
|
|
|
static void del_hw_flow_group(struct fs_node *node)
|
|
@@ -1626,22 +1634,47 @@ static u64 matched_fgs_get_version(struct list_head *match_head)
|
|
}
|
|
|
|
static struct fs_fte *
|
|
-lookup_fte_locked(struct mlx5_flow_group *g,
|
|
- const u32 *match_value,
|
|
- bool take_write)
|
|
+lookup_fte_for_write_locked(struct mlx5_flow_group *g, const u32 *match_value)
|
|
{
|
|
struct fs_fte *fte_tmp;
|
|
|
|
- if (take_write)
|
|
- nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
|
|
- else
|
|
- nested_down_read_ref_node(&g->node, FS_LOCK_PARENT);
|
|
- fte_tmp = rhashtable_lookup_fast(&g->ftes_hash, match_value,
|
|
- rhash_fte);
|
|
+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
|
|
+
|
|
+ fte_tmp = rhashtable_lookup_fast(&g->ftes_hash, match_value, rhash_fte);
|
|
if (!fte_tmp || !tree_get_node(&fte_tmp->node)) {
|
|
fte_tmp = NULL;
|
|
goto out;
|
|
}
|
|
+
|
|
+ if (!fte_tmp->node.active) {
|
|
+ tree_put_node(&fte_tmp->node, false);
|
|
+ fte_tmp = NULL;
|
|
+ goto out;
|
|
+ }
|
|
+ nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD);
|
|
+
|
|
+out:
|
|
+ up_write_ref_node(&g->node, false);
|
|
+ return fte_tmp;
|
|
+}
|
|
+
|
|
+static struct fs_fte *
|
|
+lookup_fte_for_read_locked(struct mlx5_flow_group *g, const u32 *match_value)
|
|
+{
|
|
+ struct fs_fte *fte_tmp;
|
|
+
|
|
+ if (!tree_get_node(&g->node))
|
|
+ return NULL;
|
|
+
|
|
+ rcu_read_lock();
|
|
+ fte_tmp = rhashtable_lookup(&g->ftes_hash, match_value, rhash_fte);
|
|
+ if (!fte_tmp || !tree_get_node(&fte_tmp->node)) {
|
|
+ rcu_read_unlock();
|
|
+ fte_tmp = NULL;
|
|
+ goto out;
|
|
+ }
|
|
+ rcu_read_unlock();
|
|
+
|
|
if (!fte_tmp->node.active) {
|
|
tree_put_node(&fte_tmp->node, false);
|
|
fte_tmp = NULL;
|
|
@@ -1649,14 +1682,21 @@ lookup_fte_locked(struct mlx5_flow_group *g,
|
|
}
|
|
|
|
nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD);
|
|
+
|
|
out:
|
|
- if (take_write)
|
|
- up_write_ref_node(&g->node, false);
|
|
- else
|
|
- up_read_ref_node(&g->node);
|
|
+ tree_put_node(&g->node, false);
|
|
return fte_tmp;
|
|
}
|
|
|
|
+static struct fs_fte *
|
|
+lookup_fte_locked(struct mlx5_flow_group *g, const u32 *match_value, bool write)
|
|
+{
|
|
+ if (write)
|
|
+ return lookup_fte_for_write_locked(g, match_value);
|
|
+ else
|
|
+ return lookup_fte_for_read_locked(g, match_value);
|
|
+}
|
|
+
|
|
static struct mlx5_flow_handle *
|
|
try_add_to_existing_fg(struct mlx5_flow_table *ft,
|
|
struct list_head *match_head,
|
|
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
|
|
index c6221ccbdddf..8e4ca13f4d74 100644
|
|
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
|
|
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
|
|
@@ -205,6 +205,7 @@ struct fs_fte {
|
|
enum fs_fte_status status;
|
|
struct mlx5_fc *counter;
|
|
struct rhash_head hash;
|
|
+ struct rcu_head rcu;
|
|
int modify_mask;
|
|
};
|
|
|
|
--
|
|
2.13.6
|
|
|