Projects
openEuler:C
gazelle
_service:tar_scm:0038-solve-problem-that-rte_pk...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:tar_scm:0038-solve-problem-that-rte_pktmbuf_poll_creat-in-same-nu.patch of Package gazelle
From 2a5891557162fb5743f13cb550dc7ff109476176 Mon Sep 17 00:00:00 2001 From: hantwofish <hankangkang5@huawei.com> Date: Wed, 1 Nov 2023 18:34:07 +0800 Subject: [PATCH] solve problem that rte_pktmbuf_poll_creat in same numa . --- src/lstack/core/lstack_dpdk.c | 11 +++--- src/lstack/core/lstack_protocol_stack.c | 52 ++++++++++++++++++++----- src/lstack/include/lstack_dpdk.h | 2 +- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c index 48fa67d..88a6c45 100644 --- a/src/lstack/core/lstack_dpdk.c +++ b/src/lstack/core/lstack_dpdk.c @@ -110,7 +110,7 @@ int32_t dpdk_eal_init(void) { int32_t ret; struct cfg_params *global_params = get_global_cfg_params(); - + ret = rte_eal_init(global_params->dpdk_argc, global_params->dpdk_argv); if (ret < 0) { if (rte_errno == EALREADY) { @@ -137,7 +137,7 @@ int32_t dpdk_eal_init(void) } struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf, - uint32_t mbuf_cache_size, uint16_t queue_id) + uint32_t mbuf_cache_size, uint16_t queue_id, unsigned numa_id) { int32_t ret; char pool_name[PATH_MAX]; @@ -145,12 +145,13 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf, ret = snprintf_s(pool_name, sizeof(pool_name), PATH_MAX - 1, "%s_%hu", name, queue_id); if (ret < 0) { + LSTACK_LOG(ERR, LSTACK, "snprintf_s fail ret=%d \n", ret); return NULL; } /* time stamp before pbuf_custom as priv_data */ uint16_t private_size = RTE_ALIGN(sizeof(struct mbuf_private), RTE_CACHE_LINE_SIZE); - pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf, mbuf_cache_size, private_size, MBUF_SZ, rte_socket_id()); + pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf, mbuf_cache_size, private_size, MBUF_SZ, numa_id); if (pool == NULL) { LSTACK_LOG(ERR, LSTACK, "cannot create %s pool rte_err=%d\n", pool_name, rte_errno); } @@ -611,7 +612,7 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i int32_t ret; struct rte_mempool *rxtx_pktmbuf_pool = get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx]; - + uint16_t socket_id = 0; struct cfg_params *cfg = get_global_cfg_params(); if (!cfg->use_ltran && cfg->num_process == 1) { @@ -664,7 +665,7 @@ int32_t dpdk_ethdev_start(void) int32_t dpdk_init_lstack_kni(void) { struct protocol_stack_group *stack_group = get_protocol_stack_group(); - stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, 0, 0); + stack_group->kni_pktmbuf_pool = create_pktmbuf_mempool("kni_mbuf", KNI_NB_MBUF, 0, 0, rte_socket_id()); if (stack_group->kni_pktmbuf_pool == NULL) { return -1; } diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index 3662948..2fd8d0a 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -523,6 +523,46 @@ static void gazelle_listen_thread(void *arg) recv_pkts_from_other_process(cfg_param->process_idx, arg); } +int32_t stack_group_init_mempool(void) +{ + struct cfg_params *global_cfg_parmas = get_global_cfg_params(); + struct protocol_stack_group *stack_group = get_protocol_stack_group(); + struct rte_mempool *rxtx_mbuf = NULL; + uint32_t cpu_id = 0; + unsigned numa_id = 0; + int queue_id = 0; + + LSTACK_LOG(INFO, LSTACK, + "config::num_cpu=%d num_process=%d \n", global_cfg_parmas->num_cpu, global_cfg_parmas->num_process); + + uint32_t total_mbufs = get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count; + + for (int cpu_idx = 0; cpu_idx < global_cfg_parmas->num_cpu; cpu_idx++) { + cpu_id = global_cfg_parmas->cpus[cpu_idx]; + numa_id = numa_node_of_cpu(cpu_id); + + for (int process_idx = 0; process_idx < global_cfg_parmas->num_process; process_idx++) { + queue_id = cpu_idx * global_cfg_parmas->num_process + process_idx; + if (queue_id >= PROTOCOL_STACK_MAX) { + LSTACK_LOG(ERR, LSTACK, "index is over\n"); + return -1; + } + + rxtx_mbuf = create_pktmbuf_mempool( + "rxtx_mbuf", total_mbufs / stack_group->stack_num, RXTX_CACHE_SZ, queue_id, numa_id); + if (rxtx_mbuf == NULL) { + LSTACK_LOG(ERR, LSTACK, "cpuid=%u, numid=%d , rxtx_mbuf idx= %d create_pktmbuf_mempool fail\n", + cpu_id, numa_id, queue_id); + return -1; + } + + get_protocol_stack_group()->total_rxtx_pktmbuf_pool[queue_id] = rxtx_mbuf; + } + } + + return 0; +} + int32_t stack_group_init(void) { struct protocol_stack_group *stack_group = get_protocol_stack_group(); @@ -542,14 +582,8 @@ int32_t stack_group_init(void) stack_group->stack_setup_fail = 0; if (get_global_cfg_params()->is_primary) { - uint32_t total_mbufs = get_global_cfg_params()->mbuf_count_per_conn * get_global_cfg_params()->tcp_conn_count; - for (uint16_t idx = 0; idx < get_global_cfg_params()->tot_queue_num; idx++) { - struct rte_mempool* rxtx_mbuf = create_pktmbuf_mempool("rxtx_mbuf", - total_mbufs / stack_group->stack_num, RXTX_CACHE_SZ, idx); - if (rxtx_mbuf == NULL) { - return -1; - } - get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx] = rxtx_mbuf; + if (stack_group_init_mempool() != 0) { + return -1; } } @@ -970,7 +1004,7 @@ void stack_recvlist_count(struct rpc_msg *msg) list_for_each_safe(node, temp, list) { count++; } - + msg->result = count; } diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h index 6ca4f3b..1a054d6 100644 --- a/src/lstack/include/lstack_dpdk.h +++ b/src/lstack/include/lstack_dpdk.h @@ -56,7 +56,7 @@ void dpdk_restore_pci(void); bool port_in_stack_queue(uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port); uint16_t get_port_id(void); struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf, - uint32_t mbuf_cache_size, uint16_t queue_id); + uint32_t mbuf_cache_size, uint16_t queue_id, unsigned numa_id); void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id); int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num); -- 2.23.0
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.