Projects
openEuler:C
gazelle
_service:tar_scm:0072-1.solve-the-problem-of-1w...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:tar_scm:0072-1.solve-the-problem-of-1w-connection-not-being-able-.patch of Package gazelle
From 17d320fa7a978de0d76ae13d0afb1744cf456250 Mon Sep 17 00:00:00 2001 From: hantwofish <hankangkang5@huawei.com> Date: Mon, 4 Dec 2023 15:45:09 +0800 Subject: [PATCH] 1.solve the problem of 1w connection not being able to ping 2.add debug info : rpc_mempool availd size 3.mod struct member name 3.1 rxtx_pktmbuf_pool -> rxtx_mbufpool 3.2 pool -> rpcpool; rpc_pool -> mempool --- src/common/gazelle_dfx_msg.h | 3 ++- src/lstack/core/lstack_dpdk.c | 15 +++++++++------ src/lstack/core/lstack_lwip.c | 8 ++++---- src/lstack/core/lstack_protocol_stack.c | 12 +++++++++--- src/lstack/core/lstack_stack_stat.c | 7 +++++-- src/lstack/core/lstack_thread_rpc.c | 20 +++++++++++++++----- src/lstack/include/lstack_protocol_stack.h | 5 ++++- src/lstack/include/lstack_thread_rpc.h | 11 ++++++----- src/lstack/netif/lstack_ethdev.c | 10 +++++----- src/lstack/netif/lstack_vdev.c | 2 +- src/ltran/ltran_dfx.c | 3 ++- 11 files changed, 62 insertions(+), 34 deletions(-) diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h index ac6ea5e..81d3426 100644 --- a/src/common/gazelle_dfx_msg.h +++ b/src/common/gazelle_dfx_msg.h @@ -93,7 +93,8 @@ struct gazelle_stack_aggregate_stats { struct gazelle_stat_pkts { uint16_t conn_num; - uint32_t mempool_freecnt; + uint32_t mbufpool_avail_cnt; + uint32_t rpcpool_avail_cnt; uint64_t call_msg_cnt; uint64_t recv_list_cnt; uint64_t call_alloc_fail; diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c index 1811a9d..5107f1a 100644 --- a/src/lstack/core/lstack_dpdk.c +++ b/src/lstack/core/lstack_dpdk.c @@ -207,16 +207,16 @@ static struct reg_ring_msg *create_reg_mempool(const char *name, uint16_t queue_ int32_t pktmbuf_pool_init(struct protocol_stack *stack) { - stack->rxtx_pktmbuf_pool = get_pktmbuf_mempool("rxtx_mbuf", stack->queue_id); - if (stack->rxtx_pktmbuf_pool == NULL) { - LSTACK_LOG(ERR, LSTACK, "rxtx_pktmbuf_pool is NULL\n"); + stack->rxtx_mbuf_pool = get_pktmbuf_mempool("rxtx_mbuf", stack->queue_id); + if (stack->rxtx_mbuf_pool == NULL) { + LSTACK_LOG(ERR, LSTACK, "rxtx_mbuf_pool is NULL\n"); return -1; } if (use_ltran()) { stack->reg_buf = create_reg_mempool("reg_ring_msg", stack->queue_id); if (stack->reg_buf == NULL) { - LSTACK_LOG(ERR, LSTACK, "rxtx_pktmbuf_pool is NULL\n"); + LSTACK_LOG(ERR, LSTACK, "rxtx_mbuf_pool is NULL\n"); return -1; } } @@ -292,6 +292,9 @@ int32_t create_shared_ring(struct protocol_stack *stack) int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num) { + if (rte_mempool_avail_count(pool) < MBUFPOOL_RESERVE_NUM + num) { + return -ENOMEM; + } int32_t ret = rte_pktmbuf_alloc_bulk(pool, mbufs, num); if (ret != 0) { LSTACK_LOG(ERR, LSTACK, "rte_pktmbuf_alloc_bulk fail allocNum=%d, ret=%d, info:%s \n", @@ -607,7 +610,7 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i { int32_t ret; - struct rte_mempool *rxtx_pktmbuf_pool = get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx]; + struct rte_mempool *rxtx_mbuf_pool = get_protocol_stack_group()->total_rxtx_pktmbuf_pool[idx]; uint16_t socket_id = 0; struct cfg_params *cfg = get_global_cfg_params(); @@ -617,7 +620,7 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i socket_id = cfg->process_numa[idx]; } ret = rte_eth_rx_queue_setup(eth_params->port_id, idx, eth_params->nb_rx_desc, socket_id, - ð_params->rx_conf, rxtx_pktmbuf_pool); + ð_params->rx_conf, rxtx_mbuf_pool); if (ret < 0) { LSTACK_LOG(ERR, LSTACK, "cannot setup rx_queue %hu: %s\n", idx, rte_strerror(-ret)); return -1; diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c index af9bf73..5d5ad73 100644 --- a/src/lstack/core/lstack_lwip.c +++ b/src/lstack/core/lstack_lwip.c @@ -143,7 +143,7 @@ static bool replenish_send_idlembuf(struct protocol_stack *stack, struct lwip_so return false; } - if (rte_pktmbuf_alloc_bulk(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)pbuf, replenish_cnt) != 0) { + if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)pbuf, replenish_cnt) != 0) { stack->stats.tx_allocmbuf_fail++; return true; } @@ -237,7 +237,7 @@ struct pbuf *do_lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type typ struct rte_mbuf *mbuf; struct protocol_stack *stack = get_protocol_stack(); - if (rte_pktmbuf_alloc_bulk(stack->rxtx_pktmbuf_pool, &mbuf, 1) != 0) { + if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf, 1) != 0) { stack->stats.tx_allocmbuf_fail++; return NULL; } @@ -362,7 +362,7 @@ static inline ssize_t app_direct_write(struct protocol_stack *stack, struct lwip } /* first pbuf get from send_ring. and malloc pbufs attach to first pbuf */ - if (rte_pktmbuf_alloc_bulk(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)&pbufs[1], write_num - 1) != 0) { + if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)&pbufs[1], write_num - 1) != 0) { stack->stats.tx_allocmbuf_fail++; free(pbufs); return 0; @@ -401,7 +401,7 @@ static inline ssize_t app_direct_attach(struct protocol_stack *stack, struct pbu } /* first pbuf get from send_ring. and malloc pbufs attach to first pbuf */ - if (rte_pktmbuf_alloc_bulk(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)pbufs, write_num) != 0) { + if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)pbufs, write_num) != 0) { stack->stats.tx_allocmbuf_fail++; free(pbufs); return 0; diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index dc8f143..f61e7a8 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -902,7 +902,7 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack continue; } - ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1); + ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1); if (ret != 0) { stack->stats.rx_allocmbuf_fail++; return; @@ -914,7 +914,7 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack return; } } - ret = dpdk_alloc_pktmbuf(cur_stack->rxtx_pktmbuf_pool, &mbuf_copy, 1); + ret = dpdk_alloc_pktmbuf(cur_stack->rxtx_mbuf_pool, &mbuf_copy, 1); if (ret != 0) { cur_stack->stats.rx_allocmbuf_fail++; return; @@ -947,7 +947,13 @@ void stack_mempool_size(struct rpc_msg *msg) { struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_0].p; - msg->result = rte_mempool_avail_count(stack->rxtx_pktmbuf_pool); + msg->result = rte_mempool_avail_count(stack->rxtx_mbuf_pool); +} + +void stack_rpcpool_size(struct rpc_msg *msg) +{ + struct rpc_msg_pool *rpc_mem_pool = (struct rpc_msg_pool*)msg->args[MSG_ARG_0].p; + msg->result = rte_mempool_avail_count(rpc_mem_pool->mempool); } void stack_create_shadow_fd(struct rpc_msg *msg) diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c index 3fd5903..45237b0 100644 --- a/src/lstack/core/lstack_stack_stat.c +++ b/src/lstack/core/lstack_stack_stat.c @@ -179,12 +179,15 @@ static void get_stack_stats(struct gazelle_stack_dfx_data *dfx, struct protocol_ int32_t rpc_call_result = rpc_call_msgcnt(stack); dfx->data.pkts.call_msg_cnt = (rpc_call_result < 0) ? 0 : rpc_call_result; - rpc_call_result = rpc_call_mempoolsize(stack); - dfx->data.pkts.mempool_freecnt = (rpc_call_result < 0) ? 0 : rpc_call_result; + rpc_call_result = rpc_call_mbufpoolsize(stack); + dfx->data.pkts.mbufpool_avail_cnt = (rpc_call_result < 0) ? 0 : rpc_call_result; rpc_call_result = rpc_call_recvlistcnt(stack); dfx->data.pkts.recv_list_cnt = (rpc_call_result < 0) ? 0 : rpc_call_result; + rpc_call_result = rpc_call_rpcpool_size(stack); + dfx->data.pkts.rpcpool_avail_cnt = (rpc_call_result < 0) ? 0 : rpc_call_result; + dfx->data.pkts.conn_num = stack->conn_num; } diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c index 4dc3da3..473e908 100644 --- a/src/lstack/core/lstack_thread_rpc.c +++ b/src/lstack/core/lstack_thread_rpc.c @@ -29,7 +29,7 @@ static inline __attribute__((always_inline)) struct rpc_msg *get_rpc_msg(struct { int ret; struct rpc_msg *msg = NULL; - ret = rte_mempool_get(rpc_pool->rpc_pool, (void **)&msg); + ret = rte_mempool_get(rpc_pool->mempool, (void **)&msg); if (ret < 0) { LSTACK_LOG(INFO, LSTACK, "rpc pool empty!\n"); errno = ENOMEM; @@ -54,9 +54,9 @@ static struct rpc_msg *rpc_msg_alloc(struct protocol_stack *stack, rpc_msg_func return NULL; } - g_rpc_pool->rpc_pool = create_mempool("rpc_pool", RPC_MSG_MAX, sizeof(struct rpc_msg), + g_rpc_pool->mempool = create_mempool("rpc_pool", RPC_MSG_MAX, sizeof(struct rpc_msg), 0, rte_gettid()); - if (g_rpc_pool->rpc_pool == NULL) { + if (g_rpc_pool->mempool == NULL) { get_protocol_stack_group()->call_alloc_fail++; return NULL; } @@ -67,7 +67,7 @@ static struct rpc_msg *rpc_msg_alloc(struct protocol_stack *stack, rpc_msg_func get_protocol_stack_group()->call_alloc_fail++; return NULL; } - msg->pool = g_rpc_pool; + msg->rpcpool = g_rpc_pool; pthread_spin_init(&msg->lock, PTHREAD_PROCESS_PRIVATE); msg->func = func; @@ -194,7 +194,7 @@ int32_t rpc_call_thread_regphase2(struct protocol_stack *stack, void *conn) return rpc_sync_call(&stack->rpc_queue, msg); } -int32_t rpc_call_mempoolsize(struct protocol_stack *stack) +int32_t rpc_call_mbufpoolsize(struct protocol_stack *stack) { struct rpc_msg *msg = rpc_msg_alloc(stack, stack_mempool_size); if (msg == NULL) { @@ -206,6 +206,16 @@ int32_t rpc_call_mempoolsize(struct protocol_stack *stack) return rpc_sync_call(&stack->rpc_queue, msg); } +int32_t rpc_call_rpcpool_size(struct protocol_stack *stack) +{ + struct rpc_msg *msg = rpc_msg_alloc(stack, stack_rpcpool_size); + if (msg == NULL) { + return -1; + } + msg->args[MSG_ARG_0].p = g_rpc_pool; + return rpc_sync_call(&stack->rpc_queue, msg); +} + int32_t rpc_call_recvlistcnt(struct protocol_stack *stack) { struct rpc_msg *msg = rpc_msg_alloc(stack, stack_recvlist_count); diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h index e339b8d..6638984 100644 --- a/src/lstack/include/lstack_protocol_stack.h +++ b/src/lstack/include/lstack_protocol_stack.h @@ -31,6 +31,8 @@ #define SOCK_SEND_REPLENISH_THRES (16) #define WAKEUP_MAX_NUM (32) +#define MBUFPOOL_RESERVE_NUM 5000 + struct rte_mempool; struct rte_ring; struct rte_mbuf; @@ -45,7 +47,7 @@ struct protocol_stack { cpu_set_t idle_cpuset; /* idle cpu in numa of stack, app thread bind to it */ int32_t epollfd; /* kernel event thread epoll fd */ - struct rte_mempool *rxtx_pktmbuf_pool; + struct rte_mempool *rxtx_mbuf_pool; struct rte_ring *rx_ring; struct rte_ring *tx_ring; struct rte_ring *reg_ring; @@ -168,6 +170,7 @@ void stack_fcntl(struct rpc_msg *msg); void stack_ioctl(struct rpc_msg *msg); void stack_send(struct rpc_msg *msg); void stack_mempool_size(struct rpc_msg *msg); +void stack_rpcpool_size(struct rpc_msg *msg); void stack_create_shadow_fd(struct rpc_msg *msg); void stack_replenish_sendring(struct rpc_msg *msg); void stack_get_conntable(struct rpc_msg *msg); diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h index 13f5ec2..ca8a510 100644 --- a/src/lstack/include/lstack_thread_rpc.h +++ b/src/lstack/include/lstack_thread_rpc.h @@ -26,7 +26,7 @@ #define MSG_ARG_4 (4) #define RPM_MSG_ARG_SIZE (5) -#define RPC_MSG_MAX 2048 +#define RPC_MSG_MAX 4096 #define RPC_MSG_MASK (RPC_MSG_MAX - 1) struct rpc_msg; @@ -48,14 +48,14 @@ struct rpc_msg { int8_t recall_flag : 1; int64_t result; /* func return val */ lockless_queue_node queue_node; - struct rpc_msg_pool *pool; + struct rpc_msg_pool *rpcpool; rpc_msg_func func; /* msg handle func hook */ union rpc_msg_arg args[RPM_MSG_ARG_SIZE]; /* resolve by type */ }; struct rpc_msg_pool { - struct rte_mempool *rpc_pool; + struct rte_mempool *mempool; }; struct protocol_stack; @@ -87,7 +87,8 @@ int32_t rpc_call_setsockopt(int fd, int level, int optname, const void *optval, int32_t rpc_call_fcntl(int fd, int cmd, long val); int32_t rpc_call_ioctl(int fd, long cmd, void *argp); int32_t rpc_call_replenish(struct protocol_stack *stack, struct lwip_sock *sock); -int32_t rpc_call_mempoolsize(struct protocol_stack *stack); +int32_t rpc_call_mbufpoolsize(struct protocol_stack *stack); +int32_t rpc_call_rpcpool_size(struct protocol_stack *stack); static inline __attribute__((always_inline)) void rpc_call(lockless_queue *queue, struct rpc_msg *msg) { @@ -97,7 +98,7 @@ static inline __attribute__((always_inline)) void rpc_call(lockless_queue *queue static inline __attribute__((always_inline)) void rpc_msg_free(struct rpc_msg *msg) { pthread_spin_destroy(&msg->lock); - rte_mempool_put(msg->pool->rpc_pool, (void *)msg); + rte_mempool_put(msg->rpcpool->mempool, (void *)msg); } #endif diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c index 31856f2..2e11670 100644 --- a/src/lstack/netif/lstack_ethdev.c +++ b/src/lstack/netif/lstack_ethdev.c @@ -542,9 +542,9 @@ void parse_arp_and_transefer(char* buf) int32_t ret; for (int32_t i = 0; i < stack_group->stack_num; i++) { stack = stack_group->stacks[i]; - ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1); + ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1); while (ret != 0) { - ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1); + ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1); stack->stats.rx_allocmbuf_fail++; } copy_mbuf(mbuf_copy, mbuf); @@ -571,9 +571,9 @@ void parse_tcp_and_transefer(char* buf) struct rte_mbuf *mbuf_copy = NULL; struct protocol_stack *stack = stack_group->stacks[stk_index]; - int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1); + int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1); while (ret != 0) { - ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, &mbuf_copy, 1); + ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1); stack->stats.rx_allocmbuf_fail++; } @@ -944,7 +944,7 @@ int32_t ethdev_init(struct protocol_stack *stack) if (use_ltran()) { stack->rx_ring_used = 0; - int32_t ret = fill_mbuf_to_ring(stack->rxtx_pktmbuf_pool, stack->rx_ring, RING_SIZE(VDEV_RX_QUEUE_SZ)); + int32_t ret = fill_mbuf_to_ring(stack->rxtx_mbuf_pool, stack->rx_ring, RING_SIZE(VDEV_RX_QUEUE_SZ)); if (ret != 0) { LSTACK_LOG(ERR, LSTACK, "fill mbuf to rx_ring failed ret=%d\n", ret); return ret; diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c index e1438da..ccf664a 100644 --- a/src/lstack/netif/lstack_vdev.c +++ b/src/lstack/netif/lstack_vdev.c @@ -57,7 +57,7 @@ static uint32_t ltran_rx_poll(struct protocol_stack *stack, struct rte_mbuf **pk stack->rx_ring_used += rcvd_pkts; if (unlikely(stack->rx_ring_used >= USED_RX_PKTS_WATERMARK)) { uint32_t free_cnt = LWIP_MIN(stack->rx_ring_used, RING_SIZE(DPDK_PKT_BURST_SIZE)); - int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_pktmbuf_pool, (struct rte_mbuf **)free_buf, free_cnt); + int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)free_buf, free_cnt); if (likely(ret == 0)) { nr_pkts = gazelle_ring_sp_enqueue(stack->rx_ring, (void **)free_buf, free_cnt); stack->rx_ring_used -= nr_pkts; diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c index d3ff527..58ab14c 100644 --- a/src/ltran/ltran_dfx.c +++ b/src/ltran/ltran_dfx.c @@ -618,7 +618,8 @@ static void show_lstack_stats(struct gazelle_stack_dfx_data *lstack_stat) printf("call_alloc_fail: %-12"PRIu64" ", lstack_stat->data.pkts.call_alloc_fail); printf("call_null: %-18"PRIu64" \n", lstack_stat->data.pkts.stack_stat.call_null); printf("send_pkts_fail: %-13"PRIu64" ", lstack_stat->data.pkts.stack_stat.send_pkts_fail); - printf("mempool_freecnt: %-12"PRIu32" \n", lstack_stat->data.pkts.mempool_freecnt); + printf("mbuf_pool_freecnt: %-10"PRIu32" ", lstack_stat->data.pkts.mbufpool_avail_cnt); + printf("rpc_pool_freecnt: %-12"PRIu32" \n", lstack_stat->data.pkts.rpcpool_avail_cnt); } static void gazelle_print_lstack_stat_detail(struct gazelle_stack_dfx_data *lstack_stat, -- 2.27.0
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.