for (i = 0; i < cnt; i++)
{
    tsx = NULL;

    /* Create UAS transaction for this request. */
    status = pjsip_tsx_create_uas(ua, rdata, &tsx);
    if (status != PJ_SUCCESS)
    goto on_error;

    PJ_LOG(1, (THIS_FILE, "%s: create uas tsx", __FUNCTION__));

    pj_thread_sleep(3); //msec

HI, I make a further test for transaction module operation(create/destroy) and found that memory increment fast. The memory didn't decrease to normal level that it is before test. I think maybe memory leak occured aboute transaction(tsx). I dump the pool used list and can't find obvious memory leak, Please could anyone give me some advice, thanks. Test flow: 1. Receive a register request from client 2. Just create/destroy for 30000 times by the receive rdata Codes change for test: change-1: repeat to create/destroy transaction: { int cnt = 30000; for (i = 0; i < cnt; i++) { tsx = NULL; /* Create UAS transaction for this request. */ status = pjsip_tsx_create_uas(ua, rdata, &tsx); if (status != PJ_SUCCESS) goto on_error; PJ_LOG(1, (THIS_FILE, "%s: create uas tsx", __FUNCTION__)); pj_thread_sleep(3); //msec pjsip_tsx_terminate(tsx, 500); } } change-2: for create different key for transaction sip_transaction.c: int g_tsx_seq = 200000; /* * Create transaction key for RFC3161 compliant system. */ static pj_status_t create_tsx_key_3261( pj_pool_t *pool, pj_str_t *key, pjsip_role_e role, const pjsip_method *method, const pj_str_t *branch) { ....... .......... *p++ = '$'; snprintf(seq_buf, sizeof(seq_buf), "%d", g_tsx_seq); g_tsx_seq++; pj_memcpy(p, seq_buf, strlen(seq_buf)); p += strlen(seq_buf); /* Set length */ key->slen = p - key->ptr; } Following is test result: ====>Before test: $top -p 1178 top - 18:02:21 up 20 days, 8:45, 4 users, load average: 0.08, 0.06, 0.06 Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie %Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 1015500 total, 177584 free, 369236 used, 468680 buff/cache KiB Swap: 1048572 total, 338684 free, 709888 used. 418024 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 1178 www-data 20 0 332704 27640 3432 S 13.3 2.7 0:00.20 pjsua_00000011 $pmap -d 1178 0000000000400000 2040 r-x-- 0000000000000000 0fd:00000 pjsua-i686-pc-linux-gnu 00000000007fd000 4 r---- 00000000001fd000 0fd:00000 pjsua-i686-pc-linux-gnu 00000000007fe000 112 rw--- 00000000001fe000 0fd:00000 pjsua-i686-pc-linux-gnu 000000000081a000 17176 rw--- 0000000000000000 000:00000 [ anon ] 000000000325d000 6392 rw--- 0000000000000000 000:00000 [ anon ] 00007f293c000000 132 rw--- 0000000000000000 000:00000 [ anon ] 00007f293c021000 65404 ----- 0000000000000000 000:00000 [ anon ] 00007f2944000000 140 rw--- 0000000000000000 000:00000 [ anon ] 00007f2944023000 65396 ----- 0000000000000000 000:00000 [ anon ] 00007f294b7ff000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f294b800000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f294c000000 520 rw--- 0000000000000000 000:00000 [ anon ] 00007f294c082000 65016 ----- 0000000000000000 000:00000 [ anon ] 00007f295043c000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f295043d000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2950c3d000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f2950c3e000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f295143e000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f295143f000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2951c3f000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f2951c40000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2952440000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f2952441000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2952c41000 48 r-x-- 0000000000000000 0fd:00000 libnss_files-2.17.so 00007f2952c4d000 2044 ----- 000000000000c000 0fd:00000 libnss_files-2.17.so 00007f2952e4c000 4 r---- 000000000000b000 0fd:00000 libnss_files-2.17.so 00007f2952e4d000 4 rw--- 000000000000c000 0fd:00000 libnss_files-2.17.so mapped: 332708K writeable/private: 74664K shared: 0K pool detail: 00000011 18:03:57.089 sip_endpoint.c !Dumping endpoint 0x325e818: 00000011 18:03:57.090 cachpool Dumping caching pool: 00000011 18:03:57.090 cachpool Capacity=0, max_capacity=0, used_cnt=16 00000011 18:03:57.090 cachpool Dumping all active pools: 00000011 18:03:57.090 cachpool pjsua: 13976 of 17024 (82%) used 00000011 18:03:57.090 cachpool pept0x325e770: 2549432 of 2572096 (99%) used 00000011 18:03:57.090 cachpool pjsua-app: 9720 of 11024 (88%) used 00000011 18:03:57.090 cachpool tsxlayer: 8552 of 9216 (92%) used 00000011 18:03:57.090 cachpool ua0x34d9510: 4456 of 5120 (87%) used 00000011 18:03:57.090 cachpool med-ept: 2702864 of 2797568 (96%) used 00000011 18:03:57.090 cachpool codec-mgr: 328 of 512 (64%) used 00000011 18:03:57.090 cachpool evt mgr: 432 of 512 (84%) used 00000011 18:03:57.090 cachpool evsub: 3000 of 4608 (65%) used 00000011 18:03:57.090 cachpool udp0x387cbc0: 1144 of 1536 (74%) used 00000011 18:03:57.090 cachpool glck0x387cfe0: 680 of 1024 (66%) used 00000011 18:03:57.090 cachpool rtd0x387d610: 5080 of 12096 (41%) used 00000011 18:03:57.090 cachpool acc0x3880570: 640 of 1024 (62%) used 00000011 18:03:57.090 cachpool tcptp: 1448 of 1536 (94%) used 00000011 18:03:57.090 cachpool glck0x3880db0: 680 of 1024 (66%) used 00000011 18:03:57.090 cachpool acc0x38813e0: 656 of 1024 (64%) used 00000011 18:03:57.090 cachpool Total 5303088 of 5436944 (97 %) used! 00000011 18:03:57.090 sip_endpoint.c Endpoint pool capacity=2572096, used_size=2549432 00000011 18:03:57.090 sip_transport.c Outstanding transmit buffers: 0 00000011 18:03:57.090 sip_transport.c Dumping listeners: 00000011 18:03:57.090 sip_transport.c tcptp:9001 TCP:10.0.1.99:9001 00000011 18:03:57.090 sip_transport.c Dumping transports: 00000011 18:03:57.090 sip_transport.c udp0x387cbc0 udp 0.0.0.0:9001 [published as 10.0.1.99:9001] (refcnt=1) 00000011 18:03:57.090 sip_endpoint.c Timer heap has 3 entries 00000011 18:03:57.090 endpoint.c Dumping PJMEDIA capabilities: 00000011 18:03:57.090 endpoint.c Total number of installed codecs: 0 After test: $top -p 1178 top - 18:07:20 up 20 days, 8:50, 4 users, load average: 0.00, 0.02, 0.05 Tasks: 1 total, 0 running, 1 sleeping, 0 stopped, 0 zombie %Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 1015500 total, 88368 free, 446844 used, 480288 buff/cache KiB Swap: 1048572 total, 338684 free, 709888 used. 340412 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 1178 www-data 20 0 399056 104608 3524 S 0.3 10.3 0:04.30 pjsua_00000011 $pmap -d 1178 Address Kbytes Mode Offset Device Mapping 0000000000400000 2040 r-x-- 0000000000000000 0fd:00000 pjsua-i686-pc-linux-gnu 00000000007fd000 4 r---- 00000000001fd000 0fd:00000 pjsua-i686-pc-linux-gnu 00000000007fe000 112 rw--- 00000000001fe000 0fd:00000 pjsua-i686-pc-linux-gnu 000000000081a000 17176 rw--- 0000000000000000 000:00000 [ anon ] 000000000325d000 7208 rw--- 0000000000000000 000:00000 [ anon ] 00007f293c000000 93636 rw--- 0000000000000000 000:00000 [ anon ] 00007f2941b71000 37436 ----- 0000000000000000 000:00000 [ anon ] 00007f2944000000 140 rw--- 0000000000000000 000:00000 [ anon ] 00007f2944023000 65396 ----- 0000000000000000 000:00000 [ anon ] 00007f294b7ff000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f294b800000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f294c000000 520 rw--- 0000000000000000 000:00000 [ anon ] 00007f294c082000 65016 ----- 0000000000000000 000:00000 [ anon ] 00007f295043c000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f295043d000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2950c3d000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f2950c3e000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f295143e000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f295143f000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2951c3f000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f2951c40000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2952440000 4 ----- 0000000000000000 000:00000 [ anon ] 00007f2952441000 8192 rw--- 0000000000000000 000:00000 [ anon ] 00007f2952c41000 48 r-x-- 0000000000000000 0fd:00000 libnss_files-2.17.so mapped: 399060K writeable/private: 168984K shared: 0K pool detail: 00000011 18:08:36.368 sip_endpoint.c !Dumping endpoint 0x325e818: 00000011 18:08:36.368 cachpool Dumping caching pool: 00000011 18:08:36.368 cachpool Capacity=0, max_capacity=0, used_cnt=16 00000011 18:08:36.368 cachpool Dumping all active pools: 00000011 18:08:36.368 cachpool pjsua: 13976 of 17024 (82%) used 00000011 18:08:36.368 cachpool pept0x325e770: 3654952 of 3700096 (98%) used 00000011 18:08:36.368 cachpool pjsua-app: 9720 of 11024 (88%) used 00000011 18:08:36.368 cachpool tsxlayer: 8552 of 9216 (92%) used 00000011 18:08:36.368 cachpool ua0x34d9510: 4456 of 5120 (87%) used 00000011 18:08:36.369 cachpool med-ept: 2702864 of 2797568 (96%) used 00000011 18:08:36.369 cachpool codec-mgr: 328 of 512 (64%) used 00000011 18:08:36.369 cachpool evt mgr: 432 of 512 (84%) used 00000011 18:08:36.369 cachpool evsub: 3000 of 4608 (65%) used 00000011 18:08:36.369 cachpool udp0x387cbc0: 1144 of 1536 (74%) used 00000011 18:08:36.369 cachpool glck0x387cfe0: 680 of 1024 (66%) used 00000011 18:08:36.369 cachpool rtd0x387d610: 5080 of 12096 (41%) used 00000011 18:08:36.369 cachpool acc0x3880570: 640 of 1024 (62%) used 00000011 18:08:36.369 cachpool tcptp: 1448 of 1536 (94%) used 00000011 18:08:36.369 cachpool glck0x3880db0: 680 of 1024 (66%) used 00000011 18:08:36.369 cachpool acc0x38813e0: 656 of 1024 (64%) used 00000011 18:08:36.369 cachpool Total 6408608 of 6564944 (97 %) used! 00000011 18:08:36.369 sip_endpoint.c Endpoint pool capacity=3700096, used_size=3654952 00000011 18:08:36.369 sip_transport.c Outstanding transmit buffers: 0 00000011 18:08:36.369 sip_transport.c Dumping listeners: 00000011 18:08:36.369 sip_transport.c tcptp:9001 TCP:10.0.1.99:9001 00000011 18:08:36.369 sip_transport.c Dumping transports: 00000011 18:08:36.369 sip_transport.c udp0x387cbc0 udp 0.0.0.0:9001 [published as 10.0.1.99:9001] (refcnt=1) 00000011 18:08:36.369 sip_endpoint.c Timer heap has 3 entries 00000011 18:08:36.369 endpoint.c Dumping PJMEDIA capabilities: 00000011 18:08:36.369 endpoint.c Total number of installed codecs: 0 ------------------------------------------------------------------ 发件人：pjsip-request <pjsip-request@lists.pjsip.org> 发送时间：2019年4月14日(星期日) 00:01 收件人：pjsip <pjsip@lists.pjsip.org> 主　题：pjsip Digest, Vol 140, Issue 8 Send pjsip mailing list submissions to pjsip@lists.pjsip.org To subscribe or unsubscribe via the World Wide Web, visit http://lists.pjsip.org/mailman/listinfo/pjsip_lists.pjsip.org or, via email, send a message with subject or body 'help' to pjsip-request@lists.pjsip.org You can reach the person managing the list at pjsip-owner@lists.pjsip.org When replying, please edit your Subject line so it is more specific than "Re: Contents of pjsip digest..." Today's Topics: 1. Memory leak in my register-server on basis of pjsip-2.8 (shengy2019) ---------------------------------------------------------------------- Message: 1 Date: Sat, 13 Apr 2019 11:50:57 +0800 From: "shengy2019" <shengy2019@aliyun.com> To: "pjsip" <pjsip@lists.pjsip.org> Subject: [pjsip] Memory leak in my register-server on basis of pjsip-2.8 Message-ID: <0d3fa6fe-6a94-4e5f-8758-1be23468d91e.shengy2019@aliyun.com> Content-Type: text/plain; charset="utf-8" HI, I develop a registrar-server on basis of pjsip-2.8 stack, but it seems memory leak when client keeping high concurrency(500 ~ 1000 RPS till to 10000-total) TEST Procedure: Server procedure(TCP mode): 1. receive register without auth from client, server will response 401 2. receive reg with auth, server will create a transaction(UAS), start a timer(5s) for the transaction 3. when transaction timeout, destroy the transaction by pjsip_tsx_terminate(tsx, 603); client procedure(TCP mode): 1. send a new reg without auth to server by the same source addr[ip/port] 2. receive 401-reg from server 3. send reg with auth During the test, client recycle the upping register flow(1-5) at 1000 RPS till to total 10000; the memory of server increase at a fast speed, and the memory keep a high level even when client stopped. Anyway, memory-leak occured. I dump the pool-used detail, but no obvious pool-used was found. transaction that I created is no leak. If don't create/destroy tsx, the memory is ok. So I think transaction create/destroy frequently in high concurrency, causing memory leak. Maybe it it related to pool-cache's policy. The following is the pool detail: Please anyone could give me some advice how to debug this. system config: memory-1G KiB Mem : 1015500 total, 114236 free, 454364 used, 446900 buff/cache KiB Swap: 1048572 total, 316316 free, 732256 used. 343412 avail Mem ===>Before test: $top -p 4726 PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 4726 www-data 20 0 332700 27652 3440 S 0.0 2.7 0:00.20 pjsua_00000011 $pmap -d 4726 mapped: 332704K writeable/private: 74664K shared: 0K Pool detail: 00000011 17:06:21.467 sip_endpoint.c !Dumping endpoint 0x2b72818: 00000011 17:06:21.467 cachpool Dumping caching pool: 00000011 17:06:21.467 cachpool Capacity=0, max_capacity=0, used_cnt=16 00000011 17:06:21.467 cachpool Dumping all active pools: 00000011 17:06:21.467 cachpool pjsua: 13976 of 17024 (82%) used 00000011 17:06:21.467 cachpool pept0x2b72770: 2549432 of 2572096 (99%) used 00000011 17:06:21.467 cachpool pjsua-app: 9696 of 11024 (87%) used 00000011 17:06:21.467 cachpool tsxlayer: 8552 of 9216 (92%) used 00000011 17:06:21.467 cachpool ua0x2ded510: 4456 of 5120 (87%) used 00000011 17:06:21.468 cachpool med-ept: 2702864 of 2797568 (96%) used 00000011 17:06:21.468 cachpool codec-mgr: 328 of 512 (64%) used 00000011 17:06:21.468 cachpool evt mgr: 432 of 512 (84%) used 00000011 17:06:21.468 cachpool evsub: 3000 of 4608 (65%) used 00000011 17:06:21.468 cachpool udp0x3190bc0: 1144 of 1536 (74%) used 00000011 17:06:21.468 cachpool glck0x3190fe0: 680 of 1024 (66%) used 00000011 17:06:21.468 cachpool rtd0x3191610: 5080 of 12096 (41%) used 00000011 17:06:21.468 cachpool acc0x3194570: 640 of 1024 (62%) used 00000011 17:06:21.468 cachpool tcptp: 1448 of 1536 (94%) used 00000011 17:06:21.468 cachpool glck0x3194db0: 680 of 1024 (66%) used 00000011 17:06:21.468 cachpool acc0x31953e0: 656 of 1024 (64%) used 00000011 17:06:21.468 cachpool Total 5303064 of 5436944 (97 %) used! 00000011 17:06:21.468 sip_endpoint.c Endpoint pool capacity=2572096, used_size=2549432 00000011 17:06:21.468 sip_transport.c Outstanding transmit buffers: 0 00000011 17:06:21.468 sip_transport.c Dumping listeners: 00000011 17:06:21.468 sip_transport.c tcptp:9001 TCP:10.0.1.99:9001 00000011 17:06:21.468 sip_transport.c Dumping transports: 00000011 17:06:21.468 sip_transport.c udp0x3190bc0 udp 0.0.0.0:9001 [published as 10.0.1.99:9001] (refcnt=1) 00000011 17:06:21.468 sip_endpoint.c Timer heap has 3 entries 00000011 17:06:21.468 endpoint.c Dumping PJMEDIA capabilities: 00000011 17:06:21.468 endpoint.c Total number of installed codecs: 0 ===>After test $top -p 4726 PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 4726 www-data 20 0 399052 64540 3568 S 0.3 6.4 0:07.46 pjsua_00000011 $pmap -d 4726 mapped: 399056K writeable/private: 112640K shared: 0K Pool detail: 00000011 17:12:50.052 sip_endpoint.c !Dumping endpoint 0x2b72818: 00000011 17:12:50.052 cachpool Dumping caching pool: 00000011 17:12:50.052 cachpool Capacity=0, max_capacity=0, used_cnt=16 00000011 17:12:50.052 cachpool Dumping all active pools: 00000011 17:12:50.052 cachpool pjsua: 13976 of 17024 (82%) used 00000011 17:12:50.052 cachpool pept0x2b72770: 3065432 of 3104096 (98%) used 00000011 17:12:50.052 cachpool pjsua-app: 9696 of 11024 (87%) used 00000011 17:12:50.052 cachpool tsxlayer: 8552 of 9216 (92%) used 00000011 17:12:50.052 cachpool ua0x2ded510: 4456 of 5120 (87%) used 00000011 17:12:50.052 cachpool med-ept: 2702864 of 2797568 (96%) used 00000011 17:12:50.052 cachpool codec-mgr: 328 of 512 (64%) used 00000011 17:12:50.052 cachpool evt mgr: 432 of 512 (84%) used 00000011 17:12:50.052 cachpool evsub: 3000 of 4608 (65%) used 00000011 17:12:50.052 cachpool udp0x3190bc0: 1144 of 1536 (74%) used 00000011 17:12:50.052 cachpool glck0x3190fe0: 680 of 1024 (66%) used 00000011 17:12:50.052 cachpool rtd0x3191610: 5080 of 12096 (41%) used 00000011 17:12:50.052 cachpool acc0x3194570: 640 of 1024 (62%) used 00000011 17:12:50.052 cachpool tcptp: 1448 of 1536 (94%) used 00000011 17:12:50.052 cachpool glck0x3194db0: 680 of 1024 (66%) used 00000011 17:12:50.052 cachpool acc0x31953e0: 656 of 1024 (64%) used 00000011 17:12:50.052 cachpool Total 5819064 of 5968944 (97 %) used! 00000011 17:12:50.052 sip_endpoint.c Endpoint pool capacity=3104096, used_size=3065432 00000011 17:12:50.052 sip_transport.c Outstanding transmit buffers: 0 00000011 17:12:50.052 sip_transport.c Dumping listeners: 00000011 17:12:50.052 sip_transport.c tcptp:9001 TCP:10.0.1.99:9001 00000011 17:12:50.052 sip_transport.c Dumping transports: 00000011 17:12:50.052 sip_transport.c udp0x3190bc0 udp 0.0.0.0:9001 [published as 10.0.1.99:9001] (refcnt=1) 00000011 17:12:50.052 sip_endpoint.c Timer heap has 3 entries 00000011 17:12:50.052 endpoint.c Dumping PJMEDIA capabilities: 00000011 17:12:50.052 endpoint.c Total number of installed codecs: 0