Hi Folks:
This is what happens at a lower level. Do you have insight about the cause
?
The relevant code segments and the GDB stack traces follow below.
The main() process and IO_Task() are executing concurrently.
NOTE: The IO_Trigger_Task() has been eliminated and the uv_start_poll()
call is invoked in the IO_Task()
by a proxy routine and an async. wakeup in the main() process. The wakeup
is done in the make_incoming_connection()
callback routine by the following code segment.
poll_handle->data = (void *) cdesc;
//
// Start epoll() monitoring of the connection. The poll_start_proxy()
// routine executes in the IO_Task().
//
uv_async_init(&Poll_Loop, &cdesc->async_handle, poll_start_proxy);
cdesc->async_handle.data = (void *) cdesc;
uv_async_send(&cdesc->async_handle);
* The main() process is executing uv__finish_close() which is invoked via
uv_run() and uv__run_closing_handles().
It crashes in the call to uv__finish_close()
This is the code from unix/core.c: Line 210 is the first assert()
statement.
------------------------------------------------------------------------------
static void uv__finish_close(uv_handle_t* handle) {
/* Note: while the handle is in the UV_CLOSING state now, it's still
possible
* for it to be active in the sense that uv__is_active() returns true.
* A good example is when the user calls uv_shutdown(), immediately
followed
* by uv_close(). The handle is considered active at this point because
the
* completion of the shutdown req is still pending.
*/
#ifndef SUPRESS
assert(handle->flags & UV_CLOSING);
assert(!(handle->flags & UV_CLOSED));
#endif /* SUPRESS */
handle->flags |= UV_CLOSED;
* The IO_Task() thread is executing uv_poll_start() which is called from
start_poll_proxy() in
response to an async. request from the main() process. (uv__async_event()
makes the call to
start_poll_proxy()).
This is the code that executes in the IO_Task() from network_io.c
-----------------------------------------------------------------
//
// Executes in IO_Task
//
ROUTINE void poll_start_proxy(uv_async_t *async_handle)
{
int r;
CONN_DESC *cdesc = (CONN_DESC *) async_handle->data;
uv_poll_init(&Poll_Loop, cdesc->poll_handle, cdesc->fd);
if((r = uv_poll_start(cdesc->poll_handle, UV_READABLE, poll_callback))
< 0)
{
fprintf(stderr, "PROXY: IO_TASK - Polling Initiation Error %d:
%s\n", r, uv_err_name(r));
abort();
}
uv_close( (uv_handle_t *) async_handle, NULL);
return;
}
This is the code from unix/poll.c: Line 92 is the call to uv__poll_stop()
--------------------------------------------------------------------------
int uv_poll_start(uv_poll_t* handle, int pevents, uv_poll_cb poll_cb) {
int events;
assert((pevents & ~(UV_READABLE | UV_WRITABLE)) == 0);
assert(!(handle->flags & (UV_CLOSING | UV_CLOSED)));
uv__poll_stop(handle);
if (pevents == 0)
return 0;
events = 0;
if (pevents & UV_READABLE)
events |= UV__POLLIN;
if (pevents & UV_WRITABLE)
events |= UV__POLLOUT;
uv__io_start(handle->loop, &handle->io_watcher, events);
uv__handle_start(handle);
handle->poll_cb = poll_cb;
return 0;
}
main() Stack Trace
--------------------
#0 0x00007ffff751e267 in __GI_raise (sig=sig@entry=6) at
../sysdeps/unix/sysv/linux/raise.c:55
#1 0x00007ffff751feca in __GI_abort () at abort.c:89
#2 0x00007ffff751703d in __assert_fail_base (fmt=0x7ffff7679028
"%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
assertion=assertion@entry=0x41e678 "handle->flags & UV_CLOSING",
file=file@entry=0x41e668 "src/unix/core.c",
line=line@entry=210, function=function@entry=0x41e8b0
<__PRETTY_FUNCTION__.9522> "uv__finish_close")
at assert.c:92
#3 0x00007ffff75170f2 in __GI___assert_fail
(assertion=assertion@entry=0x41e678 "handle->flags & UV_CLOSING",
file=file@entry=0x41e668 "src/unix/core.c", line=line@entry=210,
function=function@entry=0x41e8b0 <__PRETTY_FUNCTION__.9522>
"uv__finish_close") at assert.c:101
#4 0x000000000040cd60 in uv__finish_close (handle=<optimized out>) at
src/unix/core.c:210
#5 uv__run_closing_handles (loop=0x830680 <Connect_Loop>) at
src/unix/core.c:261
#6 uv_run (loop=0x830680 <Connect_Loop>, mode=UV_RUN_DEFAULT) at
src/unix/core.c:328
#7 0x0000000000403e0f in main () at main.c:184
IO_Task() Stack Trace
-----------------------
#0 uv__poll_stop (handle=0x831bc0) at src/unix/poll.c:73
#1 0x000000000040e93f in uv_poll_start (handle=0x831bc0, pevents=1,
poll_cb=0x404576 <poll_callback>)
at src/unix/poll.c:92
#2 0x00000000004047d4 in poll_start_proxy (async_handle=0x639468
<Conn_Desc_Table+40>) at network_io.c:146
#3 0x000000000040c1dd in uv__async_event (loop=0x639000 <Poll_Loop>,
w=<optimized out>, nevents=<optimized out>)
at src/unix/async.c:89
#4 0x000000000040c29e in uv__async_io (loop=0x639000 <Poll_Loop>,
w=0x6391c8 <Poll_Loop+456>,
events=<optimized out>) at src/unix/async.c:160
#5 0x000000000041590d in uv__io_poll (loop=loop@entry=0x639000
<Poll_Loop>, timeout=-1) at src/unix/linux-core.c:319
#6 0x000000000040cac7 in uv_run (loop=0x639000 <Poll_Loop>,
mode=UV_RUN_DEFAULT) at src/unix/core.c:326
#7 0x000000000040484e in IO_Task (arg=0x0) at network_io.c:171
#8 0x0000000000412eb7 in uv__thread_start (arg=<optimized out>) at
src/unix/thread.c:49
#9 0x00007ffff7bc26aa in start_thread (arg=0x7ffff6ce7700) at
pthread_create.c:333
#10 0x00007ffff75efeed in clone () at
../sysdeps/unix/sysv/linux/x86_64/clone.S:109
Best Regards,
Paul R.
On Monday, December 21, 2020 at 5:39:09 PM UTC-8 [email protected]
wrote:
> Hi Folks:
>
> I think I see the heart of my problem. Everything appears to work correctly
> when you establish an incoming TCP connection and release it and the
> related
> Libuv handles--the uv_tcp_t connection handle and the uv_poll_t poll
> handle.
> (I revised the release code to do things the right way.)
>
> The comments about coalescing of uv_async_send() calls in the
> documentation is somewhat misleading.
> They should indicate that call with the same handle are synchronous. Also,
> I suspect that
> uv_async_send() is not reentrant.
>
> When you attempt another incoming connection the following things occur.
>
> Notice in 2.2, below that uv_start_loop() executes without being called.
> This
> doesn't make sense to me--at least on the surface. Can you think of a
> possible
> reason this occurs ?
>
> 1) The connection is successfully established--with uv_accept(), and the
> socket descriptor fd is the same as was used in the previous connection,
> in the main() process. (Occurs with uv_loop_t Session_Loop.)
>
> conn_handle = (uv_tcp_t *) malloc(sizeof(uv_tcp_t));
> if(conn_handle == NULL)
> {
> fprintf(stderr, "MAIN: No Connect Handle Memory\n");
> abort();
> }
>
> uv_tcp_init(&Connect_Loop, conn_handle);
> if(uv_accept(listen_handle, (uv_stream_t *) conn_handle) == 0)
> {
> uv_os_fd_t fd;
>
> uv_fileno((const uv_handle_t*) conn_handle, &fd);
> }
>
> 2.1) A poll handle is successfully allocated in the IO_Trigger_Task()
> thread.
> (No loop involved.)
>
> poll_handle = (uv_poll_t *) malloc(sizeof(uv_poll_t));
> if(poll_handle == NULL)
> {
> fprintf(stderr, "IO_TRIGGER_TASK: No Poll HAndle Memory\n");
> abort();
> }
>
> uv_poll_init(&Poll_Loop, poll_handle, pm->info);
> if((r = uv_poll_start(poll_handle, UV_READABLE, poll_callback)) <
> 0)
> {
> fprintf(stderr, "IO_TRIGGER_TASK: Polling Initiation Error %d:
> %s\n", r, uv_err_name(r));
> abort();
> }
>
> 2.2) uv_poll_start() is invoked via a call.
>
> uv_poll_init(&Poll_Loop, poll_handle, pm->info);
> if((r = uv_poll_start(poll_handle, UV_READABLE, poll_callback)) <
> 0)
> {
> fprintf(stderr, "IO_TRIGGER_TASK: Polling Initiation Error %d:
> %s\n", r, uv_err_name(r));
> abort();
> }
>
> 2.3) uv_poll_start() executes again without being called !
>
> This is what you see in GDB which is very strange since I know there is
> only
> one instance of the IO_Trigger_Task() running and it was not called a
> second time
> because the line before line 212 didn't execute a second time.
>
> Breakpoint 1, IO_Trigger_Task (arg=0x0) at network_io.c:212
> 212 if((r = uv_poll_start(poll_handle, UV_READABLE,
> poll_callback)) < 0)
> (gdb) bt
> #0 IO_Trigger_Task (arg=0x0) at network_io.c:212
> #1 0x0000000000413017 in uv__thread_start (arg=<optimized out>) at
> src/unix/thread.c:49
> #2 0x00007ffff7bc26aa in start_thread (arg=0x7ffff64e6700) at
> pthread_create.c:333
> #3 0x00007ffff75efeed in clone () at
> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
> (gdb) s
> uv_poll_start (handle=0x7fffec0008c0, pevents=1, poll_cb=0x404599
> <poll_callback>) at src/unix/poll.c:89
> 89 assert((pevents & ~(UV_READABLE | UV_WRITABLE)) == 0);
> (gdb) s
> 86 int uv_poll_start(uv_poll_t* handle, int pevents, uv_poll_cb
> poll_cb) {
> (gdb) bt
> #0 uv_poll_start (handle=0x7fffec0008c0, pevents=1, poll_cb=0x404599
> <poll_callback>) at src/unix/poll.c:86
> #1 0x00000000004048bb in IO_Trigger_Task (arg=0x0) at network_io.c:212
> #2 0x0000000000413017 in uv__thread_start (arg=<optimized out>) at
> src/unix/thread.c:49
> #3 0x00007ffff7bc26aa in start_thread (arg=0x7ffff64e6700) at
> pthread_create.c:333
> #4 0x00007ffff75efeed in clone () at
> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
> (gdb)
>
> This is the relevant code of the IO_Trigger_Task() thread.
>
>
> for(;;)
> {
> //
> // Wait for a message from the main() process.
> //
> pm = WAIT_IO_Trigger_MSG();
>
> poll_handle = (uv_poll_t *) malloc(sizeof(uv_poll_t));
> if(poll_handle == NULL)
> {
> fprintf(stderr, "IO_TRIGGER_TASK: No Poll HAndle Memory\n");
> abort();
> }
>
> uv_poll_init(&Poll_Loop, poll_handle, pm->info);
> //
> // Start epoll() monitoring of the connection.
> //
> if((r = uv_poll_start(poll_handle, UV_READABLE, poll_callback)) <
> 0)
> {
> fprintf(stderr, "IO_TRIGGER_TASK: Polling Initiation Error %d:
> %s\n", r, uv_err_name(r));
> abort();
> }
>
> MSG_FREE(pm);
> }
>
>
>
> 2.4) The polling callback function never executes.
>
> NOTE: The polling loop, Poll_Loop, of type uv_loop_t is already running
> and was started,
> in the IO_Task() thread, at startup time as follows.
>
> uv_loop_init(&Poll_Loop);
> for(;;)
> {
> r = uv_run(&Poll_Loop, UV_RUN_DEFAULT);
> if(r)
> fprintf(stderr, "IO_TASK: Run Error %d\n", r);
> }
>
>
> This is the sequence of operations used to free the first connection.
>
> 1) Release the uv_poll_t poll handle in the IO_Task() from the
> Protocol_Task()
>
> //
> // This causes immediate socket disconnection when it is closed.
> //
> spec.l_onoff = TRUE;
> spec.l_linger = 0;
> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec, sizeof(spec) );
> //
> // poll_release_proxy() executes in the IO_Task() and releases poll
> handle.
> //
> uv_async_init(&Poll_Loop, &cdesc->async_handle, poll_release_proxy);
> cdesc->async_handle.data = (void *) cdesc;
> uv_async_send(&cdesc->async_handle);
>
> 1.1) Wait for poll handle to be freed and then release the async. handle.
>
> uv_close((uv_handle_t *) &cdesc->async_handle, NULL);
>
> 2) Release the uv_tcp_t connect handle in the main() process from the
> Protocol_Task()
>
> uv_async_init(&Connect_Loop, &cdesc->async_handle, conn_release_proxy);
> cdesc->async_handle.data = (void *) cdesc;
> uv_async_send(&cdesc->async_handle);
>
> 2.1) Wait for the connect handle to be free and then release the async.
> handle.
>
> uv_close((uv_handle_t *) &cdesc->async_handle, NULL);
>
> 3) Do protocol bookkeeping.
>
> This is the code of the proxy callback routines and the close callback
> routine.
>
> //
> // This routine executes asynchronously and frees a handle.
> // It is invoked in the follows two cases.
> //
> // * When the main process invokes poll_release_proxy()
> //
> // * When the IO_Task invokes conn_release_proxy().
> //
> ROUTINE void close_callback(uv_handle_t *handle)
> {
> SDU *msg;
>
> CONN_DESC *cdesc = (CONN_DESC *) handle->data;
>
> free(handle);
>
> ENTER_MUTEX(&Service_Q_Mutex);
> //
> // Set the state correctly and validate the state.
> //
> switch(cdesc->release_state)
> {
> case RS_POLL_HANDLE_PEND:
> cdesc->release_state = RS_POLL_HANDLE_FREE;
> break;
> case RS_CONN_HANDLE_PEND:
> cdesc->release_state = RS_CONN_HANDLE_FREE;
> break;
> default:
> fprintf(stderr, "CLOSE_PROXY - BUG: Invalid Release State = %d\n",
> cdesc->release_state);
> abort();
> }
> EXIT_MUTEX(&Service_Q_Mutex);
>
> //
> // Send a notification message to the Protocol_Task.
> //
> msg = MSG_ALLOC(0, FALSE);
> msg->class = C_NOTIFY;
> msg->type = T_HANDLE_FREE;
> msg->info = 0;
>
> SEND_SDU(cdesc, msg);
>
> return;
> }
>
>
> //
> // This routine is invoked by the IO_Task() in response to an async.
> wakeup by the Protocol_Task()
> // during TCP connection termination. It release the resources used by the
> Poll_Loop.
> //
> ROUTINE void poll_release_proxy(uv_async_t *async_handle)
> {
> CONN_DESC *cdesc = (CONN_DESC *) async_handle->data;
>
> //
> // Stop polling operations before closing the handle.
> //
> uv_poll_stop(cdesc->poll_handle);
> cdesc->poll_handle->data = (void *) cdesc;
> uv_close((uv_handle_t *) cdesc->poll_handle, close_callback);
>
> return;
> }
>
> //
> // This routine is invoked by the main process in response to an async.
> wakeup by the Protocol_Task()
> // during TCP connection termination. It release the resources used by the
> Connect_Loop.
> //
> ROUTINE void conn_release_proxy(uv_async_t *async_handle)
> {
> CONN_DESC *cdesc = (CONN_DESC *) async_handle->data;
>
> cdesc->conn_handle->data = (void *) cdesc;
> uv_close((uv_handle_t *) cdesc->conn_handle, close_callback);
>
> return;
> }
>
> Best Regards,
>
> Paul R.
>
> On Sunday, December 20, 2020 at 1:23:52 PM UTC-8 [email protected]
> wrote:
>
>> Hi Folks:
>>
>> With limited testing the problem ceases to happen if you force uv_run()
>> in the IO_Task()
>> enough to finish its pending work. As an interim measure I do this by
>> making the
>> Protocol_Task() to yield the CPU after calling uv_stop() and
>> up_poll_stop() as follows in
>> the RELEASE_CONNECTION() routine. This appears to cause IO_Task() to be
>> scheduled and run
>> but I am not all all convinced this is a reliable technique.
>>
>> //
>> // Deactive and release the poll handle.
>> // You have stop the Poll_Loop to deactivate and deallocate the
>> poll handle.
>> //
>> uv_stop(&Poll_Loop);
>>
>> uv_poll_stop(cdesc->poll_handle);
>> #ifdef CLOSE_KLUDGE2
>> //
>> // Try to let run() in the IO_Task() finish pending work by
>> yielding the CPU.
>> //
>> for(k = 0; k < 10; k++) pthread_yield();
>> #endif // CLOSE_KLUDGE2
>> uv_close((uv_handle_t *) cdesc->poll_handle, close_callback);
>>
>>
>> Best Regards,
>>
>> Paul R.
>>
>>
>> On Sunday, December 20, 2020 at 10:13:34 AM UTC-8 [email protected]
>> wrote:
>>
>>> Hi Folks:
>>>
>>> I made some progress on the problem but it is definitely not solved. The
>>> updated code
>>> and more diagnostic code are included in the message.
>>>
>>> NOTE: I am using the GIT HUB distribution from the following link on
>>> Ubuntu Linux version 15.04.
>>>
>>> https://github.com/nikhilm/uvbook
>>>
>>> The Libuv software package looks like version 1.3.0.
>>>
>>> I have had to take extraordinary measures to make connection release
>>> reliable.
>>> The relevant code is included at near end of this message and the
>>> extraordinary
>>> measures are in the CLOSE_KLUDGE sections. The difficulty arises because
>>> the
>>> Libuv loops are not used in the Protocol_Task() yet it must affect
>>> operations
>>> on those loops to release handles. It would be nice if Libuv included an
>>> API
>>> for releasing handles reliably which could be called from any task.
>>>
>>> Connection release still fails about 15% of the time in which case a
>>> crash occurs
>>> and the following diagnostic is displayed.
>>>
>>> pexd: src/unix/core.c:210: uv__finish_close: Assertion
>>> `!(handle->flags & UV_CLOSED)' failed.
>>>
>>> More diagnostic information follows. Do you know what causes this crash
>>> ?
>>>
>>> Best Regards,
>>>
>>> Paul Romero
>>>
>>>
>>> Crash Diagnostics
>>> -----------------
>>> The crash occurs when run() is executing in the IO_Task() in
>>> network_io.c according to the following
>>> GBD stack trace.
>>>
>>> #0 0x00007f281754c267 in __GI_raise (sig=sig@entry=6) at
>>> ../sysdeps/unix/sysv/linux/raise.c:55
>>> #1 0x00007f281754deca in __GI_abort () at abort.c:89
>>> #2 0x00007f281754503d in __assert_fail_base (fmt=0x7f28176a7028
>>> "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n",
>>> assertion=assertion@entry=0x41e093 "!(handle->flags & UV_CLOSED)",
>>> file=file@entry=0x41e068 "src/unix/core.c",
>>> line=line@entry=210, function=function@entry=0x41e2b0
>>> <__PRETTY_FUNCTION__.9522> "uv__finish_close") at assert.c:92
>>> #3 0x00007f28175450f2 in __GI___assert_fail
>>> (assertion=assertion@entry=0x41e093 "!(handle->flags & UV_CLOSED)",
>>> file=file@entry=0x41e068 "src/unix/core.c", line=line@entry=210,
>>> function=function@entry=0x41e2b0 <__PRETTY_FUNCTION__.9522>
>>> "uv__finish_close") at assert.c:101
>>> #4 0x000000000040c967 in uv__finish_close (handle=<optimized out>) at
>>> src/unix/core.c:210
>>> #5 uv__run_closing_handles (loop=0x638080 <Poll_Loop>) at
>>> src/unix/core.c:259
>>> #6 uv_run (loop=0x638080 <Poll_Loop>, mode=UV_RUN_DEFAULT) at
>>> src/unix/core.c:326
>>> #7 0x0000000000404962 in IO_Task (arg=0x0) at network_io.c:226
>>> #8 0x0000000000412ad7 in uv__thread_start (arg=<optimized out>) at
>>> src/unix/thread.c:49
>>> #9 0x00007f2817bf06aa in start_thread (arg=0x7f2816d15700) at
>>> pthread_create.c:333
>>> #10 0x00007f281761deed in clone () at
>>> ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
>>>
>>> However, the GDB thread information indicates that RELEASE_CONNECTION(),
>>> in protocol.c, is executing
>>> in the Protocol_Task() when the crash occurs.
>>>
>>> Id Target Id Frame
>>> 6 Thread 0x7f2817516700 (LWP 3424) syscall () at
>>> ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
>>> 5 Thread 0x7f2816514700 (LWP 3426) pthread_cond_wait@@GLIBC_2.3.2 ()
>>> at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
>>> 4 Thread 0x7f2818003700 (LWP 3423) syscall () at
>>> ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
>>> 3 Thread 0x7f2815512700 (LWP 3428) pthread_cond_wait@@GLIBC_2.3.2 ()
>>> at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
>>> 2 Thread 0x7f2815d13700 (LWP 3427) 0x0000000000404500 in
>>> RELEASE_CONNECTION (cdesc=0x6384c0 <Conn_Desc_Table>)
>>> at protocol.c:357
>>> * 1 Thread 0x7f2816d15700 (LWP 3425) 0x00007f281754c267 in __GI_raise
>>> (sig=sig@entry=6)
>>> at ../sysdeps/unix/sysv/linux/raise.c:55
>>>
>>> Line 357 of protocol.c is as follows.
>>>
>>> while(WaitClose[cdesc->index]);
>>>
>>> Wait_Close[] is only modified in two cases and only in the
>>> Protocol_Task().
>>>
>>> 1) It is initialized to a handle address in RELEASE_CONNECTION() in the
>>> Protocol_Task().
>>> 2) It is cleared in the uv_close() callback routine close_callback().
>>>
>>>
>>> Code
>>> -----
>>>
>>> #define CLOSE_KLUDGE
>>>
>>> extern uv_loop_t Poll_Loop;
>>> extern uv_loop_t Connect_Loop;
>>>
>>> #ifdef CLOSE_KLUDGE
>>> uv_handle_t *WaitClose[MAX_CONN_DESC] = { NULL };
>>> #endif // CLOSE_KLUDGE
>>>
>>> ROUTINE void close_callback(uv_handle_t *handle)
>>> {
>>> int k;
>>>
>>> free(handle);
>>>
>>> #ifdef CLOSE_KLUDGE
>>> //
>>> // Determine if the handle is being closed.
>>> //
>>> for(k = 0; k < MAX_CONN_DESC; k++)
>>> {
>>> if(WaitClose[k] == handle)
>>> {
>>> //
>>> // Closure is complete.
>>> //
>>> WaitClose[k] = NULL;
>>> break;
>>> }
>>> }
>>> #endif // CLOSE_KLUDGE
>>>
>>> return;
>>> }
>>>
>>> ROUTINE void RELEASE_CONNECTION(CONN_DESC *cdesc)
>>> {
>>> uv_async_t as_handle;
>>> struct linger spec;
>>>
>>> if(N_Sockets > 0)
>>> N_Sockets--;
>>> //
>>> // This causes immediate socket disconnection when it is closed.
>>> //
>>> spec.l_onoff = TRUE;
>>> spec.l_linger = 0;
>>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec, sizeof(spec) );
>>>
>>> if(cdesc->poll_handle)
>>> {
>>> #ifdef CLOSE_KLUDGE
>>> WaitClose[cdesc->index] = (uv_handle_t *) cdesc->poll_handle;
>>> #endif // CLOSE_KLUDGE
>>> //
>>> // Deactive and release the poll handle.
>>> // You have stop the Poll_Loop to deactivate and deallocate the
>>> poll handle.
>>> //
>>> uv_stop(&Poll_Loop);
>>>
>>> uv_poll_stop(cdesc->poll_handle);
>>> uv_close((uv_handle_t *) cdesc->poll_handle, close_callback);
>>> //
>>> // Wake up the Poll_Loop in the IO_Task()
>>> //
>>> uv_async_init(&Poll_Loop, &as_handle, NULL);
>>> uv_async_send(&as_handle);
>>> uv_close((uv_handle_t *) &as_handle, NULL);
>>> #ifdef CLOSE_KLUDGE
>>> //
>>> // Wait for the handle to be closed and deallocated.
>>> //
>>> while(WaitClose[cdesc->index]);
>>> #endif // CLOSE_KLUDGE
>>> }
>>>
>>> if(cdesc->conn_handle)
>>> {
>>> #ifdef CLOSE_KLUDGE
>>> WaitClose[cdesc->index] = (uv_handle_t *) cdesc->conn_handle;
>>> #endif // CLOSE_KLUDGE
>>> //
>>> // Close and deallocate the connect handle in order to close the
>>> socket connecction.
>>> // You have to wake up the Connect_Loop for the close_callback()
>>> // routine to execute.
>>> //
>>> uv_close((uv_handle_t *) cdesc->conn_handle, close_callback);
>>> //
>>> // Wake up the Connect_Loop in the main() process.
>>> //
>>> uv_async_init(&Connect_Loop, &as_handle, NULL);
>>> uv_async_send(&as_handle);
>>> uv_close((uv_handle_t *) &as_handle, NULL);
>>> #ifdef CLOSE_KLUDGE
>>> //
>>> // Wait for the handle and socket connection to be release and
>>> closed.
>>> //
>>> while(WaitClose[cdesc->index]);
>>> #endif // CLOSE_KLUDGE
>>> }
>>>
>>>
>>> ENTER_MUTEX(&Service_Q_Mutex);
>>> DELETE_CONN(cdesc);
>>> cdesc->fd = -1;
>>> flush_msg(&cdesc->task_input_q);
>>> EXIT_MUTEX(&Service_Q_Mutex);
>>>
>>> return;
>>> }
>>>
>>> On Sunday, December 20, 2020 at 3:47:07 AM UTC-8 [email protected]
>>> wrote:
>>>
>>>> Hi Folks:
>>>>
>>>> My Libuv based Server performs all its functions correctly except for
>>>> TCP connection termination.
>>>>
>>>> Each TCP connection has uv_tcp_t connection handle and uv_poll_t handle
>>>> whose allocation
>>>> and operation are explained below. When the Protocol_Task() thread
>>>> needs to terminate
>>>> a connection, it must stop polling, terminate the TCP socket
>>>> connection, and deallocate
>>>> the handles.
>>>>
>>>> NOTE: I am using the GIT HUB distribution from the following link on
>>>> Ubuntu Linux version 15.04.
>>>>
>>>> https://github.com/nikhilm/uvbook
>>>>
>>>> I have tried the following two approaches.
>>>>
>>>> 1) Just use uv_poll_stop() to terminate polling and uv_close() to
>>>> terminate the TCP connection.
>>>>
>>>> 2) Use uv_poll_stop() to terminate polling and the using
>>>> uv_queue_work() and uv_async_send() to
>>>> wake up the Connect_Loop, in the main() process described below, so
>>>> it can terminate the
>>>> TCP connection, by proxy, with uv_close().
>>>>
>>>> In both cases the following problem occurs. The callback routine
>>>> supplied to uv_close()
>>>> does not execute until another incoming TCP connection occurs, and in
>>>> most cases,
>>>> the Pool_Loop, in the IO_Task() described below, stops invoking it
>>>> callback routine--
>>>> poll_callback(). In case 2, a crash almost alway ensues. (I probably am
>>>> not using
>>>> uv_async_send() correctly.)
>>>>
>>>> Do I have a fundamental misunderstanding of how Libuv works or am I
>>>> doing something wrong ?
>>>>
>>>> Also, I strongly suspect using Linux recv() to read data is not optimal
>>>> when epoll() is
>>>> being used. My understanding is that there is a way to pass buffers to
>>>> epoll() such that
>>>> data will automatically be inserted in them when a UV_READABLE event
>>>> occurs. Do you have
>>>> any advice about this ?
>>>>
>>>> An overview of my Server and the relevant code follow.
>>>>
>>>> Best Regards,
>>>>
>>>> Paul Romero
>>>>
>>>> Multi-Connection TCP Server Functional Architecture Overview
>>>>
>>>> -----------------------------------------------------------------------------------------
>>>> There is a connection descriptor for each incoming TCP connection which
>>>> contains all data
>>>> needed to manage the connection and perform the relevant functions.
>>>>
>>>> When the main() process detects an incoming TCP connection, it sends a
>>>> notification message to the
>>>> IO_Trigger_Task(). The IO_Trigger_Task() then sets up epoll()
>>>> monitoring of incoming TCP data
>>>> for that connection.
>>>>
>>>> Subsequently, the IO_Task() invokes poll_callback() when incoming data
>>>> is available, reads a chunk
>>>> of data, and sends a protocol message to the Protocol_Task() when a
>>>> complete protocol message is
>>>> recognized.
>>>>
>>>> The Timer_Task() sends an expiration notification message to the
>>>> Protocol_Task() when a protocol
>>>> timer expires.
>>>>
>>>> The Protocol_Task() send messages to the Send_Op_Task() for
>>>> transmission across the network.
>>>> It spawns a DB Operation Task to perform slow data base operations and
>>>> the DB Operation Task
>>>> notifies the Protocol_Task() when the operation is complete and then
>>>> terminates.
>>>>
>>>> Loops of type uv_loop_t
>>>> -----------------------
>>>> * Connect_Loop
>>>> * Pool_Loop
>>>> * Timer_Loop`
>>>>
>>>> Tasks: All Libuv thread tasks run concurrently and are launched by
>>>> main() at startup time.
>>>>
>>>> ------------------------------------------------------------------------------------------
>>>> * main(): A Linux process that runs the Connect_Loop to detect incoming
>>>> TCP connections.
>>>> The make_incoming_connection() callback routine accepts incoming
>>>> connections and
>>>> allocates a uv_tcp_t handle on a per connection basis
>>>>
>>>> * IO_Trigger_Task(): A Libuv thread that sets up epoll() plumbing for
>>>> the IO_Task()
>>>> when an incoming TCP connection occurs. It allocates a uv_poll_t
>>>> handle, on a per
>>>> connection basis, and calls uv_poll_start() to initiate epoll()
>>>> operation with the
>>>> Poll_Loop in the IO_Task(). It configures the handle to detect
>>>> UV_READABLE events and
>>>> handles them with the poll_callback() routine. However, it does not
>>>> run the Poll_Loop.
>>>> (Basically, this task just sets up plumbing.)
>>>>
>>>> * IO_Task(): A Libuv thread that runs the Poll_Loop to handle incoming
>>>> TCP data, on a per
>>>> connection basis. The poll_callback() routine executes and uses
>>>> normal Linux recv() to read
>>>> chunks of data, in non-blocking mode, when a UV_READABLE event occurs.
>>>>
>>>> * Timer_Task(): A Libuv thread that runs the Time_Loop to handle ticks,
>>>> and whose main
>>>> function is to detect protocol timer expiration. The tick duration is
>>>> configured with
>>>> is configured with uv_timer_init() and uv_timer_start(), and ticks
>>>> are handled by the
>>>> timer_callback() routine.
>>>>
>>>> * Protocol_Task(): A Libuv thread that handles protocol messages sent
>>>> to it by the following tasks
>>>> on per connection basis: IO_Task(), Timer_Task(), DB Operation Tasks.
>>>> DB Operation Libuv thread tasks
>>>> are spawned by the Protocol_Task() to perform slow database
>>>> operations and send a notification message
>>>> to the Protocol_Task() upon completion of the operation.
>>>>
>>>> * Send_Op_Task(): A Libuv thread that transmits all network bound
>>>> messages with normal
>>>> Linux send() on a per connection basis.
>>>>
>>>>
>>>> Approach 1 Code
>>>> -------------
>>>> ROUTINE void close_callback(uv_handle_t *handle)
>>>> {
>>>>
>>>> free(handle);
>>>> return;
>>>> }
>>>>
>>>> ROUTINE void RELEASE_CONNECTION(CONN_DESC *cdesc)
>>>> {
>>>> struct linger spec;
>>>> int r;
>>>>
>>>> if(N_Sockets > 0)
>>>> N_Sockets--;
>>>>
>>>> if(cdesc->poll_handle)
>>>> {
>>>> uv_poll_stop(cdesc->poll_handle);
>>>> free((void *) cdesc->poll_handle);
>>>> }
>>>>
>>>> if(cdesc->conn_handle)
>>>> {
>>>> struct linger spec;
>>>>
>>>> spec.l_onoff = TRUE;
>>>> spec.l_linger = 0;
>>>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec,
>>>> sizeof(spec) );
>>>>
>>>> uv_close((uv_handle_t *) cdesc->conn_handle, close_callback);
>>>> }
>>>>
>>>> ENTER_MUTEX(&Service_Q_Mutex);
>>>> DELETE_CONN(cdesc);
>>>> cdesc->fd = -1;
>>>> flush_msg(&cdesc->task_input_q);
>>>> EXIT_MUTEX(&Service_Q_Mutex);
>>>>
>>>> return;
>>>> }
>>>>
>>>> Approach 2 Code
>>>> -----------------
>>>> ROUTINE void close_callback(uv_handle_t *handle)
>>>> {
>>>> free(handle);
>>>> return;
>>>> }
>>>>
>>>> typedef struct close_template {
>>>> uv_handle_t *handle;
>>>> void (*callback) (uv_handle_t *);
>>>> } CLOSE_TEMPLATE;
>>>>
>>>> ROUTINE void close_proxy(uv_work_t *data)
>>>> {
>>>> CLOSE_TEMPLATE *cparam = (CLOSE_TEMPLATE *) cparam;
>>>>
>>>> uv_close(cparam->handle, cparam->callback);
>>>> return;
>>>> }
>>>>
>>>>
>>>> extern uv_loop_t Connect_Loop;
>>>> static CLOSE_TEMPLATE close_data;
>>>>
>>>> ROUTINE void RELEASE_CONNECTION(CONN_DESC *cdesc)
>>>> {
>>>> uv_work_t wreq;
>>>> uv_async_t as_handle;
>>>> struct linger spec;
>>>>
>>>> if(N_Sockets > 0)
>>>> N_Sockets--;
>>>>
>>>> //
>>>> // Stop this. TBD: Might need to do this via proxy in the IO_Task()
>>>> Poll_Loop.
>>>> //
>>>> uv_poll_stop(cdesc->poll_handle);
>>>>
>>>> uv_async_init(&Connect_Loop, &as_handle, NULL);
>>>>
>>>> close_data.handle = (uv_handle_t *) cdesc->conn_handle;
>>>> close_data.callback = close_callback;
>>>> //
>>>> // Call uv_close() in the close_proxy()
>>>> //
>>>> wreq.data = (void *) &close_data;
>>>> uv_queue_work(&Connect_Loop, &wreq, close_proxy, NULL);
>>>>
>>>> spec.l_onoff = TRUE;
>>>> spec.l_linger = 0;
>>>> setsockopt(cdesc->fd, SOL_SOCKET, SO_LINGER, &spec, sizeof(spec) );
>>>>
>>>> uv_async_send(&as_handle);
>>>> uv_close((uv_handle_t *) &as_handle, NULL);
>>>>
>>>> free(cdesc->poll_handle);
>>>>
>>>> ENTER_MUTEX(&Service_Q_Mutex);
>>>> DELETE_CONN(cdesc);
>>>> cdesc->fd = -1;
>>>> flush_msg(&cdesc->task_input_q);
>>>> EXIT_MUTEX(&Service_Q_Mutex);
>>>>
>>>> return;
>>>> }
>>>>
>>>
--
You received this message because you are subscribed to the Google Groups
"libuv" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/libuv/893eb3ed-685e-416b-9d05-3501588095den%40googlegroups.com.
