From 96a49b724295422ceb6465553fdf0da20e5a3b6d Mon Sep 17 00:00:00 2001
From: Kumar Kaushik <kaushikk@vmware.com>
Date: Mon, 29 Jan 2018 19:30:18 -0800
Subject: [PATCH] Handling timeout events for faulty application callback

Change-Id: I9a412f6d7cf2c3345c73f0581906829bd183e445
---
 common/sockinterface.c               |  26 ++-
 include/public/vmrest.h              |   1 +
 include/vmrestdefines.h              |   9 +
 server/restengine/httpAllocStruct.c  |   4 +-
 server/restengine/httpProtocolHead.c |   7 +-
 transport/posix/socket.c             | 425 ++++++++++++++++++-----------------
 transport/posix/structs.h            |   1 +
 7 files changed, 257 insertions(+), 216 deletions(-)

diff --git a/common/sockinterface.c b/common/sockinterface.c
index 403189c..9e7d404 100644
--- a/common/sockinterface.c
+++ b/common/sockinterface.c
@@ -327,14 +327,13 @@ VmRESTHandleSocketEvent(
             break;
 
         case VM_SOCK_EVENT_TYPE_CONNECTION_TIMEOUT:
-            /**** Free Associated memory ****/
-            VMREST_LOG_DEBUG(pRESTHandle,"%s","EVENT-HANDLER: Connection timeout happened..Disconnecting client ...");
-            dwError = VmRESTOnConnectionTimeout(
+             VMREST_LOG_DEBUG(pRESTHandle,"%s","EVENT-HANDLER: Connection timeout happened..Disconnecting client");
+             dwError = VmRESTOnConnectionTimeout(
                           pRESTHandle,
                           pSocket
                           );
-            BAIL_ON_VMREST_ERROR(dwError);
-            break;
+             BAIL_ON_VMREST_ERROR(dwError);
+             break;
 
         case VM_SOCK_EVENT_TYPE_UNKNOWN:
              VMREST_LOG_DEBUG(pRESTHandle,"%s","EVENT-HANDLER: Unknown Socket Event, do nothing");
@@ -436,6 +435,7 @@ VmRESTTcpReceiveNewData(
     char*                            pszBuffer = NULL;
     uint32_t                         nProcessed = 0;
     uint32_t                         nBufLen = 0;
+    uint32_t                         ret = REST_ENGINE_SUCCESS;
     BOOLEAN                          bNextIO = FALSE;
 
     if (!pSocket || !pRESTHandle || !pQueue)
@@ -517,7 +517,7 @@ VmRESTTcpReceiveNewData(
 
 cleanup:
 
-   if (!bNextIO)
+   if (!bNextIO && dwError != REST_ENGINE_ERROR_DOUBLE_FAILURE)
    {
        VMREST_LOG_DEBUG(pRESTHandle,"%s","Calling closed connection....");
        /**** Close connection ****/ 
@@ -542,6 +542,20 @@ VmRESTTcpReceiveNewData(
 error:
 
     VMREST_LOG_ERROR(pRESTHandle,"ERROR code %u", dwError);
+
+    if ((dwError == VMREST_TRANSPORT_DEFERRED_TIMEOUT_PROCESS) && (bNextIO))
+    {
+         ret = VmRESTOnConnectionTimeout(
+                     pRESTHandle,
+                     pSocket
+                     );
+
+         if (ret != REST_ENGINE_SUCCESS)
+         {
+            VMREST_LOG_ERROR(pRESTHandle,"Double failure on deferred timeout processing dwError, = %u", dwError);
+            dwError = REST_ENGINE_ERROR_DOUBLE_FAILURE;
+         }
+    }
     goto cleanup;
 }
 
diff --git a/include/public/vmrest.h b/include/public/vmrest.h
index 3988266..5219952 100644
--- a/include/public/vmrest.h
+++ b/include/public/vmrest.h
@@ -32,6 +32,7 @@
 /**** REST ENGINE ERROR CODES ****/
 #define     REST_ENGINE_SUCCESS                            0
 #define     REST_ENGINE_FAILURE                            1
+#define     REST_ENGINE_ERROR_DOUBLE_FAILURE               99
 #define     REST_ERROR_MISSING_CONFIG                      100
 #define     REST_ERROR_INVALID_CONFIG                      101
 #define     REST_ERROR_NO_MEMORY                           102
diff --git a/include/vmrestdefines.h b/include/vmrestdefines.h
index 914422a..8a37628 100644
--- a/include/vmrestdefines.h
+++ b/include/vmrestdefines.h
@@ -63,6 +63,15 @@
 #define VMREST_TRANSPORT_COND_INIT_FAILED               61125
 #define VMREST_TRANSPORT_SERVER_THREAD_START_FAILED     61126
 
+
+#define VMREST_TRANSPORT_DEFERRED_TIMEOUT_PROCESS       61127
+#define VMREST_TRANSPORT_SOCK_READ_FAILED               61128
+#define VMREST_TRANSPORT_SOCK_WRITE_FAILED              61129
+#define VMREST_TRANSPORT_SOCK_GET_HANDLE_FAILED         61130
+#define VMREST_TRANSPORT_SOCK_SET_HANDLE_FAILED         61131
+#define VMREST_TRANSPORT_SOCK_DATA_OVER_LIMIT           61132
+#define VMREST_TRANSPORT_REMOTE_CONN_CLOSED             61133
+
 #define ERROR_TRANSPORT_INVALID_PARAMS                  61040
 #define ERROR_TRANSPORT_VALIDATION_FAILED               61041
 
diff --git a/server/restengine/httpAllocStruct.c b/server/restengine/httpAllocStruct.c
index 64cb846..27aa9ed 100644
--- a/server/restengine/httpAllocStruct.c
+++ b/server/restengine/httpAllocStruct.c
@@ -519,11 +519,13 @@ VmRESTAllocateMiscQueue(
     PMISC_HEADER_QUEUE               pMiscQueue = NULL;
 
     dwError = VmRESTAllocateMemory(
-                  sizeof(VM_REST_HTTP_MESSAGE_BODY),
+                  sizeof(MISC_HEADER_QUEUE),
                   (void**)&pMiscQueue
                   );
     BAIL_ON_VMREST_ERROR(dwError);
 
+    pMiscQueue->head = NULL;
+
     *ppMiscHeaderQueue = pMiscQueue;
 
 cleanup:
diff --git a/server/restengine/httpProtocolHead.c b/server/restengine/httpProtocolHead.c
index 18cff08..95089a3 100644
--- a/server/restengine/httpProtocolHead.c
+++ b/server/restengine/httpProtocolHead.c
@@ -1448,8 +1448,13 @@ VmRESTProcessBuffer(
     if (ret != REST_ENGINE_SUCCESS)
     {
         VMREST_LOG_ERROR(pRESTHandle,"%s","Double Failure case detected ....");
+        VMREST_LOG_ERROR(pRESTHandle,"%s","possible memory leak");
+        dwError = REST_ENGINE_ERROR_DOUBLE_FAILURE;
+    }
+    else
+    {
+        dwError = REST_ENGINE_SUCCESS;
     }
-    dwError = REST_ENGINE_SUCCESS;
     goto cleanup;
 
 }
diff --git a/transport/posix/socket.c b/transport/posix/socket.c
index 6c0e14e..207075b 100644
--- a/transport/posix/socket.c
+++ b/transport/posix/socket.c
@@ -81,13 +81,6 @@ VmSockPosixReArmTimer(
     int                              milliSec
     );
 
-static
-BOOLEAN
-VmSockPosixIsSafeToCloseConnOnTimeOut(
-    PVMREST_HANDLE                   pRESTHandle,
-    PVM_SOCKET                       pTimerSocket
-    );
-
 static
 uint32_t
 VmRESTAcceptSSLContext(
@@ -482,7 +475,9 @@ VmSockPosixWaitForEvent(
     BOOLEAN                          bLocked = FALSE;
     VM_SOCK_EVENT_TYPE               eventType = VM_SOCK_EVENT_TYPE_UNKNOWN;
     PVM_SOCKET                       pSocket = NULL;
+    BOOLEAN                          bFound = FALSE;
     BOOLEAN                          bFreeEventQueue = 0;
+    int                              i = 0;
 
     if (!pQueue || !ppSocket || !pEventType)
     {
@@ -551,7 +546,7 @@ VmSockPosixWaitForEvent(
                 BAIL_ON_VMREST_ERROR(dwError);
                 pSocket = pEventSocket;
             }
-            else if (pEventSocket->type == VM_SOCK_TYPE_LISTENER)
+            else if (pEventSocket->type == VM_SOCK_TYPE_LISTENER)    // New connection request
             {
                 VMREST_LOG_INFO(pRESTHandle,"%s","C-REST-ENGINE: ========================  NEW REQUEST ==========================");
                 dwError = VmSockPosixAcceptConnection(
@@ -572,6 +567,7 @@ VmSockPosixWaitForEvent(
                                    );
                     BAIL_ON_VMREST_ERROR(dwError);
 
+                    /**** Try SSL Handshake ****/
                     dwError  = VmRESTAcceptSSLContext(
                                    pRESTHandle,
                                    pSocket,
@@ -580,6 +576,7 @@ VmSockPosixWaitForEvent(
                     BAIL_ON_VMREST_ERROR(dwError);
                 }
 
+                /**** Start watching new connection ****/
                 dwError = VmSockPosixEventQueueAdd_inlock(
                               pQueue,
                               TRUE,
@@ -593,9 +590,12 @@ VmSockPosixWaitForEvent(
                               pSocket
                               );
                 BAIL_ON_VMREST_ERROR(dwError);
+
                 eventType = VM_SOCK_EVENT_TYPE_TCP_NEW_CONNECTION;
+
+                VMREST_LOG_DEBUG(pRESTHandle,"Timer fd %d associated with socket fd %d", pSocket->pTimerSocket->fd ,pSocket->fd);
             }
-            else if (pEventSocket->type == VM_SOCK_TYPE_SIGNAL)
+            else if (pEventSocket->type == VM_SOCK_TYPE_SIGNAL) // Shutdown library
             {
                 if (pQueue->bShutdown)
                 {
@@ -614,58 +614,108 @@ VmSockPosixWaitForEvent(
                     eventType = VM_SOCK_EVENT_TYPE_DATA_AVAILABLE;
                 }
             }
-            else if (pEventSocket->type == VM_SOCK_TYPE_TIMER)
+            else if (pEventSocket->type == VM_SOCK_TYPE_TIMER) // Time out event
             {
-                /**** No activity on the socket watched by poller till timeout ****/
-                if (VmSockPosixIsSafeToCloseConnOnTimeOut(pRESTHandle, pEventSocket))
+                eventType = VM_SOCK_EVENT_TYPE_UNKNOWN;
+                pSocket = pEventSocket;
+
+                VMREST_LOG_DEBUG(pRESTHandle, "Timeout event happened on timer fd %d", pSocket->fd);
+              
+                if (pSocket->pIoSocket != NULL)
                 {
-                    pSocket = pEventSocket;
+                    /*** Scan pQueue and look for IO event corresponding to this timer event ***/
+                    for ((i = (pQueue->iReady + 1)); i < pQueue->nReady ; i++)
+                    {
+                        struct epoll_event* pEventTemp = &pQueue->pEventArray[i];
+                        PVM_SOCKET pEventSocketTemp = (PVM_SOCKET)pEventTemp->data.ptr;
+                        if (pEventSocketTemp->fd == pEventSocket->pIoSocket->fd)
+                        {
+                            pEventSocket->pIoSocket->pTimerSocket = NULL;
+                            bFound = TRUE;
+                            break;
+                        }
+                    }
 
-                    /**** Delete timer socket from poller ****/
-                    dwError = VmSockPosixEventQueueDelete_inlock(
-                                  pQueue,
-                                  pSocket
-                                  );
-                    BAIL_ON_VMREST_ERROR(dwError);
+                    if (bFound)
+                    {
+                        VMREST_LOG_DEBUG(pRESTHandle,"Action: DEFERRED, IO sock found in queue(Succeeding),  Io Sock %d, timer %d", pSocket->pIoSocket->fd, pSocket->fd );
+                    }
+                    else
+                    {
+                        if (pSocket->pIoSocket->bInUse == TRUE)
+                        {
+                            VMREST_LOG_DEBUG(pRESTHandle,"Action: DEFERRED, IO Soc in use, IoSocket %d, timer %d", pSocket->pIoSocket->fd, pSocket->fd );
+                            pEventSocket->pIoSocket->pTimerSocket = NULL;
+                        }
+                        else
+                        {
+                             /**** We are good to close actual IO Socket here ****/
+                             VMREST_LOG_INFO(pRESTHandle,"Action: IO DELETION, IoSocket %d, timer %d", pSocket->pIoSocket->fd, pSocket->fd );
+                             eventType = VM_SOCK_EVENT_TYPE_CONNECTION_TIMEOUT;
+                             pSocket = pSocket->pIoSocket;
+                             
+                             /**** Delete IO from queue ****/
+                             VmSockPosixEventQueueDelete_inlock(
+                                           pQueue,
+                                           pSocket
+                                           );
+                        }
+                    }
+                }
 
-                    /**** Delete actual IO socket from poller ****/
-                    dwError = VmSockPosixEventQueueDelete_inlock(
-                                  pQueue,
-                                  pSocket->pIoSocket
-                                  );
-                    BAIL_ON_VMREST_ERROR(dwError);
+                /** Close and free the timer socket ****/
+                VmSockPosixCloseSocket(pRESTHandle,pEventSocket);
+                VmSockPosixReleaseSocket(pRESTHandle,pEventSocket);
 
-                    pSocket = pEventSocket->pIoSocket;
-                    eventType = VM_SOCK_EVENT_TYPE_CONNECTION_TIMEOUT;
+                if (eventType == VM_SOCK_EVENT_TYPE_UNKNOWN)
+                {
+                    pSocket = NULL;
                 }
             }
-            else
+            else  // Data available on IO Socket
             {
-                /**** Data is available over the socket ****/
-                pSocket = pEventSocket;
+                 pSocket = pEventSocket;
 
-                /**** If SSL handshake is not yet complete, do the needful ****/
-                if ((pRESTHandle->pSSLInfo->isSecure) && (!(pSocket->bSSLHandShakeCompleted)))
-                {
-                    dwError = VmRESTAcceptSSLContext(
-                                  pRESTHandle,
-                                  pSocket,
-                                  TRUE
-                                  );
-                    BAIL_ON_VMREST_ERROR(dwError);
-                }
-                else
-                {
-                    eventType = VM_SOCK_EVENT_TYPE_DATA_AVAILABLE;
+                 /**** Mark IO socket in use - timer out event cannot modify IO till this is done ****/
+                 pSocket->bInUse = TRUE;
 
-                    /**** Stop timer on the socket ****/
-                    dwError = VmSockPosixReArmTimer(
-                                   pRESTHandle,
-                                   pSocket->pTimerSocket,
-                                   0
-                                   );
-                    BAIL_ON_VMREST_ERROR(dwError);
-                }
+                 if (pSocket->pTimerSocket == NULL)
+                 {
+                     /**** Time out already occurred on this socket.. request won't be processed ****/
+                     VmSockPosixCloseSocket(pRESTHandle,pSocket);
+                     VmSockPosixReleaseSocket(pRESTHandle,pSocket);
+                 }
+                 else
+                 {
+                      /**** Process data  ****/
+                      VMREST_LOG_DEBUG(pRESTHandle,"Data notification on socket fd %d", pSocket->fd);
+
+                      /**** If SSL handshake is not yet complete, do the needful ****/
+                      if ((pRESTHandle->pSSLInfo->isSecure) && (!(pSocket->bSSLHandShakeCompleted)))
+                      {
+                          dwError = VmRESTAcceptSSLContext(
+                                        pRESTHandle,
+                                        pSocket,
+                                        TRUE
+                                        );
+                          BAIL_ON_VMREST_ERROR(dwError);
+
+                          /**** We do not need IO any more ..mark as available for timer ****/
+                          pSocket->bInUse = FALSE;
+                      }
+                      else
+                      {
+                          eventType = VM_SOCK_EVENT_TYPE_DATA_AVAILABLE;
+
+                          /*** Disarm timer associated with this IO socket .. dont delete ***/
+                          dwError = VmSockPosixReArmTimer(
+                                        pRESTHandle,
+                                        pSocket->pTimerSocket,
+                                        0
+                                        );
+                          BAIL_ON_VMREST_ERROR(dwError);
+                      }
+                 }
             }
         }
         pQueue->iReady++;
@@ -696,7 +746,7 @@ VmSockPosixWaitForEvent(
 
 error:
 
-    VMREST_LOG_ERROR(pRESTHandle,"%s","Socket layer - wait for event error");
+    VMREST_LOG_ERROR(pRESTHandle,"Error while processing socket event, dwError = %u", dwError);
     if (ppSocket)
     {
         *ppSocket = NULL;
@@ -884,8 +934,8 @@ VmSockPosixRead(
     if (nPrevBuf >= pRESTHandle->pRESTConfig->maxDataPerConnMB)
     {
         /**** Discard the request here itself. This might be the first read IO cycle ****/
-        VMREST_LOG_ERROR(pRESTHandle,"Total Data in request %u bytes is over allowed limit of %u bytes, closing connection...", nPrevBuf, pRESTHandle->pRESTConfig->maxDataPerConnMB);
-        dwError = REST_ENGINE_FAILURE;
+        VMREST_LOG_ERROR(pRESTHandle,"Total Data in request %u bytes is over allowed limit of %u bytes, closing connection with fd %d", nPrevBuf, pRESTHandle->pRESTConfig->maxDataPerConnMB, pSocket->fd);
+        dwError = VMREST_TRANSPORT_SOCK_DATA_OVER_LIMIT;
     }
     BAIL_ON_VMREST_ERROR(dwError);
 
@@ -897,8 +947,8 @@ VmSockPosixRead(
         }
         else
         {
-            VMREST_LOG_ERROR(pRESTHandle,"Unknown socket read error: errno %u, errorCode %u, nRead %d", errno, errorCode, nRead);
-            dwError = REST_ENGINE_FAILURE;
+            VMREST_LOG_ERROR(pRESTHandle,"Socket read error: errno %u, errorCode %u, nRead %d", errno, errorCode, nRead);
+            dwError = VMREST_TRANSPORT_SOCK_READ_FAILED;
         }
     }
     else
@@ -906,12 +956,12 @@ VmSockPosixRead(
         if (nRead == 0)
         {
             VMREST_LOG_ERROR(pRESTHandle,"%s","Socket Read Failed: Remote has closed the connection");
-            dwError = VM_SOCK_POSIX_ERROR_BROKEN_PIPE;
+            dwError = VMREST_TRANSPORT_SOCK_READ_FAILED;
         }
         else
         {
             VMREST_LOG_ERROR(pRESTHandle, "Socket read failed with error code %u", errorCode);
-            dwError = errorCode;
+            dwError = VMREST_TRANSPORT_SOCK_READ_FAILED;
         }
     }
     BAIL_ON_VMREST_ERROR(dwError);
@@ -936,29 +986,37 @@ VmSockPosixRead(
 
 error:
 
-    if (pSocket)
-    {
-        VMREST_LOG_ERROR(pRESTHandle,"Socket read failed with Socket fd %d, dwError = %u, nRead = %d, errno = %u, errorCode = %u", pSocket->fd, dwError, nRead, errno, errorCode);
-    }
-    else
-    {
-        VMREST_LOG_ERROR(pRESTHandle,"Socket read failed with dwError = %u, nRead = %d, errno = %u, errorCode = %u", dwError, nRead, errno, errorCode);
-    }
-
-    if (pszBufPrev && pSocket && pRESTHandle->pSockContext)
+    if (pSocket && pRESTHandle->pSockContext)
     {
         /**** Delete the socket from poller ****/
-        
-        VmSockPosixEventQueueDelete_inlock(
+            VmSockPosixEventQueueDelete_inlock(
             pRESTHandle->pSockContext->pEventQueue,
             pSocket
             );
-        
-        VmRESTFreeMemory(pszBufPrev);
-        pszBufPrev = NULL;
+
         pSocket->pszBuffer = NULL;
         pSocket->nProcessed = 0;
         pSocket->nBufData = 0;
+
+        if (pSocket->pTimerSocket)
+        {
+            VmSockPosixReArmTimer(
+                pRESTHandle,
+                pSocket->pTimerSocket,
+                ((pRESTHandle->pRESTConfig->connTimeoutSec) * 1000)
+                );
+
+            write(pSocket->pTimerSocket->fd, "NotifyPQ", 8);
+            pSocket->pTimerSocket->pIoSocket = NULL;
+            
+        }
+
+    }
+
+    if (pszBufPrev)
+    {   
+        VmRESTFreeMemory(pszBufPrev);
+        pszBufPrev = NULL;
     }
 
     if (nBufLen)
@@ -1080,8 +1138,8 @@ VmSockPosixWrite(
 
 error:
 
-    VMREST_LOG_ERROR(pRESTHandle,"%s", "Socket write failed");
     goto cleanup;
+
 }
 
 VOID
@@ -1092,10 +1150,6 @@ VmSockPosixReleaseSocket(
 {
     if (pSocket)
     {
-        if (pSocket->pTimerSocket)
-        {
-             VmSockPosixFreeSocket(pSocket->pTimerSocket);
-        }
         VmSockPosixFreeSocket(pSocket);
     }
 }
@@ -1118,22 +1172,21 @@ VmSockPosixCloseSocket(
     }
     BAIL_ON_VMREST_ERROR(dwError);
 
-    VMREST_LOG_INFO(pRESTHandle,"C-REST-ENGINE: Closing socket with fd %d", pSocket->fd);
+    VMREST_LOG_INFO(pRESTHandle,"C-REST-ENGINE: Closing socket with fd %d, Socket Type %u ( 2-Io / 5-Timer )", pSocket->fd, pSocket->type);
 
     dwError = VmRESTLockMutex(pSocket->pMutex);
     BAIL_ON_VMREST_ERROR(dwError);
 
     bLocked = TRUE;
 
-    if (pSocket->pTimerSocket && (pSocket->pTimerSocket->fd > 0))
+    if (pSocket->pTimerSocket)
     {
-        close(pSocket->pTimerSocket->fd);
-        pSocket->pTimerSocket->fd = -1;
+        pSocket->pTimerSocket->pIoSocket = NULL;
     }
 
-    if (pRESTHandle->pSSLInfo->isSecure && pSocket->ssl)
+    if (pRESTHandle->pSSLInfo->isSecure && pSocket->ssl && (pSocket->type != VM_SOCK_TYPE_TIMER))
     {
-        ret = SSL_shutdown(pSocket->ssl);
+        ret =  SSL_shutdown(pSocket->ssl);
         if (ret < 0)
         {
             errorCode = SSL_get_error(pSocket->ssl, ret);
@@ -1149,7 +1202,6 @@ VmSockPosixCloseSocket(
         pSocket->fd = -1;
     }
 
-
 cleanup:
 
     if (bLocked)
@@ -1294,6 +1346,12 @@ VmSockPosixEventQueueDelete_inlock(
     DWORD                            dwError = REST_ENGINE_SUCCESS;
     struct                           epoll_event event = {0};
 
+    if (!pSocket || !pQueue)
+    {
+        dwError = REST_ERROR_INVALID_HANDLER;
+    }
+    BAIL_ON_VMREST_ERROR(dwError);
+
     if (epoll_ctl(pQueue->epollFd, EPOLL_CTL_DEL, pSocket->fd, &event) < 0)
     {
         dwError = VM_SOCK_POSIX_ERROR_SYS_CALL_FAILED;
@@ -1341,6 +1399,7 @@ VmSockPosixAcceptConnection(
     pSocket->pTimerSocket = NULL;
     pSocket->pIoSocket = NULL;
     pSocket->bSSLHandShakeCompleted = FALSE;
+    pSocket->bInUse = FALSE;
 
     *ppSocket = pSocket;
 
@@ -1560,35 +1619,21 @@ VmSockPosixSetRequestHandle(
 
     pSocket->nProcessed = nProcessed;
 
-    /**** Rearm timer and add sockfd back to poller ****/
-    if (!bCompleted)
+    if (bCompleted)
     {
-        dwError = VmSockPosixReArmTimer(
-                      pRESTHandle,
-                      pSocket->pTimerSocket,
-                      ((pRESTHandle->pRESTConfig->connTimeoutSec) * 1000)
-                      );
-        BAIL_ON_VMREST_ERROR(dwError);
-
-        event.data.ptr = pSocket;
-        event.events = EPOLLIN;
-
-        event.events = event.events | EPOLLONESHOT;
-
-        if (epoll_ctl(pQueue->epollFd, EPOLL_CTL_MOD, pSocket->fd, &event) < 0)
+        /**** We are done with request - no need to add back to poller *****/
+        if (pSocket->pTimerSocket != NULL)
         {
-            dwError = VM_SOCK_POSIX_ERROR_SYS_CALL_FAILED;
+            pSocket->pTimerSocket->pIoSocket = NULL;
+
+            /**** Give immediate notification to timer for cleanup ****/
+            dwError = VmSockPosixReArmTimer(
+                           pRESTHandle,
+                           pSocket->pTimerSocket,
+                           1
+                           );
             BAIL_ON_VMREST_ERROR(dwError);
         }
-    }
-    else
-    {
-        /**** Delete timerfd from poller ****/
-        dwError = VmSockPosixEventQueueDelete_inlock(
-                      pQueue,
-                      pSocket->pTimerSocket
-                      );
-        BAIL_ON_VMREST_ERROR(dwError);
 
         /**** Delete actual IO socket from poller ****/
         dwError = VmSockPosixEventQueueDelete_inlock(
@@ -1596,13 +1641,37 @@ VmSockPosixSetRequestHandle(
                       pSocket
                       );
         BAIL_ON_VMREST_ERROR(dwError);
-       
-
-        if (pSocket->pTimerSocket->fd > 0)
+    }
+    else
+    {
+        /***** Add back IO socket to poller for next IO cycle ****/
+        if (pSocket->pTimerSocket == NULL)
         {
-            close(pSocket->pTimerSocket->fd);
-            pSocket->pTimerSocket->fd = -1;
-        } 
+            /**** Timeout already happened. Notify HTTP layer to send 408 Req timeout - if possible ****/
+            dwError = VMREST_TRANSPORT_DEFERRED_TIMEOUT_PROCESS;
+        }
+        else
+        {
+            /*** Rearm timer and add IO socket to poller ****/
+            dwError = VmSockPosixReArmTimer(
+                           pRESTHandle,
+                           pSocket->pTimerSocket,
+                           ((pRESTHandle->pRESTConfig->connTimeoutSec) * 1000)
+                           );
+            BAIL_ON_VMREST_ERROR(dwError);
+
+            event.data.ptr = pSocket;
+            event.events = EPOLLIN;
+
+            event.events = event.events | EPOLLONESHOT;
+
+            if (epoll_ctl(pQueue->epollFd, EPOLL_CTL_MOD, pSocket->fd, &event) < 0)
+            {
+                dwError = VM_SOCK_POSIX_ERROR_SYS_CALL_FAILED;
+                BAIL_ON_VMREST_ERROR(dwError);
+            }
+        }
+        BAIL_ON_VMREST_ERROR(dwError);
     }
 
 cleanup:
@@ -1611,6 +1680,10 @@ VmSockPosixSetRequestHandle(
     {
         VmRESTUnlockMutex(pSocket->pMutex);
     }
+    if (!bCompleted && pSocket && (pSocket->pTimerSocket != NULL))
+    {
+        pSocket->bInUse = FALSE;
+    }
 
     return dwError;
 
@@ -1702,6 +1775,13 @@ VmSockPosixReArmTimer(
     int                              nSec = 0;
     int                              nNanoSec = 0;
 
+    if (!pRESTHandle || !pTimerSocket )
+    {
+        VMREST_LOG_ERROR(pRESTHandle,"%s","Invalid params");
+        dwError = ERROR_INVALID_PARAMETER;
+    }
+    BAIL_ON_VMREST_ERROR(dwError);
+
     if (milliSec > 0)
     {
         nSec = milliSec / 1000;
@@ -1733,96 +1813,6 @@ VmSockPosixReArmTimer(
 
 }
 
-static
-BOOLEAN
-VmSockPosixIsSafeToCloseConnOnTimeOut(
-    PVMREST_HANDLE                   pRESTHandle,
-    PVM_SOCKET                       pTimerSocket
-    )
-{
-    uint32_t                         dwError = REST_ENGINE_SUCCESS;
-    struct                           epoll_event event = {0};
-    BOOLEAN                          bCloseConn = FALSE;
-    ssize_t                          nRead = 0;
-    uint32_t                         errorCode = 0;
-    uint64_t                         res = 0;
-    char                             pBuf = '\0';
-    struct                           itimerspec ts = {0};
-    PVM_SOCKET                       pSocket = NULL;
-
-    if (!pRESTHandle || !pTimerSocket)
-    {
-        VMREST_LOG_ERROR(pRESTHandle,"%s","Invalid params");
-        dwError = ERROR_INVALID_PARAMETER;
-    }
-    BAIL_ON_VMREST_ERROR(dwError);
-
-    pSocket = pTimerSocket->pIoSocket;
-    errno = 0;
-
-    if ((pRESTHandle->pSSLInfo->isSecure) && (pSocket->ssl))
-    {
-        nRead = SSL_read(pSocket->ssl, &pBuf, 0);
-        errorCode = SSL_get_error(pSocket->ssl, nRead);
-    }
-    else if (pSocket->fd > 0)
-    {
-        nRead = read(pSocket->fd, &pBuf, 0);
-        errorCode = errno;
-    }
-    VMREST_LOG_DEBUG(pRESTHandle, "IO socket read bytes %d, errorCode %u, at timer expiration", nRead, errorCode);
-
-    if (errorCode == EAGAIN || errorCode == EWOULDBLOCK || errorCode == SSL_ERROR_WANT_READ)
-    {
-        /**** Check for preceeding reset of timer due to valid IO ****/
-        if (timerfd_gettime(pTimerSocket->fd, &ts) == 0)
-        {
-            if ((ts.it_value.tv_sec == 0) && (ts.it_value.tv_nsec == 0))
-            {
-                /**** timer is still disarmed ****/
-                /**** It's safe to close connection ****/
-                bCloseConn = TRUE;
-            }
-
-            /**** Do a read on timer socket - dummy read ****/
-            do
-            {
-                errorCode = 0;
-                errno = 0;
-                nRead = 0;
-                nRead = read(pTimerSocket->fd, &res, sizeof(res));
-                errorCode = errno;
-                res = 0;
-            }while(nRead > 0);
-
-            if (!bCloseConn)
-            {
-                /**** Add timer socket back to poller ****/
-                event.data.ptr = pTimerSocket;
-                event.events = EPOLLIN;
-                event.events = event.events | EPOLLONESHOT;
-
-                if (epoll_ctl(pRESTHandle->pSockContext->pEventQueue->epollFd, EPOLL_CTL_MOD, pTimerSocket->fd, &event) < 0)
-                {
-                    dwError = VM_SOCK_POSIX_ERROR_SYS_CALL_FAILED;
-                    BAIL_ON_VMREST_ERROR(dwError);
-                }
-            }
-        }
-    }
-
-cleanup:
-
-    return bCloseConn;
-
-error:
-
-    bCloseConn = FALSE;
-
-    goto cleanup;
-
-}
-
 static
 uint32_t
 VmSockPosixCreateTimer(
@@ -1860,6 +1850,7 @@ VmSockPosixCreateTimer(
 
     pTimerSocket->type = VM_SOCK_TYPE_TIMER;
     pTimerSocket->fd = timerFd;
+    pTimerSocket->bInUse = FALSE;
     pTimerSocket->pIoSocket = pSocket;
     pTimerSocket->pRequest = NULL;
     pTimerSocket->pszBuffer = NULL;
@@ -1931,6 +1922,7 @@ VmRESTAcceptSSLContext(
 
     if (ret == 1)
     {
+        VMREST_LOG_DEBUG(pRESTHandle," SSL accept successful on socket %d, ret %d, errorCode %u", pSocket->fd, ret, errorCode);
         pSocket->bSSLHandShakeCompleted = TRUE;
         bReArm = TRUE;
     }
@@ -1940,15 +1932,22 @@ VmRESTAcceptSSLContext(
        pSocket->bSSLHandShakeCompleted = FALSE;
        bReArm = TRUE;
     }
+    else if ((ret == 0) && (errorCode == SSL_ERROR_SYSCALL))
+    {
+         VMREST_LOG_ERROR(pRESTHandle," Client closed the connection during SSL handshake, socket fd %d, ret %d, errorCode %u, errno %d", pSocket->fd, ret, errorCode, errno);
+         dwError = VMREST_TRANSPORT_SSL_ACCEPT_FAILED;
+         BAIL_ON_VMREST_ERROR(dwError);
+    }
     else
     {
-        VMREST_LOG_ERROR(pRESTHandle, "SSL handshake failed...connection will be closed for socket with fd %d, ret %d, errorCode %u", pSocket->fd, ret, errorCode);
+        VMREST_LOG_ERROR(pRESTHandle, "SSL handshake failed...connection will be closed for socket with fd %d, ret %d, errorCode %u, errno %d", pSocket->fd, ret, errorCode, errno);
         dwError = VMREST_TRANSPORT_SSL_ACCEPT_FAILED;
         BAIL_ON_VMREST_ERROR(dwError);
     }
 
      if (bReArm && bWatched)
      {
+         /**** Rearm and add the socket ****/
          dwError = VmSockPosixReArmTimer(
                        pRESTHandle,
                        pSocket->pTimerSocket,
@@ -1974,13 +1973,23 @@ VmRESTAcceptSSLContext(
 
 error:
 
-    if (bWatched && pRESTHandle && pRESTHandle->pSockContext)
+    if (pRESTHandle && pRESTHandle->pSockContext)
     {
         /**** Delete from poller ****/
         VmSockPosixEventQueueDelete_inlock(
             pRESTHandle->pSockContext->pEventQueue,
             pSocket
             );
+
+        if (bWatched && pSocket && pSocket->pTimerSocket)
+        {
+            pSocket->pTimerSocket->pIoSocket = NULL;
+            VmSockPosixReArmTimer(
+                pRESTHandle,
+                pSocket->pTimerSocket,
+                1
+                );
+        }
     }
 
     goto cleanup;
diff --git a/transport/posix/structs.h b/transport/posix/structs.h
index ad5563e..369cfcb 100644
--- a/transport/posix/structs.h
+++ b/transport/posix/structs.h
@@ -20,6 +20,7 @@ typedef struct _VM_SOCKET
     int                              fd;
     SSL*                             ssl;
     BOOLEAN                          bSSLHandShakeCompleted;
+    BOOLEAN                          bInUse;
     char*                            pszBuffer;
     uint32_t                         nBufData;
     uint32_t                         nProcessed;