ctdb-common: Log a count of dropped messages with non-blocking logging
authorMartin Schwenke <martin@meltin.net>
Thu, 1 Jun 2017 11:13:58 +0000 (21:13 +1000)
committerMartin Schwenke <martins@samba.org>
Tue, 13 Jun 2017 07:12:19 +0000 (09:12 +0200)
The non-blocking logging variants can currently silently drop messages
when the socket queue fills.

In this case, count the number of dropped messages and attempt to log
a message about dropped log messages when the next message is logged.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
ctdb/common/logging.c

index 0e3efe0a7288ec125eb0647947f214a213debdc7..f68c1b9d4579269556b88092961c289ed9a379b2 100644 (file)
@@ -239,6 +239,7 @@ struct syslog_log_state {
        /* RFC3164 says: The total length of the packet MUST be 1024
           bytes or less. */
        char buffer[1024];
+       unsigned int dropped_count;
 };
 
 /* Format messages as per RFC3164
@@ -307,19 +308,66 @@ static void syslog_log(void *private_data, int level, const char *msg)
        syslog(debug_level_to_priority(level), "%s", msg);
 }
 
-static void syslog_log_sock(void *private_data, int level, const char *msg)
+static int syslog_log_sock_maybe(struct syslog_log_state *state,
+                                int level, const char *msg)
 {
-       struct syslog_log_state *state = talloc_get_type_abort(
-               private_data, struct syslog_log_state);
        int n;
+       ssize_t ret;
 
        n = state->format(level, state, msg, state->buffer,
                          sizeof(state->buffer));
        if (n == -1) {
-               return;
+               return E2BIG;
+       }
+
+       do {
+               ret = write(state->fd, state->buffer, n);
+       } while (ret == -1 && errno == EINTR);
+
+       if (ret == -1) {
+               return errno;
+       }
+
+       return 0;
+
+}
+static void syslog_log_sock(void *private_data, int level, const char *msg)
+{
+       struct syslog_log_state *state = talloc_get_type_abort(
+               private_data, struct syslog_log_state);
+       int ret;
+
+       if (state->dropped_count > 0) {
+               char t[64] = { 0 };
+               snprintf(t, sizeof(t),
+                        "[Dropped %u log messages]\n",
+                        state->dropped_count);
+               t[sizeof(t)-1] = '\0';
+               ret = syslog_log_sock_maybe(state, level, t);
+               if (ret == EAGAIN || ret == EWOULDBLOCK) {
+                       state->dropped_count++;
+                       /*
+                        * If above failed then actually drop the
+                        * message that would be logged below, since
+                        * it would have been dropped anyway and it is
+                        * also likely to fail.  Falling through and
+                        * attempting to log the message also means
+                        * that the dropped message count will be
+                        * logged out of order.
+                        */
+                       return;
+               }
+               if (ret != 0) {
+                       /* Silent failure on any other error */
+                       return;
+               }
+               state->dropped_count = 0;
        }
 
-       sys_write_v(state->fd, state->buffer, n);
+       ret = syslog_log_sock_maybe(state, level, msg);
+       if (ret == EAGAIN || ret == EWOULDBLOCK) {
+               state->dropped_count++;
+       }
 }
 
 static int syslog_log_setup_syslog(TALLOC_CTX *mem_ctx, const char *app_name)