STATISTICS: Add tracking of the 10 hottest keys per database measured in hopcount

and add mechanisms to dump it using the ctdb dbstatistics command

diff --git a/include/ctdb_protocol.h b/include/ctdb_protocol.h
index 5c787ff6ea3962395bf2a6aff2939a64c599a794..33187c78428655a6a48ec8d0fdc06b4023c99824 100644 (file)
--- a/include/ctdb_protocol.h
+++ b/include/ctdb_protocol.h
@@ -614,6 +614,7 @@ struct ctdb_traverse_start_ext {
   ctdb statistics information
  */
 #define MAX_COUNT_BUCKETS 16
+#define MAX_HOT_KEYS      10
 
 struct ctdb_statistics {
        uint32_t num_clients;
@@ -680,10 +681,23 @@ struct ctdb_statistics_wire {
 /*
  * db statistics
  */
+struct ctdb_db_hot_key {
+       uint32_t count;
+       TDB_DATA key;
+};
 struct ctdb_db_statistics {
        uint32_t db_ro_delegations;
        uint32_t db_ro_revokes;
        uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+       uint32_t num_hot_keys;
+       struct ctdb_db_hot_key hot_keys[MAX_HOT_KEYS];
+};
+struct ctdb_db_statistics_wire {
+       uint32_t db_ro_delegations;
+       uint32_t db_ro_revokes;
+       uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+       uint32_t num_hot_keys;
+       char hot_keys[1];
 };
 
 /*

diff --git a/libctdb/control.c b/libctdb/control.c
index b4c54cd660389e3458b0f315a4c21fe75b3f3348..f927e0866a3cad93d86192027dd7dce4b43e0480 100644 (file)
--- a/libctdb/control.c
+++ b/libctdb/control.c
@@ -120,6 +120,9 @@ bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
 {
        struct ctdb_reply_control *reply;
        struct ctdb_db_statistics *s;
+       struct ctdb_db_statistics_wire *wire;
+       int i;
+       char *ptr;
 
        reply = unpack_reply_control(req, CTDB_CONTROL_GET_DB_STATISTICS);
        if (!reply) {
@@ -129,16 +132,36 @@ bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
                DEBUG(ctdb, LOG_ERR, "ctdb_getpnn_recv: status -1");
                return false;
        }
-       if (reply->datalen != sizeof(struct ctdb_db_statistics)) {
-               DEBUG(ctdb, LOG_ERR, "ctdb_getdbstat_recv: returned data is %d bytes but should be %d", reply->datalen, (int)sizeof(struct ctdb_db_statistics));
+       if (reply->datalen < offsetof(struct ctdb_db_statistics_wire, hot_keys)) {
+               DEBUG(ctdb, LOG_ERR, "ctdb_getdbstat_recv: returned data is %d bytes but should be >= %d", reply->datalen, (int)sizeof(struct ctdb_db_statistics));
                return false;
        }
 
-       s = malloc(sizeof(struct ctdb_db_statistics));
+       wire = reply->data;
+
+       s = malloc(offsetof(struct ctdb_db_statistics, hot_keys) + sizeof(struct ctdb_db_hot_key) * wire->num_hot_keys);
        if (!s) {
                return false;
        }
-       memcpy(s, reply->data, sizeof(struct ctdb_db_statistics));
+       s->db_ro_delegations = wire->db_ro_delegations;
+       s->db_ro_revokes     = wire->db_ro_revokes;
+       for (i = 0; i < MAX_COUNT_BUCKETS; i++) {
+               s->hop_count_bucket[i] = wire->hop_count_bucket[i];
+       }
+       s->num_hot_keys      = wire->num_hot_keys;
+       ptr = &wire->hot_keys[0];
+       for (i = 0; i < wire->num_hot_keys; i++) {
+               s->hot_keys[i].count = *(uint32_t *)ptr;
+               ptr += 4;
+
+               s->hot_keys[i].key.dsize = *(uint32_t *)ptr;
+               ptr += 4;
+
+               s->hot_keys[i].key.dptr = malloc(s->hot_keys[i].key.dsize);
+               memcpy(s->hot_keys[i].key.dptr, ptr, s->hot_keys[i].key.dsize);
+               ptr += s->hot_keys[i].key.dsize;
+       }
+
        *stat = s;
 
        return true;
@@ -158,9 +181,18 @@ struct ctdb_request *ctdb_getdbstat_send(struct ctdb_connection *ctdb,
 
 void ctdb_free_dbstat(struct ctdb_db_statistics *stat)
 {
+       int i;
+
        if (stat == NULL) {
                return;
        }
+
+       for (i = 0; i < stat->num_hot_keys; i++) {
+               if (stat->hot_keys[i].key.dptr != NULL) {
+                       free(stat->hot_keys[i].key.dptr);
+               }
+       }
+
        free(stat);
 }
 

diff --git a/server/ctdb_call.c b/server/ctdb_call.c
index fe7e947c036813716ef17d76a32c02745d87c446..56cb5e8b096bf06567834d408f61b6ec91d0f163 100644 (file)
--- a/server/ctdb_call.c
+++ b/server/ctdb_call.c
@@ -667,6 +667,54 @@ ctdb_defer_pinned_down_request(struct ctdb_context *ctdb, struct ctdb_db_context
        return 0;
 }
 
+static void
+ctdb_update_db_stat_hot_keys(struct ctdb_db_context *ctdb_db, TDB_DATA key, int hopcount)
+{
+       int i;
+
+       /* smallest value is always at index 0 */
+       if (hopcount <= ctdb_db->statistics.hot_keys[0].count) {
+               return;
+       }
+
+       /* see if we already know this key */
+       for (i = 0; i < MAX_HOT_KEYS; i++) {
+               if (key.dsize != ctdb_db->statistics.hot_keys[i].key.dsize) {
+                       continue;
+               }
+               if (memcmp(key.dptr, ctdb_db->statistics.hot_keys[i].key.dptr, key.dsize)) {
+                       continue;
+               }
+               /* found an entry for this key */
+               if (hopcount <= ctdb_db->statistics.hot_keys[i].count) {
+                       return;
+               }
+               ctdb_db->statistics.hot_keys[i].count = hopcount;
+               goto sort_keys;
+       }
+
+       if (ctdb_db->statistics.hot_keys[0].key.dptr != NULL) {
+               talloc_free(ctdb_db->statistics.hot_keys[0].key.dptr);
+       }
+       ctdb_db->statistics.hot_keys[0].key.dsize = key.dsize;
+       ctdb_db->statistics.hot_keys[0].key.dptr  = talloc_memdup(ctdb_db, key.dptr, key.dsize);
+       ctdb_db->statistics.hot_keys[0].count = hopcount;
+
+
+sort_keys:
+       for (i = 2; i < MAX_HOT_KEYS; i++) {
+               if (ctdb_db->statistics.hot_keys[i].count < ctdb_db->statistics.hot_keys[0].count) {
+                       hopcount = ctdb_db->statistics.hot_keys[i].count;
+                       ctdb_db->statistics.hot_keys[i].count = ctdb_db->statistics.hot_keys[0].count;
+                       ctdb_db->statistics.hot_keys[0].count = hopcount;
+
+                       key = ctdb_db->statistics.hot_keys[i].key;
+                       ctdb_db->statistics.hot_keys[i].key = ctdb_db->statistics.hot_keys[0].key;
+                       ctdb_db->statistics.hot_keys[0].key = key;
+               }
+       }
+}
+
 /*
   called when a CTDB_REQ_CALL packet comes in
 */
@@ -867,7 +915,7 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
        }
        CTDB_INCREMENT_STAT(ctdb, hop_count_bucket[bucket]);
        CTDB_INCREMENT_DB_STAT(ctdb_db, hop_count_bucket[bucket]);
-
+       ctdb_update_db_stat_hot_keys(ctdb_db, call->key, c->hopcount);
 
        /* If this database supports sticky records, then check if the
           hopcount is big. If it is it means the record is hot and we

diff --git a/server/ctdb_ltdb_server.c b/server/ctdb_ltdb_server.c
index 1d21c07a56b1ef5632dafc98725f6073a034b68f..0aa378be09b47ed00feeeb1e001c0a2f1a176b8f 100644 (file)
--- a/server/ctdb_ltdb_server.c
+++ b/server/ctdb_ltdb_server.c
@@ -1493,6 +1493,10 @@ int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
                                TDB_DATA *outdata)
 {
        struct ctdb_db_context *ctdb_db;
+       struct ctdb_db_statistics_wire *stats;
+       int i;
+       int len;
+       char *ptr;
 
        ctdb_db = find_ctdb_db(ctdb, db_id);
        if (!ctdb_db) {
@@ -1500,8 +1504,38 @@ int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
                return -1;
        }
 
-       outdata->dptr  = (uint8_t *)&(ctdb_db->statistics);
-       outdata->dsize = sizeof(ctdb_db->statistics);
+       len = offsetof(struct ctdb_db_statistics_wire, hot_keys);
+       for (i = 0; i < MAX_HOT_KEYS; i++) {
+               len += 8 + ctdb_db->statistics.hot_keys[i].key.dsize;
+       }
+
+       stats = talloc_size(outdata, len);
+       if (stats == NULL) {
+               DEBUG(DEBUG_ERR,("Failed to allocate db statistics wire structure\n"));
+               return -1;
+       }
+
+       stats->db_ro_delegations = ctdb_db->statistics.db_ro_delegations;
+       stats->db_ro_revokes     = ctdb_db->statistics.db_ro_revokes;
+       for (i = 0; i < MAX_COUNT_BUCKETS; i++) {
+               stats->hop_count_bucket[i] = ctdb_db->statistics.hop_count_bucket[i];
+       }
+       stats->num_hot_keys = MAX_HOT_KEYS;
+
+       ptr = &stats->hot_keys[0];
+       for (i = 0; i < MAX_HOT_KEYS; i++) {
+               *(uint32_t *)ptr = ctdb_db->statistics.hot_keys[i].count;
+               ptr += 4;
+
+               *(uint32_t *)ptr = ctdb_db->statistics.hot_keys[i].key.dsize;
+               ptr += 4;
+
+               memcpy(ptr, ctdb_db->statistics.hot_keys[i].key.dptr, ctdb_db->statistics.hot_keys[i].key.dsize);
+               ptr += ctdb_db->statistics.hot_keys[i].key.dsize;
+       }
+
+       outdata->dptr  = (uint8_t *)stats;
+       outdata->dsize = len;
 
        return 0;
 }

diff --git a/tools/ctdb.c b/tools/ctdb.c
index a3bcd6e44fccfa8172820c5c400e30cd55dcca12..07a47db21eddb815579621fb7cbbf9f1791452f4 100644 (file)
--- a/tools/ctdb.c
+++ b/tools/ctdb.c
@@ -619,6 +619,15 @@ static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char
                printf(" %d", dbstatistics->hop_count_bucket[i]);
        }
        printf("\n");
+       printf("Num Hot Keys:     %d\n", dbstatistics->num_hot_keys);
+       for (i = 0; i < dbstatistics->num_hot_keys; i++) {
+               int j;
+               printf("Count:%d Key:", dbstatistics->hot_keys[i].count);
+               for (j = 0; j < dbstatistics->hot_keys[i].key.dsize; j++) {
+                       printf("%02x", dbstatistics->hot_keys[i].key.dptr[j]&0xff);
+               }
+               printf("\n");
+       }
 
        ctdb_free_dbstat(dbstatistics);
        return 0;