2 * GeoIP database support
4 * Copyright 2018, Gerald Combs <gerald@wireshark.org>
6 * Wireshark - Network traffic analyzer
7 * By Gerald Combs <gerald@wireshark.org>
8 * Copyright 1998 Gerald Combs
10 * SPDX-License-Identifier: GPL-2.0-or-later
17 #include <epan/maxmind_db.h>
19 static mmdb_lookup_t mmdb_not_found;
26 #include <epan/wmem/wmem.h>
28 #include <epan/addr_resolv.h>
30 #include <epan/prefs.h>
32 #include <wsutil/report_message.h>
33 #include <wsutil/file_util.h>
34 #include <wsutil/filesystem.h>
35 #include <wsutil/ws_pipe.h>
36 #include <wsutil/strtoi.h>
39 // - Add RBL lookups? Along with the "is this a spammer" information that most RBL databases
40 // provide, you can also fetch AS information: http://www.team-cymru.org/IP-ASN-mapping.html
41 // - Switch to a different format? I was going to use g_key_file_* to parse
42 // the mmdbresolve output, but it was easier to just parse it directly.
44 static GThread *mmdbr_thread;
45 static GAsyncQueue *mmdbr_request_q; // g_allocated char *
46 static GMutex mmdbr_pipe_mtx;
48 // Hashes of mmdb_lookup_t
49 static wmem_map_t *mmdb_ipv4_map;
50 static wmem_map_t *mmdb_ipv6_map;
51 static gboolean new_entries;
54 static wmem_map_t *mmdb_str_chunk;
55 static wmem_map_t *mmdb_ipv6_chunk;
57 /* Child mmdbresolve process */
58 static ws_pipe_t mmdbr_pipe; // Requires mutex
59 static FILE *mmdbr_stdout; // Requires mutex
61 /* UAT definitions. Copied from oids.c */
62 typedef struct _maxmind_db_path_t {
66 static maxmind_db_path_t *maxmind_db_paths;
67 static guint num_maxmind_db_paths;
68 static const maxmind_db_path_t maxmind_db_system_paths[] = {
70 // XXX Properly expand "%ProgramData%\GeoIP".
71 { "C:\\ProgramData\\GeoIP" },
74 { "/usr/share/GeoIP" },
79 static uat_t *maxmind_db_paths_uat;
80 UAT_DIRECTORYNAME_CB_DEF(maxmind_mod, path, maxmind_db_path_t)
82 static GPtrArray *mmdb_file_arr; // .mmdb files
85 #define MMDB_DEBUG(...) { \
86 char *MMDB_DEBUG_MSG = g_strdup_printf(__VA_ARGS__); \
87 g_warning("mmdb: %s:%d %s", G_STRFUNC, __LINE__, MMDB_DEBUG_MSG); \
88 g_free(MMDB_DEBUG_MSG); \
91 #define MMDB_DEBUG(...)
94 static void mmdb_resolve_stop(void);
96 // Hopefully scanning a few lines asynchronously has less overhead than
97 // reading in a child thread.
98 #define RES_STATUS_ERROR "mmdbresolve.status: false"
99 #define RES_COUNTRY_ISO_CODE "country.iso_code"
100 #define RES_COUNTRY_NAMES_EN "country.names.en"
101 #define RES_CITY_NAMES_EN "city.names.en"
102 #define RES_ASN_ORG "autonomous_system_organization"
103 #define RES_ASN_NUMBER "autonomous_system_number"
104 #define RES_LOCATION_LATITUDE "location.latitude"
105 #define RES_LOCATION_LONGITUDE "location.longitude"
106 #define RES_END "# End "
108 // Interned strings and v6 addresses, similar to GLib's string chunks.
109 static const char *chunkify_string(char *key) {
110 key = g_strstrip(key);
111 char *chunk_string = (char *) wmem_map_lookup(mmdb_str_chunk, key);
114 chunk_string = wmem_strdup(wmem_epan_scope(), key);
115 wmem_map_insert(mmdb_str_chunk, chunk_string, chunk_string);
121 static const void *chunkify_v6_addr(const ws_in6_addr *addr) {
122 void *chunk_v6_bytes = (char *) wmem_map_lookup(mmdb_ipv6_chunk, addr->bytes);
124 if (!chunk_v6_bytes) {
125 chunk_v6_bytes = wmem_memdup(wmem_epan_scope(), addr->bytes, sizeof(ws_in6_addr));
126 wmem_map_insert(mmdb_ipv6_chunk, chunk_v6_bytes, chunk_v6_bytes);
129 return chunk_v6_bytes;
132 static void init_lookup(mmdb_lookup_t *lookup) {
133 mmdb_lookup_t empty_lookup = { FALSE, NULL, NULL, NULL, 0, NULL, DBL_MAX, DBL_MAX };
134 *lookup = empty_lookup;
137 static gboolean mmdbr_pipe_valid(void) {
138 g_mutex_lock(&mmdbr_pipe_mtx);
139 gboolean pipe_valid = ws_pipe_valid(&mmdbr_pipe);
140 g_mutex_unlock(&mmdbr_pipe_mtx);
144 // Writing to mmdbr_pipe.stdin_fd can block. Do so in a separate thread.
145 #define MMDB_WAIT_TIME (150 * 1000) // microseconds
147 write_mmdbr_stdin_worker(gpointer data _U_) {
149 if (!mmdbr_pipe_valid()) {
150 // Should be due to mmdb_resolve_stop.
151 MMDB_DEBUG("invalid mmdbr pipe. exiting thread.");
155 char *request = (char *) g_async_queue_timeout_pop(mmdbr_request_q, MMDB_WAIT_TIME);
160 MMDB_DEBUG("write %s ql %d", request, g_async_queue_length(mmdbr_request_q));
161 g_mutex_lock(&mmdbr_pipe_mtx);
162 ssize_t req_status = ws_write(mmdbr_pipe.stdin_fd, request, (unsigned int)strlen(request));
163 g_mutex_unlock(&mmdbr_pipe_mtx);
164 if (req_status < 0) {
165 MMDB_DEBUG("write error %s. exiting thread.", g_strerror(errno));
174 read_mmdbr_stdout(void) {
175 static char cur_addr[WS_INET6_ADDRSTRLEN];
176 static mmdb_lookup_t cur_lookup;
178 g_mutex_lock(&mmdbr_pipe_mtx);
179 if (!ws_pipe_valid(&mmdbr_pipe)) {
180 g_mutex_unlock(&mmdbr_pipe_mtx);
183 MMDB_DEBUG("read mmdbr %d", ws_pipe_data_available(mmdbr_pipe.stdout_fd));
185 int read_buf_size = 2048;
186 char *read_buf = (char *) g_malloc(read_buf_size);
188 while (ws_pipe_data_available(mmdbr_pipe.stdout_fd)) {
190 char *line = fgets(read_buf, read_buf_size, mmdbr_stdout);
191 if (!line || ferror(mmdbr_stdout)) {
192 MMDB_DEBUG("read error %s", g_strerror(errno));
197 line = g_strstrip(line);
198 size_t line_len = strlen(line);
199 MMDB_DEBUG("read %zd bytes, feof %d: %s", line_len, feof(mmdbr_stdout), line);
200 if (line_len < 1) continue;
202 char *val_start = strchr(line, ':');
203 if (val_start) val_start++;
205 if (line[0] == '[' && line_len > 2) {
206 // [init] or resolved address in square brackets.
207 line[line_len - 1] = '\0';
208 g_strlcpy(cur_addr, line + 1, WS_INET6_ADDRSTRLEN);
209 init_lookup(&cur_lookup);
210 } else if (strcmp(line, RES_STATUS_ERROR) == 0) {
211 // Error during init.
213 init_lookup(&cur_lookup);
215 } else if (val_start && g_str_has_prefix(line, RES_COUNTRY_ISO_CODE)) {
216 cur_lookup.found = TRUE;
217 cur_lookup.country_iso = chunkify_string(val_start);
218 } else if (val_start && g_str_has_prefix(line, RES_COUNTRY_NAMES_EN)) {
219 cur_lookup.found = TRUE;
220 cur_lookup.country = chunkify_string(val_start);
221 } else if (val_start && g_str_has_prefix(line, RES_CITY_NAMES_EN)) {
222 cur_lookup.found = TRUE;
223 cur_lookup.city = chunkify_string(val_start);
224 } else if (val_start && g_str_has_prefix(line, RES_ASN_ORG)) {
225 cur_lookup.found = TRUE;
226 cur_lookup.as_org = chunkify_string(val_start);
227 } else if (val_start && g_str_has_prefix(line, RES_ASN_NUMBER)) {
228 if (ws_strtou32(val_start, NULL, &cur_lookup.as_number)) {
229 cur_lookup.found = TRUE;
231 MMDB_DEBUG("Invalid as number: %s", val_start);
233 } else if (val_start && g_str_has_prefix(line, RES_LOCATION_LATITUDE)) {
234 cur_lookup.found = TRUE;
235 cur_lookup.latitude = g_ascii_strtod(val_start, NULL);
236 } else if (val_start && g_str_has_prefix(line, RES_LOCATION_LONGITUDE)) {
237 cur_lookup.found = TRUE;
238 cur_lookup.longitude = g_ascii_strtod(val_start, NULL);
239 } else if (g_str_has_prefix(line, RES_END)) {
240 if (cur_lookup.found) {
241 mmdb_lookup_t *mmdb_val = (mmdb_lookup_t *) wmem_memdup(wmem_epan_scope(), &cur_lookup, sizeof(cur_lookup));
242 if (strstr(cur_addr, ".")) {
243 MMDB_DEBUG("inserting v4 %p %s: city %s country %s", (void *) mmdb_val, cur_addr, mmdb_val->city, mmdb_val->country);
245 ws_inet_pton4(cur_addr, &addr);
246 wmem_map_insert(mmdb_ipv4_map, GUINT_TO_POINTER(addr), mmdb_val);
248 } else if (strstr(cur_addr, ":")) {
249 MMDB_DEBUG("inserting v6 %p %s: city %s country %s", (void *) mmdb_val, cur_addr, mmdb_val->city, mmdb_val->country);
251 ws_inet_pton6(cur_addr, &addr);
252 wmem_map_insert(mmdb_ipv6_map, chunkify_v6_addr(&addr), mmdb_val);
257 init_lookup(&cur_lookup);
260 g_mutex_unlock(&mmdbr_pipe_mtx);
266 * Stop our mmdbresolve process.
268 static void mmdb_resolve_stop(void) {
271 while (mmdbr_request_q && (request = (char *) g_async_queue_try_pop(mmdbr_request_q)) != NULL) {
275 if (!mmdbr_pipe_valid()) {
276 MMDB_DEBUG("not cleaning up, invalid PID %d", mmdbr_pipe.pid);
280 g_mutex_lock(&mmdbr_pipe_mtx);
281 ws_close(mmdbr_pipe.stdin_fd);
282 fclose(mmdbr_stdout);
283 MMDB_DEBUG("closing pid %d", mmdbr_pipe.pid);
284 g_spawn_close_pid(mmdbr_pipe.pid);
285 mmdbr_pipe.pid = WS_INVALID_PID;
287 g_mutex_unlock(&mmdbr_pipe_mtx);
289 g_thread_join(mmdbr_thread);
294 * Start an mmdbresolve process.
296 static void mmdb_resolve_start(void) {
297 if (!mmdbr_request_q) {
298 mmdbr_request_q = g_async_queue_new();
301 if (!mmdb_ipv4_map) {
302 mmdb_ipv4_map = wmem_map_new(wmem_epan_scope(), g_direct_hash, g_direct_equal);
305 if (!mmdb_ipv6_map) {
306 mmdb_ipv6_map = wmem_map_new(wmem_epan_scope(), ipv6_oat_hash, ipv6_equal);
309 if (!mmdb_str_chunk) {
310 mmdb_str_chunk = wmem_map_new(wmem_epan_scope(), wmem_str_hash, g_str_equal);
313 if (!mmdb_ipv6_chunk) {
314 mmdb_ipv6_chunk = wmem_map_new(wmem_epan_scope(), ipv6_oat_hash, ipv6_equal);
317 if (!mmdb_file_arr) {
318 MMDB_DEBUG("unexpected mmdb_file_arr == NULL");
324 if (mmdb_file_arr->len == 0) {
325 MMDB_DEBUG("no GeoIP databases found");
329 GPtrArray *args = g_ptr_array_new();
330 char *mmdbresolve = g_strdup_printf("%s%c%s", get_progfile_dir(), G_DIR_SEPARATOR, "mmdbresolve");
331 g_ptr_array_add(args, mmdbresolve);
332 for (guint i = 0; i < mmdb_file_arr->len; i++) {
333 g_ptr_array_add(args, g_strdup("-f"));
334 g_ptr_array_add(args, g_strdup((const gchar *)g_ptr_array_index(mmdb_file_arr, i)));
336 g_ptr_array_add(args, NULL);
338 ws_pipe_init(&mmdbr_pipe);
340 GPid pipe_pid = ws_pipe_spawn_async(&mmdbr_pipe, args);
341 MMDB_DEBUG("spawned %s pid %d", mmdbresolve, pipe_pid);
343 for (guint i = 0; i < args->len; i++) {
344 char *arg = (char *)g_ptr_array_index(args, i);
345 MMDB_DEBUG("args: %s", arg);
348 g_ptr_array_free(args, TRUE);
350 if (pipe_pid == WS_INVALID_PID) {
351 ws_pipe_init(&mmdbr_pipe);
355 // XXX Should we set O_NONBLOCK similar to dumpcap?
356 mmdbr_stdout = ws_fdopen(mmdbr_pipe.stdout_fd, "r");
357 setvbuf(mmdbr_stdout, NULL, _IONBF, 0);
359 mmdbr_thread = g_thread_new("write_mmdbr_stdin_worker", write_mmdbr_stdin_worker, NULL);
363 * Scan a directory for GeoIP databases and load them
366 maxmind_db_scan_dir(const char *dirname) {
370 if ((dir = ws_dir_open(dirname, 0, NULL)) != NULL) {
371 while ((file = ws_dir_read_name(dir)) != NULL) {
372 const char *name = ws_dir_get_name(file);
373 if (g_str_has_suffix(file, ".mmdb")) {
374 char *datname = g_strdup_printf("%s" G_DIR_SEPARATOR_S "%s", dirname, name);
375 FILE *mmdb_f = ws_fopen(datname, "r");
377 g_ptr_array_add(mmdb_file_arr, datname);
389 static void* maxmind_db_path_copy_cb(void* dest, const void* orig, size_t len _U_) {
390 const maxmind_db_path_t *m = (const maxmind_db_path_t *)orig;
391 maxmind_db_path_t *d = (maxmind_db_path_t *)dest;
393 d->path = g_strdup(m->path);
398 static void maxmind_db_path_free_cb(void* p) {
399 maxmind_db_path_t *m = (maxmind_db_path_t *)p;
403 static void maxmind_db_cleanup(void) {
408 /* If we have old data, clear out the whole thing
409 * and start again. TODO: Just update the ones that
410 * have changed for efficiency's sake. */
412 for (i = 0; i < mmdb_file_arr->len; i++) {
413 g_free(g_ptr_array_index(mmdb_file_arr, i));
415 /* finally, free the array itself */
416 g_ptr_array_free(mmdb_file_arr, TRUE);
417 mmdb_file_arr = NULL;
421 /* called every time the user presses "Apply" or "OK in the list of
422 * GeoIP directories, and also once on startup */
423 static void maxmind_db_post_update_cb(void) {
426 maxmind_db_cleanup();
428 /* allocate the array */
429 mmdb_file_arr = g_ptr_array_new();
431 /* First try the system paths */
432 for (i = 0; maxmind_db_system_paths[i].path != NULL; i++) {
433 maxmind_db_scan_dir(maxmind_db_system_paths[i].path);
436 /* Walk all the directories */
437 for (i = 0; i < num_maxmind_db_paths; i++) {
438 if (maxmind_db_paths[i].path) {
439 maxmind_db_scan_dir(maxmind_db_paths[i].path);
443 mmdb_resolve_start();
447 * Initialize GeoIP lookups
450 maxmind_db_pref_init(module_t *nameres)
452 static uat_field_t maxmind_db_paths_fields[] = {
453 UAT_FLD_DIRECTORYNAME(maxmind_mod, path, "MaxMind Database Directory", "The MaxMind database directory path"),
457 maxmind_db_paths_uat = uat_new("MaxMind Database Paths",
458 sizeof(maxmind_db_path_t),
460 FALSE, // Global, not per-profile
461 (void**)&maxmind_db_paths,
462 &num_maxmind_db_paths,
463 UAT_AFFECTS_DISSECTION, // Affects IP4 and IPv6 packets.
465 maxmind_db_path_copy_cb,
467 maxmind_db_path_free_cb,
468 maxmind_db_post_update_cb,
470 maxmind_db_paths_fields);
472 prefs_register_uat_preference(nameres,
474 "MaxMind database directories",
475 "Search paths for MaxMind address mapping databases."
476 " Wireshark will look in each directory for files ending"
478 maxmind_db_paths_uat);
481 void maxmind_db_pref_cleanup(void)
490 gboolean maxmind_db_lookup_process(void)
492 if (mmdbr_pipe_valid()) {
496 gboolean prev_ne = new_entries;
501 const mmdb_lookup_t *
502 maxmind_db_lookup_ipv4(guint32 addr) {
503 mmdb_lookup_t *result = (mmdb_lookup_t *) wmem_map_lookup(mmdb_ipv4_map, GUINT_TO_POINTER(addr));
506 // Try again, mainly so that we empty our pipe buffers.
508 result = (mmdb_lookup_t *) wmem_map_lookup(mmdb_ipv4_map, GUINT_TO_POINTER(addr));
512 if (mmdbr_pipe_valid()) {
513 char addr_str[WS_INET_ADDRSTRLEN];
514 ws_inet_ntop4(&addr, addr_str, WS_INET_ADDRSTRLEN);
515 MMDB_DEBUG("looking up %s", addr_str);
516 g_async_queue_push(mmdbr_request_q, g_strdup_printf("%s\n", addr_str));
519 result = &mmdb_not_found;
520 wmem_map_insert(mmdb_ipv4_map, GUINT_TO_POINTER(addr), result);
526 const mmdb_lookup_t *
527 maxmind_db_lookup_ipv6(const ws_in6_addr *addr) {
528 mmdb_lookup_t * result = (mmdb_lookup_t *) wmem_map_lookup(mmdb_ipv6_map, addr->bytes);
531 // Try again, mainly so that we empty our pipe buffers.
533 result = (mmdb_lookup_t *) wmem_map_lookup(mmdb_ipv6_map, addr->bytes);
537 if (mmdbr_pipe_valid()) {
538 char addr_str[WS_INET6_ADDRSTRLEN];
539 ws_inet_ntop6(addr, addr_str, WS_INET6_ADDRSTRLEN);
540 MMDB_DEBUG("looking up %s", addr_str);
541 g_async_queue_push(mmdbr_request_q, g_strdup_printf("%s\n", addr_str));
544 result = &mmdb_not_found;
545 wmem_map_insert(mmdb_ipv6_map, chunkify_v6_addr(addr), result);
552 maxmind_db_get_paths(void) {
553 GString* path_str = NULL;
556 path_str = g_string_new("");
558 for (i = 0; maxmind_db_system_paths[i].path != NULL; i++) {
559 g_string_append_printf(path_str,
560 "%s" G_SEARCHPATH_SEPARATOR_S, maxmind_db_system_paths[i].path);
563 for (i = 0; i < num_maxmind_db_paths; i++) {
564 if (maxmind_db_paths[i].path) {
565 g_string_append_printf(path_str,
566 "%s" G_SEARCHPATH_SEPARATOR_S, maxmind_db_paths[i].path);
570 g_string_truncate(path_str, path_str->len-1);
572 return g_string_free(path_str, FALSE);
575 #else // HAVE_MAXMINDDB
578 maxmind_db_pref_init(module_t *nameres _U_) {}
581 maxmind_db_pref_cleanup(void) {}
585 maxmind_db_lookup_process(void)
590 const mmdb_lookup_t *
591 maxmind_db_lookup_ipv4(guint32 addr _U_) {
592 return &mmdb_not_found;
595 const mmdb_lookup_t *
596 maxmind_db_lookup_ipv6(const ws_in6_addr *addr _U_) {
597 return &mmdb_not_found;
601 maxmind_db_get_paths(void) {
604 #endif // HAVE_MAXMINDDB
613 * indent-tabs-mode: nil
616 * ex: set shiftwidth=4 tabstop=8 expandtab:
617 * :indentSize=4:tabSize=8:noTabs=true: