[Automatic update for 2016-12-18]
[metze/wireshark/wip.git] / file.c
diff --git a/file.c b/file.c
index 44fc2335484474405c33beeac391ec4a22d9eb4f..b5920fd569a6933c07d257e3f2035210b0778a03 100644 (file)
--- a/file.c
+++ b/file.c
 
 #include <config.h>
 
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
 #include <time.h>
 
 #include <stdlib.h>
 #include <ctype.h>
 #include <errno.h>
 
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-
 #include <wsutil/tempfile.h>
 #include <wsutil/file_util.h>
 #include <wsutil/filesystem.h>
-#include <wsutil/ws_version_info.h>
+#include <ws_version_info.h>
 
 #include <wiretap/merge.h>
 
@@ -61,9 +53,8 @@
 #include <epan/dfilter/dfilter-macro.h>
 #include <epan/strutil.h>
 #include <epan/addr_resolv.h>
+#include <epan/color_filters.h>
 
-#include "color.h"
-#include "color_filters.h"
 #include "cfile.h"
 #include "file.h"
 #include "fileset.h"
@@ -96,7 +87,7 @@
 # include <winsock2.h>
 #endif
 
-#if defined(_WIN32) && defined(INET6)
+#ifdef _WIN32
 # include <ws2tcpip.h>
 #endif
 
@@ -127,6 +118,8 @@ static match_result match_wide(capture_file *cf, frame_data *fdata,
     void *criterion);
 static match_result match_binary(capture_file *cf, frame_data *fdata,
     void *criterion);
+static match_result match_regex(capture_file *cf, frame_data *fdata,
+    void *criterion);
 static match_result match_dfilter(capture_file *cf, frame_data *fdata,
     void *criterion);
 static match_result match_marked(capture_file *cf, frame_data *fdata,
@@ -145,11 +138,12 @@ static void cf_open_failure_alert_box(const char *filename, int err,
 static void cf_rename_failure_alert_box(const char *filename, int err);
 static void cf_close_failure_alert_box(const char *filename, int err);
 static void ref_time_packets(capture_file *cf);
-/* Update the progress bar this many times when reading a file. */
-#define N_PROGBAR_UPDATES   100
-/* We read around 200k/100ms don't update the progress bar more often than that */
-#define MIN_QUANTUM         200000
-#define MIN_NUMBER_OF_PACKET 1500
+
+/* Seconds spent processing packets between pushing UI updates. */
+#define PROGBAR_UPDATE_INTERVAL 0.150
+
+/* Show the progress bar after this many seconds. */
+#define PROGBAR_SHOW_DELAY 0.5
 
 /*
  * We could probably use g_signal_...() instead of the callbacks below but that
@@ -367,9 +361,6 @@ cf_open(capture_file *cf, const char *fname, unsigned int type, gboolean is_temp
   cf->prev_cap = NULL;
   cf->cum_bytes = 0;
 
-  /* Adjust timestamp precision if auto is selected, col width will be adjusted */
-  cf_timestamp_auto_precision(cf);
-  /* XXX needed ? */
   packet_list_queue_draw();
   cf_callback_invoke(cf_cb_file_opened, cf);
 
@@ -408,6 +399,7 @@ cf_add_encapsulation_type(capture_file *cf, int encap)
 void
 cf_close(capture_file *cf)
 {
+  cf->stop_flag = FALSE;
   if (cf->state == FILE_CLOSED)
     return; /* Nothing to do */
 
@@ -494,6 +486,24 @@ cf_close(capture_file *cf)
   cf_callback_invoke(cf_cb_file_closed, cf);
 }
 
+/*
+ * TRUE if the progress dialog doesn't exist and it looks like we'll
+ * take > 2s to load, FALSE otherwise.
+ */
+static inline gboolean
+progress_is_slow(progdlg_t *progdlg, GTimer *prog_timer, gint64 size, gint64 pos)
+{
+  double elapsed;
+
+  if (progdlg) return FALSE;
+  elapsed = g_timer_elapsed(prog_timer, NULL);
+  if ((elapsed / 2 > PROGBAR_SHOW_DELAY && (size / pos) > 2) /* It looks like we're going to be slow. */
+      || elapsed > PROGBAR_SHOW_DELAY) { /* We are indeed slow. */
+    return TRUE;
+  }
+  return FALSE;
+}
+
 static float
 calc_progbar_val(capture_file *cf, gint64 size, gint64 file_pos, gchar *status_str, gulong status_size)
 {
@@ -529,11 +539,11 @@ calc_progbar_val(capture_file *cf, gint64 size, gint64 file_pos, gchar *status_s
 cf_read_status_t
 cf_read(capture_file *cf, gboolean reloading)
 {
-  int                  err;
-  gchar               *err_info;
+  int                  err = 0;
+  gchar               *err_info = NULL;
   gchar               *name_ptr;
-  progdlg_t           *progbar        = NULL;
-  gboolean             stop_flag;
+  progdlg_t           *volatile progbar = NULL;
+  GTimer              *prog_timer = g_timer_new();
   GTimeVal             start_time;
   epan_dissect_t       edt;
   dfilter_t           *dfcode;
@@ -569,23 +579,18 @@ cf_read(capture_file *cf, gboolean reloading)
   /* The packet list window will be empty until the file is completly loaded */
   packet_list_freeze();
 
-  stop_flag = FALSE;
+  cf->stop_flag = FALSE;
   g_get_current_time(&start_time);
 
   epan_dissect_init(&edt, cf->epan, create_proto_tree, FALSE);
 
   TRY {
-#ifdef HAVE_LIBPCAP
-    int     displayed_once    = 0;
-#endif
     int     count             = 0;
 
     gint64  size;
     gint64  file_pos;
     gint64  data_offset;
 
-    gint64  progbar_quantum;
-    gint64  progbar_nextstep;
     float   progbar_val;
     gchar   status_str[100];
 
@@ -596,59 +601,41 @@ cf_read(capture_file *cf, gboolean reloading)
     /* Find the size of the file. */
     size = wtap_file_size(cf->wth, NULL);
 
-    /* Update the progress bar when it gets to this value. */
-    progbar_nextstep = 0;
-    /* When we reach the value that triggers a progress bar update,
-       bump that value by this amount. */
-    if (size >= 0) {
-      progbar_quantum = size/N_PROGBAR_UPDATES;
-      if (progbar_quantum < MIN_QUANTUM)
-        progbar_quantum = MIN_QUANTUM;
-    }else
-      progbar_quantum = 0;
+    g_timer_start(prog_timer);
 
     while ((wtap_read(cf->wth, &err, &err_info, &data_offset))) {
       if (size >= 0) {
         count++;
         file_pos = wtap_read_so_far(cf->wth);
 
-        /* Create the progress bar if necessary.
-         * Check whether it should be created or not every MIN_NUMBER_OF_PACKET
-         */
-        if ((progbar == NULL) && !(count % MIN_NUMBER_OF_PACKET)) {
+        /* Create the progress bar if necessary. */
+        if (progress_is_slow(progbar, prog_timer, size, file_pos)) {
           progbar_val = calc_progbar_val(cf, size, file_pos, status_str, sizeof(status_str));
           if (reloading)
             progbar = delayed_create_progress_dlg(cf->window, "Reloading", name_ptr,
-                TRUE, &stop_flag, &start_time, progbar_val);
+                TRUE, &cf->stop_flag, &start_time, progbar_val);
           else
             progbar = delayed_create_progress_dlg(cf->window, "Loading", name_ptr,
-                TRUE, &stop_flag, &start_time, progbar_val);
+                TRUE, &cf->stop_flag, &start_time, progbar_val);
         }
 
-        /* Update the progress bar, but do it only N_PROGBAR_UPDATES times;
-           when we update it, we have to run the GTK+ main loop to get it
-           to repaint what's pending, and doing so may involve an "ioctl()"
-           to see if there's any pending input from an X server, and doing
-           that for every packet can be costly, especially on a big file. */
-        if (file_pos >= progbar_nextstep) {
-          if (progbar != NULL) {
-            progbar_val = calc_progbar_val(cf, size, file_pos, status_str, sizeof(status_str));
-            /* update the packet bar content on the first run or frequently on very large files */
-#ifdef HAVE_LIBPCAP
-            if (progbar_quantum > 500000 || displayed_once == 0) {
-              if ((auto_scroll_live || displayed_once == 0 || cf->displayed_count < 1000) && cf->count != 0) {
-                displayed_once = 1;
-                packets_bar_update();
-              }
-            }
-#endif /* HAVE_LIBPCAP */
-            update_progress_dlg(progbar, progbar_val, status_str);
-          }
-          progbar_nextstep += progbar_quantum;
+        /*
+         * Update the progress bar, but do it only after
+         * PROGBAR_UPDATE_INTERVAL has elapsed. Calling update_progress_dlg
+         * and packets_bar_update will likely trigger UI paint events, which
+         * might take a while depending on the platform and display. Reset
+         * our timer *after* painting.
+         */
+        if (progbar && g_timer_elapsed(prog_timer, NULL) > PROGBAR_UPDATE_INTERVAL) {
+          progbar_val = calc_progbar_val(cf, size, file_pos, status_str, sizeof(status_str));
+          /* update the packet bar content on the first run or frequently on very large files */
+          update_progress_dlg(progbar, progbar_val, status_str);
+          packets_bar_update();
+          g_timer_start(prog_timer);
         }
       }
 
-      if (stop_flag) {
+      if (cf->stop_flag) {
         /* Well, the user decided to abort the read. He/She will be warned and
            it might be enough for him/her to work with the already loaded
            packets.
@@ -687,6 +674,7 @@ cf_read(capture_file *cf, gboolean reloading)
   /* We're done reading the file; destroy the progress bar if it was created. */
   if (progbar != NULL)
     destroy_progress_dlg(progbar);
+  g_timer_destroy(prog_timer);
 
   /* We're done reading sequentially through the file. */
   cf->state = FILE_READ_DONE;
@@ -722,7 +710,7 @@ cf_read(capture_file *cf, gboolean reloading)
     packet_list_select_first_row();
   }
 
-  if (stop_flag) {
+  if (cf->stop_flag) {
     simple_message_box(ESD_TYPE_WARN, NULL,
                   "The remaining packets in the file were discarded.\n"
                   "\n"
@@ -760,7 +748,7 @@ cf_read(capture_file *cf, gboolean reloading)
 
     case WTAP_ERR_DECOMPRESS:
       simple_error_message_box(
-                 "The compressed capture file appears to be damaged or corrupt.\n",
+                 "The compressed capture file appears to be damaged or corrupt.\n(%s)",
                  err_info != NULL ? err_info : "no information supplied");
       g_free(err_info);
       break;
@@ -1118,6 +1106,15 @@ add_packet_to_packet_list(frame_data *fdata, capture_file *cf,
   if (dfcode != NULL) {
       epan_dissect_prime_dfilter(edt, dfcode);
   }
+#if 0
+  /* Prepare coloring rules, this ensures that display filter rules containing
+   * frame.color_rule references are still processed.
+   * TODO: actually detect that situation or maybe apply other optimizations? */
+  if (edt->tree && color_filters_used()) {
+    color_filters_prime_edt(edt);
+    fdata->flags.need_colorize = 1;
+  }
+#endif
 
   /* Dissect the frame. */
   epan_dissect_run_with_taps(edt, cf->cd_t, phdr, frame_tvbuff_new(fdata, buf), fdata, cinfo);
@@ -1149,19 +1146,7 @@ add_packet_to_packet_list(frame_data *fdata, capture_file *cf,
     frame_data_set_after_dissect(fdata, &cf->cum_bytes);
     cf->prev_dis = fdata;
 
-    /* If we haven't yet seen the first frame, this is it.
-
-       XXX - we must do this before we add the row to the display,
-       as, if the display's GtkCList's selection mode is
-       GTK_SELECTION_BROWSE, when the first entry is added to it,
-       "cf_select_packet()" will be called, and it will fetch the row
-       data for the 0th row, and will get a null pointer rather than
-       "fdata", as "gtk_clist_append()" won't yet have returned and
-       thus "gtk_clist_set_row_data()" won't yet have been called.
-
-       We thus need to leave behind bread crumbs so that
-       "cf_select_packet()" can find this frame.  See the comment
-       in "cf_select_packet()". */
+    /* If we haven't yet seen the first frame, this is it. */
     if (cf->first_displayed == 0)
       cf->first_displayed = fdata->num;
 
@@ -1230,414 +1215,168 @@ read_packet(capture_file *cf, dfilter_t *dfcode, epan_dissect_t *edt,
   return row;
 }
 
-cf_status_t
-cf_merge_files(char **out_filenamep, int in_file_count,
-               char *const *in_filenames, int file_type, gboolean do_append)
-{
-  merge_in_file_t *in_files, *in_file;
-  char            *out_filename;
-  char            *tmpname;
-  int              out_fd;
-  wtap_dumper     *pdh;
-  int              open_err, read_err, write_err, close_err;
-  gchar           *err_info, *write_err_info = NULL;
-  int              err_fileno;
-  int              i;
-  gboolean         got_read_error     = FALSE, got_write_error = FALSE;
-  gint64           data_offset;
-  progdlg_t       *progbar            = NULL;
-  gboolean         stop_flag;
-  gint64           f_len, file_pos;
-  float            progbar_val;
-  GTimeVal         start_time;
-  gchar            status_str[100];
-  gint64           progbar_nextstep;
-  gint64           progbar_quantum;
-  gchar           *display_basename;
-  int              selected_frame_type;
-  gboolean         fake_interface_ids = FALSE;
-
-  /* open the input files */
-  if (!merge_open_in_files(in_file_count, in_filenames, &in_files,
-                           &open_err, &err_info, &err_fileno)) {
-    g_free(in_files);
-    cf_open_failure_alert_box(in_filenames[err_fileno], open_err, err_info,
-                              FALSE, 0);
-    return CF_ERROR;
-  }
 
-  if (*out_filenamep != NULL) {
-    out_filename = *out_filenamep;
-    out_fd = ws_open(out_filename, O_CREAT|O_TRUNC|O_BINARY, 0600);
-    if (out_fd == -1)
-      open_err = errno;
-  } else {
-    out_fd = create_tempfile(&tmpname, "wireshark");
-    if (out_fd == -1)
-      open_err = errno;
-    out_filename = g_strdup(tmpname);
-    *out_filenamep = out_filename;
-  }
-  if (out_fd == -1) {
-    err_info = NULL;
-    merge_close_in_files(in_file_count, in_files);
-    g_free(in_files);
-    cf_open_failure_alert_box(out_filename, open_err, NULL, TRUE, file_type);
-    return CF_ERROR;
-  }
-
-  selected_frame_type = merge_select_frame_type(in_file_count, in_files);
-
-  /* If we are trying to merge a number of libpcap files with different encapsulation types
-   * change the output file type to pcapng and create SHB and IDB:s for the new file use the
-   * interface index stored in in_files per file to change the phdr before writing the datablock.
-   * XXX should it be an option to convert to pcapng?
-   *
-   * We need something similar when merging pcapng files possibly with an option to say
-   * the same interface(s) used in all in files. SHBs comments should be merged together.
-   */
-  if ((selected_frame_type == WTAP_ENCAP_PER_PACKET)&&(file_type == WTAP_FILE_TYPE_SUBTYPE_PCAP)) {
-    /* Write output in pcapng format */
-    wtapng_section_t            *shb_hdr;
-    wtapng_iface_descriptions_t *idb_inf, *idb_inf_merge_file;
-    wtapng_if_descr_t            int_data, *file_int_data;
-    GString                     *comment_gstr;
-    guint                        itf_count, itf_id = 0;
-
-    fake_interface_ids = TRUE;
-    /* Create SHB info */
-    shb_hdr      = wtap_file_get_shb_info(in_files[0].wth);
-    comment_gstr = g_string_new("");
-    g_string_append_printf(comment_gstr, "%s \n",shb_hdr->opt_comment);
-    g_string_append_printf(comment_gstr, "File created by merging: \n");
-    file_type = WTAP_FILE_TYPE_SUBTYPE_PCAPNG;
-
-    for (i = 0; i < in_file_count; i++) {
-        g_string_append_printf(comment_gstr, "File%d: %s \n",i+1,in_files[i].filename);
-    }
-    shb_hdr->section_length = -1;
-    /* options */
-    shb_hdr->opt_comment   = g_string_free(comment_gstr, FALSE);  /* NULL if not available */
-    shb_hdr->shb_hardware  = NULL;        /* NULL if not available, UTF-8 string containing the        */
-                                          /*  description of the hardware used to create this section. */
-    shb_hdr->shb_os        = NULL;        /* NULL if not available, UTF-8 string containing the name   */
-                                          /*  of the operating system used to create this section.     */
-    shb_hdr->shb_user_appl = g_strdup("Wireshark"); /* NULL if not available, UTF-8 string containing the name   */
-                                          /*  of the application used to create this section.          */
-
-    /* create fake IDB info */
-    idb_inf = g_new(wtapng_iface_descriptions_t,1);
-    /* TODO make this the number of DIFFERENT encapsulation types
-     * check that snaplength is the same too?
-     */
-    idb_inf->interface_data = g_array_new(FALSE, FALSE, sizeof(wtapng_if_descr_t));
-
-    for (i = 0; i < in_file_count; i++) {
-      idb_inf_merge_file               = wtap_file_get_idb_info(in_files[i].wth);
-      for (itf_count = 0; itf_count < idb_inf_merge_file->interface_data->len; itf_count++) {
-        /* read the interface data from the in file to our combined interface data */
-        file_int_data = &g_array_index (idb_inf_merge_file->interface_data, wtapng_if_descr_t, itf_count);
-        int_data.wtap_encap            = file_int_data->wtap_encap;
-        int_data.time_units_per_second = file_int_data->time_units_per_second;
-        int_data.link_type             = file_int_data->link_type;
-        int_data.snap_len              = file_int_data->snap_len;
-        int_data.if_name               = g_strdup(file_int_data->if_name);
-        int_data.opt_comment           = NULL;
-        int_data.if_description        = NULL;
-        int_data.if_speed              = 0;
-        int_data.if_tsresol            = 6;
-        int_data.if_filter_str         = NULL;
-        int_data.bpf_filter_len        = 0;
-        int_data.if_filter_bpf_bytes   = NULL;
-        int_data.if_os                 = NULL;
-        int_data.if_fcslen             = -1;
-        int_data.num_stat_entries      = 0;          /* Number of ISB:s */
-        int_data.interface_statistics  = NULL;
-
-        g_array_append_val(idb_inf->interface_data, int_data);
-      }
-      g_free(idb_inf_merge_file);
-
-      /* Set fake interface Id in per file data */
-      in_files[i].interface_id = itf_id;
-      itf_id += itf_count;
-    }
+typedef struct _callback_data_t {
+  gint64           f_len;
+  GTimeVal         start_time;
+  progdlg_t       *progbar;
+  GTimer          *prog_timer;
+  gboolean         stop_flag;
+} callback_data_t;
 
-    pdh = wtap_dump_fdopen_ng(out_fd, file_type,
-                              selected_frame_type,
-                              merge_max_snapshot_length(in_file_count, in_files),
-                              FALSE /* compressed */, shb_hdr, idb_inf /* wtapng_iface_descriptions_t *idb_inf */, &open_err);
 
-    if (pdh == NULL) {
-      ws_close(out_fd);
-      merge_close_in_files(in_file_count, in_files);
-      g_free(in_files);
-      cf_open_failure_alert_box(out_filename, open_err, err_info, TRUE,
-                                file_type);
-      return CF_ERROR;
-    }
+static gboolean
+merge_callback(merge_event event, int num _U_,
+               const merge_in_file_t in_files[], const guint in_file_count,
+               void *data)
+{
+  guint i;
+  callback_data_t *cb_data = (callback_data_t*) data;
 
-  } else {
+  g_assert(cb_data != NULL);
 
-    pdh = wtap_dump_fdopen(out_fd, file_type,
-                           selected_frame_type,
-                           merge_max_snapshot_length(in_file_count, in_files),
-                           FALSE /* compressed */, &open_err);
-    if (pdh == NULL) {
-      ws_close(out_fd);
-      merge_close_in_files(in_file_count, in_files);
-      g_free(in_files);
-      cf_open_failure_alert_box(out_filename, open_err, err_info, TRUE,
-                                file_type);
-      return CF_ERROR;
-    }
-  }
+  switch (event) {
 
-  /* Get the sum of the sizes of all the files. */
-  f_len = 0;
-  for (i = 0; i < in_file_count; i++)
-    f_len += in_files[i].size;
+    case MERGE_EVENT_INPUT_FILES_OPENED:
+      /* do nothing */
+      break;
 
-  /* Update the progress bar when it gets to this value. */
-  progbar_nextstep = 0;
-  /* When we reach the value that triggers a progress bar update,
-     bump that value by this amount. */
-  progbar_quantum = f_len/N_PROGBAR_UPDATES;
-  /* Progress so far. */
-  progbar_val = 0.0f;
+    case MERGE_EVENT_FRAME_TYPE_SELECTED:
+      /* do nothing */
+      break;
 
-  stop_flag = FALSE;
-  g_get_current_time(&start_time);
+    case MERGE_EVENT_READY_TO_MERGE:
+      /* Get the sum of the sizes of all the files. */
+      for (i = 0; i < in_file_count; i++)
+        cb_data->f_len += in_files[i].size;
 
-  /* do the merge (or append) */
-  for (;;) {
-    if (do_append)
-      in_file = merge_append_read_packet(in_file_count, in_files, &read_err,
-                                         &err_info);
-    else
-      in_file = merge_read_packet(in_file_count, in_files, &read_err,
-                                  &err_info);
-    if (in_file == NULL) {
-      /* EOF */
-      break;
-    }
+      cb_data->prog_timer = g_timer_new();
+      g_timer_start(cb_data->prog_timer);
 
-    if (read_err != 0) {
-      /* I/O error reading from in_file */
-      got_read_error = TRUE;
+      g_get_current_time(&cb_data->start_time);
       break;
-    }
 
-    /* Get the sum of the data offsets in all of the files. */
-    data_offset = 0;
-    for (i = 0; i < in_file_count; i++)
-      data_offset += in_files[i].data_offset;
+    case MERGE_EVENT_PACKET_WAS_READ:
+      {
+        gint64 data_offset = 0;
 
-    /* Create the progress bar if necessary.
-       We check on every iteration of the loop, so that it takes no
-       longer than the standard time to create it (otherwise, for a
-       large file, we might take considerably longer than that standard
-       time in order to get to the next progress bar step). */
-    if (progbar == NULL) {
-      progbar = delayed_create_progress_dlg(NULL, "Merging", "files",
-        FALSE, &stop_flag, &start_time, progbar_val);
-    }
-
-    /* Update the progress bar, but do it only N_PROGBAR_UPDATES times;
-       when we update it, we have to run the GTK+ main loop to get it
-       to repaint what's pending, and doing so may involve an "ioctl()"
-       to see if there's any pending input from an X server, and doing
-       that for every packet can be costly, especially on a big file. */
-    if (data_offset >= progbar_nextstep) {
-        /* Get the sum of the seek positions in all of the files. */
-        file_pos = 0;
+        /* Get the sum of the data offsets in all of the files. */
+        data_offset = 0;
         for (i = 0; i < in_file_count; i++)
-          file_pos += wtap_read_so_far(in_files[i].wth);
-        progbar_val = (gfloat) file_pos / (gfloat) f_len;
-        if (progbar_val > 1.0f) {
-          /* Some file probably grew while we were reading it.
-             That "shouldn't happen", so we'll just clip the progress
-             value at 1.0. */
-          progbar_val = 1.0f;
-        }
-        if (progbar != NULL) {
-          g_snprintf(status_str, sizeof(status_str),
-                     "%" G_GINT64_MODIFIER "dKB of %" G_GINT64_MODIFIER "dKB",
-                     file_pos / 1024, f_len / 1024);
-          update_progress_dlg(progbar, progbar_val, status_str);
-        }
-        progbar_nextstep += progbar_quantum;
-    }
+          data_offset += in_files[i].data_offset;
 
-    if (stop_flag) {
-      /* Well, the user decided to abort the merge. */
-      break;
-    }
+        /* Create the progress bar if necessary.
+           We check on every iteration of the loop, so that it takes no
+           longer than the standard time to create it (otherwise, for a
+           large file, we might take considerably longer than that standard
+           time in order to get to the next progress bar step). */
+        if (cb_data->progbar == NULL) {
+          cb_data->progbar = delayed_create_progress_dlg(NULL, "Merging", "files",
+            FALSE, &cb_data->stop_flag, &cb_data->start_time, 0.0f);
+        }
 
-    /* If we have WTAP_ENCAP_PER_PACKET and the infiles are of type
-     * WTAP_FILE_TYPE_SUBTYPE_PCAP, we need to set the interface id
-     * in the paket header = the interface index we used in the IDBs
-     * interface description for this file(encapsulation type).
-     */
-    if (fake_interface_ids) {
-      struct wtap_pkthdr *phdr;
+        /*
+         * Update the progress bar, but do it only after
+         * PROGBAR_UPDATE_INTERVAL has elapsed. Calling update_progress_dlg
+         * and packets_bar_update will likely trigger UI paint events, which
+         * might take a while depending on the platform and display. Reset
+         * our timer *after* painting.
+         */
+        if (g_timer_elapsed(cb_data->prog_timer, NULL) > PROGBAR_UPDATE_INTERVAL) {
+            float  progbar_val;
+            gint64 file_pos = 0;
+            /* Get the sum of the seek positions in all of the files. */
+            for (i = 0; i < in_file_count; i++)
+              file_pos += wtap_read_so_far(in_files[i].wth);
+
+            progbar_val = (gfloat) file_pos / (gfloat) cb_data->f_len;
+            if (progbar_val > 1.0f) {
+              /* Some file probably grew while we were reading it.
+                 That "shouldn't happen", so we'll just clip the progress
+                 value at 1.0. */
+              progbar_val = 1.0f;
+            }
 
-      phdr = wtap_phdr(in_file->wth);
-      if (phdr->presence_flags & WTAP_HAS_INTERFACE_ID) {
-        phdr->interface_id += in_file->interface_id;
-      } else {
-        phdr->interface_id = in_file->interface_id;
-        phdr->presence_flags = phdr->presence_flags | WTAP_HAS_INTERFACE_ID;
+            if (cb_data->progbar != NULL) {
+              gchar status_str[100];
+              g_snprintf(status_str, sizeof(status_str),
+                         "%" G_GINT64_MODIFIER "dKB of %" G_GINT64_MODIFIER "dKB",
+                         file_pos / 1024, cb_data->f_len / 1024);
+              update_progress_dlg(cb_data->progbar, progbar_val, status_str);
+            }
+            g_timer_start(cb_data->prog_timer);
+        }
       }
-    }
-    if (!wtap_dump(pdh, wtap_phdr(in_file->wth),
-                   wtap_buf_ptr(in_file->wth), &write_err, &write_err_info)) {
-      got_write_error = TRUE;
       break;
-    }
-  }
-
-  /* We're done merging the files; destroy the progress bar if it was created. */
-  if (progbar != NULL)
-    destroy_progress_dlg(progbar);
 
-  merge_close_in_files(in_file_count, in_files);
-  if (!got_write_error) {
-    if (!wtap_dump_close(pdh, &write_err))
-      got_write_error = TRUE;
-  } else {
-    /*
-     * We already got a write error; no need to report another
-     * write error on close.
-     *
-     * Don't overwrite the earlier write error.
-     */
-    (void)wtap_dump_close(pdh, &close_err);
+    case MERGE_EVENT_DONE:
+      /* We're done merging the files; destroy the progress bar if it was created. */
+      if (cb_data->progbar != NULL)
+        destroy_progress_dlg(cb_data->progbar);
+      g_timer_destroy(cb_data->prog_timer);
+      break;
   }
 
-  if (got_read_error) {
-    /*
-     * Find the file on which we got the error, and report the error.
-     */
-    for (i = 0; i < in_file_count; i++) {
-      if (in_files[i].state == GOT_ERROR) {
-        /* Put up a message box noting that a read failed somewhere along
-           the line. */
-        display_basename = g_filename_display_basename(in_files[i].filename);
-        switch (read_err) {
-
-        case WTAP_ERR_SHORT_READ:
-          simple_error_message_box(
-                     "The capture file %s appears to have been cut short"
-                      " in the middle of a packet.", display_basename);
-          break;
-
-        case WTAP_ERR_BAD_FILE:
-          simple_error_message_box(
-                     "The capture file %s appears to be damaged or corrupt.\n(%s)",
-                     display_basename, err_info);
-          g_free(err_info);
-          break;
-
-        case WTAP_ERR_DECOMPRESS:
-          simple_error_message_box(
-                     "The compressed capture file %s appears to be damaged or corrupt.\n"
-                     "(%s)", display_basename,
-                     err_info != NULL ? err_info : "no information supplied");
-          g_free(err_info);
-          break;
+  return cb_data->stop_flag;
+}
 
-        default:
-          simple_error_message_box(
-                     "An error occurred while reading the"
-                     " capture file %s: %s.",
-                     display_basename,  wtap_strerror(read_err));
-          break;
-        }
-        g_free(display_basename);
-      }
-    }
-  }
 
-  if (got_write_error) {
-    /* Put up an alert box for the write error. */
-    if (write_err < 0) {
-      /* Wiretap error. */
-      switch (write_err) {
 
-      case WTAP_ERR_UNWRITABLE_ENCAP:
-        /*
-         * This is a problem with the particular frame we're writing and
-         * the file type and subtype we're writing; note that, and report
-         * the frame number and file type/subtype.
-         */
-        display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN");
-        simple_error_message_box(
-                      "Frame %u of \"%s\" has a network type that can't be saved in a \"%s\" file.",
-                      in_file ? in_file->packet_num : 0, display_basename,
-                      wtap_file_type_subtype_string(file_type));
-        g_free(display_basename);
-        break;
+cf_status_t
+cf_merge_files_to_tempfile(char **out_filenamep, int in_file_count,
+                           char *const *in_filenames, int file_type,
+                           gboolean do_append)
+{
+  int                        err      = 0;
+  gchar                     *err_info = NULL;
+  guint                      err_fileno;
+  merge_result               status;
+  merge_progress_callback_t  cb;
+
+  /* prepare our callback routine */
+  cb.callback_func = merge_callback;
+  cb.data = g_malloc0(sizeof(callback_data_t));
+
+  /* merge the files */
+  status = merge_files_to_tempfile(out_filenamep, "wireshark", file_type,
+                                   (const char *const *) in_filenames,
+                                   in_file_count, do_append,
+                                   IDB_MERGE_MODE_ALL_SAME, 0 /* snaplen */,
+                                   "Wireshark", &cb, &err, &err_info,
+                                   &err_fileno);
+
+  g_free(cb.data);
+
+  switch (status) {
+    case MERGE_OK:
+      break;
 
-      case WTAP_ERR_PACKET_TOO_LARGE:
-        /*
-         * This is a problem with the particular frame we're writing and
-         * the file type and subtype we're writing; note that, and report
-         * the frame number and file type/subtype.
-         */
-        display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN");
-        simple_error_message_box(
-                      "Frame %u of \"%s\" is too large for a \"%s\" file.",
-                      in_file ? in_file->packet_num : 0, display_basename,
-                      wtap_file_type_subtype_string(file_type));
-        g_free(display_basename);
-        break;
+    case MERGE_USER_ABORTED:
+      /* this isn't really an error, though we will return CF_ERROR later */
+      break;
 
-      case WTAP_ERR_UNWRITABLE_REC_TYPE:
-        /*
-         * This is a problem with the particular record we're writing and
-         * the file type and subtype we're writing; note that, and report
-         * the record number and file type/subtype.
-         */
-        display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN");
-        simple_error_message_box(
-                      "Record %u of \"%s\" has a record type that can't be saved in a \"%s\" file.",
-                      in_file ? in_file->packet_num : 0, display_basename,
-                      wtap_file_type_subtype_string(file_type));
-        g_free(display_basename);
-        break;
+    case MERGE_ERR_CANT_OPEN_INFILE:
+      cf_open_failure_alert_box(in_filenames[err_fileno], err, err_info,
+                                FALSE, 0);
+      break;
 
-      case WTAP_ERR_UNWRITABLE_REC_DATA:
-        /*
-         * This is a problem with the particular record we're writing and
-         * the file type and subtype we're writing; note that, and report
-         * the frame number and file type/subtype.
-         */
-        display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN");
-        simple_error_message_box(
-                      "Record %u of \"%s\" has data that can't be saved in a \"%s\" file.\n(%s)",
-                      in_file ? in_file->packet_num : 0, display_basename,
-                      wtap_file_type_subtype_string(file_type),
-                      write_err_info != NULL ? write_err_info : "no information supplied");
-        g_free(write_err_info);
-        g_free(display_basename);
-        break;
+    case MERGE_ERR_CANT_OPEN_OUTFILE:
+      cf_open_failure_alert_box(*out_filenamep, err, err_info, TRUE,
+                                file_type);
+      break;
 
-      default:
-        display_basename = g_filename_display_basename(out_filename);
-        simple_error_message_box(
-                      "An error occurred while writing to the file \"%s\": %s.",
-                      out_filename, wtap_strerror(write_err));
-        g_free(display_basename);
-        break;
-      }
-    } else {
-      /* OS error. */
-      write_failure_alert_box(out_filename, write_err);
-    }
+    case MERGE_ERR_CANT_READ_INFILE:      /* fall through */
+    case MERGE_ERR_BAD_PHDR_INTERFACE_ID:
+    case MERGE_ERR_CANT_WRITE_OUTFILE:
+    case MERGE_ERR_CANT_CLOSE_OUTFILE:
+    default:
+      simple_error_message_box("%s", err_info ? err_info : "unknown error");
+      break;
   }
 
-  if (got_read_error || got_write_error || stop_flag) {
+  g_free(err_info);
+
+  if (status != MERGE_OK) {
     /* Callers aren't expected to treat an error or an explicit abort
        differently - we put up error dialogs ourselves, so they don't
        have to. */
@@ -1699,10 +1438,12 @@ cf_filter_packets(capture_file *cf, gchar *dftext, gboolean force)
 
   /* Now rescan the packet list, applying the new filter, but not
      throwing away information constructed on a previous pass. */
-  if (dftext == NULL) {
-    rescan_packets(cf, "Resetting", "Filter", FALSE);
-  } else {
-    rescan_packets(cf, "Filtering", dftext, FALSE);
+  if (cf->state != FILE_CLOSED) {
+    if (dftext == NULL) {
+      rescan_packets(cf, "Resetting", "Filter", FALSE);
+    } else {
+      rescan_packets(cf, "Filtering", dftext, FALSE);
+    }
   }
 
   /* Cleanup and release all dfilter resources */
@@ -1798,7 +1539,7 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
   guint32     framenum;
   frame_data *fdata;
   progdlg_t  *progbar = NULL;
-  gboolean    stop_flag;
+  GTimer     *prog_timer = g_timer_new();
   int         count;
   frame_data *selected_frame, *preceding_frame, *following_frame, *prev_frame;
   int         selected_frame_num, preceding_frame_num, following_frame_num, prev_frame_num;
@@ -1806,8 +1547,6 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
   float       progbar_val;
   GTimeVal    start_time;
   gchar       status_str[100];
-  int         progbar_nextstep;
-  int         progbar_quantum;
   epan_dissect_t  edt;
   dfilter_t  *dfcode;
   column_info *cinfo;
@@ -1856,9 +1595,21 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
 
     /* 'reset' dissection session */
     epan_free(cf->epan);
+    if (cf->edt && cf->edt->pi.fd) {
+      /* All pointers in "per frame proto data" for the currently selected
+         packet are allocated in wmem_file_scope() and deallocated in epan_free().
+         Free them here to avoid unintended usage in packet_list_clear(). */
+      frame_data_destroy(cf->edt->pi.fd);
+    }
     cf->epan = ws_epan_new(cf);
     cf->cinfo.epan = cf->epan;
 
+    /* A new Lua tap listener may be registered in lua_prime_all_fields()
+       called via epan_new() / init_dissection() when reloading Lua plugins. */
+    if (!create_proto_tree && have_filtering_tap_listeners()) {
+      create_proto_tree = TRUE;
+    }
+
     /* We need to redissect the packets so we have to discard our old
      * packet list store. */
     packet_list_clear();
@@ -1880,17 +1631,15 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
   cf->prev_cap = NULL;
   cf->cum_bytes = 0;
 
-  /* Update the progress bar when it gets to this value. */
-  progbar_nextstep = 0;
-  /* When we reach the value that triggers a progress bar update,
-     bump that value by this amount. */
-  progbar_quantum = cf->count/N_PROGBAR_UPDATES;
+  cf_callback_invoke(cf_cb_file_rescan_started, cf);
+
+  g_timer_start(prog_timer);
   /* Count of packets at which we've looked. */
   count = 0;
   /* Progress so far. */
   progbar_val = 0.0f;
 
-  stop_flag = FALSE;
+  cf->stop_flag = FALSE;
   g_get_current_time(&start_time);
 
   /* no previous row yet */
@@ -1918,15 +1667,17 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
        time in order to get to the next progress bar step). */
     if (progbar == NULL)
       progbar = delayed_create_progress_dlg(cf->window, action, action_item, TRUE,
-                                            &stop_flag, &start_time,
+                                            &cf->stop_flag,
+                                            &start_time,
                                             progbar_val);
 
-    /* Update the progress bar, but do it only N_PROGBAR_UPDATES times;
-       when we update it, we have to run the GTK+ main loop to get it
-       to repaint what's pending, and doing so may involve an "ioctl()"
-       to see if there's any pending input from an X server, and doing
-       that for every packet can be costly, especially on a big file. */
-    if (count >= progbar_nextstep) {
+    /*
+     * Update the progress bar, but do it only after PROGBAR_UPDATE_INTERVAL
+     * has elapsed. Calling update_progress_dlg and packets_bar_update will
+     * likely trigger UI paint events, which might take a while depending on
+     * the platform and display. Reset our timer *after* painting.
+     */
+    if (g_timer_elapsed(prog_timer, NULL) > PROGBAR_UPDATE_INTERVAL) {
       /* let's not divide by zero. I should never be started
        * with count == 0, so let's assert that
        */
@@ -1939,10 +1690,10 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
         update_progress_dlg(progbar, progbar_val, status_str);
       }
 
-      progbar_nextstep += progbar_quantum;
+      g_timer_start(prog_timer);
     }
 
-    if (stop_flag) {
+    if (cf->stop_flag) {
       /* Well, the user decided to abort the filtering.  Just stop.
 
          XXX - go back to the previous filter?  Users probably just
@@ -2034,6 +1785,7 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
      was created. */
   if (progbar != NULL)
     destroy_progress_dlg(progbar);
+  g_timer_destroy(prog_timer);
 
   /* Unfreeze the packet list. */
   if (!add_to_packet_list)
@@ -2044,6 +1796,8 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
 
   packet_list_thaw();
 
+  cf_callback_invoke(cf_cb_file_rescan_finished, cf);
+
   if (selected_frame_num == -1) {
     /* The selected frame didn't pass the filter. */
     if (selected_frame == NULL) {
@@ -2109,7 +1863,7 @@ rescan_packets(capture_file *cf, const char *action, const char *action_item, gb
 
 
 /*
- * Scan trough all frame data and recalculate the ref time
+ * Scan through all frame data and recalculate the ref time
  * without rereading the file.
  * XXX - do we need a progres bar or is this fast enough?
  */
@@ -2201,7 +1955,8 @@ process_specified_records(capture_file *cf, packet_range_t *range,
     const char *string1, const char *string2, gboolean terminate_is_stop,
     gboolean (*callback)(capture_file *, frame_data *,
                          struct wtap_pkthdr *, const guint8 *, void *),
-    void *callback_args)
+    void *callback_args,
+    gboolean show_progress_bar)
 {
   guint32          framenum;
   frame_data      *fdata;
@@ -2209,30 +1964,24 @@ process_specified_records(capture_file *cf, packet_range_t *range,
   psp_return_t     ret     = PSP_FINISHED;
 
   progdlg_t       *progbar = NULL;
+  GTimer          *prog_timer = g_timer_new();
   int              progbar_count;
   float            progbar_val;
-  gboolean         progbar_stop_flag;
   GTimeVal         progbar_start_time;
   gchar            progbar_status_str[100];
-  int              progbar_nextstep;
-  int              progbar_quantum;
   range_process_e  process_this;
   struct wtap_pkthdr phdr;
 
   wtap_phdr_init(&phdr);
   ws_buffer_init(&buf, 1500);
 
-  /* Update the progress bar when it gets to this value. */
-  progbar_nextstep = 0;
-  /* When we reach the value that triggers a progress bar update,
-     bump that value by this amount. */
-  progbar_quantum = cf->count/N_PROGBAR_UPDATES;
+  g_timer_start(prog_timer);
   /* Count of packets at which we've looked. */
   progbar_count = 0;
   /* Progress so far. */
   progbar_val = 0.0f;
 
-  progbar_stop_flag = FALSE;
+  cf->stop_flag = FALSE;
   g_get_current_time(&progbar_start_time);
 
   if (range != NULL)
@@ -2248,35 +1997,34 @@ process_specified_records(capture_file *cf, packet_range_t *range,
        longer than the standard time to create it (otherwise, for a
        large file, we might take considerably longer than that standard
        time in order to get to the next progress bar step). */
-    if (progbar == NULL)
+    if (show_progress_bar && progbar == NULL)
       progbar = delayed_create_progress_dlg(cf->window, string1, string2,
                                             terminate_is_stop,
-                                            &progbar_stop_flag,
+                                            &cf->stop_flag,
                                             &progbar_start_time,
                                             progbar_val);
 
-    /* Update the progress bar, but do it only N_PROGBAR_UPDATES times;
-       when we update it, we have to run the GTK+ main loop to get it
-       to repaint what's pending, and doing so may involve an "ioctl()"
-       to see if there's any pending input from an X server, and doing
-       that for every packet can be costly, especially on a big file. */
-    if (progbar_count >= progbar_nextstep) {
+    /*
+     * Update the progress bar, but do it only after PROGBAR_UPDATE_INTERVAL
+     * has elapsed. Calling update_progress_dlg and packets_bar_update will
+     * likely trigger UI paint events, which might take a while depending on
+     * the platform and display. Reset our timer *after* painting.
+     */
+    if (progbar && g_timer_elapsed(prog_timer, NULL) > PROGBAR_UPDATE_INTERVAL) {
       /* let's not divide by zero. I should never be started
        * with count == 0, so let's assert that
        */
       g_assert(cf->count > 0);
       progbar_val = (gfloat) progbar_count / cf->count;
 
-      if (progbar != NULL) {
-        g_snprintf(progbar_status_str, sizeof(progbar_status_str),
-                   "%4u of %u packets", progbar_count, cf->count);
-        update_progress_dlg(progbar, progbar_val, progbar_status_str);
-      }
+      g_snprintf(progbar_status_str, sizeof(progbar_status_str),
+                  "%4u of %u packets", progbar_count, cf->count);
+      update_progress_dlg(progbar, progbar_val, progbar_status_str);
 
-      progbar_nextstep += progbar_quantum;
+      g_timer_start(prog_timer);
     }
 
-    if (progbar_stop_flag) {
+    if (cf->stop_flag) {
       /* Well, the user decided to abort the operation.  Just stop,
          and arrange to return PSP_STOPPED to our caller, so they know
          it was stopped explicitly. */
@@ -2316,6 +2064,7 @@ process_specified_records(capture_file *cf, packet_range_t *range,
      it was created. */
   if (progbar != NULL)
     destroy_progress_dlg(progbar);
+  g_timer_destroy(prog_timer);
 
   wtap_phdr_cleanup(&phdr);
   ws_buffer_free(&buf);
@@ -2356,6 +2105,8 @@ cf_retap_packets(capture_file *cf)
     return CF_READ_ABORTED;
   }
 
+  cf_callback_invoke(cf_cb_file_retap_started, cf);
+
   /* Do we have any tap listeners with filters? */
   filtering_tap_listeners = have_filtering_tap_listeners();
 
@@ -2381,10 +2132,12 @@ cf_retap_packets(capture_file *cf)
 
   ret = process_specified_records(cf, &range, "Recalculating statistics on",
                                   "all packets", TRUE, retap_packet,
-                                  &callback_args);
+                                  &callback_args, TRUE);
 
   epan_dissect_cleanup(&callback_args.edt);
 
+  cf_callback_invoke(cf_cb_file_retap_finished, cf);
+
   switch (ret) {
   case PSP_FINISHED:
     /* Completed successfully. */
@@ -2432,6 +2185,7 @@ print_packet(capture_file *cf, frame_data *fdata,
   int             cp_off;
   char            bookmark_name[9+10+1];  /* "__frameNNNNNNNNNN__\0" */
   char            bookmark_title[6+10+1]; /* "Frame NNNNNNNNNN__\0"  */
+  col_item_t*     col_item;
 
   /* Fill in the column information if we're printing the summary
      information. */
@@ -2469,8 +2223,9 @@ print_packet(capture_file *cf, frame_data *fdata,
     cp = &args->line_buf[0];
     line_len = 0;
     for (i = 0; i < args->num_visible_cols; i++) {
+      col_item = &cf->cinfo.columns[args->visible_cols[i]];
       /* Find the length of the string for this column. */
-      column_len = (int) strlen(cf->cinfo.col_data[args->visible_cols[i]]);
+      column_len = (int) strlen(col_item->col_data);
       if (args->col_widths[i] > column_len)
          column_len = args->col_widths[i];
 
@@ -2485,10 +2240,10 @@ print_packet(capture_file *cf, frame_data *fdata,
       }
 
       /* Right-justify the packet number column. */
-      if (cf->cinfo.col_fmt[args->visible_cols[i]] == COL_NUMBER)
-        g_snprintf(cp, column_len+1, "%*s", args->col_widths[i], cf->cinfo.col_data[args->visible_cols[i]]);
+      if (col_item->col_fmt == COL_NUMBER)
+        g_snprintf(cp, column_len+1, "%*s", args->col_widths[i], col_item->col_data);
       else
-        g_snprintf(cp, column_len+1, "%-*s", args->col_widths[i], cf->cinfo.col_data[args->visible_cols[i]]);
+        g_snprintf(cp, column_len+1, "%-*s", args->col_widths[i], col_item->col_data);
       cp += column_len;
       if (i != args->num_visible_cols - 1)
         *cp++ = ' ';
@@ -2566,7 +2321,8 @@ fail:
 }
 
 cf_print_status_t
-cf_print_packets(capture_file *cf, print_args_t *print_args)
+cf_print_packets(capture_file *cf, print_args_t *print_args,
+                 gboolean show_progress_bar)
 {
   print_callback_args_t callback_args;
   gint          data_width;
@@ -2640,14 +2396,14 @@ cf_print_packets(capture_file *cf, print_args_t *print_args)
       if (i == last_visible_col)
         callback_args.col_widths[visible_col_count] = 0;
       else {
-        callback_args.col_widths[visible_col_count] = (gint) strlen(cf->cinfo.col_title[i]);
+        callback_args.col_widths[visible_col_count] = (gint) strlen(cf->cinfo.columns[i].col_title);
         data_width = get_column_char_width(get_column_format(i));
         if (data_width > callback_args.col_widths[visible_col_count])
           callback_args.col_widths[visible_col_count] = data_width;
       }
 
       /* Find the length of the string for this column. */
-      column_len = (int) strlen(cf->cinfo.col_title[i]);
+      column_len = (int) strlen(cf->cinfo.columns[i].col_title);
       if (callback_args.col_widths[i] > column_len)
         column_len = callback_args.col_widths[visible_col_count];
 
@@ -2664,9 +2420,9 @@ cf_print_packets(capture_file *cf, print_args_t *print_args)
 
       /* Right-justify the packet number column. */
 /*      if (cf->cinfo.col_fmt[i] == COL_NUMBER)
-        g_snprintf(cp, column_len+1, "%*s", callback_args.col_widths[visible_col_count], cf->cinfo.col_title[i]);
+        g_snprintf(cp, column_len+1, "%*s", callback_args.col_widths[visible_col_count], cf->cinfo.columns[i].col_title);
       else*/
-      g_snprintf(cp, column_len+1, "%-*s", callback_args.col_widths[visible_col_count], cf->cinfo.col_title[i]);
+      g_snprintf(cp, column_len+1, "%-*s", callback_args.col_widths[visible_col_count], cf->cinfo.columns[i].col_title);
       cp += column_len;
       if (i != cf->cinfo.num_cols - 1)
         *cp++ = ' ';
@@ -2687,14 +2443,14 @@ cf_print_packets(capture_file *cf, print_args_t *print_args)
   proto_tree_needed =
       callback_args.print_args->print_dissections != print_dissections_none ||
       callback_args.print_args->print_hex ||
-      have_custom_cols(&cf->cinfo);
+      have_custom_cols(&cf->cinfo) || have_field_extractors();
   epan_dissect_init(&callback_args.edt, cf->epan, proto_tree_needed, proto_tree_needed);
 
   /* Iterate through the list of packets, printing the packets we were
      told to print. */
   ret = process_specified_records(cf, &print_args->range, "Printing",
                                   "selected packets", TRUE, print_packet,
-                                  &callback_args);
+                                  &callback_args, show_progress_bar);
   epan_dissect_cleanup(&callback_args.edt);
   g_free(callback_args.header_line_buf);
   g_free(callback_args.line_buf);
@@ -2741,6 +2497,7 @@ cf_print_packets(capture_file *cf, print_args_t *print_args)
 typedef struct {
   FILE *fh;
   epan_dissect_t edt;
+  print_args_t *print_args;
 } write_packet_callback_args_t;
 
 static gboolean
@@ -2754,7 +2511,7 @@ write_pdml_packet(capture_file *cf, frame_data *fdata,
   epan_dissect_run(&args->edt, cf->cd_t, phdr, frame_tvbuff_new(fdata, pd), fdata, NULL);
 
   /* Write out the information in that tree. */
-  write_pdml_proto_tree(&args->edt, args->fh);
+  write_pdml_proto_tree(NULL, NULL, &args->edt, args->fh);
 
   epan_dissect_reset(&args->edt);
 
@@ -2779,13 +2536,14 @@ cf_write_pdml_packets(capture_file *cf, print_args_t *print_args)
   }
 
   callback_args.fh = fh;
+  callback_args.print_args = print_args;
   epan_dissect_init(&callback_args.edt, cf->epan, TRUE, TRUE);
 
   /* Iterate through the list of packets, printing the packets we were
      told to print. */
   ret = process_specified_records(cf, &print_args->range, "Writing PDML",
                                   "selected packets", TRUE,
-                                  write_pdml_packet, &callback_args);
+                                  write_pdml_packet, &callback_args, TRUE);
 
   epan_dissect_cleanup(&callback_args.edt);
 
@@ -2857,17 +2615,18 @@ cf_write_psml_packets(capture_file *cf, print_args_t *print_args)
   }
 
   callback_args.fh = fh;
+  callback_args.print_args = print_args;
 
   /* Fill in the column information, only create the protocol tree
-     if having custom columns. */
-  proto_tree_needed = have_custom_cols(&cf->cinfo);
+     if having custom columns or field extractors. */
+  proto_tree_needed = have_custom_cols(&cf->cinfo) || have_field_extractors();
   epan_dissect_init(&callback_args.edt, cf->epan, proto_tree_needed, proto_tree_needed);
 
   /* Iterate through the list of packets, printing the packets we were
      told to print. */
   ret = process_specified_records(cf, &print_args->range, "Writing PSML",
                                   "selected packets", TRUE,
-                                  write_psml_packet, &callback_args);
+                                  write_psml_packet, &callback_args, TRUE);
 
   epan_dissect_cleanup(&callback_args.edt);
 
@@ -2938,16 +2697,17 @@ cf_write_csv_packets(capture_file *cf, print_args_t *print_args)
   }
 
   callback_args.fh = fh;
+  callback_args.print_args = print_args;
 
-  /* only create the protocol tree if having custom columns. */
-  proto_tree_needed = have_custom_cols(&cf->cinfo);
+  /* only create the protocol tree if having custom columns or field extractors. */
+  proto_tree_needed = have_custom_cols(&cf->cinfo) || have_field_extractors();
   epan_dissect_init(&callback_args.edt, cf->epan, proto_tree_needed, proto_tree_needed);
 
   /* Iterate through the list of packets, printing the packets we were
      told to print. */
   ret = process_specified_records(cf, &print_args->range, "Writing CSV",
                                   "selected packets", TRUE,
-                                  write_csv_packet, &callback_args);
+                                  write_csv_packet, &callback_args, TRUE);
 
   epan_dissect_cleanup(&callback_args.edt);
 
@@ -3005,14 +2765,15 @@ cf_write_carrays_packets(capture_file *cf, print_args_t *print_args)
   }
 
   callback_args.fh = fh;
+  callback_args.print_args = print_args;
   epan_dissect_init(&callback_args.edt, cf->epan, TRUE, TRUE);
 
   /* Iterate through the list of packets, printing the packets we were
      told to print. */
   ret = process_specified_records(cf, &print_args->range,
-                  "Writing C Arrays",
-                  "selected packets", TRUE,
-                                  carrays_write_packet, &callback_args);
+                                  "Writing C Arrays",
+                                  "selected packets", TRUE,
+                                  carrays_write_packet, &callback_args, TRUE);
 
   epan_dissect_cleanup(&callback_args.edt);
 
@@ -3033,6 +2794,81 @@ cf_write_carrays_packets(capture_file *cf, print_args_t *print_args)
   return CF_PRINT_OK;
 }
 
+static gboolean
+write_json_packet(capture_file *cf, frame_data *fdata,
+                  struct wtap_pkthdr *phdr, const guint8 *pd,
+          void *argsp)
+{
+  write_packet_callback_args_t *args = (write_packet_callback_args_t *)argsp;
+
+  /* Create the protocol tree, but don't fill in the column information. */
+  epan_dissect_run(&args->edt, cf->cd_t, phdr, frame_tvbuff_new(fdata, pd), fdata, NULL);
+
+  /* Write out the information in that tree. */
+  write_json_proto_tree(NULL, args->print_args, NULL, &args->edt, args->fh);
+
+  epan_dissect_reset(&args->edt);
+
+  return !ferror(args->fh);
+}
+
+cf_print_status_t
+cf_write_json_packets(capture_file *cf, print_args_t *print_args)
+{
+  write_packet_callback_args_t callback_args;
+  FILE         *fh;
+  psp_return_t  ret;
+
+  fh = ws_fopen(print_args->file, "w");
+  if (fh == NULL)
+    return CF_PRINT_OPEN_ERROR; /* attempt to open destination failed */
+
+  write_json_preamble(fh);
+  if (ferror(fh)) {
+    fclose(fh);
+    return CF_PRINT_WRITE_ERROR;
+  }
+
+  callback_args.fh = fh;
+  callback_args.print_args = print_args;
+  epan_dissect_init(&callback_args.edt, cf->epan, TRUE, TRUE);
+
+  /* Iterate through the list of packets, printing the packets we were
+     told to print. */
+  ret = process_specified_records(cf, &print_args->range, "Writing PDML",
+                                  "selected packets", TRUE,
+                                  write_json_packet, &callback_args, TRUE);
+
+  epan_dissect_cleanup(&callback_args.edt);
+
+  switch (ret) {
+
+  case PSP_FINISHED:
+    /* Completed successfully. */
+    break;
+
+  case PSP_STOPPED:
+    /* Well, the user decided to abort the printing. */
+    break;
+
+  case PSP_FAILED:
+    /* Error while printing. */
+    fclose(fh);
+    return CF_PRINT_WRITE_ERROR;
+  }
+
+  write_json_finale(fh);
+  if (ferror(fh)) {
+    fclose(fh);
+    return CF_PRINT_WRITE_ERROR;
+  }
+
+  /* XXX - check for an error */
+  fclose(fh);
+
+  return CF_PRINT_OK;
+}
+
 gboolean
 cf_find_packet_protocol_tree(capture_file *cf, const char *string,
                              search_direction dir)
@@ -3117,22 +2953,30 @@ match_subtree_text(proto_node *node, gpointer data)
     proto_item_fill_label(fi, label_str);
   }
 
-  /* Does that label match? */
-  label_len = strlen(label_ptr);
-  for (i = 0; i < label_len; i++) {
-    c_char = label_ptr[i];
-    if (cf->case_type)
-      c_char = g_ascii_toupper(c_char);
-    if (c_char == string[c_match]) {
-      c_match++;
-      if (c_match == string_len) {
-        /* No need to look further; we have a match */
-        mdata->frame_matched = TRUE;
-        mdata->finfo = fi;
-        return;
-      }
-    } else
-      c_match = 0;
+  if (cf->regex) {
+    if (g_regex_match(cf->regex, label_ptr, (GRegexMatchFlags) 0, NULL)) {
+      mdata->frame_matched = TRUE;
+      mdata->finfo = fi;
+      return;
+    }
+  } else {
+    /* Does that label match? */
+    label_len = strlen(label_ptr);
+    for (i = 0; i < label_len; i++) {
+      c_char = label_ptr[i];
+      if (cf->case_type)
+        c_char = g_ascii_toupper(c_char);
+      if (c_char == string[c_match]) {
+        c_match++;
+        if (c_match == string_len) {
+          /* No need to look further; we have a match */
+          mdata->frame_matched = TRUE;
+          mdata->finfo = fi;
+          return;
+        }
+      } else
+        c_match = 0;
+    }
   }
 
   /* Recurse into the subtree, if it exists */
@@ -3180,22 +3024,29 @@ match_summary_line(capture_file *cf, frame_data *fdata, void *criterion)
 
   /* Find the Info column */
   for (colx = 0; colx < cf->cinfo.num_cols; colx++) {
-    if (cf->cinfo.fmt_matx[colx][COL_INFO]) {
+    if (cf->cinfo.columns[colx].fmt_matx[COL_INFO]) {
       /* Found it.  See if we match. */
-      info_column = edt.pi.cinfo->col_data[colx];
+      info_column = edt.pi.cinfo->columns[colx].col_data;
       info_column_len = strlen(info_column);
-      for (i = 0; i < info_column_len; i++) {
-        c_char = info_column[i];
-        if (cf->case_type)
-          c_char = g_ascii_toupper(c_char);
-        if (c_char == string[c_match]) {
-          c_match++;
-          if (c_match == string_len) {
-            result = MR_MATCHED;
-            break;
-          }
-        } else
-          c_match = 0;
+      if (cf->regex) {
+        if (g_regex_match(cf->regex, info_column, (GRegexMatchFlags) 0, NULL)) {
+          result = MR_MATCHED;
+          break;
+        }
+      } else {
+        for (i = 0; i < info_column_len; i++) {
+          c_char = info_column[i];
+          if (cf->case_type)
+            c_char = g_ascii_toupper(c_char);
+          if (c_char == string[c_match]) {
+            c_match++;
+            if (c_match == string_len) {
+              result = MR_MATCHED;
+              break;
+            }
+          } else
+            c_match = 0;
+        }
       }
       break;
     }
@@ -3229,8 +3080,11 @@ cf_find_packet_data(capture_file *cf, const guint8 *string, size_t string_size,
   info.data = string;
   info.data_len = string_size;
 
-  /* String or hex search? */
-  if (cf->string) {
+  /* Regex, String or hex search? */
+  if (cf->regex) {
+    /* Regular Expression search */
+    return find_packet(cf, match_regex, NULL, dir);
+  } else if (cf->string) {
     /* String search - what type of string? */
     switch (cf->scs_type) {
 
@@ -3285,6 +3139,7 @@ match_narrow_and_wide(capture_file *cf, frame_data *fdata, void *criterion)
           result = MR_MATCHED;
           cf->search_pos = i; /* Save the position of the last character
                                  for highlighting the field. */
+          cf->search_len = (guint32)textlen;
           break;
         }
       }
@@ -3332,6 +3187,7 @@ match_narrow(capture_file *cf, frame_data *fdata, void *criterion)
         result = MR_MATCHED;
         cf->search_pos = i; /* Save the position of the last character
                                for highlighting the field. */
+        cf->search_len = (guint32)textlen;
         break;
       }
     }
@@ -3379,6 +3235,7 @@ match_wide(capture_file *cf, frame_data *fdata, void *criterion)
         result = MR_MATCHED;
         cf->search_pos = i; /* Save the position of the last character
                                for highlighting the field. */
+        cf->search_len = (guint32)textlen;
         break;
       }
       i += 1;
@@ -3422,6 +3279,7 @@ match_binary(capture_file *cf, frame_data *fdata, void *criterion)
         result = MR_MATCHED;
         cf->search_pos = i; /* Save the position of the last character
                                for highlighting the field. */
+        cf->search_len = (guint32)datalen;
         break;
       }
     }
@@ -3435,6 +3293,30 @@ match_binary(capture_file *cf, frame_data *fdata, void *criterion)
   return result;
 }
 
+static match_result
+match_regex(capture_file *cf, frame_data *fdata, void *criterion _U_)
+{
+    match_result  result = MR_NOTMATCHED;
+    GMatchInfo   *match_info = NULL;
+
+    /* Load the frame's data. */
+    if (!cf_read_record(cf, fdata)) {
+        /* Attempt to get the packet failed. */
+        return MR_ERROR;
+    }
+
+    if (g_regex_match_full(cf->regex, (const gchar *)ws_buffer_start_ptr(&cf->buf), fdata->cap_len,
+                           0, (GRegexMatchFlags) 0, &match_info, NULL))
+    {
+        gint start_pos = 0, end_pos = 0;
+        g_match_info_fetch_pos (match_info, 0, &start_pos, &end_pos);
+        cf->search_pos = end_pos - 1;
+        cf->search_len = end_pos - start_pos;
+        result = MR_MATCHED;
+    }
+    return result;
+}
+
 gboolean
 cf_find_packet_dfilter(capture_file *cf, dfilter_t *sfcode,
                        search_direction dir)
@@ -3523,14 +3405,12 @@ find_packet(capture_file *cf,
   frame_data  *fdata;
   frame_data  *new_fd = NULL;
   progdlg_t   *progbar = NULL;
-  gboolean     stop_flag;
+  GTimer      *prog_timer = g_timer_new();
   int          count;
   gboolean     found;
   float        progbar_val;
   GTimeVal     start_time;
   gchar        status_str[100];
-  int          progbar_nextstep;
-  int          progbar_quantum;
   const char  *title;
   match_result result;
 
@@ -3542,15 +3422,11 @@ find_packet(capture_file *cf,
     count = 0;
     framenum = start_fd->num;
 
-    /* Update the progress bar when it gets to this value. */
-    progbar_nextstep = 0;
-    /* When we reach the value that triggers a progress bar update,
-       bump that value by this amount. */
-    progbar_quantum = cf->count/N_PROGBAR_UPDATES;
+    g_timer_start(prog_timer);
     /* Progress so far. */
     progbar_val = 0.0f;
 
-    stop_flag = FALSE;
+    cf->stop_flag = FALSE;
     g_get_current_time(&start_time);
 
     title = cf->sfilter?cf->sfilter:"";
@@ -3562,14 +3438,15 @@ find_packet(capture_file *cf,
          time in order to get to the next progress bar step). */
       if (progbar == NULL)
          progbar = delayed_create_progress_dlg(cf->window, "Searching", title,
-           FALSE, &stop_flag, &start_time, progbar_val);
-
-      /* Update the progress bar, but do it only N_PROGBAR_UPDATES times;
-         when we update it, we have to run the GTK+ main loop to get it
-         to repaint what's pending, and doing so may involve an "ioctl()"
-         to see if there's any pending input from an X server, and doing
-         that for every packet can be costly, especially on a big file. */
-      if (count >= progbar_nextstep) {
+           FALSE, &cf->stop_flag, &start_time, progbar_val);
+
+      /*
+       * Update the progress bar, but do it only after PROGBAR_UPDATE_INTERVAL
+       * has elapsed. Calling update_progress_dlg and packets_bar_update will
+       * likely trigger UI paint events, which might take a while depending on
+       * the platform and display. Reset our timer *after* painting.
+       */
+      if (g_timer_elapsed(prog_timer, NULL) > PROGBAR_UPDATE_INTERVAL) {
         /* let's not divide by zero. I should never be started
          * with count == 0, so let's assert that
          */
@@ -3577,16 +3454,14 @@ find_packet(capture_file *cf,
 
         progbar_val = (gfloat) count / cf->count;
 
-        if (progbar != NULL) {
-          g_snprintf(status_str, sizeof(status_str),
-                     "%4u of %u packets", count, cf->count);
-          update_progress_dlg(progbar, progbar_val, status_str);
-        }
+        g_snprintf(status_str, sizeof(status_str),
+                    "%4u of %u packets", count, cf->count);
+        update_progress_dlg(progbar, progbar_val, status_str);
 
-        progbar_nextstep += progbar_quantum;
+        g_timer_start(prog_timer);
       }
 
-      if (stop_flag) {
+      if (cf->stop_flag) {
         /* Well, the user decided to abort the search.  Go back to the
            frame where we started. */
         new_fd = start_fd;
@@ -3664,6 +3539,7 @@ find_packet(capture_file *cf,
        was created. */
     if (progbar != NULL)
       destroy_progress_dlg(progbar);
+    g_timer_destroy(prog_timer);
   }
 
   if (new_fd != NULL) {
@@ -3672,6 +3548,7 @@ find_packet(capture_file *cf,
     found = packet_list_select_row_from_data(new_fd);
     cf->search_in_progress = FALSE;
     cf->search_pos = 0; /* Reset the position */
+    cf->search_len = 0; /* Reset length */
     if (!found) {
       /* We didn't find a row corresponding to this frame.
          This means that the frame isn't being displayed currently,
@@ -3691,6 +3568,12 @@ cf_goto_frame(capture_file *cf, guint fnumber)
 {
   frame_data *fdata;
 
+  if (cf == NULL || cf->frames == NULL) {
+    /* we don't have a loaded capture file - fix for bugs 11810 & 11989 */
+    statusbar_push_temporary_msg("There is no file loaded");
+    return FALSE;   /* we failed to go to that packet */
+  }
+
   fdata = frame_data_sequence_find(cf->frames, fnumber);
 
   if (fdata == NULL) {
@@ -3717,22 +3600,6 @@ cf_goto_frame(capture_file *cf, guint fnumber)
   return TRUE;  /* we got to that packet */
 }
 
-gboolean
-cf_goto_top_frame(void)
-{
-  /* Find and select */
-  packet_list_select_first_row();
-  return TRUE;  /* we got to that packet */
-}
-
-gboolean
-cf_goto_bottom_frame(void)
-{
-  /* Find and select */
-  packet_list_select_last_row();
-  return TRUE;  /* we got to that packet */
-}
-
 /*
  * Go to frame specified by currently selected protocol tree item.
  */
@@ -3765,38 +3632,6 @@ cf_select_packet(capture_file *cf, int row)
   /* Get the frame data struct pointer for this frame */
   fdata = packet_list_get_row_data(row);
 
-  if (fdata == NULL) {
-    /* XXX - if a GtkCList's selection mode is GTK_SELECTION_BROWSE, when
-       the first entry is added to it by "real_insert_row()", that row
-       is selected (see "real_insert_row()", in "ui/gtk/gtkclist.c", in both
-       our version and the vanilla GTK+ version).
-
-       This means that a "select-row" signal is emitted; this causes
-       "packet_list_select_cb()" to be called, which causes "cf_select_packet()"
-       to be called.
-
-       "cf_select_packet()" fetches, above, the data associated with the
-       row that was selected; however, as "gtk_clist_append()", which
-       called "real_insert_row()", hasn't yet returned, we haven't yet
-       associated any data with that row, so we get back a null pointer.
-
-       We can't assume that there's only one frame in the frame list,
-       either, as we may be filtering the display.
-
-       We therefore assume that, if "row" is 0, i.e. the first row
-       is being selected, and "cf->first_displayed" equals
-       "cf->last_displayed", i.e. there's only one frame being
-       displayed, that frame is the frame we want.
-
-       This means we have to set "cf->first_displayed" and
-       "cf->last_displayed" before adding the row to the
-       GtkCList; see the comment in "add_packet_to_packet_list()". */
-
-       if (row == 0 && cf->first_displayed == cf->last_displayed)
-         fdata = frame_data_sequence_find(cf->frames, cf->first_displayed);
-  }
-
-  /* If fdata _still_ isn't set simply give up. */
   if (fdata == NULL) {
     return;
   }
@@ -3918,41 +3753,45 @@ cf_unignore_frame(capture_file *cf, frame_data *frame)
 const gchar *
 cf_read_shb_comment(capture_file *cf)
 {
-  wtapng_section_t *shb_inf;
-  const gchar      *temp_str;
-
-  /* Get info from SHB */
-  shb_inf = wtap_file_get_shb_info(cf->wth);
-  if (shb_inf == NULL)
-        return NULL;
-  temp_str = shb_inf->opt_comment;
-  g_free(shb_inf);
-
-  return temp_str;
-
+  wtap_block_t shb_inf;
+  char *shb_comment;
+
+  /* Get the SHB. */
+  /* XXX - support multiple SHBs */
+  shb_inf = wtap_file_get_shb(cf->wth);
+
+  /* Get the first comment from the SHB. */
+  /* XXX - support multiple comments */
+  if (wtap_block_get_nth_string_option_value(shb_inf, OPT_COMMENT, 0, &shb_comment) != WTAP_OPTTYPE_SUCCESS)
+    return NULL;
+  return shb_comment;
 }
 
 void
 cf_update_capture_comment(capture_file *cf, gchar *comment)
 {
-  wtapng_section_t *shb_inf;
-
-  /* Get info from SHB */
-  shb_inf = wtap_file_get_shb_info(cf->wth);
-
-  /* See if the comment has changed or not */
-  if (shb_inf && shb_inf->opt_comment) {
-    if (strcmp(shb_inf->opt_comment, comment) == 0) {
+  wtap_block_t shb_inf;
+  gchar *shb_comment;
+
+  /* Get the SHB. */
+  /* XXX - support multiple SHBs */
+  shb_inf = wtap_file_get_shb(cf->wth);
+
+  /* Get the first comment from the SHB. */
+  /* XXX - support multiple comments */
+  if (wtap_block_get_nth_string_option_value(shb_inf, OPT_COMMENT, 0, &shb_comment) != WTAP_OPTTYPE_SUCCESS) {
+    /* There's no comment - add one. */
+    wtap_block_add_string_option(shb_inf, OPT_COMMENT, comment, strlen(comment));
+  } else {
+    /* See if the comment has changed or not */
+    if (strcmp(shb_comment, comment) == 0) {
       g_free(comment);
-      g_free(shb_inf);
       return;
     }
-  }
-
-  g_free(shb_inf);
 
-  /* The comment has changed, let's update it */
-  wtap_write_shb_comment(cf->wth, comment);
+    /* The comment has changed, let's update it */
+    wtap_block_set_nth_string_option_value(shb_inf, OPT_COMMENT, 0, comment, strlen(comment));
+  }
   /* Mark the file as having unsaved changes */
   cf->unsaved_changes = TRUE;
 }
@@ -4054,6 +3893,24 @@ cf_comment_types(capture_file *cf)
   return comment_types;
 }
 
+/*
+ * Add a resolved address to this file's list of resolved addresses.
+ */
+gboolean
+cf_add_ip_name_from_string(capture_file *cf, const char *addr, const char *name)
+{
+  /*
+   * XXX - support multiple resolved address lists, and add to the one
+   * attached to this file?
+   */
+  if (!add_ip_name_from_string(addr, name))
+    return FALSE;
+
+  /* OK, we have unsaved changes. */
+  cf->unsaved_changes = TRUE;
+  return TRUE;
+}
+
 #ifdef WANT_PACKET_EDITOR
 static gint
 g_direct_compare_func(gconstpointer a, gconstpointer b, gpointer user_data _U_)
@@ -4085,7 +3942,7 @@ cf_set_frame_edited(capture_file *cf, frame_data *fd,
   modified_frame_data *mfd = (modified_frame_data *)g_malloc(sizeof(modified_frame_data));
 
   mfd->phdr = *phdr;
-  mfd->pd = pd;
+  mfd->pd = (char *)pd;
 
   if (cf->edited_frames == NULL)
     cf->edited_frames = g_tree_new_full(g_direct_compare_func, NULL, NULL,
@@ -4149,11 +4006,10 @@ save_record(capture_file *cf, frame_data *fdata,
     hdr.presence_flags |= WTAP_HAS_INTERFACE_ID;
   if (phdr->presence_flags & WTAP_HAS_PACK_FLAGS)
     hdr.presence_flags |= WTAP_HAS_PACK_FLAGS;
-  hdr.ts.secs      = fdata->abs_ts.secs;
-  hdr.ts.nsecs     = fdata->abs_ts.nsecs;
+  hdr.ts           = phdr->ts;
   hdr.caplen       = phdr->caplen;
   hdr.len          = phdr->len;
-  hdr.pkt_encap    = fdata->lnk_t;
+  hdr.pkt_encap    = phdr->pkt_encap;
   /* pcapng */
   hdr.interface_id = phdr->interface_id;   /* identifier of the interface. */
   /* options */
@@ -4339,7 +4195,7 @@ cf_has_unsaved_data(capture_file *cf)
    * If this is a temporary file, or a file with unsaved changes, it
    * has unsaved data.
    */
-  return cf->is_tempfile || cf->unsaved_changes;
+  return (cf->is_tempfile && cf->count>0) || cf->unsaved_changes;
 }
 
 /*
@@ -4352,21 +4208,15 @@ rescan_file(capture_file *cf, const char *fname, gboolean is_tempfile, int *err)
   gchar               *err_info;
   gchar               *name_ptr;
   gint64               data_offset;
-  gint64               file_pos;
   progdlg_t           *progbar        = NULL;
-  gboolean             stop_flag;
+  GTimer              *prog_timer = g_timer_new();
   gint64               size;
   float                progbar_val;
   GTimeVal             start_time;
   gchar                status_str[100];
-  gint64               progbar_nextstep;
-  gint64               progbar_quantum;
   guint32              framenum;
   frame_data          *fdata;
   int                  count          = 0;
-#ifdef HAVE_LIBPCAP
-  int                  displayed_once = 0;
-#endif
 
   /* Close the old handle. */
   wtap_close(cf->wth);
@@ -4420,18 +4270,9 @@ rescan_file(capture_file *cf, const char *fname, gboolean is_tempfile, int *err)
   /* Find the size of the file. */
   size = wtap_file_size(cf->wth, NULL);
 
-  /* Update the progress bar when it gets to this value. */
-  progbar_nextstep = 0;
-  /* When we reach the value that triggers a progress bar update,
-     bump that value by this amount. */
-  if (size >= 0) {
-    progbar_quantum = size/N_PROGBAR_UPDATES;
-    if (progbar_quantum < MIN_QUANTUM)
-      progbar_quantum = MIN_QUANTUM;
-  }else
-    progbar_quantum = 0;
-
-  stop_flag = FALSE;
+  g_timer_start(prog_timer);
+
+  cf->stop_flag = FALSE;
   g_get_current_time(&start_time);
 
   framenum = 0;
@@ -4442,41 +4283,31 @@ rescan_file(capture_file *cf, const char *fname, gboolean is_tempfile, int *err)
     fdata->file_off = data_offset;
     if (size >= 0) {
       count++;
-      file_pos = wtap_read_so_far(cf->wth);
+      cf->f_datalen = wtap_read_so_far(cf->wth);
 
-      /* Create the progress bar if necessary.
-       * Check whether it should be created or not every MIN_NUMBER_OF_PACKET
-       */
-      if ((progbar == NULL) && !(count % MIN_NUMBER_OF_PACKET)) {
-        progbar_val = calc_progbar_val(cf, size, file_pos, status_str, sizeof(status_str));
+      /* Create the progress bar if necessary. */
+      if (progress_is_slow(progbar, prog_timer, size, cf->f_datalen)) {
+        progbar_val = calc_progbar_val(cf, size, cf->f_datalen, status_str, sizeof(status_str));
         progbar = delayed_create_progress_dlg(cf->window, "Rescanning", name_ptr,
-                                              TRUE, &stop_flag, &start_time, progbar_val);
+                                              TRUE, &cf->stop_flag, &start_time, progbar_val);
       }
 
-      /* Update the progress bar, but do it only N_PROGBAR_UPDATES times;
-         when we update it, we have to run the GTK+ main loop to get it
-         to repaint what's pending, and doing so may involve an "ioctl()"
-         to see if there's any pending input from an X server, and doing
-         that for every packet can be costly, especially on a big file. */
-      if (file_pos >= progbar_nextstep) {
-        if (progbar != NULL) {
-          progbar_val = calc_progbar_val(cf, size, file_pos, status_str, sizeof(status_str));
-          /* update the packet bar content on the first run or frequently on very large files */
-#ifdef HAVE_LIBPCAP
-          if (progbar_quantum > 500000 || displayed_once == 0) {
-            if ((auto_scroll_live || displayed_once == 0 || cf->displayed_count < 1000) && cf->count != 0) {
-              displayed_once = 1;
-              packets_bar_update();
-            }
-          }
-#endif /* HAVE_LIBPCAP */
-          update_progress_dlg(progbar, progbar_val, status_str);
-        }
-        progbar_nextstep += progbar_quantum;
+      /*
+       * Update the progress bar, but do it only after PROGBAR_UPDATE_INTERVAL
+       * has elapsed. Calling update_progress_dlg and packets_bar_update will
+       * likely trigger UI paint events, which might take a while depending on
+       * the platform and display. Reset our timer *after* painting.
+       */
+      if (progbar && g_timer_elapsed(prog_timer, NULL) > PROGBAR_UPDATE_INTERVAL) {
+        progbar_val = calc_progbar_val(cf, size, cf->f_datalen, status_str, sizeof(status_str));
+        /* update the packet bar content on the first run or frequently on very large files */
+        update_progress_dlg(progbar, progbar_val, status_str);
+        packets_bar_update();
+        g_timer_start(prog_timer);
       }
     }
 
-    if (stop_flag) {
+    if (cf->stop_flag) {
       /* Well, the user decided to abort the rescan.  Sadly, as this
          isn't a reread, recovering is difficult, so we'll just
          close the current capture. */
@@ -4498,6 +4329,7 @@ rescan_file(capture_file *cf, const char *fname, gboolean is_tempfile, int *err)
   /* We're done reading the file; destroy the progress bar if it was created. */
   if (progbar != NULL)
     destroy_progress_dlg(progbar);
+  g_timer_destroy(prog_timer);
 
   /* We're done reading sequentially through the file. */
   cf->state = FILE_READ_DONE;
@@ -4516,7 +4348,7 @@ rescan_file(capture_file *cf, const char *fname, gboolean is_tempfile, int *err)
 
   cf_callback_invoke(cf_cb_file_rescan_finished, cf);
 
-  if (stop_flag) {
+  if (cf->stop_flag) {
     /* Our caller will give up at this point. */
     return CF_READ_ABORTED;
   }
@@ -4674,12 +4506,15 @@ cf_save_records(capture_file *cf, const char *fname, guint save_format,
        or moving the capture file, we have to do it by writing the packets
        out in Wiretap. */
 
-    wtapng_section_t *shb_hdr = NULL;
+    GArray                      *shb_hdrs = NULL;
     wtapng_iface_descriptions_t *idb_inf = NULL;
+    GArray                      *nrb_hdrs = NULL;
     int encap;
 
-    shb_hdr = wtap_file_get_shb_info(cf->wth);
+    /* XXX: what free's this shb_hdr? */
+    shb_hdrs = wtap_file_get_shb_for_new_file(cf->wth);
     idb_inf = wtap_file_get_idb_info(cf->wth);
+    nrb_hdrs = wtap_file_get_nrb_for_new_file(cf->wth);
 
     /* Determine what file encapsulation type we should use. */
     encap = wtap_dump_file_encap_type(cf->linktypes);
@@ -4694,10 +4529,10 @@ cf_save_records(capture_file *cf, const char *fname, guint save_format,
          from which we're reading the packets that we're writing!) */
       fname_new = g_strdup_printf("%s~", fname);
       pdh = wtap_dump_open_ng(fname_new, save_format, encap, cf->snap,
-                              compressed, shb_hdr, idb_inf, &err);
+                              compressed, shb_hdrs, idb_inf, nrb_hdrs, &err);
     } else {
       pdh = wtap_dump_open_ng(fname, save_format, encap, cf->snap,
-                              compressed, shb_hdr, idb_inf, &err);
+                              compressed, shb_hdrs, idb_inf, nrb_hdrs, &err);
     }
     g_free(idb_inf);
     idb_inf = NULL;
@@ -4715,7 +4550,7 @@ cf_save_records(capture_file *cf, const char *fname, guint save_format,
     callback_args.fname = fname;
     callback_args.file_type = save_format;
     switch (process_specified_records(cf, NULL, "Saving", "packets",
-                                      TRUE, save_record, &callback_args)) {
+                                      TRUE, save_record, &callback_args, TRUE)) {
 
     case PSP_FINISHED:
       /* Completed successfully. */
@@ -4894,8 +4729,9 @@ cf_export_specified_packets(capture_file *cf, const char *fname,
   int                          err;
   wtap_dumper                 *pdh;
   save_callback_args_t         callback_args;
-  wtapng_section_t            *shb_hdr;
-  wtapng_iface_descriptions_t *idb_inf;
+  GArray                      *shb_hdrs = NULL;
+  wtapng_iface_descriptions_t *idb_inf = NULL;
+  GArray                      *nrb_hdrs = NULL;
   int                          encap;
 
   cf_callback_invoke(cf_cb_file_export_specified_packets_started, (gpointer)fname);
@@ -4907,8 +4743,10 @@ cf_export_specified_packets(capture_file *cf, const char *fname,
      written, don't special-case the operation - read each packet
      and then write it out if it's one of the specified ones. */
 
-  shb_hdr = wtap_file_get_shb_info(cf->wth);
+  /* XXX: what free's this shb_hdr? */
+  shb_hdrs = wtap_file_get_shb_for_new_file(cf->wth);
   idb_inf = wtap_file_get_idb_info(cf->wth);
+  nrb_hdrs = wtap_file_get_nrb_for_new_file(cf->wth);
 
   /* Determine what file encapsulation type we should use. */
   encap = wtap_dump_file_encap_type(cf->linktypes);
@@ -4923,10 +4761,10 @@ cf_export_specified_packets(capture_file *cf, const char *fname,
        from which we're reading the packets that we're writing!) */
     fname_new = g_strdup_printf("%s~", fname);
     pdh = wtap_dump_open_ng(fname_new, save_format, encap, cf->snap,
-                            compressed, shb_hdr, idb_inf, &err);
+                            compressed, shb_hdrs, idb_inf, nrb_hdrs, &err);
   } else {
     pdh = wtap_dump_open_ng(fname, save_format, encap, cf->snap,
-                            compressed, shb_hdr, idb_inf, &err);
+                            compressed, shb_hdrs, idb_inf, nrb_hdrs, &err);
   }
   g_free(idb_inf);
   idb_inf = NULL;
@@ -4950,7 +4788,7 @@ cf_export_specified_packets(capture_file *cf, const char *fname,
   callback_args.fname = fname;
   callback_args.file_type = save_format;
   switch (process_specified_records(cf, range, "Writing", "specified records",
-                                    TRUE, save_record, &callback_args)) {
+                                    TRUE, save_record, &callback_args, TRUE)) {
 
   case PSP_FINISHED:
     /* Completed successfully. */
@@ -5046,7 +4884,7 @@ cf_open_failure_alert_box(const char *filename, int err, gchar *err_info,
     case WTAP_ERR_UNSUPPORTED:
       /* Seen only when opening a capture file for reading. */
       simple_error_message_box(
-            "The file \"%s\" contains record data that Wireshark doesn't support.\n",
+            "The file \"%s\" contains record data that Wireshark doesn't support.\n"
             "(%s)",
             display_basename,
             err_info != NULL ? err_info : "no information supplied");