Zbee ZCL se: fix typo found by conflict hf
[metze/wireshark/wip.git] / epan / print.c
index e99ac08092ef680de3061628934297c0030395dc..3a66064e8853fee7cc002ee74542d9b7eff84bbd 100644 (file)
 #include <epan/epan_dissect.h>
 #include <epan/to_str.h>
 #include <epan/expert.h>
-#include <epan/packet-range.h>
+#include <epan/column-info.h>
+#include <epan/color_filters.h>
+#include <epan/prefs.h>
 #include <epan/print.h>
 #include <epan/charsets.h>
 #include <wsutil/filesystem.h>
-#include <wsutil/ws_version_info.h>
+#include <ws_version_info.h>
+#include <wsutil/utf8_entities.h>
 #include <ftypes/ftypes-int.h>
 
 #define PDML_VERSION "0"
@@ -50,7 +53,6 @@ typedef struct {
     print_dissections_e  print_dissections;
     gboolean             print_hex_for_data;
     packet_char_enc      encoding;
-    epan_dissect_t      *edt;
     GHashTable          *output_only_tables; /* output only these protocols */
 } print_data;
 
@@ -58,61 +60,93 @@ typedef struct {
     int             level;
     FILE           *fh;
     GSList         *src_list;
-    epan_dissect_t *edt;
+    gchar         **filter;
+    pf_flags        filter_flags;
 } write_pdml_data;
 
+typedef struct {
+    int             level;
+    FILE           *fh;
+    GSList         *src_list;
+    gchar         **filter;
+    pf_flags        filter_flags;
+    gboolean        print_hex;
+    gboolean        print_text;
+    proto_node_children_grouper_func node_children_grouper;
+} write_json_data;
+
 typedef struct {
     output_fields_t *fields;
     epan_dissect_t  *edt;
 } write_field_data_t;
 
 struct _output_fields {
-    gboolean     print_header;
-    gchar        separator;
-    gchar        occurrence;
-    gchar        aggregator;
-    GPtrArray   *fields;
-    GHashTable  *field_indicies;
-    GPtrArray  **field_values;
-    gchar        quote;
-    gboolean     includes_col_fields;
+    gboolean      print_bom;
+    gboolean      print_header;
+    gchar         separator;
+    gchar         occurrence;
+    gchar         aggregator;
+    GPtrArray    *fields;
+    GHashTable   *field_indicies;
+    GPtrArray   **field_values;
+    gchar         quote;
+    gboolean      includes_col_fields;
 };
 
 static gchar *get_field_hex_value(GSList *src_list, field_info *fi);
 static void proto_tree_print_node(proto_node *node, gpointer data);
 static void proto_tree_write_node_pdml(proto_node *node, gpointer data);
+static void proto_tree_write_node_ek(proto_node *node, gpointer data);
 static const guint8 *get_field_data(GSList *src_list, field_info *fi);
 static void pdml_write_field_hex_value(write_pdml_data *pdata, field_info *fi);
+static void json_write_field_hex_value(write_json_data *pdata, field_info *fi);
 static gboolean print_hex_data_buffer(print_stream_t *stream, const guchar *cp,
                                       guint length, packet_char_enc encoding);
+static void write_specified_fields(fields_format format,
+                                   output_fields_t *fields,
+                                   epan_dissect_t *edt, column_info *cinfo,
+                                   FILE *fh);
 static void print_escaped_xml(FILE *fh, const char *unescaped_string);
-
-static void print_pdml_geninfo(proto_tree *tree, FILE *fh);
+static void print_escaped_json(FILE *fh, const char *unescaped_string);
+static void print_escaped_ek(FILE *fh, const char *unescaped_string);
+
+typedef void (*proto_node_value_writer)(proto_node *, write_json_data *);
+static void write_json_proto_node_list(GSList *proto_node_list_head, write_json_data *data);
+static void write_json_proto_node(GSList *node_values_head,
+                                  const char *suffix,
+                                  proto_node_value_writer value_writer,
+                                  write_json_data *data);
+static void write_json_proto_node_value_list(GSList *node_values_head,
+                                             proto_node_value_writer value_writer,
+                                             write_json_data *data);
+static void write_json_proto_node_filtered(proto_node *node, write_json_data *data);
+static void write_json_proto_node_hex_dump(proto_node *node, write_json_data *data);
+static void write_json_proto_node_children(proto_node *node, write_json_data *data);
+static void write_json_proto_node_value(proto_node *node, write_json_data *data);
+static void write_json_proto_node_no_value(proto_node *node, write_json_data *data);
+static const char *proto_node_to_json_key(proto_node *node);
+
+static void print_pdml_geninfo(epan_dissect_t *edt, FILE *fh);
 
 static void proto_tree_get_node_field_values(proto_node *node, gpointer data);
 
+static gboolean json_is_first;
+
 /* Cache the protocols and field handles that the print functionality needs
    This helps break explicit dependency on the dissectors. */
 static int proto_data = -1;
 static int proto_frame = -1;
-static int hf_frame_arrival_time = -1;
-static int hf_frame_number = -1;
-static int hf_frame_len = -1;
-static int hf_frame_capture_len = -1;
 
 void print_cache_field_handles(void)
 {
     proto_data = proto_get_id_by_short_name("Data");
     proto_frame = proto_get_id_by_short_name("Frame");
-    hf_frame_arrival_time = proto_registrar_get_id_byname("frame.time");
-    hf_frame_number = proto_registrar_get_id_byname("frame.number");
-    hf_frame_len = proto_registrar_get_id_byname("frame.len");
-    hf_frame_capture_len = proto_registrar_get_id_byname("frame.cap_len");
 }
 
 gboolean
-proto_tree_print(print_args_t *print_args, epan_dissect_t *edt,
-                 GHashTable *output_only_tables, print_stream_t *stream)
+proto_tree_print(print_dissections_e print_dissections, gboolean print_hex,
+                 epan_dissect_t *edt, GHashTable *output_only_tables,
+                 print_stream_t *stream)
 {
     print_data data;
 
@@ -121,12 +155,11 @@ proto_tree_print(print_args_t *print_args, epan_dissect_t *edt,
     data.stream             = stream;
     data.success            = TRUE;
     data.src_list           = edt->pi.data_src;
-    data.encoding           = edt->pi.fd->flags.encoding;
-    data.print_dissections  = print_args->print_dissections;
+    data.encoding           = (packet_char_enc)edt->pi.fd->flags.encoding;
+    data.print_dissections  = print_dissections;
     /* If we're printing the entire packet in hex, don't
        print uninterpreted data fields in hex as well. */
-    data.print_hex_for_data = !print_args->print_hex;
-    data.edt                = edt;
+    data.print_hex_for_data = !print_hex;
     data.output_only_tables = output_only_tables;
 
     proto_tree_children_foreach(edt->tree, proto_tree_print_node, &data);
@@ -147,7 +180,7 @@ proto_tree_print_node(proto_node *node, gpointer data)
     g_assert(fi);
 
     /* Don't print invisible entries. */
-    if (PROTO_ITEM_IS_HIDDEN(node))
+    if (PROTO_ITEM_IS_HIDDEN(node) && (prefs.display_hidden_proto_items == FALSE))
         return;
 
     /* Give up if we've already gotten an error. */
@@ -229,39 +262,157 @@ void
 write_pdml_preamble(FILE *fh, const gchar *filename)
 {
     time_t t = time(NULL);
-    char *ts = asctime(localtime(&t));
-
-    ts[strlen(ts)-1] = 0; /* overwrite \n */
+    struct tm * timeinfo;
+    char *fmt_ts;
+    const char *ts;
 
-    fputs("<?xml version=\"1.0\"?>\n", fh);
-    fputs("<?xml-stylesheet type=\"text/xsl\" href=\"" PDML2HTML_XSL "\"?>\n", fh);
+    /* Create the output */
+    timeinfo = localtime(&t);
+    if (timeinfo != NULL) {
+        fmt_ts = asctime(timeinfo);
+        fmt_ts[strlen(fmt_ts)-1] = 0; /* overwrite \n */
+        ts = fmt_ts;
+    } else
+        ts = "Not representable";
+
+    fprintf(fh, "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
+    fprintf(fh, "<?xml-stylesheet type=\"text/xsl\" href=\"" PDML2HTML_XSL "\"?>\n");
     fprintf(fh, "<!-- You can find " PDML2HTML_XSL " in %s or at https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=" PDML2HTML_XSL ". -->\n", get_datafile_dir());
-    fputs("<pdml version=\"" PDML_VERSION "\" ", fh);
-    fprintf(fh, "creator=\"%s/%s\" time=\"%s\" capture_file=\"%s\">\n", PACKAGE, VERSION, ts, filename ? filename : "");
+    fprintf(fh, "<pdml version=\"" PDML_VERSION "\" creator=\"%s/%s\" time=\"%s\" capture_file=\"", PACKAGE, VERSION, ts);
+    if (filename) {
+        /* \todo filename should be converted to UTF-8. */
+        print_escaped_xml(fh, filename);
+    }
+    fprintf(fh, "\">\n");
+}
+
+/* Check if the str match the protocolfilter. json_filter is space
+   delimited string and str need to exact-match to one of the value. */
+static gboolean check_protocolfilter(gchar **protocolfilter, const char *str)
+{
+    gboolean res = FALSE;
+    gchar **ptr;
+
+    if (str == NULL || protocolfilter == NULL) {
+        return FALSE;
+    }
+
+    for (ptr = protocolfilter; *ptr; ptr++) {
+        if (strcmp(*ptr, str) == 0) {
+            res = TRUE;
+            break;
+        }
+    }
+
+    return res;
 }
 
 void
-write_pdml_proto_tree(epan_dissect_t *edt, FILE *fh)
+write_pdml_proto_tree(output_fields_t* fields, gchar **protocolfilter, pf_flags protocolfilter_flags, epan_dissect_t *edt, FILE *fh, gboolean use_color)
 {
     write_pdml_data data;
+    const color_filter_t *cfp = edt->pi.fd->color_filter;
 
-    /* Create the output */
-    data.level    = 0;
-    data.fh       = fh;
-    data.src_list = edt->pi.data_src;
-    data.edt      = edt;
+    g_assert(edt);
+    g_assert(fh);
 
-    fprintf(fh, "<packet>\n");
+    /* Create the output */
+    if (use_color && (cfp != NULL)) {
+        fprintf(fh, "<packet foreground='#%02x%02x%02x' background='#%02x%02x%02x'>\n",
+            cfp->fg_color.red, cfp->fg_color.green, cfp->fg_color.blue,
+            cfp->bg_color.red, cfp->bg_color.green, cfp->bg_color.blue);
+    }
+    else {
+        fprintf(fh, "<packet>\n");
+    }
 
     /* Print a "geninfo" protocol as required by PDML */
-    print_pdml_geninfo(edt->tree, fh);
-
-    proto_tree_children_foreach(edt->tree, proto_tree_write_node_pdml,
-                                &data);
+    print_pdml_geninfo(edt, fh);
+
+    if (fields == NULL || fields->fields == NULL) {
+        /* Write out all fields */
+        data.level    = 0;
+        data.fh       = fh;
+        data.src_list = edt->pi.data_src;
+        data.filter   = protocolfilter;
+        data.filter_flags   = protocolfilter_flags;
+
+        proto_tree_children_foreach(edt->tree, proto_tree_write_node_pdml,
+                                    &data);
+    } else {
+        /* Write out specified fields */
+        write_specified_fields(FORMAT_XML, fields, edt, NULL, fh);
+    }
 
     fprintf(fh, "</packet>\n\n");
 }
 
+void
+write_ek_proto_tree(output_fields_t* fields,
+                    gboolean print_hex, gchar **protocolfilter,
+                    pf_flags protocolfilter_flags, epan_dissect_t *edt,
+                    FILE *fh)
+{
+    write_json_data data;
+    char ts[30];
+    time_t t = time(NULL);
+    struct tm  *timeinfo;
+
+    g_assert(edt);
+    g_assert(fh);
+
+    /* Create the output */
+    timeinfo = localtime(&t);
+    if (timeinfo != NULL)
+        strftime(ts, sizeof ts, "%Y-%m-%d", timeinfo);
+    else
+        g_strlcpy(ts, "XXXX-XX-XX", sizeof ts); /* XXX - better way of saying "Not representable"? */
+
+    fprintf(fh, "{\"index\" : {\"_index\": \"packets-%s\", \"_type\": \"pcap_file\", \"_score\": null}}\n", ts);
+    /* Timestamp added for time indexing in Elasticsearch */
+    fprintf(fh, "{\"timestamp\" : \"%" G_GUINT64_FORMAT "%03d\", \"layers\" : {", (guint64)edt->pi.abs_ts.secs, edt->pi.abs_ts.nsecs/1000000);
+
+    if (fields == NULL || fields->fields == NULL) {
+        /* Write out all fields */
+        data.level    = 0;
+        data.fh       = fh;
+        data.src_list = edt->pi.data_src;
+        data.filter   = protocolfilter;
+        data.filter_flags = protocolfilter_flags;
+        data.print_hex = print_hex;
+
+        proto_tree_children_foreach(edt->tree, proto_tree_write_node_ek,
+                                    &data);
+    } else {
+        /* Write out specified fields */
+        write_specified_fields(FORMAT_EK, fields, edt, NULL, fh);
+    }
+
+    fputs("}}\n", fh);
+}
+
+void
+write_fields_proto_tree(output_fields_t* fields, epan_dissect_t *edt, column_info *cinfo, FILE *fh)
+{
+    g_assert(edt);
+    g_assert(fh);
+
+    /* Create the output */
+    write_specified_fields(FORMAT_CSV, fields, edt, cinfo, fh);
+}
+
+/* Indent to the correct level */
+static void print_indent(int level, FILE *fh)
+{
+    int i;
+    if (fh == NULL) {
+        return;
+    }
+    for (i = 0; i < level; i++) {
+        fputs("  ", fh);
+    }
+}
+
 /* Write out a tree's data, and any child nodes, as PDML */
 static void
 proto_tree_write_node_pdml(proto_node *node, gpointer data)
@@ -271,7 +422,6 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
     const gchar     *label_ptr;
     gchar            label_str[ITEM_LABEL_LENGTH];
     char            *dfilter_string;
-    int              i;
     gboolean         wrap_in_fake_protocol;
 
     /* dissection with an invisible proto tree? */
@@ -284,20 +434,13 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
           (fi->hfinfo->id == proto_data)) &&
          (pdata->level == 0));
 
-    /* Indent to the correct level */
-    for (i = -1; i < pdata->level; i++) {
-        fputs("  ", pdata->fh);
-    }
+    print_indent(pdata->level + 1, pdata->fh);
 
     if (wrap_in_fake_protocol) {
         /* Open fake protocol wrapper */
         fputs("<proto name=\"fake-field-wrapper\">\n", pdata->fh);
 
-        /* Indent to increased level before writing out field */
-        pdata->level++;
-        for (i = -1; i < pdata->level; i++) {
-            fputs("  ", pdata->fh);
-        }
+        print_indent(pdata->level + 1, pdata->fh);
     }
 
     /* Text label. It's printed as a field with no name. */
@@ -338,7 +481,6 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
     /* Uninterpreted data, i.e., the "Data" protocol, is
      * printed as a field instead of a protocol. */
     else if (fi->hfinfo->id == proto_data) {
-
         /* Write out field with data */
         fputs("<field name=\"data\" value=\"", pdata->fh);
         pdml_write_field_hex_value(pdata, fi);
@@ -383,7 +525,7 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
             print_escaped_xml(pdata->fh, label_ptr);
         }
 
-        if (PROTO_ITEM_IS_HIDDEN(node))
+        if (PROTO_ITEM_IS_HIDDEN(node) && (prefs.display_hidden_proto_items == FALSE))
             fprintf(pdata->fh, "\" hide=\"yes");
 
         fprintf(pdata->fh, "\" size=\"%d", fi->length);
@@ -403,13 +545,13 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
             fputs("\" show=\"\" value=\"",  pdata->fh);
             break;
         default:
-            dfilter_string = fvalue_to_string_repr(&fi->value, FTREPR_DISPLAY, fi->hfinfo->display, NULL);
+            dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
             if (dfilter_string != NULL) {
 
                 fputs("\" show=\"", pdata->fh);
                 print_escaped_xml(pdata->fh, dfilter_string);
             }
-            g_free(dfilter_string);
+            wmem_free(NULL, dfilter_string);
 
             /*
              * XXX - should we omit "value" for any fields?
@@ -433,7 +575,6 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
                         case FT_UINT16:
                         case FT_UINT24:
                         case FT_UINT32:
-                        case FT_BOOLEAN:
                             fprintf(pdata->fh, "%X", fvalue_get_uinteger(&fi->value));
                             break;
                         case FT_INT40:
@@ -446,6 +587,7 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
                         case FT_UINT48:
                         case FT_UINT56:
                         case FT_UINT64:
+                        case FT_BOOLEAN:
                             fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_uinteger64(&fi->value));
                             break;
                         default:
@@ -471,12 +613,33 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
         }
     }
 
-    /* We always print all levels for PDML. Recurse here. */
+    /* We print some levels for PDML. Recurse here. */
     if (node->first_child != NULL) {
-        pdata->level++;
-        proto_tree_children_foreach(node,
-                                    proto_tree_write_node_pdml, pdata);
-        pdata->level--;
+        if (pdata->filter == NULL || check_protocolfilter(pdata->filter, fi->hfinfo->abbrev)) {
+            gchar **_filter = NULL;
+            /* Remove protocol filter for children, if children should be included */
+            if ((pdata->filter_flags&PF_INCLUDE_CHILDREN) == PF_INCLUDE_CHILDREN) {
+                _filter = pdata->filter;
+                pdata->filter = NULL;
+            }
+
+            pdata->level++;
+            proto_tree_children_foreach(node,
+                                        proto_tree_write_node_pdml, pdata);
+            pdata->level--;
+
+            /* Put protocol filter back */
+            if ((pdata->filter_flags&PF_INCLUDE_CHILDREN) == PF_INCLUDE_CHILDREN) {
+                pdata->filter = _filter;
+            }
+        } else {
+            print_indent(pdata->level + 2, pdata->fh);
+
+            /* print dummy field */
+            fputs("<field name=\"filtered\" value=\"", pdata->fh);
+            print_escaped_xml(pdata->fh, fi->hfinfo->abbrev);
+            fputs("\" />\n", pdata->fh);
+        }
     }
 
     /* Take back the extra level we added for fake wrapper protocol */
@@ -485,10 +648,8 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
     }
 
     if (node->first_child != NULL) {
-        /* Indent to correct level */
-        for (i = -1; i < pdata->level; i++) {
-            fputs("  ", pdata->fh);
-        }
+        print_indent(pdata->level + 1, pdata->fh);
+
         /* Close off current element */
         /* Data and expert "protocols" use simple tags */
         if ((fi->hfinfo->id != proto_data) && (fi->hfinfo->id != proto_expert)) {
@@ -509,58 +670,731 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
     }
 }
 
+void
+write_json_preamble(FILE *fh)
+{
+    fputs("[\n", fh);
+    json_is_first = TRUE;
+}
+
+void
+write_json_finale(FILE *fh)
+{
+    fputs("\n\n]\n", fh);
+}
+
+void
+write_json_proto_tree(output_fields_t* fields,
+                      print_dissections_e print_dissections,
+                      gboolean print_hex, gchar **protocolfilter,
+                      pf_flags protocolfilter_flags, epan_dissect_t *edt,
+                      proto_node_children_grouper_func node_children_grouper,
+                      FILE *fh)
+{
+    char ts[30];
+    time_t t = time(NULL);
+    struct tm * timeinfo;
+    write_json_data data;
+
+    if (!json_is_first) {
+        fputs("\n\n  ,\n", fh);
+    } else {
+        json_is_first = FALSE;
+    }
+
+    timeinfo = localtime(&t);
+    if (timeinfo != NULL) {
+        strftime(ts, sizeof ts, "%Y-%m-%d", timeinfo);
+    } else {
+        g_strlcpy(ts, "XXXX-XX-XX", sizeof ts); /* XXX - better way of saying "Not representable"? */
+    }
+
+    fputs("  {\n", fh);
+    fprintf(fh, "    \"_index\": \"packets-%s\",\n", ts);
+    fputs("    \"_type\": \"pcap_file\",\n", fh);
+    fputs("    \"_score\": null,\n", fh);
+    fputs("    \"_source\": {\n", fh);
+    fputs("      \"layers\": ", fh);
+
+    if (fields == NULL || fields->fields == NULL) {
+        /* Write out all fields */
+        data.level    = 3;
+        data.fh       = fh;
+        data.src_list = edt->pi.data_src;
+        data.filter   = protocolfilter;
+        data.filter_flags = protocolfilter_flags;
+        data.print_hex = print_hex;
+        data.print_text = TRUE;
+        if (print_dissections == print_dissections_none) {
+            data.print_text = FALSE;
+        }
+        data.node_children_grouper = node_children_grouper;
+
+        write_json_proto_node_children(edt->tree, &data);
+    } else {
+        write_specified_fields(FORMAT_JSON, fields, edt, NULL, fh);
+    }
+
+    fputs("\n", fh);
+    fputs("    }\n", fh);
+    fputs("  }", fh);
+}
+
+/**
+ * Write a json object containing a list of key:value pairs where each key:value pair corresponds to a different json
+ * key and its associated nodes in the proto_tree.
+ * @param proto_node_list_head A 2-dimensional list containing a list of values for each different node json key. The
+ * elements themselves are a linked list of values associated with the same json key.
+ * @param data json writing metadata
+ */
+static void
+write_json_proto_node_list(GSList *proto_node_list_head, write_json_data *data)
+{
+    GSList *current_node = proto_node_list_head;
+
+    fputs("{\n", data->fh);
+    data->level++;
+
+    /*
+     * In most of the following if statements we cannot be sure if its the first or last if statement to be
+     * executed. Thus we need a way of knowing whether a key:value pair has already been printed in order to know
+     * if a comma should be printed before the next key:value pair. We use the delimiter_needed variable to store
+     * whether a comma needs to be written before a new key:value pair is written. Note that instead of checking
+     * before writing a new key:value pair if a comma is needed we could also check after writing a key:value pair
+     * whether a comma is needed but this would be considerably more complex since after each if statement a
+     * different condition would have to be checked. After the first value is written a delimiter is always needed so
+     * this value is never set back to FALSE after it has been set to TRUE.
+     */
+    gboolean delimiter_needed = FALSE;
+
+    // Loop over each list of nodes (differentiated by json key) and write the associated json key:value pair in the
+    // output.
+    while (current_node != NULL) {
+        // Get the list of values for the current json key.
+        GSList *node_values_list = (GSList *) current_node->data;
+
+        // Retrieve the json key from the first value.
+        proto_node *first_value = (proto_node *) node_values_list->data;
+        const char *json_key = proto_node_to_json_key(first_value);
+        // Check if the current json key is filtered from the output with the "-j" cli option.
+        gboolean is_filtered = data->filter != NULL && !check_protocolfilter(data->filter, json_key);
+
+        field_info *fi = first_value->finfo;
+        char *value_string_repr = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
+
+        // We assume all values of a json key have roughly the same layout. Thus we can use the first value to derive
+        // attributes of all the values.
+        gboolean has_value = value_string_repr != NULL;
+        gboolean has_children = first_value->first_child != NULL;
+        gboolean is_pseudo_text_field = fi->hfinfo->id == 0;
+
+        wmem_free(NULL, value_string_repr); // fvalue_to_string_repr returns allocated buffer
+
+        // "-x" command line option. A "_raw" suffix is added to the json key so the textual value can be printed
+        // with the original json key. If both hex and text writing are enabled the raw information of fields whose
+        // length is equal to 0 is not written to the output. If the field is a special text pseudo field no raw
+        // information is written either.
+        if (data->print_hex && (!data->print_text || fi->length > 0) && !is_pseudo_text_field) {
+            if (delimiter_needed) fputs(",\n", data->fh);
+            write_json_proto_node(node_values_list, "_raw", write_json_proto_node_hex_dump, data);
+            delimiter_needed = TRUE;
+        }
+
+        if (data->print_text && has_value) {
+            if (delimiter_needed) fputs(",\n", data->fh);
+            write_json_proto_node(node_values_list, "", write_json_proto_node_value, data);
+            delimiter_needed = TRUE;
+        }
+
+        if (has_children) {
+            if (delimiter_needed) fputs(",\n", data->fh);
+
+            // If a node has both a value and a set of children we print the value and the children in separate
+            // key:value pairs. These can't have the same key so whenever a value is already printed with the node
+            // json key we print the children with the same key with a "_tree" suffix added.
+            char *suffix = has_value ? "_tree": "";
+
+            if (is_filtered) {
+                write_json_proto_node(node_values_list, suffix, write_json_proto_node_filtered, data);
+            } else {
+                // Remove protocol filter for children, if children should be included. This functionality is enabled
+                // with the "-J" command line option. We save the filter so it can be reenabled when we are done with
+                // the current key:value pair.
+                gchar **_filter = NULL;
+                if ((data->filter_flags&PF_INCLUDE_CHILDREN) == PF_INCLUDE_CHILDREN) {
+                    _filter = data->filter;
+                    data->filter = NULL;
+                }
+
+                write_json_proto_node(node_values_list, suffix, write_json_proto_node_children, data);
+
+                // Put protocol filter back
+                if ((data->filter_flags&PF_INCLUDE_CHILDREN) == PF_INCLUDE_CHILDREN) {
+                    data->filter = _filter;
+                }
+            }
+
+            delimiter_needed = TRUE;
+        }
+
+        if (!has_value && !has_children && (data->print_text || (data->print_hex && is_pseudo_text_field))) {
+            if (delimiter_needed) fputs(",\n", data->fh);
+            write_json_proto_node(node_values_list, "", write_json_proto_node_no_value, data);
+            delimiter_needed = TRUE;
+        }
+
+        current_node = current_node->next;
+    }
+
+    data->level--;
+    fputs("\n", data->fh);
+    print_indent(data->level, data->fh);
+    fputs("}", data->fh);
+}
+
+/**
+ * Writes a single node as a key:value pair. The value_writer param can be used to specify how the node's value should
+ * be written.
+ * @param node_values_head Linked list containing all nodes associated with the same json key in this object.
+ * @param suffix Suffix that should be added to the json key.
+ * @param value_writer A function which writes the actual values of the node json key.
+ * @param data json writing metadata
+ */
+static void
+write_json_proto_node(GSList *node_values_head,
+                      const char *suffix,
+                      proto_node_value_writer value_writer,
+                      write_json_data *data)
+{
+    // Retrieve json key from first value.
+    proto_node *first_value = (proto_node *) node_values_head->data;
+    const char *json_key = proto_node_to_json_key(first_value);
+
+    print_indent(data->level, data->fh);
+    fputs("\"", data->fh);
+    print_escaped_json(data->fh, json_key);
+    print_escaped_json(data->fh, suffix);
+    fputs("\": ", data->fh);
+
+    write_json_proto_node_value_list(node_values_head, value_writer, data);
+}
+
+/**
+ * Writes a list of values of a single json key. If multiple values are passed they are wrapped in a json array.
+ * @param node_values_head Linked list containing all values that should be written.
+ * @param value_writer Function which writes the separate values.
+ * @param data json writing metadata
+ */
+static void
+write_json_proto_node_value_list(GSList *node_values_head, proto_node_value_writer value_writer, write_json_data *data)
+{
+    GSList *current_value = node_values_head;
+
+    // Write directly if only a single value is passed. Wrap in json array otherwise.
+    if (current_value->next == NULL) {
+        value_writer((proto_node *) current_value->data, data);
+    } else {
+        fputs("[\n", data->fh);
+        data->level++;
+
+        while (current_value != NULL) {
+            // Do not print delimiter before first value
+            if (current_value != node_values_head) fputs(",\n", data->fh);
+
+            print_indent(data->level, data->fh);
+            value_writer((proto_node *) current_value->data, data);
+            current_value = current_value->next;
+        }
+
+        data->level--;
+        fputs("\n", data->fh);
+        print_indent(data->level, data->fh);
+        fputs("]", data->fh);
+    }
+}
+
+/**
+ * Writes the value for a node that's filtered from the output.
+ */
+static void
+write_json_proto_node_filtered(proto_node *node, write_json_data *data)
+{
+    const char *json_key = proto_node_to_json_key(node);
+
+    fputs("{\n", data->fh);
+    data->level++;
+
+    print_indent(data->level, data->fh);
+    fputs("\"filtered\": ", data->fh);
+    fputs("\"", data->fh);
+    print_escaped_json(data->fh, json_key);
+    fputs("\"\n", data->fh);
+
+    data->level--;
+    print_indent(data->level, data->fh);
+    fputs("}", data->fh);
+}
+
+/**
+ * Writes the hex dump of a node. A json array is written containing the hex dump, position, length, bitmask and type of
+ * the node.
+ */
+static void
+write_json_proto_node_hex_dump(proto_node *node, write_json_data *data)
+{
+    field_info *fi = node->finfo;
+
+    fputs("[\"", data->fh);
+
+    if (fi->hfinfo->bitmask!=0) {
+        switch (fi->value.ftype->ftype) {
+            case FT_INT8:
+            case FT_INT16:
+            case FT_INT24:
+            case FT_INT32:
+                fprintf(data->fh, "%X", (guint) fvalue_get_sinteger(&fi->value));
+                break;
+            case FT_UINT8:
+            case FT_UINT16:
+            case FT_UINT24:
+            case FT_UINT32:
+                fprintf(data->fh, "%X", fvalue_get_uinteger(&fi->value));
+                break;
+            case FT_INT40:
+            case FT_INT48:
+            case FT_INT56:
+            case FT_INT64:
+                fprintf(data->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_sinteger64(&fi->value));
+                break;
+            case FT_UINT40:
+            case FT_UINT48:
+            case FT_UINT56:
+            case FT_UINT64:
+            case FT_BOOLEAN:
+                fprintf(data->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_uinteger64(&fi->value));
+                break;
+            default:
+                g_assert_not_reached();
+        }
+    } else {
+        json_write_field_hex_value(data, fi);
+    }
+
+    /* Dump raw hex-encoded dissected information including position, length, bitmask, type */
+    fprintf(data->fh, "\", %" G_GINT32_MODIFIER "d", fi->start);
+    fprintf(data->fh, ", %" G_GINT32_MODIFIER "d", fi->length);
+    fprintf(data->fh, ", %" G_GUINT64_FORMAT, fi->hfinfo->bitmask);
+    fprintf(data->fh, ", %" G_GINT32_MODIFIER "d", (gint32)fi->value.ftype->ftype);
+
+    fputs("]", data->fh);
+}
+
+/**
+ * Writes the children of a node. Calls write_json_proto_node_list internally which recursively writes children of nodes
+ * to the output.
+ */
+static void
+write_json_proto_node_children(proto_node *node, write_json_data *data)
+{
+    GSList *grouped_children_list = data->node_children_grouper(node);
+    write_json_proto_node_list(grouped_children_list, data);
+    g_slist_free_full(grouped_children_list, (GDestroyNotify) g_slist_free);
+}
+
+/**
+ * Writes the value of a node to the output.
+ */
+static void
+write_json_proto_node_value(proto_node *node, write_json_data *data)
+{
+    field_info *fi = node->finfo;
+    // Get the actual value of the node as a string.
+    char *value_string_repr = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
+
+    fputs("\"", data->fh);
+    print_escaped_json(data->fh, value_string_repr);
+    fputs("\"", data->fh);
+
+    wmem_free(NULL, value_string_repr);
+}
+
+/**
+ * Write the value for a node that has no value and no children. This is the empty string for all nodes except those of
+ * type FT_PROTOCOL for which the full name is written instead.
+ */
+static void
+write_json_proto_node_no_value(proto_node *node, write_json_data *data)
+{
+    field_info *fi = node->finfo;
+
+    fputs("\"", data->fh);
+
+    if (fi->hfinfo->type == FT_PROTOCOL) {
+        if (fi->rep) {
+            print_escaped_json(data->fh, fi->rep->representation);
+        } else {
+            gchar label_str[ITEM_LABEL_LENGTH];
+            proto_item_fill_label(fi, label_str);
+            print_escaped_json(data->fh, label_str);
+        }
+    }
+
+    fputs("\"", data->fh);
+}
+
+/**
+ * Groups each child of the node separately.
+ * @return Linked list where each element is another linked list containing a single node.
+ */
+GSList *
+proto_node_group_children_by_unique(proto_node *node) {
+    GSList *unique_nodes_list = NULL;
+    proto_node *current_child = node->first_child;
+
+    while (current_child != NULL) {
+        GSList *unique_node = g_slist_prepend(NULL, current_child);
+        unique_nodes_list = g_slist_prepend(unique_nodes_list, unique_node);
+        current_child = current_child->next;
+    }
+
+    return g_slist_reverse(unique_nodes_list);
+}
+
+/**
+ * Groups the children of a node by their json key. Children are put in the same group if they have the same json key.
+ * @return Linked list where each element is another linked list of nodes associated with the same json key.
+ */
+GSList *
+proto_node_group_children_by_json_key(proto_node *node)
+{
+    /**
+     * For each different json key we store a linked list of values corresponding to that json key. These lists are kept
+     * in both a linked list and a hashmap. The hashmap is used to quickly retrieve the values of a json key. The linked
+     * list is used to preserve the ordering of keys as they are encountered which is not guaranteed when only using a
+     * hashmap.
+     */
+    GSList *same_key_nodes_list = NULL;
+    GHashTable *lookup_by_json_key = g_hash_table_new(g_str_hash, g_str_equal);
+    proto_node *current_child = node->first_child;
+
+    /**
+     * For each child of the node get the key and get the list of values already associated with that key from the
+     * hashmap. If no list exist yet for that key create a new one and add it to both the linked list and hashmap. If a
+     * list already exists add the node to that list.
+     */
+    while (current_child != NULL) {
+        char *json_key = (char *) proto_node_to_json_key(current_child);
+        GSList *json_key_nodes = (GSList *) g_hash_table_lookup(lookup_by_json_key, json_key);
+
+        if (json_key_nodes == NULL) {
+            json_key_nodes = g_slist_append(json_key_nodes, current_child);
+            // Prepending in single linked list is O(1), appending is O(n). Better to prepend here and reverse at the
+            // end than potentially looping to the end of the linked list for each child.
+            same_key_nodes_list = g_slist_prepend(same_key_nodes_list, json_key_nodes);
+            g_hash_table_insert(lookup_by_json_key, json_key, json_key_nodes);
+        } else {
+            // Store and insert value again to circumvent unused_variable warning.
+            // Append in this case since most value lists will only have a single value.
+            json_key_nodes = g_slist_append(json_key_nodes, current_child);
+            g_hash_table_insert(lookup_by_json_key, json_key, json_key_nodes);
+        }
+
+        current_child = current_child->next;
+    }
+
+    // Hash table is not needed anymore since the linked list with the correct ordering is returned.
+    g_hash_table_destroy(lookup_by_json_key);
+
+    return g_slist_reverse(same_key_nodes_list);
+}
+
+/**
+ * Returns the json key of a node. Tries to use the node's abbreviated name. If the abbreviated name is not available
+ * the representation is used instead.
+ */
+static const char *
+proto_node_to_json_key(proto_node *node)
+{
+    const char *json_key;
+    // Check if node has abbreviated name.
+    if (node->finfo->hfinfo->id != hf_text_only) {
+        json_key = node->finfo->hfinfo->abbrev;
+    } else if (node->finfo->rep != NULL) {
+        json_key = node->finfo->rep->representation;
+    } else {
+        json_key = "";
+    }
+
+    return json_key;
+}
+
+/* Write out a tree's data, and any child nodes, as JSON for EK */
+static void
+proto_tree_write_node_ek(proto_node *node, gpointer data)
+{
+    field_info      *fi    = PNODE_FINFO(node);
+    field_info      *fi_parent    = PNODE_FINFO(node->parent);
+    write_json_data *pdata = (write_json_data*) data;
+    const gchar     *label_ptr;
+    gchar            label_str[ITEM_LABEL_LENGTH];
+    char            *dfilter_string;
+    int              i;
+    gchar           *abbrev_escaped = NULL;
+
+    /* dissection with an invisible proto tree? */
+    g_assert(fi);
+
+    /* Text label. It's printed as a field with no name. */
+    if (fi->hfinfo->id == hf_text_only) {
+        /* Get the text */
+        if (fi->rep) {
+            label_ptr = fi->rep->representation;
+        }
+        else {
+            label_ptr = "";
+        }
+
+        /* Show empty name since it is a required field */
+        fputs("\"", pdata->fh);
+        if (fi_parent != NULL) {
+            print_escaped_ek(pdata->fh, fi_parent->hfinfo->abbrev);
+            fputs("_", pdata->fh);
+        }
+        print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+
+        if (node->first_child != NULL) {
+            fputs("\": \"", pdata->fh);
+            print_escaped_json(pdata->fh, label_ptr);
+            fputs("\",", pdata->fh);
+
+        }
+        else {
+            if (node->next == NULL) {
+                fputs("\": \"",  pdata->fh);
+                print_escaped_json(pdata->fh, label_ptr);
+                fputs("\"", pdata->fh);
+            } else {
+                fputs("\": \"",  pdata->fh);
+                print_escaped_json(pdata->fh, label_ptr);
+                fputs("\",", pdata->fh);
+            }
+        }
+    }
+
+    /* Normal protocols and fields */
+    else {
+        /*
+         * Hex dump -x
+         */
+        if (pdata->print_hex && fi->length > 0) {
+            fputs("\"", pdata->fh);
+            if (fi_parent != NULL) {
+                print_escaped_ek(pdata->fh, fi_parent->hfinfo->abbrev);
+                fputs("_", pdata->fh);
+            }
+            print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+            fputs("_raw", pdata->fh);
+            fputs("\": \"", pdata->fh);
+
+            if (fi->hfinfo->bitmask!=0) {
+                switch (fi->value.ftype->ftype) {
+                    case FT_INT8:
+                    case FT_INT16:
+                    case FT_INT24:
+                    case FT_INT32:
+                        fprintf(pdata->fh, "%X", (guint) fvalue_get_sinteger(&fi->value));
+                        break;
+                    case FT_UINT8:
+                    case FT_UINT16:
+                    case FT_UINT24:
+                    case FT_UINT32:
+                        fprintf(pdata->fh, "%X", fvalue_get_uinteger(&fi->value));
+                        break;
+                    case FT_INT40:
+                    case FT_INT48:
+                    case FT_INT56:
+                    case FT_INT64:
+                        fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_sinteger64(&fi->value));
+                        break;
+                    case FT_UINT40:
+                    case FT_UINT48:
+                    case FT_UINT56:
+                    case FT_UINT64:
+                    case FT_BOOLEAN:
+                        fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_uinteger64(&fi->value));
+                        break;
+                    default:
+                        g_assert_not_reached();
+                }
+                fputs("\",", pdata->fh);
+            }
+            else {
+                json_write_field_hex_value(pdata, fi);
+                fputs("\",", pdata->fh);
+            }
+        }
+
+
+
+        fputs("\"", pdata->fh);
+
+        if (fi_parent != NULL) {
+            print_escaped_ek(pdata->fh, fi_parent->hfinfo->abbrev);
+            fputs("_", pdata->fh);
+        }
+        print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+
+        /* show, value, and unmaskedvalue attributes */
+        switch (fi->hfinfo->type)
+        {
+        case FT_PROTOCOL:
+            if (node->first_child != NULL) {
+                fputs("\": {", pdata->fh);
+            } else {
+                fputs("\": \"", pdata->fh);
+                if (fi->rep) {
+                    print_escaped_json(pdata->fh, fi->rep->representation);
+                }
+                else {
+                    label_ptr = label_str;
+                    proto_item_fill_label(fi, label_str);
+                    print_escaped_json(pdata->fh, label_ptr);
+                }
+                if (node->next == NULL) {
+                    fputs("\"",  pdata->fh);
+                } else {
+                    fputs("\",",  pdata->fh);
+                }
+            }
+            break;
+        case FT_NONE:
+            if (node->first_child != NULL) {
+                fputs("\": \"\",",  pdata->fh);
+            } else {
+                if (node->next == NULL) {
+                    fputs("\": \"\"",  pdata->fh);
+                } else {
+                    fputs("\": \"\",",  pdata->fh);
+                }
+            }
+            break;
+        default:
+            dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
+            if (dfilter_string != NULL) {
+                fputs("\": \"", pdata->fh);
+                print_escaped_json(pdata->fh, dfilter_string);
+            }
+            wmem_free(NULL, dfilter_string);
+
+            if (node->next == NULL && node->first_child == NULL) {
+                fputs("\"", pdata->fh);
+            } else {
+                fputs("\",", pdata->fh);
+            }
+        }
+
+    }
+
+    /* We print some levels for JSON. Recurse here. */
+    if (node->first_child != NULL) {
+
+        if (pdata->filter != NULL) {
+
+            /* to to thread the '.' and '_' equally. The '.' is replace by print_escaped_ek for '_' */
+            if (fi->hfinfo->abbrev != NULL) {
+                if (strlen(fi->hfinfo->abbrev) > 0) {
+                    abbrev_escaped = g_strdup(fi->hfinfo->abbrev);
+
+                    i = 0;
+                    while(abbrev_escaped[i]!='\0') {
+                        if(abbrev_escaped[i]=='.') {
+                            abbrev_escaped[i]='_';
+                        }
+                        i++;
+                    }
+                }
+            }
+
+            if(check_protocolfilter(pdata->filter, fi->hfinfo->abbrev) || check_protocolfilter(pdata->filter, abbrev_escaped)) {
+                gchar **_filter = NULL;
+                /* Remove protocol filter for children, if children should be included */
+                if ((pdata->filter_flags&PF_INCLUDE_CHILDREN) == PF_INCLUDE_CHILDREN) {
+                    _filter = pdata->filter;
+                    pdata->filter = NULL;
+                }
+
+                pdata->level++;
+                proto_tree_children_foreach(node, proto_tree_write_node_ek, pdata);
+                pdata->level--;
+
+                /* Put protocol filter back */
+                if ((pdata->filter_flags&PF_INCLUDE_CHILDREN) == PF_INCLUDE_CHILDREN) {
+                    pdata->filter = _filter;
+                }
+            } else {
+                /* print dummy field */
+                fputs("\"filtered\": \"", pdata->fh);
+                print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+                fputs("\"", pdata->fh);
+            }
+
+            /* release abbrev_escaped string */
+            if (abbrev_escaped != NULL) {
+                g_free(abbrev_escaped);
+            }
+
+        } else {
+            pdata->level++;
+            proto_tree_children_foreach(node,
+                                        proto_tree_write_node_ek, pdata);
+            pdata->level--;
+        }
+    }
+
+    if (node->first_child != NULL) {
+        if (fi->hfinfo->type == FT_PROTOCOL) {
+            /* Close off current element */
+            if (node->next == NULL) {
+                fputs("}", pdata->fh);
+            } else {
+                fputs("},", pdata->fh);
+            }
+        } else {
+            if (node->next != NULL) {
+                fputs(",", pdata->fh);
+            }
+        }
+    }
+}
+
 /* Print info for a 'geninfo' pseudo-protocol. This is required by
  * the PDML spec. The information is contained in Wireshark's 'frame' protocol,
  * but we produce a 'geninfo' protocol in the PDML to conform to spec.
  * The 'frame' protocol follows the 'geninfo' protocol in the PDML. */
 static void
-print_pdml_geninfo(proto_tree *tree, FILE *fh)
+print_pdml_geninfo(epan_dissect_t *edt, FILE *fh)
 {
     guint32     num, len, caplen;
-    nstime_t   *timestamp;
     GPtrArray  *finfo_array;
     field_info *frame_finfo;
     gchar      *tmp;
 
     /* Get frame protocol's finfo. */
-    finfo_array = proto_find_finfo(tree, proto_frame);
+    finfo_array = proto_find_first_finfo(edt->tree, proto_frame);
     if (g_ptr_array_len(finfo_array) < 1) {
         return;
     }
     frame_finfo = (field_info *)finfo_array->pdata[0];
     g_ptr_array_free(finfo_array, TRUE);
 
-    /* frame.number --> geninfo.num */
-    finfo_array = proto_find_finfo(tree, hf_frame_number);
-    if (g_ptr_array_len(finfo_array) < 1) {
-        return;
-    }
-    num = fvalue_get_uinteger(&((field_info*)finfo_array->pdata[0])->value);
-    g_ptr_array_free(finfo_array, TRUE);
+    /* frame.number, packet_info.num */
+    num = edt->pi.num;
 
-    /* frame.frame_len --> geninfo.len */
-    finfo_array = proto_find_finfo(tree, hf_frame_len);
-    if (g_ptr_array_len(finfo_array) < 1) {
-        return;
-    }
-    len = fvalue_get_uinteger(&((field_info*)finfo_array->pdata[0])->value);
-    g_ptr_array_free(finfo_array, TRUE);
+    /* frame.frame_len, packet_info.frame_data->pkt_len */
+    len = edt->pi.fd->pkt_len;
 
-    /* frame.cap_len --> geninfo.caplen */
-    finfo_array = proto_find_finfo(tree, hf_frame_capture_len);
-    if (g_ptr_array_len(finfo_array) < 1) {
-        return;
-    }
-    caplen = fvalue_get_uinteger(&((field_info*)finfo_array->pdata[0])->value);
-    g_ptr_array_free(finfo_array, TRUE);
-
-    /* frame.time --> geninfo.timestamp */
-    finfo_array = proto_find_finfo(tree, hf_frame_arrival_time);
-    if (g_ptr_array_len(finfo_array) < 1) {
-        return;
-    }
-    timestamp = (nstime_t *)fvalue_get(&((field_info*)finfo_array->pdata[0])->value);
-    g_ptr_array_free(finfo_array, TRUE);
+    /* frame.cap_len --> packet_info.frame_data->cap_len */
+    caplen = edt->pi.fd->cap_len;
 
     /* Print geninfo start */
     fprintf(fh,
@@ -582,12 +1416,12 @@ print_pdml_geninfo(proto_tree *tree, FILE *fh)
             "    <field name=\"caplen\" pos=\"0\" show=\"%u\" showname=\"Captured Length\" value=\"%x\" size=\"%d\"/>\n",
             caplen, caplen, frame_finfo->length);
 
-    tmp = abs_time_to_str(NULL, timestamp, ABSOLUTE_TIME_LOCAL, TRUE);
+    tmp = abs_time_to_str(NULL, &edt->pi.abs_ts, ABSOLUTE_TIME_LOCAL, TRUE);
 
     /* Print geninfo.timestamp */
     fprintf(fh,
             "    <field name=\"timestamp\" pos=\"0\" show=\"%s\" showname=\"Captured Time\" value=\"%d.%09d\" size=\"%d\"/>\n",
-            tmp, (int) timestamp->secs, timestamp->nsecs, frame_finfo->length);
+            tmp, (int)edt->pi.abs_ts.secs, edt->pi.abs_ts.nsecs, frame_finfo->length);
 
     wmem_free(NULL, tmp);
 
@@ -602,19 +1436,20 @@ write_pdml_finale(FILE *fh)
     fputs("</pdml>\n", fh);
 }
 
+
+
 void
 write_psml_preamble(column_info *cinfo, FILE *fh)
 {
     gint i;
 
-    fputs("<?xml version=\"1.0\"?>\n", fh);
-    fputs("<psml version=\"" PSML_VERSION "\" ", fh);
-    fprintf(fh, "creator=\"%s/%s\">\n", PACKAGE, VERSION);
+    fprintf(fh, "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
+    fprintf(fh, "<psml version=\"" PSML_VERSION "\" creator=\"%s/%s\">\n", PACKAGE, VERSION);
     fprintf(fh, "<structure>\n");
 
     for (i = 0; i < cinfo->num_cols; i++) {
         fprintf(fh, "<section>");
-        print_escaped_xml(fh, cinfo->col_title[i]);
+        print_escaped_xml(fh, cinfo->columns[i].col_title);
         fprintf(fh, "</section>\n");
     }
 
@@ -622,15 +1457,23 @@ write_psml_preamble(column_info *cinfo, FILE *fh)
 }
 
 void
-write_psml_columns(epan_dissect_t *edt, FILE *fh)
+write_psml_columns(epan_dissect_t *edt, FILE *fh, gboolean use_color)
 {
     gint i;
+    const color_filter_t *cfp = edt->pi.fd->color_filter;
 
-    fprintf(fh, "<packet>\n");
+    if (use_color && (cfp != NULL)) {
+        fprintf(fh, "<packet foreground='#%02x%02x%02x' background='#%02x%02x%02x'>\n",
+            cfp->fg_color.red, cfp->fg_color.green, cfp->fg_color.blue,
+            cfp->bg_color.red, cfp->bg_color.green, cfp->bg_color.blue);
+    }
+    else {
+        fprintf(fh, "<packet>\n");
+    }
 
     for (i = 0; i < edt->pi.cinfo->num_cols; i++) {
         fprintf(fh, "<section>");
-        print_escaped_xml(fh, edt->pi.cinfo->col_data[i]);
+        print_escaped_xml(fh, edt->pi.cinfo->columns[i].col_data);
         fprintf(fh, "</section>\n");
     }
 
@@ -654,7 +1497,7 @@ static gchar *csv_massage_str(const gchar *source, const gchar *exceptions)
     csv_str = g_strescape(source, exceptions);
     tmp_str = csv_str;
     /* Locate the UTF-8 right arrow character and replace it by an ASCII equivalent */
-    while ( (tmp_str = strstr(tmp_str, "\xe2\x86\x92")) != NULL ) {
+    while ( (tmp_str = strstr(tmp_str, UTF8_RIGHTWARDS_ARROW)) != NULL ) {
         tmp_str[0] = ' ';
         tmp_str[1] = '>';
         tmp_str[2] = ' ';
@@ -669,8 +1512,8 @@ static void csv_write_str(const char *str, char sep, FILE *fh)
 {
     gchar *csv_str;
 
-    /* Do not escape the UTF-8 righ arrow character */
-    csv_str = csv_massage_str(str, "\xe2\x86\x92");
+    /* Do not escape the UTF-8 right arrow character */
+    csv_str = csv_massage_str(str, UTF8_RIGHTWARDS_ARROW);
     fprintf(fh, "\"%s\"%c", csv_str, sep);
     g_free(csv_str);
 }
@@ -681,8 +1524,8 @@ write_csv_column_titles(column_info *cinfo, FILE *fh)
     gint i;
 
     for (i = 0; i < cinfo->num_cols - 1; i++)
-        csv_write_str(cinfo->col_title[i], ',', fh);
-    csv_write_str(cinfo->col_title[i], '\n', fh);
+        csv_write_str(cinfo->columns[i].col_title, ',', fh);
+    csv_write_str(cinfo->columns[i].col_title, '\n', fh);
 }
 
 void
@@ -691,8 +1534,8 @@ write_csv_columns(epan_dissect_t *edt, FILE *fh)
     gint i;
 
     for (i = 0; i < edt->pi.cinfo->num_cols - 1; i++)
-        csv_write_str(edt->pi.cinfo->col_data[i], ',', fh);
-    csv_write_str(edt->pi.cinfo->col_data[i], '\n', fh);
+        csv_write_str(edt->pi.cinfo->columns[i].col_data, ',', fh);
+    csv_write_str(edt->pi.cinfo->columns[i].col_data, '\n', fh);
 }
 
 void
@@ -813,6 +1656,10 @@ print_escaped_xml(FILE *fh, const char *unescaped_string)
     const char *p;
     char        temp_str[8];
 
+    if (fh == NULL || unescaped_string == NULL) {
+        return;
+    }
+
     for (p = unescaped_string; *p != '\0'; p++) {
         switch (*p) {
         case '&':
@@ -841,6 +1688,75 @@ print_escaped_xml(FILE *fh, const char *unescaped_string)
     }
 }
 
+static void
+print_escaped_bare(FILE *fh, const char *unescaped_string, gboolean change_dot)
+{
+    const char *p;
+    char        temp_str[8];
+
+    if (fh == NULL || unescaped_string == NULL) {
+        return;
+    }
+
+    for (p = unescaped_string; *p != '\0'; p++) {
+        switch (*p) {
+        case '"':
+            fputs("\\\"", fh);
+            break;
+        case '\\':
+            fputs("\\\\", fh);
+            break;
+        case '/':
+            fputs("\\/", fh);
+            break;
+        case '\b':
+            fputs("\\b", fh);
+            break;
+        case '\f':
+            fputs("\\f", fh);
+            break;
+        case '\n':
+            fputs("\\n", fh);
+            break;
+        case '\r':
+            fputs("\\r", fh);
+            break;
+        case '\t':
+            fputs("\\t", fh);
+            break;
+        case '.':
+            if (change_dot)
+                fputs("_", fh);
+            else
+                fputs(".", fh);
+            break;
+        default:
+            if (g_ascii_isprint(*p))
+                fputc(*p, fh);
+            else {
+                g_snprintf(temp_str, sizeof(temp_str), "\\u00%02x", (guint8)*p);
+                fputs(temp_str, fh);
+            }
+        }
+    }
+}
+
+/* Print a string, escaping out certain characters that need to
+ * escaped out for JSON. */
+static void
+print_escaped_json(FILE *fh, const char *unescaped_string)
+{
+    print_escaped_bare(fh, unescaped_string, FALSE);
+}
+
+/* Print a string, escaping out certain characters that need to
+ * escaped out for Elasticsearch title. */
+static void
+print_escaped_ek(FILE *fh, const char *unescaped_string)
+{
+    print_escaped_bare(fh, unescaped_string, TRUE);
+}
+
 static void
 pdml_write_field_hex_value(write_pdml_data *pdata, field_info *fi)
 {
@@ -866,6 +1782,31 @@ pdml_write_field_hex_value(write_pdml_data *pdata, field_info *fi)
     }
 }
 
+static void
+json_write_field_hex_value(write_json_data *pdata, field_info *fi)
+{
+    int           i;
+    const guint8 *pd;
+
+    if (!fi->ds_tvb)
+        return;
+
+    if (fi->length > tvb_captured_length_remaining(fi->ds_tvb, fi->start)) {
+        fprintf(pdata->fh, "field length invalid!");
+        return;
+    }
+
+    /* Find the data for this field. */
+    pd = get_field_data(pdata->src_list, fi);
+
+    if (pd) {
+        /* Print a simple hex dump */
+        for (i = 0 ; i < fi->length; i++) {
+            fprintf(pdata->fh, "%02x", pd[i]);
+        }
+    }
+}
+
 gboolean
 print_hex_data(print_stream_t *stream, epan_dissect_t *edt)
 {
@@ -901,7 +1842,7 @@ print_hex_data(print_stream_t *stream, epan_dissect_t *edt)
             return TRUE;
         cp = tvb_get_ptr(tvb, 0, length);
         if (!print_hex_data_buffer(stream, cp, length,
-                                   edt->pi.fd->flags.encoding))
+                                   (packet_char_enc)edt->pi.fd->flags.encoding))
             return FALSE;
     }
     return TRUE;
@@ -1075,30 +2016,28 @@ static void
 output_field_check(void *data, void *user_data)
 {
     gchar *field = (gchar *)data;
-    gboolean *all_valid = (gboolean *)user_data;
+    GSList **invalid_fields = (GSList **)user_data;
 
     if (!strncmp(field, COLUMN_FIELD_FILTER, strlen(COLUMN_FIELD_FILTER)))
         return;
 
     if (!proto_registrar_get_byname(field)) {
-        g_warning("'%s' isn't a valid field!", field);
-        *all_valid = FALSE;
+        *invalid_fields = g_slist_prepend(*invalid_fields, field);
     }
 
 }
 
-gboolean
+GSList *
 output_fields_valid(output_fields_t *fields)
 {
-    gboolean all_valid = TRUE;
-
+    GSList *invalid_fields = NULL;
     if (fields->fields == NULL) {
-        return TRUE;
+        return NULL;
     }
 
-    g_ptr_array_foreach(fields->fields, output_field_check, &all_valid);
+    g_ptr_array_foreach(fields->fields, output_field_check, &invalid_fields);
 
-    return all_valid;
+    return invalid_fields;
 }
 
 gboolean output_fields_set_option(output_fields_t *info, gchar *option)
@@ -1200,6 +2139,19 @@ gboolean output_fields_set_option(output_fields_t *info, gchar *option)
         }
         return TRUE;
     }
+    else if (0 == strcmp(option_name, "bom")) {
+        switch (*option_value) {
+        case 'n':
+            info->print_bom = FALSE;
+            break;
+        case 'y':
+            info->print_bom = TRUE;
+            break;
+        default:
+            return FALSE;
+        }
+        return TRUE;
+    }
 
     return FALSE;
 }
@@ -1207,6 +2159,7 @@ gboolean output_fields_set_option(output_fields_t *info, gchar *option)
 void output_fields_list_options(FILE *fh)
 {
     fprintf(fh, "TShark: The available options for field output \"E\" are:\n");
+    fputs("bom=y|n    Prepend output with the UTF-8 BOM (def: N: no)\n", fh);
     fputs("header=y|n    Print field abbreviations as first line of output (def: N: no)\n", fh);
     fputs("separator=/t|/s|<character>   Set the separator to use;\n     \"/t\" = tab, \"/s\" = space (def: /t: tab)\n", fh);
     fputs("occurrence=f|l|a  Select the occurrence of a field to use;\n     \"f\" = first, \"l\" = last, \"a\" = all (def: a: all)\n", fh);
@@ -1228,6 +2181,11 @@ void write_fields_preamble(output_fields_t* fields, FILE *fh)
     g_assert(fh);
     g_assert(fields->fields);
 
+    if (fields->print_bom) {
+        fputs(UTF8_BOM, fh);
+    }
+
+
     if (!fields->print_header) {
         return;
     }
@@ -1313,9 +2271,10 @@ static void proto_tree_get_node_field_values(proto_node *node, gpointer data)
     }
 }
 
-void write_fields_proto_tree(output_fields_t *fields, epan_dissect_t *edt, column_info *cinfo, FILE *fh)
+static void write_specified_fields(fields_format format, output_fields_t *fields, epan_dissect_t *edt, column_info *cinfo, FILE *fh)
 {
     gsize     i;
+    gboolean first = TRUE;
     gint      col;
     gchar    *col_name;
     gpointer  field_index;
@@ -1357,44 +2316,162 @@ void write_fields_proto_tree(output_fields_t *fields, epan_dissect_t *edt, colum
     proto_tree_children_foreach(edt->tree, proto_tree_get_node_field_values,
                                 &data);
 
-    if (fields->includes_col_fields) {
-        for (col = 0; col < cinfo->num_cols; col++) {
-            /* Prepend COLUMN_FIELD_FILTER as the field name */
-            col_name = g_strdup_printf("%s%s", COLUMN_FIELD_FILTER, cinfo->col_title[col]);
-            field_index = g_hash_table_lookup(fields->field_indicies, col_name);
-            g_free(col_name);
-
-            if (NULL != field_index) {
-                format_field_values(fields, field_index, g_strdup(cinfo->col_data[col]));
+    switch (format) {
+    case FORMAT_CSV:
+        if (fields->includes_col_fields) {
+            for (col = 0; col < cinfo->num_cols; col++) {
+                /* Prepend COLUMN_FIELD_FILTER as the field name */
+                col_name = g_strdup_printf("%s%s", COLUMN_FIELD_FILTER, cinfo->columns[col].col_title);
+                field_index = g_hash_table_lookup(fields->field_indicies, col_name);
+                g_free(col_name);
+
+                if (NULL != field_index) {
+                    format_field_values(fields, field_index, g_strdup(cinfo->columns[col].col_data));
+                }
             }
         }
-    }
 
-    for(i = 0; i < fields->fields->len; ++i) {
-        if (0 != i) {
-            fputc(fields->separator, fh);
+        for(i = 0; i < fields->fields->len; ++i) {
+            if (0 != i) {
+                fputc(fields->separator, fh);
+            }
+            if (NULL != fields->field_values[i]) {
+                GPtrArray *fv_p;
+                gchar * str;
+                gsize j;
+                fv_p = fields->field_values[i];
+                if (fields->quote != '\0') {
+                    fputc(fields->quote, fh);
+                }
+
+                /* Output the array of (partial) field values */
+                for (j = 0; j < g_ptr_array_len(fv_p); j++ ) {
+                    str = (gchar *)g_ptr_array_index(fv_p, j);
+                    fputs(str, fh);
+                    g_free(str);
+                }
+                if (fields->quote != '\0') {
+                    fputc(fields->quote, fh);
+                }
+                g_ptr_array_free(fv_p, TRUE);  /* get ready for the next packet */
+                fields->field_values[i] = NULL;
+            }
         }
-        if (NULL != fields->field_values[i]) {
-            GPtrArray *fv_p;
-            gchar * str;
-            gsize j;
-            fv_p = fields->field_values[i];
-            if (fields->quote != '\0') {
-                fputc(fields->quote, fh);
+        break;
+    case FORMAT_XML:
+        for(i = 0; i < fields->fields->len; ++i) {
+            gchar *field = (gchar *)g_ptr_array_index(fields->fields, i);
+
+            if (NULL != fields->field_values[i]) {
+                GPtrArray *fv_p;
+                gchar * str;
+                gsize j;
+                fv_p = fields->field_values[i];
+
+                /* Output the array of (partial) field values */
+                for (j = 0; j < (g_ptr_array_len(fv_p)); j+=2 ) {
+                    str = (gchar *)g_ptr_array_index(fv_p, j);
+
+                    fprintf(fh, "  <field name=\"%s\" value=", field);
+                    fputs("\"", fh);
+                    print_escaped_xml(fh, str);
+                    fputs("\"/>\n", fh);
+                    g_free(str);
+                }
+                g_ptr_array_free(fv_p, TRUE);  /* get ready for the next packet */
+                fields->field_values[i] = NULL;
             }
+        }
+        break;
+    case FORMAT_JSON:
+        fputs("{\n", fh);
+        for(i = 0; i < fields->fields->len; ++i) {
+            gchar *field = (gchar *)g_ptr_array_index(fields->fields, i);
 
-            /* Output the array of (partial) field values */
-            for (j = 0; j < g_ptr_array_len(fv_p); j++ ) {
-                str = (gchar *)g_ptr_array_index(fv_p, j);
-                fputs(str, fh);
-                g_free(str);
+            if (NULL != fields->field_values[i]) {
+                GPtrArray *fv_p;
+                gchar * str;
+                gsize j;
+                fv_p = fields->field_values[i];
+
+                /* Output the array of (partial) field values */
+                for (j = 0; j < (g_ptr_array_len(fv_p)); j += 2) {
+                    str = (gchar *) g_ptr_array_index(fv_p, j);
+
+                    if (j == 0) {
+                        if (!first) {
+                            fputs(",\n", fh);
+                        }
+                        fprintf(fh, "        \"%s\": [", field);
+                    }
+                    fputs("\"", fh);
+                    print_escaped_json(fh, str);
+                    fputs("\"", fh);
+                    g_free(str);
+
+                    if (j + 2 < (g_ptr_array_len(fv_p))) {
+                        fputs(",", fh);
+                    } else {
+                        fputs("]", fh);
+                    }
+                }
+
+                first = FALSE;
+                g_ptr_array_free(fv_p, TRUE);  /* get ready for the next packet */
+                fields->field_values[i] = NULL;
             }
-            if (fields->quote != '\0') {
-                fputc(fields->quote, fh);
+        }
+        fputc('\n',fh);
+
+        fputs("      }", fh);
+        break;
+    case FORMAT_EK:
+        for(i = 0; i < fields->fields->len; ++i) {
+            gchar *field = (gchar *)g_ptr_array_index(fields->fields, i);
+
+            if (NULL != fields->field_values[i]) {
+                GPtrArray *fv_p;
+                gchar * str;
+                gsize j;
+                fv_p = fields->field_values[i];
+
+                /* Output the array of (partial) field values */
+                for (j = 0; j < (g_ptr_array_len(fv_p)); j += 2) {
+                    str = (gchar *)g_ptr_array_index(fv_p, j);
+
+                    if (j == 0) {
+                        if (!first) {
+                            fputs(",", fh);
+                        }
+                        fputs("\"", fh);
+                        print_escaped_ek(fh, field);
+                        fputs("\": [", fh);
+                    }
+                    fputs("\"", fh);
+                    print_escaped_json(fh, str);
+                    fputs("\"", fh);
+                    g_free(str);
+
+                    if (j + 2 < (g_ptr_array_len(fv_p))) {
+                        fputs(",", fh);
+                    }
+                    else {
+                        fputs("]", fh);
+
+                        }
+                    }
+
+                first = FALSE;
+                g_ptr_array_free(fv_p, TRUE);  /* get ready for the next packet */
+                fields->field_values[i] = NULL;
             }
-            g_ptr_array_free(fv_p, TRUE);  /* get ready for the next packet */
-            fields->field_values[i] = NULL;
         }
+        break;
+
+    default:
+        fprintf(stderr, "Unknown fields format %d\n", format);
+        g_assert_not_reached();
+        break;
     }
 }
 
@@ -1440,9 +2517,11 @@ gchar* get_node_field_value(field_info* fi, epan_dissect_t* edt)
              * FT_NONE can be checked when using -T fields */
             return g_strdup("1");
         default:
-            dfilter_string = fvalue_to_string_repr(&fi->value, FTREPR_DISPLAY, fi->hfinfo->display, NULL);
+            dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
             if (dfilter_string != NULL) {
-                return dfilter_string;
+                gchar* ret = g_strdup(dfilter_string);
+                wmem_free(NULL, dfilter_string);
+                return ret;
             } else {
                 return get_field_hex_value(edt->pi.data_src, fi);
             }
@@ -1490,6 +2569,7 @@ get_field_hex_value(GSList *src_list, field_info *fi)
 output_fields_t* output_fields_new(void)
 {
     output_fields_t* fields     = g_new(output_fields_t, 1);
+    fields->print_bom           = FALSE;
     fields->print_header        = FALSE;
     fields->separator           = '\t';
     fields->occurrence          = 'a';