tshark JSON and Elasticsearch output
authorMartin Kacer <kacer.martin@gmail.com>
Mon, 13 Jun 2016 13:47:44 +0000 (15:47 +0200)
committerAnders Broman <a.broman58@gmail.com>
Fri, 17 Jun 2016 06:09:29 +0000 (06:09 +0000)
Added ouput -T for json|ek
Added -j switch fo filter EK json|ek fields.
Added -x switch to work with json|ek to insert raw fields.

Bug: 11754

Change-Id: Iad5a9092b843c074b0b774d1745fa14fca09f6b7
Reviewed-on: https://code.wireshark.org/review/15869
Reviewed-by: Jaap Keuter <jaap.keuter@xs4all.nl>
Petri-Dish: Jaap Keuter <jaap.keuter@xs4all.nl>
Tested-by: Petri Dish Buildbot <buildbot-no-reply@wireshark.org>
Reviewed-by: Anders Broman <a.broman58@gmail.com>
AUTHORS.src
doc/tshark.pod
epan/print.c
epan/print.h
tshark.c

index 1b72a49a23d8830283d77993ac41b31480ffef49..2ba675547d7de67b9d411f9078e25636cef78a81 100644 (file)
@@ -3671,6 +3671,10 @@ Barbu Paul - Gheorghe    <barbu.paul.gheorghe[AT]gmail.com> {
        Developer documentation improvements
 }
 
+Martin Kacer   <kacer.martin[AT]gmail.com> {
+       JSON and Elasticsearch tshark output
+}
+
 
 and by:
 
index 120c74514ae970f954036d4ded9be51385efbcd5..2ee54edbc34de02f955c57f94b757108a09a0a41 100644 (file)
@@ -22,6 +22,7 @@ S<[ B<-g> ]>
 S<[ B<-h> ]>
 S<[ B<-H> E<lt>input hosts fileE<gt> ]>
 S<[ B<-i> E<lt>capture interfaceE<gt>|- ]>
+S<[ B<-j> E<lt>json match filterE<gt> ]>
 S<[ B<-I> ]>
 S<[ B<-K> E<lt>keytabE<gt> ]>
 S<[ B<-l> ]>
@@ -39,7 +40,7 @@ S<[ B<-R> E<lt>Read filterE<gt> ]>
 S<[ B<-s> E<lt>capture snaplenE<gt> ]>
 S<[ B<-S> E<lt>separatorE<gt> ]>
 S<[ B<-t> a|ad|adoy|d|dd|e|r|u|ud|udoy ]>
-S<[ B<-T> fields|pdml|ps|psml|text ]>
+S<[ B<-T> ek|fields|json|pdml|ps|psml|text ]>
 S<[ B<-u> E<lt>seconds typeE<gt>]>
 S<[ B<-U> E<lt>tap_nameE<gt>]>
 S<[ B<-v> ]>
@@ -534,6 +535,14 @@ If used after an B<-i> option, it enables the monitor mode for
 the interface specified by the last B<-i> option occurring before
 this option.
 
+=item -j  E<lt>json match filterE<gt>
+
+JSON match filter used for json|ek output file types.
+JSON parent node containing multiple child nodes is only included,
+if the name is found in the filter.
+
+Example: B<-j "http tcp ip">
+
 =item -K  E<lt>keytabE<gt>
 
 Load kerberos crypto keys from the specified keytab file.
@@ -724,11 +733,19 @@ was captured
 
 The default format is relative.
 
-=item -T  fields|pdml|ps|psml|text
+=item -T  ek|fields|json|pdml|ps|psml|text
 
 Set the format of the output when viewing decoded packet data.  The
 options are one of:
 
+B<ek> Newline delimited JSON format for bulk import into Elasticsearch.
+It can be used with B<-j> including the JSON filter or with B<-x> flag
+to include raw packet data.
+Example of usage to import data into Elasticsearch:
+
+  tshark -T ek -j "http tcp ip" -x -r file.pcap > file.json
+  curl -XPUT http://elasticsearch:9200/_bulk --data-binary @file.json
+
 B<fields> The values of fields specified with the B<-e> option, in a
 form specified by the B<-E> option.  For example,
 
@@ -737,6 +754,13 @@ form specified by the B<-E> option.  For example,
 would generate comma-separated values (CSV) output suitable for importing
 into your favorite spreadsheet program.
 
+B<json> JSON file format. It can be used with B<-j> including
+the JSON filter or with B<-x> flag to influde raw packet data.
+Example of usage:
+
+  tshark -T json -r file.pcap
+  tshark -T json -j "http tcp ip" -x -r file.pcap
+
 B<pdml> Packet Details Markup Language, an XML-based format for the details of
 a decoded packet.  This information is equivalent to the packet details
 printed with the B<-V> flag.
index e657324c061b2417e591714567635c926a117e1e..92e9bfe73de8168925ac94a034ec1235599678f3 100644 (file)
@@ -62,6 +62,15 @@ typedef struct {
     epan_dissect_t *edt;
 } write_pdml_data;
 
+typedef struct {
+    int             level;
+    FILE           *fh;
+    GSList         *src_list;
+    epan_dissect_t *edt;
+    gchar          *filter;
+    gboolean        print_hex;
+} write_json_data;
+
 typedef struct {
     output_fields_t *fields;
     epan_dissect_t  *edt;
@@ -83,11 +92,16 @@ struct _output_fields {
 static gchar *get_field_hex_value(GSList *src_list, field_info *fi);
 static void proto_tree_print_node(proto_node *node, gpointer data);
 static void proto_tree_write_node_pdml(proto_node *node, gpointer data);
+static void proto_tree_write_node_json(proto_node *node, gpointer data);
+static void proto_tree_write_node_ek(proto_node *node, gpointer data);
 static const guint8 *get_field_data(GSList *src_list, field_info *fi);
 static void pdml_write_field_hex_value(write_pdml_data *pdata, field_info *fi);
+static void json_write_field_hex_value(write_json_data *pdata, field_info *fi);
 static gboolean print_hex_data_buffer(print_stream_t *stream, const guchar *cp,
                                       guint length, packet_char_enc encoding);
 static void print_escaped_xml(FILE *fh, const char *unescaped_string);
+static void print_escaped_json(FILE *fh, const char *unescaped_string);
+static void print_escaped_ek(FILE *fh, const char *unescaped_string);
 
 static void print_pdml_geninfo(proto_tree *tree, FILE *fh);
 
@@ -242,6 +256,12 @@ write_pdml_preamble(FILE *fh, const gchar *filename)
     fprintf(fh, "creator=\"%s/%s\" time=\"%s\" capture_file=\"%s\">\n", PACKAGE, VERSION, ts, filename ? filename : "");
 }
 
+void
+write_json_preamble(FILE *fh)
+{
+    fputs("{\n", fh);
+}
+
 void
 write_pdml_proto_tree(epan_dissect_t *edt, FILE *fh)
 {
@@ -264,6 +284,87 @@ write_pdml_proto_tree(epan_dissect_t *edt, FILE *fh)
     fprintf(fh, "</packet>\n\n");
 }
 
+void
+write_json_proto_tree(print_args_t *print_args, gchar *jsonfilter, epan_dissect_t *edt, FILE *fh)
+{
+    write_json_data data;
+    char ts[30];
+    time_t t = time(NULL);
+    struct tm * timeinfo;
+
+    /* Create the output */
+    data.level    = 0;
+    data.fh       = fh;
+    data.src_list = edt->pi.data_src;
+    data.edt      = edt;
+    data.filter   = jsonfilter;
+    data.print_hex = print_args->print_hex;
+
+    timeinfo = localtime(&t);
+    strftime(ts, 30, "%Y-%m-%d", timeinfo);
+
+    fprintf(fh, "  \"_index\": \"packets-%s\",\n", ts);
+    fputs("  \"_type\": \"pcap_file\",\n", fh);
+    fputs("  \"_score\": null,\n", fh);
+    fputs("  \"_source\": {\n", fh);
+    fputs("    \"layers\": {\n", fh);
+
+    proto_tree_children_foreach(edt->tree, proto_tree_write_node_json,
+                                &data);
+
+    fputs("    }\n", fh);
+
+    fputs("  },\n", fh);
+
+}
+
+void
+write_ek_proto_tree(print_args_t *print_args, gchar *jsonfilter, epan_dissect_t *edt, FILE *fh)
+{
+    write_json_data data;
+    char ts[30];
+    time_t t = time(NULL);
+    struct tm  *timeinfo;
+    nstime_t   *timestamp;
+    GPtrArray  *finfo_array;
+
+    /* Create the output */
+    data.level    = 0;
+    data.fh       = fh;
+    data.src_list = edt->pi.data_src;
+    data.edt      = edt;
+    data.filter   = jsonfilter;
+    data.print_hex = print_args->print_hex;
+
+
+    timeinfo = localtime(&t);
+    strftime(ts, 30, "%Y-%m-%d", timeinfo);
+
+
+    /* Get frame protocol's finfo. */
+    finfo_array = proto_find_finfo(edt->tree, proto_frame);
+    if (g_ptr_array_len(finfo_array) < 1) {
+        return;
+    }
+    /* frame.time --> geninfo.timestamp */
+    finfo_array = proto_find_finfo(edt->tree, hf_frame_arrival_time);
+    if (g_ptr_array_len(finfo_array) < 1) {
+        return;
+    }
+    timestamp = (nstime_t *)fvalue_get(&((field_info*)finfo_array->pdata[0])->value);
+    g_ptr_array_free(finfo_array, TRUE);
+
+
+    fprintf(fh, "{\"index\" : {\"_index\": \"packets-%s\", \"_type\": \"pcap_file\", \"_score\": null}}\n", ts);
+    /* Timestamp added for time indexing in Elasticsearch */
+    fprintf(fh, "{\"timestamp\" : \"%ld%03d\", \"layers\" : {", timestamp->secs, timestamp->nsecs/1000000);
+
+
+    proto_tree_children_foreach(edt->tree, proto_tree_write_node_ek,
+                                &data);
+    fputs("}}\n", fh);
+}
+
 /* Write out a tree's data, and any child nodes, as PDML */
 static void
 proto_tree_write_node_pdml(proto_node *node, gpointer data)
@@ -340,7 +441,6 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
     /* Uninterpreted data, i.e., the "Data" protocol, is
      * printed as a field instead of a protocol. */
     else if (fi->hfinfo->id == proto_data) {
-
         /* Write out field with data */
         fputs("<field name=\"data\" value=\"", pdata->fh);
         pdml_write_field_hex_value(pdata, fi);
@@ -511,6 +611,403 @@ proto_tree_write_node_pdml(proto_node *node, gpointer data)
     }
 }
 
+
+/* Write out a tree's data, and any child nodes, as JSON */
+static void
+proto_tree_write_node_json(proto_node *node, gpointer data)
+{
+    field_info      *fi    = PNODE_FINFO(node);
+    write_json_data *pdata = (write_json_data*) data;
+    const gchar     *label_ptr;
+    char            *dfilter_string;
+    int              i;
+
+    /* dissection with an invisible proto tree? */
+    g_assert(fi);
+
+    /* Indent to the correct level */
+    for (i = -3; i < pdata->level; i++) {
+        fputs("  ", pdata->fh);
+    }
+
+    /* Text label. It's printed as a field with no name. */
+    if (fi->hfinfo->id == hf_text_only) {
+        /* Get the text */
+        if (fi->rep) {
+            label_ptr = fi->rep->representation;
+        }
+        else {
+            label_ptr = "";
+        }
+
+        /* Show empty name since it is a required field */
+        fputs("\"", pdata->fh);
+        print_escaped_json(pdata->fh, label_ptr);
+
+        if (node->first_child != NULL) {
+            fputs("\": {\n", pdata->fh);
+        }
+        else {
+            if (node->next == NULL) {
+              fputs("\": \"\"\n",  pdata->fh);
+            } else {
+              fputs("\": \"\",\n",  pdata->fh);
+            }
+        }
+    }
+
+    /* Normal protocols and fields */
+    else {
+        /*
+         * Hex dump -x
+         */
+        if (pdata->print_hex && fi->length > 0) {
+            fputs("\"", pdata->fh);
+            print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
+            fputs("_raw", pdata->fh);
+            fputs("\": \"", pdata->fh);
+
+            if (fi->hfinfo->bitmask!=0) {
+                switch (fi->value.ftype->ftype) {
+                    case FT_INT8:
+                    case FT_INT16:
+                    case FT_INT24:
+                    case FT_INT32:
+                        fprintf(pdata->fh, "%X", (guint) fvalue_get_sinteger(&fi->value));
+                        break;
+                    case FT_UINT8:
+                    case FT_UINT16:
+                    case FT_UINT24:
+                    case FT_UINT32:
+                        fprintf(pdata->fh, "%X", fvalue_get_uinteger(&fi->value));
+                        break;
+                    case FT_INT40:
+                    case FT_INT48:
+                    case FT_INT56:
+                    case FT_INT64:
+                        fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_sinteger64(&fi->value));
+                        break;
+                    case FT_UINT40:
+                    case FT_UINT48:
+                    case FT_UINT56:
+                    case FT_UINT64:
+                    case FT_BOOLEAN:
+                        fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_uinteger64(&fi->value));
+                        break;
+                    default:
+                        g_assert_not_reached();
+                }
+                fputs("\",\n", pdata->fh);
+            }
+            else {
+                json_write_field_hex_value(pdata, fi);
+                fputs("\",\n", pdata->fh);
+            }
+
+            /* Indent to the correct level */
+            for (i = -3; i < pdata->level; i++) {
+                fputs("  ", pdata->fh);
+            }
+        }
+
+
+        fputs("\"", pdata->fh);
+
+        print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
+
+        /* show, value, and unmaskedvalue attributes */
+        switch (fi->hfinfo->type)
+        {
+        case FT_PROTOCOL:
+            if (node->first_child != NULL) {
+                fputs("\": {\n", pdata->fh);
+            }
+            break;
+        case FT_NONE:
+            if (node->first_child != NULL) {
+                fputs("\": {\n", pdata->fh);
+            } else {
+                if (node->next == NULL) {
+                  fputs("\": \"\"\n",  pdata->fh);
+                } else {
+                  fputs("\": \"\",\n",  pdata->fh);
+                }
+            }
+            break;
+        default:
+            dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
+            if (dfilter_string != NULL) {
+              if (node->first_child == NULL) {
+                fputs("\": \"", pdata->fh);
+                print_escaped_json(pdata->fh, dfilter_string);
+              } else {
+                fputs("\": {\n", pdata->fh);
+              }
+            }
+            wmem_free(NULL, dfilter_string);
+
+            if (node->first_child == NULL) {
+              if (node->next == NULL) {
+                  fputs("\"\n", pdata->fh);
+              } else {
+                  fputs("\",\n", pdata->fh);
+              }
+            }
+        }
+
+    }
+
+    /* We print some levels for JSON. Recurse here. */
+    if (node->first_child != NULL) {
+        if (pdata->filter != NULL) {
+          if(strstr(pdata->filter, fi->hfinfo->abbrev) != NULL) {
+            pdata->level++;
+            proto_tree_children_foreach(node,
+                                        proto_tree_write_node_json, pdata);
+            pdata->level--;
+          }
+        } else {
+            pdata->level++;
+            proto_tree_children_foreach(node,
+                                        proto_tree_write_node_json, pdata);
+            pdata->level--;
+        }
+    }
+
+    if (node->first_child != NULL) {
+        /* Indent to correct level */
+        for (i = -3; i < pdata->level; i++) {
+            fputs("  ", pdata->fh);
+        }
+        /* Close off current element */
+        if (node->next == NULL) {
+            fputs("}\n", pdata->fh);
+        } else {
+            fputs("},\n", pdata->fh);
+        }
+    }
+}
+
+/* Write out a tree's data, and any child nodes, as JSON for EK */
+static void
+proto_tree_write_node_ek(proto_node *node, gpointer data)
+{
+    field_info      *fi    = PNODE_FINFO(node);
+    field_info      *fi_parent    = PNODE_FINFO(node->parent);
+    write_json_data *pdata = (write_json_data*) data;
+    const gchar     *label_ptr;
+    char            *dfilter_string;
+    int              i;
+    gchar           *abbrev_escaped = NULL;
+    size_t           abbrev_escaped_len = 0;
+
+    /* dissection with an invisible proto tree? */
+    g_assert(fi);
+
+    /* Text label. It's printed as a field with no name. */
+    if (fi->hfinfo->id == hf_text_only) {
+        /* Get the text */
+        if (fi->rep) {
+            label_ptr = fi->rep->representation;
+        }
+        else {
+            label_ptr = "";
+        }
+
+        /* Show empty name since it is a required field */
+        fputs("\"", pdata->fh);
+        if (fi_parent != NULL) {
+            print_escaped_ek(pdata->fh, fi_parent->hfinfo->abbrev);
+            fputs("_", pdata->fh);
+        }
+        print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+
+        if (node->first_child != NULL) {
+            fputs("\": \"", pdata->fh);
+            print_escaped_json(pdata->fh, label_ptr);
+            fputs("\",", pdata->fh);
+
+        }
+        else {
+            if (node->next == NULL) {
+              fputs("\": \"",  pdata->fh);
+              print_escaped_json(pdata->fh, label_ptr);
+              fputs("\"", pdata->fh);
+            } else {
+              fputs("\": \"",  pdata->fh);
+              print_escaped_json(pdata->fh, label_ptr);
+               fputs("\",", pdata->fh);
+            }
+        }
+    }
+
+    /* Normal protocols and fields */
+    else {
+        /*
+         * Hex dump -x
+         */
+        if (pdata->print_hex && fi->length > 0) {
+            fputs("\"", pdata->fh);
+            if (fi_parent != NULL) {
+                print_escaped_ek(pdata->fh, fi_parent->hfinfo->abbrev);
+                fputs("_", pdata->fh);
+            }
+            print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+            fputs("_raw", pdata->fh);
+            fputs("\": \"", pdata->fh);
+
+            if (fi->hfinfo->bitmask!=0) {
+                switch (fi->value.ftype->ftype) {
+                    case FT_INT8:
+                    case FT_INT16:
+                    case FT_INT24:
+                    case FT_INT32:
+                        fprintf(pdata->fh, "%X", (guint) fvalue_get_sinteger(&fi->value));
+                        break;
+                    case FT_UINT8:
+                    case FT_UINT16:
+                    case FT_UINT24:
+                    case FT_UINT32:
+                        fprintf(pdata->fh, "%X", fvalue_get_uinteger(&fi->value));
+                        break;
+                    case FT_INT40:
+                    case FT_INT48:
+                    case FT_INT56:
+                    case FT_INT64:
+                        fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_sinteger64(&fi->value));
+                        break;
+                    case FT_UINT40:
+                    case FT_UINT48:
+                    case FT_UINT56:
+                    case FT_UINT64:
+                    case FT_BOOLEAN:
+                        fprintf(pdata->fh, "%" G_GINT64_MODIFIER "X", fvalue_get_uinteger64(&fi->value));
+                        break;
+                    default:
+                        g_assert_not_reached();
+                }
+                fputs("\",", pdata->fh);
+            }
+            else {
+                json_write_field_hex_value(pdata, fi);
+                fputs("\",", pdata->fh);
+            }
+        }
+
+
+
+        fputs("\"", pdata->fh);
+
+        if (fi_parent != NULL) {
+            print_escaped_ek(pdata->fh, fi_parent->hfinfo->abbrev);
+            fputs("_", pdata->fh);
+        }
+        print_escaped_ek(pdata->fh, fi->hfinfo->abbrev);
+
+        /* show, value, and unmaskedvalue attributes */
+        switch (fi->hfinfo->type)
+        {
+        case FT_PROTOCOL:
+            if (node->first_child != NULL) {
+                fputs("\": {", pdata->fh);
+            }
+            break;
+        case FT_NONE:
+            if (node->first_child != NULL) {
+                fputs("\": \"\",",  pdata->fh);
+            } else {
+                if (node->next == NULL) {
+                  fputs("\": \"\"",  pdata->fh);
+                } else {
+                  fputs("\": \"\",",  pdata->fh);
+                }
+            }
+            break;
+        default:
+            dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
+            if (dfilter_string != NULL) {
+              if (node->first_child == NULL) {
+                fputs("\": \"", pdata->fh);
+                print_escaped_json(pdata->fh, dfilter_string);
+              } else {
+                  fputs("\": \"\",", pdata->fh);
+              }
+            }
+            wmem_free(NULL, dfilter_string);
+
+            if (node->first_child == NULL) {
+              if (node->next == NULL) {
+                  fputs("\"", pdata->fh);
+              } else {
+                  fputs("\",", pdata->fh);
+              }
+            }
+        }
+
+    }
+
+    /* We print some levels for JSON. Recurse here. */
+    if (node->first_child != NULL) {
+
+        if (pdata->filter != NULL) {
+
+          /* to to thread the '.' and '_' equally. The '.' is replace by print_escaped_ek for '_' */
+          if (fi->hfinfo->abbrev != NULL) {
+            abbrev_escaped_len = strlen(fi->hfinfo->abbrev) + 1;
+            if (abbrev_escaped_len > 0) {
+                abbrev_escaped = g_strdup(fi->hfinfo->abbrev);
+
+                i = 0;
+                while(abbrev_escaped[i]!='\0') {
+                   if(abbrev_escaped[i]=='.')
+                   {
+                       abbrev_escaped[i]='_';
+                   }
+                   i++;
+                 }
+            }
+          }
+
+          if((strstr(pdata->filter, fi->hfinfo->abbrev) != NULL) || (strstr(pdata->filter, abbrev_escaped) != NULL)) {
+            pdata->level++;
+            proto_tree_children_foreach(node,
+                                        proto_tree_write_node_ek, pdata);
+            pdata->level--;
+          } else {
+              /* print dummy field */
+              fputs("\"filtered\": \"\"", pdata->fh);
+          }
+
+          /* release abbrev_escaped string */
+          if (abbrev_escaped != NULL) {
+              abbrev_escaped_len = 0;
+              g_free(abbrev_escaped);
+          }
+
+        } else {
+            pdata->level++;
+            proto_tree_children_foreach(node,
+                                        proto_tree_write_node_ek, pdata);
+            pdata->level--;
+        }
+    }
+
+    if (node->first_child != NULL) {
+      if (fi->hfinfo->type == FT_PROTOCOL) {
+        /* Close off current element */
+          if (node->next == NULL) {
+              fputs("}", pdata->fh);
+          } else {
+              fputs("},", pdata->fh);
+          }
+      } else {
+          if (node->next != NULL) {
+              fputs(",", pdata->fh);
+          }
+      }
+    }
+}
+
 /* Print info for a 'geninfo' pseudo-protocol. This is required by
  * the PDML spec. The information is contained in Wireshark's 'frame' protocol,
  * but we produce a 'geninfo' protocol in the PDML to conform to spec.
@@ -604,6 +1101,13 @@ write_pdml_finale(FILE *fh)
     fputs("</pdml>\n", fh);
 }
 
+void
+write_json_finale(FILE *fh)
+{
+    fputs("}\n", fh);
+
+}
+
 void
 write_psml_preamble(column_info *cinfo, FILE *fh)
 {
@@ -843,6 +1347,57 @@ print_escaped_xml(FILE *fh, const char *unescaped_string)
     }
 }
 
+/* Print a string, escaping out certain characters that need to
+ * escaped out for JSON. */
+static void
+print_escaped_json(FILE *fh, const char *unescaped_string)
+{
+    const char *p;
+    char        temp_str[8];
+
+    for (p = unescaped_string; *p != '\0'; p++) {
+        switch (*p) {
+        case '"':
+            fputs("&quot;", fh);
+            break;
+        default:
+            if (g_ascii_isprint(*p))
+                fputc(*p, fh);
+            else {
+                g_snprintf(temp_str, sizeof(temp_str), "%x", (guint8)*p);
+                fputs(temp_str, fh);
+            }
+        }
+    }
+}
+
+/* Print a string, escaping out certain characters that need to
+ * escaped out for Elasticsearch title. */
+static void
+print_escaped_ek(FILE *fh, const char *unescaped_string)
+{
+    const char *p;
+    char        temp_str[8];
+
+    for (p = unescaped_string; *p != '\0'; p++) {
+        switch (*p) {
+        case '"':
+                    fputs("&quot;", fh);
+                    break;
+        case '.':
+            fputs("_", fh);
+            break;
+        default:
+            if (g_ascii_isprint(*p))
+                fputc(*p, fh);
+            else {
+                g_snprintf(temp_str, sizeof(temp_str), "\\x%x", (guint8)*p);
+                fputs(temp_str, fh);
+            }
+        }
+    }
+}
+
 static void
 pdml_write_field_hex_value(write_pdml_data *pdata, field_info *fi)
 {
@@ -868,6 +1423,31 @@ pdml_write_field_hex_value(write_pdml_data *pdata, field_info *fi)
     }
 }
 
+static void
+json_write_field_hex_value(write_json_data *pdata, field_info *fi)
+{
+    int           i;
+    const guint8 *pd;
+
+    if (!fi->ds_tvb)
+        return;
+
+    if (fi->length > tvb_captured_length_remaining(fi->ds_tvb, fi->start)) {
+        fprintf(pdata->fh, "field length invalid!");
+        return;
+    }
+
+    /* Find the data for this field. */
+    pd = get_field_data(pdata->src_list, fi);
+
+    if (pd) {
+        /* Print a simple hex dump */
+        for (i = 0 ; i < fi->length; i++) {
+            fprintf(pdata->fh, "%02x", pd[i]);
+        }
+    }
+}
+
 gboolean
 print_hex_data(print_stream_t *stream, epan_dissect_t *edt)
 {
index 8ee35e04ba59c4c05bd965fad7c4aa5876d2b150..b152ac5f7abd63f5b0e19274ce5c280eba33bc46 100644 (file)
@@ -108,6 +108,12 @@ WS_DLL_PUBLIC void write_pdml_preamble(FILE *fh, const gchar* filename);
 WS_DLL_PUBLIC void write_pdml_proto_tree(epan_dissect_t *edt, FILE *fh);
 WS_DLL_PUBLIC void write_pdml_finale(FILE *fh);
 
+WS_DLL_PUBLIC void write_json_preamble(FILE *fh);
+WS_DLL_PUBLIC void write_json_proto_tree(print_args_t *print_args, gchar *jsonfilter, epan_dissect_t *edt, FILE *fh);
+WS_DLL_PUBLIC void write_json_finale(FILE *fh);
+
+WS_DLL_PUBLIC void write_ek_proto_tree(print_args_t *print_args, gchar *jsonfilter, epan_dissect_t *edt, FILE *fh);
+
 WS_DLL_PUBLIC void write_psml_preamble(column_info *cinfo, FILE *fh);
 WS_DLL_PUBLIC void write_psml_columns(epan_dissect_t *edt, FILE *fh);
 WS_DLL_PUBLIC void write_psml_finale(FILE *fh);
index 96f15556784c6f2b527003c9ee4f61383c7c2ad0..af38ab0702c6daf46d94486c2ede909474ba33fa 100644 (file)
--- a/tshark.c
+++ b/tshark.c
@@ -158,7 +158,9 @@ static gboolean perform_two_pass_analysis;
 typedef enum {
   WRITE_TEXT,   /* summary or detail text */
   WRITE_XML,    /* PDML or PSML */
-  WRITE_FIELDS  /* User defined list of fields */
+  WRITE_FIELDS, /* User defined list of fields */
+  WRITE_JSON,    /* JSON */
+  WRITE_EK      /* JSON bulk insert to Elasticsearch */
   /* Add CSV and the like here */
 } output_action_e;
 
@@ -175,6 +177,7 @@ static print_format_e print_format = PR_FMT_TEXT;
 static print_stream_t *print_stream;
 
 static output_fields_t* output_fields  = NULL;
+static gchar *jsonfilter = NULL;
 
 /* The line separator used between packets, changeable via the -S option */
 static const char *separator = "";
@@ -372,8 +375,10 @@ print_usage(FILE *output)
   fprintf(output, "  -P                       print packet summary even when writing to a file\n");
   fprintf(output, "  -S <separator>           the line separator to print between packets\n");
   fprintf(output, "  -x                       add output of hex and ASCII dump (Packet Bytes)\n");
-  fprintf(output, "  -T pdml|ps|psml|text|fields\n");
+  fprintf(output, "  -T pdml|ps|psml|json|ek|text|fields\n");
   fprintf(output, "                           format of text output (def: text)\n");
+  fprintf(output, "  -j <jsonfilter>          only protocols layers to include if -Tjson, -Tek selected,\n");
+  fprintf(output, "                           (e.g. \"http tcp ip\",\n");
   fprintf(output, "  -e <field>               field to print if -Tfields selected (e.g. tcp.port,\n");
   fprintf(output, "                           _ws.col.Info)\n");
   fprintf(output, "                           this option can be repeated to print multiple fields\n");
@@ -1020,7 +1025,7 @@ main(int argc, char *argv[])
  * We do *not* use a leading - because the behavior of a leading - is
  * platform-dependent.
  */
-#define OPTSTRING "+2" OPTSTRING_CAPTURE_COMMON "C:d:e:E:F:gG:hH:" "K:lnN:o:O:PqQr:R:S:t:T:u:U:vVw:W:xX:Y:z:"
+#define OPTSTRING "+2" OPTSTRING_CAPTURE_COMMON "C:d:e:E:F:gG:hH:j:" "K:lnN:o:O:PqQr:R:S:t:T:u:U:vVw:W:xX:Y:z:"
 
   static const char    optstring[] = OPTSTRING;
 
@@ -1479,6 +1484,9 @@ main(int argc, char *argv[])
         return 1;
       }
       break;
+    case 'j':
+      jsonfilter = optarg;
+      break;
     case 'W':        /* Select extra information to save in our capture file */
       /* This is patterned after the -N flag which may not be the best idea. */
       if (strchr(optarg, 'n')) {
@@ -1639,7 +1647,16 @@ main(int argc, char *argv[])
         output_action = WRITE_FIELDS;
         print_details = TRUE;   /* Need full tree info */
         print_summary = FALSE;  /* Don't allow summary */
-      } else {
+      } else if (strcmp(optarg, "json") == 0) {
+        output_action = WRITE_JSON;
+        print_details = TRUE;   /* Need details */
+        print_summary = FALSE;  /* Don't allow summary */
+      } else if (strcmp(optarg, "ek") == 0) {
+        output_action = WRITE_EK;
+        print_details = TRUE;   /* Need details */
+        print_summary = FALSE;  /* Don't allow summary */
+      }
+      else {
         cmdarg_err("Invalid -T parameter \"%s\"; it must be one of:", optarg);                   /* x */
         cmdarg_err_cont("\t\"fields\" The values of fields specified with the -e option, in a form\n"
                         "\t         specified by the -E option.\n"
@@ -1653,6 +1670,12 @@ main(int argc, char *argv[])
                         "\t         summary information of a decoded packet. This information is\n"
                         "\t         equivalent to the information shown in the one-line summary\n"
                         "\t         printed by default.\n"
+                        "\t\"json\"   Packet Summary, an JSON-based format for the details\n"
+                        "\t         summary information of a decoded packet. This information is \n"
+                        "\t         equivalent to the packet details printed with the -V flag.\n"
+                        "\t\"ek\"   Packet Summary, an EK JSON-based format for the bulk insert \n"
+                        "\t         into elastic search cluster. This information is \n"
+                        "\t         equivalent to the packet details printed with the -V flag.\n"
                         "\t\"text\"   Text of a human-readable one-line summary of each of the\n"
                         "\t         packets, or a multi-line view of the details of each of the\n"
                         "\t         packets, depending on whether the -V flag was specified.\n"
@@ -1844,8 +1867,8 @@ main(int argc, char *argv[])
   }
 
   if (print_hex) {
-    if (output_action != WRITE_TEXT) {
-      cmdarg_err("Raw packet hex data can only be printed as text or PostScript");
+    if (output_action != WRITE_TEXT && output_action != WRITE_JSON && output_action != WRITE_EK) {
+      cmdarg_err("Raw packet hex data can only be printed as text, PostScript, JSON or EK JSON");
       return 1;
     }
   }
@@ -3883,6 +3906,13 @@ write_preamble(capture_file *cf)
     write_fields_preamble(output_fields, stdout);
     return !ferror(stdout);
 
+  case WRITE_JSON:
+    write_json_preamble(stdout);
+    return !ferror(stdout);
+
+  case WRITE_EK:
+    return !ferror(stdout);
+
   default:
     g_assert_not_reached();
     return FALSE;
@@ -4186,6 +4216,8 @@ print_packet(capture_file *cf, epan_dissect_t *edt)
         write_psml_columns(edt, stdout);
         return !ferror(stdout);
       case WRITE_FIELDS: /*No non-verbose "fields" format */
+      case WRITE_JSON:
+      case WRITE_EK:
         g_assert_not_reached();
         break;
       }
@@ -4224,6 +4256,16 @@ print_packet(capture_file *cf, epan_dissect_t *edt)
       write_fields_proto_tree(output_fields, edt, &cf->cinfo, stdout);
       printf("\n");
       return !ferror(stdout);
+    case WRITE_JSON:
+      print_args.print_hex = print_hex;
+      write_json_proto_tree(&print_args, jsonfilter, edt, stdout);
+      printf("\n");
+      return !ferror(stdout);
+    case WRITE_EK:
+      print_args.print_hex = print_hex;
+      write_ek_proto_tree(&print_args, jsonfilter, edt, stdout);
+      printf("\n");
+      return !ferror(stdout);
     }
   }
   if (print_hex) {
@@ -4258,6 +4300,13 @@ write_finale(void)
     write_fields_finale(output_fields, stdout);
     return !ferror(stdout);
 
+  case WRITE_JSON:
+    write_json_finale(stdout);
+    return !ferror(stdout);
+
+  case WRITE_EK:
+    return !ferror(stdout);
+
   default:
     g_assert_not_reached();
     return FALSE;