wiretap: Fix rejecting non-EMS files

Don't cast the return value of fgetc and similar functions to a
char before checking if the return value was EOF. This can cause
false positives on a platform with signed char, and false negatives
on a platform with unsigned char.

If file_peekc or file_getc returns EOF, check to see if we're actually
at the end of the file. If so, that's an indication that the file isn't
our file type (because we skipped to the end), but it's not a read
error; it might still be some other file type.

Prevents wiretap messages of the sort "The file could not be opened:
Success."

Fixup cea4faac394a1cfb56772f1889ae642b288c66fb

diff --git a/wiretap/ems.c b/wiretap/ems.c
index 8e7b307722ad80d39512d3f0abd6a1a8b99d6054..4035f42f798297888be0f78eea88af300ace4c96 100644 (file)
--- a/wiretap/ems.c
+++ b/wiretap/ems.c
@@ -48,7 +48,7 @@ static int ems_file_type_subtype = -1;
  * Gets one character and returns in case of error.
  * Without error, peeks at next character and returns it.
  */
-static char get_and_peek(FILE_T fh) {
+static int get_and_peek(FILE_T fh) {
     int c;
 
     c = file_getc(fh);
@@ -65,7 +65,7 @@ static char get_and_peek(FILE_T fh) {
  * Skips whitespace at the beginning of a line, comment lines, and empty
  * lines.
  */
-static char peek_relevant_character(FILE_T fh) {
+static int peek_relevant_character(FILE_T fh) {
     int c;
 
     while (true) {
@@ -157,6 +157,9 @@ wtap_open_return_val ems_open(wtap *wth, int *err, gchar **err_info) {
     // skip irrelevant characters
     c = peek_relevant_character(wth->fh);
     if (c < 0) {
+        if (file_eof(wth->fh)) {
+            return WTAP_OPEN_NOT_MINE;
+        }
         *err = file_error(wth->fh, err_info);
         return WTAP_OPEN_ERROR;
     }