Log potential encoding issues (#8)

Also gracefully handle any encoding issues in request bodies, while
logging a warning.

Bump to v0.2.5.

Author: liujed

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    akita-har_logger (0.2.4)
+    akita-har_logger (0.2.5)
 
 GEM
   remote: https://rubygems.org/

lib/akita/har_logger/har_utils.rb

@@ -5,12 +5,20 @@ module HarLogger
     class HarUtils
       # Rack apparently uses 8-bit ASCII for everything, even when the string
       # is not 8-bit ASCII. This reinterprets 8-bit ASCII strings as UTF-8.
+      #
+      # If we are unable to do this reinterpretation, return the string
+      # unchanged, but log a warning that points to the caller.
       def self.fixEncoding(v)
-        if v == nil || v.encoding != Encoding::ASCII_8BIT then
-          v
-        else
-          String.new(v).force_encoding(Encoding::UTF_8)
+        if v != nil && v.encoding == Encoding::ASCII_8BIT then
+          forced = String.new(v).force_encoding(Encoding::UTF_8)
+          if forced.valid_encoding? then
+            v = forced
+          else
+            Rails.logger.warn "[#{caller_locations(1, 1)}] Unable to fix encoding: not a valid UTF-8 string. This will likely cause JSON serialization to fail."
+          end
         end
+
+        v
       end
 
       # Converts a Hash into a list of Hash objects. Each entry in the given

lib/akita/har_logger/http_request.rb

@@ -85,7 +85,7 @@ def getPostDataCharSet(env)
           return Encoding::ISO_8859_1
         end
 
-        Encoding.default_external
+        Encoding::ASCII_8BIT
       end
 
       # Obtains the posted data from an HTTP environment.
@@ -117,8 +117,27 @@ def getPostData(env)
             # body when the request specifies UTF-8. Reinterpret the content
             # body according to what the request says it is, and re-encode into
             # UTF-8.
-            result[:text] = req.body.string.encode(Encoding::UTF_8,
-                getPostDataCharSet(env))
+            #
+            # Gracefully handle any characters that are invalid in the source
+            # encoding and characters that have no UTF-8 representation by
+            # replacing with '?'. Log a warning when this happens.
+            source = req.body.string.force_encoding(getPostDataCharSet(env))
+            utf8EncodingSuccessful = false
+            if source.valid_encoding? then
+              begin
+                result[:text] = source.encode(Encoding::UTF_8)
+                utf8EncodingSuccessful = true
+              rescue Encoding::UndefinedConversionError
+                Rails.logger.warn "[#{caller_locations(0, 1)}] Unable to losslessly convert request body from #{source.encoding} to UTF-8. Characters undefined in UTF-8 will be replaced with '?'."
+              end
+            else
+              Rails.logger.warn "[#{caller_locations(0, 1)}] Request body is not valid #{source.encoding}. Invalid characters and characters undefined in UTF-8 will be replaced with '?'."
+            end
+
+            if !utf8EncodingSuccessful then
+              result[:text] = source.encode(Encoding::UTF_8,
+                  invalid: :replace, undef: :replace, replace: '?')
+            end
           end
 
           result

lib/akita/har_logger/version.rb

@@ -2,6 +2,6 @@
 
 module Akita
   module HarLogger
-    VERSION = "0.2.4"
+    VERSION = "0.2.5"
   end
 end