From e8d20ca26976428021d38d8de16716c876801238 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 17 May 2016 17:46:09 +0800 Subject: [PATCH] STORM-1845: use UTF-8 instead of default encoding --- .../src/main/java/org/apache/storm/hbase/common/Utils.java | 3 ++- .../storm/hdfs/trident/format/DelimitedRecordFormat.java | 4 +++- .../storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java | 3 ++- .../apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java | 3 ++- .../main/java/org/apache/storm/hive/common/HiveWriter.java | 3 ++- .../storm/redis/trident/state/AbstractRedisMapState.java | 3 ++- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java index 8efe09836e..df020ee265 100644 --- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java +++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; public class Utils { private static final Logger LOG = LoggerFactory.getLogger(Utils.class); @@ -44,7 +45,7 @@ public static byte[] toBytes(Object obj) { if(obj == null) { return null; } else if(obj instanceof String){ - return ((String)obj).getBytes(); + return ((String)obj).getBytes(StandardCharsets.UTF_8); } else if (obj instanceof Integer){ return Bytes.toBytes((Integer) obj); } else if (obj instanceof Long){ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java index a08664d0e4..95d0422438 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java @@ -20,6 +20,8 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.trident.tuple.TridentTuple; +import java.nio.charset.StandardCharsets; + /** * RecordFormat implementation that uses field and record delimiters. * By default uses a comma (",") as the field delimiter and a @@ -78,6 +80,6 @@ public byte[] format(TridentTuple tuple) { } } sb.append(this.recordDelimiter); - return sb.toString().getBytes(); + return sb.toString().getBytes(StandardCharsets.UTF_8); } } diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java index 958080f68b..1846b2a5a4 100644 --- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java +++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java @@ -34,6 +34,7 @@ import java.util.Date; import java.text.SimpleDateFormat; import java.io.IOException; +import java.nio.charset.StandardCharsets; public class DelimitedRecordHiveMapper implements HiveMapper { private static final Logger LOG = LoggerFactory.getLogger(DelimitedRecordHiveMapper.class); @@ -133,7 +134,7 @@ public byte[] mapRecord(TridentTuple tuple) { builder.append(fieldDelimiter); } } - return builder.toString().getBytes(); + return builder.toString().getBytes(StandardCharsets.UTF_8); } private String getPartitionsByTimeFormat() { diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java index a391fc69d9..b1ceaef57f 100644 --- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java +++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java @@ -36,6 +36,7 @@ import java.util.Date; import java.text.SimpleDateFormat; import java.io.IOException; +import java.nio.charset.StandardCharsets; public class JsonRecordHiveMapper implements HiveMapper { private static final Logger LOG = LoggerFactory.getLogger(DelimitedRecordHiveMapper.class); @@ -122,7 +123,7 @@ public byte[] mapRecord(TridentTuple tuple) { obj.put(field,tuple.getValueByField(field)); } } - return obj.toJSONString().getBytes(); + return obj.toJSONString().getBytes(StandardCharsets.UTF_8); } private String getPartitionsByTimeFormat() { diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java index 4df1c60c22..9ce24d44bd 100644 --- a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java +++ b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java @@ -27,6 +27,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.nio.charset.StandardCharsets; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.hcatalog.streaming.*; @@ -403,7 +404,7 @@ private byte[] generateRecord(Tuple tuple) { buf.append(o); buf.append(","); } - return buf.toString().getBytes(); + return buf.toString().getBytes(StandardCharsets.UTF_8); } /** diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java index f5bd459ee1..1d73ca2470 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.nio.charset.StandardCharsets; /** * AbstractRedisMapState is base class of any RedisMapState, which implements IBackingMap. @@ -99,7 +100,7 @@ private List deserializeValues(List> keys, List values) List result = new ArrayList(keys.size()); for (String value : values) { if (value != null) { - result.add((T) getSerializer().deserialize(value.getBytes())); + result.add((T) getSerializer().deserialize(value.getBytes(StandardCharsets.UTF_8))); } else { result.add(null); }