diff --git a/camel-core/src/main/java/org/apache/camel/model/dataformat/CsvDataFormat.java b/camel-core/src/main/java/org/apache/camel/model/dataformat/CsvDataFormat.java index cfd4f6bada435..b6fdae09ba340 100644 --- a/camel-core/src/main/java/org/apache/camel/model/dataformat/CsvDataFormat.java +++ b/camel-core/src/main/java/org/apache/camel/model/dataformat/CsvDataFormat.java @@ -31,7 +31,7 @@ /** * Represents a CSV (Comma Separated Values) {@link org.apache.camel.spi.DataFormat} * - * @version + * @version */ @XmlRootElement(name = "csv") @XmlAccessorType(XmlAccessType.FIELD) @@ -48,6 +48,8 @@ public class CsvDataFormat extends DataFormatDefinition { private Boolean skipFirstLine; @XmlAttribute private Boolean lazyLoad; + @XmlAttribute + private Boolean useMaps; public CsvDataFormat() { super("csv"); @@ -111,6 +113,14 @@ public void setLazyLoad(Boolean lazyLoad) { this.lazyLoad = lazyLoad; } + public Boolean getUseMaps() { + return useMaps; + } + + public void setUseMaps(Boolean useMaps) { + this.useMaps = useMaps; + } + @Override protected DataFormat createDataFormat(RouteContext routeContext) { DataFormat csvFormat = super.createDataFormat(routeContext); @@ -150,5 +160,9 @@ protected void configureDataFormat(DataFormat dataFormat, CamelContext camelCont if (lazyLoad != null) { setProperty(camelContext, dataFormat, "lazyLoad", lazyLoad); } + + if (useMaps != null) { + setProperty(camelContext, dataFormat, "useMaps", useMaps); + } } } diff --git a/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvDataFormat.java b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvDataFormat.java index 649bc81e84559..848ac49775014 100644 --- a/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvDataFormat.java +++ b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvDataFormat.java @@ -46,7 +46,7 @@ * Autogeneration can be disabled. In this case, only the fields defined in * csvConfig are written on the output. * - * @version + * @version */ public class CsvDataFormat implements DataFormat { private CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY; @@ -58,6 +58,7 @@ public class CsvDataFormat implements DataFormat { * Lazy row loading with iterator for big files. */ private boolean lazyLoad; + private boolean useMaps; public void marshal(Exchange exchange, Object object, OutputStream outputStream) throws Exception { if (delimiter != null) { @@ -105,12 +106,18 @@ public Object unmarshal(Exchange exchange, InputStream inputStream) throws Excep reader = IOHelper.buffered(new InputStreamReader(inputStream, IOHelper.getCharsetName(exchange))); CSVParser parser = new CSVParser(reader, strategy); - if (skipFirstLine) { - // read one line ahead and skip it - parser.getLine(); + CsvLineConverter lineConverter; + if (useMaps) { + lineConverter = CsvLineConverters.getMapLineConverter(parser.getLine()); + } else { + lineConverter = CsvLineConverters.getListConverter(); + if (skipFirstLine) { + // read one line ahead and skip it + parser.getLine(); + } } - CsvIterator csvIterator = new CsvIterator(parser, reader); + @SuppressWarnings("unchecked") CsvIterator csvIterator = new CsvIterator(parser, reader, lineConverter); return lazyLoad ? csvIterator : loadAllAsList(csvIterator); } catch (Exception e) { error = true; @@ -122,9 +129,9 @@ public Object unmarshal(Exchange exchange, InputStream inputStream) throws Excep } } - private List> loadAllAsList(CsvIterator iter) { + private List loadAllAsList(CsvIterator iter) { try { - List> list = new ArrayList>(); + List list = new ArrayList(); while (iter.hasNext()) { list.add(iter.next()); } @@ -145,7 +152,7 @@ public void setDelimiter(String delimiter) { } this.delimiter = delimiter; } - + public CSVConfig getConfig() { return config; } @@ -191,6 +198,20 @@ public void setLazyLoad(boolean lazyLoad) { this.lazyLoad = lazyLoad; } + public boolean isUseMaps() { + return useMaps; + } + + /** + * Sets whether or not the result of the unmarshalling should be a {@code java.util.Map} instead of a {@code java.util.List}. It uses the first line as a + * header line and uses it as keys of the maps. + * + * @param useMaps {@code true} in order to use {@code java.util.Map} instead of {@code java.util.List}, {@code false} otherwise. + */ + public void setUseMaps(boolean useMaps) { + this.useMaps = useMaps; + } + private synchronized void updateFieldsInConfig(Set set, Exchange exchange) { for (Object value : set) { if (value != null) { @@ -203,5 +224,4 @@ private synchronized void updateFieldsInConfig(Set set, Exchange exchange) { } } } - -} \ No newline at end of file +} diff --git a/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvIterator.java b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvIterator.java index fece08271cd16..9d26edba38579 100644 --- a/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvIterator.java +++ b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvIterator.java @@ -20,9 +20,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.Reader; -import java.util.Arrays; import java.util.Iterator; -import java.util.List; import java.util.NoSuchElementException; import org.apache.camel.util.IOHelper; @@ -30,15 +28,17 @@ /** */ -public class CsvIterator implements Iterator>, Closeable { +public class CsvIterator implements Iterator, Closeable { private final CSVParser parser; private final Reader reader; + private final CsvLineConverter lineConverter; private String[] line; - public CsvIterator(CSVParser parser, Reader reader) throws IOException { + public CsvIterator(CSVParser parser, Reader reader, CsvLineConverter lineConverter) throws IOException { this.parser = parser; this.reader = reader; + this.lineConverter = lineConverter; line = parser.getLine(); } @@ -48,11 +48,11 @@ public boolean hasNext() { } @Override - public List next() { + public T next() { if (!hasNext()) { throw new NoSuchElementException(); } - List result = Arrays.asList(line); + T result = lineConverter.convertLine(line); try { line = parser.getLine(); } catch (IOException e) { diff --git a/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvLineConverter.java b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvLineConverter.java new file mode 100644 index 0000000000000..7f91636c80cc8 --- /dev/null +++ b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvLineConverter.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.csv; + +/** + * This interface helps converting a single CSV line into another representation. + * + * @param Class for representing a single line + */ +public interface CsvLineConverter { + /** + * Converts a single CSV line. + * + * @param line CSV line + * @return Another representation of the CSV line + */ + public T convertLine(String[] line); +} diff --git a/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvLineConverters.java b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvLineConverters.java new file mode 100644 index 0000000000000..1f3ad30096135 --- /dev/null +++ b/components/camel-csv/src/main/java/org/apache/camel/dataformat/csv/CsvLineConverters.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.csv; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * This {@code CsvLineConverters} class provides common implementations of the {@code CsvLineConverter} interface. + */ +public final class CsvLineConverters { + /** + * Provides an implementation of {@code CsvLineConverter} that converts a line into a {@code List}. + * + * @return List-based {@code CsvLineConverter} implementation + */ + public static CsvLineConverter> getListConverter() { + return ListLineConverter.SINGLETON; + } + + /** + * Provides an implementation of {@code CsvLineConverter} that converts a line into a {@code Map}. + *

+ * It requires to have unique {@code headers} values as well as the same number of item in each line. + * + * @param headers Headers of the CSV file + * @return Map-based {@code CsvLineConverter} implementation + */ + public static CsvLineConverter> getMapLineConverter(String[] headers) { + return new MapLineConverter(headers); + } + + private static class ListLineConverter implements CsvLineConverter> { + public static final ListLineConverter SINGLETON = new ListLineConverter(); + + @Override + public List convertLine(String[] line) { + return Arrays.asList(line); + } + } + + private static class MapLineConverter implements CsvLineConverter> { + private final String[] headers; + + private MapLineConverter(String[] headers) { + this.headers = checkHeaders(headers); + } + + @Override + public Map convertLine(String[] line) { + if (line.length != headers.length) { + throw new IllegalStateException("This line does not have the same number of items than the header"); + } + + Map result = new HashMap(line.length); + for (int i = 0; i < line.length; i++) { + result.put(headers[i], line[i]); + } + return result; + } + + private static String[] checkHeaders(String[] headers) { + // Check that we have headers + if (headers == null || headers.length == 0) { + throw new IllegalArgumentException("Missing headers for the CSV parsing"); + } + + // Check that there is no duplicates + Set headerSet = new HashSet(headers.length); + Collections.addAll(headerSet, headers); + if (headerSet.size() != headers.length) { + throw new IllegalArgumentException("There are duplicate headers"); + } + + return headers; + } + } + + private CsvLineConverters() { + // Prevent instantiation + } +} diff --git a/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvIteratorTest.java b/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvIteratorTest.java index 091faf65c2c88..cd5425ae182cb 100644 --- a/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvIteratorTest.java +++ b/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvIteratorTest.java @@ -19,29 +19,27 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; +import java.util.List; import java.util.NoSuchElementException; - -import mockit.Expectations; -import mockit.Injectable; import org.apache.commons.csv.CSVParser; import org.junit.Assert; import org.junit.Test; +import mockit.Expectations; +import mockit.Injectable; public class CsvIteratorTest { public static final String HDD_CRASH = "HDD crash"; @Test - public void closeIfError( - @Injectable final InputStreamReader reader, - @Injectable final CSVParser parser) throws IOException { + public void closeIfError(@Injectable final InputStreamReader reader, @Injectable final CSVParser parser) throws IOException { new Expectations() { { parser.getLine(); - result = new String[] {"1"}; + result = new String[]{"1"}; parser.getLine(); - result = new String[] {"2"}; + result = new String[]{"2"}; parser.getLine(); result = new IOException(HDD_CRASH); @@ -52,7 +50,7 @@ public void closeIfError( }; @SuppressWarnings("resource") - CsvIterator iterator = new CsvIterator(parser, reader); + CsvIterator> iterator = new CsvIterator>(parser, reader, CsvLineConverters.getListConverter()); Assert.assertTrue(iterator.hasNext()); Assert.assertEquals(Arrays.asList("1"), iterator.next()); Assert.assertTrue(iterator.hasNext()); @@ -75,15 +73,14 @@ public void closeIfError( } @Test - public void normalCycle(@Injectable final InputStreamReader reader, - @Injectable final CSVParser parser) throws IOException { + public void normalCycle(@Injectable final InputStreamReader reader, @Injectable final CSVParser parser) throws IOException { new Expectations() { { parser.getLine(); - result = new String[] {"1"}; + result = new String[]{"1"}; parser.getLine(); - result = new String[] {"2"}; + result = new String[]{"2"}; parser.getLine(); result = null; @@ -92,9 +89,9 @@ public void normalCycle(@Injectable final InputStreamReader reader, reader.close(); } }; - + @SuppressWarnings("resource") - CsvIterator iterator = new CsvIterator(parser, reader); + CsvIterator> iterator = new CsvIterator>(parser, reader, CsvLineConverters.getListConverter()); Assert.assertTrue(iterator.hasNext()); Assert.assertEquals(Arrays.asList("1"), iterator.next()); @@ -109,6 +106,5 @@ public void normalCycle(@Injectable final InputStreamReader reader, } catch (NoSuchElementException e) { // okay } - } } diff --git a/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvLineConvertersTest.java b/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvLineConvertersTest.java new file mode 100644 index 0000000000000..fcb2c4ff28714 --- /dev/null +++ b/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvLineConvertersTest.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.csv; + +import java.util.List; +import java.util.Map; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class CsvLineConvertersTest { + @Test + public void shouldConvertAsList() { + CsvLineConverter converter = CsvLineConverters.getListConverter(); + + Object result = converter.convertLine(new String[]{"foo", "bar"}); + + assertTrue(result instanceof List); + List list = (List) result; + assertEquals(2, list.size()); + assertEquals("foo", list.get(0)); + assertEquals("bar", list.get(1)); + } + + @Test + public void shouldConvertAsMap() { + CsvLineConverter converter = CsvLineConverters.getMapLineConverter(new String[]{"HEADER_1", "HEADER_2"}); + + Object result = converter.convertLine(new String[]{"foo", "bar"}); + + assertTrue(result instanceof Map); + Map map = (Map) result; + assertEquals(2, map.size()); + assertEquals("foo", map.get("HEADER_1")); + assertEquals("bar", map.get("HEADER_2")); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldNotConvertAsMapWithNullHeaders() { + CsvLineConverters.getMapLineConverter(null); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldNotConvertAsMapWithNoHeaders() { + CsvLineConverters.getMapLineConverter(new String[]{}); + } + + @Test(expected = IllegalStateException.class) + public void shouldNotConvertAsMapWithInvalidLine() { + CsvLineConverter converter = CsvLineConverters.getMapLineConverter(new String[]{"HEADER_1", "HEADER_2"}); + + converter.convertLine(new String[]{"foo"}); + } +} diff --git a/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvUnmarshalMapLineTest.java b/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvUnmarshalMapLineTest.java new file mode 100644 index 0000000000000..5f02dc30a439e --- /dev/null +++ b/components/camel-csv/src/test/java/org/apache/camel/dataformat/csv/CsvUnmarshalMapLineTest.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.csv; + +import java.util.List; +import java.util.Map; +import org.apache.camel.EndpointInject; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.test.spring.CamelSpringTestSupport; +import org.junit.Test; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +/** + * Spring based test for the CsvDataFormat demonstrating the usage of + * the useMaps option. + */ +public class CsvUnmarshalMapLineTest extends CamelSpringTestSupport { + + @EndpointInject(uri = "mock:result") + private MockEndpoint result; + + @SuppressWarnings("unchecked") + @Test + public void testCsvUnMarshal() throws Exception { + result.expectedMessageCount(1); + + // the first line contains the column names which we intend to skip + template.sendBody("direct:start", "OrderId|Item|Amount\n123|Camel in Action|1\n124|ActiveMQ in Action|2"); + + assertMockEndpointsSatisfied(); + + List> body = result.getReceivedExchanges().get(0).getIn().getBody(List.class); + assertEquals(2, body.size()); + assertEquals("123", body.get(0).get("OrderId")); + assertEquals("Camel in Action", body.get(0).get("Item")); + assertEquals("1", body.get(0).get("Amount")); + assertEquals("124", body.get(1).get("OrderId")); + assertEquals("ActiveMQ in Action", body.get(1).get("Item")); + assertEquals("2", body.get(1).get("Amount")); + } + + @Test + public void testCsvUnMarshalNoLine() throws Exception { + result.expectedMessageCount(1); + + // the first and last line we intend to skip + template.sendBody("direct:start", "OrderId|Item|Amount\n"); + + assertMockEndpointsSatisfied(); + + List body = result.getReceivedExchanges().get(0).getIn().getBody(List.class); + assertEquals(0, body.size()); + } + + @Override + protected ClassPathXmlApplicationContext createApplicationContext() { + return new ClassPathXmlApplicationContext("org/apache/camel/dataformat/csv/CsvUnmarshalMapLineSpringTest-context.xml"); + } +} diff --git a/components/camel-csv/src/test/resources/org/apache/camel/dataformat/csv/CsvUnmarshalMapLineSpringTest-context.xml b/components/camel-csv/src/test/resources/org/apache/camel/dataformat/csv/CsvUnmarshalMapLineSpringTest-context.xml new file mode 100644 index 0000000000000..17e73259df6ec --- /dev/null +++ b/components/camel-csv/src/test/resources/org/apache/camel/dataformat/csv/CsvUnmarshalMapLineSpringTest-context.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + +