Camel Bindy Streaming Payload and Writing to File

Question

I have a route which supposes to read a huge XML file and then write a CSV file with a header. XML Record needs to be transformed first so I map it to java POJO and then marshal it again to write into a csv file.

I can't load all of the records in memory as the file contains more 200k records.

Issue: I am only seeing the last record being added to the CSV file. Not sure why it's not appending the data into the existing file.

Any idea how to make it work. The header is required in CSV.I am not seeing any other option to directly transform the stream and write headers along with to CSV without unmarshalling it to Pojo first. I tried using BeanIO as well, which requires me to add a Header record and not sure how that can be injected into a stream.

   from("{{xml.files.route}}")

            .split(body().tokenizeXML("EMPLOYEE", null))
            .streaming()
            .unmarshal().jacksonXml(Employee.class)

            .marshal(bindyDataFormat)

            .to("file://C:/Files/Test/emp/csv/?fileName=test.csv")

            .end();

If I try to append into the existing file then CSV file appends headers to each iteration of records.

 .to("file://C:/Files/Test/emp/csv/?fileName=test.csv&fileExist=append")

Answer 1

Your problem here is related to camel-bindy and not the file-component. It kinda expects you to marshal collection objects instead of individual objects hence if you marshal each object individually and have @CsvRecord(generateHeaderColumns = true ) on your Employee class then you'll get headers every time you marshal an individual Employee object.

You could set generateHeaderColumns to false and start the file with headers string manually. One way to obtain headers for Bindy annotated class is to get fields annotated with DataField using org.apache.commons.lang3.reflect.FieldUtils from apache-commons and construct headers string based on position, columnName and fieldName.

I usually prefer camel-stream over file-component when I need to stream something to a file but using file-component with appends probably works just as well.

Example:

package com.example;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;

import org.apache.camel.RoutesBuilder;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.dataformat.bindy.annotation.DataField;
import org.apache.camel.dataformat.bindy.csv.BindyCsvDataFormat;
import org.apache.camel.test.junit4.CamelTestSupport;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.Test;

public class ExampleTest  extends CamelTestSupport {
    
    @Test
    public void testStreamEmployeesToCsvFile(){
        
        List<Employee> body = new ArrayList<>();
        body.add(new Employee("John", "Doe", 1965));
        body.add(new Employee("Mary", "Sue", 1987));
        body.add(new Employee("Gary", "Sue", 1991));

        template.sendBody("direct:streamEmployeesToCSV", body);
    }

    @Override
    protected RoutesBuilder createRouteBuilder() throws Exception {
        return new RouteBuilder(){

            @Override
            public void configure() throws Exception {
                
                BindyCsvDataFormat csvDataFormat = new BindyCsvDataFormat(Employee.class);
                System.out.println(getCSVHeadersForClass(Employee.class, ","));

                from("direct:streamEmployeesToCSV")
                    .setProperty("Employees", body())
                    // a bit hacky due to camel writing first entry and headers 
                    // on the same line for some reason with (camel 2.25.2)
                    .setBody().constant("")
                    .to("file:target/testoutput?fileName=test.csv&fileExist=Override")
                    .setBody().constant(getCSVHeadersForClass(Employee.class, ","))
                    .to("stream:file?fileName=./target/testoutput/test.csv")
                    .split(exchangeProperty("Employees"))
                        .marshal(csvDataFormat)
                        .to("stream:file?fileName=./target/testoutput/test.csv")
                    .end()
                    .log("Done");
            }

            private String getCSVHeadersForClass(Class clazz, String separator ) {

                Field[] fieldsArray = FieldUtils.getFieldsWithAnnotation(clazz, DataField.class);
                List<Field> fields = new ArrayList<>(Arrays.asList(fieldsArray));

                fields.sort(new Comparator<Field>(){

                    @Override
                    public int compare(Field lhsField, Field rhsField) {

                        DataField lhs = lhsField.getAnnotation(DataField.class);
                        DataField rhs = rhsField.getAnnotation(DataField.class);

                        return lhs.pos() < rhs.pos() ? -1 : (lhs.pos() > rhs.pos()) ? 1 : 0;
                    }
                });

                String[] fieldHeaders = new String[fields.size()];
                for (int i = 0; i < fields.size(); i++) {
                    DataField dataField = fields.get(i).getAnnotation(DataField.class);

                    if(dataField.columnName().equals(""))
                        fieldHeaders[i] = fields.get(i).getName();
                    else
                        fieldHeaders[i] = dataField.columnName();
                }
                
                String csvHeaders = "";
                for (int i = 0; i < fieldHeaders.length; i++) {
                    csvHeaders += fieldHeaders[i];
                    csvHeaders += i < fieldHeaders.length - 1 ? separator : "";
                }
                return csvHeaders;
            }
        };
    }
}

<dependency>
    <groupId>org.apache.commons</groupId>
    <artifactId>commons-lang3</artifactId>
    <version>${apache-commons.version}</version>
</dependency>