Using webMethods Java service(SAX Parser) to split a huge XML file in to multiple small files.

I initially wanted to use STAX Parser which is much easy to use and efficient (From looking at some sample codes online) than the SAX parser but i finally decided to use the SAX Parser. I am currently using a webMethods 8.2.2 env with 1.6 Java so i had to use the SAX Parser to split files.

My requirement was to split a 3 GB XML file in to small XML files. This huge XML had a list of orders in them and below is a sample structure of my XML.

<?xml version="1.0" encoding="UTF-8?>

<rootNode>

       <childNode>

               <ordernumber>12354</ordernumber>



       </childNode>

       <childNode>

               <ordernumber>12355</ordernumber>



       </childNode>

       <childNode>

               <ordernumber>12356</ordernumber>



       </childNode>

</rootNode>

The result should look like this:

file1:

<?xml version="1.0" encoding="UTF-8?>

<rootNode>

       <childNode>

               <ordernumber>12354</ordernumber>



       </childNode>

       <childNode>

               <ordernumber>12355</ordernumber>



       </childNode>

</rootNode>

file2:

<?xml version="1.0" encoding="UTF-8?>

<rootNode>

       <childNode>

               <ordernumber>12356</ordernumber>



       </childNode>

</rootNode>

Java Service Code:

final IDataCursor pipelineCursor = pipeline.getCursor();
  String fileName = IDataUtil.getString( pipelineCursor, "fileName" );
  final String targetDirectotryForSplitFiles = IDataUtil.getString( pipelineCursor, "targetDirectotryForSplitFiles" );
  final String searchElement = IDataUtil.getString( pipelineCursor, "searchElement" );
  final String rootName = IDataUtil.getString( pipelineCursor, "rootName" );
  pipelineCursor.destroy();
    
  try {   
    DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
    DocumentBuilder docBuilder;
    docBuilder = docFactory.newDocumentBuilder();    
    Document doc = docBuilder.parse(fileName);
    NodeList list = doc.getElementsByTagName(searchElement);
    
    final int chunkSize=list.getLength()*7;
    
    SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
    SAXParser saxParser = saxParserFactory.newSAXParser();   
    
    DefaultHandler defaultHandler=new DefaultHandler(){
    int count=0;
    int fileNameCount=0;
    ArrayList outList=new ArrayList();
    String searchTerm="close";
    public void startElement(String uri, String localName, String qName,Attributes attributes) throws SAXException {  
    if (qName.equalsIgnoreCase(searchElement)) {
     String tag = "<" + qName;
     for (int i = 0; i < attributes.getLength(); i++) {
    
      tag += " " + attributes.getLocalName(i) + "="
      + attributes.getValue(i);
     }
    
     tag += ">";
     outList.add(tag);
     searchTerm = "open";  
    }
    else{    
     if(qName.compareTo(rootName)==0){
      
     }
     else{
      outList.add("<" + qName + ">");
     }     
     }
    }  
    public void characters(char ch[], int start, int length)throws SAXException {     
     if (searchTerm.equals("open")) {
      String escapeCharPreserved=StringEscapeUtils.escapeXml(new String(ch, start, length));
      outList.add(escapeCharPreserved);
     }
    }
    public void endElement(String uri, String localName, String qName)  
    throws SAXException {
           outList.add("</" + qName + ">");
           count++;
           if (qName.equalsIgnoreCase("PCRBRecord")) {
               searchTerm = "close";
               
               if(count>chunkSize){
         try {
           writeToFile(outList,targetDirectotryForSplitFiles+"/rest_"+fileNameCount+".xml",rootName);
           fileNameCount++;
           outList.clear();
           count=0;
         } catch (IOException e) {
          IDataUtil.put(pipelineCursor, "filerror", e.toString());
         }
        }
              }
     }
    };
   
   saxParser.parse(fileName, defaultHandler);   
  } catch (ParserConfigurationException e) {
   IDataUtil.put(pipelineCursor, "result", e.toString());
  } catch (SAXException e) {
   IDataUtil.put(pipelineCursor, "result", e.toString());
  } catch (IOException e) {
   IDataUtil.put(pipelineCursor, "result", e.toString());
  }

Shared Code:

private static void writeToFile(ArrayList inputLines, String fileName, String rootName) throws IOException{
  
  ArrayList templist=new ArrayList();
  templist.add("<?xml version="1.0" encoding="UTF-8"?>");
  templist.add("<"+rootName+">");
  templist.addAll(inputLines);
  templist.add("</"+rootName+">");
  FileWriter writer = new FileWriter(fileName); 
  for(String str: templist) {
    writer.write(str);
  }
  writer.close();
 }

Imports:

import java.io.*;
import java.util.ArrayList;
import javax.xml.parsers.*;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.commons.lang.StringEscapeUtils;

List files and directories recursively

This java service will list all the files and the files in the sub-directories with full path names.

IDataCursor pipelineCursor = pipeline.getCursor();

String    dirPath = IDataUtil.getString( pipelineCursor, "dirPath" );

pipelineCursor.destroy();

         

List<String> output=new ArrayList<String>();

         

List<String> tempPlaceHolder=new ArrayList<String>();

output=extract(dirPath,tempPlaceHolder);

  

IDataUtil.put( pipelineCursor, "dirList", output.toArray(new String[output.size()]));

pipelineCursor.destroy();

Shared Code:

public static List<String> extract(String directoryName, List<String> outFiles){

    
File directory=new File(directoryName);
File[] fileList=directory.listFiles();

String outFileName;
for(int i=0;i<fileList.length;i++){
 if(fileList[i].isFile()){
 outFileName=fileList[i].toString();
 outFiles.add(outFileName);
        }
 else{

      outFileName=fileList[i].toString();
      outFiles.add(outFileName);
      extract(outFileName,outFiles);               

     }           
    }

return outFiles;
}

Java service to create a Tar file using Apache Commons Compress library.

There is often a need to create a Tar file by archiving logs files or other files. You could create a tar file using Native Java libraries as well as third party libraries. The advantage of using third party libraries is that it  provides out of the box methods to write the files. The disadvantage would be to go through the license agreements on the jars and make sure you are not violating any of their agreements.

IDataCursor pipelineCursor = pipeline.getCursor();
String directoryName = IDataUtil.getString( pipelineCursor, "directoryName" );
pipelineCursor.destroy();
  
File dirName=new File(directoryName);
File[] fileNames=dirName.listFiles();  
OutputStream tarfileOutstream;  
try {
      tarfileOutstream = new FileOutputStream(new File("E:/aarremreddy/Tasks/Compress/test.tar"));
      ArchiveOutputStream aos = new ArchiveStreamFactory().createArchiveOutputStream(ArchiveStreamFactory.TAR, tarfileOutstream);
     
      for(int i=0;i<fileNames.length;i++){
 File tarInputfile= fileNames[i];
 TarArchiveEntry archiveEntry = new TarArchiveEntry(tarInputfile);
 archiveEntry.setSize(tarInputfile.length());
 aos.putArchiveEntry(archiveEntry);
 IOUtils.copy(new FileInputStream(tarInputfile),aos);
 aos.closeArchiveEntry();
      }

    aos.finish(); 
    tarfileOutstream.close();
    IDataUtil.put(pipelineCursor, "result", "Success");
} 
catch (FileNotFoundException e) {
 IDataUtil.put(pipelineCursor, "result", e.toString());
} catch (ArchiveException e) {
 IDataUtil.put(pipelineCursor, "result", e.toString());
} catch (IOException e) {
 IDataUtil.put(pipelineCursor, "result", e.toString());
}

Imports:

import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.utils.IOUtils;

Jars:

commons-compress-1.4.1.jar;
commons-io-2.4.jar;