Android Open Source - Profiterole Splitter

• Back to Development/map  ↑

From Project

Back to project page Profiterole.

License

The source code is released under:

Apache License

Java Source Code

package profiterole.mapreduce;
/* w w w .j av  a  2 s . c  om*/
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.File;
import java.util.LinkedList;
import java.util.List;

/** works linearly by all input to split it to workable units
 * 
 * @author Boris Farber
 *
 */
public class Splitter {
    
  public static List<File> folderToFileList(File path) {

    List<File> arr = new LinkedList<File>();

    if(path == null) {
      return arr;
    }

    File[] files = path.listFiles();
    for (int i = 0; i < files.length; i++) {
      if (files[i].isFile()) { // this line weeds out other
        // directories/folders
        arr.add(files[i]);
      }
    }
    return arr;
  }
  
  public static void splitFolder(File sourceFolder, File destFolder, int splitlen) {
    // TODO fill the method thus add ability to work with many files
    // thus support working with many files and inverted index
    List<File> files = folderToFileList(sourceFolder);
    
    
    
  }
  
  public static int splitFile(File sourceFile, File destFolder, int splitlen) {
    int count=1,data;
    try
    {
      long leninfile=0,leng=0;
          
      InputStream infile = new BufferedInputStream(new FileInputStream(sourceFile)); 
      data=infile.read();
      
      while(data != -1) {
        File filename=new File(destFolder + "//"+ sourceFile.getName() + count + ".sp");
        OutputStream outfile = new BufferedOutputStream(new FileOutputStream(filename)); 
        while(data!=-1 && leng<splitlen) {
          outfile.write(data);
          leng++;
          data=infile.read();
        }
        
        leninfile+=leng;
        leng=0;
        outfile.close();
        count++;
      }
    }
    catch(Exception e) {
      e.printStackTrace();
      return 0;
    }
    
    return 1;
  }
    
  public static void main(String args[]) {
    
    File destFolder = new File(System.getProperty("user.dir") + "//MapJobs//");    
    destFolder.mkdir();
    
    splitFile(new File(System.getProperty("user.dir") + "//pg1661.txt"), destFolder, 10000);
  
    // TODO big data
    //File destFolder = new File("C://Projects//BigData//Sample//Process");    
    //destFolder.mkdir();
    //splitFile(new File("C://Projects//BigData//Sample//Z1_data.xml"), destFolder, 1000000);  
  }
}

Java Source Code List

• Back to Development/map  ↑