apache spark Map Partitions With Index - Java Big Data

Java examples for Big Data:apache spark

  1. HOME
  2. Java
  3. Big Data
  4. apache spark

Description

apache spark Map Partitions With Index

Demo Code



import java.util.Iterator;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;

public class MapPartitionsWithIndexEx {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName(
                "MapPartitionsWithIndexEx").setMaster("local[*]");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaRDD<String> rawInputRdd = sc.textFile("cjem.csv");

        Function2 removeHeader = new Function2<Integer, Iterator<String>, Iterator<String>>() {
            @Override//  w ww.j av a  2 s .  c  o m
            public Iterator<String> call(Integer ind,
                    Iterator<String> iterator) throws Exception {
                if (ind == 0 && iterator.hasNext()) {
                    iterator.next();
                    return iterator;
                } else
                    return iterator;
            }
        };
        JavaRDD<String> inputRdd = rawInputRdd.mapPartitionsWithIndex(
                removeHeader, false);

        sc.close();

    }

}

Related Tutorials