counting of words in input list of sentences using apache spark - Java Big Data

Java examples for Big Data:apache spark

  1. HOME
  2. Java
  3. Big Data
  4. apache spark

Description

counting of words in input list of sentences using apache spark

Demo Code

    import java.util.Arrays;
    import java.util.List;
    import java.util.Map;

    import org.apache.spark.SparkConf;
    import org.apache.spark.api.java.JavaSparkContext;

    import scala.Tuple2;

    public class WordCountDemo {
        public static void main(String[] args) {
    final SparkConf sparkConf = new SparkConf().setAppName("Word Count Demo").setMaster("local");
    /*from w w  w  .  jav a  2 s  .  co  m*/
    try(final JavaSparkContext jSC = new JavaSparkContext(sparkConf)) {
      final List<String> sentences = Arrays.asList(
          "All Programming Tutorials",
          "Getting Started With Apache Spark",
          "Developing Java Applications In Apache Spark",
          "Getting Started With RDDs In Apache Spark"
      );
      final Map<String, Object> wordsCount = jSC.parallelize(sentences)
                                .flatMap((x) -> Arrays.asList(x.split(" ")))
                                .mapToPair((x) -> new Tuple2<String, Integer>(x, 1))
                                .countByKey();
      
      System.out.println(wordsCount);
    }
  }
    }

Related Tutorials