Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.Duration; import org.apache.spark.streaming.api.java.JavaDStream; import org.apache.spark.streaming.api.java.JavaPairDStream; import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; import org.apache.spark.streaming.api.java.JavaStreamingContext; import org.apache.spark.streaming.receiver.Receiver; import scala.Tuple2; import java.io.BufferedReader; import java.io.FileReader; import java.io.InputStreamReader; import java.net.ConnectException; import java.net.Socket; import java.util.regex.Pattern; /** * Custom Receiver that receives data over a socket. Received bytes is interpreted as * text and \n delimited lines are considered as records. They are then counted and printed. * * Usage: JavaCustomReceiver <master> <hostname> <port> * <master> is the Spark master URL. In local mode, <master> should be 'local[n]' with n > 1. * <hostname> and <port> of the TCP server that Spark Streaming would connect to receive data. * * To run this on your local machine, you need to first run a Netcat server * `$ nc -lk 9999` * and then run the example * `$ bin/run-example org.apache.spark.examples.streaming.JavaCustomReceiver localhost 9999` */ public class JavaCustomReceiver extends Receiver<String> { private static final Pattern SPACE = Pattern.compile(" "); public static void main(String[] args) { if (args.length < 2) { System.err.println("Usage: JavaCustomReceiver <hostname> <port>"); System.exit(1); } // StreamingExamples.setStreamingLogLevels(); // Create the context with a 1 second batch size SparkConf sparkConf = new SparkConf().setAppName("JavaCustomReceiver"); JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000)); // Create a input stream with the custom receiver on target ip:port and count the // words in input stream of \n delimited text (eg. generated by 'nc') JavaReceiverInputDStream<String> lines = ssc .receiverStream(new JavaCustomReceiver(args[0], Integer.parseInt(args[1]))); JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() { public Iterable<String> call(String x) { return Lists.newArrayList(SPACE.split(x)); } }); JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() { public Tuple2<String, Integer> call(String s) { return new Tuple2<String, Integer>(s, 1); } }).reduceByKey(new Function2<Integer, Integer, Integer>() { public Integer call(Integer i1, Integer i2) { return i1 + i2; } }); wordCounts.print(); ssc.start(); ssc.awaitTermination(); } // ============= Receiver code that receives data over a socket ============== String host = null; int port = -1; public JavaCustomReceiver(String host_, int port_) { super(StorageLevel.MEMORY_AND_DISK_2()); host = host_; port = port_; } public void onStart() { // Start the thread that receives data over a connection new Thread() { @Override public void run() { receive(); } }.start(); } public void onStop() { // There is nothing much to do as the thread calling receive() // is designed to stop by itself isStopped() returns false } /** Create a socket connection and receive data until receiver is stopped */ private void receive() { Socket socket = null; String userInput = null; try { // connect to the server socket = new Socket(host, port); // BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); // Path pt=new Path("hdfs://192.168.0.1:9000/equinox-sanjose.20120119-netflow.txt"); // FileSystem fs = FileSystem.get(new Configuration()); // BufferedReader in=new BufferedReader(new InputStreamReader(fs.open(pt))); Path pt = new Path("hdfs://192.168.0.1:9000/user/hduser/equinox-sanjose.20120119-netflow.txt"); Configuration conf = new Configuration(); conf.addResource(new Path("/usr/local/hadoop/conf/core-site.xml")); conf.addResource(new Path("/usr/local/hadoop/conf/hdfs-site.xml")); // FileSystem fs = FileSystem.get(conf); FileSystem fs = pt.getFileSystem(conf); System.out.println(fs.getHomeDirectory()); BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(pt))); // BufferedReader in = new BufferedReader( // new FileReader( // "/home/hduser/spark_scratchPad/equinox-sanjose.20120119-netflow.txt")); // // Until stopped or connection broken continue reading while (!isStopped() && (userInput = in.readLine()) != null) { System.out.println("Received data '" + userInput + "'"); store(userInput); } in.close(); socket.close(); // Restart in an attempt to connect again when server is active again restart("Trying to connect again"); } catch (ConnectException ce) { // restart if could not connect to server restart("Could not connect", ce); } catch (Throwable t) { restart("Error receiving data", t); } } }