eu.crowdrec.contest.sender.RequestSender.java Source code

Java tutorial

Introduction

Here is the source code for eu.crowdrec.contest.sender.RequestSender.java

Source

/*
Copyright (c) 2014, TU Berlin
Permission is hereby granted, free of charge, to any person obtaining 
a copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
DEALINGS IN THE SOFTWARE.
*/

package eu.crowdrec.contest.sender;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.SequenceInputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.Vector;
import java.util.zip.GZIPInputStream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class simulates an online evaluation by re-playing an previously recorded stream.
 * 
 * The request sender reads line by line an input file and sends each line to a recommender server.
 * The recommendation responses are collected and stored in an output file, optimized for the evaluator.
 * The lines for the evaluator contain the most ID of the request and the response of the recommender service.
 * If an output file for the evaluator is created, the sender must parse the input file.
 * Currently raw CLEF newsREEL files as well as idomaar data files are supported.
 * 
 * 
 * @author andreas
 * @author thanh
 */

public class RequestSender {

    /** the methods used for creating http connections */
    public static enum HttpLib {
        HTTPCLIENT, JAVA
    }

    /** define the library used for creating a http connection */
    public static HttpLib httpLib = HttpLib.HTTPCLIENT;

    /** the httpClient instance used for creating the connection */
    private final static HttpConnectionManager httpConnectionManager = new MultiThreadedHttpConnectionManager();
    static {
        // configure the http client - httpclient should run in a multi-threaded environment
        final HttpConnectionManagerParams httpConnectionManagerParams = new HttpConnectionManagerParams();
        httpConnectionManagerParams.setDefaultMaxConnectionsPerHost(100);
        httpConnectionManagerParams.setMaxTotalConnections(100);
        httpConnectionManager.setParams(httpConnectionManagerParams);
    }
    final static HttpClient httpClient = new HttpClient(httpConnectionManager);

    /**
     * should a thread pool be used when sending requests
     */
    public static boolean useThreadPool = true;

    /**
     * the default logger
     */
    private static final Logger logger = LoggerFactory.getLogger(RequestSender.class);

    /**
     * Send a line from a logFile to an HTTP server (single-threaded).
     * 
     * @param logline the line that should by sent
     * 
     * @param connection the connection to the http server, must not be null
     * 
     * @return the response or null (if an error has been detected)
     */
    static private String excutePost(final String logline, final String serverURL) {

        // split the logLine into several token
        String[] token = logline.split("\t");

        String type = token[0];
        String property = token[3];
        String entity = token[4];

        // encode the content as URL parameters.
        String urlParameters = "";
        try {
            urlParameters = String.format("type=%s&properties=%s&entities=%s", URLEncoder.encode(type, "UTF-8"),
                    URLEncoder.encode(property, "UTF-8"), URLEncoder.encode(entity, "UTF-8"));
        } catch (UnsupportedEncodingException e1) {
            logger.warn(e1.toString());
        }

        // initialize a HTTP connection to the server
        // reuse of connection objects is delegated to the JVM
        HttpURLConnection connection = null;

        try {
            final URL url = new URL(serverURL);
            connection = (HttpURLConnection) url.openConnection();
            connection.setRequestMethod("POST");
            connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
            connection.setRequestProperty("Content-Language", "en-US");
            connection.setUseCaches(false);
            connection.setDoInput(true);
            connection.setDoOutput(true);
            connection.setRequestProperty("Content-Length", "" + Integer.toString(urlParameters.getBytes().length));

            // Send request
            DataOutputStream wr = new DataOutputStream(connection.getOutputStream());
            wr.writeBytes(urlParameters);
            wr.flush();
            wr.close();

            // Get Response
            BufferedReader rd = null;
            InputStream is = null;
            try {
                is = connection.getInputStream();
                rd = new BufferedReader(new InputStreamReader(is));

                StringBuffer response = new StringBuffer();
                for (String line = rd.readLine(); line != null; line = rd.readLine()) {
                    response.append(line);
                    response.append(" ");
                }
                return response.toString();
            } catch (IOException e) {
                logger.warn("receivind response failed, ignored.");
            } finally {
                if (is != null) {
                    is.close();
                }
                if (rd != null) {
                    rd.close();
                }
            }
        } catch (MalformedURLException e) {
            System.err.println("invalid server URL, program stopped.");
            System.exit(-1);
        } catch (IOException e) {
            System.err.println("i/o error connecting the http server, program stopped. e=" + e);
            System.exit(-1);
        } catch (Exception e) {
            System.err.println("general error connecting the http server, program stopped. e:" + e);
            e.printStackTrace();
            System.exit(-1);
        } finally {
            // close the connection
            if (connection != null) {
                connection.disconnect();
            }
        }
        return null;
    }

    /**
     * Send a line from a logFile to an HTTP server.
     * 
     * @param logline the line that should by sent
     * 
     * @param connection the connection to the http server, must not be null
     * 
     * @return the response or null (if an error has been detected)
     */
    static private String excutePostWithHttpClient(final String logline, final String serverURL) {

        // split the logLine into several token
        String[] token = logline.split("\t");

        // define the URL parameter
        String urlParameters = "";

        boolean oldMethod = token.length < 4;
        if (oldMethod) {
            try {
                String type = logline.contains("\"event_type\": \"recommendation_request\"")
                        ? "recommendation_request"
                        : "event_notification";
                String newLine = logline.substring(0, logline.length() - 1)
                        + ", \"limit\":6, \"type\":\"impression\"}";
                urlParameters = String.format("type=%s&body=%s", URLEncoder.encode(type, "UTF-8"),
                        URLEncoder.encode(newLine, "UTF-8"));

            } catch (UnsupportedEncodingException e1) {
                logger.warn(e1.toString());
            }
        } else {
            String type = token[0];
            String property = token[3];
            String entity = token[4];

            // encode the content as URL parameters.
            try {
                urlParameters = String.format("type=%s&properties=%s&entities=%s", URLEncoder.encode(type, "UTF-8"),
                        URLEncoder.encode(property, "UTF-8"), URLEncoder.encode(entity, "UTF-8"));
            } catch (UnsupportedEncodingException e1) {
                logger.warn(e1.toString());
            }
        }

        PostMethod postMethod = null;
        try {
            StringRequestEntity requestEntity = new StringRequestEntity(urlParameters,
                    "application/x-www-form-urlencoded", "UTF-8");

            postMethod = new PostMethod(serverURL);
            postMethod.setParameter("useCache", "false");
            postMethod.setRequestEntity(requestEntity);

            int statusCode = httpClient.executeMethod(postMethod);
            String response = statusCode == 200 ? postMethod.getResponseBodyAsString() : "statusCode:" + statusCode;

            return response.trim();
        } catch (IOException e) {
            logger.warn("receivind response failed, ignored.");
        } finally {
            if (postMethod != null) {
                postMethod.releaseConnection();
            }
        }
        return null;
    }

    /**
     * read logFile then sends line by line to server.
     * 
     * @param inLogFileName
     *            path to log file. That can be a zip file or text file.
     * @param outLogFile
     *            path to outLog file. The outLog file should be analyzed by the evaluator.
     *            if the filename is null; no output will be generated
     * @param serverURL
     *            URL of the server
     */
    public static void sender(final String inLogFileName, final String outLogFile, final String serverURL) {

        // handle the log file
        // check type of file

        // try to read the defined logFile
        BufferedReader br = null;
        BufferedWriter bw = null;
        try {
            // if outLogFile name is not null, create an output file
            if (outLogFile != null && outLogFile.length() > 0) {
                bw = new BufferedWriter(new FileWriter(new File(outLogFile), false));
            }

            // support a list of files in a directory
            File inLogFile = new File(inLogFileName);
            InputStream is;
            if (inLogFile.isFile()) {
                is = new FileInputStream(inLogFileName);
                // support gZip files
                if (inLogFile.getName().toLowerCase().endsWith(".gz")) {
                    is = new GZIPInputStream(is);
                }
            } else {
                // if the input is a directory, consider all files based on a pattern
                File[] childs = inLogFile.listFiles(new FilenameFilter() {

                    @Override
                    public boolean accept(File dir, String name) {
                        final String fileName = name.toLowerCase();
                        return fileName.endsWith("data.idomaar.txt.gz") || fileName.endsWith("data.idomaar.txt")
                                || fileName.endsWith(".data.gz");
                    }
                });
                if (childs == null || childs.length == 0) {
                    throw new IOException("invalid inLogFileName or empty directory");
                }
                Arrays.sort(childs, new Comparator<File>() {

                    @Override
                    public int compare(File o1, File o2) {
                        return o1.getName().compareTo(o2.getName());
                    }
                });
                Vector<InputStream> isChilds = new Vector<InputStream>();
                for (int i = 0; i < childs.length; i++) {
                    InputStream tmpIS = new FileInputStream(childs[i]);
                    // support gZip files
                    if (childs[i].getName().toLowerCase().endsWith(".gz")) {
                        tmpIS = new GZIPInputStream(tmpIS);
                    }
                    isChilds.add(tmpIS);
                }
                is = new SequenceInputStream(isChilds.elements());
            }

            // read the log file line by line
            br = new BufferedReader(new InputStreamReader(is));
            try {
                for (String line = br.readLine(); line != null; line = br.readLine()) {

                    // ignore invalid lines and header
                    if (line.startsWith("null") || line.startsWith("#")) {
                        continue;
                    }

                    if (useThreadPool) {
                        RequestSenderThread t = new RequestSenderThread(line, serverURL, bw);
                        try {
                            // try to limit the speed of sending requests
                            if (Thread.activeCount() > 1000) {
                                if (logger.isDebugEnabled()) {
                                    logger.debug("Thread.activeCount() = " + Thread.activeCount());
                                }
                                Thread.sleep(200);
                            }
                        } catch (Exception e) {
                            logger.info(e.toString());
                        }
                        t.start();
                    } else {
                        // send parameters to http server and get response.
                        final long startTime = System.currentTimeMillis();
                        String result = HttpLib.HTTPCLIENT.equals(httpLib)
                                ? excutePostWithHttpClient(line, serverURL)
                                : excutePost(line, serverURL);

                        // analyze whether an answer is expected
                        boolean answerExpected = false;
                        if (line.contains("\"event_type\": \"recommendation_request\"")) {
                            answerExpected = true;
                        }
                        if (answerExpected) {
                            if (logger.isInfoEnabled()) {
                                logger.info("serverResponse: " + result);
                            }

                            // if the output file is not null, write the output in a synchronized way
                            if (bw != null) {
                                String[] data = LogFileUtils.extractEvaluationRelevantDataFromInputLine(line);
                                String requestId = data[0];
                                String userId = data[1];
                                String itemId = data[2];
                                String domainId = data[3];
                                String timeStamp = data[4];
                                long responseTime = System.currentTimeMillis() - startTime;
                                synchronized (bw) {
                                    try {
                                        bw.write("prediction\t" + requestId + "\t" + timeStamp + "\t" + responseTime
                                                + "\t" + itemId + "\t" + userId + "\t" + domainId + "\t" + result);
                                        bw.newLine();
                                    } catch (Exception e) {
                                        e.printStackTrace();
                                    }
                                }
                            }
                        }
                    }
                }
            } catch (IOException e) {
                logger.warn(e.toString(), e);
            }
        } catch (FileNotFoundException e) {
            logger.error("logFile not found e:" + e.toString());
        } catch (IOException e) {
            logger.error("reading the logFile failed e:" + e.toString());
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    logger.debug("close read-log file failed");
                }
            }
            if (bw != null) {
                try {
                    // wait for ensuring that all request are finished
                    // this simplifies the management of thread and worked fine for all test machines
                    Thread.sleep(5000);
                    bw.flush();
                } catch (Exception e) {
                    logger.debug("close write-log file failed");
                }
            }
        }
    }

    /**
     * Start the logFile sender.
     * 
     * @param args
     *            String[]{hostname:port, inLogfileName, outLogFileName}
     */
    public static void main(String[] args) {

        if (args.length == 2 || args.length == 3) {
            String args2 = args.length == 3 ? args[2] : null;
            long startTime = System.currentTimeMillis();
            sender(args[1], args2, args[0]);
            try {
                Thread.sleep(5000);
            } catch (Exception e) {
            }
            System.out.println("finished (threadPool=" + useThreadPool + "): "
                    + (System.currentTimeMillis() - startTime) + " (Finished: " + new Date() + ")");
        } else {
            System.err.println("wrong number of parameters.");
            System.err.println("usage: java RequestSender <hostName>:<port> <logfileName>");
            System.err.println(".gz files are supported.");
        }
    }

    /**
     * A class for sending messages concurrently
     * @author andreas
     *
     */
    public static class RequestSenderThread extends Thread {

        // member variables
        final String line;
        final String serverURL;
        final BufferedWriter bw;

        /**
         * Constructor
         * @param line
         * @param serverURL
         * @param bw
         */
        public RequestSenderThread(final String line, final String serverURL, final BufferedWriter bw) {
            this.line = line;
            this.serverURL = serverURL;
            this.bw = bw;
        }

        /**
         * Execute a HTTP post request based on a line from the input file.
         * If in output file is provided, parse the input line and create a line for the evaluator.
         * The file writing is synchronized. Since file writing is done after having received the response, the file writing should not be the bottleneck, but it may reorder the lines in the output file.
         * 
         * @see java.lang.Thread#run()
         */
        @Override
        public void run() {

            final long startTime = System.currentTimeMillis();

            // select the lib and send the http request
            String result = HttpLib.HTTPCLIENT.equals(httpLib) ? excutePostWithHttpClient(line, serverURL)
                    : excutePost(line, serverURL);

            // if the output file is not null, write the output in a synchronized way
            boolean answerExpected = false;
            if (line.contains("\"event_type\": \"recommendation_request\"")) {
                answerExpected = true;
            }

            if (answerExpected) {
                if (logger.isInfoEnabled()) {
                    logger.info("serverResponse: " + result);
                }

                // if the output file is not null, write the output in a synchronized way
                if (bw != null) {
                    String[] data = LogFileUtils.extractEvaluationRelevantDataFromInputLine(line);
                    String requestId = data[0];
                    String userId = data[1];
                    String itemId = data[2];
                    String domainId = data[3];
                    String timeStamp = data[4];
                    long responseTime = System.currentTimeMillis() - startTime;
                    synchronized (bw) {
                        try {
                            bw.write("prediction\t" + requestId + "\t" + timeStamp + "\t" + responseTime + "\t"
                                    + itemId + "\t" + userId + "\t" + domainId + "\t" + result);
                            bw.newLine();
                        } catch (Exception e) {
                            logger.warn(e.toString(), e);
                        }
                    }
                }
            }
        }
    }
}