org.apache.asterix.experiment.client.SpatialQueryGenerator.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.asterix.experiment.client.SpatialQueryGenerator.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.asterix.experiment.client;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.Socket;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.asterix.experiment.action.base.IAction;
import org.apache.asterix.experiment.action.base.SequentialActionList;
import org.apache.asterix.experiment.action.derived.RunAQLStringAction;
import org.apache.asterix.experiment.action.derived.TimedAction;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.hyracks.api.util.ExperimentProfilerUtils;

public class SpatialQueryGenerator {

    private static final int SKIP_LINE_COUNT = 199;

    private final ExecutorService threadPool;

    private final int partitionRangeStart;

    private final int duration;

    private final String restHost;

    private final int restPort;

    private final String orchHost;

    private final int orchPort;

    private final String openStreetMapFilePath;

    private final boolean isIndexOnlyPlan;

    public SpatialQueryGenerator(SpatialQueryGeneratorConfig config) {
        threadPool = Executors.newCachedThreadPool(new ThreadFactory() {

            private final AtomicInteger count = new AtomicInteger();

            @Override
            public Thread newThread(Runnable r) {
                int tid = count.getAndIncrement();
                Thread t = new Thread(r, "DataGeneratorThread: " + tid);
                t.setDaemon(true);
                return t;
            }
        });

        partitionRangeStart = config.getPartitionRangeStart();
        duration = config.getDuration();
        restHost = config.getRESTHost();
        restPort = config.getRESTPort();
        orchHost = config.getQueryOrchestratorHost();
        orchPort = config.getQueryOrchestratorPort();
        openStreetMapFilePath = config.getOpenStreetMapFilePath();
        isIndexOnlyPlan = config.getIsIndexOnlyPlan();
    }

    public void start() throws Exception {
        final Semaphore sem = new Semaphore(0);
        threadPool.submit(new QueryGenerator(sem, restHost, restPort, orchHost, orchPort, partitionRangeStart,
                duration, openStreetMapFilePath, isIndexOnlyPlan));
        sem.acquire();
    }

    public static class QueryGenerator implements Runnable {

        private static final Logger LOGGER = Logger.getLogger(QueryGenerator.class.getName());

        private final HttpClient httpClient;
        private final Semaphore sem;
        private final String restHost;
        private final int restPort;
        private final String orchHost;
        private final int orchPort;
        private final int partition;
        private final int queryDuration;
        private final String openStreetMapFilePath;
        private final float[] radiusType = new float[] { 0.00001f, 0.0001f, 0.001f, 0.01f, 0.1f };
        private int radiusIter = 0;
        private final Random randGen;
        private BufferedReader br;
        private long queryCount;
        private Random random = new Random(211);
        private final boolean isIndexOnlyPlan;
        private String outputFilePath;
        private FileOutputStream outputFos;

        public QueryGenerator(Semaphore sem, String restHost, int restPort, String orchHost, int orchPort,
                int partitionRangeStart, int queryDuration, String openStreetMapFilePath, boolean isIndexOnlyPlan) {
            httpClient = new DefaultHttpClient();
            this.sem = sem;
            this.restHost = restHost;
            this.restPort = restPort;
            this.orchHost = orchHost;
            this.orchPort = orchPort;
            this.partition = partitionRangeStart;
            this.queryDuration = queryDuration * 1000;
            this.openStreetMapFilePath = openStreetMapFilePath;
            this.queryCount = 0;
            this.randGen = new Random(partitionRangeStart);
            this.isIndexOnlyPlan = isIndexOnlyPlan;
        }

        @Override
        public void run() {
            LOGGER.info("\nQueryGen[" + partition + "] running with the following parameters: \n"
                    + "queryGenDuration : " + queryDuration + "\n" + "isIndexOnlyPlan : " + isIndexOnlyPlan);

            try {
                outputFilePath = openStreetMapFilePath.substring(0,
                        openStreetMapFilePath.lastIndexOf(File.separator)) + File.separator + "QueryGenResult-"
                        + Inet4Address.getLocalHost().getHostAddress() + ".txt";
                outputFos = ExperimentProfilerUtils.openOutputFile(outputFilePath);
            } catch (Exception e) {
                e.printStackTrace();
                return;
            }

            try {
                if (openStreetMapFilePath != null) {
                    this.br = new BufferedReader(new FileReader(openStreetMapFilePath));
                }
                try {
                    //connect to orchestrator socket
                    Socket orchSocket = null;
                    orchSocket = new Socket(orchHost, orchPort);
                    try {
                        //send reached message to orchestrator
                        sendReached(orchSocket);

                        //wait until receiving resume message from the orchestrator server
                        receiveResume(orchSocket);

                        if (LOGGER.isLoggable(Level.INFO)) {
                            LOGGER.info("QueryGen[" + partition + "] starts sending queries...");
                        }
                        //send queries during query duration
                        long startTS = System.currentTimeMillis();
                        long prevTS = startTS;
                        long curTS = startTS;
                        while (curTS - startTS < queryDuration) {
                            sendQuery();
                            queryCount++;
                            curTS = System.currentTimeMillis();
                            if (LOGGER.isLoggable(Level.INFO) && queryCount % 100 == 0) {
                                LOGGER.info("QueryGen[" + partition + "][TimeToQuery100] " + (curTS - prevTS)
                                        + " in milliseconds");
                                prevTS = curTS;
                            }
                        }
                        if (LOGGER.isLoggable(Level.INFO)) {
                            LOGGER.info("QueryGen[" + partition + "][QueryCount] " + queryCount + " in "
                                    + (queryDuration / 1000) + " seconds");
                        }

                        if (outputFos != null) {
                            ExperimentProfilerUtils.closeOutputFile(outputFos);
                        }

                        //send reqched message to orchestrator
                        sendReached(orchSocket);
                    } finally {
                        if (orchSocket != null) {
                            orchSocket.close();
                        }
                    }
                } catch (Throwable t) {
                    t.printStackTrace();
                    outputFos.write("Error during sending query\n".getBytes());
                    throw t;
                } finally {
                    if (openStreetMapFilePath != null) {
                        br.close();
                    }
                    if (outputFos != null) {
                        ExperimentProfilerUtils.closeOutputFile(outputFos);
                    }
                }
            } catch (Throwable t) {
                System.err.println("Error connecting to rest API server " + restHost + ":" + restPort);
                t.printStackTrace();
            } finally {
                sem.release();
            }
        }

        private void sendQuery() throws IOException {
            //prepare radius and center point
            int skipLineCount = SKIP_LINE_COUNT;
            int lineCount = 0;
            String line = null;
            ;
            String strPoints[] = null;
            float x = 0, y = 0;
            int beginX = -180, endX = 180, beginY = -90, endY = 90;
            if (openStreetMapFilePath != null) {
                while (lineCount < skipLineCount) {
                    if ((line = br.readLine()) == null) {
                        //reopen file
                        br.close();
                        br = new BufferedReader(new FileReader(openStreetMapFilePath));
                        line = br.readLine();
                    }
                    strPoints = line.split(",");
                    if (strPoints.length != 2) {
                        continue;
                    }
                    lineCount++;
                }
                y = Float.parseFloat(strPoints[0]) / 10000000; //latitude (y value)
                x = Float.parseFloat(strPoints[1]) / 10000000; //longitude (x value)
            } else {
                int xMajor = beginX + random.nextInt(endX - beginX);
                int xMinor = random.nextInt(100);
                x = xMajor + ((float) xMinor) / 100;

                int yMajor = beginY + random.nextInt(endY - beginY);
                int yMinor = random.nextInt(100);
                y = yMajor + ((float) yMinor) / 100;
            }

            //create action
            SequentialActionList sAction = new SequentialActionList();
            IAction rangeQueryAction = new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort,
                    getRangeQueryAQL(radiusType[radiusIter++ % radiusType.length], x, y, isIndexOnlyPlan),
                    outputFos), outputFos);
            sAction.add(rangeQueryAction);

            //perform
            sAction.perform();
        }

        private String getRangeQueryAQL(float radius, float x, float y, boolean isIndexOnlyPlan) {
            StringBuilder sb = new StringBuilder();
            sb.append("use dataverse experiments; ");
            sb.append("count( ");
            sb.append("for $x in dataset Tweets").append(" ");
            sb.append("let $n :=  create-circle( ");
            sb.append("point(\"").append(x).append(", ").append(y).append("\") ");
            sb.append(", ");
            sb.append(String.format("%f", radius));
            sb.append(" )");
            if (isIndexOnlyPlan) {
                sb.append("where spatial-intersect($x.sender-location, $n) ");
            } else {
                sb.append(
                        "where spatial-intersect($x.sender-location, $n) and $x.btree-extra-field1 <= int32(\"2147483647\") ");
            }
            sb.append("return $x ");
            sb.append(");");
            return sb.toString();
        }

        private void sendReached(Socket s) throws IOException {
            new DataOutputStream(s.getOutputStream()).writeInt(OrchestratorDGProtocol.REACHED.ordinal());
            s.getOutputStream().flush();
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info("Sent " + OrchestratorDGProtocol.REACHED + " to " + s.getRemoteSocketAddress());
            }
        }

        private void receiveResume(Socket s) throws IOException {
            int msg = new DataInputStream(s.getInputStream()).readInt();
            OrchestratorDGProtocol msgType = OrchestratorDGProtocol.values()[msg];
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info("Received " + msgType + " from " + s.getRemoteSocketAddress());
            }
            if (msgType != OrchestratorDGProtocol.RESUME) {
                throw new IllegalStateException("Unknown protocol message received: " + msgType);
            }
        }

        private void sendStopped(Socket s) throws IOException {
            new DataOutputStream(s.getOutputStream()).writeInt(OrchestratorDGProtocol.STOPPED.ordinal());
            s.getOutputStream().flush();
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info("Sent " + OrchestratorDGProtocol.STOPPED + " to " + s.getRemoteSocketAddress());
            }
        }

    }
}