com.streak.logging.utils.AnalysisUtility.java Source code

Java tutorial

Introduction

Here is the source code for com.streak.logging.utils.AnalysisUtility.java

Source

/*
 * Copyright 2012 Rewardly Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.streak.logging.utils;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.servlet.http.HttpServletRequest;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.json.JSONException;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpResponse;
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.utils.SystemProperty;
import com.google.appengine.api.utils.SystemProperty.Environment.Value;
import com.streak.datastore.analysis.builtin.BuiltinDatastoreExportConfiguration;
import com.streak.logging.analysis.LogsExportConfiguration;
import com.streak.logging.analysis.LogsFieldExporter;
import com.streak.logging.analysis.LogsFieldExporterSet;

public class AnalysisUtility {

    public static boolean isDev() {
        return SystemProperty.environment.value() == Value.Development;
    }

    public static long round(long x, long roundMultiple) {
        return (x / roundMultiple) * roundMultiple;
    }

    public static boolean areParametersValid(String... params) {
        if (params == null) {
            return false;
        }

        for (String p : params) {
            if (p == null || p.isEmpty()) {
                return false;
            }
        }
        return true;
    }

    public static String extractParameterOrThrow(HttpServletRequest req, String paramName) {
        String param = req.getParameter(paramName);
        if (!areParametersValid(param)) {
            throw new InvalidTaskParameterException("Couldn't find required parameter " + paramName);
        }
        return param;
    }

    public static String escapeAndQuoteFieldCsv(String field) {
        return "\"" + field.replace("\"", "\"\"").replace("\r\n", "\r").replace("\n", "\r") + "\"";
    }

    public static String createLogTableKey(String schemaHash, long startTime, long endTime) {
        return String.format("log_%s_%020d_%020d", schemaHash, startTime, endTime);
    }

    public static String createSchemaKey(String schemaHash, long startTime, long endTime) {
        return createLogTableKey(schemaHash, startTime, endTime) + ".schema";
    }

    public static long getEndMsFromKey(String key) {
        String[] keyParts = key.split("_");
        return Long.parseLong(keyParts[keyParts.length - 1]);
    }

    public static void fetchCloudStorageLogUris(String bucketName, String schemaHash, long startMs, long endMs,
            HttpRequestFactory requestFactory, List<String> urisToProcess, boolean readSchemas) throws IOException {
        String startKey = AnalysisUtility.createLogTableKey(schemaHash, startMs, startMs);
        String endKey = AnalysisUtility.createLogTableKey(schemaHash, endMs, endMs);
        fetchCloudStorageUris(bucketName, startKey, endKey, requestFactory, urisToProcess, readSchemas);
    }

    public static void fetchCloudStorageUris(String bucketName, String startKey, String endKey,
            HttpRequestFactory requestFactory, List<String> urisToProcess, boolean readSchemas) throws IOException {
        String bucketUri = "http://commondatastorage.googleapis.com/" + bucketName;
        HttpRequest request = requestFactory.buildGetRequest(new GenericUrl(bucketUri + "?marker=" + startKey));
        HttpResponse response = request.execute();

        try {
            Document responseDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
                    .parse(response.getContent());
            XPath xPath = XPathFactory.newInstance().newXPath();
            NodeList nodes = (NodeList) xPath.evaluate("//Contents/Key/text()", responseDoc,
                    XPathConstants.NODESET);
            for (int i = 0; i < nodes.getLength(); i++) {
                String key = nodes.item(i).getNodeValue();
                if (key.compareTo(endKey) >= 0) {
                    break;
                }
                if (key.endsWith(".schema") ^ readSchemas) {
                    continue;
                }
                if (readSchemas) {
                    key = key.substring(0, key.length() - ".schema".length());
                }
                urisToProcess.add("gs://" + bucketName + "/" + key);
            }
        } catch (SAXException e) {
            throw new IOException("Error parsing cloud storage response", e);
        } catch (ParserConfigurationException e) {
            throw new IOException("Error configuring cloud storage parser", e);
        } catch (XPathExpressionException e) {
            throw new IOException("Error finding keys", e);
        }
    }

    public static String getRequestBaseName(HttpServletRequest req) {
        String path = req.getRequestURI();
        return path.substring(0, path.lastIndexOf("/"));
    }

    public static LogsExportConfiguration instantiateLogExporterConfig(String logsExportConfigurationClassStr) {
        Class<?> exporterConfigClass;
        try {
            exporterConfigClass = Class.forName(logsExportConfigurationClassStr);
        } catch (ClassNotFoundException e) {
            throw new InvalidTaskParameterException(
                    "Got invalid BigqueryFieldExporterSet class name: " + logsExportConfigurationClassStr);
        }

        if (!LogsExportConfiguration.class.isAssignableFrom(exporterConfigClass)) {
            throw new InvalidTaskParameterException(
                    "Got logsExportConfiguration parameter " + logsExportConfigurationClassStr
                            + " that doesn't implement: " + LogsExportConfiguration.class.getSimpleName());
        }

        LogsExportConfiguration exporterConfig;
        try {
            exporterConfig = (LogsExportConfiguration) exporterConfigClass.newInstance();
        } catch (InstantiationException e) {
            throw new InvalidTaskParameterException(
                    "Couldn't instantiate BigqueryFieldExporter set class " + logsExportConfigurationClassStr);
        } catch (IllegalAccessException e) {
            throw new InvalidTaskParameterException("LogsExportConfiguration class "
                    + logsExportConfigurationClassStr + " has no visible default constructor");
        }
        return exporterConfig;
    }

    public static String computeSchemaHash(LogsFieldExporterSet exporterSet) {
        try {
            List<LogsFieldExporter> exporters = exporterSet.getExporters();
            MessageDigest md = MessageDigest.getInstance("MD5");
            for (LogsFieldExporter exporter : exporters) {
                for (int i = 0; i < exporter.getFieldCount(); i++) {
                    md.update(exporter.getFieldName(i).getBytes("UTF-8"));
                    md.update(exporter.getFieldType(i).getBytes("UTF-8"));
                }
            }
            byte[] array = md.digest();
            StringBuffer sb = new StringBuffer();
            for (int i = 0; i < array.length; ++i) {
                sb.append(Integer.toHexString((array[i] & 0xFF) | 0x100).substring(1, 3));
            }
            return sb.toString().substring(0, 6);
        } catch (NoSuchAlgorithmException nsae) {
            throw new RuntimeException("Couldn't find MD5 algorithm for schema hash", nsae);
        } catch (UnsupportedEncodingException e) {
            throw new RuntimeException("Couldn't get UTF-8 encoding for schema hash", e);
        }
    }

    public static TableSchema createSchema(LogsFieldExporterSet exporterSet) {
        Set<String> fieldNames = new HashSet<String>();

        List<LogsFieldExporter> exporters = exporterSet.getExporters();
        TableSchema schema = new TableSchema();
        schema.setFields(new ArrayList<TableFieldSchema>());

        for (LogsFieldExporter exporter : exporters) {
            for (int i = 0; i < exporter.getFieldCount(); i++) {
                String fieldName = exporter.getFieldName(i);
                String fieldMode = "NULLABLE";
                String fieldType = exporter.getFieldType(i).toLowerCase().intern();

                if (fieldNames.contains(fieldName)) {
                    throw new InvalidFieldException(
                            "BigqueryFieldExporterSet " + exporterSet.getClass().getCanonicalName()
                                    + " defines multiple fields with name " + exporter.getFieldName(i));
                }
                fieldNames.add(fieldName);

                if (exporter.getFieldRepeated(i)) {
                    fieldMode = "REPEATED";
                }

                List<TableFieldSchema> subfields = null;
                if (fieldType.equals("record")) {
                    subfields = exporter.getFieldFields(i);
                }

                TableFieldSchema tfs = new TableFieldSchema();
                tfs.setName(fieldName);
                tfs.setType(fieldType);
                tfs.setMode(fieldMode);

                if (subfields != null) {
                    tfs.setFields(subfields);
                }
                schema.getFields().add(tfs);
            }
        }
        return schema;
    }

    public static void putJsonValueFormatted(Map<String, Object> row, String fieldName, Object fieldValue,
            String fieldType) throws JSONException {
        // These strings have been interned so == works for comparison
        if ("string" == fieldType) {
            String stringValue;
            if (fieldValue instanceof Text) {
                stringValue = ((Text) fieldValue).getValue();
            } else {
                stringValue = fieldValue.toString();
            }
            row.put(fieldName, stringValue);
        } else if ("float" == fieldType) {
            row.put(fieldName, fieldValue);
        } else if ("integer" == fieldType) {
            if (fieldValue instanceof Date) {
                @SuppressWarnings("unused")
                long dateLong = ((Date) fieldValue).getTime();
                row.put(fieldName, fieldValue);
            } else {
                row.put(fieldName, fieldValue);
            }
        } else {
            row.put(fieldName, fieldValue);
        }
    }

    public static String failureJson(String message) {
        return "{\"success\":false, \"message\":\"" + message + "\"}";
    }

    public static String successJson(String message) {
        return "{\"success\":true, \"message\":\"" + message + "\"}";
    }

    public static BuiltinDatastoreExportConfiguration instantiateDatastoreExportConfig(
            String builtinDatastoreExportConfig) {
        Class<?> exportConfigClass;
        try {
            exportConfigClass = Class.forName(builtinDatastoreExportConfig);
        } catch (ClassNotFoundException e) {
            throw new InvalidTaskParameterException(
                    "Got invalid BuiltinDatastoreExportConfig class name: " + builtinDatastoreExportConfig);
        }
        if (!BuiltinDatastoreExportConfiguration.class.isAssignableFrom(exportConfigClass)) {
            throw new InvalidTaskParameterException("Got bigqueryFieldExporterSet parameter "
                    + builtinDatastoreExportConfig + " that doesn't implement BigqueryFieldExporterSet");
        }
        BuiltinDatastoreExportConfiguration exportConfig;
        try {
            exportConfig = (BuiltinDatastoreExportConfiguration) exportConfigClass.newInstance();
        } catch (InstantiationException e) {
            throw new InvalidTaskParameterException(
                    "Couldn't instantiate BigqueryFieldExporter set class " + builtinDatastoreExportConfig);
        } catch (IllegalAccessException e) {
            throw new InvalidTaskParameterException("BigqueryFieldExporter class " + builtinDatastoreExportConfig
                    + " has no visible default constructor");
        }
        return exportConfig;
    }

    public static String getPreBackupName(long timestamp, String backupNamePrefix) {
        return backupNamePrefix + timestamp + "_";
    }
}