Java tutorial
/* * Copyright 2012 Rewardly Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.streak.logging.utils; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import org.json.JSONException; import org.w3c.dom.Document; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.api.services.bigquery.model.TableSchema; import com.google.appengine.api.datastore.Text; import com.google.appengine.api.utils.SystemProperty; import com.google.appengine.api.utils.SystemProperty.Environment.Value; import com.streak.datastore.analysis.builtin.BuiltinDatastoreExportConfiguration; import com.streak.logging.analysis.LogsExportConfiguration; import com.streak.logging.analysis.LogsFieldExporter; import com.streak.logging.analysis.LogsFieldExporterSet; public class AnalysisUtility { public static boolean isDev() { return SystemProperty.environment.value() == Value.Development; } public static long round(long x, long roundMultiple) { return (x / roundMultiple) * roundMultiple; } public static boolean areParametersValid(String... params) { if (params == null) { return false; } for (String p : params) { if (p == null || p.isEmpty()) { return false; } } return true; } public static String extractParameterOrThrow(HttpServletRequest req, String paramName) { String param = req.getParameter(paramName); if (!areParametersValid(param)) { throw new InvalidTaskParameterException("Couldn't find required parameter " + paramName); } return param; } public static String escapeAndQuoteFieldCsv(String field) { return "\"" + field.replace("\"", "\"\"").replace("\r\n", "\r").replace("\n", "\r") + "\""; } public static String createLogTableKey(String schemaHash, long startTime, long endTime) { return String.format("log_%s_%020d_%020d", schemaHash, startTime, endTime); } public static String createSchemaKey(String schemaHash, long startTime, long endTime) { return createLogTableKey(schemaHash, startTime, endTime) + ".schema"; } public static long getEndMsFromKey(String key) { String[] keyParts = key.split("_"); return Long.parseLong(keyParts[keyParts.length - 1]); } public static void fetchCloudStorageLogUris(String bucketName, String schemaHash, long startMs, long endMs, HttpRequestFactory requestFactory, List<String> urisToProcess, boolean readSchemas) throws IOException { String startKey = AnalysisUtility.createLogTableKey(schemaHash, startMs, startMs); String endKey = AnalysisUtility.createLogTableKey(schemaHash, endMs, endMs); fetchCloudStorageUris(bucketName, startKey, endKey, requestFactory, urisToProcess, readSchemas); } public static void fetchCloudStorageUris(String bucketName, String startKey, String endKey, HttpRequestFactory requestFactory, List<String> urisToProcess, boolean readSchemas) throws IOException { String bucketUri = "http://commondatastorage.googleapis.com/" + bucketName; HttpRequest request = requestFactory.buildGetRequest(new GenericUrl(bucketUri + "?marker=" + startKey)); HttpResponse response = request.execute(); try { Document responseDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder() .parse(response.getContent()); XPath xPath = XPathFactory.newInstance().newXPath(); NodeList nodes = (NodeList) xPath.evaluate("//Contents/Key/text()", responseDoc, XPathConstants.NODESET); for (int i = 0; i < nodes.getLength(); i++) { String key = nodes.item(i).getNodeValue(); if (key.compareTo(endKey) >= 0) { break; } if (key.endsWith(".schema") ^ readSchemas) { continue; } if (readSchemas) { key = key.substring(0, key.length() - ".schema".length()); } urisToProcess.add("gs://" + bucketName + "/" + key); } } catch (SAXException e) { throw new IOException("Error parsing cloud storage response", e); } catch (ParserConfigurationException e) { throw new IOException("Error configuring cloud storage parser", e); } catch (XPathExpressionException e) { throw new IOException("Error finding keys", e); } } public static String getRequestBaseName(HttpServletRequest req) { String path = req.getRequestURI(); return path.substring(0, path.lastIndexOf("/")); } public static LogsExportConfiguration instantiateLogExporterConfig(String logsExportConfigurationClassStr) { Class<?> exporterConfigClass; try { exporterConfigClass = Class.forName(logsExportConfigurationClassStr); } catch (ClassNotFoundException e) { throw new InvalidTaskParameterException( "Got invalid BigqueryFieldExporterSet class name: " + logsExportConfigurationClassStr); } if (!LogsExportConfiguration.class.isAssignableFrom(exporterConfigClass)) { throw new InvalidTaskParameterException( "Got logsExportConfiguration parameter " + logsExportConfigurationClassStr + " that doesn't implement: " + LogsExportConfiguration.class.getSimpleName()); } LogsExportConfiguration exporterConfig; try { exporterConfig = (LogsExportConfiguration) exporterConfigClass.newInstance(); } catch (InstantiationException e) { throw new InvalidTaskParameterException( "Couldn't instantiate BigqueryFieldExporter set class " + logsExportConfigurationClassStr); } catch (IllegalAccessException e) { throw new InvalidTaskParameterException("LogsExportConfiguration class " + logsExportConfigurationClassStr + " has no visible default constructor"); } return exporterConfig; } public static String computeSchemaHash(LogsFieldExporterSet exporterSet) { try { List<LogsFieldExporter> exporters = exporterSet.getExporters(); MessageDigest md = MessageDigest.getInstance("MD5"); for (LogsFieldExporter exporter : exporters) { for (int i = 0; i < exporter.getFieldCount(); i++) { md.update(exporter.getFieldName(i).getBytes("UTF-8")); md.update(exporter.getFieldType(i).getBytes("UTF-8")); } } byte[] array = md.digest(); StringBuffer sb = new StringBuffer(); for (int i = 0; i < array.length; ++i) { sb.append(Integer.toHexString((array[i] & 0xFF) | 0x100).substring(1, 3)); } return sb.toString().substring(0, 6); } catch (NoSuchAlgorithmException nsae) { throw new RuntimeException("Couldn't find MD5 algorithm for schema hash", nsae); } catch (UnsupportedEncodingException e) { throw new RuntimeException("Couldn't get UTF-8 encoding for schema hash", e); } } public static TableSchema createSchema(LogsFieldExporterSet exporterSet) { Set<String> fieldNames = new HashSet<String>(); List<LogsFieldExporter> exporters = exporterSet.getExporters(); TableSchema schema = new TableSchema(); schema.setFields(new ArrayList<TableFieldSchema>()); for (LogsFieldExporter exporter : exporters) { for (int i = 0; i < exporter.getFieldCount(); i++) { String fieldName = exporter.getFieldName(i); String fieldMode = "NULLABLE"; String fieldType = exporter.getFieldType(i).toLowerCase().intern(); if (fieldNames.contains(fieldName)) { throw new InvalidFieldException( "BigqueryFieldExporterSet " + exporterSet.getClass().getCanonicalName() + " defines multiple fields with name " + exporter.getFieldName(i)); } fieldNames.add(fieldName); if (exporter.getFieldRepeated(i)) { fieldMode = "REPEATED"; } List<TableFieldSchema> subfields = null; if (fieldType.equals("record")) { subfields = exporter.getFieldFields(i); } TableFieldSchema tfs = new TableFieldSchema(); tfs.setName(fieldName); tfs.setType(fieldType); tfs.setMode(fieldMode); if (subfields != null) { tfs.setFields(subfields); } schema.getFields().add(tfs); } } return schema; } public static void putJsonValueFormatted(Map<String, Object> row, String fieldName, Object fieldValue, String fieldType) throws JSONException { // These strings have been interned so == works for comparison if ("string" == fieldType) { String stringValue; if (fieldValue instanceof Text) { stringValue = ((Text) fieldValue).getValue(); } else { stringValue = fieldValue.toString(); } row.put(fieldName, stringValue); } else if ("float" == fieldType) { row.put(fieldName, fieldValue); } else if ("integer" == fieldType) { if (fieldValue instanceof Date) { @SuppressWarnings("unused") long dateLong = ((Date) fieldValue).getTime(); row.put(fieldName, fieldValue); } else { row.put(fieldName, fieldValue); } } else { row.put(fieldName, fieldValue); } } public static String failureJson(String message) { return "{\"success\":false, \"message\":\"" + message + "\"}"; } public static String successJson(String message) { return "{\"success\":true, \"message\":\"" + message + "\"}"; } public static BuiltinDatastoreExportConfiguration instantiateDatastoreExportConfig( String builtinDatastoreExportConfig) { Class<?> exportConfigClass; try { exportConfigClass = Class.forName(builtinDatastoreExportConfig); } catch (ClassNotFoundException e) { throw new InvalidTaskParameterException( "Got invalid BuiltinDatastoreExportConfig class name: " + builtinDatastoreExportConfig); } if (!BuiltinDatastoreExportConfiguration.class.isAssignableFrom(exportConfigClass)) { throw new InvalidTaskParameterException("Got bigqueryFieldExporterSet parameter " + builtinDatastoreExportConfig + " that doesn't implement BigqueryFieldExporterSet"); } BuiltinDatastoreExportConfiguration exportConfig; try { exportConfig = (BuiltinDatastoreExportConfiguration) exportConfigClass.newInstance(); } catch (InstantiationException e) { throw new InvalidTaskParameterException( "Couldn't instantiate BigqueryFieldExporter set class " + builtinDatastoreExportConfig); } catch (IllegalAccessException e) { throw new InvalidTaskParameterException("BigqueryFieldExporter class " + builtinDatastoreExportConfig + " has no visible default constructor"); } return exportConfig; } public static String getPreBackupName(long timestamp, String backupNamePrefix) { return backupNamePrefix + timestamp + "_"; } }