Java tutorial
/* * Copyright 2012 Rewardly Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.streak.logging.analysis; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.json.JSONException; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.appengine.api.log.LogQuery; import com.google.appengine.api.log.LogQuery.Version; import com.google.appengine.api.log.LogService; import com.google.appengine.api.log.LogServiceException; import com.google.appengine.api.log.LogServiceFactory; import com.google.appengine.api.log.RequestLogs; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskAlreadyExistsException; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions.Method; import com.streak.logging.utils.AnalysisConstants; import com.streak.logging.utils.AnalysisUtility; import com.streak.logging.utils.BigqueryIngester; import com.streak.logging.utils.InvalidFieldException; @SuppressWarnings("serial") public class LogExportDirectToBigqueryTask extends HttpServlet { private static final String TASK_URL = "/bqlogging/logExportDirectToBigqueryTask"; private static final Logger log = Logger.getLogger("bqlogging"); private long MAX_BYTES_PER_POST = 1 * 1000 * 1000; // not exactly a megabyte, leave some buffer public static void enqueueMultipleTasksForManyRanges(String logsExporterConfigurationClassName) { LogsExportConfiguration config = AnalysisUtility .instantiateLogExporterConfig(logsExporterConfigurationClassName); long now = System.currentTimeMillis(); long logRangeEndMs = AnalysisUtility.round(now, config.getMillisPerExport()); long logRangeStartMs = logRangeEndMs - config.getMillisPerExport(); for (int i = 0; i < AnalysisConstants.NUM_TASKS_TO_GENERATE_PER_ENQUEUE; i++) { Queue queue; if (!AnalysisUtility.areParametersValid(config.getQueueName())) { queue = QueueFactory.getDefaultQueue(); } else { queue = QueueFactory.getQueue(config.getQueueName()); } TaskOptions t = TaskOptions.Builder.withUrl(TASK_URL); t.param(AnalysisConstants.LOGS_EXPORTER_CONFIGURATION_PARAM, logsExporterConfigurationClassName); t.param(AnalysisConstants.LOG_RANGE_START_MS, Long.toString(logRangeStartMs)); t.param(AnalysisConstants.LOG_RANGE_END_MS, Long.toString(logRangeEndMs)); t.etaMillis(logRangeEndMs + AnalysisConstants.MILLIS_TO_DELAY_TASKS_BEFORE_RUNNING); t.method(Method.GET); String name = LogExportDirectToBigqueryTask.class.getSimpleName() + "_" + Long.toString(logRangeStartMs) + "_" + Long.toString(logRangeEndMs); log.warning("exportTaskName: " + name); t.taskName(name); try { queue.addAsync(t); } catch (TaskAlreadyExistsException te) { // we've already enqueued a task for this window, so don't worry about it } logRangeEndMs += config.getMillisPerExport(); logRangeStartMs += config.getMillisPerExport(); } } public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("text/plain"); long logRangeStartMs = Long.parseLong(req.getParameter(AnalysisConstants.LOG_RANGE_START_MS)); long logRangeEndMs = Long.parseLong(req.getParameter(AnalysisConstants.LOG_RANGE_END_MS)); log.warning(logRangeStartMs + " start time"); log.warning(logRangeEndMs + " end time"); String logsExporterConfig = req.getParameter(AnalysisConstants.LOGS_EXPORTER_CONFIGURATION_PARAM); LogsExportConfiguration exportConfig = AnalysisUtility.instantiateLogExporterConfig(logsExporterConfig); LogsFieldExporterSet exporterSet = exportConfig.getExporterSet(); List<LogsFieldExporter> exporters = exporterSet.getExporters(); Iterable<RequestLogs> logs = null; try { logs = queryForLogs(logRangeStartMs, logRangeEndMs, exportConfig, exporterSet); } catch (LogServiceException e) { // this task just needs to be retried, set a custom error code in case you want to alter how it shows up for reporting setFailedTaskResponseCode(resp, exportConfig); e.printStackTrace(); return; } try { streamToBigquery(logRangeStartMs, logRangeEndMs, exportConfig, exporterSet, exporters, logs); } catch (GoogleJsonResponseException e) { // this task just needs to be retried, set a custom error code in case you want to alter how it shows up for reporting setFailedTaskResponseCode(resp, exportConfig); e.printStackTrace(); return; } } public void setFailedTaskResponseCode(HttpServletResponse resp, LogsExportConfiguration exportConfig) { Integer respCode = exportConfig.getCustomTaskFailureResponseCode(); if (respCode == null) { respCode = 503; } resp.setStatus(respCode); } public void streamToBigquery(long logRangeStartMs, long logRangeEndMs, LogsExportConfiguration exportConfig, LogsFieldExporterSet exporterSet, List<LogsFieldExporter> exporters, Iterable<RequestLogs> logs) throws UnsupportedEncodingException, IOException, GoogleJsonResponseException { List<Map<String, Object>> rows = new ArrayList<>(); List<String> insertIds = new ArrayList<>(); int resultsCount = 0; long singleExportBytes = 2; for (RequestLogs log : logs) { Map<String, Object> row = new HashMap<>(); if (exporterSet.skipLog(log)) { continue; } for (LogsFieldExporter exporter : exporters) { exporter.processLog(log); for (int fieldIndex = 0; fieldIndex < exporter.getFieldCount(); fieldIndex++) { String fieldName = exporter.getFieldName(fieldIndex); String fieldType = exporter.getFieldType(fieldIndex); Object fieldValue = exporter.getField(fieldName); if (fieldValue == null && !exporter.getFieldNullable(fieldIndex)) { throw new InvalidFieldException("Exporter " + exporter.getClass().getCanonicalName() + " didn't return field for " + fieldName); } try { AnalysisUtility.putJsonValueFormatted(row, fieldName, fieldValue, fieldType); } catch (JSONException e) { e.printStackTrace(); } } } long rowBytes = row.toString().getBytes("UTF-8").length + 1; // Assumes a comma for every array item but w/e, conservative is fine if (singleExportBytes + rowBytes > MAX_BYTES_PER_POST) { BigqueryIngester.streamingRowIngestion(rows, insertIds, exportConfig.getBigqueryTableId(logRangeStartMs, logRangeEndMs), exportConfig.getBigqueryDatasetId(), exportConfig.getBigqueryProjectId(), exportConfig.getBigquery()); rows = new ArrayList<Map<String, Object>>(); insertIds = new ArrayList<>(); singleExportBytes = 0; } singleExportBytes += rowBytes; rows.add(row); insertIds.add(log.getRequestId()); resultsCount++; if (resultsCount == 19 && AnalysisUtility.isDev()) { break; // stupid dev server bug: https://code.google.com/p/googleappengine/issues/detail?id=8987 } } if (!rows.isEmpty()) { BigqueryIngester.streamingRowIngestion(rows, insertIds, exportConfig.getBigqueryTableId(logRangeStartMs, logRangeEndMs), exportConfig.getBigqueryDatasetId(), exportConfig.getBigqueryProjectId(), exportConfig.getBigquery()); } log.warning(resultsCount + " rows exported"); } public Iterable<RequestLogs> queryForLogs(long logRangeStartMs, long logRangeEndMs, LogsExportConfiguration exportConfig, LogsFieldExporterSet exporterSet) { LogService ls = LogServiceFactory.getLogService(); LogQuery lq = new LogQuery(); lq = lq.startTimeMillis(logRangeStartMs).endTimeMillis(logRangeEndMs).includeAppLogs(true); if (exportConfig.getLogLevel() != null) { lq = lq.minLogLevel(exportConfig.getLogLevel()); } List<Version> appVersions = exporterSet.applicationVersionsToExport(); if (appVersions != null && appVersions.size() > 0) { lq = lq.versions(appVersions); } Iterable<RequestLogs> logs = ls.fetch(lq); return logs; } }