Java tutorial
/* * Copyright (C) 2015 Jrg Prante * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.xbib.elasticsearch.jdbc.strategy.standard; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.joda.time.DateTime; import org.xbib.elasticsearch.common.metrics.MetricsLogger; import org.xbib.elasticsearch.common.util.LocaleUtil; import org.xbib.elasticsearch.common.util.StrategyLoader; import org.xbib.elasticsearch.jdbc.strategy.Context; import org.xbib.elasticsearch.jdbc.strategy.JDBCSource; import org.xbib.elasticsearch.jdbc.strategy.Sink; import org.xbib.elasticsearch.common.util.SQLCommand; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.TimeZone; import java.util.concurrent.Future; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; /** * The context consists of the parameters that span source and sink settings. * It represents the state, for supporting the task execution, and scripting. */ public class StandardContext<S extends JDBCSource> implements Context<S, Sink> { private final static Logger logger = LogManager.getLogger("importer.jdbc.context.standard"); private Settings settings; private S source; private Sink sink; private State state = State.IDLE; private DateTime dateOfThrowable; private Throwable throwable; private final static List<Future> futures = new LinkedList<>(); @Override public String strategy() { return "standard"; } @Override public StandardContext newInstance() { return new StandardContext(); } @Override public State getState() { return state; } @Override public StandardContext setSettings(Settings settings) { this.settings = settings; if (settings.getAsBoolean("metrics.enabled", false) && futures.isEmpty()) { Thread thread = new MetricsThread(); ScheduledThreadPoolExecutor scheduledthreadPoolExecutor = new ScheduledThreadPoolExecutor(1); futures.add(scheduledthreadPoolExecutor.scheduleAtFixedRate(thread, 0L, settings.getAsTime("metrics.interval", TimeValue.timeValueSeconds(30)).seconds(), TimeUnit.SECONDS)); logger.info("metrics thread started"); } return this; } @Override public Settings getSettings() { return settings; } @Override public StandardContext setSource(S source) { this.source = source; Map<String, String> map = settings.getAsMap(); if (map.containsKey("metrics.lastexecutionstart")) { DateTime lastexecutionstart = DateTime.parse(settings.get("metrics.lastexecutionstart")); source.getMetric().setLastExecutionStart(lastexecutionstart); } if (map.containsKey("metrics.lastexecutionend")) { DateTime lastexecutionend = DateTime.parse(settings.get("metrics.lastexecutionend")); source.getMetric().setLastExecutionEnd(lastexecutionend); } if (map.containsKey("metrics.counter")) { int counter = Integer.parseInt(settings.get("metrics.counter")); if (counter > 0) { source.getMetric().setCounter(counter); } } return this; } @Override public S getSource() { return source; } @Override public StandardContext setSink(Sink sink) { this.sink = sink; return this; } @Override public Sink getSink() { return sink; } public StandardContext setThrowable(Throwable throwable) { this.throwable = throwable; this.dateOfThrowable = new DateTime(); return this; } public Throwable getThrowable() { return throwable; } public DateTime getDateOfThrowable() { return dateOfThrowable; } @Override public void execute() throws Exception { try { state = State.BEFORE_FETCH; beforeFetch(); state = State.FETCH; fetch(); } finally { state = State.AFTER_FETCH; afterFetch(); state = State.IDLE; } } @Override @SuppressWarnings("unchecked") public void beforeFetch() throws Exception { logger.debug("before fetch"); Sink sink = createSink(); S source = createSource(); prepareContext(source, sink); sink.setContext(this); source.setContext(this); getSink().beforeFetch(); getSource().beforeFetch(); } @Override public void fetch() throws Exception { logger.debug("fetch"); try { getSource().fetch(); } catch (Throwable e) { setThrowable(e); logger.error("at fetch: " + e.getMessage(), e); } } @Override public void afterFetch() throws Exception { logger.debug("after fetch"); writeState(); try { getSource().afterFetch(); } catch (Throwable e) { setThrowable(e); logger.error("after fetch: " + e.getMessage(), e); } try { getSink().afterFetch(); } catch (Throwable e) { setThrowable(e); logger.error("after fetch: " + e.getMessage(), e); } } @Override public void shutdown() { logger.info("shutdown in progress"); for (Future future : futures) { future.cancel(true); } if (source != null) { try { source.shutdown(); } catch (Exception e) { logger.error("source shutdown: " + e.getMessage(), e); } } if (sink != null) { try { sink.shutdown(); } catch (Exception e) { logger.error("sink shutdown: " + e.getMessage(), e); } } logger.info("shutdown completed"); writeState(); } protected void writeState() { String statefile = settings.get("statefile"); if (statefile == null || source == null || source.getMetric() == null) { return; } try { File file = new File(statefile); if (file.getParentFile() != null && !file.getParentFile().exists()) { file.getParentFile().mkdirs(); } if (!file.exists() || file.canWrite()) { Writer writer = new FileWriter(statefile); FormatDateTimeFormatter formatter = Joda.forPattern("dateOptionalTime"); Settings.Builder settingsBuilder = Settings.settingsBuilder().put(settings) .put("metrics.lastexecutionstart", formatter.printer().print(source.getMetric().getLastExecutionStart())) .put("metrics.lastexecutionend", formatter.printer().print(source.getMetric().getLastExecutionEnd())) .put("metrics.counter", source.getMetric().getCounter()); XContentBuilder builder = jsonBuilder().prettyPrint().startObject().field("type", "jdbc") .field("jdbc").map(settingsBuilder.build().getAsStructuredMap()).endObject(); writer.write(builder.string()); writer.close(); if (file.length() > 0) { logger.info("state persisted to {}", statefile); } else { logger.error("state file truncated!"); } } else { logger.warn("can't write to {}", statefile); } } catch (IOException e) { logger.error(e.getMessage(), e); } } @SuppressWarnings("unchecked") protected S createSource() { S source = (S) StrategyLoader.newSource(strategy()); logger.info("found source class {}", source); String url = settings.get("url"); String user = settings.get("user"); String password = settings.get("password"); String locale = settings.get("locale", LocaleUtil.fromLocale(Locale.getDefault())); String timezone = settings.get("timezone", TimeZone.getDefault().getID()); source.setUrl(url).setUser(user).setPassword(password).setLocale(LocaleUtil.toLocale(locale)) .setTimeZone(TimeZone.getTimeZone(timezone)); return source; } protected Sink createSink() throws IOException { Sink sink = StrategyLoader.newSink(strategy()); logger.info("found sink class {}", sink); return sink; } @SuppressWarnings("unchecked") protected void prepareContext(S source, Sink sink) throws IOException { Map<String, Object> params = settings.getAsStructuredMap(); List<SQLCommand> sql = SQLCommand.parse(params); String rounding = XContentMapValues.nodeStringValue(params.get("rounding"), null); int scale = XContentMapValues.nodeIntegerValue(params.get("scale"), 2); boolean autocommit = XContentMapValues.nodeBooleanValue(params.get("autocommit"), false); int fetchsize = 10; String fetchSizeStr = XContentMapValues.nodeStringValue(params.get("fetchsize"), null); if ("min".equals(fetchSizeStr)) { fetchsize = Integer.MIN_VALUE; // for MySQL streaming mode } else if (fetchSizeStr != null) { try { fetchsize = Integer.parseInt(fetchSizeStr); } catch (Exception e) { // ignore unparseable } } else { // if MySQL, enable streaming mode hack by default String url = XContentMapValues.nodeStringValue(params.get("url"), null); if (url != null && url.startsWith("jdbc:mysql")) { fetchsize = Integer.MIN_VALUE; // for MySQL streaming mode } } int maxrows = XContentMapValues.nodeIntegerValue(params.get("max_rows"), 0); int maxretries = XContentMapValues.nodeIntegerValue(params.get("max_retries"), 3); TimeValue maxretrywait = XContentMapValues.nodeTimeValue(params.get("max_retries_wait"), TimeValue.timeValueSeconds(30)); String resultSetType = XContentMapValues.nodeStringValue(params.get("resultset_type"), "TYPE_FORWARD_ONLY"); String resultSetConcurrency = XContentMapValues.nodeStringValue(params.get("resultset_concurrency"), "CONCUR_UPDATABLE"); boolean shouldIgnoreNull = XContentMapValues.nodeBooleanValue(params.get("ignore_null_values"), false); boolean shouldDetectGeo = XContentMapValues.nodeBooleanValue(params.get("detect_geo"), true); boolean shouldDetectJson = XContentMapValues.nodeBooleanValue(params.get("detect_json"), true); boolean shouldPrepareDatabaseMetadata = XContentMapValues .nodeBooleanValue(params.get("prepare_database_metadata"), false); boolean shouldPrepareResultSetMetadata = XContentMapValues .nodeBooleanValue(params.get("prepare_resultset_metadata"), false); Map<String, Object> columnNameMap = (Map<String, Object>) params.get("column_name_map"); int queryTimeout = XContentMapValues.nodeIntegerValue(params.get("query_timeout"), 1800); Map<String, Object> connectionProperties = (Map<String, Object>) params.get("connection_properties"); boolean shouldTreatBinaryAsString = XContentMapValues.nodeBooleanValue(params.get("treat_binary_as_string"), false); source.setRounding(rounding).setScale(scale).setStatements(sql).setAutoCommit(autocommit) .setMaxRows(maxrows).setFetchSize(fetchsize).setRetries(maxretries).setMaxRetryWait(maxretrywait) .setResultSetType(resultSetType).setResultSetConcurrency(resultSetConcurrency) .shouldIgnoreNull(shouldIgnoreNull).shouldDetectGeo(shouldDetectGeo) .shouldDetectJson(shouldDetectJson).shouldPrepareDatabaseMetadata(shouldPrepareDatabaseMetadata) .shouldPrepareResultSetMetadata(shouldPrepareResultSetMetadata).setColumnNameMap(columnNameMap) .setQueryTimeout(queryTimeout).setConnectionProperties(connectionProperties) .shouldTreatBinaryAsString(shouldTreatBinaryAsString); setSource(source); setSink(sink); } private final static MetricsLogger metricsLogger = new MetricsLogger(); public void log() { try { if (source != null) { metricsLogger.writeMetrics(settings, source.getMetric()); } if (sink != null) { metricsLogger.writeMetrics(settings, sink.getMetric()); } } catch (Exception e) { // ignore log errors } } class MetricsThread extends Thread { public void run() { log(); } } }