Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.isis.objectstore.nosql.db.file.server; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.OutputStream; import java.io.PrintWriter; import java.net.ConnectException; import java.net.InetAddress; import java.net.ServerSocket; import java.net.Socket; import java.net.SocketTimeoutException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.zip.CRC32; import java.util.zip.CheckedInputStream; import java.util.zip.CheckedOutputStream; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.configuration.CompositeConfiguration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.SystemConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.isis.core.commons.lang.ObjectExtensions; import org.apache.isis.objectstore.nosql.NoSqlStoreException; public class FileServer { private static final Logger LOG = LoggerFactory.getLogger(FileServer.class); private static final String DEFAULT_HOST = "localhost"; private static final int DEFAULT_SERVICE_PORT = 9100; private static final int DEFAULT_CONTROL_PORT = 9101; private static final int DEFAULT_SYNC_PORT = 9102; private static final int BACKLOG = 0; private static final int INIT = 1; private static final int RECOVERY_LOG = 2; public static void main(final String[] args) throws IOException, ParseException { final Options options = new Options(); options.addOption("h", "help", false, "Show this help"); options.addOption("m", "mode", true, "mode: normal | secondary | recovery | archive"); final CommandLineParser parser = new BasicParser(); final CommandLine cmd = parser.parse(options, args); if (cmd.hasOption('h')) { printHelp(options); return; } final String mode = cmd.getOptionValue("m"); final List<String> argList = ObjectExtensions.asT(cmd.getArgList()); if ("recovery".equals(mode)) { final FileServer fileServer = new FileServer(); fileServer.startRecovery(argList); } else if ("archive".equals(mode)) { final FileServer fileServer = new FileServer(); fileServer.startArchive(argList); } else if ("secondary".equals(mode)) { final FileServer fileServer = new FileServer(); fileServer.startSecondary(); } else if (mode == null || "normal".equals(mode)) { final FileServer fileServer = new FileServer(); fileServer.startNormal(); } else { printHelp(options); } } private static void printHelp(final Options options) { final HelpFormatter help = new HelpFormatter(); help.printHelp("FileSever [OPTIONS] [FIRST RECOVERY FILES] [LAST RECOVERY FILES]", options); } private FileServerProcessor server; private CompositeConfiguration config; private boolean awaitConnections = true; private boolean isQuiescent = false; private long requests; public FileServer() { org.apache.log4j.PropertyConfigurator.configure("config/logging.properties"); try { config = new CompositeConfiguration(); config.addConfiguration(new SystemConfiguration()); config.addConfiguration(new PropertiesConfiguration("config/server.properties")); final String data = config.getString("fileserver.data"); final String services = config.getString("fileserver.services"); final String logs = config.getString("fileserver.logs"); final String archive = config.getString("fileserver.archive"); Util.setDirectory(data, services, logs, archive); server = new FileServerProcessor(); } catch (final ConfigurationException e) { LOG.error("configuration failure", e); System.out.println(e.getMessage()); System.exit(0); } } private void startNormal() { new Thread("control") { @Override public void run() { startControl(); }; }.start(); new Thread("service") { @Override public void run() { startService(); }; }.start(); new Thread("log-rolling") { @Override public void run() { startLogRolling(); } }.start(); if (config.getBoolean("fileserver.sync", false)) { new Thread("sync") { @Override public void run() { startSyncing(); }; }.start(); } else { LOG.info("not syncing to secondary server"); } } private void startService() { final String serviceHost = config.getString("fileserver.host", DEFAULT_HOST); final int servicePort = config.getInt("fileserver.port", DEFAULT_SERVICE_PORT); final int connectionTimeout = config.getInt("fileserver.connection.timeout", 5000); final int readTimeout = config.getInt("fileserver.read.timeout", 5000); ServerSocket socket = null; try { LOG.debug("setting up service socket on " + serviceHost + ":" + servicePort); final InetAddress address = InetAddress.getByName(serviceHost); socket = new ServerSocket(servicePort, BACKLOG, address); socket.setSoTimeout(connectionTimeout); LOG.info("file service listenting on " + socket.getInetAddress().getHostAddress() + " port " + socket.getLocalPort()); LOG.debug("file service listenting on " + socket); final LogRange logFileRange = Util.logFileRange(); if (!logFileRange.noLogFile()) { final long lastRecoveryFile = logFileRange.getLast(); final File file = Util.logFile(lastRecoveryFile); LOG.info("replaying last recovery file: " + file.getAbsolutePath()); recover(file); } server.startup(); } catch (final UnknownHostException e) { LOG.error("Unknown host " + serviceHost, e); System.exit(0); } catch (final IOException e) { LOG.error("start failure - networking not set up for " + serviceHost, e); System.exit(0); } catch (final RuntimeException e) { LOG.error("start failure", e); System.exit(0); } do { try { while (isQuiescent) { try { Thread.sleep(300); } catch (final InterruptedException ignore) { } } final Socket connection = socket.accept(); LOG.debug("connection from " + connection); connection.setSoTimeout(readTimeout); serviceConnection(connection, readTimeout); } catch (final SocketTimeoutException expected) { } catch (final IOException e) { LOG.error("networking problem", e); } } while (awaitConnections); } private void serviceConnection(final Socket connection, final int readTimeout) { try { final InputStream input = connection.getInputStream(); final OutputStream output = connection.getOutputStream(); final ServerConnection pipe = new ServerConnection(input, output); requests++; server.process(pipe); pipe.logComplete(); } catch (final NoSqlStoreException e) { if (e.getCause() instanceof SocketTimeoutException) { LOG.error("read timed out after " + (readTimeout / 1000.0) + " seconds", e); } else { LOG.error("file server failure", e); } } catch (final IOException e) { LOG.error("networking failure", e); } catch (final RuntimeException e) { LOG.error("request failure", e); } finally { try { connection.close(); } catch (final IOException e) { LOG.warn("failure to close connection", e); } } } private void startSyncing() { final String syncHost = config.getString("fileserver.sync-host", DEFAULT_HOST); final int syncPort = config.getInt("fileserver.sync-port", DEFAULT_SYNC_PORT); final int connectionTimeout = config.getInt("fileserver.connection.timeout", 5000); LOG.info("preparing to sync to secondary server on " + syncHost + " port " + syncPort); final InetAddress address; try { address = InetAddress.getByName(syncHost); } catch (final UnknownHostException e) { LOG.error("Unknown host " + syncHost, e); System.exit(0); return; } while (awaitConnections) { Socket socket = null; try { socket = new Socket(address, syncPort); LOG.info("sync connected to " + socket.getInetAddress().getHostAddress() + " port " + socket.getLocalPort()); final CRC32 crc32 = new CRC32(); final DataOutput output = new DataOutputStream( new CheckedOutputStream(socket.getOutputStream(), crc32)); final DataInput input = new DataInputStream(socket.getInputStream()); output.writeByte(INIT); long logId = input.readLong(); do { final long nextLogId = logId + 1; final File file = Util.logFile(nextLogId); if (file.exists() && server.getLogger().isWritten(nextLogId)) { logId++; output.writeByte(RECOVERY_LOG); crc32.reset(); output.writeLong(logId); LOG.info("sending recovery file: " + file.getName()); final BufferedInputStream fileInput = new BufferedInputStream(new FileInputStream(file)); final byte[] buffer = new byte[8092]; int read; while ((read = fileInput.read(buffer)) > 0) { output.writeInt(read); output.write(buffer, 0, read); } output.writeInt(0); output.writeLong(crc32.getValue()); } try { Thread.sleep(300); } catch (final InterruptedException ignore) { } while (isQuiescent) { try { Thread.sleep(300); } catch (final InterruptedException ignore) { } } } while (awaitConnections); } catch (final ConnectException e) { LOG.warn("not yet connected to secondary server at " + syncHost + " port " + syncPort); try { Thread.sleep(connectionTimeout); } catch (final InterruptedException ignore) { } } catch (final IOException e) { LOG.error("start failure - networking not set up for " + syncHost, e); try { Thread.sleep(300); } catch (final InterruptedException ignore) { } } catch (final RuntimeException e) { LOG.error("start failure", e); try { Thread.sleep(300); } catch (final InterruptedException ignore) { } } } } private void startControl() { final String controlHost = config.getString("fileserver.control-host", DEFAULT_HOST); final int controlPort = config.getInt("fileserver.control-port", DEFAULT_CONTROL_PORT); final int connectionTimeout = config.getInt("fileserver.connection.timeout", 5000); ServerSocket socket = null; try { LOG.debug("setting up control socket on " + controlHost + ":" + controlPort); final InetAddress address = InetAddress.getByName(controlHost); socket = new ServerSocket(controlPort, 0, address); socket.setSoTimeout(connectionTimeout); LOG.info("file control listenting on " + socket.getInetAddress().getHostAddress() + " port " + socket.getLocalPort()); LOG.debug("file control listenting on " + socket); } catch (final UnknownHostException e) { LOG.error("Unknown host " + controlHost, e); System.exit(0); } catch (final IOException e) { LOG.error("start failure - networking not set up for " + controlHost, e); System.exit(0); } catch (final RuntimeException e) { LOG.error("start failure", e); System.exit(0); } do { try { final Socket connection = socket.accept(); LOG.info("control connection from " + connection); controlConnection(connection); } catch (final SocketTimeoutException expected) { } catch (final IOException e) { LOG.error("networking problem", e); } } while (awaitConnections); } private void controlConnection(final Socket connection) { try { final InputStream input = connection.getInputStream(); final BufferedReader reader = new BufferedReader(new InputStreamReader(input)); final OutputStream output = connection.getOutputStream(); final PrintWriter print = new PrintWriter(output); print.print("> "); print.flush(); String line; while ((line = reader.readLine()) != null) { if ("shutdown".equals(line)) { awaitConnections = false; print.println("Server shutdown initiated..."); print.flush(); server.shutdown(); break; } else if ("quiesce".equals(line)) { isQuiescent = true; final String message = "Placing server in a quiescent state"; LOG.info(message); print.println(message); print.print("> "); print.flush(); } else if ("resume".equals(line)) { if (isQuiescent) { isQuiescent = false; final String message = "Resuming from a quiescent state"; LOG.info(message); print.println(message); } else { print.println("Can't resume as not currently in a quiescent state"); } print.print("> "); print.flush(); } else if ("quit".equals(line)) { print.println("Bye"); print.flush(); break; } else if ("status".equals(line)) { print.println("requests: " + requests); print.println("quiescent: " + isQuiescent); print.print("> "); print.flush(); } else { print.println("Unknown command, valid commands are: quit, quiesce, status, resume, shutdown"); print.print("> "); print.flush(); } } } catch (final IOException e) { LOG.error("networking failure", e); } catch (final RuntimeException e) { LOG.error("request failure", e); } finally { try { connection.close(); } catch (final IOException e) { LOG.warn("failure to close connection", e); } } } private void startRecovery(final List<String> list) { LOG.info("starting recovery"); final LogRange logFileRange = Util.logFileRange(); if (logFileRange.noLogFile()) { System.err.println("No recovery files found"); System.exit(0); } final long lastId = logFileRange.getLast(); LOG.info("last log file is " + Util.logFile(lastId).getName()); long startId = lastId; long endId = lastId; final int size = list.size(); if (size > 0) { startId = Long.valueOf(list.get(0)); if (size > 1) { endId = Long.valueOf(list.get(1)); } } if (startId < logFileRange.getFirst() || startId > lastId || endId > lastId) { System.err.println( "File IDs invalid: they must be between " + logFileRange.getFirst() + " and " + lastId); System.exit(0); } if (startId > endId) { System.err.println("File IDs invalid: start must be before the end"); System.exit(0); } Util.ensureDirectoryExists(); for (long id = startId; id <= endId; id++) { final File file = Util.logFile(id); LOG.info("recovering data from " + file.getName()); recover(file); } LOG.info("recovery complete"); } private void startArchive(final List<String> list) { LOG.info("starting archiving"); final LogRange logFileRange = Util.logFileRange(); if (logFileRange.noLogFile()) { System.err.println("No recovery files found"); System.exit(0); } final long lastId = logFileRange.getLast(); LOG.info("last log file is " + Util.logFile(lastId).getName()); long endId = lastId - 1; final int size = list.size(); if (size > 0) { endId = Long.valueOf((String) list.get(0)); } if (endId >= lastId) { System.err.println("File ID invalid: they must be less that " + lastId); System.exit(0); } final long startId = logFileRange.getFirst(); for (long id = startId; id <= endId; id++) { final File file = Util.logFile(id); LOG.info("moving " + file.getName()); final File destination = Util.archiveLogFile(id); file.renameTo(destination); } LOG.info("archive complete"); } private void startSecondary() { final String serviceHost = config.getString("fileserver.sync-host", DEFAULT_HOST); final int servicePort = config.getInt("fileserver.sync-port", DEFAULT_SYNC_PORT); Util.ensureDirectoryExists(); ServerSocket socket = null; try { LOG.debug("setting up syncing socket on " + serviceHost + ":" + servicePort); final InetAddress address = InetAddress.getByName(serviceHost); socket = new ServerSocket(servicePort, 0, address); LOG.info( "listenting on " + socket.getInetAddress().getHostAddress() + " port " + socket.getLocalPort()); LOG.debug("listenting on " + socket); do { syncConnection(socket.accept(), 0); } while (awaitConnections); } catch (final UnknownHostException e) { LOG.error("Unknown host " + serviceHost, e); System.exit(0); } catch (final IOException e) { LOG.error("start failure - networking not set up for " + serviceHost, e); System.exit(0); } catch (final RuntimeException e) { LOG.error("start failure", e); System.exit(0); } } private void syncConnection(final Socket connection, final int readTimeout) { try { final CRC32 crc32 = new CRC32(); final DataOutput output = new DataOutputStream(connection.getOutputStream()); final DataInput input = new DataInputStream(new CheckedInputStream(connection.getInputStream(), crc32)); if (input.readByte() != INIT) { return; } final LogRange logFileRange = Util.logFileRange(); final long lastId = logFileRange.noLogFile() ? -1 : logFileRange.getLast(); output.writeLong(lastId); do { if (input.readByte() != RECOVERY_LOG) { return; } crc32.reset(); final long logId = input.readLong(); final File file = Util.tmpLogFile(logId); LOG.info("syncing recovery file: " + file.getName()); final BufferedOutputStream fileOutput = new BufferedOutputStream(new FileOutputStream(file)); final byte[] buffer = new byte[8092]; int length; while ((length = input.readInt()) > 0) { input.readFully(buffer, 0, length); fileOutput.write(buffer, 0, length); } fileOutput.close(); final long calculatedChecksum = crc32.getValue(); final long sentChecksum = input.readLong(); if (calculatedChecksum != sentChecksum) { throw new NoSqlStoreException("Checksum didn't match during download of " + file.getName()); } recover(file); final File renameTo = Util.logFile(logId); file.renameTo(renameTo); } while (true); } catch (final NoSqlStoreException e) { LOG.error("file server failure", e); } catch (final IOException e) { LOG.error("networking failure", e); } catch (final RuntimeException e) { LOG.error("request failure", e); } finally { try { connection.close(); } catch (final IOException e) { LOG.warn("failure to close connection", e); } } // TODO restart } private void recover(final File file) { LineNumberReader reader = null; try { reader = new LineNumberReader(new InputStreamReader(new FileInputStream(file), Util.ENCODING)); while (true) { final String line = reader.readLine(); if (line == null) { break; } if (!line.startsWith("#transaction started")) { throw new NoSqlStoreException( "No transaction start found: " + line + " (" + reader.getLineNumber() + ")"); } readTransaction(reader); } } catch (final IOException e) { throw new NoSqlStoreException(e); } finally { if (reader != null) { try { reader.close(); } catch (final IOException e) { throw new NoSqlStoreException(e); } } } } private void readTransaction(final LineNumberReader reader) throws IOException { final ArrayList<FileContent> files = new ArrayList<FileContent>(); final DataFileWriter content = new DataFileWriter(files); String header; while ((header = reader.readLine()) != null) { if (header.startsWith("#transaction ended")) { LOG.debug("transaction read in (ending " + reader.getLineNumber() + ")"); content.writeData(); reader.readLine(); return; } if (header.startsWith("S")) { final String[] split = header.substring(1).split(" "); final String key = split[0]; final String name = split[1]; server.saveService(key, name); reader.readLine(); } else if (header.startsWith("B")) { final String[] split = header.substring(1).split(" "); final String name = split[0]; final long nextBatch = Long.valueOf(split[1]); server.saveNextBatch(name, nextBatch); reader.readLine(); } else { FileContent elementData; elementData = readElementData(header, reader); files.add(elementData); } } LOG.warn("transaction has no ending marker so is incomplete and will not be restored (ending " + reader.getLineNumber() + ")"); } private FileContent readElementData(final String header, final LineNumberReader reader) throws IOException { final StringBuffer content = new StringBuffer(); String line; while ((line = reader.readLine()) != null) { if (line.length() == 0) { break; } content.append(line); content.append('\n'); } final char command = header.charAt(0); final String[] split = header.substring(1).split(" "); final String type = split[0]; final String id = split[1]; final String version = split[2]; return new FileContent(command, id, null, version, type, content.toString()); } private void startLogRolling() { final int rollPeriod = config.getInt("fileserver.log-period", 5); final long sleepTime = rollPeriod * 60 * 1000; while (awaitConnections) { final LogWriter logger = server.getLogger(); if (logger != null) { logger.startNewFile(); } try { Thread.sleep(sleepTime); } catch (final InterruptedException ignore) { } } } }