Java tutorial
/* * MuninMX * Written by Enrico Kern, kern@clavain.com * www.clavain.com * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * * * * special thanks to the unbelievable machine company GmbH * www.unbelievable-machine.com */ package com.clavain; import com.clavain.alerts.Alert; import com.clavain.alerts.msg.PushOverMessage; import com.clavain.alerts.msg.ShortTextMessage; import com.clavain.alerts.msg.TTSMessage; import com.clavain.alerts.ratelimiter.PushOverLimiter; import com.clavain.alerts.ratelimiter.SMSLimiter; import com.clavain.alerts.ratelimiter.TTSLimiter; import com.clavain.checks.ReturnDebugTrace; import com.mongodb.BasicDBObject; import com.mongodb.DB; import com.mongodb.DBCollection; import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; import com.mongodb.ServerAddress; import com.mongodb.WriteConcern; import com.clavain.executors.MongoExecutor; import com.clavain.executors.MongoEssentialExecutor; import com.clavain.handlers.JettyLauncher; import com.clavain.json.ServiceCheck; import com.clavain.munin.MuninNode; import com.clavain.munin.MuninPlugin; import com.clavain.rca.Analyzer; import com.clavain.workers.PluginUpdater; import java.io.FileInputStream; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.util.ArrayList; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import static com.clavain.utils.Database.*; import static com.clavain.utils.Quartz.*; import com.clavain.utils.SocketCheck; import com.clavain.workers.MongoWorker; import java.util.Collections; import java.util.concurrent.CopyOnWriteArrayList; import org.quartz.Scheduler; import org.quartz.SchedulerFactory; import org.quartz.impl.StdSchedulerFactory; import static com.clavain.utils.Generic.getUnixtime; import com.clavain.workers.NewNodeWatcher; import static com.clavain.utils.Database.dbScheduleAllCustomJobs; import com.clavain.workers.DataRetentionWorker; import java.io.File; import java.io.ObjectInputStream; import com.clavain.checks.ReturnServiceCheck; import com.clavain.executors.MongoCheckExecutor; import com.clavain.workers.ErrorNotifyExecutor; import com.google.gson.Gson; import java.util.List; /** * * @author enricokern */ public class muninmxcd { public static Logger logger = Logger.getRootLogger(); public static Properties p = null; public static LinkedBlockingQueue<BasicDBObject> mongo_queue = new LinkedBlockingQueue<BasicDBObject>(); public static LinkedBlockingQueue<BasicDBObject> mongo_essential_queue = new LinkedBlockingQueue<BasicDBObject>(); public static LinkedBlockingQueue<BasicDBObject> mongo_check_queue = new LinkedBlockingQueue<BasicDBObject>(); public static boolean logMore = false; public static MongoClient m; public static DB db; public static DBCollection col; public static String version = "1.0 <Codename: Regret>"; public static Connection conn = null; public static CopyOnWriteArrayList<MuninNode> v_munin_nodes; public static CopyOnWriteArrayList<MuninPlugin> v_cinterval_plugins; public static Scheduler sched; public static Scheduler sched_custom; public static Scheduler sched_checks; public static CopyOnWriteArrayList<SocketCheck> v_sockets; // alerting public static LinkedBlockingQueue<PushOverMessage> notification_pushover_queue = new LinkedBlockingQueue<PushOverMessage>(); public static LinkedBlockingQueue<ShortTextMessage> notification_sms_queue = new LinkedBlockingQueue<ShortTextMessage>(); public static LinkedBlockingQueue<TTSMessage> notification_tts_queue = new LinkedBlockingQueue<TTSMessage>(); public static LinkedBlockingQueue<ReturnServiceCheck> check_error_queue = new LinkedBlockingQueue<ReturnServiceCheck>(); public static CopyOnWriteArrayList<Alert> v_alerts = new CopyOnWriteArrayList<>(); // RCA public static CopyOnWriteArrayList<Analyzer> v_analyzer = new CopyOnWriteArrayList<>(); public static int rcajobs_running = 0; public static int maxnodes = 100000; public static int maxchecks = 100000; public static int socketTimeout = 30000; // CHECKS public static CopyOnWriteArrayList<ServiceCheck> v_serviceChecks = new CopyOnWriteArrayList<>();; public static List<Integer> errorProcessing = new CopyOnWriteArrayList<Integer>(); public static String[] initialArgs; /** * @param args the command line arguments */ public static void main(String[] args) { if (args.length < 1) { System.err.println("Usage: java -jar MuninMXcd.jar <full path to config>"); System.exit(1); } try { initialArgs = args; p = new Properties(); FileInputStream propInFile = null; propInFile = new FileInputStream(args[0]); p.loadFromXML(propInFile); String logfile = p.getProperty("log.file"); socketTimeout = Integer.parseInt(p.getProperty("socket.timeout")); PatternLayout layout = new PatternLayout("%d{ISO8601} %-5p %m%n"); ConsoleAppender consoleAppender = new ConsoleAppender(layout); logger.addAppender(consoleAppender); FileAppender fileAppender = new FileAppender(layout, logfile, false); logger.addAppender(fileAppender); logger.info("MuninMX Collector Daemon - " + version + " starting up..."); logger.info("Loading configuration from <" + args[0] + ">"); String l_strLogLevel = p.getProperty("log.level"); // ALL | DEBUG | INFO | WARN | ERROR | FATAL | OFF: if (l_strLogLevel.equals("ALL")) { logger.setLevel(Level.ALL); } else if (l_strLogLevel.equals("DEBUG")) { logger.setLevel(Level.DEBUG); } else if (l_strLogLevel.equals("INFO")) { logger.setLevel(Level.INFO); } else if (l_strLogLevel.equals("WARN")) { logger.setLevel(Level.WARN); } else if (l_strLogLevel.equals("ERROR")) { logger.setLevel(Level.ERROR); } else if (l_strLogLevel.equals("FATAL")) { logger.setLevel(Level.FATAL); } else { logger.setLevel(Level.OFF); } if (p.getProperty("log.more") != null) { if (p.getProperty("log.more").equals("true")) { logMore = true; } } } catch (Exception ex) { System.err.println("Failed to Init basic logging infastructure: " + ex.getLocalizedMessage()); System.exit(1); } try { // connect to db logger.info("Connecting to TokuMX"); MongoClientOptions.Builder builder = new MongoClientOptions.Builder(); builder.connectionsPerHost(400); builder.autoConnectRetry(true); builder.threadsAllowedToBlockForConnectionMultiplier(10); // speed up inserts, we dont care if we miss some if the shit hits the fan builder.writeConcern(WriteConcern.NONE); m = new MongoClient(new ServerAddress(p.getProperty("mongo.host")), builder.build()); // connect to mysql connectToDatabase(p); // PreFilling Nodes, max 100 in concurrent logger.info("Loading initial MuninNode details. This can take a few minutes..."); v_munin_nodes = new CopyOnWriteArrayList<>(); v_cinterval_plugins = new CopyOnWriteArrayList<>(); v_sockets = new CopyOnWriteArrayList<>(); java.sql.Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery("SELECT * FROM nodes"); while (rs.next()) { MuninNode mn = new MuninNode(); mn.setHostname(rs.getString("hostname")); mn.setNodename(rs.getString("hostname")); mn.setNode_id(rs.getInt("id")); mn.setPort(rs.getInt("port")); mn.setUser_id(rs.getInt("user_id")); mn.setQueryInterval(rs.getInt("query_interval")); mn.setStr_via(rs.getString("via_host")); mn.setAuthpw(rs.getString("authpw")); mn.setGroup(rs.getString("groupname")); v_munin_nodes.add(mn); logger.info("* " + mn.getHostname() + " queued for pluginfetch"); } // launching quartz scheduler logger.info("Launching Scheduler"); SchedulerFactory sf = new StdSchedulerFactory("quartz.properties"); sched = sf.getScheduler(); sched.start(); // launching quartz scheduler for custom interval logger.info("Launching Custom Interval Scheduler"); SchedulerFactory sfc = new StdSchedulerFactory("customquartz.properties"); sched_custom = sfc.getScheduler(); sched_custom.start(); // starting API server new Thread(new JettyLauncher()).start(); int sleepTime = Integer.parseInt(p.getProperty("startup.sleeptime")); int startupIterations = Integer.parseInt(p.getProperty("startup.iterations")); // scheduling jobs int i = 0; for (MuninNode it_mn : v_munin_nodes) { if (i == startupIterations) { Thread.sleep(sleepTime); i = 0; logger.info("Waiting " + sleepTime + "ms for new scheduling slot"); } scheduleJob(it_mn); i++; } // schedule custom interval jobs dbScheduleAllCustomJobs(); // add all alerts dbAddAllAlerts(); // Service Checks logger.info("Launching Service Check Scheduler"); SchedulerFactory sfsc = new StdSchedulerFactory("checksquartz.properties"); sched_checks = sfsc.getScheduler(); sched_checks.start(); // load service checks from database stmt = conn.createStatement(); rs = stmt.executeQuery("SELECT * FROM service_checks"); while (rs.next()) { Gson gson = new Gson(); ServiceCheck tc = gson.fromJson(rs.getString("json"), ServiceCheck.class); tc.setCid(rs.getInt("id")); tc.setUser_id(rs.getInt("user_id")); v_serviceChecks.add(tc); logger.info("* " + tc.getCheckname() + " Service Check added"); } // queue service checks for (ServiceCheck it_sc : v_serviceChecks) { scheduleServiceCheck(it_sc); } // starting MongoExecutor new Thread(new MongoExecutor()).start(); // starting MongoExecutor for Package Tracking and Essential Informations new Thread(new MongoEssentialExecutor()).start(); // starting MongoExecutor for Service Checks new Thread(new MongoCheckExecutor()).start(); // starting newnodewatcher new Thread(new NewNodeWatcher()).start(); // start pushover sending message new Thread(new PushOverLimiter()).start(); // SMS Limiter new Thread(new SMSLimiter()).start(); // TTS Limiter new Thread(new TTSLimiter()).start(); // start DataRetention Worker new Thread(new DataRetentionWorker()).start(); // start Error Notify Inspector new Thread(new ErrorNotifyExecutor()).start(); int curTime; int toTime; int mb = 1024 * 1024; while (true) { Thread.sleep(5000); System.out.println("Mongo Queue Size: " + mongo_queue.size()); System.out.println("Mongo Check Queue Size: " + mongo_check_queue.size()); System.out.println("Mongo Essential Queue Size: " + mongo_essential_queue.size()); Runtime runtime = Runtime.getRuntime(); //Print used memory System.out.println("Used Memory:" + (runtime.totalMemory() - runtime.freeMemory()) / mb); //Print free memory System.out.println("Free Memory:" + runtime.freeMemory() / mb); //Print total available memory System.out.println("Total Memory:" + runtime.totalMemory() / mb); //Print Maximum available memory System.out.println("Max Memory:" + runtime.maxMemory() / mb); System.out.println(" "); if (p.getProperty("kill.sockets").equals("true")) { System.out.println("Sockets: " + v_sockets.size()); // check for sockets that we can kill curTime = getUnixtime(); for (SocketCheck sc : v_sockets) { toTime = curTime - 120; if (sc.getSocketCreated() < toTime) { if (!sc.getSocket().isClosed()) { logger.info("timing out socket... from: " + sc.getHostname()); sc.closeSocket(); v_sockets.remove(sc); } else { v_sockets.remove(sc); } } } } } } catch (Exception ex) { System.err.println("Something went wrong as fuck: " + ex.getLocalizedMessage()); logger.fatal("Something went wrong as fuck. exiting: " + ex.getLocalizedMessage()); ex.printStackTrace(); System.exit(1); } } // Establish a connection to the database private static boolean connectToDatabase(Properties p) { try { logger.info("Connecting to MySQL"); conn = DriverManager.getConnection("jdbc:mysql://" + p.getProperty("mysql.host") + ":" + p.getProperty("mysql.port") + "/" + p.getProperty("mysql.db") + "?" + "user=" + p.getProperty("mysql.user") + "&password=" + p.getProperty("mysql.pass") + "&autoReconnect=true&failOverReadOnly=false&maxReconnects=10"); return (true); } catch (Exception ex) { // handle any errors logger.fatal("Error connecting to database: " + ex.getMessage()); return (false); } } }