org.apache.lens.regression.util.Util.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.lens.regression.util.Util.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.lens.regression.util;

import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;

import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.ws.rs.core.Response;
import javax.xml.bind.*;
import javax.xml.datatype.DatatypeConfigurationException;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.lens.api.APIResult;
import org.apache.lens.api.StringList;
import org.apache.lens.api.jaxb.LensJAXBContext;
import org.apache.lens.api.metastore.ObjectFactory;
import org.apache.lens.api.metastore.XProperties;
import org.apache.lens.api.metastore.XProperty;

import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.jcraft.jsch.*;

import lombok.extern.slf4j.Slf4j;

@Slf4j
public class Util {

    private static final String PROPERTY_FILE = "lens.properties";
    private static String localFilePath = "target/";
    private static String localFile;
    private static String backupFile;

    private Util() {

    }

    public static Properties getPropertiesObj(String filename) {
        try {
            Properties properties = new Properties();
            log.info("filename: {}", filename);
            InputStream confStream = Util.class.getResourceAsStream("/" + filename);
            properties.load(confStream);
            confStream.close();
            return properties;
        } catch (IOException e) {
            log.info("Error in getProperies:", e);
        }
        return null;
    }

    public static String getProperty(String property) {
        Properties prop = Util.getPropertiesObj(PROPERTY_FILE);
        return prop.getProperty(property);
    }

    public static String runRemoteCommand(String command) throws JSchException, IOException {
        StringBuilder outputBuffer = new StringBuilder();
        StringBuilder print = new StringBuilder();

        String userName = Util.getProperty("lens.remote.username");
        String host = Util.getProperty("lens.remote.host");
        String password = Util.getProperty("lens.remote.password");

        log.info("Running command : {} on host : {} with user as {}", command, host, userName);

        JSch jsch = new JSch();
        Session session = jsch.getSession(userName, host, 22);
        session.setPassword(password);

        Properties config = new Properties();
        config.put("StrictHostKeyChecking", "no");

        session.setServerAliveInterval(10000);
        session.setConfig(config);
        session.connect(0);

        ChannelExec channel = (ChannelExec) session.openChannel("exec");
        InputStream commandOutput = channel.getInputStream();

        channel.setCommand(command);
        channel.connect();
        int readByte = commandOutput.read();
        char toAppend = ' ';
        while (readByte != 0xffffffff) {
            toAppend = (char) readByte;
            outputBuffer.append(toAppend);
            readByte = commandOutput.read();
            if (toAppend == '\n') {
                log.info(print.toString());
                print = new StringBuilder();
            } else {
                print.append(toAppend);
            }
        }
        channel.disconnect();
        session.disconnect();
        return outputBuffer.toString();
    }

    public static void writeFile(String fileName, String str) {
        try {
            PrintWriter out = new PrintWriter(fileName, "UTF-8");
            out.println(str);
            out.close();
        } catch (IOException e) {
            log.info("File Exception : ", e);
        }
    }

    public static APIResult getApiResult(Response response) {
        APIResult result = response.readEntity(APIResult.class);
        return result;
    }

    @SuppressWarnings("unchecked")
    public static <T> Object extractObject(String queryString, Class<T> c) throws IllegalAccessException {
        JAXBContext jaxbContext = null;
        Unmarshaller unmarshaller = null;
        StringReader reader = new StringReader(queryString);
        try {
            jaxbContext = new LensJAXBContext(ObjectFactory.class) {
            };
            unmarshaller = jaxbContext.createUnmarshaller();
            return (T) ((JAXBElement<?>) unmarshaller.unmarshal(reader)).getValue();
        } catch (JAXBException e) {
            System.out.println("Exception : " + e);
            return null;
        }
    }

    @SuppressWarnings("unchecked")
    public static <T> Object getObject(String queryString, Class<T> c) throws IllegalAccessException {
        JAXBContext jaxbContext = null;
        Unmarshaller unmarshaller = null;
        StringReader reader = new StringReader(queryString);
        try {
            jaxbContext = new LensJAXBContext(c);
            unmarshaller = jaxbContext.createUnmarshaller();
            return (T) unmarshaller.unmarshal(reader);
        } catch (JAXBException e) {
            System.out.println("Exception : " + e);
            return null;
        }
    }

    @SuppressWarnings("unchecked")
    public static <T> String convertObjectToXml(T object, Class<T> clazz, String functionName)
            throws SecurityException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {

        JAXBElement<T> root = null;
        StringWriter stringWriter = new StringWriter();
        ObjectFactory methodObject = new ObjectFactory();

        Method method = methodObject.getClass().getMethod(functionName, clazz);
        root = (JAXBElement<T>) method.invoke(methodObject, object);
        try {
            getMarshaller(clazz).marshal(root, stringWriter);
        } catch (JAXBException e) {
            log.error("Not able to marshall", e);
        }
        return stringWriter.toString();
    }

    public static Marshaller getMarshaller(Class clazz) {
        JAXBContext jaxbContext = null;
        try {
            jaxbContext = new LensJAXBContext(clazz);
            Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
            jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);

            return jaxbMarshaller;
        } catch (JAXBException e) {
            log.error("Error : ", e);
        }

        return null;
    }

    public static HashMap<String, String> stringListToMap(StringList stringList) {
        HashMap<String, String> map = new HashMap<String, String>();
        if (stringList == null) {
            return null;
        }
        List<String> list = stringList.getElements();
        for (String element : list) {
            StringTokenizer tk = new StringTokenizer(element, "=");
            map.put(tk.nextToken(), tk.nextToken());
        }
        return map;
    }

    public static HashMap<String, String> stringListToMap(String paramList) throws Exception {
        StringList stringList = (StringList) Util.getObject(paramList, StringList.class);
        return stringListToMap(stringList);
    }

    public static void changeConfig(HashMap<String, String> map, String remotePath) throws Exception {

        Path p = Paths.get(remotePath);
        String fileName = p.getFileName().toString();
        backupFile = localFilePath + "backup-" + fileName;
        localFile = localFilePath + fileName;
        log.info("Copying " + remotePath + " to " + localFile);
        remoteFile("get", remotePath, localFile);
        Files.copy(new File(localFile).toPath(), new File(backupFile).toPath(), REPLACE_EXISTING);

        DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
        Document doc = docBuilder.parse(new FileInputStream(localFile));
        doc.normalize();

        NodeList rootNodes = doc.getElementsByTagName("configuration");
        Node root = rootNodes.item(0);
        Element rootElement = (Element) root;
        NodeList property = rootElement.getElementsByTagName("property");

        for (int i = 0; i < property.getLength(); i++) { //Deleting redundant properties from the document
            Node prop = property.item(i);
            Element propElement = (Element) prop;
            Node propChild = propElement.getElementsByTagName("name").item(0);

            Element nameElement = (Element) propChild;
            if (map.containsKey(nameElement.getTextContent())) {
                rootElement.removeChild(prop);
                i--;
            }
        }

        Iterator<Entry<String, String>> ab = map.entrySet().iterator();
        while (ab.hasNext()) {
            Entry<String, String> entry = ab.next();
            String propertyName = entry.getKey();
            String propertyValue = entry.getValue();
            System.out.println(propertyName + " " + propertyValue + "\n");
            Node newNode = doc.createElement("property");
            rootElement.appendChild(newNode);
            Node newName = doc.createElement("name");
            Element newNodeElement = (Element) newNode;

            newName.setTextContent(propertyName);
            newNodeElement.appendChild(newName);

            Node newValue = doc.createElement("value");
            newValue.setTextContent(propertyValue);
            newNodeElement.appendChild(newValue);
        }
        prettyPrint(doc);
        remoteFile("put", remotePath, localFile);
    }

    /*
     * function to save the changes in the document
     */
    public static final void prettyPrint(Document xml)
            throws TransformerFactoryConfigurationError, TransformerException {
        xml.normalize();
        Transformer transformer = TransformerFactory.newInstance().newTransformer();
        transformer.setOutputProperty(OutputKeys.INDENT, "yes");
        transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
        StreamResult output = new StreamResult(new File(localFile));
        Source input = new DOMSource(xml);
        transformer.transform(input, output);
    }

    /*
     * function to download or upload a file to a remote server
     */
    public static void remoteFile(String function, String remotePath, String localPath)
            throws JSchException, SftpException {

        String serverUrl = getProperty("lens.remote.host");
        String serverUname = getProperty("lens.remote.username");
        String serverPass = getProperty("lens.remote.password");

        JSch jsch = new JSch();
        Session session = jsch.getSession(serverUname, serverUrl);

        UserInfo ui = null;
        session.setUserInfo(ui);
        session.setPassword(serverPass);

        Properties config = new Properties();
        config.put("StrictHostKeyChecking", "no");
        session.setConfig(config);
        session.connect();

        Channel channel = session.openChannel("sftp");
        channel.connect();

        ChannelSftp sftpChannel = (ChannelSftp) channel;
        if (function.equals("get")) {
            sftpChannel.get(remotePath, localPath);
        } else if (function.equals("put")) {
            sftpChannel.put(localPath, remotePath);
        }
        sftpChannel.exit();
        session.disconnect();

    }

    public static void changeConfig(String remotePath) throws JSchException, SftpException {
        String fileName = Paths.get(remotePath).getFileName().toString();
        backupFile = localFilePath + "backup-" + fileName;
        log.info("Copying " + backupFile + " to " + remotePath);
        remoteFile("put", remotePath, backupFile);
    }

    public static Map<String, String> mapFromXProperties(XProperties xProperties) {
        Map<String, String> properties = new HashMap<String, String>();
        if (xProperties != null && xProperties.getProperty() != null && !xProperties.getProperty().isEmpty()) {
            for (XProperty xp : xProperties.getProperty()) {
                properties.put(xp.getName(), xp.getValue());
            }
        }
        return properties;
    }

    public static XProperties xPropertiesFromMap(Map<String, String> map) {
        ObjectFactory xCF = new ObjectFactory();
        if (map != null && !map.isEmpty()) {
            XProperties xp = xCF.createXProperties();
            List<XProperty> xpList = xp.getProperty();
            for (Map.Entry<String, String> e : map.entrySet()) {
                XProperty property = xCF.createXProperty();
                property.setName(e.getKey());
                property.setValue(e.getValue());
                xpList.add(property);
            }

            return xp;
        }
        return null;
    }

    public static List<XProperty> xPropertyFromMap(Map<String, String> map) {
        List<XProperty> xpList = new ArrayList<XProperty>();
        if (map != null && !map.isEmpty()) {
            for (Map.Entry<String, String> e : map.entrySet()) {
                XProperty property = new XProperty();
                property.setName(e.getKey());
                property.setValue(e.getValue());
                xpList.add(property);
            }
        }
        return xpList;
    }

    public static XMLGregorianCalendar getXMLGregorianCalendar(Date d) {
        if (d == null) {
            return null;
        }

        GregorianCalendar c1 = new GregorianCalendar();
        c1.setTime(d);
        try {
            return DatatypeFactory.newInstance().newXMLGregorianCalendar(c1);
        } catch (DatatypeConfigurationException e) {
            log.info("Error converting date " + d, e);
            return null;
        }
    }

    public static String getJobIdFromProgressMsg(String progressMsg) {
        Pattern jobIdPattern = Pattern.compile("(job_\\d*_\\d*)");
        Matcher matcher = jobIdPattern.matcher(progressMsg);
        String jobid = null;
        if (matcher.find()) {
            jobid = matcher.group(1);
        }
        return jobid;
    }

    public static String getMapredJobPrority(String url, String jobId) throws IOException, IllegalAccessException,
            InstantiationException, ParserConfigurationException, SAXException {

        String xmlPathExpr = "/conf/property[name='mapreduce.job.priority']/value";
        String jobConfXml = sendHttpGetRequest(url.replace("JOB_ID", jobId));
        String priority = getValueFromXml(jobConfXml, xmlPathExpr);
        return priority;
    }

    public static String sendHttpGetRequest(String url) throws IOException {
        HttpGet request = new HttpGet(url);
        request.addHeader("Accept", "application/xml");
        HttpResponse response = new DefaultHttpClient().execute(request);
        BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), "UTF-8"));
        StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null) {
            result.append(line);
        }
        return result.toString();
    }

    public static Document getDocument(String xmlContent)
            throws ParserConfigurationException, SAXException, IOException {

        DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder();
        InputSource is = new InputSource();
        is.setCharacterStream(new StringReader(xmlContent));
        Document doc = db.parse(is);
        return doc;
    }

    public static String getValueFromXml(String xmlContent, String xmlPathExpr) {

        String value = null;
        try {
            Document doc = getDocument(xmlContent);
            XPath xpath = XPathFactory.newInstance().newXPath();
            value = xpath.compile(xmlPathExpr).evaluate(doc);

        } catch (ParserConfigurationException e) {
            log.error("error", e);
        } catch (SAXException e) {
            log.error("error", e);
        } catch (IOException e) {
            log.error("error", e);
        } catch (XPathExpressionException e) {
            log.error("error", e);
        }
        return value;
    }

    public static String getCurrentDate(String format) {
        SimpleDateFormat sdf = new SimpleDateFormat(format);
        return sdf.format(new Date());
    }

    public static String getFormattedDate(String epochTime, String format) {
        SimpleDateFormat sdf = new SimpleDateFormat(format);
        return sdf.format(Long.parseLong(epochTime));
    }

    public static String modifyDate(String dateString, String format, int calenderIntervalUnit, int diff)
            throws ParseException {

        SimpleDateFormat sdf = new SimpleDateFormat(format);
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(sdf.parse(dateString));
        calendar.add(calenderIntervalUnit, diff);
        return sdf.format(calendar.getTime());
    }

    public static XMLGregorianCalendar getGregorianCalendar(String date, String format)
            throws ParseException, DatatypeConfigurationException {
        GregorianCalendar cal = new GregorianCalendar();
        cal.setTime(new SimpleDateFormat(format).parse(date));
        cal.setTimeZone(TimeZone.getTimeZone("UTC"));
        XMLGregorianCalendar calendar = DatatypeFactory.newInstance().newXMLGregorianCalendar(cal);
        return calendar;
    }

    public static XMLGregorianCalendar getGregorianCalendar(String date)
            throws ParseException, DatatypeConfigurationException {
        return getGregorianCalendar(date, "yyyy-MM-dd HH:mm:ss"); //"yyyy-MM-dd'T'HH:mm:ss.SSSX"
    }

    public static String getDateStringFromGregorainCalender(XMLGregorianCalendar xcal, String format) {
        Date dt = xcal.toGregorianCalendar().getTime();
        SimpleDateFormat sdf = new SimpleDateFormat(format);
        return sdf.format(dt);
    }

}