com.haskins.cloudtrailviewer.core.EventLoader.java Source code

Java tutorial

Introduction

Here is the source code for com.haskins.cloudtrailviewer.core.EventLoader.java

Source

/*    
CloudTrail Viewer, is a Java desktop application for reading AWS CloudTrail logs
files.
    
Copyright (C) 2015  Mark P. Haskins
    
This program is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software Foundation,
either version 3 of the License, or (at your option) any later version.
    
This program is distributed in the hope that it will be useful,but WITHOUT ANY 
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 
PARTICULAR PURPOSE.  See the GNU General Public License for more details.
    
You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package com.haskins.cloudtrailviewer.core;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.haskins.cloudtrailviewer.dao.AccountDao;
import com.haskins.cloudtrailviewer.model.AwsAccount;
import com.haskins.cloudtrailviewer.model.event.Event;
import com.haskins.cloudtrailviewer.model.filter.AllFilter;
import com.haskins.cloudtrailviewer.model.filter.CompositeFilter;
import com.haskins.cloudtrailviewer.model.load.LoadFileRequest;
import com.haskins.cloudtrailviewer.utils.EventUtils;
import com.haskins.cloudtrailviewer.utils.GeoIpUtils;

import javax.swing.*;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipException;

/**
 * Class responsible for Loading events into the database.
 * 
 * @author mark
 */
public class EventLoader implements Serializable {

    private final static Logger LOGGER = Logger.getLogger("CloudTrail");

    private final static int BUFFER_SIZE = 32;
    private static final long serialVersionUID = -6006847187319711509L;

    private final FilteredEventDatabase eventDb;

    private final List<EventLoaderListener> listeners = new ArrayList<>();

    /**
     * Default Constructor
     * @param database Database that events should be added too
     */
    public EventLoader(FilteredEventDatabase database) {
        eventDb = database;
        listeners.add(eventDb);
    }

    /**
     * registers a listener to the Loaded
     * @param listener reference to a listener
     */
    public void addEventLoaderListener(EventLoaderListener listener) {
        listeners.add(listener);
    }

    /**
     * Loads Events fro the local file system
     * @param request object containing Request information
     */
    public void loadEventsFromLocalFiles(final LoadFileRequest request) {

        SwingWorker worker = new SwingWorker<Void, Void>() {

            @Override
            public Void doInBackground() {

                if (request.getFilter() == null) {

                    CompositeFilter filters = new CompositeFilter();
                    filters.addFilter(new AllFilter());
                    request.setFilter(filters);
                }

                int count = 0;

                List<String> filenames = request.getFilenames();
                int total = filenames.size();

                for (String filename : filenames) {

                    count++;

                    for (EventLoaderListener l : listeners) {
                        l.processingFile(count, total);
                    }

                    try (InputStream stream = loadEventFromLocalFile(filename)) {
                        processStream(stream, request.getFilter());
                    } catch (IOException ioe) {
                        LOGGER.log(Level.WARNING, "Failed to load file : " + filename, ioe);
                    } catch (Exception e) {
                        LOGGER.log(Level.WARNING, "Failed to load file : " + filename, e);
                    }
                }

                for (EventLoaderListener l : listeners) {
                    l.finishedLoading();
                }

                return null;
            }
        };

        worker.execute();
    }

    /**
     * Loads events from S3
     * @param request Object containing request information
     */
    public void loadEventsFromS3(final LoadFileRequest request) {

        SwingWorker worker = new SwingWorker<Void, Void>() {

            @Override
            public Void doInBackground() {

                int count = 0;

                List<AwsAccount> accounts = AccountDao.getAllAccounts(true);
                if (accounts.isEmpty()) {
                    return null;
                }

                AwsAccount account = accounts.get(0);

                String key = account.getKey();
                String secret = account.getSecret();

                AWSCredentials credentials = new BasicAWSCredentials(key, secret);

                AmazonS3 s3Client = new AmazonS3Client(credentials);
                String bucketName = account.getBucket();

                if (request.getFilter() == null) {
                    CompositeFilter filters = new CompositeFilter();
                    filters.addFilter(new AllFilter());
                    request.setFilter(filters);
                }

                List<String> filenames = request.getFilenames();
                int total = filenames.size();

                for (String filename : filenames) {

                    count++;

                    for (EventLoaderListener l : listeners) {
                        l.processingFile(count, total);
                    }

                    try (InputStream stream = loadEventFromS3(s3Client, bucketName, filename)) {
                        processStream(stream, request.getFilter());

                    } catch (IOException ioe) {
                        LOGGER.log(Level.WARNING, "Failed to load file : " + filename, ioe);
                    } catch (Exception e) {
                        LOGGER.log(Level.WARNING, "Failed to load file : " + filename, e);
                    }
                }

                for (EventLoaderListener l : listeners) {
                    l.finishedLoading();
                }

                return null;
            }
        };

        worker.execute();
    }

    private InputStream loadEventFromLocalFile(final String file) throws IOException {

        byte[] encoded = Files.readAllBytes(Paths.get(file));
        return new ByteArrayInputStream(encoded);
    }

    ////////////////////////////////////////////////////////////////////////////
    ///// private methods
    ////////////////////////////////////////////////////////////////////////////
    private InputStream loadEventFromS3(AmazonS3 s3Client, String bucketName, final String key) throws IOException {

        S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, key));
        return s3Object.getObjectContent();
    }

    private String uncompress(InputStream stream) {

        StringBuilder json = new StringBuilder();

        try (GZIPInputStream gzis = new GZIPInputStream(stream, BUFFER_SIZE)) {

            try (BufferedReader bf = new BufferedReader(new InputStreamReader(gzis, "UTF-8"))) {

                String line;
                while ((line = bf.readLine()) != null) {
                    json.append(line);
                }
            }
        } catch (ZipException ex) {
            json.append(loadUncompressedFile(stream));
        } catch (UnsupportedEncodingException ex) {
            LOGGER.log(Level.WARNING, "File encoding not recognised : ", ex);
        } catch (IOException ex) {
            LOGGER.log(Level.WARNING, "Problem uncompressing file data : ", ex);
        }

        return json.toString();
    }

    private String loadUncompressedFile(InputStream stream) {

        StringBuilder json = new StringBuilder();

        Scanner scanner = new Scanner(stream, "UTF-8");
        while (scanner.hasNext()) {
            String line = scanner.next();
            json.append(line.replaceAll("(\\r|\\n|\\t)", ""));
        }

        // check if the first character is a { otherwise add one
        String firstChars = json.substring(0, 2);
        if (firstChars.equalsIgnoreCase("Re")) {
            json.insert(0, "{\"");
        } else if (firstChars.equalsIgnoreCase("\"R")) {
            json.insert(0, "{");
        }

        return json.toString();
    }

    private List<Event> createEvents(String json_string) {

        Gson g = new Gson();

        List<Event> events = new ArrayList<>();

        JsonObject jsonObject = new JsonParser().parse(json_string).getAsJsonObject();
        JsonArray records = (JsonArray) jsonObject.get("Records");

        for (Object record : records) {

            try {
                JsonObject obj = (JsonObject) record;
                Event e = g.fromJson(obj, Event.class);
                events.add(e);
            } catch (Exception e) {
                LOGGER.log(Level.WARNING, "Create Event from JSON : ", e);
            }
        }

        return events;
    }

    private void processStream(InputStream stream, CompositeFilter filter) {

        List<Event> events = createEvents(uncompress(stream));
        for (Event event : events) {

            GeoIpUtils.getInstance().populateGeoData(event);

            EventUtils.addTimestamp(event);
            if (filter.passes(event)) {

                event.getResourceInfo();
                eventDb.addEvent(event);
            }
        }
    }
}