com.importio.kinesis.elasticsearch.KinesisTransformer.java Source code

Introduction

Here is the source code for com.importio.kinesis.elasticsearch.KinesisTransformer.java

Source

/*
 * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Amazon Software License (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 * http://aws.amazon.com/asl/
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package com.importio.kinesis.elasticsearch;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.List;

import com.amazonaws.services.kinesis.connectors.elasticsearch.ElasticsearchObject;
import com.amazonaws.services.kinesis.connectors.elasticsearch.ElasticsearchTransformer;
import com.amazonaws.services.kinesis.connectors.interfaces.ICollectionTransformer;
import com.amazonaws.services.kinesis.model.Record;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.common.collect.Lists;
import com.importio.kinesis.elasticsearch.data.GeoPoint;
import com.importio.kinesis.elasticsearch.data.Page;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.AddressNotFoundException;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.record.City;
import com.maxmind.geoip2.record.Country;
import com.maxmind.geoip2.record.Location;
import com.maxmind.geoip2.record.Subdivision;

import lombok.AccessLevel;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.apachecommons.CommonsLog;

@CommonsLog
@FieldDefaults(level = AccessLevel.PRIVATE)
public class KinesisTransformer extends ElasticsearchTransformer<Page>
        implements ICollectionTransformer<Page, ElasticsearchObject> {

    /** The internal event type **/
    static final String EVENT_TYPE = "page";

    static final String INDEX_NAME = System.getProperty("es.index", "kinesis");
    static final String INDEX_TYPE = System.getProperty("es.type", EVENT_TYPE);

    final ObjectMapper mapper;
    DatabaseReader geoReader;

    @SneakyThrows(IOException.class)
    public KinesisTransformer() {

        // create Jackson mapper
        mapper = new ObjectMapper();
        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false);
        mapper.setSerializationInclusion(Include.NON_NULL);

        // creates the DatabaseReader object, which should be reused across lookups.
        geoReader = new DatabaseReader.Builder(getClass().getResourceAsStream("/GeoLite2-City.mmdb")).build();

    }

    @Override
    public ElasticsearchObject fromClass(Page record) throws IOException {

        String source = null;
        boolean create = true;
        try {
            source = mapper.writeValueAsString(record);
        } catch (JsonProcessingException e) {
            String message = "Error parsing record to JSON";
            log.error(message, e);
            throw new IOException(message, e);
        }

        ElasticsearchObject elasticsearchObject = new ElasticsearchObject(INDEX_NAME, INDEX_TYPE, null, source);
        elasticsearchObject.setCreate(create);

        return elasticsearchObject;
    }

    @Override
    public Collection<Page> toClass(Record record) throws IOException {

        List<Page> list = Lists.newArrayList();
        BufferedReader reader = new BufferedReader(
                new InputStreamReader(new ByteArrayInputStream(record.getData().array())));
        String line;
        while ((line = reader.readLine()) != null) {
            try {
                Page page = mapper.readValue(line, Page.class);

                if (!EVENT_TYPE.equals(page.getMetaEventType())) {
                    continue;
                }

                if (page.getMetaIp() != null && !page.getMetaIp().equals("127.0.0.1")) {
                    setGeoFields(page);
                }

                list.add(page);
            } catch (Exception e) {
                log.error("Failed conversion: " + line, e);
            }
        }
        return list;
    }

    private void setGeoFields(Page page) throws UnknownHostException, IOException, GeoIp2Exception {
        try {
            InetAddress ipAddress = InetAddress.getByName(page.getMetaIp());

            // Replace "city" with the appropriate method for your
            // database, e.g.,
            // "country".
            CityResponse response = geoReader.city(ipAddress);

            if (response != null) { // should throw an exception,
                // belt and braces

                Country country = response.getCountry();
                if (country != null) {
                    page.setCountry(country.getName());
                }

                Subdivision subdivision = response.getMostSpecificSubdivision();
                if (subdivision != null) {
                    page.setSubdivision(subdivision.getName()); // 'Minnesota'
                }

                City city = response.getCity();
                if (city != null) {
                    page.setCity(city.getName()); // 'Minneapolis'
                }

                Location location = response.getLocation();
                if (location != null && location.getLatitude() != null && location.getLatitude() != null) {
                    GeoPoint geo = new GeoPoint();
                    geo.setLat(location.getLatitude()); // 44.9733
                    geo.setLon(location.getLongitude()); // -93.2323
                    page.setGeoPoint(geo);
                }
            }
        } catch (AddressNotFoundException e) {
            // ignore
        }
    }
}