cfel.service.PortalImportTask.java Source code

Java tutorial

Introduction

Here is the source code for cfel.service.PortalImportTask.java

Source

/*******************************************************************************
 * Copyright (c) 2014 IBM Corporation and Others
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *   IBM Corporation - initial API and implementation
 *******************************************************************************/

package cfel.service;

import java.awt.image.BufferedImage;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Set;
import java.util.TimerTask;

import javax.imageio.ImageIO;
import javax.servlet.ServletContext;

import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONException;
import org.apache.wink.json4j.JSONObject;

import cfel.util.ImageUtil;
import cfel.util.WSClientUtil;

import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;

public class PortalImportTask extends TimerTask {
    public static final String ACTIVITY_SEARCH_URL = Config.PORTAL_URL + "/api.php/activity/search.json";
    public static final String ELMEMBER_SEARCH_API = Config.PORTAL_URL + "/api.php/elmember/search";
    public static final String FILE_URL = "/file/";
    public static final DateFormat FORMAT_DATE_ISO = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
    public static final DateFormat FORMAT_DATE = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z",
            Locale.ENGLISH);
    private static final String[] IMAGE_TYPES = new String[] { "thumbnail", "marker", "card" };

    private DatabaseServive mDS;
    private DBCollection mPhotoCollection;

    private final ServletContext mServletContext;

    public PortalImportTask(ServletContext servletContext) {
        super();
        this.mServletContext = servletContext;
        try {
            mDS = DatabaseServive.getInstance();
            mPhotoCollection = mDS.getCollection("photo");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    @Override
    public void run() {
        System.out.println("PortalImportTask started at " + new Date().toString());
        if (mPhotoCollection == null) {
            System.err.println("mongod is not running");
            return;
        }
        PortalActivity activity = new PortalActivity();
        Set<String> imageUrlSet = new HashSet<String>();
        for (int temp = 0; activity.hasNext() && temp < 999; temp++) {// while (activity.hasNext()) {
            JSONArray dataList = activity.getNext();
            if (dataList == null) {
                imageUrlSet = null;
                System.err.println("Error loading activities");
                break;
            }
            for (int i = 0; i < dataList.length(); i++) {
                try {
                    JSONObject obj = dataList.getJSONObject(i);
                    String imageUrl = getImageURL(obj);
                    if (imageUrl != null) {
                        imageUrlSet.add(imageUrl);
                        importActivity(obj, imageUrl);
                    }
                } catch (JSONException e) {
                    e.printStackTrace();
                }
            }
        }
        fixThumbnailImages();
        if (imageUrlSet != null) {
            fixActivities(imageUrlSet);
        }
        System.out.println("PortalImportTask finished at " + new Date().toString());
    }

    private void importActivity(JSONObject activity, String portal_image_url) {
        // System.out.println(activity.toString());
        try {
            JSONObject member = activity.getJSONObject("member");
            if (member.containsKey("id")) {
                String title = activity.getString("body").replace(activity.getString("image_large_url"), "").trim();
                DBObject photo = new BasicDBObject();
                photo.put("portal_image_url", portal_image_url);
                if (mPhotoCollection.count(photo) > 0) {
                    System.err.println(portal_image_url + " already exists");
                    return;
                } else {
                    System.out.println("Inserting " + portal_image_url);
                }
                String imageSourceUrl = Config.PORTAL_URL.startsWith("http:")
                        ? portal_image_url.replace("https://", "http://")
                        : portal_image_url;
                BufferedImage[] images = ImageUtil.convertImage(new URL(imageSourceUrl));
                if (images == null) {
                    System.err.println(portal_image_url + " corrupted");
                    return;
                }
                saveThumbnailImages(photo, images, "/" + getFileId(portal_image_url) + ".png");
                photo.put("uploader_user_id", member.getString("id"));
                photo.put("title", title);
                Object exifObj = getExifData(portal_image_url);
                Date date = null;
                if (exifObj instanceof JSONObject) {
                    JSONObject exif = (JSONObject) exifObj;
                    if (exif.containsKey("date")) {
                        try {
                            date = FORMAT_DATE_ISO.parse(exif.getString("date"));
                        } catch (ParseException e) {
                            e.printStackTrace();
                        }
                    }
                    if (exif.containsKey("location")) {
                        photo.put("exif_location", exif.getJSONArray("location"));
                    }
                }
                if (date == null && activity.containsKey("created_at")) {
                    try {
                        date = FORMAT_DATE.parse(activity.getString("created_at"));
                    } catch (ParseException e) {
                        e.printStackTrace();
                    }
                }
                if (date != null) {
                    photo.put("exif_date", date);

                }
                mPhotoCollection.insert(photo);
                // System.out.println(photo);
                // System.out.println();
            }
        } catch (JSONException | MalformedURLException e) {
            e.printStackTrace();
        }
    }

    private static String getImageURL(JSONObject activity) {
        String result = null;
        try {
            Object image_url = activity.get("image_large_url");
            if (image_url instanceof String && !image_url.toString().endsWith("/no_image.gif")
                    && activity.containsKey("member")) {
                result = (String) image_url;
                if (!result.matches("^https?://.*")) {
                    result = Config.PORTAL_ROOT + result;
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return result;
    }

    private static final String EXIF_SEARCH_URL = Config.PORTAL_URL + "/api.php/imgexf/search?apiKey="
            + Config.API_KEY;

    private JSONObject getExifData(String image_url) {
        Object obj = WSClientUtil.getJSON(EXIF_SEARCH_URL + "&name=" + getFileId(image_url));
        return (obj instanceof JSONObject) ? (JSONObject) obj : null;
    }

    private String getFileId(String image_url) {
        String[] part = image_url.split("/");
        String[] subPart = part[part.length - 1].split("\\.");
        return subPart[0];
    }

    private void saveThumbnailImages(DBObject photo, BufferedImage[] images, String suffix) {
        for (int i = 0; i < IMAGE_TYPES.length; i++) {
            String fileName = IMAGE_TYPES[i] + suffix;
            OutputStream os = mDS.getFileOutputStream(fileName, "image/png");
            try {
                if (os != null) {
                    ImageIO.write(images[i], "png", os);
                    photo.put(IMAGE_TYPES[i] + "_image_url", FILE_URL + fileName);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                if (os != null) {
                    try {
                        os.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                images[i].flush();
            }
        }
    }

    private void fixThumbnailImages() {
        DBObject query = new BasicDBObject("card_image_url", new BasicDBObject("$exists", false));
        for (Iterator<DBObject> it = mPhotoCollection.find(query); it.hasNext();) {
            DBObject photo = it.next();
            if (!photo.containsField("portal_image_url")) {
                System.err.println("No portal_image_url in " + photo);
                continue;
            }
            String portal_image_url = photo.get("portal_image_url").toString();
            if (!portal_image_url.matches("^https?://.*")) {
                portal_image_url = Config.ALBUM_ROOT + portal_image_url;
            }
            try {
                URL url = new URL(portal_image_url);
                System.err.println("Missing thumbnail images for " + url);
                BufferedImage[] images = ImageUtil.convertImage(url);
                if (images != null) {
                    DBObject updates = new BasicDBObject();
                    saveThumbnailImages(updates, images, url.getPath().replace(".", "_") + ".png");
                    mPhotoCollection.update(photo, new BasicDBObject("$set", updates));
                } else {
                    System.err.println(url + " corrupted");
                }
            } catch (MalformedURLException e) {
                System.err.println("Unknown image URL" + portal_image_url);
                continue;
            }
        }
    }

    private void fixActivities(Set<String> imageSet) {
        DBObject query = new BasicDBObject("$and",
                new DBObject[] { new BasicDBObject("portal_image_url", new BasicDBObject("$exists", true)),
                        new BasicDBObject("uploader_user_id", new BasicDBObject("$exists", true)) });
        for (Iterator<DBObject> it = mPhotoCollection.find(query); it.hasNext();) {
            DBObject photo = it.next();
            boolean temporary_removed = !imageSet.contains(photo.get("portal_image_url").toString());
            boolean removed = Boolean.TRUE.equals(photo.get("permanently_removed")) || temporary_removed;
            if (temporary_removed != Boolean.TRUE.equals(photo.get("temporary_removed"))) {
                mPhotoCollection.update(photo,
                        new BasicDBObject("$set", new BasicDBObject("temporary_removed", temporary_removed)));
            }
            if (removed != Boolean.TRUE.equals(photo.get("removed"))) {
                mPhotoCollection.update(photo, new BasicDBObject("$set", new BasicDBObject("removed", removed)));
                System.out.println("removed:" + removed + " " + photo);
            }
        }
    }
}