org.dbpedia.spotlight.evaluation.SpotterMemoryEvaluator.java Source code

Introduction

Here is the source code for org.dbpedia.spotlight.evaluation.SpotterMemoryEvaluator.java

Source

/*
 * Copyright 2012 DBpedia Spotlight Development Team
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  Check our project website for information on how to acknowledge the authors and how to contribute to the project: http://spotlight.dbpedia.org
 */

package org.dbpedia.spotlight.evaluation;

import net.sf.json.JSONException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dbpedia.spotlight.exceptions.ConfigurationException;
import org.dbpedia.spotlight.exceptions.InitializationException;
import org.dbpedia.spotlight.exceptions.SpottingException;
import org.dbpedia.spotlight.model.SpotlightConfiguration;
import org.dbpedia.spotlight.model.Text;
import org.dbpedia.spotlight.spot.Spotter;
import org.dbpedia.spotlight.spot.lingpipe.LingPipeSpotter;
import org.dbpedia.spotlight.spot.opennlp.ExactSurfaceFormDictionary;
import org.dbpedia.spotlight.spot.opennlp.OpenNLPChunkerSpotter;
import org.dbpedia.spotlight.spot.opennlp.SurfaceFormDictionary;

import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.Scanner;

/**
 * Evaluator for memory consumption of {@link org.dbpedia.spotlight.spot.Spotter}s (and spot selectors).
 *
 * @author Joachim Daiber
 */
public class SpotterMemoryEvaluator {

    private final static Log LOG = LogFactory.getLog(SpotterMemoryEvaluator.class);

    public static void main(String[] args) throws IOException, JSONException, ConfigurationException,
            InitializationException, org.json.JSONException, SpottingException {

        File dictionary = new File(
                "/Users/jodaiber/Desktop/lrec_2012_spotting/surface_forms-Wikipedia-TitRedDis.thresh3.spotterDictionary");

        Spotter spotter = null;

        //
        if (args.length == 0) {
            LOG.error("server.properties is requested to continue...");
            return;
        }

        SpotlightConfiguration configuration = new SpotlightConfiguration(args[0]);

        int spotterNr = 0;

        switch (spotterNr) {
        case 0: {
            String openNLPDir = "/Users/jodaiber/Desktop/DBpedia/";
            SurfaceFormDictionary sfDictProbThresh3 = ExactSurfaceFormDictionary.fromLingPipeDictionary(dictionary,
                    false);
            System.out.println("Dictionary size: " + sfDictProbThresh3.size());
            File stopwordsFile = new File(openNLPDir + "stopwords.txt");
            spotter = OpenNLPChunkerSpotter.fromDir(openNLPDir, configuration.getI18nLanguageCode(),
                    sfDictProbThresh3, stopwordsFile);
            break;
        }
        case 1: {
            spotter = new LingPipeSpotter(dictionary, configuration.getAnalyzer());
            break;
        }
        }

        System.out.println("Using Spotter " + spotter.getName());

        System.out.println("Running GC.");
        System.gc();
        System.gc();
        System.gc();
        System.gc();

        int i = 0;

        LinkedList<Long> consumption = new LinkedList<Long>();

        for (File textFile : new File("/data/spotlight/csaw/original/crawledDocs").listFiles()) {

            if (!textFile.getName().endsWith(".txt"))
                continue;

            i++;
            if (i == 100)
                break;

            spotter.extract(new Text(new Scanner(textFile).useDelimiter("\\A").next()));

            consumption.addLast(
                    (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024));
            System.out.println("Memory consumption: " + consumption.getLast());
        }

        long total = 0;
        for (long step : consumption) {
            total += step;
        }

        System.out.println("Mean consumption: " + (total / consumption.size()));

    }
}