edu.anu.spice.Evaluation.java Source code

Introduction

Here is the source code for edu.anu.spice.Evaluation.java

Source

/*
 * Copyright (c) 2016, Peter Anderson <peter.anderson@anu.edu.au>
 *
 * This file is part of Semantic Propositional Image Caption Evaluation
 * (SPICE).
 * 
 * SPICE is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
    
 * SPICE is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
    
 * You should have received a copy of the GNU Affero General Public
 * License along with SPICE.  If not, see <http://www.gnu.org/licenses/>.
 * 
 */

package edu.anu.spice;

import org.json.simple.JSONAware;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;

import edu.anu.spice.TupleSet.Count;

public class Evaluation implements JSONAware {
    public double f; // f-1 score
    public double pr; // precision
    public double re; // recall
    public int tp; // true positives
    public int fp; // false positives
    public int fn; // false negatives
    public int numImages;

    public Evaluation() {
        f = 0.0;
        pr = 0.0;
        re = 0.0;
        tp = 0;
        fp = 0;
        fn = 0;
        numImages = 0;
    }

    protected void calcFScore(boolean allowNan) {
        double beta = 1.0;
        int ref_n = this.fn + this.tp;
        int test_n = this.fp + this.tp;
        if (ref_n > 0) {
            this.re = (double) tp / (double) ref_n;
            if (test_n > 0) {
                this.pr = (double) tp / (double) test_n;
            }
            if (this.pr > 0 && this.re > 0) {
                this.f = (1.0 + beta * beta) * (this.pr * this.re) / (beta * beta * this.pr + this.re);
            }
        }
        if (allowNan && ref_n == 0) {
            this.f = Double.NaN;
            this.pr = Double.NaN;
            this.re = Double.NaN;
        }
    }

    public Evaluation(TupleSet candidates, TupleSet references, boolean allowNan, boolean useSynsets) {
        this();
        this.numImages = 1;
        Count intersection;
        if (useSynsets) {
            intersection = candidates.match_similar(references);
        } else {
            intersection = candidates.match_exact(references);
        }
        this.tp = intersection.n;
        this.fp = candidates.size() - this.tp;
        this.fn = references.size() - this.tp;
        this.calcFScore(allowNan);
    }

    public String toString(String delim) {
        StringBuilder sb = new StringBuilder();
        sb.append(f + delim);
        sb.append(pr + delim);
        sb.append(re + delim);
        return sb.toString().trim();
    }

    public String toString() {
        return this.toString(",");
    }

    @SuppressWarnings("unchecked")
    @Override
    public String toJSONString() {
        JSONObject jsonObj = new JSONObject();
        jsonObj.put("tp", new Integer(tp));
        jsonObj.put("fp", new Integer(fp));
        jsonObj.put("fn", new Integer(fn));
        jsonObj.put("f", new Double(f));
        jsonObj.put("pr", new Double(pr));
        jsonObj.put("re", new Double(re));
        jsonObj.put("numImages", new Integer(numImages));
        return JSONValue.toJSONString(jsonObj);
    }

}