de.tudarmstadt.ukp.experiments.argumentation.sequence.significance.SignificanceMain.java Source code

Introduction

Here is the source code for de.tudarmstadt.ukp.experiments.argumentation.sequence.significance.SignificanceMain.java

Source

/*
 * Copyright 2016
 * Ubiquitous Knowledge Processing (UKP) Lab
 * Technische Universitt Darmstadt
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package de.tudarmstadt.ukp.experiments.argumentation.sequence.significance;

import com.google.common.collect.Table;
import com.google.common.collect.TreeBasedTable;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * @author Ivan Habernal
 */
public class SignificanceMain {
    private static final double P_VALUE = 0.001;
    private static final String TOKEN_LEVEL_PREDICTIONS_CSV = "tokenLevelPredictions.csv";

    public static void main(String[] args) throws Exception {
        // aggregated
        File mainFolder = new File(args[0]);

        //        compareMethods(mainFolder, "cv_");
        //        compareMethods(mainFolder, "cd_hs");
        //        compareMethods(mainFolder, "cd_ms");
        //        compareMethods(mainFolder, "cd_pis");
        //        compareMethods(mainFolder, "cd_pps");
        //        compareMethods(mainFolder, "cd_rs");
        //        compareMethods(mainFolder, "cd_sse");
        compareMethods(mainFolder, "cr_longToShort");
        compareMethods(mainFolder, "cr_shortToLong");

    }

    /**
     * Test significance between all methods in the folder and prints them as a CSV table
     *
     * @param mainFolder         folder
     * @param folderResultPrefix prefix of folders to test, i.e. "cv_"
     * @throws IOException
     */
    public static void compareMethods(File mainFolder, final String folderResultPrefix) throws IOException {
        File[] cvResults = mainFolder.listFiles(new FileFilter() {
            @Override
            public boolean accept(File pathname) {
                return pathname.isDirectory() && pathname.getName().startsWith(folderResultPrefix);
            }
        });

        Table<String, String, Boolean> table = TreeBasedTable.create();

        for (int i = 0; i < cvResults.length; i++) {
            for (int j = 0; j < cvResults.length; j++) {
                if (i != j) {
                    File one = cvResults[i];
                    File two = cvResults[j];
                    double pValue = TestSignificanceTwoFiles.compareId2OutcomeFiles(
                            new File(one, TOKEN_LEVEL_PREDICTIONS_CSV), new File(two, TOKEN_LEVEL_PREDICTIONS_CSV),
                            new OutcomeSuccessMapReaderSequenceTokenCSVImpl());

                    // order of methods does not matter
                    List<String> methods = new ArrayList<>();
                    methods.add(one.getName());
                    methods.add(two.getName());
                    Collections.sort(methods);

                    table.put(methods.get(0), methods.get(1), pValue < P_VALUE);

                    // and make symmetric matrix
                    table.put(methods.get(1), methods.get(0), pValue < P_VALUE);
                }
            }
        }

        System.out.println(tableToCsv(table));
    }

    /**
     * Prints table to output string as CSV
     *
     * @param out   output
     * @param <T>   value type
     * @param table table
     * @throws IOException
     */
    public static <T> String tableToCsv(Table<String, String, Boolean> table) throws IOException {
        StringWriter sw = new StringWriter();
        CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT);

        List<String> firstRow = new ArrayList<>();
        firstRow.add(" ");
        firstRow.addAll(table.columnKeySet());
        printer.printRecord(firstRow);

        for (String rowKey : table.rowKeySet()) {
            printer.print(rowKey);
            for (String columnKey : table.columnKeySet()) {
                printer.print(table.get(rowKey, columnKey));
            }
            printer.println();
        }

        printer.close();

        return sw.toString();
    }
}