Example usage for weka.classifiers Evaluation KBMeanInformation

List of usage examples for weka.classifiers Evaluation KBMeanInformation

Introduction

In this page you can find the example usage for weka.classifiers Evaluation KBMeanInformation.

Prototype

public final double KBMeanInformation() throws Exception 

Source Link

Document

Return the Kononenko & Bratko Information score in bits per instance.

Usage

From source file:FlexDMThread.java

License:Open Source License

public void run() {
    try {//  ww w  .j a  va  2  s .  c o m
        //Get the data from the source

        FlexDM.getMainData.acquire();
        Instances data = dataset.getSource().getDataSet();
        FlexDM.getMainData.release();

        //Set class attribute if undefined
        if (data.classIndex() == -1) {
            data.setClassIndex(data.numAttributes() - 1);
        }

        //Process hyperparameters for classifier
        String temp = "";
        for (int i = 0; i < classifier.getNumParams(); i++) {
            temp += classifier.getParameter(i).getName();
            temp += " ";
            if (classifier.getParameter(i).getValue() != null) {
                temp += classifier.getParameter(i).getValue();
                temp += " ";
            }
        }

        String[] options = weka.core.Utils.splitOptions(temp);

        //Print to console- experiment is starting
        if (temp.equals("")) { //no parameters
            temp = "results_no_parameters";
            try {
                System.out.println("STARTING CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName().substring(dataset.getName().lastIndexOf("\\") + 1)
                        + " with no parameters");
            } catch (Exception e) {
                System.out.println("STARTING CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName() + " with no parameters");
            }
        } else { //parameters
            try {
                System.out.println("STARTING CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName().substring(dataset.getName().lastIndexOf("\\") + 1)
                        + " with parameters " + temp);
            } catch (Exception e) {
                System.out.println("STARTING CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName() + " with parameters " + temp);
            }
        }

        //Create classifier, setting parameters
        weka.classifiers.Classifier x = createObject(classifier.getName());
        x.setOptions(options);
        x.buildClassifier(data);

        //Process the test selection
        String[] tempTest = dataset.getTest().split("\\s");

        //Create evaluation object for training and testing classifiers
        Evaluation eval = new Evaluation(data);
        StringBuffer predictions = new StringBuffer();

        //Train and evaluate classifier
        if (tempTest[0].equals("testset")) { //specified test file
            //Build classifier
            x.buildClassifier(data);

            //Open test file, load data
            //DataSource testFile = new DataSource(dataset.getTest().substring(7).trim());
            // Instances testSet = testFile.getDataSet();
            FlexDM.getTestData.acquire();
            Instances testSet = dataset.getTestFile().getDataSet();
            FlexDM.getTestData.release();

            //Set class attribute if undefined
            if (testSet.classIndex() == -1) {
                testSet.setClassIndex(testSet.numAttributes() - 1);
            }

            //Evaluate model
            Object[] array = { predictions, new Range(), new Boolean(true) };
            eval.evaluateModel(x, testSet, array);
        } else if (tempTest[0].equals("xval")) { //Cross validation
            //Build classifier
            x.buildClassifier(data);

            //Cross validate
            eval.crossValidateModel(x, data, Integer.parseInt(tempTest[1]), new Random(1), predictions,
                    new Range(), true);
        } else if (tempTest[0].equals("leavexval")) { //Leave one out cross validation
            //Build classifier
            x.buildClassifier(data);

            //Cross validate
            eval.crossValidateModel(x, data, data.numInstances() - 1, new Random(1), predictions, new Range(),
                    true);
        } else if (tempTest[0].equals("percent")) { //Percentage split of single data set
            //Set training and test sizes from percentage
            int trainSize = (int) Math.round(data.numInstances() * Double.parseDouble(tempTest[1]));
            int testSize = data.numInstances() - trainSize;

            //Load specified data
            Instances train = new Instances(data, 0, trainSize);
            Instances testSet = new Instances(data, trainSize, testSize);

            //Build classifier
            x.buildClassifier(train);

            //Train and evaluate model
            Object[] array = { predictions, new Range(), new Boolean(true) };
            eval.evaluateModel(x, testSet, array);
        } else { //Evaluate on training data
            //Test and evaluate model
            Object[] array = { predictions, new Range(), new Boolean(true) };
            eval.evaluateModel(x, data, array);
        }

        //create datafile for results
        String filename = dataset.getDir() + "/" + classifier.getDirName() + "/" + temp + ".txt";
        PrintWriter writer = new PrintWriter(filename, "UTF-8");

        //Print classifier, dataset, parameters info to file
        try {
            writer.println("CLASSIFIER: " + classifier.getName() + "\n DATASET: " + dataset.getName()
                    + "\n PARAMETERS: " + temp);
        } catch (Exception e) {
            writer.println("CLASSIFIER: " + classifier.getName() + "\n DATASET: " + dataset.getName()
                    + "\n PARAMETERS: " + temp);
        }

        //Add evaluation string to file
        writer.println(eval.toSummaryString());
        //Process result options
        if (checkResults("stats")) { //Classifier statistics
            writer.println(eval.toClassDetailsString());
        }
        if (checkResults("model")) { //The model
            writer.println(x.toString());
        }
        if (checkResults("matrix")) { //Confusion matrix
            writer.println(eval.toMatrixString());
        }
        if (checkResults("entropy")) { //Entropy statistics
            //Set options req'd to get the entropy stats
            String[] opt = new String[4];
            opt[0] = "-t";
            opt[1] = dataset.getName();
            opt[2] = "-k";
            opt[3] = "-v";

            //Evaluate model
            String entropy = Evaluation.evaluateModel(x, opt);

            //Grab the relevant info from the results, print to file
            entropy = entropy.substring(entropy.indexOf("=== Stratified cross-validation ===") + 35,
                    entropy.indexOf("=== Confusion Matrix ==="));
            writer.println("=== Entropy Statistics ===");
            writer.println(entropy);
        }
        if (checkResults("predictions")) { //The models predictions
            writer.println("=== Predictions ===\n");
            if (!dataset.getTest().contains("xval")) { //print header of predictions table if req'd
                writer.println(" inst#     actual  predicted error distribution ()");
            }
            writer.println(predictions.toString()); //print predictions to file
        }

        writer.close();

        //Summary file is semaphore controlled to ensure quality
        try { //get a permit
              //grab the summary file, write the classifiers details to it
            FlexDM.writeFile.acquire();
            PrintWriter p = new PrintWriter(new FileWriter(summary, true));
            if (temp.equals("results_no_parameters")) { //change output based on parameters
                temp = temp.substring(8);
            }

            //write percent correct, classifier name, dataset name to summary file
            p.write(dataset.getName() + ", " + classifier.getName() + ", " + temp + ", " + eval.correct() + ", "
                    + eval.incorrect() + ", " + eval.unclassified() + ", " + eval.pctCorrect() + ", "
                    + eval.pctIncorrect() + ", " + eval.pctUnclassified() + ", " + eval.kappa() + ", "
                    + eval.meanAbsoluteError() + ", " + eval.rootMeanSquaredError() + ", "
                    + eval.relativeAbsoluteError() + ", " + eval.rootRelativeSquaredError() + ", "
                    + eval.SFPriorEntropy() + ", " + eval.SFSchemeEntropy() + ", " + eval.SFEntropyGain() + ", "
                    + eval.SFMeanPriorEntropy() + ", " + eval.SFMeanSchemeEntropy() + ", "
                    + eval.SFMeanEntropyGain() + ", " + eval.KBInformation() + ", " + eval.KBMeanInformation()
                    + ", " + eval.KBRelativeInformation() + ", " + eval.weightedTruePositiveRate() + ", "
                    + eval.weightedFalsePositiveRate() + ", " + eval.weightedTrueNegativeRate() + ", "
                    + eval.weightedFalseNegativeRate() + ", " + eval.weightedPrecision() + ", "
                    + eval.weightedRecall() + ", " + eval.weightedFMeasure() + ", "
                    + eval.weightedAreaUnderROC() + "\n");
            p.close();

            //release semaphore
            FlexDM.writeFile.release();
        } catch (InterruptedException e) { //bad things happened
            System.err.println("FATAL ERROR OCCURRED: Classifier: " + cNum + " - " + classifier.getName()
                    + " on dataset " + dataset.getName());
        }

        //output we have successfully finished processing classifier
        if (temp.equals("no_parameters")) { //no parameters
            try {
                System.out.println("FINISHED CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName().substring(dataset.getName().lastIndexOf("\\") + 1)
                        + " with no parameters");
            } catch (Exception e) {
                System.out.println("FINISHED CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName() + " with no parameters");
            }
        } else { //with parameters
            try {
                System.out.println("FINISHED CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName().substring(dataset.getName().lastIndexOf("\\") + 1)
                        + " with parameters " + temp);
            } catch (Exception e) {
                System.out.println("FINISHED CLASSIFIER " + cNum + " - " + classifier.getName() + " on dataset "
                        + dataset.getName() + " with parameters " + temp);
            }
        }

        try { //get a permit
              //grab the log file, write the classifiers details to it
            FlexDM.writeLog.acquire();
            PrintWriter p = new PrintWriter(new FileWriter(log, true));

            Date date = new Date();
            Format formatter = new SimpleDateFormat("dd/MM/YYYY HH:mm:ss");
            //formatter.format(date)

            if (temp.equals("results_no_parameters")) { //change output based on parameters
                temp = temp.substring(8);
            }

            //write details to log file
            p.write(dataset.getName() + ", " + dataset.getTest() + ", \"" + dataset.getResult_string() + "\", "
                    + classifier.getName() + ", " + temp + ", " + formatter.format(date) + "\n");
            p.close();

            //release semaphore
            FlexDM.writeLog.release();
        } catch (InterruptedException e) { //bad things happened
            System.err.println("FATAL ERROR OCCURRED: Classifier: " + cNum + " - " + classifier.getName()
                    + " on dataset " + dataset.getName());
        }

        s.release();

    } catch (Exception e) {
        //an error occurred
        System.err.println("FATAL ERROR OCCURRED: " + e.toString() + "\nClassifier: " + cNum + " - "
                + classifier.getName() + " on dataset " + dataset.getName());
        s.release();
    }

}

From source file:adams.flow.core.EvaluationHelper.java

License:Open Source License

/**
 * Returns a statistical value from the evaluation object.
 *
 * @param eval   the evaluation object to get the value from
 * @param statistic   the type of value to return
 * @param classIndex   the class label index, for statistics like AUC
 * @return      the determined value, Double.NaN if not found
 * @throws Exception   if evaluation fails
 *//*ww  w .j  a va  2 s  . c o m*/
public static double getValue(Evaluation eval, EvaluationStatistic statistic, int classIndex) throws Exception {
    switch (statistic) {
    case NUMBER_CORRECT:
        return eval.correct();
    case NUMBER_INCORRECT:
        return eval.incorrect();
    case NUMBER_UNCLASSIFIED:
        return eval.unclassified();
    case PERCENT_CORRECT:
        return eval.pctCorrect();
    case PERCENT_INCORRECT:
        return eval.pctIncorrect();
    case PERCENT_UNCLASSIFIED:
        return eval.pctUnclassified();
    case KAPPA_STATISTIC:
        return eval.kappa();
    case MEAN_ABSOLUTE_ERROR:
        return eval.meanAbsoluteError();
    case ROOT_MEAN_SQUARED_ERROR:
        return eval.rootMeanSquaredError();
    case RELATIVE_ABSOLUTE_ERROR:
        return eval.relativeAbsoluteError();
    case ROOT_RELATIVE_SQUARED_ERROR:
        return eval.rootRelativeSquaredError();
    case CORRELATION_COEFFICIENT:
        return eval.correlationCoefficient();
    case SF_PRIOR_ENTROPY:
        return eval.SFPriorEntropy();
    case SF_SCHEME_ENTROPY:
        return eval.SFSchemeEntropy();
    case SF_ENTROPY_GAIN:
        return eval.SFEntropyGain();
    case SF_MEAN_PRIOR_ENTROPY:
        return eval.SFMeanPriorEntropy();
    case SF_MEAN_SCHEME_ENTROPY:
        return eval.SFMeanSchemeEntropy();
    case SF_MEAN_ENTROPY_GAIN:
        return eval.SFMeanEntropyGain();
    case KB_INFORMATION:
        return eval.KBInformation();
    case KB_MEAN_INFORMATION:
        return eval.KBMeanInformation();
    case KB_RELATIVE_INFORMATION:
        return eval.KBRelativeInformation();
    case TRUE_POSITIVE_RATE:
        return eval.truePositiveRate(classIndex);
    case NUM_TRUE_POSITIVES:
        return eval.numTruePositives(classIndex);
    case FALSE_POSITIVE_RATE:
        return eval.falsePositiveRate(classIndex);
    case NUM_FALSE_POSITIVES:
        return eval.numFalsePositives(classIndex);
    case TRUE_NEGATIVE_RATE:
        return eval.trueNegativeRate(classIndex);
    case NUM_TRUE_NEGATIVES:
        return eval.numTrueNegatives(classIndex);
    case FALSE_NEGATIVE_RATE:
        return eval.falseNegativeRate(classIndex);
    case NUM_FALSE_NEGATIVES:
        return eval.numFalseNegatives(classIndex);
    case IR_PRECISION:
        return eval.precision(classIndex);
    case IR_RECALL:
        return eval.recall(classIndex);
    case F_MEASURE:
        return eval.fMeasure(classIndex);
    case MATTHEWS_CORRELATION_COEFFICIENT:
        return eval.matthewsCorrelationCoefficient(classIndex);
    case AREA_UNDER_ROC:
        return eval.areaUnderROC(classIndex);
    case AREA_UNDER_PRC:
        return eval.areaUnderPRC(classIndex);
    case WEIGHTED_TRUE_POSITIVE_RATE:
        return eval.weightedTruePositiveRate();
    case WEIGHTED_FALSE_POSITIVE_RATE:
        return eval.weightedFalsePositiveRate();
    case WEIGHTED_TRUE_NEGATIVE_RATE:
        return eval.weightedTrueNegativeRate();
    case WEIGHTED_FALSE_NEGATIVE_RATE:
        return eval.weightedFalseNegativeRate();
    case WEIGHTED_IR_PRECISION:
        return eval.weightedPrecision();
    case WEIGHTED_IR_RECALL:
        return eval.weightedRecall();
    case WEIGHTED_F_MEASURE:
        return eval.weightedFMeasure();
    case WEIGHTED_MATTHEWS_CORRELATION_COEFFICIENT:
        return eval.weightedMatthewsCorrelation();
    case WEIGHTED_AREA_UNDER_ROC:
        return eval.weightedAreaUnderROC();
    case WEIGHTED_AREA_UNDER_PRC:
        return eval.weightedAreaUnderPRC();
    case UNWEIGHTED_MACRO_F_MEASURE:
        return eval.unweightedMacroFmeasure();
    case UNWEIGHTED_MICRO_F_MEASURE:
        return eval.unweightedMicroFmeasure();
    case BIAS:
        return eval.getPluginMetric(Bias.class.getName()).getStatistic(Bias.NAME);
    case RSQUARED:
        return eval.getPluginMetric(RSquared.class.getName()).getStatistic(RSquared.NAME);
    case SDR:
        return eval.getPluginMetric(SDR.class.getName()).getStatistic(SDR.NAME);
    case RPD:
        return eval.getPluginMetric(RPD.class.getName()).getStatistic(RPD.NAME);
    default:
        throw new IllegalArgumentException("Unhandled statistic field: " + statistic);
    }
}