Example usage for java.util ArrayList addAll

List of usage examples for java.util ArrayList addAll

  1. HOME
  2. Java
  3. java
  4. java.util.*
  5. ArrayList
  6. addAll

Introduction

In this page you can find the example usage for java.util ArrayList addAll.

Prototype

public boolean addAll(Collection<? extends E> c)

Source Link

Document

Appends all of the elements in the specified collection to the end of this list, in the order that they are returned by the specified collection's Iterator.

Usage

From source file:POP3Mail.java

public static void main(String[] args) {
    try {//  w  w  w .ja  v a2s . c om
        LogManager.getLogManager().readConfiguration(new FileInputStream("logging.properties"));
    } catch (Exception e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }
    String server = null;
    String username = null;
    String password = null;
    if (new File("mail.properties").exists()) {
        Properties properties = new Properties();
        try {
            properties.load(new FileInputStream(new File("mail.properties")));
            server = properties.getProperty("server");
            username = properties.getProperty("username");
            password = properties.getProperty("password");
            ArrayList<String> list = new ArrayList<String>(
                    Arrays.asList(new String[] { server, username, password }));
            list.addAll(Arrays.asList(args));
            args = list.toArray(new String[list.size()]);
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
    if (args.length < 3) {
        System.err
                .println("Usage: POP3Mail <pop3 server hostname> <username> <password> [TLS [true=implicit]]");
        System.exit(1);
    }

    server = args[0];
    username = args[1];
    password = args[2];
    String proto = null;
    int messageid = -1;
    boolean implicit = false;
    for (int i = 3; i < args.length; ++i) {
        if (args[i].equals("-m")) {
            i += 1;
            messageid = Integer.parseInt(args[i]);
        }
    }

    // String proto = args.length > 3 ? args[3] : null;
    // boolean implicit = args.length > 4 ? Boolean.parseBoolean(args[4])
    // : false;

    POP3Client pop3;

    if (proto != null) {
        System.out.println("Using secure protocol: " + proto);
        pop3 = new POP3SClient(proto, implicit);
    } else {
        pop3 = new POP3Client();
    }
    pop3.setDefaultPort(110);
    System.out.println("Connecting to server " + server + " on " + pop3.getDefaultPort());

    // We want to timeout if a response takes longer than 60 seconds
    pop3.setDefaultTimeout(60000);
    // suppress login details
    pop3.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out), true));

    try {
        pop3.connect(server);
    } catch (IOException e) {
        System.err.println("Could not connect to server.");
        e.printStackTrace();
        System.exit(1);
    }

    try {
        if (!pop3.login(username, password)) {
            System.err.println("Could not login to server.  Check password.");
            pop3.disconnect();
            System.exit(1);
        }
        PrintWriter printWriter = new PrintWriter(new FileWriter("messages.csv"), true);
        POP3MessageInfo[] messages = null;
        POP3MessageInfo[] identifiers = null;
        if (messageid == -1) {
            messages = pop3.listMessages();
            identifiers = pop3.listUniqueIdentifiers();
        } else {
            messages = new POP3MessageInfo[] { pop3.listMessage(messageid) };
        }
        if (messages == null) {
            System.err.println("Could not retrieve message list.");
            pop3.disconnect();
            return;
        } else if (messages.length == 0) {
            System.out.println("No messages");
            pop3.logout();
            pop3.disconnect();
            return;
        }
        new File("../json").mkdirs();
        int count = 0;
        for (POP3MessageInfo msginfo : messages) {
            if (msginfo.number != identifiers[count].number) {
                throw new RuntimeException();
            }
            msginfo.identifier = identifiers[count].identifier;
            BufferedReader reader = (BufferedReader) pop3.retrieveMessageTop(msginfo.number, 0);
            ++count;
            if (count % 100 == 0) {
                logger.finest(String.format("%d %s", msginfo.number, msginfo.identifier));
            }
            System.out.println(String.format("%d %s", msginfo.number, msginfo.identifier));
            if (reader == null) {
                System.err.println("Could not retrieve message header.");
                pop3.disconnect();
                System.exit(1);
            }
            if (printMessageInfo(reader, msginfo.number, printWriter)) {
            }
        }
        printWriter.close();
        pop3.logout();
        pop3.disconnect();
    } catch (IOException e) {
        e.printStackTrace();
        return;
    }
}

From source file:ch.epfl.lsir.xin.test.SVDPPTest.java

/**
 * @param args/*from  w  w w .  j  ava 2s  .c  om*/
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//SVDPP");

    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File("conf//SVDPlusPlus.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    logger.flush();
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    double totalPrecision = 0;
    double totalRecall = 0;
    double totalMAP = 0;
    double totalNDCG = 0;
    double totalMRR = 0;
    double totalAUC = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    logger.flush();
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }

    for (int folder = 1; folder <= F; folder++) {
        System.out.println("Folder: " + folder);
        logger.println("Folder: " + folder);
        logger.flush();
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            if (testRatings.get(i).getValue() < 5)
                continue;
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a SVD++ recommendation model.");
        logger.flush();
        SVDPlusPlus algo = new SVDPlusPlus(trainRatingMatrix, false,
                ".//localModels//" + config.getString("NAME"));
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();

        //rating prediction accuracy
        double RMSE = 0;
        double MAE = 0;
        double precision = 0;
        double recall = 0;
        double map = 0;
        double ndcg = 0;
        double mrr = 0;
        double auc = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()), false);
            if (prediction > algo.getMaxRating())
                prediction = algo.getMaxRating();
            if (prediction < algo.getMinRating())
                prediction = algo.getMinRating();
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
        System.out.println("Folder --- MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Folder --- MAE: "
                + MAE + " RMSE: " + RMSE);
        //ranking accuracy
        if (algo.getTopN() > 0) {
            HashMap<Integer, ArrayList<ResultUnit>> results = new HashMap<Integer, ArrayList<ResultUnit>>();
            for (int i = 0; i < trainRatingMatrix.getRow(); i++) {
                ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
                if (rec == null)
                    continue;
                int total = testRatingMatrix.getUserRatingNumber(i);
                if (total == 0)//this user is ignored
                    continue;
                results.put(i, rec);
            }
            RankResultGenerator generator = new RankResultGenerator(results, algo.getTopN(), testRatingMatrix);
            precision = generator.getPrecisionN();
            totalPrecision = totalPrecision + precision;
            recall = generator.getRecallN();
            totalRecall = totalRecall + recall;
            map = generator.getMAPN();
            totalMAP = totalMAP + map;
            ndcg = generator.getNDCGN();
            totalNDCG = totalNDCG + ndcg;
            mrr = generator.getMRRN();
            totalMRR = totalMRR + mrr;
            auc = generator.getAUC();
            totalAUC = totalAUC + auc;
            System.out.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
            logger.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
        }

        logger.flush();
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    System.out.println("Precision@N: " + totalPrecision / F);
    System.out.println("Recall@N: " + totalRecall / F);
    System.out.println("MAP@N: " + totalMAP / F);
    System.out.println("MRR@N: " + totalMRR / F);
    System.out.println("NDCG@N: " + totalNDCG / F);
    System.out.println("AUC@N: " + totalAUC / F);

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F + "\n" + "Precision@N: " + totalPrecision / F + "\n"
            + "Recall@N: " + totalRecall / F + "\n" + "MAP@N: " + totalMAP / F + "\n" + "MRR@N: " + totalMRR / F
            + "\n" + "NDCG@N: " + totalNDCG / F + "\n" + "AUC@N: " + totalAUC / F);
    logger.flush();
    logger.close();
}

From source file:ch.epfl.lsir.xin.test.ItemBasedCFTest.java

/**
 * @param args/* w  w w .  j  a v  a2s  .  c o  m*/
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//ItemBasedCF");
    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File(".//conf//ItemBasedCF.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    double totalPrecision = 0;
    double totalRecall = 0;
    double totalMAP = 0;
    double totalNDCG = 0;
    double totalMRR = 0;
    double totalAUC = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }

    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }

    for (int folder = 1; folder <= F; folder++) {
        logger.println("Folder: " + folder);
        System.out.println("Folder: " + folder);
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        trainRatingMatrix.calculateGlobalAverage();
        trainRatingMatrix.calculateItemsMean();
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());
        logger.println("Initialize a item based collaborative filtering recommendation model.");
        ItemBasedCF algo = new ItemBasedCF(trainRatingMatrix);
        algo.setLogger(logger);
        algo.build();//if read local model, no need to build the model
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();

        //rating prediction accuracy
        double RMSE = 0;
        double MAE = 0;
        double precision = 0;
        double recall = 0;
        double map = 0;
        double ndcg = 0;
        double mrr = 0;
        double auc = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()), false);
            if (prediction > algo.getMaxRating())
                prediction = algo.getMaxRating();
            if (prediction < algo.getMinRating())
                prediction = algo.getMinRating();

            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
        System.out.println("Folder --- MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Folder --- MAE: "
                + MAE + " RMSE: " + RMSE);

        //ranking accuracy
        if (algo.getTopN() > 0) {
            HashMap<Integer, ArrayList<ResultUnit>> results = new HashMap<Integer, ArrayList<ResultUnit>>();
            for (int i = 0; i < trainRatingMatrix.getRow(); i++) {
                //               ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
                //               results.put(i, rec);
                ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
                if (rec == null)
                    continue;
                int total = testRatingMatrix.getUserRatingNumber(i);
                if (total == 0)//this user is ignored
                    continue;
                results.put(i, rec);
            }
            RankResultGenerator generator = new RankResultGenerator(results, algo.getTopN(), testRatingMatrix,
                    trainRatingMatrix);
            precision = generator.getPrecisionN();
            totalPrecision = totalPrecision + precision;
            recall = generator.getRecallN();
            totalRecall = totalRecall + recall;
            map = generator.getMAPN();
            totalMAP = totalMAP + map;
            ndcg = generator.getNDCGN();
            totalNDCG = totalNDCG + ndcg;
            mrr = generator.getMRRN();
            totalMRR = totalMRR + mrr;
            auc = generator.getAUC();
            totalAUC = totalAUC + auc;
            System.out.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
            logger.append("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc + "\n");
        }
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    System.out.println("Precision@N: " + totalPrecision / F);
    System.out.println("Recall@N: " + totalRecall / F);
    System.out.println("MAP@N: " + totalMAP / F);
    System.out.println("MRR@N: " + totalMRR / F);
    System.out.println("NDCG@N: " + totalNDCG / F);
    System.out.println("AUC@N: " + totalAUC / F);
    System.out.println("similarity: " + config.getString("SIMILARITY"));
    //MAE: 0.7227232762922241 RMSE: 0.9225576790122603 (MovieLens 100K, shrinkage 2500, neighbor size 40, PCC)
    //MAE: 0.7250636319353241 RMSE: 0.9242305485411567 (MovieLens 100K, shrinkage 25, neighbor size 40, PCC)
    //MAE: 0.7477213243604459 RMSE: 0.9512195004171138 (MovieLens 100K, shrinkage 2500, neighbor size 40, COSINE)

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F + "\n" + "Precision@N: " + totalPrecision / F + "\n"
            + "Recall@N: " + totalRecall / F + "\n" + "MAP@N: " + totalMAP / F + "\n" + "MRR@N: " + totalMRR / F
            + "\n" + "NDCG@N: " + totalNDCG / F + "\n" + "AUC@N: " + totalAUC / F);
    logger.flush();
    logger.close();
}

From source file:ch.epfl.lsir.xin.test.SocialRegTest.java

/**
 * @param args/*from  w w  w. j  ava  2  s  .  c  o  m*/
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//SocialReg");
    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File("conf//SocialReg.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    logger.flush();
    DataLoaderFile loader = new DataLoaderFile(".//data//Epinions-ratings.txt");
    loader.readSimple();
    //read social information
    loader.readRelation(".//data//Epinions-trust.txt");
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    double totalPrecision = 0;
    double totalRecall = 0;
    double totalMAP = 0;
    double totalNDCG = 0;
    double totalMRR = 0;
    double totalAUC = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    logger.flush();
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }

    for (int folder = 1; folder <= F; folder++) {
        System.out.println("Folder: " + folder);
        logger.println("Folder: " + folder);
        logger.flush();
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = dataset.getUserIDMapping();
        HashMap<String, Integer> itemIDIndexMapping = dataset.getItemIDMapping();
        //         for( int i = 0 ; i < dataset.getUserIDs().size() ; i++ )
        //         {
        //            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        //         }
        //         for( int i = 0 ; i < dataset.getItemIDs().size() ; i++ )
        //         {
        //            itemIDIndexMapping.put(dataset.getItemIDs().get(i) , i);
        //         }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a social regularization recommendation model.");
        logger.flush();
        SocialReg algo = new SocialReg(trainRatingMatrix, dataset.getRelationships(), false,
                ".//localModels//" + config.getString("NAME"));
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();

        System.out.println(trainRatings.size() + " vs. " + testRatings.size());

        //rating prediction accuracy
        double RMSE = 0;
        double MAE = 0;
        double precision = 0;
        double recall = 0;
        double map = 0;
        double ndcg = 0;
        double mrr = 0;
        double auc = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()));
            if (prediction > algo.getMaxRating())
                prediction = algo.getMaxRating();
            if (prediction < algo.getMinRating())
                prediction = algo.getMinRating();
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
        System.out.println("Folder --- MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Folder --- MAE: "
                + MAE + " RMSE: " + RMSE);
        //ranking accuracy
        //         if( algo.getTopN() > 0 )
        //         {
        //            HashMap<Integer , ArrayList<ResultUnit>> results = new HashMap<Integer , ArrayList<ResultUnit>>();
        //            for( int i = 0 ; i < trainRatingMatrix.getRow() ; i++ )
        //            {
        //               ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
        //               results.put(i, rec);
        //            }
        //            RankResultGenerator generator = new RankResultGenerator(results , algo.getTopN() , testRatingMatrix);
        //            precision = generator.getPrecisionN();
        //            totalPrecision = totalPrecision + precision;
        //            recall = generator.getRecallN();
        //            totalRecall = totalRecall + recall;
        //            map = generator.getMAPN();
        //            totalMAP = totalMAP + map;
        //            ndcg = generator.getNDCGN();
        //            totalNDCG = totalNDCG + ndcg;
        //            mrr = generator.getMRRN();
        //            totalMRR = totalMRR + mrr;
        //            auc = generator.getAUC();
        //            totalAUC = totalAUC + auc;
        //            System.out.println("Folder --- precision: " + precision + " recall: " + 
        //            recall + " map: " + map + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
        //            logger.println("Folder --- precision: " + precision + " recall: " + 
        //                  recall + " map: " + map + " ndcg: " + ndcg + " mrr: " + 
        //                  mrr + " auc: " + auc);
        //         }

        logger.flush();
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    System.out.println("Precision@N: " + totalPrecision / F);
    System.out.println("Recall@N: " + totalRecall / F);
    System.out.println("MAP@N: " + totalMAP / F);
    System.out.println("MRR@N: " + totalMRR / F);
    System.out.println("NDCG@N: " + totalNDCG / F);
    System.out.println("AUC@N: " + totalAUC / F);

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F + "\n" + "Precision@N: " + totalPrecision / F + "\n"
            + "Recall@N: " + totalRecall / F + "\n" + "MAP@N: " + totalMAP / F + "\n" + "MRR@N: " + totalMRR / F
            + "\n" + "NDCG@N: " + totalNDCG / F + "\n" + "AUC@N: " + totalAUC / F);
    logger.flush();
    logger.close();
}

From source file:ch.epfl.lsir.xin.test.BiasedMFTest.java

/**
 * @param args/*from  w ww  .  j a  va  2  s  .c  om*/
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//BiasedMF");

    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File("conf//biasedMF.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    logger.flush();
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    double totalPrecision = 0;
    double totalRecall = 0;
    double totalMAP = 0;
    double totalNDCG = 0;
    double totalMRR = 0;
    double totalAUC = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    logger.flush();
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }

    for (int folder = 1; folder <= F; folder++) {
        System.out.println("Folder: " + folder);
        logger.println("Folder: " + folder);
        logger.flush();
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            //            if( testRatings.get(i).getValue() < 5 )
            //               continue;
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a biased matrix factorization recommendation model.");
        logger.flush();
        BiasedMF algo = new BiasedMF(trainRatingMatrix, false, ".//localModels//" + config.getString("NAME"));
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();

        //rating prediction accuracy
        double RMSE = 0;
        double MAE = 0;
        double precision = 0;
        double recall = 0;
        double map = 0;
        double ndcg = 0;
        double mrr = 0;
        double auc = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()), false);
            if (prediction > algo.getMaxRating())
                prediction = algo.getMaxRating();
            if (prediction < algo.getMinRating())
                prediction = algo.getMinRating();
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
        System.out.println("Folder --- MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Folder --- MAE: "
                + MAE + " RMSE: " + RMSE);
        //ranking accuracy
        if (algo.getTopN() > 0) {
            HashMap<Integer, ArrayList<ResultUnit>> results = new HashMap<Integer, ArrayList<ResultUnit>>();
            for (int i = 0; i < trainRatingMatrix.getRow(); i++) {
                ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
                if (rec == null)
                    continue;
                int total = testRatingMatrix.getUserRatingNumber(i);
                if (total == 0)//this user is ignored
                    continue;
                results.put(i, rec);
            }
            RankResultGenerator generator = new RankResultGenerator(results, algo.getTopN(), testRatingMatrix,
                    trainRatingMatrix);
            precision = generator.getPrecisionN();
            totalPrecision = totalPrecision + precision;
            recall = generator.getRecallN();
            totalRecall = totalRecall + recall;
            map = generator.getMAPN();
            totalMAP = totalMAP + map;
            ndcg = generator.getNDCGN();
            totalNDCG = totalNDCG + ndcg;
            mrr = generator.getMRRN();
            totalMRR = totalMRR + mrr;
            auc = generator.getAUC();
            totalAUC = totalAUC + auc;
            System.out.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
            logger.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
        }

        logger.flush();
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    System.out.println("Precision@N: " + totalPrecision / F);
    System.out.println("Recall@N: " + totalRecall / F);
    System.out.println("MAP@N: " + totalMAP / F);
    System.out.println("MRR@N: " + totalMRR / F);
    System.out.println("NDCG@N: " + totalNDCG / F);
    System.out.println("AUC@N: " + totalAUC / F);

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F + "\n" + "Precision@N: " + totalPrecision / F + "\n"
            + "Recall@N: " + totalRecall / F + "\n" + "MAP@N: " + totalMAP / F + "\n" + "MRR@N: " + totalMRR / F
            + "\n" + "NDCG@N: " + totalNDCG / F + "\n" + "AUC@N: " + totalAUC / F);
    logger.flush();
    logger.close();
}

From source file:ch.epfl.lsir.xin.test.UserBasedCFTest.java

/**
 * @param args//  www .j a v a 2s .c  om
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//UserBasedCF");
    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File(".//conf//UserBasedCF.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    double totalPrecision = 0;
    double totalRecall = 0;
    double totalMAP = 0;
    double totalNDCG = 0;
    double totalMRR = 0;
    double totalAUC = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }

    for (int folder = 1; folder <= F; folder++) {
        logger.println("Folder: " + folder);
        System.out.println("Folder: " + folder);
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        trainRatingMatrix.calculateGlobalAverage();
        trainRatingMatrix.calculateUsersMean();
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            //            if( testRatings.get(i).getValue() < 5 )
            //               continue;
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        logger.println("Initialize a user based collaborative filtering recommendation model.");
        UserBasedCF algo = new UserBasedCF(trainRatingMatrix, false,
                ".//localModels//" + config.getString("NAME"));
        algo.setLogger(logger);
        algo.build();//if read local model, no need to build the model
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        System.out.println(trainRatings.size() + " vs. " + testRatings.size());
        logger.flush();

        //rating prediction accuracy
        double RMSE = 0;
        double MAE = 0;
        double precision = 0;
        double recall = 0;
        double map = 0;
        double ndcg = 0;
        double mrr = 0;
        double auc = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()), false);

            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            if (prediction > algo.getMaxRating())
                prediction = algo.getMaxRating();
            if (prediction < algo.getMinRating())
                prediction = algo.getMinRating();
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
        System.out.println("Folder --- MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Folder --- MAE: "
                + MAE + " RMSE: " + RMSE);
        logger.flush();
        //ranking accuracy
        if (algo.getTopN() > 0) {
            HashMap<Integer, ArrayList<ResultUnit>> results = new HashMap<Integer, ArrayList<ResultUnit>>();
            for (int i = 0; i < testRatingMatrix.getRow(); i++) {
                ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
                if (rec == null)
                    continue;
                int total = testRatingMatrix.getUserRatingNumber(i);
                if (total == 0)//this user is ignored
                    continue;
                results.put(i, rec);
                //               for( Map.Entry<Integer, Double> entry : testRatingMatrix.getRatingMatrix().get(i).entrySet() )
                //               {
                //                  System.out.print( entry.getKey() + "(" + entry.getValue() + ") , ");
                //               }
                //               System.out.println();
                //               for( int j = 0 ; j < rec.size() ; j++ )
                //               {
                //                  System.out.print(rec.get(j).getItemIndex() + "(" + rec.get(j).getPrediciton() +
                //                        ") , ");
                //               }
                //               System.out.println("**********");
            }
            RankResultGenerator generator = new RankResultGenerator(results, algo.getTopN(), testRatingMatrix,
                    trainRatingMatrix);
            precision = generator.getPrecisionN();
            totalPrecision = totalPrecision + precision;
            recall = generator.getRecallN();
            totalRecall = totalRecall + recall;
            map = generator.getMAPN();
            totalMAP = totalMAP + map;
            ndcg = generator.getNDCGN();
            totalNDCG = totalNDCG + ndcg;
            mrr = generator.getMRRN();
            totalMRR = totalMRR + mrr;
            auc = generator.getAUC();
            totalAUC = totalAUC + auc;
            System.out.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
            logger.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
        }
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    System.out.println("Precision@N: " + totalPrecision / F);
    System.out.println("Recall@N: " + totalRecall / F);
    System.out.println("MAP@N: " + totalMAP / F);
    System.out.println("MRR@N: " + totalMRR / F);
    System.out.println("NDCG@N: " + totalNDCG / F);
    System.out.println("AUC@N: " + totalAUC / F);
    // MovieLens100k
    //MAE: 0.7343907480119425 RMSE: 0.9405808357192891 (MovieLens 100K, shrinkage 25, neighbor size 60, PCC)
    //MAE: 0.7522376630596646 RMSE: 0.9520931265724659 (MovieLens 100K, no shrinkage , neighbor size 40, COSINE)
    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F + "\n" + "Precision@N: " + totalPrecision / F + "\n"
            + "Recall@N: " + totalRecall / F + "\n" + "MAP@N: " + totalMAP / F + "\n" + "MRR@N: " + totalMRR / F
            + "\n" + "NDCG@N: " + totalNDCG / F + "\n" + "AUC@N: " + totalAUC / F);
    logger.flush();
    logger.close();

}

From source file:ch.epfl.lsir.xin.test.MFTest.java

/**
 * @param args//from   ww w.  ja v  a 2 s  . co  m
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//MF");

    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File("conf//MF.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    logger.flush();
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    double totalPrecision = 0;
    double totalRecall = 0;
    double totalMAP = 0;
    double totalNDCG = 0;
    double totalMRR = 0;
    double totalAUC = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    logger.flush();
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }

    for (int folder = 1; folder <= F; folder++) {
        System.out.println("Folder: " + folder);
        logger.println("Folder: " + folder);
        logger.flush();
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            //            if( testRatings.get(i).getValue() < 5 )
            //               continue;
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a matrix factorization based recommendation model.");
        logger.flush();
        MatrixFactorization algo = new MatrixFactorization(trainRatingMatrix, false,
                ".//localModels//" + config.getString("NAME"));
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();

        //rating prediction accuracy
        double RMSE = 0;
        double MAE = 0;
        double precision = 0;
        double recall = 0;
        double map = 0;
        double ndcg = 0;
        double mrr = 0;
        double auc = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()), false);
            if (prediction > algo.getMaxRating())
                prediction = algo.getMaxRating();
            if (prediction < algo.getMinRating())
                prediction = algo.getMinRating();
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
        System.out.println("Folder --- MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Folder --- MAE: "
                + MAE + " RMSE: " + RMSE);
        //ranking accuracy
        if (algo.getTopN() > 0) {
            HashMap<Integer, ArrayList<ResultUnit>> results = new HashMap<Integer, ArrayList<ResultUnit>>();
            for (int i = 0; i < trainRatingMatrix.getRow(); i++) {
                ArrayList<ResultUnit> rec = algo.getRecommendationList(i);
                if (rec == null)
                    continue;
                int total = testRatingMatrix.getUserRatingNumber(i);
                if (total == 0)//this user is ignored
                    continue;
                results.put(i, rec);
                //               for( Map.Entry<Integer, Double> entry : testRatingMatrix.getRatingMatrix().get(i).entrySet() )
                //               {
                //                  System.out.print( entry.getKey() + "(" + entry.getValue() + ") , ");
                //               }
                //               System.out.println();
                //               for( int j = 0 ; j < rec.size() ; j++ )
                //               {
                //                  System.out.print(rec.get(j).getItemIndex() + "(" + rec.get(j).getPrediciton() +
                //                        ") , ");
                //               }
                //               System.out.println("**********");
            }
            RankResultGenerator generator = new RankResultGenerator(results, algo.getTopN(), testRatingMatrix,
                    trainRatingMatrix);
            precision = generator.getPrecisionN();
            totalPrecision = totalPrecision + precision;
            recall = generator.getRecallN();
            totalRecall = totalRecall + recall;
            map = generator.getMAPN();
            totalMAP = totalMAP + map;
            ndcg = generator.getNDCGN();
            totalNDCG = totalNDCG + ndcg;
            mrr = generator.getMRRN();
            totalMRR = totalMRR + mrr;
            auc = generator.getAUC();
            totalAUC = totalAUC + auc;
            System.out.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
            logger.println("Folder --- precision: " + precision + " recall: " + recall + " map: " + map
                    + " ndcg: " + ndcg + " mrr: " + mrr + " auc: " + auc);
        }

        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " MAE: " + MAE
                + " RMSE: " + RMSE);
        logger.flush();
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    System.out.println("Precision@N: " + totalPrecision / F);
    System.out.println("Recall@N: " + totalRecall / F);
    System.out.println("MAP@N: " + totalMAP / F);
    System.out.println("MRR@N: " + totalMRR / F);
    System.out.println("NDCG@N: " + totalNDCG / F);
    System.out.println("AUC@N: " + totalAUC / F);

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F + "\n" + "Precision@N: " + totalPrecision / F + "\n"
            + "Recall@N: " + totalRecall / F + "\n" + "MAP@N: " + totalMAP / F + "\n" + "MRR@N: " + totalMRR / F
            + "\n" + "NDCG@N: " + totalNDCG / F + "\n" + "AUC@N: " + totalAUC / F);
    logger.flush();
    logger.close();

}

From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] pArgs) throws Exception {
    String id = "proportion-of-children-under-5-years-who-have-ever-breastfed-by-county-xls-2005-6";
    String sId = EncodeURIComponent(id);
    String url = "https://africaopendata.org/api/action";

    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", "Test");
    dataset.put("id", id);

    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);
    LOG.info(/*from w  w w. jav  a 2 s  .c o m*/
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));

        if (doc != null && doc.containsKey("result")) {
            Document result = new Document();
            LOG.info(doc.get("result").getClass().toString());
            if (doc.get("result") instanceof Document) {
                LOG.info("!!! Document result !!!!");
                result = (Document) doc.get("result");
            } else if (doc.get("result") instanceof ArrayList) {
                LOG.info("!!! Document list !!!!");

                result = (Document) (((ArrayList) doc.get("result")).get(0));
            } else {
                LOG.info("!!! NOT FOUND !!!!");
                result = null;
            }
            LOG.info("result find!");
            if (result != null) {
                dataset.put("title", result.get("title"));
                dataset.put("author", result.get("author"));
                dataset.put("author_email", result.get("author_email"));
                dataset.put("license_id", result.get("license_id"));
            }

            ArrayList<String> tags = new ArrayList<String>();
            ArrayList<String> meta = new ArrayList<String>();
            ArrayList<String> other_tags = new ArrayList<String>();

            if (result.containsKey("author") && result.get("author") != null)
                other_tags.add(result.get("author").toString());
            if (result.containsKey("title") && result.get("title") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
            if (result.containsKey("description") && result.get("description") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));

            ArrayList<Document> resources = new ArrayList<Document>();
            if (result != null && result.containsKey("resources")) {
                resources = (ArrayList<Document>) result.get("resources");
                for (Document resource : resources) {
                    if (resource.getString("format").toUpperCase().equals("JSON")) {
                        dataset.put("dataset-type", "JSON");
                        dataset.put("url", resource.get("url"));
                        dataset.put("created", resource.get("created"));
                        dataset.put("description", resource.get("description"));
                        dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                        meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
                    }
                }
            }

            if (result != null && result.containsKey("tags")) {
                ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
                for (Document tag : tagsFromCKAN) {
                    if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    } else if (tag.containsKey("display_name")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    }
                }

            }

            dataset.put("tags", tags);
            dataset.put("meta", meta);
            dataset.put("resources", resources);
            dataset.put("other_tags", other_tags);

        }

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    //GetMetaByDocument("http://catalog.data.gov/api/action/package_show?id=1e68f387-5f1c-46c0-a0d1-46044ffef5bf");
}

From source file:net.sf.tweety.cli.plugins.CliMain.java

public static void main(String[] args) {

    // check, if first call parameter is for the helptext
    if (args.length == 0) {
        System.out.println("Welcome to the Tweety command line interface.");
        System.out.println("Obtain help with command --help");
        System.exit(0);//www .  ja v a  2s.  c  o m
    } else if ((args.length == 1 && args[0].equals("--help"))) {
        printHelpText();
        System.exit(0);
    } else if (args.length == 1 && !args[0].contains("--help")) {
        System.out.println("No valid input, call with --help for helptext");
        System.exit(0);
    }

    // create new plugin manager
    PluginManager pm = PluginManagerFactory.createPluginManager();
    // create plugin manager util
    PluginManagerUtil pmu = new PluginManagerUtil(pm);

    // System.out.println(pmu.getPlugins());

    // collected parameter
    ArrayList<ArrayList<String>> collectedparams = new ArrayList<ArrayList<String>>();

    // list of available plugins
    Map<String, String> availablePlugins = new HashMap<String, String>();

    // try to configure CLI
    try {
        availablePlugins = configCLI();
    } catch (ConfigurationException e) {
        System.out.println("Something went wrong with your Configuration: ");
        e.printStackTrace();
    } catch (FileNotFoundException e) {
        System.out.println("No such configuration file: ");
        e.printStackTrace();
    }

    // handle all input parameter
    for (int i = 0; i < args.length; i++) {
        // The called plugin
        if (args[i].equals(ARG__CALLED_PLUGIN) || args[i].equals(ARG__CALLED_PLUGIN_SHORT)) {
            String calledPlugin = "";
            while ((i + 1) < args.length && !args[i + 1].startsWith("-")) {
                calledPlugin += args[++i];
            }
            plugin = calledPlugin;
        }

        // the input files
        else if (args[i].equals(ARG__INPUT_FILES) || args[i].equals(ARG__INPUT_FILES_SHORT)) {
            ArrayList<String> inFiles = new ArrayList<String>();
            while ((i + 1) < args.length && !args[i + 1].startsWith("-")) {
                inFiles.add(args[++i]);
            }

            String[] files = new String[inFiles.size()];
            inFiles.toArray(files);

            File[] inf = new File[inFiles.size()];

            for (int k = 0; k < inf.length; k++) {
                inf[k] = new File(files[k]).getAbsoluteFile();
            }

            inputFiles = inf;
        }

        // output file
        else if (args[i].equals(ARG__OUTPUT_FILE) || args[i].equals(ARG__OUTPUT_FILE_SHORT)) {
            // outputFile not used!
            outputFile = args[++i];
        }

        // collecting given command parameters
        else if (args[i].startsWith("-")) {
            ArrayList<String> temp = new ArrayList<String>();
            temp.add(args[i]);
            while ((i + 1) < args.length && !args[i + 1].startsWith("-")) {
                temp.add(args[++i]);

            }
            collectedparams.add(temp);
        } // else if (args[i].equals(ARG__DEBUG_FLAG)
          // ||args[i].equals(ARG__DEBUG_FLAG_SHORT)){
          // debug = true;
          // }
    }

    // check whether the called plugin is present
    boolean pluginPresent = false;
    for (TweetyPlugin tp : pmu.getPlugins(TweetyPlugin.class)) {
        if (tp.getCommand().equalsIgnoreCase(plugin)) {
            pluginPresent = true;
            System.out.println("Called plugin present");
        }
    }
    // TODO: move loading into own method
    // trying to load plugin if not present
    // old method for loading plugins
    if (!pluginPresent) {
        System.out.print("Trying to find plugin...");
        if (availablePlugins.containsKey(plugin)) {
            pm.addPluginsFrom(new File(availablePlugins.get(plugin)).toURI());
            System.out.print("success.\n");
        } else {
            System.out.print("no such plugin available.\n");
        }
    }

    // Test: print all plugins
    // System.out.println("Plugin loaded due to call parameter: " +
    // pm.getPlugin(TweetyPlugin.class, new
    // OptionCapabilities("Tweety Plugin", plugin) ));
    // System.out.println("Print all plugins: " + pmu.getPlugins());
    // System.out.println("Given plugin call parameter: " + plugin);

    // each plugin MUST implement the capabilites "Tweety Plugin" and the
    // variable "call parameter" to select called plugin from plugin pool
    TweetyPlugin tp = pm.getPlugin(TweetyPlugin.class, new OptionCapabilities("Tweety Plugin", plugin));

    // for (TweetyPlugin tp : pmu.getPlugins(TweetyPlugin.class)) {
    if (tp.getCommand().equalsIgnoreCase(plugin)) {
        System.out.println("Valid plugin found.");
        // each input parameter is checked against the called plugin
        // whether it is valid
        ArrayList<CommandParameter> ip = new ArrayList<CommandParameter>();

        System.out.print("Trying to instantiate parameters...");
        try {
            ip.addAll(instantiateParameters(tp, collectedparams));
            System.out.print("done.\n");
        } catch (CloneNotSupportedException e) {
            e.printStackTrace();
        }
        PluginOutput out = new PluginOutput();

        System.out.println("Execute Plugin...");
        out = tp.execute(inputFiles, ip.toArray(new CommandParameter[ip.size()]));

        if (outputFile != null) {

            try {
                FileWriter fw = new FileWriter(outputFile);
                fw.write(out.getOutput());
                fw.close();
                System.out.println("Output written to file: " + outputFile);
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }

        } else {
            System.out.println("No output file given, writing to console...");
            System.out.println("Output: \n" + out.getOutput());
        }
    } else {
        System.out.println("Faulty parameters. Please check input.");

    }

}

From source file:deck36.storm.plan9.nodejs.HighFiveStreamJoinTopology.java

public static void main(String[] args) throws Exception {

    String env = null;/*from   w  w w  . j av  a 2s . co m*/

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"local".equals(env))
            if (!"prod".equals(env)) {
                System.out.println("Usage: $0 (local|dev|prod)\n");
                System.exit(1);
            }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("storm_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only CBT-related events by using the rounting key 'cbt.#'

    String badgeName = HighFiveStreamJoinTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf, "$.deck36_storm.HighFiveStreamJoinBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf, "$.deck36_storm.HighFiveStreamJoinBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (node, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.nodejs.executor"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.HighFiveStreamJoinBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.HighFiveStreamJoinBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for HighFive badge: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    builder.setBolt("badge", new MultilangAdapterBolt(command, "badge"), parallelism_hint)
            .setNumTasks(num_tasks).shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router", new Plan9RabbitMQRouterBolt(
            (String) JsonPath.read(conf, "$.deck36_storm.HighFiveStreamJoinBolt.rabbitmq.target_exchange"),
            "HighFive" // RabbitMQ routing key
    ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env) || "local".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}