Java tutorial
/* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ package modelcreation; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.lang.reflect.Array; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.math3.distribution.TDistribution; import org.apache.commons.math3.stat.regression.OLSMultipleLinearRegression; import org.apache.commons.math3.util.FastMath; /** * * @author suleyman */ public class ModelCreation { public static String TOKEN = "c01f377372564f10854877919d090ffe"; /** * @param args the command line arguments */ public static void main(String[] args) { int size = writeDataIntoFile(); double[][] x = new double[size][2]; double[] y = new double[size]; readDataFromFile(x, y); // TTest tTest = new TTest(); // System.out.println("p value for home value = " + tTest.tTest(x[0], y)); // System.out.println("p value for away value = " + tTest.tTest(x[1], y)); // System.out.println("Average mean squared error: " + apply10FoldCrossValidation(x, y)); // double[] predictions = new double[size]; // for (int i = 0; i < size; i++) { // predictions[i] = 0.5622255342802198 + (1.0682845275289186E-9 * x[i][0]) + (-9.24614306976538E-10 * x[i][1]); // // //System.out.print("Actual: " + y[i]); // //System.out.println(" Predicted: " + predicted); // } // // System.out.println(calculateMeanSquaredError(y, predictions)); OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression(); regression.newSampleData(y, x); regression.setNoIntercept(true); printRegressionStatistics(regression); //Team[] teams2014 = getTeams(354); //Team[] teams2015 = getTeams(398, 2015); //Team[] teams = concatTeams(teams2014, teams2015); // HashMap<Integer, ArrayList<Integer>> marketValueGoalsDataset = createMarketValueGoalsDataset(teams2014); // // SimpleRegression regression = new SimpleRegression(); // // Set<Integer> marketValues = marketValueGoalsDataset.keySet(); // for (Integer marketValue:marketValues) { // ArrayList<Integer> goals = marketValueGoalsDataset.get(marketValue); // int totalGoals = 0; // for(Integer goal:goals) { // regression.addData(marketValue, goal); // totalGoals += goal; // } // double avg = (double) totalGoals / goals.size(); // System.out.println("Team Value: " + marketValue + ", Goal Average: " + avg); // } // // System.out.println("Intercept: " + regression.getIntercept()); // System.out.println("Slope: " + regression.getSlope()); // System.out.println("R^2: " + regression.getRSquare()); //LinearRegression.calculateLinearRegression(marketValueGoalsDataset); } public static double apply10FoldCrossValidation(double[][] x, double[] y) { int subSize = y.length / 10; ArrayList<Integer> indeces = new ArrayList(); for (int i = 0; i < y.length; i++) { indeces.add(i); } Collections.shuffle(indeces); double[] meanSquaredErrors = new double[10]; int count = 0; for (int i = 0; i < 10; i++) { System.out.println("-------------Fold " + i + "--------------"); double[][] subXTest = new double[subSize][2]; double[] subYTest = new double[subSize]; double[][] subXTraining = new double[y.length - subSize][2]; double[] subYTraining = new double[y.length - subSize]; for (int j = 0; j < i * subSize; j++) { int index = indeces.get(count); count++; subXTraining[j][0] = x[index][0]; subXTraining[j][1] = x[index][1]; subYTraining[j] = y[index]; } for (int j = 0; j < subSize; j++) { int index = indeces.get(count); count++; subXTest[j][0] = x[index][0]; subXTest[j][1] = x[index][1]; subYTest[j] = y[index]; } for (int j = i * subSize; j < y.length - subSize; j++) { int index = indeces.get(count); count++; subXTraining[j][0] = x[index][0]; subXTraining[j][1] = x[index][1]; subYTraining[j] = y[index]; } count = 0; OLSMultipleLinearRegression regression = new OLSMultipleLinearRegression(); regression.newSampleData(subYTraining, subXTraining); regression.setNoIntercept(true); meanSquaredErrors[i] = evaluateModel(regression, subXTest, subYTest); } double sum = 0; for (int i = 0; i < meanSquaredErrors.length; i++) { sum += meanSquaredErrors[i]; } return (double) sum / meanSquaredErrors.length; } public static double evaluateModel(OLSMultipleLinearRegression regression, double[][] subXTest, double[] subYTest) { System.out.println("Adjusted R^2 = " + regression.calculateAdjustedRSquared()); System.out.println("R^2 = " + regression.calculateRSquared()); System.out.println("Residual Sum Of Squares = " + regression.calculateResidualSumOfSquares()); System.out.println("Total Sum of Squares = " + regression.calculateTotalSumOfSquares()); double[] parameters = regression.estimateRegressionParameters(); double[] predictions = new double[subYTest.length]; for (int i = 0; i < subYTest.length; i++) { double prediction = parameters[0] + (parameters[1] * subXTest[i][0]) + (parameters[2] * subXTest[i][1]); predictions[i] = prediction; } double meanSquaredError = calculateMeanSquaredError(subYTest, predictions); System.out.println("Mean Squared Error = " + meanSquaredError); return meanSquaredError; } public static double calculateMeanSquaredError(double[] actuals, double[] predictions) { int size = actuals.length; double sum_sq = 0; for (int i = 0; i < size; i++) { double err = predictions[i] - actuals[i]; sum_sq += (err * err); } double mse = (double) sum_sq / size; return mse; } public static void printRegressionStatistics(OLSMultipleLinearRegression regression) { System.out.println("Adjusted R^2 = " + regression.calculateAdjustedRSquared()); System.out.println("R^2 = " + regression.calculateRSquared()); System.out.println("Residual Sum Of Squares = " + regression.calculateResidualSumOfSquares()); System.out.println("Total Sum of Squares = " + regression.calculateTotalSumOfSquares()); double[] standardErrors = regression.estimateRegressionParametersStandardErrors(); double[] residuals = regression.estimateResiduals(); double[] parameters = regression.estimateRegressionParameters(); int residualdf = residuals.length - parameters.length; for (int i = 0; i < parameters.length; i++) { double coeff = parameters[i]; double tstat = parameters[i] / regression.estimateRegressionParametersStandardErrors()[i]; double pvalue = new TDistribution(residualdf).cumulativeProbability(-FastMath.abs(tstat)) * 2; System.out.println("Coefficient(" + i + ") : " + coeff); System.out.println("Standard Error(" + i + ") : " + standardErrors[i]); System.out.println("t-stats(" + i + ") : " + tstat); System.out.println("p-value(" + i + ") : " + pvalue); } } public static void readDataFromFile(double[][] x, double[] y) { try { BufferedReader in = new BufferedReader(new FileReader("fixtures.txt")); int index = 0; String line; while ((line = in.readLine()) != null) { String[] row = line.split(","); x[index][0] = Double.parseDouble(row[0]); x[index][1] = Double.parseDouble(row[1]); y[index] = Double.parseDouble(row[2]); index++; } } catch (Exception ex) { System.out.println(ex.getMessage()); } } public static int writeDataIntoFile() { int count = 0; Team[] teams2014 = getTeams(354); Fixture[] fixtures2014 = Fixture.getAllFixtures(354); try { BufferedWriter out = new BufferedWriter(new FileWriter("fixtures.txt")); for (Fixture fixture : fixtures2014) { if (fixture.result.goalsHomeTeam != fixture.result.goalsAwayTeam && fixture.status.equals("FINISHED")) { int homeTeamValue = 0; int awayTeamValue = 0; for (Team team : teams2014) { if (team.name.equals(fixture.homeTeamName)) { homeTeamValue = Integer.parseInt(team.squadMarketValue .substring(0, team.squadMarketValue.length() - 2).replace(",", "")); } else if (team.name.equals(fixture.awayTeamName)) { awayTeamValue = Integer.parseInt(team.squadMarketValue .substring(0, team.squadMarketValue.length() - 2).replace(",", "")); } } String result = ""; if (fixture.result.goalsHomeTeam > fixture.result.goalsAwayTeam) { result = "1"; } else { result = "0"; } out.write(homeTeamValue + "," + awayTeamValue + "," + result + "\n"); count++; } } out.close(); } catch (Exception ex) { System.out.println(ex.getMessage()); } return count; } public static <T> T[] concatArrays(T[] a, T[] b) { int aLen = a.length; int bLen = b.length; T[] c = (T[]) Array.newInstance(a.getClass().getComponentType(), aLen + bLen); System.arraycopy(a, 0, c, 0, aLen); System.arraycopy(b, 0, c, aLen, bLen); return c; } public static HashMap<Integer, ArrayList<Integer>> createMarketValueGoalsDataset(Team[] teams) { HashMap<Integer, ArrayList<Integer>> marketValueGoalsDataset = new HashMap(); for (Team team : teams) { ArrayList<Integer> goals = new ArrayList(); for (Fixture fixture : team.fixtures) { if (fixture.status.equals("FINISHED")) { if (team.name.equals(fixture.homeTeamName)) { goals.add(fixture.result.goalsHomeTeam); } else { goals.add(fixture.result.goalsAwayTeam); } } } String valueStr = team.squadMarketValue.substring(0, team.squadMarketValue.length() - 2).replace(",", ""); int marketValue = Integer.parseInt(valueStr); marketValueGoalsDataset.put(marketValue, goals); } return marketValueGoalsDataset; } public static Team[] getTeams(int seasonId) { String response = ""; try { URL api = new URL("http://api.football-data.org/v1/soccerseasons/" + seasonId + "/teams"); URLConnection connection = api.openConnection(); connection.setRequestProperty("X-Auth-Token", TOKEN); BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); String inputLine; while ((inputLine = in.readLine()) != null) { response += inputLine; } in.close(); } catch (MalformedURLException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } if (response.isEmpty()) { System.out.println("Response is empty!"); return null; } Team[] teams = Team.createTeams(response); return teams; } public static Team[] getTeamsWithFixtures(int seasonId, int seasonYear) { String response = ""; try { URL api = new URL("http://api.football-data.org/v1/soccerseasons/" + seasonId + "/teams"); URLConnection connection = api.openConnection(); connection.setRequestProperty("X-Auth-Token", TOKEN); BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); String inputLine; while ((inputLine = in.readLine()) != null) { response += inputLine; } in.close(); } catch (MalformedURLException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } if (response.isEmpty()) { System.out.println("Response is empty!"); return null; } Team[] teams = Team.createTeamsWithFixtures(response, seasonYear); return teams; } public static class Team { String name; String code; String shortName; String squadMarketValue; String crestUrl; Fixture[] fixtures; @SerializedName("_links") Links links; public void setFixtures(Fixture[] fixtures) { this.fixtures = fixtures; } public static Team[] createTeams(String json) { Gson gson = new Gson(); JsonParser parser = new JsonParser(); JsonObject object = (JsonObject) parser.parse(json); JsonArray teamsArray = (JsonArray) object.get("teams"); Team[] teams = gson.fromJson(teamsArray, Team[].class); return teams; } public static Team[] createTeamsWithFixtures(String json, int season) { Gson gson = new Gson(); JsonParser parser = new JsonParser(); JsonObject object = (JsonObject) parser.parse(json); JsonArray teamsArray = (JsonArray) object.get("teams"); Team[] teams = gson.fromJson(teamsArray, Team[].class); Fixture.addFixtureToTeams(teams, season); return teams; } } public static class Fixture { Links _links; Date date; String status; int matchDay; String homeTeamName; String awayTeamName; Result result; public static void addFixtureToTeams(Team[] teams, int season) { for (Team team : teams) { String fixtureUrl = team.links.fixtures.url + "?season=" + season; String response = ""; try { URL api = new URL(fixtureUrl); URLConnection connection = api.openConnection(); connection.setRequestProperty("X-Auth-Token", TOKEN); BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); String inputLine = ""; while ((inputLine = in.readLine()) != null) { response += inputLine; } in.close(); } catch (MalformedURLException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } if (response.isEmpty()) { System.out.println("Response is empty!"); return; } Fixture[] fixtures = createFixtures(response); team.setFixtures(fixtures); } } public static Fixture[] getAllFixtures(int seasonId) { String fixtureUrl = "http://api.football-data.org/v1/soccerseasons/" + seasonId + "/fixtures"; String response = ""; try { URL api = new URL(fixtureUrl); URLConnection connection = api.openConnection(); connection.setRequestProperty("X-Auth-Token", TOKEN); BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); String inputLine = ""; while ((inputLine = in.readLine()) != null) { response += inputLine; } in.close(); } catch (MalformedURLException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(ModelCreation.class.getName()).log(Level.SEVERE, null, ex); } if (response.isEmpty()) { System.out.println("Response is empty!"); return null; } Fixture[] fixtures = createFixtures(response); return fixtures; } public static Fixture[] createFixtures(String json) { Gson gson = new Gson(); JsonParser parser = new JsonParser(); JsonObject object = (JsonObject) parser.parse(json); JsonArray teamsArray = (JsonArray) object.get("fixtures"); Fixture[] fixtures = gson.fromJson(teamsArray, Fixture[].class); return fixtures; } } public static class Result { int goalsHomeTeam; int goalsAwayTeam; } public static class Links { Link self; Link fixtures; Link players; public static class Link { @SerializedName("href") String url; } } }