List of usage examples for weka.core Instances attributeToDoubleArray
publicdouble[] attributeToDoubleArray(int index)
From source file:cezeri.feature.selection.FeatureSelectionRanker.java
private static TFeatureRank[] correlation(Instances data, int type) { TFeatureRank[] ret = new TFeatureRank[data.numAttributes() - 1]; String[] attributeNames = FactoryInstance.getAttributeList(data); double[] out = data.attributeToDoubleArray(data.classIndex()); for (int i = 0; i < data.numAttributes() - 1; i++) { TFeatureRank obj = new TFeatureRank(); obj.featureName = attributeNames[i]; obj.index = i + ""; if (type == TCorelation.ARE) { obj.value = Math.abs(FactoryStatistic.ARE(data.attributeToDoubleArray(i), out)); }/*from www . ja v a 2 s. c o m*/ if (type == TCorelation.CRCF) { obj.value = Math.abs(FactoryStatistic.CRCF(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.IOA) { obj.value = Math.abs(FactoryStatistic.IOA(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.KENDALL) { obj.value = Math.abs(FactoryStatistic.KENDALL(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.MAE) { obj.value = Math.abs(FactoryStatistic.MAE(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.MPE) { obj.value = Math.abs(FactoryStatistic.MPE(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.MSE) { obj.value = Math.abs(FactoryStatistic.MSE(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.NSEC) { obj.value = Math.abs(FactoryStatistic.NSEC(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.PEARSON) { obj.value = Math.abs(FactoryStatistic.PEARSON(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.R) { obj.value = Math.abs(FactoryStatistic.R(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.R2) { obj.value = Math.abs(FactoryStatistic.R2(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.RAE) { obj.value = Math.abs(FactoryStatistic.RAE(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.RELATIVE_NSEC) { obj.value = Math.abs(FactoryStatistic.RELATIVE_NSEC(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.RMSE) { obj.value = Math.abs(FactoryStatistic.RMSE(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.RRSE) { obj.value = Math.abs(FactoryStatistic.RRSE(data.attributeToDoubleArray(i), out)); } if (type == TCorelation.SPEARMAN) { obj.value = Math.abs(FactoryStatistic.SPEARMAN(data.attributeToDoubleArray(i), out)); } // if (type==FactoryCorrelation.KENDALL) { // obj.value=Math.abs(FactoryCorrelation.rankKendallTauBeta(data.attributeToDoubleArray(i), out)); // } // if (type==FactoryCorrelation.PEARSON) { // obj.value=Math.abs(FactoryCorrelation.pearson(data.attributeToDoubleArray(i), out)); // } // if (type==FactoryCorrelation.SPEARMAN) { // obj.value=Math.abs(FactoryCorrelation.spearman(data.attributeToDoubleArray(i), out)); // } ret[i] = obj; } ArrayList<TFeatureRank> lst = toArrayList(ret); Collections.sort(lst, new CustomComparatorForFeatureRank()); ret = toArray(lst); return ret; }
From source file:cezeri.utils.FactoryInstance.java
public static Instances getSubsetData(Instances data, String[] attList) { Instances temp = new Instances(data); for (int i = 0; i < data.numAttributes(); i++) { if (!temp.attribute(0).equals(temp.classAttribute())) { temp.deleteAttributeAt(0);/*ww w .j a va 2 s .c o m*/ } } double[][] m = new double[attList.length + 1][data.numInstances()]; for (int i = 0; i < attList.length; i++) { int n = attList.length - 1 - i; String str = attList[n]; Attribute t = data.attribute(str); double[] d = data.attributeToDoubleArray(t.index()); m[n] = d; temp.insertAttributeAt(t, 0); } m[attList.length] = data.attributeToDoubleArray(data.classIndex()); m = CMatrix.getInstance(m).transpose().get2DArrayDouble(); FastVector att = new FastVector(); for (int i = 0; i < temp.numAttributes(); i++) { att.addElement(temp.attribute(i)); } Instances ret = new Instances(temp.relationName(), att, m.length); for (int i = 0; i < m.length; i++) { Instance ins = new Instance(m[0].length); for (int j = 0; j < m[0].length; j++) { ins.setValue(j, m[i][j]); } ret.add(ins); } ret.setClassIndex(temp.classIndex()); return ret; }
From source file:de.ugoe.cs.cpdp.dataprocessing.SimulationFilter.java
License:Apache License
@Override public void apply(Instances testdata, Instances traindata) { Instances newDataSet = new Instances(traindata); traindata.delete();// ww w . ja v a2 s.com HashMap<Double, Instance> artifactNames = new HashMap<Double, Instance>(); // This is to add all data, where the first occurence of the file has a bug ArrayList<Double> firstOccurenceArtifactNames = new ArrayList<Double>(); // Sort dataset (StateID is connected to the date of commit: Lower StateID // means earlier commit than a higher stateID) Attribute wekaAttribute = newDataSet.attribute("Artifact.Target.StateID"); newDataSet.sort(wekaAttribute); /* * Logical summary: If there is an instance that dont have a bug, put it into the hashmap * (only unique values in there) * * If there is an instance, that hava a bug look up if it is in the hashmap already (this * means: it does not had a bug before!): If this is true add it to a new dataset and remove * it from the hashmap, so that new changes from "nonBug" -> "bug" for this file can be * found. * * If the instance has a bug and is not in the hashmap (this means: The file has a bug with * its first occurence or this file only has bugs and not an instance with no bug), then (if * it is not in the arrayList above) add it to the new dataset. This way it is possible to * get the first occurence of a file, which has a bug */ for (int i = 0; i < newDataSet.numInstances(); i++) { Instance wekaInstance = newDataSet.instance(i); double newBugLabel = wekaInstance.classValue(); Attribute wekaArtifactName = newDataSet.attribute("Artifact.Name"); Double artifactName = wekaInstance.value(wekaArtifactName); if (newBugLabel == 0.0 && artifactNames.keySet().contains(artifactName)) { artifactNames.put(artifactName, wekaInstance); } else if (newBugLabel == 0.0 && !artifactNames.keySet().contains(artifactName)) { artifactNames.put(artifactName, wekaInstance); } else if (newBugLabel == 1.0 && artifactNames.keySet().contains(artifactName)) { traindata.add(wekaInstance); artifactNames.remove(artifactName); } else if (newBugLabel == 1.0 && !artifactNames.keySet().contains(artifactName)) { if (!firstOccurenceArtifactNames.contains(artifactName)) { traindata.add(wekaInstance); firstOccurenceArtifactNames.add(artifactName); } } } // If we have a file, that never had a bug (this is, when it is NOT in the // new created dataset, but it is in the HashMap from above) add it to // the new dataset double[] artifactNamesinNewDataSet = traindata.attributeToDoubleArray(0); HashMap<Double, Instance> artifactNamesCopy = new HashMap<Double, Instance>(artifactNames); for (Double artifactName : artifactNames.keySet()) { for (int i = 0; i < artifactNamesinNewDataSet.length; i++) { if (artifactNamesinNewDataSet[i] == artifactName) { artifactNamesCopy.remove(artifactName); } } } for (Double artifact : artifactNamesCopy.keySet()) { traindata.add(artifactNamesCopy.get(artifact)); } }
From source file:j48.C45PruneableClassifierTreeG.java
License:Open Source License
/** * sorts the int array in ascending order by attribute indexed * by a in dataset data. //from ww w. ja v a 2 s .c o m * @param the data the indices represent * @param the index of the attribute to sort by * @return array of sorted indicies */ private int[] sortByAttribute(Instances data, int a) { double[] attList = data.attributeToDoubleArray(a); int[] temp = Utils.sort(attList); return temp; }
From source file:lu.lippmann.cdb.datasetview.tabs.TimeSeriesSimilarityPanel.java
License:Open Source License
private Instances buildFeatureDS(final Instances dataSet) throws Exception { final int numAttributes = dataSet.numAttributes(); final java.util.List<String> namesOfFeaturesToConsider = new ArrayList<String>(); namesOfFeaturesToConsider.addAll(WekaDataStatsUtil.getAttributeNames(dataSet)); namesOfFeaturesToConsider.removeAll(WekaDataStatsUtil.getDateAttributeNames(dataSet)); final double[][] simMatrix = new double[numAttributes][numAttributes]; for (int i = 0; i < numAttributes; i++) { final double[] arrayI = dataSet.attributeToDoubleArray(i); if (this.withNormalization) MathsUtil.normalize(arrayI); simMatrix[i][i] = 0d;/*from w w w . j a v a 2 s .c o m*/ ; for (int j = i + 1; j < numAttributes; j++) { final double[] arrayJ = dataSet.attributeToDoubleArray(j); if (this.withNormalization) MathsUtil.normalize(arrayJ); simMatrix[i][j] = new DynamicTimeWarping(arrayI, arrayJ).getDistance(); //System.out.println(i+" "+j); } } for (int i = 0; i < numAttributes; i++) { for (int j = 0; j < i + 1; j++) { simMatrix[i][j] = simMatrix[j][i]; } } /*for (int i=0;i<numAttributes;i++) { System.out.println(i+" -> "+FormatterUtil.buildStringFromArrayOfDoubles(simMatrix[i])); }*/ final ArrayList<Attribute> attrs = new ArrayList<Attribute>(numAttributes + 1); for (int i = 0; i < numAttributes; i++) { attrs.add(new Attribute(dataSet.attribute(i).name() + "-feat")); } attrs.add(new Attribute(FEATUREDESC_ATTRNAME, namesOfFeaturesToConsider)); final Instances ds = new Instances("featuresComparisonDs", attrs, 0); ds.setClassIndex(attrs.size() - 1); for (int i = 0; i < simMatrix.length; i++) { final DenseInstance di = new DenseInstance(1.0d, ArraysUtil.concat(simMatrix[i], new double[] { 0d })); di.setDataset(ds); di.setValue(simMatrix.length, dataSet.attribute(i).name()); ds.add(di); } return ds; }
From source file:lu.lippmann.cdb.datasetview.tabs.UnsupervisedFeatureEvaluationTabView.java
License:Open Source License
private static Instances buildDerivatedDataset(final Instances dataSet, final List<String> possibleValues, final List<Integer> valueForEachFeature) throws Exception { final int numInstances = dataSet.numInstances(); final ArrayList<Attribute> attrs = new ArrayList<Attribute>(numInstances + 2); attrs.add(new Attribute(FEATUREDESC_ATTRNAME, (java.util.List<String>) null)); for (int i = 0; i < numInstances; i++) { attrs.add(new Attribute(i + "_eval")); }//from w ww. j av a 2 s .c om attrs.add(new Attribute("__", possibleValues)); final Instances newds = new Instances("unsupervisedFeaturesEval", attrs, 0); final int numAttributes = dataSet.numAttributes(); for (int j = 0; j < numAttributes; j++) { double[] val = ArraysUtil.concat(dataSet.attributeToDoubleArray(j), new double[] { 0.0d }); val = ArraysUtil.concat(new double[] { 0.0d }, val); newds.add(new DenseInstance(1.0d, val)); } for (int j = 0; j < numAttributes; j++) { newds.instance(j).setValue(0, dataSet.attribute(j).name()); newds.instance(j).setValue(numInstances + 1, possibleValues.get(valueForEachFeature.get(j))); } newds.setClassIndex(numInstances + 1); return newds; }
From source file:lu.lippmann.cdb.ext.hydviga.HydroRunner.java
License:Open Source License
private static void showDistances(final Instances dataSet, final int tsidx) { for (int maxValues : new int[] { 1000, 5000, 10000 }) { System.out.println(""); System.out.println("DTW from " + dataSet.attribute(tsidx).name() + " (first " + maxValues + " values)"); final double[] startArray1 = ArraysUtil.firstValues(dataSet.attributeToDoubleArray(tsidx), maxValues); for (int i = 0; i < dataSet.numAttributes(); i++) { if (dataSet.attribute(i).isDate()) continue; if (i == tsidx) continue; final double[] startArray2 = ArraysUtil.lastValues(dataSet.attributeToDoubleArray(i), maxValues); final double distance = new DynamicTimeWarping(startArray1, startArray2).getDistance(); System.out.println("\t to " + dataSet.attribute(i).name() + ": " + distance); }// w w w.j a v a 2 s .com System.out.println(""); System.out.println(""); System.out.println("DTW from " + dataSet.attribute(tsidx).name() + " (last " + maxValues + " values)"); final double[] endArray1 = ArraysUtil.lastValues(dataSet.attributeToDoubleArray(tsidx), maxValues); for (int i = 0; i < dataSet.numAttributes(); i++) { if (dataSet.attribute(i).isDate()) continue; if (i == tsidx) continue; final double[] endArray2 = ArraysUtil.lastValues(dataSet.attributeToDoubleArray(i), maxValues); final double distance = new DynamicTimeWarping(endArray1, endArray2).getDistance(); System.out.println("\t to " + dataSet.attribute(i).name() + ": " + distance); } System.out.println(""); } }
From source file:lu.lippmann.cdb.ext.hydviga.ui.GapFillingFrame.java
License:Open Source License
/** * Constructor./*from w w w.j a v a 2 s.co m*/ */ public GapFillingFrame(final AbstractTabView atv, final Instances dataSet, final Attribute attr, final int dateIdx, final int valuesBeforeAndAfter, final int position, final int gapsize, final StationsDataProvider gcp, final boolean inBatchMode) { super(); setTitle("Gap filling for " + attr.name() + " (" + dataSet.attribute(dateIdx).formatDate(dataSet.instance(position).value(dateIdx)) + " -> " + dataSet.attribute(dateIdx).formatDate(dataSet.instance(position + gapsize).value(dateIdx)) + ")"); LogoHelper.setLogo(this); this.atv = atv; this.dataSet = dataSet; this.attr = attr; this.dateIdx = dateIdx; this.valuesBeforeAndAfter = valuesBeforeAndAfter; this.position = position; this.gapsize = gapsize; this.gcp = gcp; final Instances testds = WekaDataProcessingUtil.buildFilteredDataSet(dataSet, 0, dataSet.numAttributes() - 1, Math.max(0, position - valuesBeforeAndAfter), Math.min(position + gapsize + valuesBeforeAndAfter, dataSet.numInstances() - 1)); this.attrNames = WekaTimeSeriesUtil.getNamesOfAttributesWithoutGap(testds); this.isGapSimulated = (this.attrNames.contains(attr.name())); this.originaldataSet = new Instances(dataSet); if (this.isGapSimulated) { setTitle(getTitle() + " [SIMULATED GAP]"); /*final JXLabel fictiveGapLabel=new JXLabel(" FICTIVE GAP"); fictiveGapLabel.setForeground(Color.RED); fictiveGapLabel.setFont(new Font(fictiveGapLabel.getFont().getName(), Font.PLAIN,fictiveGapLabel.getFont().getSize()*2)); final JXPanel fictiveGapPanel=new JXPanel(); fictiveGapPanel.setLayout(new BorderLayout()); fictiveGapPanel.add(fictiveGapLabel,BorderLayout.CENTER); getContentPane().add(fictiveGapPanel,BorderLayout.NORTH);*/ this.attrNames.remove(attr.name()); this.originalDataBeforeGapSimulation = dataSet.attributeToDoubleArray(attr.index()); for (int i = position; i < position + gapsize; i++) dataSet.instance(i).setMissing(attr); } final Object[] attrNamesObj = this.attrNames.toArray(); this.centerPanel = new JXPanel(); this.centerPanel.setLayout(new BorderLayout()); getContentPane().add(this.centerPanel, BorderLayout.CENTER); //final JXPanel algoPanel=new JXPanel(); //getContentPane().add(algoPanel,BorderLayout.NORTH); final JXPanel filterPanel = new JXPanel(); //filterPanel.setLayout(new BoxLayout(filterPanel, BoxLayout.Y_AXIS)); filterPanel.setLayout(new GridBagLayout()); final GridBagConstraints gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 0; gbc.weightx = 1; gbc.weighty = 1; gbc.fill = GridBagConstraints.HORIZONTAL; gbc.insets = new Insets(10, 10, 10, 10); getContentPane().add(filterPanel, BorderLayout.WEST); final JXComboBox algoCombo = new JXComboBox(Algo.values()); algoCombo.setBorder(new TitledBorder("Algorithm")); filterPanel.add(algoCombo, gbc); gbc.gridy++; final JXLabel infoLabel = new JXLabel("Usable = with no missing values on the period"); //infoLabel.setBorder(new TitledBorder("")); filterPanel.add(infoLabel, gbc); gbc.gridy++; final JList<Object> timeSeriesList = new JList<Object>(attrNamesObj); timeSeriesList.setBorder(new TitledBorder("Usable time series")); timeSeriesList.setSelectionMode(DefaultListSelectionModel.MULTIPLE_INTERVAL_SELECTION); final JScrollPane jcpMap = new JScrollPane(timeSeriesList); jcpMap.setPreferredSize(new Dimension(225, 150)); jcpMap.setMinimumSize(new Dimension(225, 150)); filterPanel.add(jcpMap, gbc); gbc.gridy++; final JXPanel mapPanel = new JXPanel(); mapPanel.setBorder(new TitledBorder("")); mapPanel.setLayout(new BorderLayout()); mapPanel.add(gcp.getMapPanel(Arrays.asList(attr.name()), this.attrNames, true), BorderLayout.CENTER); filterPanel.add(mapPanel, gbc); gbc.gridy++; final JXLabel mssLabel = new JXLabel( "<html>Most similar usable serie: <i>[... computation ...]</i></html>"); mssLabel.setBorder(new TitledBorder("")); filterPanel.add(mssLabel, gbc); gbc.gridy++; final JXLabel nsLabel = new JXLabel("<html>Nearest usable serie: <i>[... computation ...]</i></html>"); nsLabel.setBorder(new TitledBorder("")); filterPanel.add(nsLabel, gbc); gbc.gridy++; final JXLabel ussLabel = new JXLabel("<html>Upstream serie: <i>[... computation ...]</i></html>"); ussLabel.setBorder(new TitledBorder("")); filterPanel.add(ussLabel, gbc); gbc.gridy++; final JXLabel dssLabel = new JXLabel("<html>Downstream serie: <i>[... computation ...]</i></html>"); dssLabel.setBorder(new TitledBorder("")); filterPanel.add(dssLabel, gbc); gbc.gridy++; final JCheckBox hideOtherSeriesCB = new JCheckBox("Hide the others series"); hideOtherSeriesCB.setSelected(DEFAULT_HIDE_OTHER_SERIES_OPTION); filterPanel.add(hideOtherSeriesCB, gbc); gbc.gridy++; final JCheckBox showErrorCB = new JCheckBox("Show error on plot"); filterPanel.add(showErrorCB, gbc); gbc.gridy++; final JCheckBox zoomCB = new JCheckBox("Auto-adjusted size"); zoomCB.setSelected(DEFAULT_ZOOM_OPTION); filterPanel.add(zoomCB, gbc); gbc.gridy++; final JCheckBox multAxisCB = new JCheckBox("Multiple axis"); filterPanel.add(multAxisCB, gbc); gbc.gridy++; final JCheckBox showEnvelopeCB = new JCheckBox("Show envelope (all algorithms, SLOW)"); filterPanel.add(showEnvelopeCB, gbc); gbc.gridy++; final JXButton showModelButton = new JXButton("Show the model"); filterPanel.add(showModelButton, gbc); gbc.gridy++; showModelButton.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { final JXFrame dialog = new JXFrame(); dialog.setTitle("Model"); LogoHelper.setLogo(dialog); dialog.getContentPane().removeAll(); dialog.getContentPane().setLayout(new BorderLayout()); final JTextPane modelTxtPane = new JTextPane(); modelTxtPane.setText(gapFiller.getModel()); modelTxtPane.setBackground(Color.WHITE); modelTxtPane.setEditable(false); final JScrollPane jsp = new JScrollPane(modelTxtPane, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED); jsp.setSize(new Dimension(400 - 20, 400 - 20)); dialog.getContentPane().add(jsp, BorderLayout.CENTER); dialog.setSize(new Dimension(400, 400)); dialog.setLocationRelativeTo(centerPanel); dialog.pack(); dialog.setVisible(true); } }); algoCombo.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); showModelButton.setEnabled(gapFiller.hasExplicitModel()); } catch (final Exception e1) { e1.printStackTrace(); } } }); timeSeriesList.addListSelectionListener(new ListSelectionListener() { @Override public void valueChanged(final ListSelectionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); mapPanel.removeAll(); final List<String> currentlySelected = new ArrayList<String>(); currentlySelected.add(attr.name()); for (final Object sel : timeSeriesList.getSelectedValues()) currentlySelected.add(sel.toString()); mapPanel.add(gcp.getMapPanel(currentlySelected, attrNames, true), BorderLayout.CENTER); } catch (final Exception e1) { e1.printStackTrace(); } } }); hideOtherSeriesCB.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); } catch (final Exception e1) { e1.printStackTrace(); } } }); showErrorCB.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); } catch (Exception e1) { e1.printStackTrace(); } } }); zoomCB.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); } catch (Exception e1) { e1.printStackTrace(); } } }); showEnvelopeCB.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); } catch (Exception e1) { e1.printStackTrace(); } } }); multAxisCB.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), timeSeriesList.getSelectedIndices(), hideOtherSeriesCB.isSelected(), showErrorCB.isSelected(), zoomCB.isSelected(), showEnvelopeCB.isSelected(), multAxisCB.isSelected()); } catch (Exception e1) { e1.printStackTrace(); } } }); this.inBatchMode = inBatchMode; if (!inBatchMode) { try { refresh(Algo.valueOf(algoCombo.getSelectedItem().toString()), new int[0], DEFAULT_HIDE_OTHER_SERIES_OPTION, false, DEFAULT_ZOOM_OPTION, false, false); showModelButton.setEnabled(gapFiller.hasExplicitModel()); } catch (Exception e1) { e1.printStackTrace(); } } if (!inBatchMode) { /* automatically select computed series */ new AbstractSimpleAsync<Void>(true) { @Override public Void execute() throws Exception { mostSimilar = WekaTimeSeriesSimilarityUtil.findMostSimilarTimeSerie(testds, attr, attrNames, false); mssLabel.setText("<html>Most similar usable serie: <b>" + mostSimilar + "</b></html>"); nearest = gcp.findNearestStation(attr.name(), attrNames); nsLabel.setText("<html>Nearest usable serie: <b>" + nearest + "</b></html>"); upstream = gcp.findUpstreamStation(attr.name(), attrNames); if (upstream != null) { ussLabel.setText("<html>Upstream usable serie: <b>" + upstream + "</b></html>"); } else { ussLabel.setText("<html>Upstream usable serie: <b>N/A</b></html>"); } downstream = gcp.findDownstreamStation(attr.name(), attrNames); if (downstream != null) { dssLabel.setText("<html>Downstream usable serie: <b>" + downstream + "</b></html>"); } else { dssLabel.setText("<html>Downstream usable serie: <b>N/A</b></html>"); } timeSeriesList.setSelectedIndices( new int[] { attrNames.indexOf(mostSimilar), attrNames.indexOf(nearest), attrNames.indexOf(upstream), attrNames.indexOf(downstream) }); return null; } @Override public void onSuccess(final Void result) { } @Override public void onFailure(final Throwable caught) { caught.printStackTrace(); } }.start(); } else { try { mostSimilar = WekaTimeSeriesSimilarityUtil.findMostSimilarTimeSerie(testds, attr, attrNames, false); } catch (Exception e1) { e1.printStackTrace(); } nearest = gcp.findNearestStation(attr.name(), attrNames); upstream = gcp.findUpstreamStation(attr.name(), attrNames); downstream = gcp.findDownstreamStation(attr.name(), attrNames); } }
From source file:lu.lippmann.cdb.ext.hydviga.ui.GapFillingFrame.java
License:Open Source License
private GapFillingCase refresh(final Algo algo, final int[] indexesOfUsedSeries, final boolean hideOthers, final boolean showError, final boolean zoom, final boolean showEnvelope, final boolean multAxis) throws Exception { if (!inBatchMode) this.centerPanel.removeAll(); int[] arr = new int[] { attr.index(), dateIdx }; for (final int iii : indexesOfUsedSeries) { arr = ArraysUtil.concat(arr, new int[] { dataSet.attribute(this.attrNames.get(iii)).index() }); }/* w w w. j a va2 s .c o m*/ Instances filteredDs = WekaDataProcessingUtil.buildFilteredByAttributesDataSet(dataSet, arr); //System.out.println(filteredDs.toSummaryString()); Attribute original = null; Instances filteredDsWithOriginal = null; if (this.isGapSimulated) { original = new Attribute("original"); filteredDsWithOriginal = new Instances(filteredDs); filteredDsWithOriginal.insertAttributeAt(original, filteredDsWithOriginal.numAttributes() - 1); final Attribute origAttr = filteredDsWithOriginal.attribute(original.name()); for (int ii = position - 1; ii < position + gapsize + 1; ii++) { filteredDsWithOriginal.instance(ii).setValue(origAttr, this.originalDataBeforeGapSimulation[ii]); } } filteredDs = WekaDataProcessingUtil.buildFilteredDataSet(filteredDs, 0, filteredDs.numAttributes() - 1, Math.max(0, this.position - this.valuesBeforeAndAfter), Math.min(this.position + this.gapsize + this.valuesBeforeAndAfter, filteredDs.numInstances() - 1)); this.gapFiller = GapFillerFactory.getGapFiller(algo); final Instances completedds = this.gapFiller.fillGaps(filteredDs); final Instances diff = WekaTimeSeriesUtil.buildDiff(filteredDs, completedds); final int valuesToCheckForError = this.valuesBeforeAndAfter / 4; double maeByEnlargingGap = Double.NaN; double maeByAddingAGapBefore = Double.NaN; double maeByAddingAGapAfter = Double.NaN; double maeByComparingWithOriginal = Double.NaN; double rmseByEnlargingGap = Double.NaN; double rmseByAddingAGapBefore = Double.NaN; double rmseByAddingAGapAfter = Double.NaN; double rmseByComparingWithOriginal = Double.NaN; double rsrByEnlargingGap = Double.NaN; double rsrByAddingAGapBefore = Double.NaN; double rsrByAddingAGapAfter = Double.NaN; double rsrByComparingWithOriginal = Double.NaN; double pbiasByEnlargingGap = Double.NaN; double pbiasByAddingAGapBefore = Double.NaN; double pbiasByAddingAGapAfter = Double.NaN; double pbiasByComparingWithOriginal = Double.NaN; double nsByEnlargingGap = Double.NaN; double nsByAddingAGapBefore = Double.NaN; double nsByAddingAGapAfter = Double.NaN; double nsByComparingWithOriginal = Double.NaN; double indexOfAgreementByEnlargingGap = Double.NaN; double indexOfAgreementByAddingAGapBefore = Double.NaN; double indexOfAgreementByAddingAGapAfter = Double.NaN; double indexOfAgreementByComparingWithOriginal = Double.NaN; if (this.isGapSimulated) { //System.out.println(attr.index()+" begin="+(this.position)+" end="+(this.position+this.gapsize)); final Instances correctedDataSet = buildCorrectedDataset(diff); final double[] cad = correctedDataSet.attributeToDoubleArray(attr.index()); maeByComparingWithOriginal = MathsUtil.mae(this.originalDataBeforeGapSimulation, cad, this.position, this.position + this.gapsize); rmseByComparingWithOriginal = MathsUtil.rmse(this.originalDataBeforeGapSimulation, cad, this.position, this.position + this.gapsize); rsrByComparingWithOriginal = MathsUtil.rsr(this.originalDataBeforeGapSimulation, cad, this.position, this.position + this.gapsize); pbiasByComparingWithOriginal = MathsUtil.pbias(this.originalDataBeforeGapSimulation, cad, this.position, this.position + this.gapsize); nsByComparingWithOriginal = MathsUtil.nashSutcliffe(this.originalDataBeforeGapSimulation, cad, this.position, this.position + this.gapsize); indexOfAgreementByComparingWithOriginal = MathsUtil.indexOfAgreement( this.originalDataBeforeGapSimulation, cad, this.position, this.position + this.gapsize); } else { maeByEnlargingGap = this.gapFiller.evaluateMAEByEnlargingGap(filteredDs, valuesToCheckForError); maeByAddingAGapBefore = this.gapFiller.evaluateMAEByAddingAGapBefore(filteredDs, valuesToCheckForError); maeByAddingAGapAfter = this.gapFiller.evaluateMAEByAddingAGapAfter(filteredDs, valuesToCheckForError); rmseByEnlargingGap = this.gapFiller.evaluateRMSEByEnlargingGap(filteredDs, valuesToCheckForError); rmseByAddingAGapBefore = this.gapFiller.evaluateRMSEByAddingAGapBefore(filteredDs, valuesToCheckForError); rmseByAddingAGapAfter = this.gapFiller.evaluateRMSEByAddingAGapAfter(filteredDs, valuesToCheckForError); nsByEnlargingGap = this.gapFiller.evaluateNSByEnlargingGap(filteredDs, valuesToCheckForError); nsByAddingAGapBefore = this.gapFiller.evaluateNSByAddingAGapBefore(filteredDs, valuesToCheckForError); nsByAddingAGapAfter = this.gapFiller.evaluateNSByAddingAGapAfter(filteredDs, valuesToCheckForError); } if (hideOthers) { if (this.isGapSimulated) { filteredDs = WekaDataProcessingUtil.buildFilteredByAttributesDataSet(filteredDsWithOriginal, new int[] { 0, 1, filteredDsWithOriginal.attribute(original.name()).index() }); filteredDs = WekaDataProcessingUtil.buildFilteredDataSet(filteredDs, 0, filteredDs.numAttributes() - 1, Math.max(0, this.position - this.valuesBeforeAndAfter), Math.min(this.position + this.gapsize + this.valuesBeforeAndAfter, filteredDs.numInstances() - 1)); } else { filteredDs = WekaDataProcessingUtil.buildFilteredByAttributesDataSet(filteredDs, new int[] { 0, 1 }); } } final Instances decomposition = WekaTimeSeriesUtil.buildMergedDataSet(filteredDs, diff); final Attribute diffAttribute = decomposition.attribute(attr.name() + "_diff"); final List<XYAnnotation> aaa = new ArrayList<XYAnnotation>(); if (showError) { showError(this.isGapSimulated ? maeByComparingWithOriginal : maeByEnlargingGap, decomposition, diffAttribute, aaa); } if (showEnvelope) { final MainViewLoadingFrame loadingFrame = new MainViewLoadingFrame(); loadingFrame.setVisible(true); loadingFrame.pack(); loadingFrame.repaint(); showEnvelope(arr, aaa); loadingFrame.setVisible(false); } if (!inBatchMode) { final ChartPanel cp; /*if (showError) { cp=TimeSeriesChartUtil.buildChartPanelForAllAttributesInterval(decomposition,WekaDataStatsUtil.getFirstDateAttributeIdx(decomposition),mae,diffAttribute.index()); } else {*/ cp = TimeSeriesChartUtil.buildChartPanelForAllAttributes(decomposition, multAxis, WekaDataStatsUtil.getFirstDateAttributeIdx(decomposition), null, aaa); /*}*/ final Marker gapBeginMarker = new ValueMarker( dataSet.instance(Math.max(0, position - 1)).value(dateIdx)); gapBeginMarker.setPaint(Color.RED); gapBeginMarker.setLabel("Gap begin"); gapBeginMarker.setLabelAnchor(RectangleAnchor.TOP_LEFT); gapBeginMarker.setLabelTextAnchor(TextAnchor.TOP_RIGHT); cp.getChart().getXYPlot().addDomainMarker(gapBeginMarker); final Marker gapEndMarker = new ValueMarker( dataSet.instance(Math.min(dataSet.numInstances() - 1, position + gapsize)).value(dateIdx)); gapEndMarker.setPaint(Color.RED); gapEndMarker.setLabel("Gap end"); gapEndMarker.setLabelAnchor(RectangleAnchor.TOP_RIGHT); gapEndMarker.setLabelTextAnchor(TextAnchor.TOP_LEFT); cp.getChart().getXYPlot().addDomainMarker(gapEndMarker); if (!zoom) { final NumberAxis na = (NumberAxis) (cp.getChart().getXYPlot().getRangeAxis()); na.setRange(0, WekaDataStatsUtil.getMaxValue(dataSet, attrNames)); } String errorInfo; if (!this.isGapSimulated) { errorInfo = "By enlarging the gap:\t MAE=" + FormatterUtil.DECIMAL_FORMAT_4.format(maeByEnlargingGap) + "\t RMSE=" + FormatterUtil.DECIMAL_FORMAT_4.format(rmseByEnlargingGap) + "\t NASH-SUTCLIFFE=" + FormatterUtil.DECIMAL_FORMAT_4.format(nsByEnlargingGap) + "\nBy adding a gap before:\t MAE=" + FormatterUtil.DECIMAL_FORMAT_4.format(maeByAddingAGapBefore) + "\t RMSE=" + FormatterUtil.DECIMAL_FORMAT_4.format(rmseByAddingAGapBefore) + "\t NASH-SUTCLIFFE=" + FormatterUtil.DECIMAL_FORMAT_4.format(nsByAddingAGapBefore) + "\nBy adding a gap after:\t MAE=" + FormatterUtil.DECIMAL_FORMAT_4.format(maeByAddingAGapAfter) + "\t RMSE=" + FormatterUtil.DECIMAL_FORMAT_4.format(rmseByAddingAGapAfter) + "\t NASH-SUTCLIFFE=" + FormatterUtil.DECIMAL_FORMAT_4.format(nsByAddingAGapAfter); } else { errorInfo = "MAE: " + FormatterUtil.DECIMAL_FORMAT_4.format(maeByComparingWithOriginal); errorInfo += "\n"; errorInfo += "RMSE: " + FormatterUtil.DECIMAL_FORMAT_4.format(rmseByComparingWithOriginal); errorInfo += "\n"; errorInfo += "RSR: " + FormatterUtil.DECIMAL_FORMAT_4.format(rsrByComparingWithOriginal); errorInfo += "\n"; errorInfo += "PBIAS: " + FormatterUtil.DECIMAL_FORMAT_4.format(pbiasByComparingWithOriginal); errorInfo += "\n"; errorInfo += "NASH-SUTCLIFFE: " + FormatterUtil.DECIMAL_FORMAT_4.format(nsByComparingWithOriginal); errorInfo += "\n"; errorInfo += "INDEX OF AGREEMENT: " + FormatterUtil.DECIMAL_FORMAT_4.format(indexOfAgreementByComparingWithOriginal); } cp.setBorder(new TitledBorder("")); final JTextArea errorTextArea = new JTextArea(errorInfo); errorTextArea.setBorder(new TitledBorder("")); this.centerPanel.add(errorTextArea, BorderLayout.NORTH); this.centerPanel.add(cp, BorderLayout.CENTER); final JXPanel cmdPanel = new JXPanel(); final JXButton okButton = new JXButton("Ok"); okButton.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { final Instances correctedDataSet = buildCorrectedDataset(diff); final DataChange change = new DataChange(correctedDataSet, TabView.DataChangeTypeEnum.Update); atv.pushDataChange(change); setVisible(false); } }); cmdPanel.add(okButton); final JXButton cancelButton = new JXButton("Cancel"); cancelButton.addActionListener(new ActionListener() { @Override public void actionPerformed(final ActionEvent e) { setVisible(false); } }); cmdPanel.add(cancelButton); this.centerPanel.add(cmdPanel, BorderLayout.SOUTH); this.centerPanel.updateUI(); getContentPane().repaint(); } final double globalMAE = (this.isGapSimulated) ? maeByComparingWithOriginal : ((maeByEnlargingGap + maeByAddingAGapBefore + maeByAddingAGapAfter) / 3); final double globalRMSE = (this.isGapSimulated) ? rmseByComparingWithOriginal : ((rmseByEnlargingGap + rmseByAddingAGapBefore + rmseByAddingAGapAfter) / 3); final double globalRSR = (this.isGapSimulated) ? rsrByComparingWithOriginal : ((rsrByEnlargingGap + rsrByAddingAGapBefore + rsrByAddingAGapAfter) / 3); final double globalPBIAS = (this.isGapSimulated) ? pbiasByComparingWithOriginal : ((pbiasByEnlargingGap + pbiasByAddingAGapBefore + pbiasByAddingAGapAfter) / 3); final double globalNS = (this.isGapSimulated) ? nsByComparingWithOriginal : ((nsByEnlargingGap + nsByAddingAGapBefore + nsByAddingAGapAfter) / 3); final double globalIndexOfAgreement = (this.isGapSimulated) ? indexOfAgreementByComparingWithOriginal : ((indexOfAgreementByEnlargingGap + indexOfAgreementByAddingAGapBefore + indexOfAgreementByAddingAGapAfter) / 3); // usage logs for stats final long firstTimestamp = (long) dataSet.instance(position).value(dateIdx); final boolean isDuringRising; if (nearest == null) isDuringRising = GapsUtil.isDuringRising(dataSet, position, gapsize, new int[] { dateIdx, attr.index() }); else isDuringRising = GapsUtil.isDuringRising(dataSet, position, gapsize, new int[] { dateIdx, attr.index(), dataSet.attribute(nearest).index() }); return new GapFillingCase(DateUtil.getSeason(firstTimestamp), DateUtil.getYear(firstTimestamp), algo, gapsize, position, attr, gcp.getCoordinates(attr.name())[0], gcp.getCoordinates(attr.name())[1], gcp.findDownstreamStation(attr.name()) != null, gcp.findUpstreamStation(attr.name()) != null, globalMAE, globalRMSE, globalRSR, globalPBIAS, globalNS, globalIndexOfAgreement, ArraysUtil.contains(indexesOfUsedSeries, attrNames.indexOf(mostSimilar)), ArraysUtil.contains(indexesOfUsedSeries, attrNames.indexOf(nearest)), ArraysUtil.contains(indexesOfUsedSeries, attrNames.indexOf(downstream)), ArraysUtil.contains(indexesOfUsedSeries, attrNames.indexOf(upstream)), isDuringRising, GapsUtil.measureHighMiddleLowInterval(dataSet, attr.index(), position - 1)); }
From source file:lu.lippmann.cdb.lab.timeseries.WekaTimeSeriesSimilarityUtil.java
License:Open Source License
public static String findMostSimilarTimeSerie(final Instances testds, final Attribute attr, final List<String> attrNames, final boolean THREADED_NN_LOOKUP) { //System.out.println("findMostSimilarTimeSerie: for "+attr.name()+" in "+attrNames); //System.out.println("findMostSimilarTimeSerie: (before) rows -> "+testds.numInstances()); final List<String> attrNamesWithCurrent = new ArrayList<>(attrNames); attrNamesWithCurrent.add(attr.name()); final Instances reducedds = WekaDataProcessingUtil.buildDataSetWithoutRowsWithMissingValues(testds, attrNamesWithCurrent); // TODO: optimize remove into this method //System.out.println(reducedds.numAttributes()); //System.out.println("findMostSimilarTimeSerie: (after) rows -> "+reducedds.numInstances()); final double[] startArray1 = reducedds.attributeToDoubleArray(reducedds.attribute(attr.name()).index()); if (startArray1.length == 0) System.out.println("no val for " + attr.name() + "??"); double min = Double.POSITIVE_INFINITY; int idx = -1; if (!THREADED_NN_LOOKUP) { for (int i = 0; i < reducedds.numAttributes(); i++) { //System.out.println(i); if (!attrNames.contains(reducedds.attribute(i).name())) { //System.out.println(reducedds.attribute(i).name()+" not in "+attrNames); continue; }//from w ww . ja v a 2 s.com final double[] startArray2 = reducedds.attributeToDoubleArray(i); final double distance = new DynamicTimeWarping(startArray1, startArray2).getDistance(); if (distance < min) { min = distance; idx = i; } //System.out.println("\t to "+reducedds.attribute(i).name()+": "+distance); } } else { final List<Callable<Object[]>> tasks = new ArrayList<Callable<Object[]>>(); for (int i = 0; i < reducedds.numAttributes(); i++) { if (!attrNames.contains(reducedds.attribute(i).name())) continue; final int ii = i; tasks.add(new Callable<Object[]>() { @Override public Object[] call() throws Exception { final double[] startArray2 = reducedds.attributeToDoubleArray(ii); final double distance = new DynamicTimeWarping(startArray1, startArray2).getDistance(); //System.out.println("\t to "+reducedds.attribute(ii).name()+": "+distance); return new Object[] { distance, ii }; } }); } try { final List<Future<Object[]>> res = EXECUTOR_SERVICE.invokeAll(tasks); for (final Future<Object[]> r : res) { final Object[] oo = r.get(); final double distance = (Double) oo[0]; final int curridx = (Integer) oo[1]; if (distance < min) { min = distance; idx = curridx; } } } catch (Exception e1) { e1.printStackTrace(); } } final String foundSerie = (idx != -1) ? reducedds.attribute(idx).name() : null; //System.out.println("found -> "+foundSerie); return foundSerie; }