Java tutorial
/** * Copyright 2012 Universitat Pompeu Fabra. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * */ package org.onexus.ui.workspace.internal.wizards.data; import org.apache.commons.lang3.StringUtils; import org.apache.wicket.extensions.wizard.WizardModel; import org.apache.wicket.extensions.wizard.WizardStep; import org.apache.wicket.markup.html.form.ListMultipleChoice; import org.apache.wicket.markup.html.form.RadioChoice; import org.apache.wicket.model.IModel; import org.apache.wicket.model.PropertyModel; import org.apache.wicket.request.mapper.parameter.PageParameters; import org.onexus.collection.api.Collection; import org.onexus.collection.api.Field; import org.onexus.collection.api.Link; import org.onexus.collection.api.types.Text; import org.onexus.collection.api.utils.LinkUtils; import org.onexus.data.api.IDataManager; import org.onexus.resource.api.Folder; import org.onexus.resource.api.IResourceManager; import org.onexus.resource.api.Loader; import org.onexus.resource.api.ORI; import org.onexus.resource.api.Parameter; import org.onexus.resource.api.Property; import org.onexus.resource.api.Resource; import org.onexus.ui.api.pages.resource.ResourcesPage; import org.onexus.ui.api.wizards.AbstractWizard; import javax.inject.Inject; import javax.inject.Named; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; public class CreateCollectionWizard extends AbstractWizard { @Inject private IDataManager dataManager; @Inject private IResourceManager resourceManager; // Formats private static final String CSV = "Comma separated values"; private static final String TSV = "Tab separated values"; private static final List<String> FORMATS = Arrays.asList(new String[] { TSV, CSV }); // Maximum lines to load to deduce the datatype private static final int MAXIMUM_LINES = 10000; private static final int MAXIMUM_UNIQUE_VALUES = 1500; private String selected = TSV; private List<String> primaryKeys = new ArrayList<String>(); private ORI sourceURI; // Data information private String headers[]; private Map<String, Set<String>> sampleData; private int nullEmpty = 0; private int nullDash = 0; private int nullString = 0; private int nullNA = 0; public CreateCollectionWizard(String id, IModel<? extends Resource> model) { super(id); sourceURI = model.getObject().getORI(); WizardModel wizardModel = new WizardModel(); wizardModel.add(new ChooseFormat()); wizardModel.add(new PrimaryKeys()); init(wizardModel); } private void readData() throws IOException { String separator = " "; if (selected.equals(CSV)) { separator = ","; } if (selected.equals(TSV)) { separator = "\t"; } Iterator<InputStream> streams = dataManager.load(sourceURI).iterator(); if (!streams.hasNext()) { return; } BufferedReader fr = new BufferedReader(new InputStreamReader(streams.next())); // Get headers headers = fr.readLine().split(separator); // Build values map sampleData = new HashMap<String, Set<String>>(); for (String header : headers) { sampleData.put(header, new HashSet<String>()); } String line = fr.readLine(); for (int i = 0; i < MAXIMUM_LINES && line != null; i++) { String values[] = line.split(separator); for (int h = 0; h < headers.length && h < values.length; h++) { if (values[h] == null || values[h].isEmpty()) { nullEmpty++; } else if (values[h].equalsIgnoreCase("-")) { nullDash++; } else if (values[h].equalsIgnoreCase("NULL")) { nullString++; } else if (values[h].equalsIgnoreCase("NA")) { nullNA++; } Set<String> columnValues = sampleData.get(headers[h]); if (columnValues.size() < MAXIMUM_UNIQUE_VALUES) { sampleData.get(headers[h]).add(values[h]); } } line = fr.readLine(); } fr.close(); } @Override public void onFinish() { super.onFinish(); // Create collection Collection collection = newCollection(); // Collect fields from other collections in the same folder Map<String, Field> otherFields = collectFields(); List<Field> fields = new ArrayList<Field>(); for (String header : headers) { String shortName, title; if (otherFields.containsKey(header)) { Field field = otherFields.get(header); shortName = field.getLabel(); title = field.getTitle(); } else { String lower = StringUtils.lowerCase(header); shortName = StringUtils.abbreviate(lower, 20); title = StringUtils.capitalize(lower); } Field field = new Field(header, shortName, title, deduceClass(sampleData.get(header))); if (header.toLowerCase().contains("pvalue") || header.toLowerCase().contains("qvalue")) { field.setProperties(Arrays.asList(new Property[] { new Property("BROWSER_DECORATOR", "PVALUE2") })); } if (primaryKeys.contains(header)) { field.setPrimaryKey(Boolean.TRUE); } fields.add(field); } collection.setFields(fields); // Deduce links from other collections in the same folder Map<String, Link> otherLinks = collectLinks(); List<Link> links = new ArrayList<Link>(); List<Collection> allProjectCollections = new ArrayList<Collection>(); addAllCollections(allProjectCollections, resourceManager.getProject(sourceURI.getProjectUrl()).getORI()); for (String header : headers) { if (otherLinks.containsKey(header)) { Link otherLink = otherLinks.get(header); Link link = new Link(); link.setCollection(otherLink.getCollection()); link.getFields().add(otherLink.getFields().get(0)); links.add(link); } else { for (Collection col : allProjectCollections) { Field field = col.getField(header); if (field != null && (header.toLowerCase().endsWith("id") || header.toLowerCase().endsWith("key"))) { // Only link to collections without any link if (col.getLinks() == null || col.getLinks().isEmpty()) { Link link = new Link(); link.setCollection(new ORI((String) null, col.getORI().getPath())); link.getFields().add(header); links.add(link); } } } } } collection.setLinks(links); Loader loader = new Loader(); loader.setPlugin("tsv-loader"); List<Parameter> parameters = new ArrayList<Parameter>(); parameters.add(new Parameter("data", sourceURI.getPath())); if (nullEmpty > nullDash && nullEmpty > nullString && nullEmpty > nullNA) { parameters.add(new Parameter("NULL_VALUE", "")); } if (nullString > nullDash && nullString > nullEmpty && nullString > nullNA) { parameters.add(new Parameter("NULL_VALUE", "NULL")); } if (nullNA > nullDash && nullNA > nullString && nullNA > nullEmpty) { parameters.add(new Parameter("NULL_VALUE", "NA")); } loader.setParameters(parameters); collection.setLoader(loader); resourceManager.save(collection); PageParameters params = new PageParameters().add(ResourcesPage.PARAMETER_RESOURCE, collection.getORI()); setResponsePage(ResourcesPage.class, params); } private Map<String, Link> collectLinks() { Map<String, Link> links = new HashMap<String, Link>(); List<Collection> collections = new ArrayList<Collection>(); addAllCollections(collections, sourceURI.getParent()); for (Collection collection : collections) { if (collection.getLinks() != null) { for (Link link : collection.getLinks()) { // Only simple links (not composed) if (link.getFields().size() == 1) { String field = LinkUtils.getFromFieldName(link.getFields().get(0)); links.put(field, link); } } } } return links; } private Map<String, Field> collectFields() { Map<String, Field> fields = new HashMap<String, Field>(); List<Collection> collections = new ArrayList<Collection>(); addAllCollections(collections, sourceURI.getParent()); for (Collection collection : collections) { for (Field field : collection.getFields()) { fields.put(field.getId(), field); } } return fields; } private Collection newCollection() { String sourceName = sourceURI.getPath(); String collectionName; int punt = sourceName.lastIndexOf('.'); if (punt != -1) { collectionName = sourceName.substring(0, punt); } else { collectionName = sourceName + ".col"; } ORI collectionURI = new ORI(sourceURI.getProjectUrl(), collectionName); Collection collection = new Collection(); collection.setORI(collectionURI); collection.setTitle(collectionName); return collection; } private void addAllCollections(List<Collection> collections, ORI parentUri) { collections.addAll(resourceManager.loadChildren(Collection.class, parentUri)); List<Folder> folders = resourceManager.loadChildren(Folder.class, parentUri); for (Folder folder : folders) { addAllCollections(collections, folder.getORI()); } } public String getSelected() { return selected; } public void setSelected(String selected) { this.selected = selected; } private static Class<?> deduceClass(Set<String> values) { boolean longString = false; boolean integerType = true; boolean doubleType = false; for (String value : values) { // Skip null and empty values if (value == null || value.isEmpty() || value.equals("-")) { continue; } if (integerType) { try { Integer.valueOf(value); } catch (NumberFormatException e) { integerType = false; doubleType = true; } } if (doubleType) { try { Double.valueOf(value); } catch (NumberFormatException e) { doubleType = false; } } if (value.length() > 127) { longString = true; } } if (integerType) { return Integer.class; } if (doubleType) { return Double.class; } if (longString) { return Text.class; } else { return String.class; } } public List<String> getPrimaryKeys() { return primaryKeys; } public void setPrimaryKeys(List<String> primaryKeys) { this.primaryKeys = primaryKeys; } private final class ChooseFormat extends WizardStep { public ChooseFormat() { super("Create collection", "Choose one file format"); RadioChoice<String> commandOptions = new RadioChoice<String>("formats", new PropertyModel<String>(CreateCollectionWizard.this, "selected"), FORMATS); add(commandOptions); } @Override public void applyState() { try { readData(); } catch (IOException e) { error(e.getMessage()); } } } private final class PrimaryKeys extends WizardStep { public PrimaryKeys() { super("Create collection", "Select primary keys"); } @Override protected void onBeforeRender() { addOrReplace(new ListMultipleChoice<String>("primaryKeys", Arrays.asList(headers))); super.onBeforeRender(); } } }