Java tutorial
/* * Copyright (C) 2009 Google Inc. * Copyright (C) 2010 University of Washington. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package org.opendatakit.aggregate.parser; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.javarosa.core.model.instance.TreeElement; import org.opendatakit.aggregate.constants.ParserConsts; import org.opendatakit.aggregate.constants.ServletConsts; import org.opendatakit.aggregate.constants.TaskLockType; import org.opendatakit.aggregate.constants.common.FormActionStatusTimestamp; import org.opendatakit.aggregate.constants.common.GeoPointConsts; import org.opendatakit.aggregate.datamodel.FormDataModel; import org.opendatakit.aggregate.datamodel.FormDataModel.ElementType; import org.opendatakit.aggregate.datamodel.TopLevelDynamicBase; import org.opendatakit.aggregate.exception.ODKFormAlreadyExistsException; import org.opendatakit.aggregate.exception.ODKFormNotFoundException; import org.opendatakit.aggregate.exception.ODKIncompleteSubmissionData; import org.opendatakit.aggregate.exception.ODKIncompleteSubmissionData.Reason; import org.opendatakit.aggregate.exception.ODKParseException; import org.opendatakit.aggregate.form.FormDefinition; import org.opendatakit.aggregate.form.FormFactory; import org.opendatakit.aggregate.form.IForm; import org.opendatakit.aggregate.form.MiscTasks; import org.opendatakit.aggregate.form.SubmissionAssociationTable; import org.opendatakit.aggregate.form.XFormParameters; import org.opendatakit.common.datamodel.DynamicBase; import org.opendatakit.common.persistence.CommonFieldsBase; import org.opendatakit.common.persistence.DataField; import org.opendatakit.common.persistence.Datastore; import org.opendatakit.common.persistence.EntityKey; import org.opendatakit.common.persistence.PersistConsts; import org.opendatakit.common.persistence.TaskLock; import org.opendatakit.common.persistence.exception.ODKDatastoreException; import org.opendatakit.common.persistence.exception.ODKEntityPersistException; import org.opendatakit.common.persistence.exception.ODKTaskLockException; import org.opendatakit.common.security.User; import org.opendatakit.common.web.CallingContext; /** * Parses an XML definition of an XForm based on java rosa types * * @author wbrunette@gmail.com * @author mitchellsundt@gmail.com * @author chrislrobert@gmail.com * */ public class FormParserForJavaRosa extends BaseFormParserForJavaRosa { private static final Log log = LogFactory.getLog(FormParserForJavaRosa.class.getName()); private static final long FIFTEEN_MINUTES_IN_MILLISECONDS = 15 * 60 * 1000L; // arbitrary limit on the table-creation process, to prevent infinite loops // that exhaust memory private static final int MAX_FORM_CREATION_ATTEMPTS = 100; private String fdmSubmissionUri; private int elementCount = 0; private int phantomCount = 0; private final Map<FormDataModel, Integer> fieldLengths = new HashMap<FormDataModel, Integer>(); /** * Constructor that parses and xform from the input stream supplied and * creates the proper ODK Aggregate Form definition. * * @param formName * @param formXmlData * @param inputXml * @param fileName * @param uploadedFormItems * @param warnings * -- the builder that will hold all the non-fatal form-creation * warnings * @param cc * @throws ODKFormAlreadyExistsException * @throws ODKIncompleteSubmissionData * @throws ODKDatastoreException * @throws ODKParseException */ public FormParserForJavaRosa(String formName, MultiPartFormItem formXmlData, String inputXml, String fileName, MultiPartFormData uploadedFormItems, StringBuilder warnings, CallingContext cc) throws ODKFormAlreadyExistsException, ODKIncompleteSubmissionData, ODKDatastoreException, ODKParseException { super(inputXml, formName, false); if (formXmlData == null) { throw new ODKIncompleteSubmissionData(Reason.MISSING_XML); } // Construct the base table prefix candidate from the // submissionElementDefn.formId. String persistenceStoreFormId = submissionElementDefn.formId; if (persistenceStoreFormId.indexOf(':') != -1) { // this is likely an xmlns-style URI (http://..../) // remove the scheme://domain.org/ from this name, as it is likely // to be common across all forms. Use the remainder as the base // table // prefix candidate. persistenceStoreFormId = submissionElementDefn.formId .substring(submissionElementDefn.formId.indexOf(':') + 1); int idxSlashAfterDomain = persistenceStoreFormId.indexOf('/', 2); if (idxSlashAfterDomain != -1) { // remove the domain from the xmlns -- we'll use the string // after the // domain for the tablespace. persistenceStoreFormId = persistenceStoreFormId.substring(idxSlashAfterDomain + 1); } } // First, replace all slash substitutions with underscores. // Then replace all non-alphanumerics with underscores. // Then trim any leading underscores. persistenceStoreFormId = persistenceStoreFormId.replace(ParserConsts.FORWARD_SLASH_SUBSTITUTION, "_"); persistenceStoreFormId = persistenceStoreFormId.replaceAll("[^\\p{Digit}\\p{Lu}\\p{Ll}\\p{Lo}]", "_"); persistenceStoreFormId = persistenceStoreFormId.replaceAll("^_*", ""); initHelper(uploadedFormItems, formXmlData, inputXml, persistenceStoreFormId, warnings, cc); } enum AuxType { NONE, BC_REF, REF_BLOB, GEO_LAT, GEO_LNG, GEO_ALT, GEO_ACC, LONG_STRING_REF, REF_TEXT }; private String generatePhantomKey(String uriSubmissionFormModel) { return String.format("elem+%1$s(%2$08d-phantom:%3$08d)", uriSubmissionFormModel, elementCount, ++phantomCount); } private void setPrimaryKey(FormDataModel m, String uriSubmissionFormModel, AuxType aux) { String pkString; if (aux != AuxType.NONE) { pkString = String.format("elem+%1$s(%2$08d-%3$s)", uriSubmissionFormModel, elementCount, aux.toString().toLowerCase()); } else { ++elementCount; pkString = String.format("elem+%1$s(%2$08d)", uriSubmissionFormModel, elementCount); } m.setStringField(m.primaryKey, pkString); } private void initHelper(MultiPartFormData uploadedFormItems, MultiPartFormItem xformXmlData, String inputXml, String persistenceStoreFormId, StringBuilder warnings, CallingContext cc) throws ODKDatastoreException, ODKFormAlreadyExistsException, ODKParseException, ODKIncompleteSubmissionData { // /////////////////// // Step 0: ensure that form is not in the process of being deleted // we can't create or update a FormInfo record if there is a pending // deletion for this same id. { FormActionStatusTimestamp formDeletionStatus; formDeletionStatus = MiscTasks.getFormDeletionStatusTimestampOfFormId(rootElementDefn.formId, cc); if (formDeletionStatus != null) { throw new ODKFormAlreadyExistsException( "This form and its data have not yet been fully deleted from the server. Please wait a few minutes and retry."); } if (!submissionElementDefn.formId.equals(rootElementDefn.formId)) { formDeletionStatus = MiscTasks.getFormDeletionStatusTimestampOfFormId(submissionElementDefn.formId, cc); if (formDeletionStatus != null) { throw new ODKFormAlreadyExistsException( "This form and its data have not yet been fully deleted from the server. Please wait a few minutes and retry."); } } } // gain single-access lock record in database... String lockedResourceName = rootElementDefn.formId; String creationLockId = UUID.randomUUID().toString(); Datastore ds = cc.getDatastore(); User user = cc.getCurrentUser(); int i = 0; boolean locked = false; while (!locked) { if ((++i) % 10 == 0) { log.warn("excessive wait count for form creation lock. Count: " + i); try { Thread.sleep(PersistConsts.MIN_SETTLE_MILLISECONDS); } catch (InterruptedException e) { // we remain in the loop even if we get kicked out. } } else if (i != 1) { try { Thread.sleep(PersistConsts.MIN_SETTLE_MILLISECONDS); } catch (InterruptedException e) { // we remain in the loop even if we get kicked out. } } try { TaskLock formCreationTaskLock = ds.createTaskLock(user); if (formCreationTaskLock.obtainLock(creationLockId, lockedResourceName, TaskLockType.CREATE_FORM)) { locked = true; } formCreationTaskLock = null; } catch (ODKTaskLockException e) { e.printStackTrace(); } } // we hold the lock while we create the form here... try { guardedInitHelper(uploadedFormItems, xformXmlData, inputXml, persistenceStoreFormId, warnings, cc); } finally { // release the form creation serialization lock try { for (i = 0; i < 10; i++) { TaskLock formCreationTaskLock = ds.createTaskLock(user); if (formCreationTaskLock.releaseLock(creationLockId, lockedResourceName, TaskLockType.CREATE_FORM)) { break; } formCreationTaskLock = null; try { Thread.sleep(PersistConsts.MIN_SETTLE_MILLISECONDS); } catch (InterruptedException e) { // just move on, this retry mechanism // is to make things nice } } } catch (ODKTaskLockException e) { e.printStackTrace(); } } } public static void updateFormXmlVersion(IForm thisForm, String incomingFormXml, Long modelVersion, CallingContext cc) throws ODKDatastoreException { String revisedXml = xmlWithTimestampComment(xmlWithoutTimestampComment(incomingFormXml), cc.getServerURL()); // update the uiVersion and the form definition file... thisForm.setFormXml(thisForm.getFormFilename(cc), revisedXml, modelVersion, cc); } /** * Return the string by which we uniquely identify a table in the datastore. * This is the schema name concatenated with the table name. Used during table * creation to track the mapping from datastore tables to CommonFieldsBase * objects. * * @param tbl * @return */ private String tableKey(CommonFieldsBase tbl) { return tbl.getSchemaName() + "." + tbl.getTableName(); } private void guardedInitHelper(MultiPartFormData uploadedFormItems, MultiPartFormItem xformXmlData, String incomingFormXml, String persistenceStoreFormId, StringBuilder warnings, CallingContext cc) throws ODKDatastoreException, ODKFormAlreadyExistsException, ODKParseException, ODKIncompleteSubmissionData { // /////////////// // Step 1: create or fetch the Form (FormInfo) submission // // This allows us to delete the form if upload goes bad... // form downloads are immediately enabled unless the upload specifies // that they shouldn't be. String isIncompleteFlag = uploadedFormItems.getSimpleFormField(ServletConsts.TRANSFER_IS_INCOMPLETE); boolean isDownloadEnabled = (isIncompleteFlag == null || isIncompleteFlag.trim().length() == 0); /* true if newly created */ boolean newlyCreatedXForm = false; /* true if we are modifying this form definition. */ boolean updateForm; /* true if the form definition changes, but is compatible */ @SuppressWarnings("unused") boolean differentForm = false; IForm formInfo = null; /* * originationGraceTime: if a previously loaded form was last updated prior * to the originationGraceTime, then require a version change if the new * form is not identical (is changed). */ Date originationGraceTime = new Date(System.currentTimeMillis() - FIFTEEN_MINUTES_IN_MILLISECONDS); /* * originationTime: the time of the form's first upload to the system */ Date originationTime; try { formInfo = FormFactory.retrieveFormByFormId(rootElementDefn.formId, cc); // formId matches... Boolean thisIsEncryptedForm = formInfo.isEncryptedForm(); if (thisIsEncryptedForm == null) { thisIsEncryptedForm = false; } if (isFileEncryptedForm != thisIsEncryptedForm) { // they either both need to be encrypted, or both need to not be // encrypted... throw new ODKFormAlreadyExistsException( "Form encryption status cannot be altered. Form Id must be changed."); } // isEncryptedForm matches... XFormParameters thisRootElementDefn = formInfo.getRootElementDefn(); String thisTitle = formInfo.getViewableName(); String thisMd5Hash = formInfo.getMd5HashFormXml(cc); String md5Hash = CommonFieldsBase.newMD5HashUri(incomingFormXml); boolean same = thisRootElementDefn.equals(rootElementDefn) && (thisMd5Hash == null || md5Hash.equals(thisMd5Hash)); if (same) { // version matches if (thisMd5Hash == null) { // IForm record does not have any attached form definition XML // attach it, set the title, and flag the form as updating // NOTE: this is an error path and not a normal flow updateFormXmlVersion(formInfo, incomingFormXml, rootElementDefn.modelVersion, cc); formInfo.setViewableName(title); updateForm = true; originationTime = new Date(); } else { // The md5Hash of the form file being uploaded matches that // of a fully populated IForm record. // Do not allow changing the title... if (!title.equals(thisTitle)) { throw new ODKFormAlreadyExistsException( "Form title cannot be changed without updating the form version"); } updateForm = false; String existingFormXml = formInfo.getFormXml(cc); // get the upload time of the existing form definition originationTime = FormParserForJavaRosa.xmlTimestamp(existingFormXml); } } else { String existingFormXml = formInfo.getFormXml(cc); // get the upload time of the existing form definition originationTime = FormParserForJavaRosa.xmlTimestamp(existingFormXml); if (FormParserForJavaRosa.xmlWithoutTimestampComment(incomingFormXml) .equals(FormParserForJavaRosa.xmlWithoutTimestampComment(existingFormXml))) { // (version and file match). // The text of the form file being uploaded matches that of a // fully-populated IForm record once the ODK Aggregate // TimestampComment is removed. // Do not allow changing the title... if (!title.equals(thisTitle)) { throw new ODKFormAlreadyExistsException( "Form title cannot be changed without updating the form version."); } updateForm = false; } else { // file is different... // determine if the form is storage-equivalent and if version is // increasing... DifferenceResult diffresult = FormParserForJavaRosa.compareXml(this, existingFormXml, formInfo.getViewableName(), originationTime.after(originationGraceTime)); if (diffresult == DifferenceResult.XFORMS_DIFFERENT) { // form is not storage-compatible throw new ODKFormAlreadyExistsException(); } if (diffresult == DifferenceResult.XFORMS_MISSING_VERSION) { throw new ODKFormAlreadyExistsException( "Form definition file has changed but does not specify a form version. Update the form version and resubmit."); } if (diffresult == DifferenceResult.XFORMS_EARLIER_VERSION) { throw new ODKFormAlreadyExistsException( "Form version is not lexically greater than existing form version. Update the form version and resubmit."); } // update the title and form definition file as needed... if (!thisTitle.equals(title)) { formInfo.setViewableName(title); } updateFormXmlVersion(formInfo, incomingFormXml, rootElementDefn.modelVersion, cc); // mark this as a different form... differentForm = true; updateForm = true; originationTime = new Date(); } } } catch (ODKFormNotFoundException e) { // form is not found -- create it formInfo = FormFactory.createFormId(incomingFormXml, rootElementDefn, isFileEncryptedForm, isDownloadEnabled, title, cc); updateForm = false; newlyCreatedXForm = true; originationTime = new Date(); } // and upload all the media files associated with the form. // Allow updates if the form version has changed (updateForm is true) // or if the originationTime is after the originationGraceTime // e.g., the form version was changed within the last 15 minutes. boolean allowUpdates = updateForm || originationTime.after(originationGraceTime); // If an update is attempted and we don't allow updates, // throw an ODKFormAlreadyExistsException // NOTE: we store new files during this process, in the // expectation that the user simply forgot to update the // version and will do so shortly and upload that revised // form. Set<Map.Entry<String, MultiPartFormItem>> fileSet = uploadedFormItems.getFileNameEntrySet(); for (Map.Entry<String, MultiPartFormItem> itm : fileSet) { if (itm.getValue() == xformXmlData) continue;// ignore the xform -- stored above. // update the images if the form version changed, otherwise throw an // error. if (itm.getValue().getFilename().contains("settings")) { log.info("Adding settings: " + itm.getValue().getFilename()); if (formInfo.setSettingsFile(itm.getValue(), allowUpdates, cc)) { // needed update if (!allowUpdates) { // but we didn't update the form... throw new ODKFormAlreadyExistsException( "Form settings file(s) have changed. Please update the form version and resubmit."); } } continue; } if (formInfo.setXFormMediaFile(itm.getValue(), allowUpdates, cc)) { // needed update if (!allowUpdates) { // but we didn't update the form... throw new ODKFormAlreadyExistsException( "Form media file(s) have changed. Please update the form version and resubmit."); } } } // NOTE: because of caching, we only update the form definition file at // intervals of no more than every 3 seconds. So if you upload a // media file, then immediately upload an altered version, we don't // necessarily increment the uiVersion. // Determine the information about the submission... formInfo.setIsComplete(true); formInfo.persist(cc); formInfo.setAccessEntry(cc); Datastore ds = cc.getDatastore(); User user = cc.getCurrentUser(); FormDefinition fdDefined = null; try { fdDefined = FormDefinition.getFormDefinition(submissionElementDefn.formId, cc); } catch (IllegalStateException e) { e.printStackTrace(); throw new ODKFormAlreadyExistsException( "Internal error: the form already exists but has a bad form definition. Delete it."); } if (fdDefined != null) { // get most recent form-deletion statuses if (newlyCreatedXForm) { throw new ODKFormAlreadyExistsException( "Internal error: Completely new file has pre-existing form definition"); } // we're done -- updated the file and media; form definition doesn't need // updating. return; } // we don't have an existing form definition // -- create a submission association table entry mapping to what will // be // the model. // -- then create the model and iterate on manifesting it in the // database. SubmissionAssociationTable sa = SubmissionAssociationTable .assertSubmissionAssociation(formInfo.getKey().getKey(), submissionElementDefn.formId, cc); fdmSubmissionUri = sa.getUriSubmissionDataModel(); // so we have the formInfo record, but no data model backing it. // Find the submission associated with this form... final List<FormDataModel> fdmList = new ArrayList<FormDataModel>(); // List of successfully asserted relations. // Use a HashMap<String,CommonFieldsBase>(). This allows us to // use the tableKey(CommonFieldsBase) (schema name + table name) // to identify the successfully asserted relations, rather than // the object identity of the CommonFieldsBase objects, which is // inappropriate for our use. final HashMap<String, CommonFieldsBase> assertedRelations = new HashMap<String, CommonFieldsBase>(); try { // //////////////////////////////////////////////// // Step 2: Now build up the parse tree for the form... // final FormDataModel fdm = FormDataModel.assertRelation(cc); // we haven't actually constructed the fdm record yet, so use the // relation when creating the entity key... final EntityKey k = new EntityKey(fdm, fdmSubmissionUri); NamingSet opaque = new NamingSet(); // construct the data model with table and column placeholders. // assumes that the root is a non-repeating group element. final String tableNamePlaceholder = opaque.getTableName(fdm.getSchemaName(), persistenceStoreFormId, "", "CORE"); constructDataModel(opaque, k, fdmList, fdm, k.getKey(), 1, persistenceStoreFormId, "", tableNamePlaceholder, submissionElement, warnings, cc); // find a good set of names... // this also ensures that the table names don't overlap existing // tables // in the datastore. opaque.resolveNames(ds, user); // debug output // for ( FormDataModel m : fdmList ) { // m.print(System.out); // } // and revise the data model with those names... for (FormDataModel m : fdmList) { String tablePlaceholder = m.getPersistAsTable(); if (tablePlaceholder == null) continue; String columnPlaceholder = m.getPersistAsColumn(); String tableName = opaque.resolveTablePlaceholder(tablePlaceholder); String columnName = opaque.resolveColumnPlaceholder(tablePlaceholder, columnPlaceholder); m.setPersistAsColumn(columnName); m.setPersistAsTable(tableName); } // /////////////////////////////////////////// // Step 3: create the backing tables... // // OK. At this point, the construction gets a bit ugly. // We need to handle the possibility that the table // needs to be split into phantom tables. // That happens if the table exceeds the maximum row // size for the persistence layer. // we do this by constructing the form definition from the fdmList // and then testing for successful creation of each table it // defines. // If that table cannot be created, we subdivide it, rearranging // the structure of the fdmList. Repeat until no errors. // Very error prone!!! // FormDefinition fd = null; try { int nAttempts = 0; for (;;) { // place a limit on this process if (++nAttempts > MAX_FORM_CREATION_ATTEMPTS) { log.error("Aborting form-creation due to fail-safe limit (" + MAX_FORM_CREATION_ATTEMPTS + " attempts)!"); throw new ODKParseException("Unable to create form data tables after " + MAX_FORM_CREATION_ATTEMPTS + " attempts."); } fd = new FormDefinition(sa, submissionElementDefn.formId, fdmList, cc); List<CommonFieldsBase> badTables = new ArrayList<CommonFieldsBase>(); for (CommonFieldsBase tbl : fd.getBackingTableSet()) { try { // patch up tbl with desired lengths of string // fields... for (FormDataModel m : fdmList) { if (m.getElementType().equals(ElementType.STRING)) { DataField f = m.getBackingKey(); Integer i = fieldLengths.get(m); if (f != null && i != null) { f.setMaxCharLen(new Long(i)); } } } // CommonFieldsBase objects are re-constructed with each // call to new FormDefinition(...). We need to ensure the // datastore contains the table that each of these objects // refers to. // // Optimization: // // If assertedRelations contains a hit for tableKey(tbl), // then we can assume that the table exists in the datastore // and just update the CommonFieldsBase object in the // assertedRelations map. // // Otherwise, we need to see if we can create it. // // Later on, before we do any more work, we need to // sweep through and call ds.assertRelation() to ensure // that all the CommonFieldsBase objects are fully // initialized (because we don't really know what is // done in the persistence layers during the // ds.assertRelation() call). // if (assertedRelations.containsKey(tableKey(tbl))) { assertedRelations.put(tableKey(tbl), tbl); } else { ds.assertRelation(tbl, user); assertedRelations.put(tableKey(tbl), tbl); } } catch (Exception e1) { // assume it is because the table is too wide... log.warn("Create failed -- assuming phantom table required " + tableKey(tbl) + " Exception: " + e1.toString()); // we expect the following dropRelation to fail, // as the most likely state of the system is // that the table was unable to be created. try { ds.dropRelation(tbl, user); } catch (Exception e2) { // no-op } if ((tbl instanceof DynamicBase) || (tbl instanceof TopLevelDynamicBase)) { /* we know how to subdivide these -- we can recover from this */ badTables.add(tbl); } else { /* there must be something amiss with the database... */ throw e1; } } } for (CommonFieldsBase tbl : badTables) { // dang. We need to create phantom tables... orderlyDivideTable(fdmList, FormDataModel.assertRelation(cc), tbl, opaque, cc); } if (badTables.isEmpty()) { // OK. We created everything and have no re-work. // // Since this might be the N'th time through this // loop, we may have incompletely initialized the // CommonFieldsBase entries in the assertedRelations map. // // Go through that now, asserting each relation. // This ensures that all those entries are // properly initialized. // // Since this was once successful, it should still be. // If it isn't then any database error thrown is // not recoverable. for (CommonFieldsBase tbl : assertedRelations.values()) { ds.assertRelation(tbl, user); } break; } /* * reset the derived fields so that the FormDefinition construction * will work. */ for (FormDataModel m : fdmList) { m.resetDerivedFields(); } } } catch (Exception e) { /* * either something is amiss in the database or there was some sort of * internal error. Try to drop all the successfully created database * tables. */ try { log.warn("Aborting form-creation do to exception: " + e.toString() + ". Datastore exceptions are expected in the following stack trace; other exceptions may indicate a problem:"); e.printStackTrace(); /* if everything were OK, assertedRelations should be empty... */ if (!assertedRelations.isEmpty()) { log.error("assertedRelations not fully unwound!"); Iterator<Entry<String, CommonFieldsBase>> iter = assertedRelations.entrySet().iterator(); while (iter.hasNext()) { Entry<String, CommonFieldsBase> entry = iter.next(); CommonFieldsBase tbl = entry.getValue(); try { log.error("--dropping " + entry.getKey()); ds.dropRelation(tbl, user); } catch (Exception e3) { log.error("--Exception while dropping " + entry.getKey() + " exception: " + e3.toString()); // do nothing... e3.printStackTrace(); } // we tried our best... twice. // Remove the definition whether // or not we were successful. // No point in ever trying again. iter.remove(); } } // scorched earth -- get all the tables and try to drop them all... if (fd != null) { for (CommonFieldsBase tbl : fd.getBackingTableSet()) { try { ds.dropRelation(tbl, user); assertedRelations.remove(tableKey(tbl)); } catch (Exception e3) { // the above may fail because the table was never created... // do nothing... log.warn( "If the following stack trace is not a complaint about a table not existing, it is likely a problem!"); e3.printStackTrace(); } } } } catch (Exception e4) { // just log error... popping out to original exception log.error("dropping of relations unexpectedly failed with exception: " + e4.toString()); e4.printStackTrace(); } throw new ODKParseException("Error processing new form: " + e.toString()); } // TODO: if the above gets killed, how do we clean up? } catch (ODKParseException e) { formInfo.deleteForm(cc); throw e; } catch (ODKDatastoreException e) { formInfo.deleteForm(cc); throw e; } // //////////////////////////////////////////// // Step 4: record the data model... // // if we get here, we were able to create the tables -- record the // form description.... ds.putEntities(fdmList, user); // TODO: if above write fails, how do we clean this up? // and update the complete flag to indicate that upload was fully // successful. sa.setIsPersistenceModelComplete(true); ds.putEntity(sa, user); // And wait until the data is propagated across all server instances. // // Rather than relying on MemCache, we insert this delay here so that // any caller that is creating a form can know that the form definition // has been propagated across the front-ends (subject to fast/slow // clocks). // This assumes that server clock rates never cause drifts of more than // the // network transmission latency between the requester and the server // over // the PersistConsts.MAX_SETTLE_MILLISECONDS time period. // // After this delay interval, the caller can be confident that the form // is visible by whatever server receives the caller's next request // (and this is also true during unit tests). try { Thread.sleep(PersistConsts.MAX_SETTLE_MILLISECONDS); } catch (InterruptedException e) { e.printStackTrace(); } } /** * The creation of the tbl relation has failed. We need to split it into * multiple sub-tables and try again. * * @param fdmList * @param fdmRelation * @param tbl * @param newPhantomTableName */ private void orderlyDivideTable(List<FormDataModel> fdmList, FormDataModel fdmRelation, CommonFieldsBase tbl, NamingSet opaque, CallingContext cc) { log.info("Attempting to divide " + tbl.getTableName()); // Find out how many columns it has... int nCol = tbl.getFieldList().size(); if (tbl instanceof TopLevelDynamicBase) { nCol = nCol - TopLevelDynamicBase.ADDITIONAL_COLUMN_COUNT - 1; } else if (tbl instanceof DynamicBase) { nCol = nCol - DynamicBase.ADDITIONAL_COLUMN_COUNT - 1; } if (nCol < 2) { log.error("Too few columns to subdivide! " + tbl.getTableName()); throw new IllegalStateException("Too few columns to subdivide instance table! " + tbl.getSchemaName() + "." + tbl.getTableName()); } // because of how groups are assigned arbitrarily to tables, we can have a // table // that contains N groups but not the parent of those groups. So in order to // re-subdivide such a table, we need to scan the entire fdmList and build a // set of // parents of the elements contained in the table, provided those parents // are // each fully resident within the table. Set<FormDataModel> tblContentParents = new HashSet<FormDataModel>(); for (FormDataModel m : fdmList) { if (!tbl.equals(m.getBackingObjectPrototype())) { // this isn't in the table being split. continue; } FormDataModel parentTable = m; while (parentTable.getParent() != null) { FormDataModel parent = parentTable.getParent(); if (!tbl.equals(parent.getBackingObjectPrototype())) { // the parent is not within the tbl // so parentTable dominates break; } // and now check that the parent has not been // spread across multiple data tables already. // If it has, we want to keep the previous parent. boolean fragmented = false; for (FormDataModel child : parent.getChildren()) { if (!tbl.equals(child.getBackingObjectPrototype()) && FormDataModel.isFieldStoredWithinDataTable(child.getElementType())) { // the child isn't in the table being split // and the child is one that should be stored // in a data table, so this parent is already // fragmented across multiple tables. fragmented = true; break; } } if (fragmented) { // stick with the current parentTable break; } // daisy-chain up to parent // we must have had an element or a subordinate group... parentTable = parent; } tblContentParents.add(parentTable); } // OK. We have all the elements that can be further split or reallocated // in tblContentParents we should have found something... if (tblContentParents.size() == 0) { log.error("Unable to locate model for backing table! " + tbl.getTableName()); throw new IllegalStateException("Unable to locate model for backing table"); } // go through the tblContentParents dividing them into groups and // raw elements. List<FormDataModel> topElementChange = new ArrayList<FormDataModel>(); List<FormDataModel> groups = new ArrayList<FormDataModel>(); for (;;) { for (FormDataModel m : tblContentParents) { // geopoints, phantoms and groups don't have backing keys if (m.getBackingKey() != null) { topElementChange.add(m); } else { groups.add(m); } } // order the list of groups from high to low... Collections.sort(groups, new Comparator<FormDataModel>() { @Override public int compare(FormDataModel o1, FormDataModel o2) { int c1 = recursivelyCountChildrenInSameTable(o1); int c2 = recursivelyCountChildrenInSameTable(o2); if (c1 > c2) return -1; if (c1 < c2) return 1; return 0; } }); int firstGroupSize = (groups.size() == 0) ? 0 : recursivelyCountChildrenInSameTable(groups.get(0)); if (groups.size() != 0 && ((groups.size() + topElementChange.size() == 1) || firstGroupSize > (3 * nCol) / 4)) { // the parent group is dominated by a very large subgroup // switch to split that subgroup. FormDataModel parentTable = groups.get(0); groups.clear(); topElementChange.clear(); tblContentParents.clear(); for (FormDataModel m : parentTable.getChildren()) { if (!tbl.equals(m.getBackingObjectPrototype())) { // this isn't in the table... continue; } tblContentParents.add(m); } // and then repeat this loop to assign the elements of this // group to the groups and topElementChange lists. // Note that we don't have to patch up the parentTable we are // moving off of, because the tbl will continue to exist. We // just need to move some of its contents to a second table, // either by moving a nested group or geopoint off, or by // creating a phantom table. } else { // OK either the groups are not dominated by a large group so // we can potentially reallocate them across the new and // existing tables or we have more than one raw element so we // can push some of the raw elements into phantom tables. // Proceed to try each of these in turn. break; } } // Table in which to move fields... String newTable; try { newTable = opaque.generateUniqueTableName(tbl.getSchemaName(), tbl.getTableName(), cc); } catch (ODKDatastoreException e) { e.printStackTrace(); log.error("Unable to interrogate database for new table to split backing table! " + tbl.getTableName()); throw new IllegalStateException("unable to interrogate database"); } // Try to move a set of groups into the new table such that the new // table is about 50% of the size of the original table. if (groups.size() > 0) { // list is already ordered from high to low... // go through the list moving the larger groups into tables // until close to half of the elements are moved... log.info("Multiple groups -- splitting to different tables"); int cleaveCount = 0; for (FormDataModel m : groups) { int groupSize = recursivelyCountChildrenInSameTable(m); if (cleaveCount + groupSize > (3 * nCol) / 4) { // this group is too big to add into this particular split // see if there is a smaller group... continue; } recursivelyReassignChildren(m, tbl, newTable); cleaveCount += groupSize; // and if we have cleaved over half, (divide and conquer), retry // it with the database. if (cleaveCount > (nCol / 2)) { log.info("Cleaved along groups. New table: " + newTable + " columnCount: " + cleaveCount); return; } } // and otherwise, if we did cleave anything off, try anyway... // the next time through, we won't have any groups and will need // to create phantom tables, so it is worth trying for this here // now... if (cleaveCount > 0) log.info("Cleaved along groups. New table: " + newTable + " columnCount: " + cleaveCount); return; } log.info("Unable to cleave along groups; attempting phantom table! " + tbl.getTableName()); // Urgh! we don't have any nested groups we can cleave off. // Create a phantom table. We need to preserve the parent-child // relationship and the ordinal ordering even for the // external tables like choices and binary objects. // // To do that, we want to move contiguous child elements into a phantom // table. // To do that, we need to identify the parents of the elements in the table, // even if those parents are already split across tables. Then, we assume // that the // parent with the greatest number of elements also has the greatest number // of // contiguous elements. // for each topElementChange, tally its presence under its parent. Map<FormDataModel, Integer> distinctParents = new HashMap<FormDataModel, Integer>(); for (FormDataModel m : topElementChange) { FormDataModel parent = m.getParent(); Integer a = distinctParents.get(parent); if (a == null) { distinctParents.put(parent, 1); } else { distinctParents.put(parent, a + 1); } } // scan the set of tallies to find the maximum tally. // That will be the parentTable that we will be splitting // with a phantom table. int max = 0; FormDataModel parentTable = null; for (Map.Entry<FormDataModel, Integer> e : distinctParents.entrySet()) { if (e.getValue() > max) { parentTable = e.getKey(); } } // OK. We have the parent table. // // The children array is ordered by ordinal number. // Find the longest contiguous span to cleave off // or partially cleave off. // // This improves the split outcomes for forms with // many multiple choice elements, repeat groups or // media attachments. int idxStart; List<FormDataModel> children = parentTable.getChildren(); // spanCount tracks, for a given key=firstIndexOfSpan, // the count of contiguous values. Map<Integer, Integer> spanCount = new HashMap<Integer, Integer>(); int firstIndexOfSpan = 0; for (idxStart = 0; idxStart < children.size(); ++idxStart) { FormDataModel m = children.get(idxStart); if (!FormDataModel.isFieldStoredWithinDataTable(m.getElementType())) { continue; } if (!tbl.equals(m.getBackingObjectPrototype())) { // the contiguous span is broken... firstIndexOfSpan = idxStart + 1; // the next element... continue; } int elements = recursivelyCountChildrenInSameTable(m); if (elements == 0) { // the contiguous span is broken... firstIndexOfSpan = idxStart + 1; // the next element... } else { Integer v = spanCount.get(firstIndexOfSpan); if (v == null) { spanCount.put(firstIndexOfSpan, 1); } else { spanCount.put(firstIndexOfSpan, v + elements); } } } // find the longest span int maxSpanCount = 0; idxStart = -1; for (Map.Entry<Integer, Integer> spanEntry : spanCount.entrySet()) { if (spanEntry.getValue() > maxSpanCount) { idxStart = spanEntry.getKey(); maxSpanCount = spanEntry.getValue(); } } // now move up to half the desired original table columns of // this span into the phantom table. Typically, we will just // move all of this span into the phantom table because of // question groups, multiple choice or media questions breaking // up the continuity before the 50% mark. String phantomURI = generatePhantomKey(fdmSubmissionUri); int desiredOriginalTableColCount = (nCol / 2); if (idxStart == -1) { log.error( "Failed to split at half the eligible records to move to phantom table " + tbl.getTableName()); throw new IllegalStateException( "Failed to split at half the eligible records to move to phantom table!"); } { // the contiguous elements after idxStart should be moved // to be "under" the phantom table. FormDataModel firstToMove = children.get(idxStart); long remainingOrdinalNumber = firstToMove.getOrdinalNumber(); final long startingOrdinal = remainingOrdinalNumber; // data record... FormDataModel d = cc.getDatastore().createEntityUsingRelation(fdmRelation, cc.getCurrentUser()); fdmList.add(d); d.setStringField(fdmRelation.primaryKey, phantomURI); d.setOrdinalNumber(remainingOrdinalNumber); d.setUriSubmissionDataModel(fdmSubmissionUri); d.setParentUriFormDataModel(parentTable.getUri()); d.setElementName(null); d.setElementType(FormDataModel.ElementType.PHANTOM); d.setPersistAsColumn(null); d.setPersistAsTable(newTable); d.setPersistAsSchema(fdmRelation.getSchemaName()); // OK -- update ordinals of the elements being moved... long ordinalNumber = 0L; int records = 0; for (; idxStart < children.size(); ++idxStart) { if (records >= desiredOriginalTableColCount) { // we have moved the desired number of columns to // the new table -- stop! break; } FormDataModel m = children.get(idxStart); if (!FormDataModel.isFieldStoredWithinDataTable(m.getElementType())) { m.setParentUriFormDataModel(phantomURI); m.setOrdinalNumber(++ordinalNumber); continue; } if (!tbl.equals(m.getBackingObjectPrototype())) { // we need to stop because this item is // already moved out elsewhere and // phantom tables are always contiguous... break; } int elements = recursivelyCountChildrenInSameTable(m); if (elements == 0) { // stop also if this element is already // elsewhere. break; } m.setParentUriFormDataModel(phantomURI); m.setOrdinalNumber(++ordinalNumber); recursivelyReassignChildren(m, tbl, newTable); records += recursivelyCountChildrenInSameTable(m); } // and update the remaining ordinals in the original set... for (; idxStart < children.size(); ++idxStart) { FormDataModel m = children.get(idxStart); m.setOrdinalNumber(++remainingOrdinalNumber); } log.info("Created phantom for " + tbl.getTableName() + " beginning at " + Long.toString(startingOrdinal) + " with a total of " + records + " cleaved"); if (log.isDebugEnabled()) { log.debug("Dump after phantom-split of form list"); for (FormDataModel m : fdmList) { m.print(System.err); } } } } private int recursivelyCountChildrenInSameTable(FormDataModel parent) { int count = 0; for (FormDataModel m : parent.getChildren()) { if (parent.getPersistAsTable().equals(m.getPersistAsTable()) && parent.getPersistAsSchema().equals(m.getPersistAsSchema())) { count += recursivelyCountChildrenInSameTable(m); } } if (parent.getPersistAsColumn() != null) { count++; } return count; } private void recursivelyReassignChildren(FormDataModel biggest, CommonFieldsBase tbl, String newPhantomTableName) { if (!tbl.equals(biggest.getBackingObjectPrototype())) return; biggest.setPersistAsTable(newPhantomTableName); for (FormDataModel m : biggest.getChildren()) { recursivelyReassignChildren(m, tbl, newPhantomTableName); } } /** * Used to recursively process the xform definition tree to create the form * data model. * * @param treeElement * java rosa tree element * * @param parentKey * key from the parent form for proper entity group usage in gae * * @param parent * parent form element * * @throws ODKEntityPersistException * @throws ODKParseException * */ private void constructDataModel(final NamingSet opaque, final EntityKey k, final List<FormDataModel> dmList, final FormDataModel fdm, String parent, int ordinal, String tablePrefix, String nrGroupPrefix, String tableName, TreeElement treeElement, StringBuilder warnings, CallingContext cc) throws ODKEntityPersistException, ODKParseException { // for debugging: printTreeElementInfo(treeElement); FormDataModel d; FormDataModel.ElementType et; String persistAsTable = tableName; String originalPersistAsColumn = opaque.getColumnName(persistAsTable, nrGroupPrefix, treeElement.getName()); String persistAsColumn = originalPersistAsColumn; switch (treeElement.getDataType()) { case org.javarosa.core.model.Constants.DATATYPE_TEXT: /** * Text question type. */ et = FormDataModel.ElementType.STRING; break; case org.javarosa.core.model.Constants.DATATYPE_INTEGER: /** * Numeric question type. These are numbers without decimal points */ et = FormDataModel.ElementType.INTEGER; break; case org.javarosa.core.model.Constants.DATATYPE_DECIMAL: /** * Decimal question type. These are numbers with decimals */ et = FormDataModel.ElementType.DECIMAL; break; case org.javarosa.core.model.Constants.DATATYPE_DATE: /** * Date question type. This has only date component without time. */ et = FormDataModel.ElementType.JRDATE; break; case org.javarosa.core.model.Constants.DATATYPE_TIME: /** * Time question type. This has only time element without date */ et = FormDataModel.ElementType.JRTIME; break; case org.javarosa.core.model.Constants.DATATYPE_DATE_TIME: /** * Date and Time question type. This has both the date and time components */ et = FormDataModel.ElementType.JRDATETIME; break; case org.javarosa.core.model.Constants.DATATYPE_CHOICE: /** * This is a question with alist of options where not more than one option * can be selected at a time. */ et = FormDataModel.ElementType.STRING; // et = FormDataModel.ElementType.SELECT1; // persistAsColumn = null; // persistAsTable = opaque.getTableName(fdm.getSchemaName(), // tablePrefix, nrGroupPrefix, treeElement.getName()); break; case org.javarosa.core.model.Constants.DATATYPE_CHOICE_LIST: /** * This is a question with alist of options where more than one option can * be selected at a time. */ et = FormDataModel.ElementType.SELECTN; opaque.removeColumnName(persistAsTable, persistAsColumn); persistAsColumn = null; persistAsTable = opaque.getTableName(fdm.getSchemaName(), tablePrefix, nrGroupPrefix, treeElement.getName()); break; case org.javarosa.core.model.Constants.DATATYPE_BOOLEAN: /** * Question with true and false answers. */ et = FormDataModel.ElementType.BOOLEAN; break; case org.javarosa.core.model.Constants.DATATYPE_GEOPOINT: /** * Question with location answer. */ et = FormDataModel.ElementType.GEOPOINT; opaque.removeColumnName(persistAsTable, persistAsColumn); persistAsColumn = null; // structured field break; case org.javarosa.core.model.Constants.DATATYPE_BARCODE: /** * Question with barcode string answer. */ et = FormDataModel.ElementType.STRING; break; case org.javarosa.core.model.Constants.DATATYPE_BINARY: /** * Question with external binary answer. */ et = FormDataModel.ElementType.BINARY; opaque.removeColumnName(persistAsTable, persistAsColumn); persistAsColumn = null; persistAsTable = opaque.getTableName(fdm.getSchemaName(), tablePrefix, nrGroupPrefix, treeElement.getName() + "_BN"); break; case org.javarosa.core.model.Constants.DATATYPE_NULL: /* * for nodes that have * no data, or data * type otherwise * unknown */ if (treeElement.isRepeatable()) { // repeatable group... opaque.removeColumnName(persistAsTable, persistAsColumn); persistAsColumn = null; et = FormDataModel.ElementType.REPEAT; persistAsTable = opaque.getTableName(fdm.getSchemaName(), tablePrefix, nrGroupPrefix, treeElement.getName()); } else if (treeElement.getNumChildren() == 0 && dmList.size() != 0) { // assume fields that don't have children are string fields. // but exclude the top-level group, as somebody might define an // empty // form. // the developer likely has not set a type for the field. et = FormDataModel.ElementType.STRING; log.warn("Element " + getTreeElementPath(treeElement) + " does not have a type"); warnings.append("<tr><td>"); warnings.append(getTreeElementPath(treeElement)); warnings.append("</td></tr>"); } else { /* one or more children -- this is a non-repeating group */ opaque.removeColumnName(persistAsTable, persistAsColumn); persistAsColumn = null; et = FormDataModel.ElementType.GROUP; } break; default: case org.javarosa.core.model.Constants.DATATYPE_UNSUPPORTED: et = FormDataModel.ElementType.STRING; break; } Datastore ds = cc.getDatastore(); User user = cc.getCurrentUser(); // data record... d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.NONE); dmList.add(d); final String groupURI = d.getUri(); d.setOrdinalNumber(Long.valueOf(ordinal)); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(parent); d.setElementName(treeElement.getName()); d.setElementType(et); d.setPersistAsColumn(persistAsColumn); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); if (et.equals(ElementType.STRING)) { // track the preferred string lengths of the string fields Integer len = getNodesetStringLength(treeElement); if (len != null) { fieldLengths.put(d, len); } } // and patch up the tree elements that have multiple fields... switch (et) { case BINARY_CONTENT_REF_BLOB: case BOOLEAN: case DECIMAL: case INTEGER: case JRDATE: case JRDATETIME: case JRTIME: case PHANTOM: case REF_BLOB: case SELECT1: case SELECTN: case STRING: // This case keeps lint messages down... break; case BINARY: // binary elements have two additional tables associated with them // -- the _REF and _BLB tables (in addition to _BIN above). persistAsTable = opaque.getTableName(fdm.getSchemaName(), tablePrefix, nrGroupPrefix, treeElement.getName() + "_REF"); // record for VersionedBinaryContentRefBlob.. d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.BC_REF); dmList.add(d); final String bcbURI = d.getUri(); d.setOrdinalNumber(1L); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(groupURI); d.setElementName(treeElement.getName()); d.setElementType(FormDataModel.ElementType.BINARY_CONTENT_REF_BLOB); d.setPersistAsColumn(null); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); persistAsTable = opaque.getTableName(fdm.getSchemaName(), tablePrefix, nrGroupPrefix, treeElement.getName() + "_BLB"); // record for RefBlob... d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.REF_BLOB); dmList.add(d); d.setOrdinalNumber(1L); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(bcbURI); d.setElementName(treeElement.getName()); d.setElementType(FormDataModel.ElementType.REF_BLOB); d.setPersistAsColumn(null); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); break; case GEOPOINT: // geopoints are stored as 4 fields (_LAT, _LNG, _ALT, _ACC) in the // persistence layer. // the geopoint attribute itself has no column, but is a placeholder // within // the data model for the expansion set of these 4 fields. persistAsColumn = opaque.getColumnName(persistAsTable, nrGroupPrefix, treeElement.getName() + "_LAT"); d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.GEO_LAT); dmList.add(d); d.setOrdinalNumber(Long.valueOf(GeoPointConsts.GEOPOINT_LATITUDE_ORDINAL_NUMBER)); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(groupURI); d.setElementName(treeElement.getName()); d.setElementType(FormDataModel.ElementType.DECIMAL); d.setPersistAsColumn(persistAsColumn); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); persistAsColumn = opaque.getColumnName(persistAsTable, nrGroupPrefix, treeElement.getName() + "_LNG"); d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.GEO_LNG); dmList.add(d); d.setOrdinalNumber(Long.valueOf(GeoPointConsts.GEOPOINT_LONGITUDE_ORDINAL_NUMBER)); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(groupURI); d.setElementName(treeElement.getName()); d.setElementType(FormDataModel.ElementType.DECIMAL); d.setPersistAsColumn(persistAsColumn); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); persistAsColumn = opaque.getColumnName(persistAsTable, nrGroupPrefix, treeElement.getName() + "_ALT"); d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.GEO_ALT); dmList.add(d); d.setOrdinalNumber(Long.valueOf(GeoPointConsts.GEOPOINT_ALTITUDE_ORDINAL_NUMBER)); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(groupURI); d.setElementName(treeElement.getName()); d.setElementType(FormDataModel.ElementType.DECIMAL); d.setPersistAsColumn(persistAsColumn); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); persistAsColumn = opaque.getColumnName(persistAsTable, nrGroupPrefix, treeElement.getName() + "_ACC"); d = ds.createEntityUsingRelation(fdm, user); setPrimaryKey(d, fdmSubmissionUri, AuxType.GEO_ACC); dmList.add(d); d.setOrdinalNumber(Long.valueOf(GeoPointConsts.GEOPOINT_ACCURACY_ORDINAL_NUMBER)); d.setUriSubmissionDataModel(k.getKey()); d.setParentUriFormDataModel(groupURI); d.setElementName(treeElement.getName()); d.setElementType(FormDataModel.ElementType.DECIMAL); d.setPersistAsColumn(persistAsColumn); d.setPersistAsTable(persistAsTable); d.setPersistAsSchema(fdm.getSchemaName()); break; case GROUP: // non-repeating group - this modifies the group prefix, // and all children are emitted. if (!parent.equals(k.getKey())) { // incorporate the group name only if it isn't the top-level // group. if (nrGroupPrefix.length() == 0) { nrGroupPrefix = treeElement.getName(); } else { nrGroupPrefix = nrGroupPrefix + "_" + treeElement.getName(); } } // OK -- group with at least one element -- assume no value... // TreeElement list has the begin and end tags for the nested // groups. // Swallow the end tag by looking to see if the prior and current // field names are the same. TreeElement prior = null; int trueOrdinal = 0; for (int i = 0; i < treeElement.getNumChildren(); ++i) { TreeElement current = (TreeElement) treeElement.getChildAt(i); // TODO: make this pay attention to namespace of the tag... if ((prior != null) && (prior.getName().equals(current.getName()))) { // it is the end-group tag... seems to happen with two // adjacent repeat // groups log.info("repeating tag at " + i + " skipping " + current.getName()); prior = current; } else { constructDataModel(opaque, k, dmList, fdm, groupURI, ++trueOrdinal, tablePrefix, nrGroupPrefix, persistAsTable, current, warnings, cc); prior = current; } } break; case REPEAT: // repeating group - clears group prefix // and all children are emitted. // TreeElement list has the begin and end tags for the nested // groups. // Swallow the end tag by looking to see if the prior and current // field names are the same. prior = null; trueOrdinal = 0; for (int i = 0; i < treeElement.getNumChildren(); ++i) { TreeElement current = (TreeElement) treeElement.getChildAt(i); // TODO: make this pay attention to namespace of the tag... if ((prior != null) && (prior.getName().equals(current.getName()))) { // it is the end-group tag... seems to happen with two // adjacent repeat // groups log.info("repeating tag at " + i + " skipping " + current.getName()); prior = current; } else { constructDataModel(opaque, k, dmList, fdm, groupURI, ++trueOrdinal, tablePrefix, "", persistAsTable, current, warnings, cc); prior = current; } } break; } } }