List of usage examples for java.util HashSet remove
public boolean remove(Object o)
From source file:org.tigase.messenger.phone.pro.service.XMPPService.java
private final void updateJaxmppInstances() { final HashSet<BareJID> accountsJids = new HashSet<BareJID>(); for (JaxmppCore jaxmpp : multiJaxmpp.get()) { accountsJids.add(jaxmpp.getSessionObject().getUserBareJid()); }// w w w . ja v a 2 s .c o m final AccountManager am = AccountManager.get(this); for (Account account : am.getAccountsByType(Authenticator.ACCOUNT_TYPE)) { BareJID accountJid = BareJID.bareJIDInstance(account.name); Jaxmpp jaxmpp = multiJaxmpp.get(accountJid); if (jaxmpp == null) { jaxmpp = createJaxmpp(accountJid, account.hashCode()); multiJaxmpp.add(jaxmpp); } // workaround for unknown certificate error jaxmpp.getSessionObject().setProperty("jaxmpp#ThrowedException", null); String password = am.getPassword(account); String nickname = am.getUserData(account, AccountsConstants.FIELD_NICKNAME); String hostname = am.getUserData(account, AccountsConstants.FIELD_HOSTNAME); String resource = am.getUserData(account, AccountsConstants.FIELD_RESOURCE); hostname = hostname == null ? null : hostname.trim(); jaxmpp.getSessionObject().setUserProperty(SessionObject.PASSWORD, password); jaxmpp.getSessionObject().setUserProperty(SessionObject.NICKNAME, nickname); if (hostname != null && TextUtils.isEmpty(hostname)) hostname = null; // sessionObject.setUserProperty(SessionObject.DOMAIN_NAME, // hostname); if (TextUtils.isEmpty(resource)) resource = null; jaxmpp.getSessionObject().setUserProperty(SessionObject.RESOURCE, resource); MobileModeFeature.updateSettings(account, jaxmpp, this); boolean disabled = !Boolean.parseBoolean(am.getUserData(account, AccountsConstants.FIELD_ACTIVE)); jaxmpp.getSessionObject().setUserProperty("CC:DISABLED", disabled); if (disabled) { if (jaxmpp.isConnected()) { this.disconnectJaxmpp(jaxmpp, true); } } else { if (!jaxmpp.isConnected()) { this.connectJaxmpp(jaxmpp, 1L); } } accountsJids.remove(accountJid); } for (BareJID accountJid : accountsJids) { final Jaxmpp jaxmpp = multiJaxmpp.get(accountJid); if (jaxmpp != null) { multiJaxmpp.remove(jaxmpp); (new AsyncTask<Void, Void, Void>() { @Override protected Void doInBackground(Void... params) { try { jaxmpp.disconnect(); // clear presences for account? // app.clearPresences(jaxmpp.getSessionObject(), // false); // is this needed any more?? // JaxmppService.this.rosterProvider.resetStatus(jaxmpp.getSessionObject()); } catch (Exception ex) { Log.e(TAG, "Can't disconnect", ex); } return null; } }).execute(); } } dataRemover.removeUnusedData(this); }
From source file:org.apache.sysml.hops.cost.CostEstimator.java
private double rGetTimeEstimate(ProgramBlock pb, HashMap<String, VarStats> stats, HashSet<String> memoFunc, boolean recursive) throws DMLRuntimeException { double ret = 0; if (pb instanceof WhileProgramBlock) { WhileProgramBlock tmp = (WhileProgramBlock) pb; if (recursive) for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); ret *= DEFAULT_NUMITER;/* ww w .j a va2 s . co m*/ } else if (pb instanceof IfProgramBlock) { IfProgramBlock tmp = (IfProgramBlock) pb; if (recursive) { for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); if (tmp.getChildBlocksElseBody() != null) for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) { ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); ret /= 2; //weighted sum } } } else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock { ForProgramBlock tmp = (ForProgramBlock) pb; if (recursive) for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); ret *= getNumIterations(stats, tmp.getIterablePredicateVars()); } else if (pb instanceof FunctionProgramBlock && !(pb instanceof ExternalFunctionProgramBlock)) //see generic { FunctionProgramBlock tmp = (FunctionProgramBlock) pb; if (recursive) for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); } else { ArrayList<Instruction> tmp = pb.getInstructions(); for (Instruction inst : tmp) { if (inst instanceof CPInstruction) //CP { //obtain stats from createvar, cpvar, rmvar, rand maintainCPInstVariableStatistics((CPInstruction) inst, stats); //extract statistics (instruction-specific) Object[] o = extractCPInstStatistics(inst, stats); VarStats[] vs = (VarStats[]) o[0]; String[] attr = (String[]) o[1]; //if(LOG.isDebugEnabled()) // LOG.debug(inst); //call time estimation for inst ret += getCPInstTimeEstimate(inst, vs, attr); if (inst instanceof FunctionCallCPInstruction) //functions { FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst; String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName()); //awareness of recursive functions, missing program if (!memoFunc.contains(fkey) && pb.getProgram() != null) { if (LOG.isDebugEnabled()) LOG.debug("Begin Function " + fkey); memoFunc.add(fkey); Program prog = pb.getProgram(); FunctionProgramBlock fpb = prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName()); ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive); memoFunc.remove(fkey); if (LOG.isDebugEnabled()) LOG.debug("End Function " + fkey); } } } else if (inst instanceof MRJobInstruction) //MR { //obtain stats for job maintainMRJobInstVariableStatistics(inst, stats); //extract input statistics Object[] o = extractMRJobInstStatistics(inst, stats); VarStats[] vs = (VarStats[]) o[0]; //if(LOG.isDebugEnabled()) // LOG.debug(inst); if (LOG.isDebugEnabled()) LOG.debug("Begin MRJob type=" + ((MRJobInstruction) inst).getJobType()); //call time estimation for complex MR inst ret += getMRJobInstTimeEstimate(inst, vs, null); if (LOG.isDebugEnabled()) LOG.debug("End MRJob"); //cleanup stats for job cleanupMRJobVariableStatistics(inst, stats); } } } return ret; }
From source file:com.ibm.bi.dml.hops.cost.CostEstimator.java
/** * /*w w w . j a va 2 s . c om*/ * @param pb * @param vars * @param stats * @return * @throws DMLRuntimeException * @throws DMLUnsupportedOperationException */ private double rGetTimeEstimate(ProgramBlock pb, HashMap<String, VarStats> stats, HashSet<String> memoFunc, boolean recursive) throws DMLRuntimeException, DMLUnsupportedOperationException { double ret = 0; if (pb instanceof WhileProgramBlock) { WhileProgramBlock tmp = (WhileProgramBlock) pb; if (recursive) for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); ret *= DEFAULT_NUMITER; } else if (pb instanceof IfProgramBlock) { IfProgramBlock tmp = (IfProgramBlock) pb; if (recursive) { for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); if (tmp.getChildBlocksElseBody() != null) for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) { ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); ret /= 2; //weighted sum } } } else if (pb instanceof ForProgramBlock) //includes ParFORProgramBlock { ForProgramBlock tmp = (ForProgramBlock) pb; if (recursive) for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); ret *= getNumIterations(stats, tmp.getIterablePredicateVars()); } else if (pb instanceof FunctionProgramBlock && !(pb instanceof ExternalFunctionProgramBlock)) //see generic { FunctionProgramBlock tmp = (FunctionProgramBlock) pb; if (recursive) for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive); } else { ArrayList<Instruction> tmp = pb.getInstructions(); for (Instruction inst : tmp) { if (inst instanceof CPInstruction) //CP { //obtain stats from createvar, cpvar, rmvar, rand maintainCPInstVariableStatistics((CPInstruction) inst, stats); //extract statistics (instruction-specific) Object[] o = extractCPInstStatistics(inst, stats); VarStats[] vs = (VarStats[]) o[0]; String[] attr = (String[]) o[1]; //if(LOG.isDebugEnabled()) // LOG.debug(inst); //call time estimation for inst ret += getCPInstTimeEstimate(inst, vs, attr); if (inst instanceof FunctionCallCPInstruction) //functions { FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst; String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName()); //awareness of recursive functions, missing program if (!memoFunc.contains(fkey) && pb.getProgram() != null) { if (LOG.isDebugEnabled()) LOG.debug("Begin Function " + fkey); memoFunc.add(fkey); Program prog = pb.getProgram(); FunctionProgramBlock fpb = prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName()); ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive); memoFunc.remove(fkey); if (LOG.isDebugEnabled()) LOG.debug("End Function " + fkey); } } } else if (inst instanceof MRJobInstruction) //MR { //obtain stats for job maintainMRJobInstVariableStatistics(inst, stats); //extract input statistics Object[] o = extractMRJobInstStatistics(inst, stats); VarStats[] vs = (VarStats[]) o[0]; //if(LOG.isDebugEnabled()) // LOG.debug(inst); if (LOG.isDebugEnabled()) LOG.debug("Begin MRJob type=" + ((MRJobInstruction) inst).getJobType()); //call time estimation for complex MR inst ret += getMRJobInstTimeEstimate(inst, vs, null); if (LOG.isDebugEnabled()) LOG.debug("End MRJob"); //cleanup stats for job cleanupMRJobVariableStatistics(inst, stats); } } } return ret; }
From source file:org.apache.hadoop.hbase.procedure2.ProcedureExecutor.java
private void loadProcedures(final ProcedureIterator procIter, final boolean abortOnCorruption) throws IOException { // 1. Build the rollback stack int runnablesCount = 0; while (procIter.hasNext()) { Procedure proc = procIter.next(); if (!proc.hasParent() && !proc.isFinished()) { rollbackStack.put(proc.getProcId(), new RootProcedureState()); }// w w w .jav a 2s . com // add the procedure to the map proc.beforeReplay(getEnvironment()); procedures.put(proc.getProcId(), proc); // add the nonce to the map if (proc.getNonceKey() != null) { nonceKeysToProcIdsMap.put(proc.getNonceKey(), proc.getProcId()); } if (proc.getState() == ProcedureState.RUNNABLE) { runnablesCount++; } } // 2. Initialize the stacks ArrayList<Procedure> runnableList = new ArrayList(runnablesCount); HashSet<Procedure> waitingSet = null; procIter.reset(); while (procIter.hasNext()) { Procedure proc = procIter.next(); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Loading procedure state=%s isFailed=%s: %s", proc.getState(), proc.hasException(), proc)); } Long rootProcId = getRootProcedureId(proc); if (rootProcId == null) { // The 'proc' was ready to run but the root procedure was rolledback? runnables.addBack(proc); continue; } if (!proc.hasParent() && proc.isFinished()) { if (LOG.isDebugEnabled()) { LOG.debug(String.format("The procedure is completed state=%s isFailed=%s", proc.getState(), proc.hasException())); } assert !rollbackStack.containsKey(proc.getProcId()); procedures.remove(proc.getProcId()); completed.put(proc.getProcId(), newResultFromProcedure(proc)); continue; } if (proc.hasParent() && !proc.isFinished()) { Procedure parent = procedures.get(proc.getParentProcId()); // corrupted procedures are handled later at step 3 if (parent != null) { parent.incChildrenLatch(); } } RootProcedureState procStack = rollbackStack.get(rootProcId); procStack.loadStack(proc); switch (proc.getState()) { case RUNNABLE: runnableList.add(proc); break; case WAITING_TIMEOUT: if (waitingSet == null) { waitingSet = new HashSet<Procedure>(); } waitingSet.add(proc); break; case FINISHED: if (proc.hasException()) { // add the proc to the runnables to perform the rollback runnables.addBack(proc); break; } case ROLLEDBACK: case INITIALIZING: String msg = "Unexpected " + proc.getState() + " state for " + proc; LOG.error(msg); throw new UnsupportedOperationException(msg); default: break; } } // 3. Validate the stacks int corruptedCount = 0; Iterator<Map.Entry<Long, RootProcedureState>> itStack = rollbackStack.entrySet().iterator(); while (itStack.hasNext()) { Map.Entry<Long, RootProcedureState> entry = itStack.next(); RootProcedureState procStack = entry.getValue(); if (procStack.isValid()) continue; for (Procedure proc : procStack.getSubprocedures()) { LOG.error("corrupted procedure: " + proc); procedures.remove(proc.getProcId()); runnableList.remove(proc); if (waitingSet != null) waitingSet.remove(proc); corruptedCount++; } itStack.remove(); } if (abortOnCorruption && corruptedCount > 0) { throw new IOException("found " + corruptedCount + " procedures on replay"); } // 4. Push the runnables if (!runnableList.isEmpty()) { // TODO: See ProcedureWALFormatReader#hasFastStartSupport // some procedure may be started way before this stuff. for (int i = runnableList.size() - 1; i >= 0; --i) { Procedure proc = runnableList.get(i); if (!proc.hasParent()) { sendProcedureLoadedNotification(proc.getProcId()); } if (proc.wasExecuted()) { runnables.addFront(proc); } else { // if it was not in execution, it can wait. runnables.addBack(proc); } } } }
From source file:org.opendatakit.builder.CsvUtil.java
/** * Export the given tableId. Exports three csv files to the output/csv * directory under the appName:/* ww w . j a va2s .co m*/ * <ul> * <li>tableid.fileQualifier.csv - data table</li> * <li>tableid.fileQualifier.definition.csv - data table column definition</li> * <li>tableid.fileQualifier.properties.csv - key-value store of this table</li> * </ul> * If fileQualifier is null or an empty string, then it emits to * <ul> * <li>tableid.csv - data table</li> * <li>tableid.definition.csv - data table column definition</li> * <li>tableid.properties.csv - key-value store of this table</li> * </ul> * <p> * Used in ExportTask * * @param exportListener We send it progress updates * @param db the database handle * @param tableId the id of the table to export * @param orderedDefns a list of the columns in the table * @param fileQualifier the prefix that the user wants to put before the output filename * @return whether it was successful * @throws ServicesAvailabilityException if the database is down */ @SuppressWarnings("unused") public boolean exportSeparable(ExportListener exportListener, DbHandle db, String tableId, OrderedColumns orderedDefns, String fileQualifier) throws ServicesAvailabilityException { // building array of columns to select and header row for output file // then we are including all the metadata columns. ArrayList<String> columns = new ArrayList<>(); WebLogger.getLogger(appName).i(TAG, "exportSeparable: tableId: " + tableId + " fileQualifier: " + (fileQualifier == null ? "<null>" : fileQualifier)); // put the user-relevant metadata columns in leftmost columns columns.add(DataTableColumns.ID); columns.add(DataTableColumns.FORM_ID); columns.add(DataTableColumns.LOCALE); columns.add(DataTableColumns.SAVEPOINT_TYPE); columns.add(DataTableColumns.SAVEPOINT_TIMESTAMP); columns.add(DataTableColumns.SAVEPOINT_CREATOR); // add the data columns for (ColumnDefinition cd : orderedDefns.getColumnDefinitions()) { if (cd.isUnitOfRetention()) { columns.add(cd.getElementKey()); } } // And now add all remaining export columns String[] exportColumns = supervisor.getDatabase().getExportColumns(); for (String colName : exportColumns) { if (columns.contains(colName)) { continue; } columns.add(colName); } File tableInstancesFolder = new File(ODKFileUtils.getInstancesFolder(appName, tableId)); HashSet<File> instancesWithData = new HashSet<>(); if (tableInstancesFolder.exists() && tableInstancesFolder.isDirectory()) { File[] subDirectories = tableInstancesFolder.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname.isDirectory() && pathname.list().length != 0; } }); instancesWithData.addAll(Arrays.asList(subDirectories)); } OutputStreamWriter output = null; File outputCsv = null; try { // both files go under the output/csv directory... outputCsv = new File(ODKFileUtils.getOutputTableCsvFile(appName, tableId, fileQualifier)); if (!outputCsv.mkdirs()) { throw new IOException(); } // emit properties files File definitionCsv = new File( ODKFileUtils.getOutputTableDefinitionCsvFile(appName, tableId, fileQualifier)); File propertiesCsv = new File( ODKFileUtils.getOutputTablePropertiesCsvFile(appName, tableId, fileQualifier)); if (!writePropertiesCsv(db, tableId, orderedDefns, definitionCsv, propertiesCsv)) { return false; } // getting data String whereString = DataTableColumns.SAVEPOINT_TYPE + " IS NOT NULL AND (" + DataTableColumns.CONFLICT_TYPE + " IS NULL OR " + DataTableColumns.CONFLICT_TYPE + " = " + Integer.toString(ConflictType.LOCAL_UPDATED_UPDATED_VALUES) + ")"; BindArgs emptyArgs = new BindArgs(new Object[0]); String[] emptyArray = new String[0]; UserTable table = supervisor.getDatabase().simpleQuery(appName, db, tableId, orderedDefns, whereString, emptyArgs, emptyArray, null, null, null, null, null); // emit data table... File file = new File(outputCsv, tableId + (fileQualifier != null && !fileQualifier.isEmpty() ? "." + fileQualifier : "") + ".csv"); FileOutputStream out = new FileOutputStream(file); output = new OutputStreamWriter(out, CharEncoding.UTF_8); RFC4180CsvWriter cw = new RFC4180CsvWriter(output); // don't have to worry about quotes in elementKeys... cw.writeNext(columns.toArray(new String[columns.size()])); String[] row = new String[columns.size()]; for (int i = 0; i < table.getNumberOfRows(); i++) { if (i % 5 == 0) { exportListener.updateProgressDetail(i); } Row dataRow = table.getRowAtIndex(i); for (int j = 0; j < columns.size(); ++j) { row[j] = dataRow.getDataByKey(columns.get(j)); } cw.writeNext(row); /* * Copy all attachment files into the output directory tree. * Don't worry about whether they are referenced in the current * row. This is a simplification (and biases toward preserving * data). */ String instanceId = table.getRowId(i); File tableInstanceFolder = new File(ODKFileUtils.getInstanceFolder(appName, tableId, instanceId)); if (instancesWithData.contains(tableInstanceFolder)) { File outputInstanceFolder = new File( ODKFileUtils.getOutputCsvInstanceFolder(appName, tableId, instanceId)); if (!outputInstanceFolder.mkdirs()) { throw new IOException(); } ODKFileUtils.copyDirectory(tableInstanceFolder, outputInstanceFolder); instancesWithData.remove(tableInstanceFolder); } } cw.flush(); cw.close(); return true; } catch (IOException ignored) { try { File outputCsvFolder = new File(ODKFileUtils.getOutputCsvFolder(appName)); while (ODKFileUtils.directoryContains(outputCsvFolder, outputCsv)) { ODKFileUtils.deleteDirectory(outputCsv); outputCsv = outputCsv.getParentFile(); } } catch (IOException e1) { WebLogger.getLogger(appName).printStackTrace(e1); return false; } return false; } finally { try { if (output != null) { output.close(); } } catch (IOException ignored) { // we couldn't even open the file } } }
From source file:org.apache.hadoop.hive.ql.metadata.Hive.java
public void reloadFunctions() throws HiveException { HashSet<String> registryFunctions = new HashSet<String>(FunctionRegistry.getFunctionNames(".+\\..+")); for (Function function : getAllFunctions()) { String functionName = function.getFunctionName(); try {/*from w ww . j a va2 s . c om*/ LOG.info("Registering function " + functionName + " " + function.getClassName()); String qualFunc = FunctionUtils.qualifyFunctionName(functionName, function.getDbName()); FunctionRegistry.registerPermanentFunction(qualFunc, function.getClassName(), false, FunctionTask.toFunctionResource(function.getResourceUris())); registryFunctions.remove(qualFunc); } catch (Exception e) { LOG.warn("Failed to register persistent function " + functionName + ":" + function.getClassName() + ". Ignore and continue."); } } // unregister functions from local system registry that are not in getAllFunctions() for (String functionName : registryFunctions) { try { FunctionRegistry.unregisterPermanentFunction(functionName); } catch (Exception e) { LOG.warn("Failed to unregister persistent function " + functionName + "on reload. Ignore and continue."); } } }
From source file:org.apache.sysml.runtime.compress.CompressedMatrixBlock.java
/** * Compress block.// www. j a v a 2 s .c o m * * @param k number of threads * @throws DMLRuntimeException if DMLRuntimeException occurs */ public void compress(int k) throws DMLRuntimeException { //check for redundant compression if (isCompressed()) { throw new DMLRuntimeException("Redundant compression, block already compressed."); } Timing time = new Timing(true); _stats = new CompressionStatistics(); // SAMPLE-BASED DECISIONS: // Decisions such as testing if a column is amenable to bitmap // compression or evaluating co-coding potentionls are made based on a // subset of the rows. For large datasets, sampling might take a // significant amount of time. So, we generate only one sample and use // it for the entire compression process. //prepare basic meta data and deep copy / transpose input final int numRows = getNumRows(); final int numCols = getNumColumns(); final boolean sparse = isInSparseFormat(); final double sp = OptimizerUtils.getSparsity(numRows, numCols, getNonZeros()); MatrixBlock rawblock = !TRANSPOSE_INPUT ? new MatrixBlock(this) : LibMatrixReorg.transpose(this, new MatrixBlock(numCols, numRows, sparse), k); //construct sample-based size estimator CompressedSizeEstimator bitmapSizeEstimator = SizeEstimatorFactory.getSizeEstimator(rawblock, numRows); // The current implementation of this method is written for correctness, // not for performance or for minimal use of temporary space. // We start with a full set of columns. HashSet<Integer> remainingCols = new HashSet<Integer>(); for (int i = 0; i < numCols; i++) remainingCols.add(i); // PHASE 1: Classify columns by compression type // We start by determining which columns are amenable to bitmap compression double uncompressedColumnSize = getUncompressedSize(numRows, 1, sp); // information about the bitmap amenable columns List<Integer> bitmapCols = new ArrayList<Integer>(); List<Integer> uncompressedCols = new ArrayList<Integer>(); List<Integer> colsCards = new ArrayList<Integer>(); List<Long> compressedSizes = new ArrayList<Long>(); HashMap<Integer, Double> compressionRatios = new HashMap<Integer, Double>(); // Classify columns according to ration (size uncompressed / size compressed), // where a column is compressible if ratio > 1. CompressedSizeInfo[] sizeInfos = (k > 1) ? computeCompressedSizeInfos(bitmapSizeEstimator, numCols, k) : computeCompressedSizeInfos(bitmapSizeEstimator, numCols); for (int col = 0; col < numCols; col++) { long compressedSize = sizeInfos[col].getMinSize(); double compRatio = uncompressedColumnSize / compressedSize; if (compRatio > 1) { bitmapCols.add(col); compressionRatios.put(col, compRatio); colsCards.add(sizeInfos[col].getEstCarinality()); compressedSizes.add(compressedSize); } else uncompressedCols.add(col); } _stats.timePhase1 = time.stop(); if (LOG.isDebugEnabled()) { LOG.debug("Compression statistics:"); LOG.debug("--compression phase 1: " + _stats.timePhase1); } // PHASE 2: Grouping columns // Divide the bitmap columns into column groups. List<int[]> bitmapColGrps = PlanningCoCoder.findCocodesByPartitioning(bitmapSizeEstimator, bitmapCols, colsCards, compressedSizes, numRows, isInSparseFormat() ? sp : 1, k); _stats.timePhase2 = time.stop(); if (LOG.isDebugEnabled()) LOG.debug("--compression phase 2: " + _stats.timePhase2); if (INVESTIGATE_ESTIMATES) { double est = 0; for (int[] groupIndices : bitmapColGrps) est += bitmapSizeEstimator.estimateCompressedColGroupSize(groupIndices).getMinSize(); est += uncompressedCols.size() * uncompressedColumnSize; _stats.estSize = est; } // PHASE 3: Compress and correct sample-based decisions ColGroup[] colGroups = (k > 1) ? compressColGroups(rawblock, bitmapSizeEstimator, compressionRatios, numRows, sp, bitmapColGrps, k) : compressColGroups(rawblock, bitmapSizeEstimator, compressionRatios, numRows, sp, bitmapColGrps); allocateColGroupList(); for (int j = 0; j < colGroups.length; j++) { if (colGroups[j] != null) { for (int col : colGroups[j].getColIndices()) remainingCols.remove(col); _colGroups.add(colGroups[j]); } } _stats.timePhase3 = time.stop(); if (LOG.isDebugEnabled()) LOG.debug("--compression phase 3: " + _stats.timePhase3); // Phase 4: Cleanup // The remaining columns are stored uncompressed as one big column group if (!remainingCols.isEmpty()) { ArrayList<Integer> list = new ArrayList<Integer>(remainingCols); ColGroupUncompressed ucgroup = new ColGroupUncompressed(list, rawblock); _colGroups.add(ucgroup); } _stats.size = estimateCompressedSizeInMemory(); _stats.ratio = estimateSizeInMemory() / _stats.size; //final cleanup (discard uncompressed block) rawblock.cleanupBlock(true, true); this.cleanupBlock(true, true); _stats.timePhase4 = time.stop(); if (LOG.isDebugEnabled()) { LOG.debug("--compression phase 4: " + _stats.timePhase4); LOG.debug("--num col groups: " + _colGroups.size()); LOG.debug("--compressed size: " + _stats.size); LOG.debug("--compression ratio: " + _stats.ratio); } }
From source file:imitationNLG.SFX.java
public Double evaluateGeneration(HashMap<String, JAROW> classifierAttrs, HashMap<String, HashMap<String, JAROW>> classifierWords, ArrayList<DatasetInstance> trainingData, ArrayList<DatasetInstance> testingData, HashMap<String, HashSet<String>> availableAttributeActions, HashMap<String, HashMap<String, HashSet<Action>>> availableWordActions, HashMap<Integer, HashSet<String>> nGrams, boolean printResults, int epoch) { System.out.println("Evaluate argument generation "); int totalArgDistance = 0; ArrayList<ScoredFeaturizedTranslation<IString, String>> generations = new ArrayList<>(); ArrayList<ArrayList<Action>> generationActions = new ArrayList<>(); HashMap<ArrayList<Action>, DatasetInstance> generationActionsMap = new HashMap<>(); ArrayList<ArrayList<Sequence<IString>>> finalReferences = new ArrayList<>(); ArrayList<String> predictedStrings = new ArrayList<>(); ArrayList<String> predictedStringMRs = new ArrayList<>(); ArrayList<Double> attrCoverage = new ArrayList<>(); ArrayList<ArrayList<String>> predictedAttrLists = new ArrayList<>(); HashSet<HashMap<String, HashSet<String>>> mentionedAttrs = new HashSet<HashMap<String, HashSet<String>>>(); for (DatasetInstance di : testingData) { String predicate = di.getMeaningRepresentation().getPredicate(); ArrayList<Action> predictedActionList = new ArrayList<>(); ArrayList<Action> predictedWordList = new ArrayList<>(); //PHRASE GENERATION EVALUATION String predictedAttr = ""; ArrayList<String> predictedAttrValues = new ArrayList<>(); ArrayList<String> predictedAttributes = new ArrayList<>(); HashSet<String> attrValuesToBeMentioned = new HashSet<>(); HashSet<String> attrValuesAlreadyMentioned = new HashSet<>(); HashMap<String, ArrayList<String>> valuesToBeMentioned = new HashMap<>(); for (String attribute : di.getMeaningRepresentation().getAttributes().keySet()) { for (String value : di.getMeaningRepresentation().getAttributes().get(attribute)) { attrValuesToBeMentioned.add(attribute.toLowerCase() + "=" + value.toLowerCase()); }/*from w w w .java2s.co m*/ valuesToBeMentioned.put(attribute, new ArrayList<>(di.getMeaningRepresentation().getAttributes().get(attribute))); } if (attrValuesToBeMentioned.isEmpty()) { attrValuesToBeMentioned.add("empty=empty"); } HashSet<String> attrValuesToBeMentionedCopy = new HashSet<>(attrValuesToBeMentioned); while (!predictedAttr.equals(SFX.TOKEN_END) && predictedAttrValues.size() < maxAttrRealizationSize) { if (!predictedAttr.isEmpty()) { attrValuesToBeMentioned.remove(predictedAttr); } Instance attrTrainingVector = SFX.this.createAttrInstance(predicate, "@TOK@", predictedAttrValues, predictedActionList, attrValuesAlreadyMentioned, attrValuesToBeMentioned, di.getMeaningRepresentation(), availableAttributeActions); if (attrTrainingVector != null) { Prediction predictAttr = classifierAttrs.get(predicate).predict(attrTrainingVector); if (predictAttr.getLabel() != null) { predictedAttr = predictAttr.getLabel().trim(); String predictedValue = ""; if (!predictedAttr.equals(SFX.TOKEN_END)) { predictedValue = chooseNextValue(predictedAttr, attrValuesToBeMentioned, trainingData); HashSet<String> rejectedAttrs = new HashSet<String>(); while (predictedValue.isEmpty() && !predictedAttr.equals(SFX.TOKEN_END)) { rejectedAttrs.add(predictedAttr); predictedAttr = SFX.TOKEN_END; double maxScore = -Double.MAX_VALUE; for (String attr : predictAttr.getLabel2Score().keySet()) { if (!rejectedAttrs.contains(attr) && (Double .compare(predictAttr.getLabel2Score().get(attr), maxScore) > 0)) { maxScore = predictAttr.getLabel2Score().get(attr); predictedAttr = attr; } } if (!predictedAttr.equals(SFX.TOKEN_END)) { predictedValue = chooseNextValue(predictedAttr, attrValuesToBeMentioned, trainingData); } } } if (!predictedAttr.equals(SFX.TOKEN_END)) { predictedAttr += "=" + predictedValue; } predictedAttrValues.add(predictedAttr); String attribute = predictedAttrValues.get(predictedAttrValues.size() - 1).split("=")[0]; String attrValue = predictedAttrValues.get(predictedAttrValues.size() - 1); predictedAttributes.add(attrValue); //GENERATE PHRASES if (!attribute.equals(SFX.TOKEN_END)) { if (classifierWords.get(predicate).containsKey(attribute)) { String predictedWord = ""; boolean isValueMentioned = false; String valueTBM = ""; if (attrValue.contains("=")) { valueTBM = attrValue.substring(attrValue.indexOf('=') + 1); } if (valueTBM.isEmpty()) { isValueMentioned = true; } ArrayList<String> subPhrase = new ArrayList<>(); while (!predictedWord.equals(RoboCup.TOKEN_END) && predictedWordList.size() < maxWordRealizationSize) { ArrayList<String> predictedAttributesForInstance = new ArrayList<>(); for (int i = 0; i < predictedAttributes.size() - 1; i++) { predictedAttributesForInstance.add(predictedAttributes.get(i)); } if (!predictedAttributes.get(predictedAttributes.size() - 1) .equals(attrValue)) { predictedAttributesForInstance .add(predictedAttributes.get(predictedAttributes.size() - 1)); } Instance wordTrainingVector = createWordInstance(predicate, new Action("@TOK@", attrValue), predictedAttributesForInstance, predictedActionList, isValueMentioned, attrValuesAlreadyMentioned, attrValuesToBeMentioned, di.getMeaningRepresentation(), availableWordActions.get(predicate), nGrams, false); if (wordTrainingVector != null) { if (classifierWords.get(predicate) != null) { if (classifierWords.get(predicate).get(attribute) != null) { Prediction predictWord = classifierWords.get(predicate) .get(attribute).predict(wordTrainingVector); if (predictWord.getLabel() != null) { predictedWord = predictWord.getLabel().trim(); predictedActionList.add(new Action(predictedWord, attrValue)); if (!predictedWord.equals(SFX.TOKEN_END)) { subPhrase.add(predictedWord); predictedWordList.add(new Action(predictedWord, attrValue)); } } else { predictedWord = SFX.TOKEN_END; predictedActionList.add(new Action(predictedWord, attrValue)); } } else { predictedWord = SFX.TOKEN_END; predictedActionList.add(new Action(predictedWord, attrValue)); } } } if (!isValueMentioned) { if (!predictedWord.equals(SFX.TOKEN_END)) { if (predictedWord.startsWith(SFX.TOKEN_X) && (valueTBM.matches("\"[xX][0-9]+\"") || valueTBM.matches("[xX][0-9]+") || valueTBM.startsWith(SFX.TOKEN_X))) { isValueMentioned = true; } else if (!predictedWord.startsWith(SFX.TOKEN_X) && !(valueTBM.matches("\"[xX][0-9]+\"") || valueTBM.matches("[xX][0-9]+") || valueTBM.startsWith(SFX.TOKEN_X))) { String valueToCheck = valueTBM; if (valueToCheck.equals("no") || valueToCheck.equals("yes") || valueToCheck.equals("yes or no") || valueToCheck.equals("none") || valueToCheck.equals("dont_care") || valueToCheck.equals("empty")) { if (attribute.contains("=")) { valueToCheck = attribute.replace("=", ":"); } else { valueToCheck = attribute + ":" + valueTBM; } } if (!valueToCheck.equals("empty:empty") && valueAlignments.containsKey(valueToCheck)) { for (ArrayList<String> alignedStr : valueAlignments .get(valueToCheck).keySet()) { if (endsWith(subPhrase, alignedStr)) { isValueMentioned = true; break; } } } } } if (isValueMentioned) { attrValuesAlreadyMentioned.add(attrValue); attrValuesToBeMentioned.remove(attrValue); } } String mentionedAttrValue = ""; if (!predictedWord.startsWith(SFX.TOKEN_X)) { for (String attrValueTBM : attrValuesToBeMentioned) { if (attrValueTBM.contains("=")) { String value = attrValueTBM .substring(attrValueTBM.indexOf('=') + 1); if (!(value.matches("\"[xX][0-9]+\"") || value.matches("[xX][0-9]+") || value.startsWith(SFX.TOKEN_X))) { String valueToCheck = value; if (valueToCheck.equals("no") || valueToCheck.equals("yes") || valueToCheck.equals("yes or no") || valueToCheck.equals("none") || valueToCheck.equals("dont_care") || valueToCheck.equals("empty")) { valueToCheck = attrValueTBM.replace("=", ":"); } if (!valueToCheck.equals("empty:empty") && valueAlignments.containsKey(valueToCheck)) { for (ArrayList<String> alignedStr : valueAlignments .get(valueToCheck).keySet()) { if (endsWith(subPhrase, alignedStr)) { mentionedAttrValue = attrValueTBM; break; } } } } } } } if (!mentionedAttrValue.isEmpty()) { attrValuesAlreadyMentioned.add(attrValue); attrValuesToBeMentioned.remove(mentionedAttrValue); } } if (predictedWordList.size() >= maxWordRealizationSize && !predictedActionList .get(predictedActionList.size() - 1).getWord().equals(SFX.TOKEN_END)) { predictedWord = SFX.TOKEN_END; predictedActionList.add(new Action(predictedWord, predictedAttrValues.get(predictedAttrValues.size() - 1))); } } else { String predictedWord = SFX.TOKEN_END; predictedActionList.add(new Action(predictedWord, attrValue)); } } } else { predictedAttr = SFX.TOKEN_END; } } } ArrayList<String> predictedAttrs = new ArrayList<>(); for (String attributeValuePair : predictedAttrValues) { predictedAttrs.add(attributeValuePair.split("=")[0]); } ArrayList<Action> cleanActionList = new ArrayList<Action>(); for (Action action : predictedActionList) { if (!action.getWord().equals(SFX.TOKEN_END) && !action.getWord().equals(SFX.TOKEN_START)) { cleanActionList.add(action); } } for (int i = 0; i < cleanActionList.size(); i++) { for (ArrayList<Action> surrounds : punctPatterns.keySet()) { boolean matches = true; int m = 0; for (int s = 0; s < surrounds.size(); s++) { if (surrounds.get(s) != null) { if (i + s < cleanActionList.size()) { if (!cleanActionList.get(i + s).getWord().equals(surrounds.get(s) .getWord()) /*|| !cleanActionList.get(i).getAttribute().equals(surrounds.get(s).getAttribute())*/) { matches = false; s = surrounds.size(); } else { m++; } } else { matches = false; s = surrounds.size(); } } } if (matches && m > 0) { cleanActionList.add(i + 2, punctPatterns.get(surrounds)); } } } String predictedString = ""; ArrayList<String> predictedAttrList = new ArrayList<String>(); HashSet<String> redundants = new HashSet<String>(); for (Action action : cleanActionList) { if (action.getWord().startsWith(SFX.TOKEN_X)) { predictedString += di.getMeaningRepresentation().getDelexMap().get(action.getWord()) + " "; //predictedString += "x "; if (di.getMeaningRepresentation().getDelexMap().get(action.getWord()) == null || di.getMeaningRepresentation().getDelexMap().get(action.getWord()).equals("null")) { redundants.add(action.getWord()); } } else { predictedString += action.getWord() + " "; } if (predictedAttrList.isEmpty()) { predictedAttrList.add(action.getAttribute()); } else if (!predictedAttrList.get(predictedAttrList.size() - 1).equals(action.getAttribute())) { predictedAttrList.add(action.getAttribute()); } } predictedAttrLists.add(predictedAttrList); if (attrValuesToBeMentionedCopy.size() != 0.0) { double redundAttrs = 0.0; double missingAttrs = 0.0; for (String attr : predictedAttrList) { if (!attrValuesToBeMentionedCopy.contains(attr)) { redundAttrs += 1.0; } } for (String attr : attrValuesToBeMentionedCopy) { if (!predictedAttrList.contains(attr)) { missingAttrs += 1.0; } } double attrSize = (double) attrValuesToBeMentionedCopy.size(); attrCoverage.add((redundAttrs + missingAttrs) / attrSize); } if (predicate.startsWith("?")) { predictedString = predictedString.trim() + "?"; } else { predictedString = predictedString.trim() + "."; } predictedString = predictedString.replaceAll("\\?", " \\? ").replaceAll(":", " : ") .replaceAll("\\.", " \\. ").replaceAll(",", " , ").replaceAll(" ", " ").trim(); if (!mentionedAttrs.contains(di.getMeaningRepresentation().getAttributes())) { predictedStrings.add(predictedString); predictedStringMRs.add(di.getMeaningRepresentation().getMRstr()); mentionedAttrs.add(di.getMeaningRepresentation().getAttributes()); } Sequence<IString> translation = IStrings .tokenize(NISTTokenizer.tokenize(predictedString.toLowerCase())); ScoredFeaturizedTranslation<IString, String> tran = new ScoredFeaturizedTranslation<>(translation, null, 0); generations.add(tran); generationActions.add(predictedActionList); generationActionsMap.put(predictedActionList, di); ArrayList<Sequence<IString>> references = new ArrayList<>(); for (ArrayList<Action> realization : di.getEvalRealizations()) { String cleanedWords = ""; for (Action nlWord : realization) { if (!nlWord.equals(new Action(SFX.TOKEN_START, "")) && !nlWord.equals(new Action(SFX.TOKEN_END, ""))) { if (nlWord.getWord().startsWith(SFX.TOKEN_X)) { cleanedWords += di.getMeaningRepresentation().getDelexMap().get(nlWord.getWord()) + " "; } else { cleanedWords += nlWord.getWord() + " "; } } } cleanedWords = cleanedWords.trim(); if (!cleanedWords.endsWith(".")) { cleanedWords += "."; } cleanedWords = cleanedWords.replaceAll("\\?", " \\? ").replaceAll(":", " : ") .replaceAll("\\.", " \\. ").replaceAll(",", " , ").replaceAll(" ", " ").trim(); references.add(IStrings.tokenize(NISTTokenizer.tokenize(cleanedWords))); } finalReferences.add(references); //EVALUATE ATTRIBUTE SEQUENCE HashSet<ArrayList<String>> goldAttributeSequences = new HashSet<>(); for (DatasetInstance di2 : testingData) { if (di2.getMeaningRepresentation().getAttributes() .equals(di.getMeaningRepresentation().getAttributes())) { goldAttributeSequences.addAll(di2.getEvalMentionedAttributeSequences().values()); } } int minTotArgDistance = Integer.MAX_VALUE; for (ArrayList<String> goldArgs : goldAttributeSequences) { int totArgDistance = 0; HashSet<Integer> matchedPositions = new HashSet<>(); for (int i = 0; i < predictedAttrs.size(); i++) { if (!predictedAttrs.get(i).equals(SFX.TOKEN_START) && !predictedAttrs.get(i).equals(SFX.TOKEN_END)) { int minArgDistance = Integer.MAX_VALUE; int minArgPos = -1; for (int j = 0; j < goldArgs.size(); j++) { if (!matchedPositions.contains(j)) { if (goldArgs.get(j).equals(predictedAttrs.get(i))) { int argDistance = Math.abs(j - i); if (argDistance < minArgDistance) { minArgDistance = argDistance; minArgPos = j; } } } } if (minArgPos == -1) { totArgDistance += 100; } else { matchedPositions.add(minArgPos); totArgDistance += minArgDistance; } } } ArrayList<String> predictedCopy = (ArrayList<String>) predictedAttrs.clone(); for (String goldArg : goldArgs) { if (!goldArg.equals(SFX.TOKEN_END)) { boolean contained = predictedCopy.remove(goldArg); if (!contained) { totArgDistance += 1000; } } } if (totArgDistance < minTotArgDistance) { minTotArgDistance = totArgDistance; } } totalArgDistance += minTotArgDistance; } previousResults = generationActions; crossAvgArgDistances.add(totalArgDistance / (double) testingData.size()); NISTMetric NIST = new NISTMetric(finalReferences); BLEUMetric BLEU = new BLEUMetric(finalReferences, 4, false); BLEUMetric BLEUsmooth = new BLEUMetric(finalReferences, 4, true); Double nistScore = NIST.score(generations); Double bleuScore = BLEU.score(generations); Double bleuSmoothScore = BLEUsmooth.score(generations); double finalCoverage = 0.0; for (double c : attrCoverage) { finalCoverage += c; } finalCoverage /= (double) attrCoverage.size(); crossNIST.add(nistScore); crossBLEU.add(bleuScore); crossBLEUSmooth.add(bleuSmoothScore); System.out.println("Avg arg distance: \t" + totalArgDistance / (double) testingData.size()); System.out.println("NIST: \t" + nistScore); System.out.println("BLEU: \t" + bleuScore); System.out.println("COVERAGE: \t" + finalCoverage); System.out.println("g: " + generations); System.out.println("attr: " + predictedAttrLists); System.out.println("BLEU smooth: \t" + bleuSmoothScore); previousBLEU = bleuScore; if (printResults) { BufferedWriter bw = null; File f = null; try { f = new File("random_SFX" + dataset + "TextsAfter" + (epoch) + "_" + JDAggerForSFX.earlyStopMaxFurtherSteps + "_" + JDAggerForSFX.p + "epochsTESTINGDATA.txt"); } catch (NullPointerException e) { System.err.println("File not found." + e); } try { bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f))); } catch (FileNotFoundException e) { System.err.println("Error opening file for writing! " + e); } try { bw.write("BLEU:" + bleuScore); bw.write("\n"); } catch (IOException e) { System.err.println("Write error!"); } for (int i = 0; i < predictedStrings.size(); i++) { try { //Grafoume to String sto arxeio //SFX HOTEL TEXTS WITH LOLS -> 3 //SFX RESTAURANT TEXTS WITH LOLS -> 5 bw.write("MR;" + predictedStringMRs.get(i).replaceAll(";", ",") + ";"); if (dataset.equals("hotel")) { bw.write("LOLS_SFHOT;"); } else { bw.write("LOLS_SFRES;"); } //bw.write("@@srcdoc@@" + (i + 1)); /*String out = predictedStrings.get(i).replaceAll(" i ", " I ").replaceAll(" -ly ", "ly ").replaceAll(" s ", "s ").replaceAll("\\?", " \\? ").replaceAll(":", " : ").replaceAll("\\.", " \\. ").replaceAll(",", " , ").replaceAll(" ", " "); out = out.substring(0, 1).toUpperCase() + out.substring(1); bw.write(out + ";"); if (dataset.equals("hotel")) { bw.write("WEN_SFHOT;"); } else { bw.write("WEN_SFRES;"); } if (!wenDaToGen.containsKey(predictedStringMRs.get(i).trim().toLowerCase())) { System.out.println(wenDaToGen.keySet()); System.out.println(predictedStringMRs.get(i).trim().toLowerCase()); System.exit(0); } out = wenDaToGen.get(predictedStringMRs.get(i).trim().toLowerCase()).replaceAll(" i ", " I ").replaceAll(" -ly ", "ly ").replaceAll(" s ", "s ").replaceAll("\\?", " \\? ").replaceAll(":", " : ").replaceAll("\\.", " \\. ").replaceAll(",", " , ").replaceAll(" ", " "); out = out.substring(0, 1).toUpperCase() + out.substring(1); bw.write(out + ";");*/ //bw.write("@@judgeFluency@@-1"); //bw.write("@@judgeInform@@-1"); //bw.write("@@judgeQuality@@-1"); bw.write("\n"); } catch (IOException e) { System.err.println("Write error!"); } } try { bw.close(); } catch (IOException e) { System.err.println("Error closing file."); } catch (Exception e) { } } return bleuScore; }
From source file:imitationNLG.SFX.java
public Object[] createTrainingDatasets(ArrayList<DatasetInstance> trainingData, HashMap<String, HashSet<String>> availableAttributeActions, HashMap<String, HashMap<String, HashSet<Action>>> availableWordActions, HashMap<Integer, HashSet<String>> nGrams) { HashMap<String, ArrayList<Instance>> predicateAttrTrainingData = new HashMap<>(); HashMap<String, HashMap<String, ArrayList<Instance>>> predicateWordTrainingData = new HashMap<>(); if (!availableWordActions.isEmpty() && !predicates.isEmpty()/* && !arguments.isEmpty()*/) { for (String predicate : predicates) { predicateAttrTrainingData.put(predicate, new ArrayList<Instance>()); predicateWordTrainingData.put(predicate, new HashMap<String, ArrayList<Instance>>()); }/* ww w . j av a 2 s . c om*/ for (DatasetInstance di : trainingData) { //System.out.println("BEGIN"); String predicate = di.getMeaningRepresentation().getPredicate(); //for (ArrayList<Action> realization : di.getEvalRealizations()) { ArrayList<Action> realization = di.getTrainRealization(); //System.out.println(realization); HashSet<String> attrValuesAlreadyMentioned = new HashSet<>(); HashSet<String> attrValuesToBeMentioned = new HashSet<>(); for (String attribute : di.getMeaningRepresentation().getAttributes().keySet()) { //int a = 0; for (String value : di.getMeaningRepresentation().getAttributes().get(attribute)) { /*if (value.startsWith("\"x")) { value = "x" + a; a++; } else if (value.startsWith("\"")) { value = value.substring(1, value.length() - 1).replaceAll(" ", "_"); }*/ attrValuesToBeMentioned.add(attribute.toLowerCase() + "=" + value.toLowerCase()); } } if (attrValuesToBeMentioned.isEmpty()) { attrValuesToBeMentioned.add("empty=empty"); } ArrayList<String> attrs = new ArrayList<>(); boolean isValueMentioned = false; String valueTBM = ""; String attrValue = ""; ArrayList<String> subPhrase = new ArrayList<>(); for (int w = 0; w < realization.size(); w++) { if (!realization.get(w).getAttribute().equals(SFX.TOKEN_PUNCT)) { if (!realization.get(w).getAttribute().equals(attrValue)) { if (!attrValue.isEmpty()) { attrValuesToBeMentioned.remove(attrValue); } Instance attrTrainingVector = SFX.this.createAttrInstance(predicate, realization.get(w).getAttribute(), attrs, new ArrayList<Action>(realization.subList(0, w)), attrValuesAlreadyMentioned, attrValuesToBeMentioned, di.getMeaningRepresentation(), availableAttributeActions); if (attrTrainingVector != null) { /*System.out.println(realization.get(w).getAttribute() + " >>>> " + attrTrainingVector.getCorrectLabels()); for (String f : attrTrainingVector.getGeneralFeatureVector().keySet()) { if (f.startsWith("feature_attrValue_5gram_") || f.startsWith("feature_word_5gram_")) { System.out.println(">> " + f); } }*/ predicateAttrTrainingData.get(predicate).add(attrTrainingVector); } attrs.add(realization.get(w).getAttribute()); attrValue = realization.get(w).getAttribute(); subPhrase = new ArrayList<>(); isValueMentioned = false; valueTBM = ""; if (attrValue.contains("=")) { valueTBM = attrValue.substring(attrValue.indexOf('=') + 1); } if (valueTBM.isEmpty()) { isValueMentioned = true; } } if (!attrValue.equals(SFX.TOKEN_END)) { ArrayList<String> predictedAttributesForInstance = new ArrayList<>(); for (int i = 0; i < attrs.size() - 1; i++) { predictedAttributesForInstance.add(attrs.get(i)); } if (!attrs.get(attrs.size() - 1).equals(attrValue)) { predictedAttributesForInstance.add(attrs.get(attrs.size() - 1)); } Instance wordTrainingVector = createWordInstance(predicate, realization.get(w), predictedAttributesForInstance, new ArrayList<Action>(realization.subList(0, w)), isValueMentioned, attrValuesAlreadyMentioned, attrValuesToBeMentioned, di.getMeaningRepresentation(), availableWordActions.get(predicate), nGrams, false); if (wordTrainingVector != null) { String attribute = attrValue; if (attribute.contains("=")) { attribute = attrValue.substring(0, attrValue.indexOf('=')); } if (!predicateWordTrainingData.get(predicate).containsKey(attribute)) { predicateWordTrainingData.get(predicate).put(attribute, new ArrayList<Instance>()); } /*System.out.println(realization.get(w) + " >>>> " + wordTrainingVector.getCorrectLabels()); for (String f : wordTrainingVector.getGeneralFeatureVector().keySet()) { if (f.startsWith("feature_attrValue_5gram_") || f.startsWith("feature_word_5gram_")) { System.out.println(">> " + f); } }*/ predicateWordTrainingData.get(predicate).get(attribute).add(wordTrainingVector); if (!realization.get(w).getWord().equals(SFX.TOKEN_START) && !realization.get(w).getWord().equals(SFX.TOKEN_END)) { subPhrase.add(realization.get(w).getWord()); } } if (!isValueMentioned) { if (realization.get(w).getWord().startsWith(SFX.TOKEN_X) && (valueTBM.matches("[xX][0-9]+") || valueTBM.matches("\"[xX][0-9]+\"") || valueTBM.startsWith(SFX.TOKEN_X))) { isValueMentioned = true; } else if (!realization.get(w).getWord().startsWith(SFX.TOKEN_X) && !(valueTBM.matches("[xX][0-9]+") || valueTBM.matches("\"[xX][0-9]+\"") || valueTBM.startsWith(SFX.TOKEN_X))) { String valueToCheck = valueTBM; if (valueToCheck.equals("no") || valueToCheck.equals("yes") || valueToCheck.equals("yes or no") || valueToCheck.equals("none") || valueToCheck.equals("dont_care") || valueToCheck.equals("empty")) { String attribute = attrValue; if (attribute.contains("=")) { attribute = attrValue.substring(0, attrValue.indexOf('=')); } valueToCheck = attribute + ":" + valueTBM; } if (!valueToCheck.equals("empty:empty") && valueAlignments.containsKey(valueToCheck)) { for (ArrayList<String> alignedStr : valueAlignments.get(valueToCheck) .keySet()) { if (endsWith(subPhrase, alignedStr)) { isValueMentioned = true; break; } } } } if (isValueMentioned) { attrValuesAlreadyMentioned.add(attrValue); attrValuesToBeMentioned.remove(attrValue); } } String mentionedAttrValue = ""; if (!realization.get(w).getWord().startsWith(SFX.TOKEN_X)) { for (String attrValueTBM : attrValuesToBeMentioned) { if (attrValueTBM.contains("=")) { String value = attrValueTBM.substring(attrValueTBM.indexOf('=') + 1); if (!(value.matches("\"[xX][0-9]+\"") || value.matches("[xX][0-9]+") || value.startsWith(SFX.TOKEN_X))) { String valueToCheck = value; if (valueToCheck.equals("no") || valueToCheck.equals("yes") || valueToCheck.equals("yes or no") || valueToCheck.equals("none") || valueToCheck.equals("dont_care") || valueToCheck.equals("empty")) { valueToCheck = attrValueTBM.replace("=", ":"); } if (!valueToCheck.equals("empty:empty") && valueAlignments.containsKey(valueToCheck)) { for (ArrayList<String> alignedStr : valueAlignments .get(valueToCheck).keySet()) { if (endsWith(subPhrase, alignedStr)) { mentionedAttrValue = attrValueTBM; break; } } } } } } } if (!mentionedAttrValue.isEmpty()) { attrValuesAlreadyMentioned.add(mentionedAttrValue); attrValuesToBeMentioned.remove(mentionedAttrValue); } } } } //} } } Object[] results = new Object[2]; results[0] = predicateAttrTrainingData; results[1] = predicateWordTrainingData; return results; }
From source file:org.mindswap.swoop.renderer.entity.ConciseFormatEntityRenderer.java
/** * //from ww w . j a v a 2 s . co m * Returns a list of changes that are required to fufill the requirements of clicking a * delete link. A delete operation may involve various steps of changes and may also * cause some other changes to happen. The list will contain all the atomic changes * required for this operation * * @param hLink the link clicked on the editor pane * @return A list of OntologyChange objects * @throws OWLException */ public List handleDeleteLink(String hLink) throws OWLException { List changes = new ArrayList(); OWLOntology ontology = reasoner.getOntology(); // parse DELETE hyper-link int pos1 = hLink.indexOf(":"); int pos2 = hLink.indexOf(":", pos1 + 1); String hashCode = hLink.substring(pos1 + 1, pos2); String titleCode = hLink.substring(pos2 + 1, hLink.length()); Object obj = OWLDescHash.get(hashCode); if (titleCode.equals("P-FUN")) { // Remove Functional Attribute // *** Note: Change is effective immediately for property attribute SetFunctional change = new SetFunctional(ontology, (OWLProperty) obj, false, null); swoopModel.addUncommittedChange(change); return new ArrayList(); } if (titleCode.equals("P-IFUN")) { // Remove InverseFunctional Attribute // *** Note: Change is effective immediately for property attribute SetInverseFunctional change = new SetInverseFunctional(ontology, (OWLObjectProperty) obj, false, null); swoopModel.addUncommittedChange(change); return new ArrayList(); } if (titleCode.equals("P-TRA")) { // Remove Transitive Attribute // *** Note: Change is effective immediately for property attribute SetTransitive change = new SetTransitive(ontology, (OWLObjectProperty) obj, false, null); swoopModel.addUncommittedChange(change); return new ArrayList(); } if (titleCode.equals("P-SYM")) { // Remove Symmetric Attribute // *** Note: Change is effective immediately for property attribute SetSymmetric change = new SetSymmetric(ontology, (OWLObjectProperty) obj, false, null); swoopModel.addUncommittedChange(change); return new ArrayList(); } if (titleCode.equals("A-ANN")) { // Remove Annotation Instance OWLEntity currEntity = swoopModel.getSelectedEntity(); if (obj instanceof OWLAnnotationInstance) { OWLAnnotationInstance oai = (OWLAnnotationInstance) obj; RemoveAnnotationInstance change = new RemoveAnnotationInstance(ontology, currEntity, oai.getProperty(), oai.getContent(), null); swoopModel.addUncommittedChange(change); } return new ArrayList(); } if (titleCode.equals("I-SAM") || titleCode.equals("I-DIF")) { // delete sameAs axiom Set indSet = new HashSet(); indSet.add((OWLIndividual) displayedEntity); indSet.add((OWLIndividual) obj); OWLIndividualAxiom indAxiom = null; if (titleCode.equals("I-SAM")) indAxiom = ontology.getOWLDataFactory().getOWLSameIndividualsAxiom(indSet); else indAxiom = ontology.getOWLDataFactory().getOWLDifferentIndividualsAxiom(indSet); RemoveIndividualAxiom change = new RemoveIndividualAxiom(ontology, indAxiom, null); changes.add(change); } if (titleCode.equals("C-EQU")) { // delete equivalent class OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; RemoveEquivalentClass change = new RemoveEquivalentClass(ontology, displayedClass, desc, null); changes.add(change); } else { Iterator descIter = ((Collection) obj).iterator(); while (descIter.hasNext()) { OWLDescription desc = (OWLDescription) descIter.next(); RemoveEquivalentClass change = new RemoveEquivalentClass(ontology, displayedClass, desc, null); changes.add(change); } } } else if (titleCode.equals("C-DIS")) { // delete disjoint class OWLClass displayedClass = (OWLClass) displayedEntity; Set disSet = new HashSet(); if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; disSet.add(desc); disSet.add(displayedClass); } else { disSet.add(displayedClass); disSet.addAll((Collection) obj); } OWLDisjointClassesAxiom disAxiom = ontology.getOWLDataFactory().getOWLDisjointClassesAxiom(disSet); RemoveClassAxiom change = new RemoveClassAxiom(ontology, disAxiom, null); changes.add(change); } else if (titleCode.equals("C-SUB")) { // delete super-class OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; if (displayedClass.getSuperClasses(ontology).contains(obj)) { // add RemoveSuperClass change in this case RemoveSuperClass change = new RemoveSuperClass(ontology, displayedClass, desc, null); changes.add(change); } else { // remove specific axiom in the other case OWLSubClassAxiom axiom = ontology.getOWLDataFactory().getOWLSubClassAxiom(displayedClass, desc); RemoveClassAxiom change2 = new RemoveClassAxiom(ontology, axiom, null); changes.add(change2); } } else { Iterator descIter = ((Collection) obj).iterator(); while (descIter.hasNext()) { OWLDescription desc = (OWLDescription) descIter.next(); if (displayedClass.getSuperClasses(ontology).contains(obj)) { RemoveSuperClass change = new RemoveSuperClass(ontology, displayedClass, desc, null); changes.add(change); } else { // remove specific axiom if present OWLSubClassAxiom axiom = ontology.getOWLDataFactory().getOWLSubClassAxiom(displayedClass, desc); RemoveClassAxiom change2 = new RemoveClassAxiom(ontology, axiom, null); changes.add(change2); } } } } else if (titleCode.equals("C-SUP")) { // delete super-class OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLClass) { // super classes can be defined in two ways // check to see what kind of change needs to be put OWLClass desc = (OWLClass) obj; if (desc.getSuperClasses(ontology).contains(displayedClass)) { RemoveSuperClass change = new RemoveSuperClass(ontology, desc, displayedClass, null); changes.add(change); } else { // remove specific axiom if present OWLSubClassAxiom axiom = ontology.getOWLDataFactory().getOWLSubClassAxiom(desc, displayedClass); RemoveClassAxiom change2 = new RemoveClassAxiom(ontology, axiom, null); changes.add(change2); } } else { OWLDescription desc = (OWLDescription) obj; // remove specific axiom if present OWLSubClassAxiom axiom = ontology.getOWLDataFactory().getOWLSubClassAxiom(desc, displayedClass); RemoveClassAxiom change2 = new RemoveClassAxiom(ontology, axiom, null); changes.add(change2); } } else if (titleCode.equals("C-INT")) { // delete intersection element // remove whole intersection and add remaining elements OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; deleteFromBooleanDesc(ontology, displayedClass, desc, OWLAnd.class, changes); } else { Iterator descIter = ((Collection) obj).iterator(); while (descIter.hasNext()) { OWLClass desc = (OWLClass) descIter.next(); deleteFromBooleanDesc(ontology, displayedClass, desc, OWLAnd.class, changes); } } } else if (titleCode.equals("C-UNI")) { // delete union element // remove whole union and add remaining elements OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; deleteFromBooleanDesc(ontology, displayedClass, desc, OWLOr.class, changes); } else { Iterator descIter = ((Collection) obj).iterator(); while (descIter.hasNext()) { OWLClass desc = (OWLClass) descIter.next(); deleteFromBooleanDesc(ontology, displayedClass, desc, OWLOr.class, changes); } } } else if (titleCode.equals("C-NOT")) { // delete complement element OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLDescription) { BooleanElementChange change = new BooleanElementChange(OWLNot.class, "Remove", ontology, displayedClass, (OWLDescription) obj, null); changes.add(change); } } else if (titleCode.equals("I-ONE")) { // delete one-of element OWLClass displayedClass = (OWLClass) displayedEntity; if (obj instanceof OWLIndividual) { OWLIndividual desc = (OWLIndividual) obj; updateEnumerations(ontology, displayedClass, desc, changes); } } else if (titleCode.equals("I-INS")) { // delete instance RemoveIndividualClass change = new RemoveIndividualClass(ontology, (OWLIndividual) obj, (OWLDescription) displayedEntity, null); changes.add(change); } else if (titleCode.equals("C-TYP")) { // delete type RemoveIndividualClass change = new RemoveIndividualClass(ontology, (OWLIndividual) displayedEntity, (OWLDescription) obj, null); changes.add(change); } else if (titleCode.equals("C-HASDOM")) { // delete property domain OWLProperty prop = (OWLProperty) displayedEntity; if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; RemoveDomain change = new RemoveDomain(ontology, prop, desc, null); changes.add(change); } } else if (titleCode.equals("P-DOM")) { // delete property domain OWLClass cla = (OWLClass) displayedEntity; if (obj instanceof OWLProperty) { RemoveDomain change = new RemoveDomain(ontology, (OWLProperty) obj, cla, null); changes.add(change); } } else if (titleCode.equals("C-HASRAN")) { // delete property range // check if datatype or object property if (displayedEntity instanceof OWLObjectProperty) { OWLObjectProperty prop = (OWLObjectProperty) displayedEntity; if (obj instanceof OWLDescription) { OWLDescription desc = (OWLDescription) obj; RemoveObjectPropertyRange change = new RemoveObjectPropertyRange(ontology, prop, desc, null); changes.add(change); } } else { OWLDataProperty prop = (OWLDataProperty) displayedEntity; if (obj instanceof OWLDataRange) { OWLDataRange dran = (OWLDataRange) obj; RemoveDataPropertyRange change = new RemoveDataPropertyRange(ontology, prop, dran, null); changes.add(change); } } } else if (titleCode.equals("P-RAN")) { // delete property range OWLClass cla = (OWLClass) displayedEntity; if (obj instanceof OWLObjectProperty) { RemoveObjectPropertyRange change = new RemoveObjectPropertyRange(ontology, (OWLObjectProperty) obj, cla, null); changes.add(change); } } else if (titleCode.equals("P-SUB")) { // remove super property if (obj instanceof OWLProperty) { if (((OWLProperty) displayedEntity).getSuperProperties(ontology).contains(obj)) { RemoveSuperProperty change = new RemoveSuperProperty(ontology, (OWLProperty) displayedEntity, (OWLProperty) obj, null); changes.add(change); } else { // remove specific axiom if present OWLSubPropertyAxiom axiom = ontology.getOWLDataFactory() .getOWLSubPropertyAxiom((OWLProperty) displayedEntity, (OWLProperty) obj); RemovePropertyAxiom change2 = new RemovePropertyAxiom(ontology, axiom, null); changes.add(change2); } } } else if (titleCode.equals("P-SUP")) { // remove sub property if (obj instanceof OWLProperty) { if (((OWLProperty) obj).getSuperProperties(ontology).contains(displayedEntity)) { RemoveSuperProperty change = new RemoveSuperProperty(ontology, (OWLProperty) obj, (OWLProperty) displayedEntity, null); changes.add(change); } else { // remove specific axiom if present OWLSubPropertyAxiom axiom = ontology.getOWLDataFactory() .getOWLSubPropertyAxiom((OWLProperty) obj, (OWLProperty) displayedEntity); RemovePropertyAxiom change2 = new RemovePropertyAxiom(ontology, axiom, null); changes.add(change2); } } } else if (titleCode.equals("P-EQU")) { // remove equivalent property if (obj instanceof OWLProperty) { Set propSet = new HashSet(); propSet.add((OWLProperty) obj); propSet.add((OWLProperty) displayedEntity); OWLEquivalentPropertiesAxiom axiom = ontology.getOWLDataFactory() .getOWLEquivalentPropertiesAxiom(propSet); RemovePropertyAxiom change = new RemovePropertyAxiom(ontology, axiom, null); changes.add(change); } } else if (titleCode.equals("P-INV")) { // remove inverse property if (obj instanceof OWLObjectProperty) { OWLObjectProperty prop = (OWLObjectProperty) displayedEntity; OWLObjectProperty inverse = (OWLObjectProperty) obj; RemoveInverse change = new RemoveInverse(ontology, prop, inverse, null); changes.add(change); } } else if (titleCode.startsWith(("P-VAL"))) { // remove property value pair // also obtain value from hash table String valueHashKey = titleCode.substring(titleCode.lastIndexOf(":") + 1, titleCode.length()); Object value = OWLDescHash.get(valueHashKey); if (obj instanceof OWLObjectProperty) { RemoveObjectPropertyInstance change = new RemoveObjectPropertyInstance(ontology, (OWLIndividual) displayedEntity, (OWLObjectProperty) obj, (OWLIndividual) value, null); changes.add(change); } else if (obj instanceof OWLDataProperty) { RemoveDataPropertyInstance change = new RemoveDataPropertyInstance(ontology, (OWLIndividual) displayedEntity, (OWLDataProperty) obj, (OWLDataValue) value, null); changes.add(change); } } else if (titleCode.startsWith("RULE")) { System.out.println("Deleting Rule"); //String n3 = swoopModel.getRuleExpr().publishRulesToPychinko(); // obj is OWLRule // go through the rulemap and remove this rule OWLRule rule = (OWLRule) obj; //get the rule and its friggin expressivity (exactly why are they bundled together?) OWLRuleAtom consAtom = (OWLRuleAtom) rule.getConsequents().iterator().next(); OWLObject key = null; if (consAtom instanceof OWLRuleClassAtom) { key = ((OWLRuleClassAtom) consAtom).getDescription(); } else { if (consAtom instanceof OWLRuleDataPropertyAtom) { key = ((OWLRuleDataPropertyAtom) consAtom).getProperty(); } else { if (consAtom instanceof OWLRuleObjectPropertyAtom) { key = ((OWLRuleObjectPropertyAtom) consAtom).getProperty(); } } } HashSet rulesSet = (HashSet) swoopModel.getRuleExpr().getRuleMap().get(key); //find the rule we want to delete RuleValue rvDelete = null; Iterator it = rulesSet.iterator(); while (it.hasNext()) { RuleValue rv = (RuleValue) it.next(); if (rv.getRule().equals(obj)) { rvDelete = rv; } } rulesSet.remove(rvDelete); swoopModel.getRuleExpr().getRuleMap().put(key, rulesSet); } return changes; }