List of usage examples for java.util LinkedHashMap entrySet
public Set<Map.Entry<K, V>> entrySet()
From source file:org.jumpmind.db.platform.AbstractDdlBuilder.java
/** * Processes the changes to the structure of tables. * * @param currentModel/*from w w w . jav a2 s.c o m*/ * The current database schema * @param desiredModel * The desired database schema * @param changes * The change objects */ protected void processTableStructureChanges(Database currentModel, Database desiredModel, Collection<TableChange> changes, StringBuilder ddl) { filterChanges(changes); LinkedHashMap<String, List<TableChange>> changesPerTable = new LinkedHashMap<String, List<TableChange>>(); LinkedHashMap<String, List<ForeignKey>> unchangedFKs = new LinkedHashMap<String, List<ForeignKey>>(); boolean caseSensitive = delimitedIdentifierModeOn; // we first sort the changes for the tables // however since the changes might contain source or target tables // we use the names rather than the table objects for (Iterator<TableChange> changeIt = changes.iterator(); changeIt.hasNext();) { TableChange change = changeIt.next(); String name = change.getChangedTable().getName(); if (!caseSensitive) { name = name.toUpperCase(); } List<TableChange> changesForTable = (List<TableChange>) changesPerTable.get(name); if (changesForTable == null) { changesForTable = new ArrayList<TableChange>(); changesPerTable.put(name, changesForTable); unchangedFKs.put(name, getUnchangedForeignKeys(currentModel, desiredModel, name)); } changesForTable.add(change); } // we also need to drop the foreign keys of the unchanged tables // referencing the changed tables addRelevantFKsFromUnchangedTables(currentModel, desiredModel, changesPerTable.keySet(), unchangedFKs); // we're dropping the unchanged foreign keys for (Iterator<Map.Entry<String, List<ForeignKey>>> tableFKIt = unchangedFKs.entrySet().iterator(); tableFKIt .hasNext();) { Map.Entry<String, List<ForeignKey>> entry = tableFKIt.next(); Table targetTable = desiredModel.findTable((String) entry.getKey(), caseSensitive); for (Iterator<ForeignKey> fkIt = entry.getValue().iterator(); fkIt.hasNext();) { writeExternalForeignKeyDropStmt(targetTable, fkIt.next(), ddl); } } // We're using a copy of the current model so that the table structure // changes can modify it Database copyOfCurrentModel = copy(currentModel); for (Iterator<Map.Entry<String, List<TableChange>>> tableChangeIt = changesPerTable.entrySet() .iterator(); tableChangeIt.hasNext();) { Map.Entry<String, List<TableChange>> entry = tableChangeIt.next(); processTableStructureChanges(copyOfCurrentModel, desiredModel, entry.getKey(), entry.getValue(), ddl); } // and finally we're re-creating the unchanged foreign keys for (Iterator<Map.Entry<String, List<ForeignKey>>> tableFKIt = unchangedFKs.entrySet().iterator(); tableFKIt .hasNext();) { Map.Entry<String, List<ForeignKey>> entry = tableFKIt.next(); Table targetTable = desiredModel.findTable((String) entry.getKey(), caseSensitive); for (Iterator<ForeignKey> fkIt = entry.getValue().iterator(); fkIt.hasNext();) { writeExternalForeignKeyCreateStmt(desiredModel, targetTable, fkIt.next(), ddl); } } }
From source file:lucee.runtime.config.ConfigWebFactory.java
/** * @param configServer/*from ww w . j a v a 2 s. co m*/ * @param config * @param doc * @throws IOException */ private static void loadCustomTagsMappings(ConfigServerImpl configServer, ConfigImpl config, Document doc, int mode) { boolean hasAccess = ConfigWebUtil.hasAccess(config, SecurityManager.TYPE_CUSTOM_TAG); boolean hasCS = configServer != null; Element customTag = getChildByName(doc.getDocumentElement(), "custom-tag"); Element[] ctMappings = getChildren(customTag, "mapping"); // String virtualx="/custom-tag/"; // do patch cache String strDoPathcache = customTag.getAttribute("use-cache-path"); if (hasAccess && !StringUtil.isEmpty(strDoPathcache, true)) { config.setUseCTPathCache(Caster.toBooleanValue(strDoPathcache.trim(), true)); } else if (hasCS) { config.setUseCTPathCache(configServer.useCTPathCache()); } // do custom tag local search if (mode == ConfigImpl.MODE_STRICT) { config.setDoLocalCustomTag(false); } else { String strDoCTLocalSearch = customTag.getAttribute("custom-tag-local-search"); if (hasAccess && !StringUtil.isEmpty(strDoCTLocalSearch)) { config.setDoLocalCustomTag(Caster.toBooleanValue(strDoCTLocalSearch.trim(), true)); } else if (hasCS) { config.setDoLocalCustomTag(configServer.doLocalCustomTag()); } } // do custom tag deep search if (mode == ConfigImpl.MODE_STRICT) { config.setDoCustomTagDeepSearch(false); } else { String strDoCTDeepSearch = customTag.getAttribute("custom-tag-deep-search"); if (hasAccess && !StringUtil.isEmpty(strDoCTDeepSearch)) { config.setDoCustomTagDeepSearch(Caster.toBooleanValue(strDoCTDeepSearch.trim(), false)); } else if (hasCS) { config.setDoCustomTagDeepSearch(configServer.doCustomTagDeepSearch()); } } // extensions if (mode == ConfigImpl.MODE_STRICT) { config.setCustomTagExtensions(new String[] { "cfc" }); } else { String strExtensions = customTag.getAttribute("extensions"); if (hasAccess && !StringUtil.isEmpty(strExtensions)) { try { String[] arr = ListUtil.toStringArray(ListUtil.listToArrayRemoveEmpty(strExtensions, ",")); config.setCustomTagExtensions(ListUtil.trimItems(arr)); } catch (PageException e) { } } else if (hasCS) { config.setCustomTagExtensions(configServer.getCustomTagExtensions()); } } // Web Mapping boolean hasSet = false; Mapping[] mappings = null; if (hasAccess && ctMappings.length > 0) { mappings = new Mapping[ctMappings.length]; for (int i = 0; i < ctMappings.length; i++) { Element ctMapping = ctMappings[i]; String physical = ctMapping.getAttribute("physical"); String archive = ctMapping.getAttribute("archive"); boolean readonly = toBoolean(ctMapping.getAttribute("readonly"), false); boolean hidden = toBoolean(ctMapping.getAttribute("hidden"), false); //boolean trusted = toBoolean(ctMapping.getAttribute("trusted"), false); short inspTemp = inspectTemplate(ctMapping); int clMaxEl = toInt(ctMapping.getAttribute("classloader-max-elements"), 100); String primary = ctMapping.getAttribute("primary"); boolean physicalFirst = archive == null || !primary.equalsIgnoreCase("archive"); hasSet = true; mappings[i] = new MappingImpl(config, ConfigWebAdmin.createVirtual(ctMapping), physical, archive, inspTemp, physicalFirst, hidden, readonly, true, false, true, null, clMaxEl); // print.out(mappings[i].isPhysicalFirst()); } config.setCustomTagMappings(mappings); } // Server Mapping if (hasCS) { Mapping[] originals = configServer.getCustomTagMappings(); Mapping[] clones = new Mapping[originals.length]; LinkedHashMap map = new LinkedHashMap(); Mapping m; for (int i = 0; i < clones.length; i++) { m = ((MappingImpl) originals[i]).cloneReadOnly(config); map.put(toKey(m), m); // clones[i]=((MappingImpl)m[i]).cloneReadOnly(config); } if (mappings != null) { for (int i = 0; i < mappings.length; i++) { m = mappings[i]; map.put(toKey(m), m); } } if (originals.length > 0) { clones = new Mapping[map.size()]; Iterator it = map.entrySet().iterator(); Map.Entry entry; int index = 0; while (it.hasNext()) { entry = (Entry) it.next(); clones[index++] = (Mapping) entry.getValue(); // print.out("c:"+clones[index-1]); } hasSet = true; // print.err("set:"+clones.length); config.setCustomTagMappings(clones); } } if (!hasSet) { // MappingImpl m=new // MappingImpl(config,"/default-customtags/","{lucee-web}/customtags/",null,false,true,false,false,true,false,true); // config.setCustomTagMappings(new // Mapping[]{m.cloneReadOnly(config)}); } }
From source file:com.mothsoft.alexis.dao.DocumentDaoImpl.java
@SuppressWarnings("unchecked") private List<ImportantTerm> getImportantTerms(FullTextQuery fullTextQuery, int count, boolean filterStopWords) { final Long start = System.currentTimeMillis(); final List<Object[]> results = fullTextQuery.list(); final LinkedHashMap<String, Tuple<Integer, Float>> termCountMap = new LinkedHashMap<String, Tuple<Integer, Float>>(); final FullTextSession fullTextSession = Search.getFullTextSession((Session) this.em.getDelegate()); final SearchFactory searchFactory = fullTextSession.getSearchFactory(); final IndexReaderAccessor ira = searchFactory.getIndexReaderAccessor(); final IndexReader reader = ira.open(com.mothsoft.alexis.domain.Document.class); final IndexSearcher searcher = new IndexSearcher(reader); final List<ImportantTerm> importantTerms; final int numDocs; try {//from ww w.j a v a2s. c o m numDocs = reader.numDocs(); Term luceneTerm = new Term(CONTENT_TEXT_FIELD_NAME); if (logger.isDebugEnabled()) { logger.debug(String.format("Found %d matching Lucene documents of %d in reader", results.size(), numDocs)); } // loop over all the matching documents for (final Object[] ith : results) { int docId = ((Number) ith[0]).intValue(); final TermFreqVector tfv = reader.getTermFreqVector(docId, CONTENT_TEXT_FIELD_NAME); if (tfv == null) { continue; } final String[] terms = tfv.getTerms(); final int[] freqs = tfv.getTermFrequencies(); // total document size int size = 0; for (int freq : freqs) { size += freq; } if (logger.isDebugEnabled()) { logger.debug( String.format("Lucene document %d has %d terms, to be merged with running count %d", docId, size, termCountMap.size())); } // loop over the terms and aggregate the counts and tf-idf int i = 0; for (final String term : terms) { if (StopWords.ENGLISH.contains(term)) { continue; } luceneTerm = luceneTerm.createTerm(term); final int termCount = freqs[i++]; final Tuple<Integer, Float> countScore; if (termCountMap.containsKey(term)) { countScore = termCountMap.get(term); countScore.t1 += termCount; countScore.t2 += (TFIDF.score(term, termCount, size, numDocs, searcher.docFreq(luceneTerm))); } else { countScore = new Tuple<Integer, Float>(); countScore.t1 = termCount; countScore.t2 = (TFIDF.score(term, termCount, size, numDocs, searcher.docFreq(luceneTerm))); termCountMap.put(term, countScore); } } } if (logger.isDebugEnabled()) { logger.debug("Completed Lucene document processing."); } importantTerms = new ArrayList<ImportantTerm>(termCountMap.size()); // find max TF-IDF float maxTfIdf = 0.0f; for (final Tuple<Integer, Float> ith : termCountMap.values()) { if (ith.t2 > maxTfIdf) { maxTfIdf = ith.t2; } } for (final Map.Entry<String, Tuple<Integer, Float>> entry : termCountMap.entrySet()) { final int ithCount = entry.getValue().t1; final float ithTfIdf = entry.getValue().t2; importantTerms.add(new ImportantTerm(entry.getKey(), ithCount, ithTfIdf, maxTfIdf)); } if (logger.isDebugEnabled()) { logger.debug("Completed term aggregation, will clear term map"); } termCountMap.clear(); } catch (IOException e) { throw new RuntimeException(e); } finally { try { searcher.close(); } catch (IOException e) { logger.warn("Failed to close searcher: " + e, e); } ira.close(reader); } if (logger.isDebugEnabled()) { logger.debug("Sorting terms"); } Collections.sort(importantTerms, new Comparator<ImportantTerm>() { @Override public int compare(ImportantTerm term1, ImportantTerm term2) { return -1 * term1.getTfIdf().compareTo(term2.getTfIdf()); } }); if (logger.isDebugEnabled()) { logger.debug("Term sort complete"); } if (importantTerms.isEmpty() || importantTerms.size() < count) { if (logger.isDebugEnabled()) { logger.debug("Will return full list."); } logger.debug("Timer: " + (System.currentTimeMillis() - start)); return importantTerms; } else { if (logger.isDebugEnabled()) { logger.debug( "Will return sublist containing " + count + " of " + importantTerms.size() + " terms."); } logger.debug("Timer: " + (System.currentTimeMillis() - start)); return importantTerms.subList(0, count); } }
From source file:lucee.runtime.config.ConfigWebFactory.java
/** * @param configServer//from w w w .ja va 2 s. c o m * @param config * @param doc * @throws IOException */ private static void loadComponent(ConfigServer configServer, ConfigImpl config, Document doc, int mode) { Element component = getChildByName(doc.getDocumentElement(), "component"); boolean hasAccess = ConfigWebUtil.hasAccess(config, SecurityManager.TYPE_SETTING); boolean hasSet = false; boolean hasCS = configServer != null; // String virtual="/component/"; if (component != null && hasAccess) { // component-default-import String strCDI = component.getAttribute("component-default-import"); if (StringUtil.isEmpty(strCDI, true) && configServer != null) { strCDI = ((ConfigServerImpl) configServer).getComponentDefaultImport().toString(); } if (!StringUtil.isEmpty(strCDI, true)) config.setComponentDefaultImport(strCDI); // Base String strBase = component.getAttribute("base"); if (StringUtil.isEmpty(strBase, true) && configServer != null) { strBase = configServer.getBaseComponentTemplate(); } config.setBaseComponentTemplate(strBase); // deep search if (mode == ConfigImpl.MODE_STRICT) { config.setDoComponentDeepSearch(false); } else { String strDeepSearch = component.getAttribute("deep-search"); if (!StringUtil.isEmpty(strDeepSearch)) { config.setDoComponentDeepSearch(Caster.toBooleanValue(strDeepSearch.trim(), false)); } else if (hasCS) { config.setDoComponentDeepSearch(((ConfigServerImpl) configServer).doComponentDeepSearch()); } } // Dump-Template String strDumpRemplate = component.getAttribute("dump-template"); if ((strDumpRemplate == null || strDumpRemplate.trim().length() == 0) && configServer != null) { strDumpRemplate = configServer.getComponentDumpTemplate(); } config.setComponentDumpTemplate(strDumpRemplate); // data-member-default-access if (mode == ConfigImpl.MODE_STRICT) { config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); } else { String strDmda = component.getAttribute("data-member-default-access"); if (strDmda != null && strDmda.trim().length() > 0) { strDmda = strDmda.toLowerCase().trim(); if (strDmda.equals("remote")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_REMOTE); else if (strDmda.equals("public")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_PUBLIC); else if (strDmda.equals("package")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_PACKAGE); else if (strDmda.equals("private")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); } else if (configServer != null) { config.setComponentDataMemberDefaultAccess(configServer.getComponentDataMemberDefaultAccess()); } } // trigger-properties if (mode == ConfigImpl.MODE_STRICT) { config.setTriggerComponentDataMember(true); } else { Boolean tp = Caster.toBoolean(component.getAttribute("trigger-data-member"), null); if (tp != null) config.setTriggerComponentDataMember(tp.booleanValue()); else if (configServer != null) { config.setTriggerComponentDataMember(configServer.getTriggerComponentDataMember()); } } // local search if (mode == ConfigImpl.MODE_STRICT) { config.setComponentLocalSearch(false); } else { Boolean ls = Caster.toBoolean(component.getAttribute("local-search"), null); if (ls != null) config.setComponentLocalSearch(ls.booleanValue()); else if (configServer != null) { config.setComponentLocalSearch(((ConfigServerImpl) configServer).getComponentLocalSearch()); } } // use cache path Boolean ucp = Caster.toBoolean(component.getAttribute("use-cache-path"), null); if (ucp != null) config.setUseComponentPathCache(ucp.booleanValue()); else if (configServer != null) { config.setUseComponentPathCache(((ConfigServerImpl) configServer).useComponentPathCache()); } // use component shadow if (mode == ConfigImpl.MODE_STRICT) { config.setUseComponentShadow(false); } else { Boolean ucs = Caster.toBoolean(component.getAttribute("use-shadow"), null); if (ucs != null) config.setUseComponentShadow(ucs.booleanValue()); else if (configServer != null) { config.setUseComponentShadow(configServer.useComponentShadow()); } } } else if (configServer != null) { config.setBaseComponentTemplate(configServer.getBaseComponentTemplate()); config.setComponentDumpTemplate(configServer.getComponentDumpTemplate()); if (mode == ConfigImpl.MODE_STRICT) { config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); config.setTriggerComponentDataMember(true); } else { config.setComponentDataMemberDefaultAccess(configServer.getComponentDataMemberDefaultAccess()); config.setTriggerComponentDataMember(configServer.getTriggerComponentDataMember()); } } if (mode == ConfigImpl.MODE_STRICT) { config.setDoComponentDeepSearch(false); config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); config.setTriggerComponentDataMember(true); config.setComponentLocalSearch(false); config.setUseComponentShadow(false); } // Web Mapping Element[] cMappings = getChildren(component, "mapping"); hasSet = false; Mapping[] mappings = null; if (hasAccess && cMappings.length > 0) { mappings = new Mapping[cMappings.length]; for (int i = 0; i < cMappings.length; i++) { Element cMapping = cMappings[i]; String physical = cMapping.getAttribute("physical"); String archive = cMapping.getAttribute("archive"); boolean readonly = toBoolean(cMapping.getAttribute("readonly"), false); boolean hidden = toBoolean(cMapping.getAttribute("hidden"), false); //boolean trusted = toBoolean(cMapping.getAttribute("trusted"), false); short inspTemp = inspectTemplate(cMapping); String virtual = ConfigWebAdmin.createVirtual(cMapping); int clMaxEl = toInt(cMapping.getAttribute("classloader-max-elements"), 100); String primary = cMapping.getAttribute("primary"); boolean physicalFirst = archive == null || !primary.equalsIgnoreCase("archive"); hasSet = true; mappings[i] = new MappingImpl(config, virtual, physical, archive, inspTemp, physicalFirst, hidden, readonly, true, false, true, null, clMaxEl); } config.setComponentMappings(mappings); } // Server Mapping if (hasCS) { Mapping[] originals = ((ConfigServerImpl) configServer).getComponentMappings(); Mapping[] clones = new Mapping[originals.length]; LinkedHashMap map = new LinkedHashMap(); Mapping m; for (int i = 0; i < clones.length; i++) { m = ((MappingImpl) originals[i]).cloneReadOnly(config); map.put(toKey(m), m); // clones[i]=((MappingImpl)m[i]).cloneReadOnly(config); } if (mappings != null) { for (int i = 0; i < mappings.length; i++) { m = mappings[i]; map.put(toKey(m), m); } } if (originals.length > 0) { clones = new Mapping[map.size()]; Iterator it = map.entrySet().iterator(); Map.Entry entry; int index = 0; while (it.hasNext()) { entry = (Entry) it.next(); clones[index++] = (Mapping) entry.getValue(); // print.out("c:"+clones[index-1]); } hasSet = true; // print.err("set:"+clones.length); config.setComponentMappings(clones); } } if (!hasSet) { MappingImpl m = new MappingImpl(config, "/default", "{lucee-web}/components/", null, ConfigImpl.INSPECT_UNDEFINED, true, false, false, true, false, true, null); config.setComponentMappings(new Mapping[] { m.cloneReadOnly(config) }); } }
From source file:lucee.runtime.config.XMLConfigWebFactory.java
/** * @param configServer/*from w w w. j av a 2s .c om*/ * @param config * @param doc * @throws IOException */ private static void loadCustomTagsMappings(ConfigServerImpl configServer, ConfigImpl config, Document doc, int mode) { boolean hasAccess = ConfigWebUtil.hasAccess(config, SecurityManager.TYPE_CUSTOM_TAG); boolean hasCS = configServer != null; Element customTag = getChildByName(doc.getDocumentElement(), "custom-tag"); Element[] ctMappings = getChildren(customTag, "mapping"); // String virtualx="/custom-tag/"; // do patch cache String strDoPathcache = customTag.getAttribute("use-cache-path"); if (hasAccess && !StringUtil.isEmpty(strDoPathcache, true)) { config.setUseCTPathCache(Caster.toBooleanValue(strDoPathcache.trim(), true)); } else if (hasCS) { config.setUseCTPathCache(configServer.useCTPathCache()); } // do custom tag local search if (mode == ConfigImpl.MODE_STRICT) { config.setDoLocalCustomTag(false); } else { String strDoCTLocalSearch = customTag.getAttribute("custom-tag-local-search"); if (hasAccess && !StringUtil.isEmpty(strDoCTLocalSearch)) { config.setDoLocalCustomTag(Caster.toBooleanValue(strDoCTLocalSearch.trim(), true)); } else if (hasCS) { config.setDoLocalCustomTag(configServer.doLocalCustomTag()); } } // do custom tag deep search if (mode == ConfigImpl.MODE_STRICT) { config.setDoCustomTagDeepSearch(false); } else { String strDoCTDeepSearch = customTag.getAttribute("custom-tag-deep-search"); if (hasAccess && !StringUtil.isEmpty(strDoCTDeepSearch)) { config.setDoCustomTagDeepSearch(Caster.toBooleanValue(strDoCTDeepSearch.trim(), false)); } else if (hasCS) { config.setDoCustomTagDeepSearch(configServer.doCustomTagDeepSearch()); } } // extensions if (mode == ConfigImpl.MODE_STRICT) { config.setCustomTagExtensions(Constants.getComponentExtensions()); } else { String strExtensions = customTag.getAttribute("extensions"); if (hasAccess && !StringUtil.isEmpty(strExtensions)) { try { String[] arr = ListUtil.toStringArray(ListUtil.listToArrayRemoveEmpty(strExtensions, ",")); config.setCustomTagExtensions(ListUtil.trimItems(arr)); } catch (PageException e) { } } else if (hasCS) { config.setCustomTagExtensions(configServer.getCustomTagExtensions()); } } // Web Mapping boolean hasSet = false; Mapping[] mappings = null; if (hasAccess && ctMappings.length > 0) { mappings = new Mapping[ctMappings.length]; for (int i = 0; i < ctMappings.length; i++) { Element ctMapping = ctMappings[i]; String physical = ctMapping.getAttribute("physical"); String archive = ctMapping.getAttribute("archive"); boolean readonly = toBoolean(ctMapping.getAttribute("readonly"), false); boolean hidden = toBoolean(ctMapping.getAttribute("hidden"), false); //boolean trusted = toBoolean(ctMapping.getAttribute("trusted"), false); short inspTemp = inspectTemplate(ctMapping); //int clMaxEl = toInt(ctMapping.getAttribute("classloader-max-elements"), 100); String primary = ctMapping.getAttribute("primary"); boolean physicalFirst = archive == null || !primary.equalsIgnoreCase("archive"); hasSet = true; mappings[i] = new MappingImpl(config, XMLConfigAdmin.createVirtual(ctMapping), physical, archive, inspTemp, physicalFirst, hidden, readonly, true, false, true, null, -1, -1); // print.out(mappings[i].isPhysicalFirst()); } config.setCustomTagMappings(mappings); } // Server Mapping if (hasCS) { Mapping[] originals = configServer.getCustomTagMappings(); Mapping[] clones = new Mapping[originals.length]; LinkedHashMap map = new LinkedHashMap(); Mapping m; for (int i = 0; i < clones.length; i++) { m = ((MappingImpl) originals[i]).cloneReadOnly(config); map.put(toKey(m), m); // clones[i]=((MappingImpl)m[i]).cloneReadOnly(config); } if (mappings != null) { for (int i = 0; i < mappings.length; i++) { m = mappings[i]; map.put(toKey(m), m); } } if (originals.length > 0) { clones = new Mapping[map.size()]; Iterator it = map.entrySet().iterator(); Map.Entry entry; int index = 0; while (it.hasNext()) { entry = (Entry) it.next(); clones[index++] = (Mapping) entry.getValue(); // print.out("c:"+clones[index-1]); } hasSet = true; // print.err("set:"+clones.length); config.setCustomTagMappings(clones); } } if (!hasSet) { // MappingImpl m=new // MappingImpl(config,"/default-customtags/","{lucee-web}/customtags/",null,false,true,false,false,true,false,true); // config.setCustomTagMappings(new // Mapping[]{m.cloneReadOnly(config)}); } }
From source file:com.datatorrent.stram.plan.physical.PhysicalPlan.java
private void redoPartitions(PMapping currentMapping, String note) { Partitioner<Operator> partitioner = getPartitioner(currentMapping); if (partitioner == null) { LOG.warn("No partitioner for {}", currentMapping.logicalOperator); return;// w w w .ja v a2s . co m } RepartitionContext mainPC = new RepartitionContext(partitioner, currentMapping, 0); if (mainPC.newPartitions.isEmpty()) { LOG.warn("Empty partition list after repartition: {}", currentMapping.logicalOperator); return; } int memoryPerPartition = currentMapping.logicalOperator.getValue(OperatorContext.MEMORY_MB); for (Map.Entry<OutputPortMeta, StreamMeta> stream : currentMapping.logicalOperator.getOutputStreams() .entrySet()) { if (stream.getValue().getLocality() != Locality.THREAD_LOCAL && stream.getValue().getLocality() != Locality.CONTAINER_LOCAL) { memoryPerPartition += stream.getKey().getValue(PortContext.BUFFER_MEMORY_MB); } } for (OperatorMeta pp : currentMapping.parallelPartitions) { for (Map.Entry<OutputPortMeta, StreamMeta> stream : pp.getOutputStreams().entrySet()) { if (stream.getValue().getLocality() != Locality.THREAD_LOCAL && stream.getValue().getLocality() != Locality.CONTAINER_LOCAL) { memoryPerPartition += stream.getKey().getValue(PortContext.BUFFER_MEMORY_MB); } } memoryPerPartition += pp.getValue(OperatorContext.MEMORY_MB); } int requiredMemoryMB = (mainPC.newPartitions.size() - mainPC.currentPartitions.size()) * memoryPerPartition; if (requiredMemoryMB > availableMemoryMB) { LOG.warn("Insufficient headroom for repartitioning: available {}m required {}m", availableMemoryMB, requiredMemoryMB); return; } List<Partition<Operator>> addedPartitions = new ArrayList<Partition<Operator>>(); // determine modifications of partition set, identify affected operator instance(s) for (Partition<Operator> newPartition : mainPC.newPartitions) { PTOperator op = mainPC.currentPartitionMap.remove(newPartition); if (op == null) { addedPartitions.add(newPartition); } else { // check whether mapping was changed for (DefaultPartition<Operator> pi : mainPC.currentPartitions) { if (pi == newPartition && pi.isModified()) { // existing partition changed (operator or partition keys) // remove/add to update subscribers and state mainPC.currentPartitionMap.put(newPartition, op); addedPartitions.add(newPartition); } } } } // remaining entries represent deprecated partitions this.undeployOpers.addAll(mainPC.currentPartitionMap.values()); // downstream dependencies require redeploy, resolve prior to modifying plan Set<PTOperator> deps = this.getDependents(mainPC.currentPartitionMap.values()); this.undeployOpers.addAll(deps); // dependencies need redeploy, except operators excluded in remove this.deployOpers.addAll(deps); // process parallel partitions before removing operators from the plan LinkedHashMap<PMapping, RepartitionContext> partitionContexts = Maps.newLinkedHashMap(); Stack<OperatorMeta> parallelPartitions = new Stack<LogicalPlan.OperatorMeta>(); parallelPartitions.addAll(currentMapping.parallelPartitions); pendingLoop: while (!parallelPartitions.isEmpty()) { OperatorMeta ppMeta = parallelPartitions.pop(); for (StreamMeta s : ppMeta.getInputStreams().values()) { if (currentMapping.parallelPartitions.contains(s.getSource().getOperatorMeta()) && parallelPartitions.contains(s.getSource().getOperatorMeta())) { parallelPartitions.push(ppMeta); parallelPartitions.remove(s.getSource().getOperatorMeta()); parallelPartitions.push(s.getSource().getOperatorMeta()); continue pendingLoop; } } LOG.debug("Processing parallel partition {}", ppMeta); PMapping ppm = this.logicalToPTOperator.get(ppMeta); Partitioner<Operator> ppp = getPartitioner(ppm); if (ppp == null) { partitionContexts.put(ppm, null); } else { RepartitionContext pc = new RepartitionContext(ppp, ppm, mainPC.newPartitions.size()); if (pc.newPartitions == null) { throw new IllegalStateException( "Partitioner returns null for parallel partition " + ppm.logicalOperator); } partitionContexts.put(ppm, pc); } } // plan updates start here, after all changes were identified // remove obsolete operators first, any freed resources // can subsequently be used for new/modified partitions List<PTOperator> copyPartitions = Lists.newArrayList(currentMapping.partitions); // remove deprecated partitions from plan for (PTOperator p : mainPC.currentPartitionMap.values()) { copyPartitions.remove(p); removePartition(p, currentMapping); mainPC.operatorIdToPartition.remove(p.getId()); } currentMapping.partitions = copyPartitions; // add new operators for (Partition<Operator> newPartition : addedPartitions) { PTOperator p = addPTOperator(currentMapping, newPartition, mainPC.minCheckpoint); mainPC.operatorIdToPartition.put(p.getId(), newPartition); } // process parallel partition changes for (Map.Entry<PMapping, RepartitionContext> e : partitionContexts.entrySet()) { if (e.getValue() == null) { // no partitioner, add required operators for (int i = 0; i < addedPartitions.size(); i++) { LOG.debug("Automatically adding to parallel partition {}", e.getKey()); // set activation windowId to confirm to upstream checkpoints addPTOperator(e.getKey(), null, mainPC.minCheckpoint); } } else { RepartitionContext pc = e.getValue(); // track previous parallel partition mapping Map<Partition<Operator>, Partition<Operator>> prevMapping = Maps.newHashMap(); for (int i = 0; i < mainPC.currentPartitions.size(); i++) { prevMapping.put(pc.currentPartitions.get(i), mainPC.currentPartitions.get(i)); } // determine which new partitions match upstream, remaining to be treated as new operator Map<Partition<Operator>, Partition<Operator>> newMapping = Maps.newHashMap(); Iterator<Partition<Operator>> itMain = mainPC.newPartitions.iterator(); Iterator<Partition<Operator>> itParallel = pc.newPartitions.iterator(); while (itMain.hasNext() && itParallel.hasNext()) { newMapping.put(itParallel.next(), itMain.next()); } for (Partition<Operator> newPartition : pc.newPartitions) { PTOperator op = pc.currentPartitionMap.remove(newPartition); if (op == null) { pc.addedPartitions.add(newPartition); } else if (prevMapping.get(newPartition) != newMapping.get(newPartition)) { // upstream partitions don't match, remove/add to replace with new operator pc.currentPartitionMap.put(newPartition, op); pc.addedPartitions.add(newPartition); } else { // check whether mapping was changed - based on DefaultPartition implementation for (DefaultPartition<Operator> pi : pc.currentPartitions) { if (pi == newPartition && pi.isModified()) { // existing partition changed (operator or partition keys) // remove/add to update subscribers and state mainPC.currentPartitionMap.put(newPartition, op); pc.addedPartitions.add(newPartition); } } } } if (!pc.currentPartitionMap.isEmpty()) { // remove obsolete partitions List<PTOperator> cowPartitions = Lists.newArrayList(e.getKey().partitions); for (PTOperator p : pc.currentPartitionMap.values()) { cowPartitions.remove(p); removePartition(p, e.getKey()); pc.operatorIdToPartition.remove(p.getId()); } e.getKey().partitions = cowPartitions; } // add new partitions for (Partition<Operator> newPartition : pc.addedPartitions) { PTOperator oper = addPTOperator(e.getKey(), newPartition, mainPC.minCheckpoint); pc.operatorIdToPartition.put(oper.getId(), newPartition); } getPartitioner(e.getKey()).partitioned(pc.operatorIdToPartition); } } updateStreamMappings(currentMapping); for (PMapping pp : partitionContexts.keySet()) { updateStreamMappings(pp); } deployChanges(); if (mainPC.currentPartitions.size() != mainPC.newPartitions.size()) { StramEvent ev = new StramEvent.PartitionEvent(currentMapping.logicalOperator.getName(), mainPC.currentPartitions.size(), mainPC.newPartitions.size()); ev.setReason(note); this.ctx.recordEventAsync(ev); } partitioner.partitioned(mainPC.operatorIdToPartition); }
From source file:lucee.runtime.config.XMLConfigWebFactory.java
/** * @param configServer/*from w w w . j av a 2 s.c om*/ * @param config * @param doc * @throws IOException */ private static void loadComponent(ConfigServer configServer, ConfigImpl config, Document doc, int mode) { Element component = getChildByName(doc.getDocumentElement(), "component"); boolean hasAccess = ConfigWebUtil.hasAccess(config, SecurityManager.TYPE_SETTING); boolean hasSet = false; boolean hasCS = configServer != null; // String virtual="/component/"; if (component != null && hasAccess) { // component-default-import String strCDI = component.getAttribute("component-default-import"); if (StringUtil.isEmpty(strCDI, true) && configServer != null) { strCDI = ((ConfigServerImpl) configServer).getComponentDefaultImport().toString(); } if (!StringUtil.isEmpty(strCDI, true)) config.setComponentDefaultImport(strCDI); // Base CFML String strBase = component.getAttribute("base-cfml"); if (StringUtil.isEmpty(strBase, true)) strBase = component.getAttribute("base"); if (StringUtil.isEmpty(strBase, true) && configServer != null) { strBase = configServer.getBaseComponentTemplate(CFMLEngine.DIALECT_CFML); } config.setBaseComponentTemplate(CFMLEngine.DIALECT_CFML, strBase); // Base Lucee strBase = component.getAttribute("base-lucee"); if (StringUtil.isEmpty(strBase, true)) { if (configServer != null) strBase = configServer.getBaseComponentTemplate(CFMLEngine.DIALECT_LUCEE); else strBase = "/lucee/Component.lucee"; } config.setBaseComponentTemplate(CFMLEngine.DIALECT_LUCEE, strBase); // deep search if (mode == ConfigImpl.MODE_STRICT) { config.setDoComponentDeepSearch(false); } else { String strDeepSearch = component.getAttribute("deep-search"); if (!StringUtil.isEmpty(strDeepSearch)) { config.setDoComponentDeepSearch(Caster.toBooleanValue(strDeepSearch.trim(), false)); } else if (hasCS) { config.setDoComponentDeepSearch(((ConfigServerImpl) configServer).doComponentDeepSearch()); } } // Dump-Template String strDumpRemplate = component.getAttribute("dump-template"); if ((strDumpRemplate == null || strDumpRemplate.trim().length() == 0) && configServer != null) { strDumpRemplate = configServer.getComponentDumpTemplate(); } config.setComponentDumpTemplate(strDumpRemplate); // data-member-default-access if (mode == ConfigImpl.MODE_STRICT) { config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); } else { String strDmda = component.getAttribute("data-member-default-access"); if (strDmda != null && strDmda.trim().length() > 0) { strDmda = strDmda.toLowerCase().trim(); if (strDmda.equals("remote")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_REMOTE); else if (strDmda.equals("public")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_PUBLIC); else if (strDmda.equals("package")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_PACKAGE); else if (strDmda.equals("private")) config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); } else if (configServer != null) { config.setComponentDataMemberDefaultAccess(configServer.getComponentDataMemberDefaultAccess()); } } // trigger-properties if (mode == ConfigImpl.MODE_STRICT) { config.setTriggerComponentDataMember(true); } else { Boolean tp = Caster.toBoolean(component.getAttribute("trigger-data-member"), null); if (tp != null) config.setTriggerComponentDataMember(tp.booleanValue()); else if (configServer != null) { config.setTriggerComponentDataMember(configServer.getTriggerComponentDataMember()); } } // local search if (mode == ConfigImpl.MODE_STRICT) { config.setComponentLocalSearch(false); } else { Boolean ls = Caster.toBoolean(component.getAttribute("local-search"), null); if (ls != null) config.setComponentLocalSearch(ls.booleanValue()); else if (configServer != null) { config.setComponentLocalSearch(((ConfigServerImpl) configServer).getComponentLocalSearch()); } } // use cache path Boolean ucp = Caster.toBoolean(component.getAttribute("use-cache-path"), null); if (ucp != null) config.setUseComponentPathCache(ucp.booleanValue()); else if (configServer != null) { config.setUseComponentPathCache(((ConfigServerImpl) configServer).useComponentPathCache()); } // use component shadow if (mode == ConfigImpl.MODE_STRICT) { config.setUseComponentShadow(false); } else { Boolean ucs = Caster.toBoolean(component.getAttribute("use-shadow"), null); if (ucs != null) config.setUseComponentShadow(ucs.booleanValue()); else if (configServer != null) { config.setUseComponentShadow(configServer.useComponentShadow()); } } } else if (configServer != null) { config.setBaseComponentTemplate(CFMLEngine.DIALECT_CFML, configServer.getBaseComponentTemplate(CFMLEngine.DIALECT_CFML)); config.setBaseComponentTemplate(CFMLEngine.DIALECT_LUCEE, configServer.getBaseComponentTemplate(CFMLEngine.DIALECT_LUCEE)); config.setComponentDumpTemplate(configServer.getComponentDumpTemplate()); if (mode == ConfigImpl.MODE_STRICT) { config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); config.setTriggerComponentDataMember(true); } else { config.setComponentDataMemberDefaultAccess(configServer.getComponentDataMemberDefaultAccess()); config.setTriggerComponentDataMember(configServer.getTriggerComponentDataMember()); } } if (mode == ConfigImpl.MODE_STRICT) { config.setDoComponentDeepSearch(false); config.setComponentDataMemberDefaultAccess(Component.ACCESS_PRIVATE); config.setTriggerComponentDataMember(true); config.setComponentLocalSearch(false); config.setUseComponentShadow(false); } // Web Mapping Element[] cMappings = getChildren(component, "mapping"); hasSet = false; Mapping[] mappings = null; if (hasAccess && cMappings.length > 0) { mappings = new Mapping[cMappings.length]; for (int i = 0; i < cMappings.length; i++) { Element cMapping = cMappings[i]; String physical = cMapping.getAttribute("physical"); String archive = cMapping.getAttribute("archive"); boolean readonly = toBoolean(cMapping.getAttribute("readonly"), false); boolean hidden = toBoolean(cMapping.getAttribute("hidden"), false); //boolean trusted = toBoolean(cMapping.getAttribute("trusted"), false); int listMode = ConfigWebUtil.toListenerMode(cMapping.getAttribute("listener-mode"), -1); int listType = ConfigWebUtil.toListenerType(cMapping.getAttribute("listener-type"), -1); short inspTemp = inspectTemplate(cMapping); String virtual = XMLConfigAdmin.createVirtual(cMapping); //int clMaxEl = toInt(cMapping.getAttribute("classloader-max-elements"), 100); String primary = cMapping.getAttribute("primary"); boolean physicalFirst = archive == null || !primary.equalsIgnoreCase("archive"); hasSet = true; mappings[i] = new MappingImpl(config, virtual, physical, archive, inspTemp, physicalFirst, hidden, readonly, true, false, true, null, listMode, listType); } config.setComponentMappings(mappings); } // Server Mapping if (hasCS) { Mapping[] originals = ((ConfigServerImpl) configServer).getComponentMappings(); Mapping[] clones = new Mapping[originals.length]; LinkedHashMap map = new LinkedHashMap(); Mapping m; for (int i = 0; i < clones.length; i++) { m = ((MappingImpl) originals[i]).cloneReadOnly(config); map.put(toKey(m), m); // clones[i]=((MappingImpl)m[i]).cloneReadOnly(config); } if (mappings != null) { for (int i = 0; i < mappings.length; i++) { m = mappings[i]; map.put(toKey(m), m); } } if (originals.length > 0) { clones = new Mapping[map.size()]; Iterator it = map.entrySet().iterator(); Map.Entry entry; int index = 0; while (it.hasNext()) { entry = (Entry) it.next(); clones[index++] = (Mapping) entry.getValue(); // print.out("c:"+clones[index-1]); } hasSet = true; // print.err("set:"+clones.length); config.setComponentMappings(clones); } } if (!hasSet) { MappingImpl m = new MappingImpl(config, "/default", "{lucee-web}/components/", null, ConfigImpl.INSPECT_UNDEFINED, true, false, false, true, false, true, null, -1, -1); config.setComponentMappings(new Mapping[] { m.cloneReadOnly(config) }); } }
From source file:com.github.cmisbox.core.Queue.java
private void synchAllWatches() throws Exception { Storage storage = Storage.getInstance(); CMISRepository cmisRepository = CMISRepository.getInstance(); UI ui = UI.getInstance();/* ww w . ja v a2 s .com*/ Config config = Config.getInstance(); if (ui.isAvailable()) { ui.setStatus(Status.SYNCH); } List<String[]> updates = new ArrayList<String[]>(); Changes changes = cmisRepository.getContentChanges(storage.getRootIds()); LinkedHashMap<String, File> downloadList = new LinkedHashMap<String, File>(); boolean errors = false; for (ChangeItem item : changes.getEvents()) { try { if (item == null) { continue; } String id = "workspace://SpacesStore/" + item.getId(); String type = item.getT(); StoredItem storedItem = storage.findById(id); if (type.equals("D")) { if (storedItem != null) { File f = new File(config.getWatchParent() + storedItem.getPath()); f.delete(); storage.delete(storedItem, true); } } else if (type.equals("C") || type.equals("U")) { CmisObject remoteObject = cmisRepository.findObject(id); remoteObject.refresh(); if (remoteObject.getType().getBaseTypeId().equals(BaseTypeId.CMIS_FOLDER)) { Folder folder = (Folder) remoteObject; File newFile = new File(this.resolvePath(folder.getFolderParent()), folder.getName()); if (storedItem == null) { storage.add(newFile, folder, false); } else { if ((folder.getLastModificationDate().getTimeInMillis() > storedItem .getRemoteModified()) && !storedItem.getName().equals(folder.getName())) { if (new File(storedItem.getAbsolutePath()).renameTo(newFile)) { storage.localUpdate(storedItem, newFile, folder); } else { if (ui.isAvailable()) { ui.notify(Messages.renameError + " " + storedItem.getAbsolutePath() + " -> " + newFile.getAbsolutePath()); } this.log.error("Unable to rename " + storedItem.getAbsolutePath() + " to " + newFile.getAbsolutePath()); } } } } else { Document document = (Document) remoteObject; this.log.debug("preparing to update or create " + document.getName()); File newFile = new File(this.resolvePath(document.getParents().get(0)), document.getName()); if (storedItem == null) { downloadList.put(id, newFile); } else { File current = new File(storedItem.getAbsolutePath()); if (storedItem.getLocalModified() < document.getLastModificationDate() .getTimeInMillis()) { if (!current.getAbsolutePath().equals(newFile.getAbsolutePath())) { if (!current.renameTo(newFile)) { if (ui.isAvailable()) { ui.notify(Messages.renameError + " " + storedItem.getAbsolutePath() + " -> " + newFile.getAbsolutePath()); } this.log.error("Unable to rename " + storedItem.getAbsolutePath() + " to " + newFile.getAbsolutePath()); } } downloadList.put(id, newFile); } } } } } catch (Exception e1) { errors = true; this.log.error("Error getting remote chahges for " + item, e1); } } if (ui.isAvailable() && (downloadList.size() > 0)) { ui.notify(Messages.downloading + " " + downloadList.size() + " " + Messages.files); } for (Entry<String, File> e : downloadList.entrySet()) { try { storage.deleteById(e.getKey()); e.getValue().delete(); cmisRepository.download(cmisRepository.getDocument(e.getKey()), e.getValue()); storage.add(e.getValue(), cmisRepository.getDocument(e.getKey())); } catch (Exception e1) { errors = true; this.log.error("Error downloading " + e, e1); if (ui.isAvailable()) { ui.notify(Messages.errorDownloading + " " + e); } } } if (!errors) { config.setChangeLogToken(changes.getToken()); } if (ui.isAvailable()) { ui.setStatus(Status.OK); if (updates.size() == 1) { ui.notify(updates.get(0)[0] + " " + Messages.updatedBy + " " + updates.get(0)[1]); } else if (updates.size() > 1) { ui.notify(Messages.updated + " " + updates.size() + Messages.files); } } }
From source file:org.jamocha.dn.memory.javaimpl.MemoryHandlerPlusTemp.java
private static void performJoin(final AddressNodeFilterSet nodeFilterSet, final LinkedHashMap<Edge, StackElement> edgeToStack, final Edge originEdge) { final Node targetNode = originEdge.getTargetNode(); final StackElement originElement = edgeToStack.get(originEdge); final Counter counter = ((MemoryHandlerMain) originEdge.getTargetNode().getMemory()).counter; for (final AddressFilter filter : nodeFilterSet.getFilters()) { final SlotInFactAddress[] addresses = filter.getAddressesInTarget(); final List<StackElement> stack = new ArrayList<>(addresses.length); final PredicateWithArguments<ParameterLeaf> predicate = filter.getFunction(); final CounterColumn counterColumn = (CounterColumn) filter.getCounterColumn(); final boolean existential = (counterColumn != null); // determine new edges final Set<Edge> newEdges = new HashSet<>(); stack.add(originElement);//from w w w. j a v a 2 s .c o m for (final SlotInFactAddress address : addresses) { final Edge edge = targetNode.delocalizeAddress(address.getFactAddress()).getEdge(); final StackElement element = edgeToStack.get(edge); if (element != originElement) { // (!a || b) <=> (a -> b) if (!existential || edge.getSourceNode().getOutgoingExistentialEdges().contains(edge)) { if (newEdges.add(edge)) { stack.add(element); } } else { // full join regular edges if FilterElement is existential fullJoin(edgeToStack, originElement, edge, element); } } } // if existential, perform slightly different join not copying but only changing // counters. if (existential) { loop(new FunctionPointer() { @Override public void apply(final JamochaArray<Row> tr, final StackElement originElement) { final int paramLength = predicate.getParamTypes().length; final Object[] params = new Object[paramLength]; // determine parameters for (int i = 0; i < paramLength; ++i) { final SlotInFactAddress slotInTargetAddress = addresses[i]; final org.jamocha.dn.memory.FactAddress targetAddress = slotInTargetAddress .getFactAddress(); final AddressPredecessor upwardsAddress = targetNode.delocalizeAddress(targetAddress); final StackElement se = edgeToStack.get(upwardsAddress.getEdge()); params[i] = se.getValue(upwardsAddress, slotInTargetAddress.getSlotAddress()); } // increment counter if facts match predicate if (predicate.evaluate(params)) { final Row row = originElement.getRow(); // use counter to set counterColumn to 1 if counterColumn is not null counter.increment(row, counterColumn, 1); } } }, stack); } else { // replace TR in originElement with new temporary result originElement.memStack.set(0, loop(new FunctionPointer() { @Override public void apply(final JamochaArray<Row> tr, final StackElement originElement) { final int paramLength = predicate.getParamTypes().length; final Object[] params = new Object[paramLength]; // determine parameters for (int i = 0; i < paramLength; ++i) { final SlotInFactAddress slotInTargetAddress = addresses[i]; final org.jamocha.dn.memory.FactAddress targetAddress = slotInTargetAddress .getFactAddress(); final AddressPredecessor upwardsAddress = targetNode.delocalizeAddress(targetAddress); final StackElement se = edgeToStack.get(upwardsAddress.getEdge()); params[i] = se.getValue(upwardsAddress, slotInTargetAddress.getSlotAddress()); } // copy result to new TR if facts match predicate if (predicate.evaluate(params)) { // copy current row from old TR final Row row = originElement.getRow().copy(); // insert information from new inputs for (final Edge edge : newEdges) { // source is some temp, destination new TR final StackElement se = edgeToStack.get(edge); row.copy(se.getOffset(), se.getRow()); } // copy the result to new TR tr.add(row); } } }, stack)); } // point all inputs that were joint during this turn to the TR StackElement for (final Edge incomingEdge : newEdges) { edgeToStack.put(incomingEdge, originElement); } if (!originElement.checkRowBounds()) { return; } } // full join with all inputs not pointing to TR now for (final Map.Entry<Edge, StackElement> entry : edgeToStack.entrySet()) { final StackElement se = entry.getValue(); if (se == originElement) continue; final Edge nodeInput = entry.getKey(); fullJoin(edgeToStack, originElement, nodeInput, se); } }
From source file:eionet.cr.staging.exp.ExportRunner.java
/** * Export row./*from ww w .j av a2 s .c om*/ * * @param rs * the rs * @param rowIndex * the row index * @param repoConn * the repo conn * @param vf * the vf * @throws SQLException * the sQL exception * @throws RepositoryException * the repository exception * @throws DAOException */ private void exportRow(ResultSet rs, int rowIndex, RepositoryConnection repoConn, ValueFactory vf) throws SQLException, RepositoryException, DAOException { if (rowIndex == 1) { loadExistingConcepts(); } // Prepare subject URI on the basis of the template in the query configuration. String subjectUri = queryConf.getObjectUriTemplate(); if (StringUtils.isBlank(subjectUri)) { throw new IllegalArgumentException( "The object URI template in the query configuration must not be blank!"); } subjectUri = StringUtils.replace(subjectUri, "<dataset>", datasetIdentifier); // Prepare the map of ObjectDTO to be added to the subject later. LinkedHashMap<URI, ArrayList<Value>> valuesByPredicate = new LinkedHashMap<URI, ArrayList<Value>>(); // Add rdf:type predicate-value. addPredicateValue(valuesByPredicate, rdfTypeURI, objectTypeURI); // Add the DataCube dataset predicate-value. Assume this point cannot be reached if dataset value is empty. addPredicateValue(valuesByPredicate, datasetPredicateURI, datasetValueURI); // Add predicate-value pairs for hidden properties. if (hiddenProperties != null) { for (ObjectHiddenProperty hiddenProperty : hiddenProperties) { addPredicateValue(valuesByPredicate, hiddenProperty.getPredicateURI(), hiddenProperty.getValueValue()); } } boolean hasIndicatorMapping = false; // Loop through the query configuration's column mappings, construct ObjectDTO for each. for (Entry<String, ObjectProperty> entry : queryConf.getColumnMappings().entrySet()) { String colName = entry.getKey(); String colValue = rs.getString(colName); ObjectProperty property = entry.getValue(); if (property.getId().equals(INDICATOR)) { hasIndicatorMapping = true; } if (StringUtils.isBlank(colValue)) { if (property.getId().equals(BREAKDOWN)) { colValue = DEFAULT_BREAKDOWN_CODE; } else if (property.getId().equals(INDICATOR)) { colValue = DEFAULT_INDICATOR_CODE; } } if (StringUtils.isNotBlank(colValue)) { // Replace property place-holders in subject ID subjectUri = StringUtils.replace(subjectUri, "<" + property.getId() + ">", colValue); URI predicateURI = property.getPredicateURI(); if (predicateURI != null) { String propertyValue = property.getValueTemplate(); if (propertyValue == null) { propertyValue = colValue; } else { // Replace the column value place-holder in the value template (the latter cannot be specified by user) propertyValue = StringUtils.replace(propertyValue, "<value>", colValue); } recordMissingConcepts(property, colValue, propertyValue); Value value = null; if (property.isLiteralRange()) { try { String dataTypeUri = property.getDataType(); value = vf.createLiteral(propertyValue, dataTypeUri == null ? null : vf.createURI(dataTypeUri)); } catch (IllegalArgumentException e) { value = vf.createLiteral(propertyValue); } } else { value = vf.createURI(propertyValue); } addPredicateValue(valuesByPredicate, predicateURI, value); } } } // If there was no column mapping for the indicator, but a fixed indicator URI has been provided then use the latter. if (!hasIndicatorMapping && indicatorValueURI != null) { addPredicateValue(valuesByPredicate, indicatorPredicateURI, indicatorValueURI); } // If <indicator> column placeholder not replaced yet, then use the fixed indicator URI if given. if (subjectUri.indexOf("<indicator>") != -1) { String indicatorCode = StringUtils.substringAfterLast(queryConf.getIndicatorUri(), "/"); if (StringUtils.isBlank(indicatorCode)) { // No fixed indicator URI given either, resort to the default. indicatorCode = DEFAULT_INDICATOR_CODE; } subjectUri = StringUtils.replace(subjectUri, "<indicator>", indicatorCode); } // If <breakdown> column placeholder not replaced yet, then use the default. if (subjectUri.indexOf("<breakdown>") != -1) { subjectUri = StringUtils.replace(subjectUri, "<breakdown>", DEFAULT_BREAKDOWN_CODE); } // Loop over predicate-value pairs and create the triples in the triple store. if (!valuesByPredicate.isEmpty()) { int tripleCountBefore = tripleCount; URI subjectURI = vf.createURI(subjectUri); for (Entry<URI, ArrayList<Value>> entry : valuesByPredicate.entrySet()) { ArrayList<Value> values = entry.getValue(); if (values != null && !values.isEmpty()) { URI predicateURI = entry.getKey(); for (Value value : values) { repoConn.add(subjectURI, predicateURI, value, graphURI); graphs.add(graphURI.stringValue()); tripleCount++; if (tripleCount % 5000 == 0) { LOGGER.debug(tripleCount + " triples exported so far"); } // Time periods should be harvested afterwards. if (Predicates.DAS_TIMEPERIOD.equals(predicateURI.stringValue())) { if (value instanceof URI) { timePeriods.add(value.stringValue()); } } } } } if (tripleCount > tripleCountBefore) { subjectCount++; } } }