List of usage examples for java.util List containsAll
boolean containsAll(Collection<?> c);
From source file:org.openmrs.api.ConceptServiceTest.java
/** * @see ConceptService#getConceptsByName(String,Locale) * @verifies return concepts for all countries and global language given language only locale *//* w ww . ja va 2 s. c o m*/ @Test public void getConceptsByName_shouldReturnConceptsForAllCountriesAndGlobalLanguageGivenLanguageOnlyLocale() throws Exception { //given String name = "Concept"; Concept concept1 = new Concept(); concept1.addName(new ConceptName(name, new Locale("en", "US"))); Context.getConceptService().saveConcept(concept1); Concept concept2 = new Concept(); concept2.addName(new ConceptName(name, new Locale("en", "GB"))); Context.getConceptService().saveConcept(concept2); Concept concept3 = new Concept(); concept3.addName(new ConceptName(name, new Locale("en"))); Context.getConceptService().saveConcept(concept3); updateSearchIndex(); //when List<Concept> concepts = Context.getConceptService().getConceptsByName(name, new Locale("en"), false); //then Assert.assertEquals(3, concepts.size()); Assert.assertTrue(concepts.containsAll(Arrays.asList(concept1, concept2, concept3))); }
From source file:org.nuunframework.kernel.Kernel.java
/** * //from ww w. ja v a 2 s .c o m */ private void checkPlugins() { logger.info("Plugins initialisation "); plugins.clear(); List<Class<? extends Plugin>> pluginClasses = new ArrayList<Class<? extends Plugin>>(); for (Plugin plugin : fetchedPlugins) { String pluginName = plugin.name(); logger.info("checking Plugin {}.", pluginName); if (!Strings.isNullOrEmpty(pluginName)) { Object ok = plugins.put(pluginName, plugin); if (ok == null) { // Check for required param // ======================== Collection<KernelParamsRequest> kernelParamsRequests = plugin.kernelParamsRequests(); Collection<String> computedMandatoryParams = new HashSet<String>(); for (KernelParamsRequest kernelParamsRequest : kernelParamsRequests) { if (kernelParamsRequest.requestType == KernelParamsRequestType.MANDATORY) { computedMandatoryParams.add(kernelParamsRequest.keyRequested); } } if (kernelParamsAndAlias.containsAllKeys(computedMandatoryParams)) // if (kernelParams.keySet().containsAll(computedMandatoryParams)) { pluginClasses.add(plugin.getClass()); } else { logger.error("plugin {} miss parameter/s : {}", pluginName, kernelParamsRequests.toString()); throw new KernelException( "plugin " + pluginName + " miss parameter/s : " + kernelParamsRequests.toString()); } } else { logger.error( "Can not have 2 Plugin {} of the same type {}. please fix this before the kernel can start.", pluginName, plugin.getClass().getName()); throw new KernelException( "Can not have 2 Plugin %s of the same type %s. please fix this before the kernel can start.", pluginName, plugin.getClass().getName()); } } else { logger.warn("Plugin {} has no correct name it won't be installed.", plugin.getClass()); throw new KernelException("Plugin %s has no correct name it won't be installed.", pluginName); } } // Check for required and dependent plugins for (Plugin plugin : plugins.values()) { { Collection<Class<? extends Plugin>> pluginDependenciesRequired = plugin.requiredPlugins(); if (pluginDependenciesRequired != null && !pluginDependenciesRequired.isEmpty() && !pluginClasses.containsAll(pluginDependenciesRequired)) { logger.error("plugin {} misses the following plugin/s as dependency/ies {}", plugin.name(), pluginDependenciesRequired.toString()); throw new KernelException("plugin %s misses the following plugin/s as dependency/ies %s", plugin.name(), pluginDependenciesRequired.toString()); } } { Collection<Class<? extends Plugin>> dependentPlugin = plugin.dependentPlugins(); if (dependentPlugin != null && !dependentPlugin.isEmpty() && !pluginClasses.containsAll(dependentPlugin)) { logger.error("plugin {} misses the following plugin/s as dependee/s {}", plugin.name(), dependentPlugin.toString()); throw new KernelException("plugin %s misses the following plugin/s as dependee/s %s", plugin.name(), dependentPlugin.toString()); } } } }
From source file:org.voltdb.planner.ParsedSelectStmt.java
private boolean orderByColumnsCoverUniqueKeys() { // In theory, if EVERY table in the query has a uniqueness constraint // (primary key or other unique index) on columns that are all listed in the ORDER BY values, // the result is deterministic. // This holds regardless of whether the associated index is actually used in the selected plan, // so this check is plan-independent. HashMap<String, List<AbstractExpression>> baseTableAliases = new HashMap<String, List<AbstractExpression>>(); for (ParsedColInfo col : m_orderColumns) { AbstractExpression expr = col.expression; List<AbstractExpression> baseTVEs = expr.findBaseTVEs(); if (baseTVEs.size() != 1) { // Table-spanning ORDER BYs -- like ORDER BY A.X + B.Y are not helpful. // Neither are (nonsense) constant (table-less) expressions. continue; }//from w ww.j a v a2s. c o m // This loops exactly once. AbstractExpression baseTVE = baseTVEs.get(0); String nextTableAlias = ((TupleValueExpression) baseTVE).getTableAlias(); assert (nextTableAlias != null); List<AbstractExpression> perTable = baseTableAliases.get(nextTableAlias); if (perTable == null) { perTable = new ArrayList<AbstractExpression>(); baseTableAliases.put(nextTableAlias, perTable); } perTable.add(expr); } if (m_tableAliasMap.size() > baseTableAliases.size()) { // FIXME: This would be one of the tricky cases where the goal would be to prove that the // row with no ORDER BY component came from the right side of a 1-to-1 or many-to-1 join. return false; } boolean allScansAreDeterministic = true; for (Entry<String, List<AbstractExpression>> orderedAlias : baseTableAliases.entrySet()) { List<AbstractExpression> orderedAliasExprs = orderedAlias.getValue(); StmtTableScan tableScan = m_tableAliasMap.get(orderedAlias.getKey()); if (tableScan == null) { assert (false); return false; } if (tableScan instanceof StmtSubqueryScan) { return false; // don't yet handle FROM clause subquery, here. } Table table = ((StmtTargetTableScan) tableScan).getTargetTable(); // This table's scans need to be proven deterministic. allScansAreDeterministic = false; // Search indexes for one that makes the order by deterministic for (Index index : table.getIndexes()) { // skip non-unique indexes if (!index.getUnique()) { continue; } // get the list of expressions for the index List<AbstractExpression> indexExpressions = new ArrayList<AbstractExpression>(); String jsonExpr = index.getExpressionsjson(); // if this is a pure-column index... if (jsonExpr.isEmpty()) { for (ColumnRef cref : index.getColumns()) { Column col = cref.getColumn(); TupleValueExpression tve = new TupleValueExpression(table.getTypeName(), orderedAlias.getKey(), col.getName(), col.getName(), col.getIndex()); indexExpressions.add(tve); } } // if this is a fancy expression-based index... else { try { indexExpressions = AbstractExpression.fromJSONArrayString(jsonExpr, tableScan); } catch (JSONException e) { e.printStackTrace(); assert (false); continue; } } // If the sort covers the index, then it's a unique sort. //TODO: The statement's equivalence sets would be handy here to recognize cases like // WHERE B.unique_id = A.b_id // ORDER BY A.unique_id, A.b_id if (orderedAliasExprs.containsAll(indexExpressions)) { allScansAreDeterministic = true; break; } } // ALL tables' scans need to have proved deterministic if (!allScansAreDeterministic) { return false; } } return true; }
From source file:org.finra.herd.dao.BusinessObjectDataDaoTest.java
@Test public void testGetBusinessObjectDataEntitiesByKey() { // Execute the same set of tests on the sets of business object data entities with and without subpartition values. for (List<String> subPartitionValues : Arrays.asList(SUBPARTITION_VALUES, NO_SUBPARTITION_VALUES)) { // Create two business object data entities that differ on both format version and data version. List<BusinessObjectDataEntity> businessObjectDataEntities = Arrays.asList( businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, subPartitionValues, SECOND_DATA_VERSION, false, BDATA_STATUS), businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, SECOND_FORMAT_VERSION, PARTITION_VALUE, subPartitionValues, THIRD_DATA_VERSION, false, BDATA_STATUS)); // Retrieve the first business object data entity by specifying the entire business object data key. List<BusinessObjectDataEntity> resultBusinessObjectDataEntities = businessObjectDataDao .getBusinessObjectDataEntities(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, subPartitionValues, SECOND_DATA_VERSION)); // Validate the results. assertNotNull(resultBusinessObjectDataEntities); assertEquals(1, resultBusinessObjectDataEntities.size()); assertEquals(businessObjectDataEntities.get(0).getId(), resultBusinessObjectDataEntities.get(0).getId()); // Retrieve both business object data entities by not specifying both format and data versions. resultBusinessObjectDataEntities = businessObjectDataDao.getBusinessObjectDataEntities( new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, null, PARTITION_VALUE, subPartitionValues, null)); // Validate the results. assertNotNull(resultBusinessObjectDataEntities); assertTrue(resultBusinessObjectDataEntities.containsAll(businessObjectDataEntities)); assertTrue(businessObjectDataEntities.containsAll(resultBusinessObjectDataEntities)); }//from w w w . j av a2 s . co m }
From source file:edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor.java
/*** * Propagated equivalent classes from the child to the current operator, based * on the used variables of the current operator. * /*from w w w . j a va 2 s. c o m*/ * @param op * , the current operator * @param ctx * , the optimization context which keeps track of all equivalent classes. * @param usedVariables * , used variables. * @throws AlgebricksException */ private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx, List<LogicalVariable> usedVariables) throws AlgebricksException { ILogicalOperator op2 = op.getInputs().get(0).getValue(); Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx); List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(); ctx.putFDList(op, fds); Map<LogicalVariable, EquivalenceClass> chldClasses = getOrComputeEqClasses(op2, ctx); // Propagate equivalent classes that contain the used variables. for (LogicalVariable v : usedVariables) { EquivalenceClass ec = eqClasses.get(v); if (ec == null) { EquivalenceClass oc = chldClasses.get(v); if (oc == null) { continue; } List<LogicalVariable> m = new LinkedList<LogicalVariable>(); for (LogicalVariable v2 : oc.getMembers()) { if (usedVariables.contains(v2)) { m.add(v2); } } EquivalenceClass nc; if (oc.representativeIsConst()) { nc = new EquivalenceClass(m, oc.getConstRepresentative()); } else if (m.contains(oc.getVariableRepresentative())) { nc = new EquivalenceClass(m, oc.getVariableRepresentative()); } else { nc = new EquivalenceClass(m, v); } for (LogicalVariable v3 : m) { eqClasses.put(v3, nc); } } } // Propagates equivalent classes that contain expressions that use the used variables. // Note that for the case variable $v is not in the used variables but it is // equivalent to field-access($t, i) and $t is a used variable, the equivalent // class should still be propagated (kept). Set<LogicalVariable> usedVarSet = new HashSet<LogicalVariable>(usedVariables); for (Entry<LogicalVariable, EquivalenceClass> entry : chldClasses.entrySet()) { EquivalenceClass ec = entry.getValue(); for (ILogicalExpression expr : ec.getExpressionMembers()) { Set<LogicalVariable> exprUsedVars = new HashSet<LogicalVariable>(); expr.getUsedVariables(exprUsedVars); exprUsedVars.retainAll(usedVarSet); // Check if the expression member uses a used variable. if (!exprUsedVars.isEmpty()) { for (LogicalVariable v : ec.getMembers()) { eqClasses.put(v, ec); // If variable members contain a used variable, the representative // variable should be a used variable. if (usedVarSet.contains(v)) { ec.setVariableRepresentative(v); } } } } } List<FunctionalDependency> chldFds = getOrComputeFDs(op2, ctx); for (FunctionalDependency fd : chldFds) { if (!usedVariables.containsAll(fd.getHead())) { continue; } List<LogicalVariable> tl = new LinkedList<LogicalVariable>(); for (LogicalVariable v : fd.getTail()) { if (usedVariables.contains(v)) { tl.add(v); } } if (!tl.isEmpty()) { FunctionalDependency newFd = new FunctionalDependency(fd.getHead(), tl); fds.add(newFd); } } }
From source file:org.biomart.configurator.controller.MartController.java
private Mart requestCreateDataSetFromTarget(MartRegistry registry, String dsName, DataLinkInfo dlinkInfo) throws MartBuilderException { // create dataset // MartController dataset = null; Mart mart = new Mart(registry, dsName, null); // dataset = (MartController)mart.getWrapper(); // add datasettable McSQL mcSql = new McSQL(); JdbcLinkObject dbconObj = dlinkInfo.getJdbcLinkObject().getDsInfoMap().keySet().iterator().next() .getJdbcLinkObject();/*from w ww .j a va 2 s .com*/ // dataset.setConnectionObject(dbconObj); // dataset name is the martName + "_" + last part of the database name String databaseName = dbconObj.getSchemaName(); String[] _names = databaseName.split(Resources.get("tablenameSubSep"), 2); String datasetName; if (_names.length == 1) datasetName = dsName + "_" + _names[0]; else datasetName = dsName + "_" + _names[1]; String sqlDsName = dsName; if (dlinkInfo.isTargetTableNamePartitioned()) { int index = dsName.lastIndexOf(Resources.get("tablenameSubSep")); if (index >= 0) { sqlDsName = dsName.substring(0, index); } } Set<String> subPartitionTables = mcSql.getPartitionedTables(dbconObj, sqlDsName); Map<String, List<String>> tblColMap = mcSql.getMartTablesInfo(dbconObj, dsName); if (tblColMap.isEmpty()) return null; // get all main tables and keys TODO improve Map<String, List<String>> mainKeyMap = new HashMap<String, List<String>>(); List<String> orderedMainTableList = new ArrayList<String>(); List<String> mainTableList = new ArrayList<String>(); // get all mainTable for (Map.Entry<String, List<String>> entry : tblColMap.entrySet()) { if (!isMainTable(entry.getKey())) continue; mainTableList.add(entry.getKey()); } // build the mainKeyMap and order the mainTable String[] mainArray = new String[mainTableList.size()]; for (String tblName : mainTableList) { if (!isMainTable(tblName)) continue; List<String> keyList = new ArrayList<String>(); for (String colStr : tblColMap.get(tblName)) { if (colStr.endsWith(Resources.get("keySuffix"))) // if(colStr.indexOf(Resources.get("keySuffix"))>=0) keyList.add(colStr); } mainKeyMap.put(tblName, keyList); if (keyList.size() < 1) { JOptionPane.showMessageDialog(null, "no key column in " + tblName); return null; } mainArray[keyList.size() - 1] = tblName; } orderedMainTableList = Arrays.asList(mainArray); if (orderedMainTableList.isEmpty()) { JOptionPane.showMessageDialog(null, "check " + dbconObj.getDatabaseName()); return null; } for (Map.Entry<String, List<String>> entry : tblColMap.entrySet()) { String tblName = entry.getKey(); DatasetTable dstable = (DatasetTable) mart.getTableByName(tblName); // create datasettable when it is null if (dstable == null) { // get table type DatasetTableType dstType; if (isMainTable(tblName)) { if (orderedMainTableList.indexOf(tblName) > 0) dstType = DatasetTableType.MAIN_SUBCLASS; else dstType = DatasetTableType.MAIN; } else dstType = DatasetTableType.DIMENSION; dstable = new DatasetTable(mart, tblName, dstType); dstable.addInPartitions(datasetName); mart.addTable(dstable); // columns for (String colStr : entry.getValue()) { // all are fake wrapped column DatasetColumn column = dstable.getColumnByName(colStr); if (column == null) { column = new DatasetColumn(dstable, colStr); column.addInPartitions(datasetName); dstable.addColumn(column); } // set pk or fk if colStr is a key if (colStr.indexOf(Resources.get("keySuffix")) >= 0) { if (dstType.equals(DatasetTableType.DIMENSION)) { ForeignKey fkObject = new ForeignKey(column); // KeyController fk = new KeyController(fkObject); fkObject.setStatus(ComponentStatus.INFERRED); if (!dstable.getForeignKeys().contains(fkObject)) { dstable.getForeignKeys().add(fkObject); // dstable.getForeignKeys().add(fk); } } else if (dstType.equals(DatasetTableType.MAIN)) { PrimaryKey pkObject = new PrimaryKey(column); // KeyController pk = new KeyController(pkObject); pkObject.setStatus(ComponentStatus.INFERRED); dstable.setPrimaryKey(pkObject); } else { if (isColPkinTable(colStr, tblName, mainKeyMap, orderedMainTableList)) { PrimaryKey pkObject = new PrimaryKey(column); // KeyController pk = new KeyController(pkObject); pkObject.setStatus(ComponentStatus.INFERRED); dstable.setPrimaryKey(pkObject); } else { ForeignKey fkObject = new ForeignKey(column); // KeyController fk = new KeyController(fkObject); fkObject.setStatus(ComponentStatus.INFERRED); if (!dstable.getForeignKeys().contains(fkObject)) { dstable.getForeignKeys().add(fkObject); // dstable.getForeignKeys().add(fk); } } } } } // end of columns } // end of if if (dlinkInfo.isTargetTableNamePartitioned()) { for (String value : subPartitionTables) dstable.addSubPartition(value); } } // relations main -- dimension first for (String mainTStr : orderedMainTableList) { DatasetTable mainTable = mart.getTableByName(mainTStr); for (DatasetTable table : mart.getDatasetTables()) { if (!table.getType().equals(DatasetTableType.DIMENSION)) continue; PrimaryKey pk = mainTable.getPrimaryKey(); // if pk and fk have the same columns (same name for now), matched List<String> pkColList = new ArrayList<String>(); for (Column tmpCol1 : pk.getColumns()) pkColList.add(tmpCol1.getName()); for (ForeignKey fk : table.getForeignKeys()) { List<String> fkColList = new ArrayList<String>(); for (Column tmpCol2 : fk.getColumns()) fkColList.add(tmpCol2.getName()); if (pkColList.size() == fkColList.size() && pkColList.containsAll(fkColList)) { if (!Relation.isRelationExist(pk, fk)) try { Relation rel = new RelationTarget(pk, fk, Cardinality.MANY_A); // pk.getObject().addRelation(rel); // fk.getObject().addRelation(rel); rel.setOriginalCardinality(Cardinality.MANY_A); rel.setStatus(ComponentStatus.INFERRED); } catch (AssociationException e) { e.printStackTrace(); } } } } } // end of main -- dimension relation // sub main relation for (int i = 0; i < orderedMainTableList.size() - 1; i++) { DatasetTable firstDst = mart.getTableByName(orderedMainTableList.get(i)); DatasetTable secondDst = mart.getTableByName(orderedMainTableList.get(i + 1)); PrimaryKey pk = firstDst.getPrimaryKey(); // if pk and fk have the same columns (same name for now), matched List<String> pkColList = new ArrayList<String>(); for (Column tmpCol1 : pk.getColumns()) pkColList.add(tmpCol1.getName()); for (ForeignKey fk : secondDst.getForeignKeys()) { List<String> fkColList = new ArrayList<String>(); for (Column tmpCol2 : fk.getColumns()) fkColList.add(tmpCol2.getName()); if (pkColList.size() == fkColList.size() && pkColList.containsAll(fkColList)) { if (!Relation.isRelationExist(pk, fk)) try { Relation rel = new RelationTarget(pk, fk, Cardinality.MANY_A); // pk.getObject().addRelation(rel); // fk.getObject().addRelation(rel); rel.setOriginalCardinality(Cardinality.MANY_A); rel.setStatus(ComponentStatus.INFERRED); rel.setSubclassRelation(true, Relation.DATASET_WIDE); } catch (AssociationException e) { e.printStackTrace(); } catch (ValidationException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } } mart.setCentralTable(mart.getTableByName(orderedMainTableList.get(0))); // create a default partitiontable PartitionTable pt = new PartitionTable(mart, PartitionType.SCHEMA); // new PartitionTableController(pt); List<String> rowItem = new ArrayList<String>(); rowItem.add(dbconObj.getConnectionBase()); rowItem.add(dbconObj.getDatabaseName()); rowItem.add(dbconObj.getSchemaName()); rowItem.add(dbconObj.getUserName()); rowItem.add(dbconObj.getPassword()); rowItem.add(datasetName); rowItem.add(XMLElements.FALSE_VALUE.toString()); rowItem.add(datasetName); rowItem.add(Resources.BIOMART_VERSION); // add empty string till col 14 rowItem.add(""); rowItem.add(""); rowItem.add(""); rowItem.add(""); rowItem.add(""); rowItem.add(""); pt.addNewRow(rowItem); mart.addPartitionTable(pt); return mart; }
From source file:org.biomart.configurator.controller.MartController.java
private Mart requestCreateDataSetFromTargetPartitioned(MartRegistry registry, MartInVirtualSchema martInVS, DataLinkInfo dlinkInfo) throws MartBuilderException { // mart name is the database name; JdbcLinkObject dbconObj = dlinkInfo.getJdbcLinkObject().getDsInfoMap().keySet().iterator().next() .getJdbcLinkObject();/*from w ww .j a v a2s .c o m*/ String databaseName = dbconObj.getSchemaName(); Mart mart = new Mart(registry, martInVS.getName(), null); // create a default partitiontable PartitionTable pt = new PartitionTable(mart, PartitionType.SCHEMA); mart.addPartitionTable(pt); McSQL mcSql = new McSQL(); // load tables for all datasets for (DatasetFromUrl dsurl : dlinkInfo.getJdbcLinkObject().getDsInfoMap().get(martInVS)) { // assume that they all follow the name convention String tablename = dsurl.getName(); int index = tablename.indexOf(Resources.get("tablenameSep")); String dsName = tablename.substring(0, index); Map<String, List<String>> tblColMap = mcSql.getMartTablesInfo(dbconObj, dsName); if (tblColMap.isEmpty()) return null; // get all main tables and keys TODO improve Map<String, List<String>> mainKeyMap = new HashMap<String, List<String>>(); List<String> orderedMainTableList = new ArrayList<String>(); List<String> mainTableList = new ArrayList<String>(); // get all mainTable for (Map.Entry<String, List<String>> entry : tblColMap.entrySet()) { if (!isMainTable(entry.getKey())) continue; mainTableList.add(entry.getKey()); } // build the mainKeyMap and order the mainTable String[] mainArray = new String[mainTableList.size()]; for (String tblName : mainTableList) { if (!isMainTable(tblName)) continue; List<String> keyList = new ArrayList<String>(); for (String colStr : tblColMap.get(tblName)) { if (colStr.endsWith(Resources.get("keySuffix"))) // if(colStr.indexOf(Resources.get("keySuffix"))>=0) keyList.add(colStr); } mainKeyMap.put(tblName, keyList); if (keyList.size() < 1) { JOptionPane.showMessageDialog(null, "no key column in " + tblName); return null; } mainArray[keyList.size() - 1] = tblName; } orderedMainTableList = Arrays.asList(mainArray); if (orderedMainTableList.isEmpty()) { JOptionPane.showMessageDialog(null, "check " + dbconObj.getDatabaseName()); return null; } for (Map.Entry<String, List<String>> entry : tblColMap.entrySet()) { String tblName = entry.getKey(); // put it in partition format index = tblName.indexOf(Resources.get("tablenameSep")); String _tblName = tblName.substring(index + 2); String tblNamePartitioned = "(p0c" + PartitionUtils.DATASETNAME + ")" + "__" + _tblName; DatasetTable dstable = (DatasetTable) mart.getTableByName(tblNamePartitioned); // create datasettable when it is null if (dstable == null) { // get table type DatasetTableType dstType; if (isMainTable(tblNamePartitioned)) { if (orderedMainTableList.indexOf(tblNamePartitioned) > 0) dstType = DatasetTableType.MAIN_SUBCLASS; else dstType = DatasetTableType.MAIN; } else dstType = DatasetTableType.DIMENSION; dstable = new DatasetTable(mart, tblNamePartitioned, dstType); dstable.addInPartitions(dsName); mart.addTable(dstable); } // end of if else { dstable.addInPartitions(dsName); } DatasetTableType dstType = dstable.getType(); // columns for (String colStr : entry.getValue()) { // all are fake wrapped column DatasetColumn column = dstable.getColumnByName(colStr); if (column == null) { column = new DatasetColumn(dstable, colStr); column.addInPartitions(dsName); dstable.addColumn(column); } else { column.addInPartitions(dsName); } // set pk or fk if colStr is a key if (colStr.indexOf(Resources.get("keySuffix")) >= 0) { if (dstType.equals(DatasetTableType.DIMENSION)) { ForeignKey fkObject = new ForeignKey(column); // KeyController fk = new KeyController(fkObject); fkObject.setStatus(ComponentStatus.INFERRED); if (!dstable.getForeignKeys().contains(fkObject)) { dstable.getForeignKeys().add(fkObject); // dstable.getForeignKeys().add(fk); } } else if (dstType.equals(DatasetTableType.MAIN)) { PrimaryKey pkObject = new PrimaryKey(column); // KeyController pk = new KeyController(pkObject); pkObject.setStatus(ComponentStatus.INFERRED); dstable.setPrimaryKey(pkObject); } else { if (isColPkinTable(colStr, tblName, mainKeyMap, orderedMainTableList)) { PrimaryKey pkObject = new PrimaryKey(column); // KeyController pk = new KeyController(pkObject); pkObject.setStatus(ComponentStatus.INFERRED); dstable.setPrimaryKey(pkObject); } else { ForeignKey fkObject = new ForeignKey(column); // KeyController fk = new KeyController(fkObject); fkObject.setStatus(ComponentStatus.INFERRED); if (!dstable.getForeignKeys().contains(fkObject)) { dstable.getForeignKeys().add(fkObject); // dstable.getForeignKeys().add(fk); } } } } } // end of columns } // relations main -- dimension first for (String mainTStr : orderedMainTableList) { DatasetTable mainTable = mart.getTableByName(mainTStr); for (DatasetTable table : mart.getDatasetTables()) { if (!table.getType().equals(DatasetTableType.DIMENSION)) continue; PrimaryKey pk = mainTable.getPrimaryKey(); // if pk and fk have the same columns (same name for now), matched List<String> pkColList = new ArrayList<String>(); for (Column tmpCol1 : pk.getColumns()) pkColList.add(tmpCol1.getName()); for (ForeignKey fk : table.getForeignKeys()) { List<String> fkColList = new ArrayList<String>(); for (Column tmpCol2 : fk.getColumns()) fkColList.add(tmpCol2.getName()); if (pkColList.size() == fkColList.size() && pkColList.containsAll(fkColList)) { if (!Relation.isRelationExist(pk, fk)) try { Relation rel = new RelationTarget(pk, fk, Cardinality.MANY_A); // pk.getObject().addRelation(rel); // fk.getObject().addRelation(rel); rel.setOriginalCardinality(Cardinality.MANY_A); rel.setStatus(ComponentStatus.INFERRED); } catch (AssociationException e) { e.printStackTrace(); } } } } } // end of main -- dimension relation // sub main relation for (int i = 0; i < orderedMainTableList.size() - 1; i++) { DatasetTable firstDst = mart.getTableByName(orderedMainTableList.get(i)); DatasetTable secondDst = mart.getTableByName(orderedMainTableList.get(i + 1)); PrimaryKey pk = firstDst.getPrimaryKey(); // if pk and fk have the same columns (same name for now), matched List<String> pkColList = new ArrayList<String>(); for (Column tmpCol1 : pk.getColumns()) pkColList.add(tmpCol1.getName()); for (ForeignKey fk : secondDst.getForeignKeys()) { List<String> fkColList = new ArrayList<String>(); for (Column tmpCol2 : fk.getColumns()) fkColList.add(tmpCol2.getName()); if (pkColList.size() == fkColList.size() && pkColList.containsAll(fkColList)) { if (!Relation.isRelationExist(pk, fk)) try { Relation rel = new RelationTarget(pk, fk, Cardinality.MANY_A); // pk.getObject().addRelation(rel); // fk.getObject().addRelation(rel); rel.setOriginalCardinality(Cardinality.MANY_A); rel.setStatus(ComponentStatus.INFERRED); rel.setSubclassRelation(true, Relation.DATASET_WIDE); } catch (AssociationException e) { e.printStackTrace(); } catch (ValidationException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } } List<String> rowItem = new ArrayList<String>(); rowItem.add(dbconObj.getConnectionBase()); rowItem.add(dbconObj.getDatabaseName()); rowItem.add(dbconObj.getSchemaName()); rowItem.add(dbconObj.getUserName()); rowItem.add(dbconObj.getPassword()); rowItem.add(dsName); rowItem.add(XMLElements.FALSE_VALUE.toString()); rowItem.add(dsName); rowItem.add(Resources.BIOMART_VERSION); // add empty string till col 14 rowItem.add(""); rowItem.add(""); rowItem.add(""); rowItem.add(""); rowItem.add(""); rowItem.add(""); pt.addNewRow(rowItem); } mart.setCentralTable(mart.getOrderedMainTableList().get(0)); return mart; }
From source file:org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor.java
/*** * Propagated equivalent classes from the child to the current operator, * based on the used variables of the current operator. * * @param op/* w w w. ja va 2 s. c om*/ * , the current operator * @param ctx * , the optimization context which keeps track of all equivalent * classes. * @param usedVariables * , used variables. * @throws AlgebricksException */ private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx, List<LogicalVariable> usedVariables) throws AlgebricksException { ILogicalOperator op2 = op.getInputs().get(0).getValue(); Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx); List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(); ctx.putFDList(op, fds); Map<LogicalVariable, EquivalenceClass> chldClasses = getOrComputeEqClasses(op2, ctx); // Propagate equivalent classes that contain the used variables. for (LogicalVariable v : usedVariables) { EquivalenceClass ec = eqClasses.get(v); if (ec == null) { EquivalenceClass oc = chldClasses.get(v); if (oc == null) { continue; } List<LogicalVariable> m = new LinkedList<LogicalVariable>(); for (LogicalVariable v2 : oc.getMembers()) { if (usedVariables.contains(v2)) { m.add(v2); } } EquivalenceClass nc; if (oc.representativeIsConst()) { nc = new EquivalenceClass(m, oc.getConstRepresentative()); } else if (m.contains(oc.getVariableRepresentative())) { nc = new EquivalenceClass(m, oc.getVariableRepresentative()); } else { nc = new EquivalenceClass(m, v); } for (LogicalVariable v3 : m) { eqClasses.put(v3, nc); } } } // Propagates equivalent classes that contain expressions that use the // used variables. // Note that for the case variable $v is not in the used variables but // it is // equivalent to field-access($t, i) and $t is a used variable, the // equivalent // class should still be propagated (kept). Set<LogicalVariable> usedVarSet = new HashSet<LogicalVariable>(usedVariables); for (Entry<LogicalVariable, EquivalenceClass> entry : chldClasses.entrySet()) { EquivalenceClass ec = entry.getValue(); for (ILogicalExpression expr : ec.getExpressionMembers()) { Set<LogicalVariable> exprUsedVars = new HashSet<LogicalVariable>(); expr.getUsedVariables(exprUsedVars); exprUsedVars.retainAll(usedVarSet); // Check if the expression member uses a used variable. if (!exprUsedVars.isEmpty()) { for (LogicalVariable v : ec.getMembers()) { eqClasses.put(v, ec); // If variable members contain a used variable, the // representative // variable should be a used variable. if (usedVarSet.contains(v)) { ec.setVariableRepresentative(v); } } } } } List<FunctionalDependency> chldFds = getOrComputeFDs(op2, ctx); for (FunctionalDependency fd : chldFds) { if (!usedVariables.containsAll(fd.getHead())) { continue; } List<LogicalVariable> tl = new LinkedList<LogicalVariable>(); for (LogicalVariable v : fd.getTail()) { if (usedVariables.contains(v)) { tl.add(v); } } if (!tl.isEmpty()) { FunctionalDependency newFd = new FunctionalDependency(fd.getHead(), tl); fds.add(newFd); } } }
From source file:org.apache.giraph.master.BspServiceMaster.java
/** * Wait for a set of workers to signal that they are done with the * barrier./*ww w . j av a 2s.co m*/ * * @param finishedWorkerPath Path to where the workers will register their * hostname and id * @param workerInfoList List of the workers to wait for * @param event Event to wait on for a chance to be done. * @param ignoreDeath In case if worker died after making it through * barrier, we will ignore death if set to true. * @return True if barrier was successful, false if there was a worker * failure */ private boolean barrierOnWorkerList(String finishedWorkerPath, List<WorkerInfo> workerInfoList, BspEvent event, boolean ignoreDeath) { try { getZkExt().createOnceExt(finishedWorkerPath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); } catch (KeeperException e) { throw new IllegalStateException( "barrierOnWorkerList: KeeperException - Couldn't create " + finishedWorkerPath, e); } catch (InterruptedException e) { throw new IllegalStateException( "barrierOnWorkerList: InterruptedException - Couldn't create " + finishedWorkerPath, e); } List<String> hostnameIdList = new ArrayList<String>(workerInfoList.size()); for (WorkerInfo workerInfo : workerInfoList) { hostnameIdList.add(workerInfo.getHostnameId()); } String workerInfoHealthyPath = getWorkerInfoHealthyPath(getApplicationAttempt(), getSuperstep()); List<String> finishedHostnameIdList; long nextInfoMillis = System.currentTimeMillis(); final int defaultTaskTimeoutMsec = 10 * 60 * 1000; // from TaskTracker final int taskTimeoutMsec = getContext().getConfiguration().getInt("mapred.task.timeout", defaultTaskTimeoutMsec); List<WorkerInfo> deadWorkers = new ArrayList<>(); while (true) { try { finishedHostnameIdList = getZkExt().getChildrenExt(finishedWorkerPath, true, false, false); } catch (KeeperException e) { throw new IllegalStateException("barrierOnWorkerList: KeeperException - Couldn't get " + "children of " + finishedWorkerPath, e); } catch (InterruptedException e) { throw new IllegalStateException("barrierOnWorkerList: IllegalException - Couldn't get " + "children of " + finishedWorkerPath, e); } if (LOG.isDebugEnabled()) { LOG.debug("barrierOnWorkerList: Got finished worker list = " + finishedHostnameIdList + ", size = " + finishedHostnameIdList.size() + ", worker list = " + workerInfoList + ", size = " + workerInfoList.size() + " from " + finishedWorkerPath); } if (LOG.isInfoEnabled() && (System.currentTimeMillis() > nextInfoMillis)) { nextInfoMillis = System.currentTimeMillis() + 30000; LOG.info("barrierOnWorkerList: " + finishedHostnameIdList.size() + " out of " + workerInfoList.size() + " workers finished on superstep " + getSuperstep() + " on path " + finishedWorkerPath); if (workerInfoList.size() - finishedHostnameIdList.size() < MAX_PRINTABLE_REMAINING_WORKERS) { Set<String> remainingWorkers = Sets.newHashSet(hostnameIdList); remainingWorkers.removeAll(finishedHostnameIdList); LOG.info("barrierOnWorkerList: Waiting on " + remainingWorkers); } } getContext().setStatus(getGraphTaskManager().getGraphFunctions() + " - " + finishedHostnameIdList.size() + " finished out of " + workerInfoList.size() + " on superstep " + getSuperstep()); if (finishedHostnameIdList.containsAll(hostnameIdList)) { break; } for (WorkerInfo deadWorker : deadWorkers) { if (!finishedHostnameIdList.contains(deadWorker.getHostnameId())) { LOG.error("barrierOnWorkerList: no results arived from " + "worker that was pronounced dead: " + deadWorker + " on superstep " + getSuperstep()); return false; } } // Wait for a signal or timeout event.waitMsecs(taskTimeoutMsec / 2); event.reset(); getContext().progress(); // Did a worker die? try { deadWorkers.addAll(superstepChosenWorkerAlive(workerInfoHealthyPath, workerInfoList)); if (!ignoreDeath && deadWorkers.size() > 0) { String errorMessage = "******* WORKERS " + deadWorkers + " FAILED *******"; // If checkpointing is not used, we should fail the job if (!getConfiguration().useCheckpointing()) { setJobStateFailed(errorMessage); } else { LOG.error("barrierOnWorkerList: Missing chosen " + "workers " + deadWorkers + " on superstep " + getSuperstep()); // Log worker failure to command line getGraphTaskManager().getJobProgressTracker().logInfo(errorMessage); } return false; } } catch (KeeperException e) { throw new IllegalStateException( "barrierOnWorkerList: KeeperException - " + "Couldn't get " + workerInfoHealthyPath, e); } catch (InterruptedException e) { throw new IllegalStateException( "barrierOnWorkerList: InterruptedException - " + "Couldn't get " + workerInfoHealthyPath, e); } } return true; }
From source file:org.hawkular.metrics.core.service.MetricsServiceITest.java
@Test public void shouldReceiveInsertedDataNotifications() throws Exception { String tenantId = "shouldReceiveInsertedDataNotifications"; ImmutableList<MetricType<?>> metricTypes = ImmutableList.of(GAUGE, COUNTER, AVAILABILITY); AvailabilityType[] availabilityTypes = AvailabilityType.values(); int numberOfPoints = 10; List<Metric<?>> actual = Collections.synchronizedList(new ArrayList<>()); CountDownLatch latch = new CountDownLatch(metricTypes.size() * numberOfPoints); metricsService.insertedDataEvents().filter(metric -> metric.getMetricId().getTenantId().equals(tenantId)) .subscribe(metric -> {/*from www .j a va2 s . com*/ actual.add(metric); latch.countDown(); }); List<Metric<?>> expected = new ArrayList<>(); for (MetricType<?> metricType : metricTypes) { for (int i = 0; i < numberOfPoints; i++) { if (metricType == GAUGE) { DataPoint<Double> dataPoint = new DataPoint<>((long) i, (double) i); MetricId<Double> metricId = new MetricId<>(tenantId, GAUGE, "gauge"); Metric<Double> metric = new Metric<>(metricId, ImmutableList.of(dataPoint)); metricsService.addDataPoints(GAUGE, Observable.just(metric)).subscribe(); expected.add(metric); } else if (metricType == COUNTER) { DataPoint<Long> dataPoint = new DataPoint<>((long) i, (long) i); MetricId<Long> metricId = new MetricId<>(tenantId, COUNTER, "counter"); Metric<Long> metric = new Metric<>(metricId, ImmutableList.of(dataPoint)); metricsService.addDataPoints(COUNTER, Observable.just(metric)).subscribe(); expected.add(metric); } else if (metricType == AVAILABILITY) { AvailabilityType availabilityType = availabilityTypes[i % availabilityTypes.length]; DataPoint<AvailabilityType> dataPoint = new DataPoint<>((long) i, availabilityType); MetricId<AvailabilityType> metricId = new MetricId<>(tenantId, AVAILABILITY, "avail"); Metric<AvailabilityType> metric = new Metric<>(metricId, ImmutableList.of(dataPoint)); metricsService.addDataPoints(AVAILABILITY, Observable.just(metric)).subscribe(); expected.add(metric); } else { fail("Unexpected metric type: " + metricType); } } } assertTrue(latch.await(1, MINUTES), "Did not receive all notifications"); assertEquals(actual.size(), expected.size()); assertTrue(actual.containsAll(expected)); }