List of usage examples for java.util HashSet size
public int size()
From source file:evaluation.Evaluator.java
int allLabels(String list1[], String list2[]) { HashSet<Integer> labels_per_instance = new HashSet<Integer>(); for (int i = 0; i < list1.length; i++) labels_per_instance.add(new Integer(Integer.parseInt(list1[i]))); for (int i = 0; i < list2.length; i++) labels_per_instance.add(new Integer(Integer.parseInt(list2[i]))); return labels_per_instance.size(); }
From source file:thingynet.hierarchy.HierarchyTest.java
@Test public void getDescendantsShouldReturnExpectedHierarchies() { Hierarchy parent = hierarchyService.createRoot(PARENT, null); Hierarchy child = hierarchyService.createChild(parent, CHILD, null); Hierarchy sibling = hierarchyService.createChild(parent, SIBLING, null); Hierarchy grandChild = hierarchyService.createChild(child, GRAND_CHILD, null); Hierarchy greatGrandChild = hierarchyService.createChild(grandChild, GREAT_GRAND_CHILD, null); Hierarchy other = hierarchyService.createRoot(OTHER, null); hierarchyService.createChild(other, CHILD, null); HashSet<String> expectedIds = new HashSet<>(); expectedIds.add(child.getPath());// ww w. j a va2 s . c o m expectedIds.add(sibling.getPath()); expectedIds.add(grandChild.getPath()); expectedIds.add(greatGrandChild.getPath()); Iterable<Hierarchy> actual = hierarchyService.getDescendants(parent); while (actual.iterator().hasNext()) { Hierarchy descendant = actual.iterator().next(); assertThat(expectedIds.remove(descendant.getPath()), is(true)); } assertThat(expectedIds.size(), is(0)); }
From source file:org.apache.hadoop.hive.ql.parse.TaskCompiler.java
@SuppressWarnings({ "nls", "unchecked" })
public void compile(final ParseContext pCtx, final List<Task<? extends Serializable>> rootTasks,
final HashSet<ReadEntity> inputs, final HashSet<WriteEntity> outputs) throws SemanticException {
Context ctx = pCtx.getContext();
GlobalLimitCtx globalLimitCtx = pCtx.getGlobalLimitCtx();
List<Task<MoveWork>> mvTask = new ArrayList<Task<MoveWork>>();
List<LoadTableDesc> loadTableWork = pCtx.getLoadTableWork();
List<LoadFileDesc> loadFileWork = pCtx.getLoadFileWork();
boolean isCStats = pCtx.getQueryProperties().isAnalyzeRewrite();
int outerQueryLimit = pCtx.getQueryProperties().getOuterQueryLimit();
if (pCtx.getFetchTask() != null) {
return;//from w ww . java 2s .c o m
}
optimizeOperatorPlan(pCtx, inputs, outputs);
/*
* In case of a select, use a fetch task instead of a move task.
* If the select is from analyze table column rewrite, don't create a fetch task. Instead create
* a column stats task later.
*/
if (pCtx.getQueryProperties().isQuery() && !isCStats) {
if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
}
LoadFileDesc loadFileDesc = loadFileWork.get(0);
String cols = loadFileDesc.getColumns();
String colTypes = loadFileDesc.getColumnTypes();
TableDesc resultTab = pCtx.getFetchTableDesc();
if (resultTab == null) {
String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
}
FetchWork fetch = new FetchWork(loadFileDesc.getSourcePath(), resultTab, outerQueryLimit);
fetch.setSource(pCtx.getFetchSource());
fetch.setSink(pCtx.getFetchSink());
pCtx.setFetchTask((FetchTask) TaskFactory.get(fetch, conf));
// For the FetchTask, the limit optimization requires we fetch all the rows
// in memory and count how many rows we get. It's not practical if the
// limit factor is too big
int fetchLimit = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVELIMITOPTMAXFETCH);
if (globalLimitCtx.isEnable() && globalLimitCtx.getGlobalLimit() > fetchLimit) {
LOG.info("For FetchTask, LIMIT " + globalLimitCtx.getGlobalLimit() + " > " + fetchLimit
+ ". Doesn't qualify limit optimiztion.");
globalLimitCtx.disableOpt();
}
if (outerQueryLimit == 0) {
// Believe it or not, some tools do generate queries with limit 0 and than expect
// query to run quickly. Lets meet their requirement.
LOG.info("Limit 0. No query execution needed.");
return;
}
} else if (!isCStats) {
for (LoadTableDesc ltd : loadTableWork) {
Task<MoveWork> tsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf);
mvTask.add(tsk);
// Check to see if we are stale'ing any indexes and auto-update them if we want
if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) {
IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, inputs, conf);
try {
List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater.generateUpdateTasks();
for (Task<? extends Serializable> updateTask : indexUpdateTasks) {
tsk.addDependentTask(updateTask);
}
} catch (HiveException e) {
console.printInfo("WARNING: could not auto-update stale indexes, which are not in sync");
}
}
}
boolean oneLoadFile = true;
for (LoadFileDesc lfd : loadFileWork) {
if (pCtx.getQueryProperties().isCTAS()) {
assert (oneLoadFile); // should not have more than 1 load file for
// CTAS
// make the movetask's destination directory the table's destination.
Path location;
String loc = pCtx.getCreateTable().getLocation();
if (loc == null) {
// get the table's default location
Path targetPath;
try {
String[] names = Utilities.getDbTableName(pCtx.getCreateTable().getTableName());
if (!db.databaseExists(names[0])) {
throw new SemanticException("ERROR: The database " + names[0] + " does not exist.");
}
Warehouse wh = new Warehouse(conf);
targetPath = wh.getTablePath(db.getDatabase(names[0]), names[1]);
} catch (HiveException e) {
throw new SemanticException(e);
} catch (MetaException e) {
throw new SemanticException(e);
}
location = targetPath;
} else {
location = new Path(loc);
}
lfd.setTargetDir(location);
oneLoadFile = false;
}
mvTask.add(TaskFactory.get(new MoveWork(null, null, null, lfd, false), conf));
}
}
generateTaskTree(rootTasks, pCtx, mvTask, inputs, outputs);
/*
* If the query was the result of analyze table column compute statistics rewrite, create
* a column stats task instead of a fetch task to persist stats to the metastore.
*/
if (isCStats) {
genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadTableWork, loadFileWork, rootTasks, outerQueryLimit);
}
// For each task, set the key descriptor for the reducer
for (Task<? extends Serializable> rootTask : rootTasks) {
GenMapRedUtils.setKeyAndValueDescForTaskTree(rootTask);
}
// If a task contains an operator which instructs bucketizedhiveinputformat
// to be used, please do so
for (Task<? extends Serializable> rootTask : rootTasks) {
setInputFormat(rootTask);
}
optimizeTaskPlan(rootTasks, pCtx, ctx);
decideExecMode(rootTasks, ctx, globalLimitCtx);
if (pCtx.getQueryProperties().isCTAS()) {
// generate a DDL task and make it a dependent task of the leaf
CreateTableDesc crtTblDesc = pCtx.getCreateTable();
crtTblDesc.validate(conf);
// clear the mapredWork output file from outputs for CTAS
// DDLWork at the tail of the chain will have the output
Iterator<WriteEntity> outIter = outputs.iterator();
while (outIter.hasNext()) {
switch (outIter.next().getType()) {
case DFS_DIR:
case LOCAL_DIR:
outIter.remove();
break;
default:
break;
}
}
Task<? extends Serializable> crtTblTask = TaskFactory.get(new DDLWork(inputs, outputs, crtTblDesc),
conf);
// find all leaf tasks and make the DDLTask as a dependent task of all of
// them
HashSet<Task<? extends Serializable>> leaves = new LinkedHashSet<Task<? extends Serializable>>();
getLeafTasks(rootTasks, leaves);
assert (leaves.size() > 0);
for (Task<? extends Serializable> task : leaves) {
if (task instanceof StatsTask) {
// StatsTask require table to already exist
for (Task<? extends Serializable> parentOfStatsTask : task.getParentTasks()) {
parentOfStatsTask.addDependentTask(crtTblTask);
}
for (Task<? extends Serializable> parentOfCrtTblTask : crtTblTask.getParentTasks()) {
parentOfCrtTblTask.removeDependentTask(task);
}
crtTblTask.addDependentTask(task);
} else {
task.addDependentTask(crtTblTask);
}
}
}
if (globalLimitCtx.isEnable() && pCtx.getFetchTask() != null) {
LOG.info("set least row check for FetchTask: " + globalLimitCtx.getGlobalLimit());
pCtx.getFetchTask().getWork().setLeastNumRows(globalLimitCtx.getGlobalLimit());
}
if (globalLimitCtx.isEnable() && globalLimitCtx.getLastReduceLimitDesc() != null) {
LOG.info("set least row check for LimitDesc: " + globalLimitCtx.getGlobalLimit());
globalLimitCtx.getLastReduceLimitDesc().setLeastRows(globalLimitCtx.getGlobalLimit());
List<ExecDriver> mrTasks = Utilities.getMRTasks(rootTasks);
for (ExecDriver tsk : mrTasks) {
tsk.setRetryCmdWhenFail(true);
}
List<SparkTask> sparkTasks = Utilities.getSparkTasks(rootTasks);
for (SparkTask sparkTask : sparkTasks) {
sparkTask.setRetryCmdWhenFail(true);
}
}
Interner<TableDesc> interner = Interners.newStrongInterner();
for (Task<? extends Serializable> rootTask : rootTasks) {
GenMapRedUtils.internTableDesc(rootTask, interner);
}
}
From source file:org.nuxeo.ecm.automation.core.impl.OperationServiceImpl.java
@Override public OperationType[] getOperations() { HashSet<OperationType> values = new HashSet<>(operations.lookup().values()); return values.toArray(new OperationType[values.size()]); }
From source file:org.apache.geode.internal.util.CollectionUtilsJUnitTest.java
@Test public void testAddAllCollectionEnumerationWithUnmodified() { final HashSet<String> set = new HashSet<>(); set.add("one"); set.add("two"); final Vector<String> v = new Vector<>(); v.add("one"); boolean modified = CollectionUtils.addAll(set, v.elements()); assertTrue(!modified);//from w ww .j av a2 s . co m assertEquals(2, set.size()); }
From source file:org.mskcc.cbio.portal.dao.DaoMutation.java
public static Collection<Map<String, Object>> countSamplesWithProteinPosStarts( Collection<String> proteinPosStarts, Collection<Integer> internalProfileIds) throws DaoException { Connection con = null;/* w w w . j a v a2s. c o m*/ PreparedStatement pstmt = null; ResultSet rs = null; //performance fix: mutation table contains geneId; by filtering on a geneId set before table join, the temporary table needed is smaller. //a geneticProfileId set filter alone can in some cases let almost all mutations into the temporary table HashSet<String> geneIdSet = new HashSet<String>(); if (proteinPosStarts != null) { Pattern geneIdPattern = Pattern.compile("\\(\\s*(\\d+)\\s*,"); for (String proteinPos : proteinPosStarts) { Matcher geneIdMatcher = geneIdPattern.matcher(proteinPos); if (geneIdMatcher.find()) { geneIdSet.add(geneIdMatcher.group(1)); } } } if (geneIdSet.size() == 0 || internalProfileIds.size() == 0) return new ArrayList<Map<String, Object>>(); //empty IN() clause would be a SQL error below try { con = JdbcUtil.getDbConnection(DaoMutation.class); String sql = "SELECT ONCOTATOR_PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID) " + "FROM mutation INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + "WHERE mutation.ENTREZ_GENE_ID IN (" + StringUtils.join(geneIdSet, ",") + ") " + "AND GENETIC_PROFILE_ID IN (" + StringUtils.join(internalProfileIds, ",") + ") " + "AND (mutation.ENTREZ_GENE_ID, ONCOTATOR_PROTEIN_POS_START) IN (" + StringUtils.join(proteinPosStarts, ",") + ") " + "GROUP BY ONCOTATOR_PROTEIN_POS_START, GENETIC_PROFILE_ID"; pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); Collection<Map<String, Object>> data = new ArrayList<Map<String, Object>>(); while (rs.next()) { Map<String, Object> d = new HashMap<String, Object>(); String proteinPosStart = rs.getString(1); Integer geneticProfileId = rs.getInt(2); Long entrez = rs.getLong(3); Integer count = rs.getInt(4); // can you get a cancerStudy's name? // this is computing a join and in not optimal GeneticProfile geneticProfile = DaoGeneticProfile.getGeneticProfileById(geneticProfileId); Integer cancerStudyId = geneticProfile.getCancerStudyId(); CancerStudy cancerStudy = DaoCancerStudy.getCancerStudyByInternalId(cancerStudyId); String name = cancerStudy.getName(); String cancerType = cancerStudy.getTypeOfCancerId(); CanonicalGene gene = DaoGeneOptimized.getInstance().getGene(entrez); String hugo = gene.getHugoGeneSymbolAllCaps(); d.put("protein_pos_start", proteinPosStart); d.put("protein_start_with_hugo", hugo + "_" + proteinPosStart); d.put("hugo", hugo); d.put("cancer_study", name); d.put("cancer_type", cancerType); d.put("count", count); data.add(d); } return data; } catch (SQLException e) { throw new DaoException(e); } finally { JdbcUtil.closeAll(DaoMutation.class, con, pstmt, rs); } }
From source file:org.apache.hadoop.chukwa.dataloader.FSMDataLoader.java
public void load(ChukwaConfiguration conf, FileSystem fs, FileStatus[] fileList) throws IOException { if (executor == null) { try {/*from www . ja v a2 s .c o m*/ this.size = Integer.parseInt(conf.get(DATA_LOADER_THREAD_LIMIT)); } catch (Exception e) { this.size = 1; } executor = Executors.newFixedThreadPool(size); } if (completion == null) { completion = new ExecutorCompletionService(executor); } try { // Locate directory output directories of the current demux, and create a unique directory list. HashSet<Path> inputPaths = new HashSet<Path>(); HashSet<Path> outputPaths = new HashSet<Path>(); int counter = 0; for (int i = 0; i < fileList.length; i++) { Path temp = fileList[i].getPath().getParent(); if (!inputPaths.contains(temp)) { inputPaths.add(temp); } } String outputDir = conf.get("chukwa.tmp.data.dir") + File.separator + "fsm_" + System.currentTimeMillis() + "_"; if (inputPaths.size() > 0) { Configuration fsmConf = new Configuration(); // Run fsm map reduce job for dn, tt, and jobhist. for (String mapper : mappers) { String[] args = new String[inputPaths.size() + 3]; args[0] = "-in"; int k = 2; boolean hasData = false; for (Path temp : inputPaths) { String tempPath = temp.toUri().toString(); if ((mapper.intern() == mappers[0].intern() && tempPath.indexOf("ClientTraceDetailed") > 0) || (mapper.intern() == mappers[1].intern() && tempPath.indexOf("ClientTraceDetailed") > 0) || (mapper.intern() == mappers[2].intern() && tempPath.indexOf("TaskData") > 0) || (mapper.intern() == mappers[2].intern() && tempPath.indexOf("JobData") > 0)) { args[k] = tempPath; k++; hasData = true; } } args[1] = k - 2 + ""; fsmConf.set("chukwa.salsa.fsm.mapclass", mapper); args[k] = outputDir + mapper; Path outputPath = new Path(args[k]); outputPaths.add(outputPath); if (hasData) { int res = ToolRunner.run(fsmConf, new FSMBuilder(), args); } } } // Find the mapreduce output and load to MDL. for (Path outputPath : outputPaths) { Path searchDir = new Path(outputPath.toUri().toString() + "/*/*/*.evt"); log.info("Search dir:" + searchDir.toUri().toString()); FileStatus[] outputList = fs.globStatus(searchDir); if (outputList != null) { for (int j = 0; j < outputList.length; j++) { String outputFile = outputList[j].getPath().toUri().toString(); log.info("FSM -> MDL loading: " + outputFile); completion.submit(new MetricDataLoader(conf, fs, outputFile)); counter++; } } else { log.warn("No output to load."); } } for (int i = 0; i < counter; i++) { completion.take().get(); } // Clean up mapreduce output of fsm. for (Path dir : outputPaths) { fs.delete(dir, true); } } catch (Exception e) { log.error(ExceptionUtil.getStackTrace(e)); throw new IOException(); } finally { } }
From source file:org.paxle.se.search.impl.SearchProviderManager.java
private void search(ISearchRequest request, ISearchResultCollector results) throws InterruptedException, ExecutionException, SearchException { if (request == null) throw new NullPointerException("The search-request object must not be null"); final CompletionService<ISearchResult> execCompletionService = new ExecutorCompletionService<ISearchResult>( this.execService); // determining all search-providers that should be used for the query HashSet<String> allowedProviderPIDs = new HashSet<String>(request.getProviderIDs()); // loop through all providers and pass the request to each one List<String> usedProviderPIDs = new ArrayList<String>(); for (Entry<String, ServiceReference> providerEntry : this.providersRefs.entrySet()) { final String providerPID = providerEntry.getKey(); final ServiceReference providerRef = providerEntry.getValue(); if (allowedProviderPIDs.size() > 0 && !allowedProviderPIDs.contains(providerPID)) { this.logger.debug(String.format("SEProvider '%s' is skipped for search request '%d'.", providerPID, Integer.valueOf(request.getRequestID()))); continue; }/*from w ww .j a va2 s. co m*/ usedProviderPIDs.add(providerPID); execCompletionService.submit(new SearchProviderCallable(this.ctx, providerRef, request)); } if (allowedProviderPIDs.size() == 0) { // store the providers we have used to process the search-request request.setProviderIDs(usedProviderPIDs); } // loop through all providers and collect the results long searchTimeout = request.getTimeout(); for (int i = 0; i < usedProviderPIDs.size(); ++i) { final long start = System.currentTimeMillis(); // waiting for the next search result final Future<ISearchResult> future = execCompletionService.poll(searchTimeout, TimeUnit.MILLISECONDS); if (future != null) { final ISearchResult r = future.get(); if (r != null) { final String providerPID = r.getProviderID(); final int size = r.getSize(); this.logger .debug(String.format("SEProvider '%s' returned '%d' results for search-request '%d'.", providerPID, Integer.valueOf(size), Integer.valueOf(request.getRequestID()))); results.collect(r); } } final long diff = System.currentTimeMillis() - start; if ((searchTimeout -= diff) <= 0) break; } }
From source file:org.rhq.plugins.agent.AgentJavaServiceWrapperDiscoveryComponent.java
/** * Looks for the JSW relative to the agent home directory. * * @param context// w ww .j av a 2s. co m * @param version * @param baseName * @param discoveries where the new details are stored if the JSW is discovered * * @return <code>true</code> if this method discovers the JSW; <code>false</code> if not */ private boolean findInAgentHome(ResourceDiscoveryContext<AgentServerComponent<?>> context, String version, String baseName, HashSet<DiscoveredResourceDetails> discoveries) { try { EmsAttribute home = context.getParentResourceComponent().getAgentBean() .getAttribute("AgentHomeDirectory"); home.refresh(); Object agentHome = home.getValue(); if (agentHome != null) { File file = new File(agentHome.toString(), baseName); if (file.exists()) { discoveries.add(createDetails(context, version, file)); } } return discoveries.size() > 0; } catch (Exception e) { log.debug("Cannot use agent home to find JSW. Cause: " + e); return false; } }
From source file:com.trigger_context.Main_Service.java
public void senderSync(DataInputStream in, DataOutputStream out, String folder) { String tfolder = folder + (folder.charAt(folder.length() - 1) == '/' ? "" : "/"); File f = new File(folder); File file[] = f.listFiles();//from w w w. ja va 2s.c o m // noti(file.toString(),""); String md5 = null; HashMap<String, File> hm = new HashMap<String, File>(); HashSet<String> A = new HashSet<String>(); for (File element : file) { hm.put(md5 = calculateMD5(element), element); A.add(md5); } // noti(hm.toString(),""); int numB = 0; try { numB = in.readInt(); } catch (IOException e) { // TODO Auto-generated catch block noti("error reading 1st int in sendersync", ""); e.printStackTrace(); } HashSet<String> B = new HashSet<String>(); for (int i = 0; i < numB; i++) { try { B.add(in.readUTF()); } catch (IOException e1) { noti("error in readins md5", ""); e1.printStackTrace(); } } HashSet<String> aMb = new HashSet<String>(A); aMb.removeAll(B); int l1 = aMb.size(); try { out.writeInt(l1); } catch (IOException e) { // TODO Auto-generated catch block noti("error in writing 1st int", ""); e.printStackTrace(); } Iterator<String> itr = aMb.iterator(); while (itr.hasNext()) { f = hm.get(itr.next()); sendFile(out, f.getPath()); } HashSet<String> bMa = new HashSet<String>(B); bMa.removeAll(A); int l2 = bMa.size(); try { out.writeInt(l2); } catch (IOException e) { // TODO Auto-generated catch block noti("error in writing 2nd int", ""); e.printStackTrace(); } itr = bMa.iterator(); while (itr.hasNext()) { md5 = itr.next(); try { out.writeUTF(md5); } catch (IOException e) { // TODO Auto-generated catch block noti("error in sending md5", ""); e.printStackTrace(); } recvFile(in, folder); } }