List of usage examples for java.util IdentityHashMap IdentityHashMap
public IdentityHashMap(Map<? extends K, ? extends V> m)
From source file:gridool.db.sql.ParallelSQLExecJob.java
public Map<GridTask, GridNode> map(final GridRouter router, final JobConf jobConf) throws GridException { final long mapStartTime = System.currentTimeMillis(); // phase #1 preparation (prepare tables) DistributionCatalog catalog = registry.getDistributionCatalog(); final SQLTranslator translator = new SQLTranslator(catalog); final String mapQuery = translator.translateQuery(jobConf.mapQuery); final GridNode[] masters = catalog.getMasters(DistributionCatalog.defaultDistributionKey); router.resolve(masters);//from www . ja v a 2s . c om final int numNodes = masters.length; if (numNodes == 0) { return Collections.emptyMap(); } DBAccessor dba = registry.getDbAccessor(); LockManager lockMgr = registry.getLockManager(); GridNode localNode = config.getLocalNode(); ReadWriteLock rwlock = lockMgr.obtainLock(DBAccessor.SYS_TABLE_SYMBOL); final String outputName = jobConf.getOutputName(); runPreparation(dba, mapQuery, masters, localNode, rwlock, outputName); // phase #2 map tasks final InetAddress localHostAddr = NetUtils.getLocalHost(); final int port = config.getFileReceiverPort(); final Map<GridTask, GridNode> map = new IdentityHashMap<GridTask, GridNode>(numNodes); final Map<String, ParallelSQLMapTask> reverseMap = new HashMap<String, ParallelSQLMapTask>(numNodes); for (int tasknum = 0; tasknum < numNodes; tasknum++) { GridNode node = masters[tasknum]; String taskTableName = getTaskResultTableName(outputName, tasknum); ParallelSQLMapTask task = new ParallelSQLMapTask(this, node, tasknum, mapQuery, taskTableName, localHostAddr, port, catalog); map.put(task, node); String taskId = task.getTaskId(); reverseMap.put(taskId, task); } this.remainingTasks = reverseMap; this.finishedNodes = new HashSet<GridNode>(numNodes); this.outputName = outputName; this.createMergeViewDDL = constructMergeViewDDL(masters, outputName); this.reduceQuery = getReduceQuery(jobConf.reduceQuery, outputName, translator); this.outputMethod = jobConf.getOutputMethod(); this.destroyQuery = constructDestroyQuery(masters, outputName, outputMethod); this.mapStarted = System.currentTimeMillis(); this.waitForStartSpeculativeTask = (numNodes <= 1) ? -1L : jobConf.getWaitForStartSpeculativeTask(); this.thresholdForSpeculativeTask = Math.max(1, (int) (numNodes * jobConf.getInvokeSpeculativeTasksFactor())); this.failoverActive = jobConf.isFailoverActive(); this.copyintoExecs = ExecutorFactory.newFixedThreadPool(jobConf.getCopyIntoTableConcurrency(), "CopyIntoTableThread", true); this.timings = new Timings(mapStartTime, numNodes); if (LOG.isDebugEnabled()) { LOG.debug("\n[Map query]\n" + mapQuery + "\n[Reduce query]\n" + reduceQuery); } return map; }
From source file:org.logic2j.core.impl.DefaultTermAdapter.java
/** * Allow changing default behaviour of {@link #toTermInternal(Object, org.logic2j.core.api.TermAdapter.FactoryMode)} when FactoryMode is ATOM, * and the first argument exactly matches one of the keys of the map: will return the value * as the atom./* www . j av a2 s . c o m*/ * * @param theAtoms */ public void setPredefinedAtoms(Map<String, Object> theAtoms) { this.predefinedAtoms = new IdentityHashMap<String, Object>(theAtoms.size()); for (final Entry<String, Object> entry : theAtoms.entrySet()) { this.predefinedAtoms.put(entry.getKey().intern(), entry.getValue()); } }
From source file:xbird.storage.DbCollection.java
public Map<String, DTMDocument> listDocuments(String filterExp, boolean lazy, DynamicContext dynEnv) throws DbException { final Collection<File> files = FileUtils.listFiles(getDirectory(), new String[] { IDocumentTable.DTM_SEGMENT_FILE_SUFFIX }, false); final Map<String, DTMDocument> colls = new IdentityHashMap<String, DTMDocument>(files.size()); for (File f : files) { String fname = FileUtils.getFileName(f); String dname = fname.substring(0, fname.lastIndexOf('.')); if (filterExp != null && !dname.matches(filterExp)) { continue; }/* w ww . ja v a 2 s.c o m*/ final DTMDocument doc; if (lazy) { doc = new LazyDTMDocument(dname, this, dynEnv); } else { doc = getDocument(null, dname, dynEnv); } colls.put(dname, doc); } return colls; }
From source file:gridool.db.partitioning.csv.CsvHashPartitioningJob.java
public Map<GridTask, GridNode> map(final GridTaskRouter router, final PartitioningJobConf ops) throws GridException { assert (registry != null); final String[] lines = ops.getLines(); final String csvFileName = ops.getFileName(); final boolean append = !ops.isFirst(); final DBPartitioningJobConf jobConf = ops.getJobConf(); // partitioning resources final String tableName; final int tablePartitionNo; final PrimaryKey primaryKey; final Collection<ForeignKey> foreignKeys; final int[] pkeyIndicies; final String[] parentTableFkIndexNames; final int numParentForeignKeys; final boolean hasParentTable; final int numForeignKeys; final String[] fkIdxNames; final int[][] fkPositions; final int[] childTablesPartitionNo; final LRUMap<String, List<NodeWithPartitionNo>>[] fkCaches; {/* w ww . ja v a 2 s . c o m*/ tableName = jobConf.getTableName(); DistributionCatalog catalog = registry.getDistributionCatalog(); tablePartitionNo = catalog.getTablePartitionNo(tableName, true); Pair<PrimaryKey, Collection<ForeignKey>> primaryForeignKeys = ops.getPrimaryForeignKeys(); primaryKey = primaryForeignKeys.getFirst(); foreignKeys = primaryForeignKeys.getSecond(); pkeyIndicies = primaryKey.getColumnPositions(true); parentTableFkIndexNames = getParentTableFkIndexNames(primaryKey); hasParentTable = (parentTableFkIndexNames != null); numParentForeignKeys = hasParentTable ? parentTableFkIndexNames.length : 0; numForeignKeys = foreignKeys.size(); assert (numParentForeignKeys != 0); fkIdxNames = getFkIndexNames(foreignKeys, numForeignKeys); fkPositions = getFkPositions(foreignKeys, numForeignKeys); childTablesPartitionNo = getChildTablesPartitionNo(foreignKeys, numForeignKeys, catalog); fkCaches = getFkIndexCaches(numForeignKeys); } // COPY INTO control resources final char filedSeparator = jobConf.getFieldSeparator(); final char quoteChar = jobConf.getStringQuote(); // working resources final ILocalDirectory index = registry.getDirectory(); for (String idxName : fkIdxNames) { setFkIndexCacheSizes(index, idxName, FK_INDEX_CACHE_SIZE); } final String[] fields = new String[getMaxColumnCount(primaryKey, foreignKeys)]; assert (fields.length > 0); final FixedArrayList<String> fieldList = new FixedArrayList<String>(fields); final Charset charset = Charset.forName("UTF-8"); final StringBuilder strBuf = new StringBuilder(64); final int totalRecords = lines.length; final String[] fkeysFields = new String[numForeignKeys]; final byte[][] distkeys = new byte[numForeignKeys][]; final GridNode[] fkMappedNodes = new GridNode[numForeignKeys]; final int numNodes = router.getGridSize(); final Map<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> nodeAssignMap = new HashMap<GridNode, Pair<MutableInt, FastByteArrayOutputStream>>( numNodes); final Map<GridNode, MutableInt> mappedNodes = new HashMap<GridNode, MutableInt>(numNodes); for (int i = 0; i < totalRecords; i++) { String line = lines[i]; lines[i] = null; final byte[] lineBytes = line.getBytes(charset); { CsvUtils.retrieveFields(line, pkeyIndicies, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String pkeysField = combineFields(fields, pkeyIndicies.length, strBuf); // "primary" fragment mapping final byte[] distkey = StringUtils.getBytes(pkeysField); mapPrimaryFragment(distkey, mappedNodes, tablePartitionNo, router); if (hasParentTable) { // "derived by parent" fragment mapping for (int kk = 0; kk < numParentForeignKeys; kk++) { String idxName = parentTableFkIndexNames[kk]; mapDerivedByParentFragment(distkey, mappedNodes, index, idxName); } } } if (numForeignKeys > 0) { // "derived by child" fragment mapping for (int jj = 0; jj < numForeignKeys; jj++) { int[] pos = fkPositions[jj]; CsvUtils.retrieveFields(line, pos, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String fkeysField = combineFields(fields, pos.length, strBuf); byte[] distkey = StringUtils.getBytes(fkeysField); fkMappedNodes[jj] = mapDerivedByChildFragment(distkey, mappedNodes, childTablesPartitionNo[jj], router); fkeysFields[jj] = fkeysField; distkeys[jj] = distkey; } // store information for derived fragment mapping for (int kk = 0; kk < numForeignKeys; kk++) { final String fkIdxName = fkIdxNames[kk]; final String fkeysField = fkeysFields[kk]; final byte[] distkey = distkeys[kk]; final GridNode fkMappedNode = fkMappedNodes[kk]; final LRUMap<String, List<NodeWithPartitionNo>> fkCache = fkCaches[kk]; List<NodeWithPartitionNo> storedNodeInfo = fkCache.get(fkeysField); for (final Map.Entry<GridNode, MutableInt> e : mappedNodes.entrySet()) { final GridNode node = e.getKey(); if (node.equals(fkMappedNode)) { continue; // no need to map } int hiddenValue = e.getValue().intValue(); NodeWithPartitionNo nodeInfo = new NodeWithPartitionNo(node, hiddenValue); if (storedNodeInfo == null) { storedNodeInfo = new ArrayList<NodeWithPartitionNo>(8); fkCache.put(fkeysField, storedNodeInfo); } else if (storedNodeInfo.contains(nodeInfo)) {// Note that node has unique hiddenValue to persist continue; } storedNodeInfo.add(nodeInfo); byte[] value = serialize(node, hiddenValue); storeDerivedFragmentationInfo(distkey, value, index, fkIdxName); } } } if (mappedNodes.isEmpty()) { throw new IllegalStateException("Could not map records for table: '" + tableName + '\''); } // bind a record to nodes mapRecord(lineBytes, totalRecords, numNodes, nodeAssignMap, mappedNodes, filedSeparator); mappedNodes.clear(); } final Map<GridTask, GridNode> taskmap = new IdentityHashMap<GridTask, GridNode>(numNodes); final Map<GridNode, MutableInt> assignedRecMap = new HashMap<GridNode, MutableInt>(numNodes); for (final Map.Entry<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> e : nodeAssignMap.entrySet()) { GridNode node = e.getKey(); Pair<MutableInt, FastByteArrayOutputStream> pair = e.getValue(); MutableInt numRecords = pair.first; assignedRecMap.put(node, numRecords); FastByteArrayOutputStream rows = pair.second; byte[] b = rows.toByteArray(); pair.clear(); GridTask task = new FileAppendTask(this, csvFileName, b, append, true); taskmap.put(task, node); } for (final GridNode node : router.getAllNodes()) { if (!assignedRecMap.containsKey(node)) { assignedRecMap.put(node, new MutableInt(0)); } } this.assignedRecMap = assignedRecMap; return taskmap; }
From source file:gridool.db.partitioning.phihash.csv.distmm.InMemoryLocalCsvHashPartitioningJob.java
public Map<GridTask, GridNode> map(final GridRouter router, final Pair<PartitioningJobConf, GridRouter> args) throws GridException { assert (registry != null); final PartitioningJobConf ops = args.getFirst(); final String[] lines = ops.getLines(); final String csvFileName = ops.getFileName(); final DBPartitioningJobConf jobConf = ops.getJobConf(); final GridRouter origRouter = args.getSecond(); // partitioning resources final String tableName; final int tablePartitionNo; final PrimaryKey primaryKey; final Collection<ForeignKey> foreignKeys; final int[] pkeyIndicies; final String[] parentTableFkIndexNames; final int numParentForeignKeys; final boolean hasParentTable; final int numForeignKeys; final String[] fkIdxNames; final int[][] fkPositions; final int[] childTablesPartitionNo; final LRUMap<String, List<NodeWithPartitionNo>>[] fkCaches; {/*from w w w .j a va2 s. c om*/ tableName = jobConf.getTableName(); DistributionCatalog catalog = registry.getDistributionCatalog(); tablePartitionNo = catalog.getTablePartitionNo(tableName, true); Pair<PrimaryKey, Collection<ForeignKey>> primaryForeignKeys = ops.getPrimaryForeignKeys(); primaryKey = primaryForeignKeys.getFirst(); foreignKeys = primaryForeignKeys.getSecond(); pkeyIndicies = primaryKey.getColumnPositions(true); parentTableFkIndexNames = GridDbUtils.getParentTableFkIndexNames(primaryKey); hasParentTable = (parentTableFkIndexNames != null); numParentForeignKeys = hasParentTable ? parentTableFkIndexNames.length : 0; numForeignKeys = foreignKeys.size(); assert (numParentForeignKeys != 0); fkIdxNames = GridDbUtils.getFkIndexNames(foreignKeys, numForeignKeys); fkPositions = GridDbUtils.getFkPositions(foreignKeys, numForeignKeys); childTablesPartitionNo = GridDbUtils.getChildTablesPartitionNo(foreignKeys, numForeignKeys, catalog); fkCaches = GridDbUtils.getFkIndexCaches(numForeignKeys); } // COPY INTO control resources final char filedSeparator = jobConf.getFieldSeparator(); final char quoteChar = jobConf.getStringQuote(); // working resources final InMemoryMappingIndex index = registry.getMappingIndex(); if (parentTableFkIndexNames != null) { prepareIndex(index, parentTableFkIndexNames); } final GridNode localNode = config.getLocalNode(); final String[] fields = new String[GridDbUtils.getMaxColumnCount(primaryKey, foreignKeys)]; assert (fields.length > 0); final FixedArrayList<String> fieldList = new FixedArrayList<String>(fields); final Charset charset = Charset.forName("UTF-8"); final StringBuilder strBuf = new StringBuilder(64); final int totalRecords = lines.length; final String[] fkeysFields = new String[numForeignKeys]; final GridNode[] fkMappedNodes = new GridNode[numForeignKeys]; final int numNodes = origRouter.getGridSize(); final Map<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> nodeAssignMap = new HashMap<GridNode, Pair<MutableInt, FastByteArrayOutputStream>>( numNodes); final Map<GridNode, MutableInt> mappedNodes = new HashMap<GridNode, MutableInt>(numNodes); final Map<GridNode, List<DerivedFragmentInfo>> idxShippingMap = new HashMap<GridNode, List<DerivedFragmentInfo>>( numNodes); for (int i = 0; i < totalRecords; i++) { String line = lines[i]; lines[i] = null; // "primary" fragment mapping mapPrimaryFragment(mappedNodes, tablePartitionNo, localNode); if (hasParentTable) { // "derived by parent" fragment mapping CsvUtils.retrieveFields(line, pkeyIndicies, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String pkeysField = GridDbUtils.combineFields(fields, pkeyIndicies.length, strBuf); for (int kk = 0; kk < numParentForeignKeys; kk++) { String idxName = parentTableFkIndexNames[kk]; mapDerivedByParentFragment(pkeysField, mappedNodes, index, idxName); } } if (numForeignKeys > 0) { // "derived by child" fragment mapping for (int jj = 0; jj < numForeignKeys; jj++) { int[] pos = fkPositions[jj]; CsvUtils.retrieveFields(line, pos, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String fkeysField = GridDbUtils.combineFields(fields, pos.length, strBuf); byte[] distkey = StringUtils.getBytes(fkeysField); fkMappedNodes[jj] = mapDerivedByChildFragment(distkey, mappedNodes, childTablesPartitionNo[jj], origRouter); fkeysFields[jj] = fkeysField; } // store information for derived fragment mapping for (int kk = 0; kk < numForeignKeys; kk++) { final String fkIdxName = fkIdxNames[kk]; final String fkeysField = fkeysFields[kk]; final GridNode fkMappedNode = fkMappedNodes[kk]; List<DerivedFragmentInfo> shipIdxList = idxShippingMap.get(fkMappedNode); if (shipIdxList == null) { shipIdxList = new ArrayList<DerivedFragmentInfo>(2000); idxShippingMap.put(fkMappedNode, shipIdxList); } final LRUMap<String, List<NodeWithPartitionNo>> fkCache = fkCaches[kk]; List<NodeWithPartitionNo> storedNodeInfo = fkCache.get(fkeysField); for (final Map.Entry<GridNode, MutableInt> e : mappedNodes.entrySet()) { GridNode node = e.getKey(); int hiddenValue = e.getValue().intValue(); NodeWithPartitionNo nodeInfo = new NodeWithPartitionNo(node, hiddenValue); if (storedNodeInfo == null) { storedNodeInfo = new ArrayList<NodeWithPartitionNo>(8); fkCache.put(fkeysField, storedNodeInfo); } else if (storedNodeInfo.contains(nodeInfo)) {// Note that node has unique hiddenValue to persist continue; } storedNodeInfo.add(nodeInfo); // index shipping DerivedFragmentInfo fragInfo = new DerivedFragmentInfo(fkIdxName, fkeysField, nodeInfo); shipIdxList.add(fragInfo); } } } if (mappedNodes.isEmpty()) { throw new IllegalStateException("Could not map records for table: '" + tableName + '\''); } // bind a record to nodes byte[] lineBytes = line.getBytes(charset); mapRecord(lineBytes, totalRecords, numNodes, nodeAssignMap, mappedNodes, filedSeparator); mappedNodes.clear(); } final int numTasks = idxShippingMap.size(); final Map<GridTask, GridNode> taskmap = new IdentityHashMap<GridTask, GridNode>(numTasks); for (final Map.Entry<GridNode, List<DerivedFragmentInfo>> e : idxShippingMap.entrySet()) { GridNode node = e.getKey(); List<DerivedFragmentInfo> storeList = e.getValue(); GridTask task = new InMemoryBuildIndexTask(this, storeList); taskmap.put(task, node); } final ExecutorService sendExecs = ExecutorFactory.newFixedThreadPool(GridXferClient.SENDER_CONCURRENCY, "FileSender", true); final GridXferClient dfsClient = registry.getDfsService().getDFSClient(); final int recvPort = config.getFileReceiverPort(); final HashMap<GridNode, MutableInt> assignedRecMap = new HashMap<GridNode, MutableInt>(numNodes); for (final Map.Entry<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> e : nodeAssignMap.entrySet()) { GridNode node = e.getKey(); GridNode trgNode = origRouter.resolve(node); Pair<MutableInt, FastByteArrayOutputStream> pair = e.getValue(); MutableInt numRecords = pair.first; assignedRecMap.put(trgNode, numRecords); FastByteArrayOutputStream rows = pair.second; pair.clear(); GridDbUtils.sendfile(sendExecs, dfsClient, csvFileName, rows, trgNode, recvPort); } for (final GridNode node : origRouter.getAllNodes()) { if (!assignedRecMap.containsKey(node)) { assignedRecMap.put(node, new MutableInt(0)); } } this.assignedRecMap = assignedRecMap; ExecutorUtils.shutdownAndAwaitTermination(sendExecs); return taskmap; }
From source file:gridool.db.partitioning.phihash.csv.normal.CsvHashPartitioningJob.java
public Map<GridTask, GridNode> map(final GridRouter router, final PartitioningJobConf ops) throws GridException { assert (registry != null); final String[] lines = ops.getLines(); final String csvFileName = ops.getFileName(); final DBPartitioningJobConf jobConf = ops.getJobConf(); // partitioning resources final String tableName; final int tablePartitionNo; final PrimaryKey primaryKey; final Collection<ForeignKey> foreignKeys; final int[] pkeyIndicies; final String[] parentTableFkIndexNames; final int numParentForeignKeys; final boolean hasParentTable; final int numForeignKeys; final String[] fkIdxNames; final int[][] fkPositions; final int[] childTablesPartitionNo; final LRUMap<String, List<NodeWithPartitionNo>>[] fkCaches; {// w ww .ja v a 2 s . co m tableName = jobConf.getTableName(); DistributionCatalog catalog = registry.getDistributionCatalog(); tablePartitionNo = catalog.getTablePartitionNo(tableName, true); Pair<PrimaryKey, Collection<ForeignKey>> primaryForeignKeys = ops.getPrimaryForeignKeys(); primaryKey = primaryForeignKeys.getFirst(); foreignKeys = primaryForeignKeys.getSecond(); pkeyIndicies = primaryKey.getColumnPositions(true); parentTableFkIndexNames = GridDbUtils.getParentTableFkIndexNames(primaryKey); hasParentTable = (parentTableFkIndexNames != null); numParentForeignKeys = hasParentTable ? parentTableFkIndexNames.length : 0; numForeignKeys = foreignKeys.size(); assert (numParentForeignKeys != 0); fkIdxNames = GridDbUtils.getFkIndexNames(foreignKeys, numForeignKeys); fkPositions = GridDbUtils.getFkPositions(foreignKeys, numForeignKeys); childTablesPartitionNo = GridDbUtils.getChildTablesPartitionNo(foreignKeys, numForeignKeys, catalog); fkCaches = GridDbUtils.getFkIndexCaches(numForeignKeys); } // COPY INTO control resources final char filedSeparator = jobConf.getFieldSeparator(); final char quoteChar = jobConf.getStringQuote(); // working resources final ILocalDirectory index = registry.getDirectory(); for (String idxName : fkIdxNames) { setFkIndexCacheSizes(index, idxName, FK_INDEX_CACHE_SIZE); } final String[] fields = new String[GridDbUtils.getMaxColumnCount(primaryKey, foreignKeys)]; assert (fields.length > 0); final FixedArrayList<String> fieldList = new FixedArrayList<String>(fields); final Charset charset = Charset.forName("UTF-8"); final StringBuilder strBuf = new StringBuilder(64); final int totalRecords = lines.length; final String[] fkeysFields = new String[numForeignKeys]; final byte[][] distkeys = new byte[numForeignKeys][]; final GridNode[] fkMappedNodes = new GridNode[numForeignKeys]; final int numNodes = router.getGridSize(); final Map<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> nodeAssignMap = new HashMap<GridNode, Pair<MutableInt, FastByteArrayOutputStream>>( numNodes); final Map<GridNode, MutableInt> mappedNodes = new HashMap<GridNode, MutableInt>(numNodes); for (int i = 0; i < totalRecords; i++) { String line = lines[i]; lines[i] = null; final byte[] lineBytes = line.getBytes(charset); { CsvUtils.retrieveFields(line, pkeyIndicies, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String pkeysField = GridDbUtils.combineFields(fields, pkeyIndicies.length, strBuf); // "primary" fragment mapping final byte[] distkey = StringUtils.getBytes(pkeysField); mapPrimaryFragment(distkey, mappedNodes, tablePartitionNo, router); if (hasParentTable) { // "derived by parent" fragment mapping for (int kk = 0; kk < numParentForeignKeys; kk++) { String idxName = parentTableFkIndexNames[kk]; mapDerivedByParentFragment(distkey, mappedNodes, index, idxName); } } } if (numForeignKeys > 0) { // "derived by child" fragment mapping for (int jj = 0; jj < numForeignKeys; jj++) { int[] pos = fkPositions[jj]; CsvUtils.retrieveFields(line, pos, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String fkeysField = GridDbUtils.combineFields(fields, pos.length, strBuf); byte[] distkey = StringUtils.getBytes(fkeysField); fkMappedNodes[jj] = mapDerivedByChildFragment(distkey, mappedNodes, childTablesPartitionNo[jj], router); fkeysFields[jj] = fkeysField; distkeys[jj] = distkey; } // store information for derived fragment mapping for (int kk = 0; kk < numForeignKeys; kk++) { final String fkIdxName = fkIdxNames[kk]; final String fkeysField = fkeysFields[kk]; final byte[] distkey = distkeys[kk]; final GridNode fkMappedNode = fkMappedNodes[kk]; final LRUMap<String, List<NodeWithPartitionNo>> fkCache = fkCaches[kk]; List<NodeWithPartitionNo> storedNodeInfo = fkCache.get(fkeysField); for (final Map.Entry<GridNode, MutableInt> e : mappedNodes.entrySet()) { final GridNode node = e.getKey(); if (node.equals(fkMappedNode)) { continue; // no need to map } int hiddenValue = e.getValue().intValue(); NodeWithPartitionNo nodeInfo = new NodeWithPartitionNo(node, hiddenValue); if (storedNodeInfo == null) { storedNodeInfo = new ArrayList<NodeWithPartitionNo>(8); fkCache.put(fkeysField, storedNodeInfo); } else if (storedNodeInfo.contains(nodeInfo)) {// Note that node has unique hiddenValue to persist continue; } storedNodeInfo.add(nodeInfo); byte[] value = NodeWithPartitionNo.serialize(node, hiddenValue); storeDerivedFragmentationInfo(distkey, value, index, fkIdxName); } } } if (mappedNodes.isEmpty()) { throw new IllegalStateException("Could not map records for table: '" + tableName + '\''); } // bind a record to nodes mapRecord(lineBytes, totalRecords, numNodes, nodeAssignMap, mappedNodes, filedSeparator); mappedNodes.clear(); } final ExecutorService sendExecs = ExecutorFactory.newFixedThreadPool(GridXferClient.SENDER_CONCURRENCY, "FileSender", true); final GridXferClient dfsClient = registry.getDfsService().getDFSClient(); final int recvPort = config.getFileReceiverPort(); final Map<GridTask, GridNode> taskmap = new IdentityHashMap<GridTask, GridNode>(numNodes); final Map<GridNode, MutableInt> assignedRecMap = new HashMap<GridNode, MutableInt>(numNodes); for (final Map.Entry<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> e : nodeAssignMap.entrySet()) { GridNode node = e.getKey(); GridNode trgNode = router.resolve(node); Pair<MutableInt, FastByteArrayOutputStream> pair = e.getValue(); MutableInt numRecords = pair.first; assignedRecMap.put(trgNode, numRecords); FastByteArrayOutputStream rows = pair.second; pair.clear(); GridDbUtils.sendfile(sendExecs, dfsClient, csvFileName, rows, trgNode, recvPort); } for (final GridNode node : router.getAllNodes()) { if (!assignedRecMap.containsKey(node)) { assignedRecMap.put(node, new MutableInt(0)); } } this.assignedRecMap = assignedRecMap; ExecutorUtils.shutdownAndAwaitTermination(sendExecs); return taskmap; }
From source file:gridool.db.partitioning.phihash.csv.grace.CsvGraceHashPartitioningJob.java
public Map<GridTask, GridNode> map(final GridRouter router, final PartitioningJobConf ops) throws GridException { assert (registry != null); final String[] lines = ops.getLines(); final String csvFileName = ops.getFileName(); final DBPartitioningJobConf jobConf = ops.getJobConf(); final int numBuckets = jobConf.getNumberOfBuckets(); if (numBuckets <= 0) { throw new GridException("Illegal number of buckets for grace hash partitioning: " + numBuckets); }/*ww w .ja v a 2s . co m*/ if (!HashUtils.isPowerOfTwo(numBuckets)) { throw new GridException("number of buckets is not power of two: " + numBuckets); } final int bucketShift = HashUtils.shiftsForNextPowerOfTwo(numBuckets); // partitioning resources final String tableName; final int tablePartitionNo; final PrimaryKey primaryKey; final Collection<ForeignKey> foreignKeys; final int[] pkeyIndicies; final String[] parentTableFkIndexNames; final int numParentForeignKeys; final boolean hasParentTable; final int numForeignKeys; final String[] fkIdxNames; final int[][] fkPositions; final int[] childTablesPartitionNo; final LRUMap<String, List<NodeWithPartitionNo>>[] fkCaches; { tableName = jobConf.getTableName(); DistributionCatalog catalog = registry.getDistributionCatalog(); tablePartitionNo = catalog.getTablePartitionNo(tableName, true); Pair<PrimaryKey, Collection<ForeignKey>> primaryForeignKeys = ops.getPrimaryForeignKeys(); primaryKey = primaryForeignKeys.getFirst(); foreignKeys = primaryForeignKeys.getSecond(); pkeyIndicies = primaryKey.getColumnPositions(true); parentTableFkIndexNames = GridDbUtils.getParentTableFkIndexNames(primaryKey); hasParentTable = (parentTableFkIndexNames != null); numParentForeignKeys = hasParentTable ? parentTableFkIndexNames.length : 0; numForeignKeys = foreignKeys.size(); assert (numParentForeignKeys != 0); fkIdxNames = GridDbUtils.getFkIndexNames(foreignKeys, numForeignKeys); fkPositions = GridDbUtils.getFkPositions(foreignKeys, numForeignKeys); childTablesPartitionNo = GridDbUtils.getChildTablesPartitionNo(foreignKeys, numForeignKeys, catalog); fkCaches = GridDbUtils.getFkIndexCaches(numForeignKeys); } // COPY INTO control resources final char filedSeparator = jobConf.getFieldSeparator(); final char quoteChar = jobConf.getStringQuote(); // working resources final String[] fields = new String[GridDbUtils.getMaxColumnCount(primaryKey, foreignKeys)]; assert (fields.length > 0); final FixedArrayList<String> fieldList = new FixedArrayList<String>(fields); final Charset charset = Charset.forName("UTF-8"); final StringBuilder strBuf = new StringBuilder(64); final int totalRecords = lines.length; final String[] fkeysFields = new String[numForeignKeys]; final GridNode[] fkMappedNodes = new GridNode[numForeignKeys]; // loading indices for the buckets int bucket = ops.getBucketNumber(); final InMemoryMappingIndex index = hasParentTable ? InMemoryIndexHelper.loadIndex(bucket, parentTableFkIndexNames, registry) : null; final Map<String, OutputStream> outputMap = ops.getOutputMap(); final int numNodes = router.getGridSize(); final Map<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> nodeAssignMap = new HashMap<GridNode, Pair<MutableInt, FastByteArrayOutputStream>>( numNodes); final Map<GridNode, MutableInt> mappedNodes = new HashMap<GridNode, MutableInt>(numNodes); for (int i = 0; i < totalRecords; i++) { String line = lines[i]; lines[i] = null; final byte[] lineBytes = line.getBytes(charset); { CsvUtils.retrieveFields(line, pkeyIndicies, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String pkeysField = GridDbUtils.combineFields(fields, pkeyIndicies.length, strBuf); // "primary" fragment mapping mapPrimaryFragment(pkeysField, mappedNodes, tablePartitionNo, router); if (hasParentTable) { // "derived by parent" fragment mapping for (int kk = 0; kk < numParentForeignKeys; kk++) { String idxName = parentTableFkIndexNames[kk]; mapDerivedByParentFragment(pkeysField, mappedNodes, index, idxName); } } } if (numForeignKeys > 0) { // "derived by child" fragment mapping for (int jj = 0; jj < numForeignKeys; jj++) { int[] pos = fkPositions[jj]; CsvUtils.retrieveFields(line, pos, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String fkeysField = GridDbUtils.combineFields(fields, pos.length, strBuf); byte[] distkey = StringUtils.getBytes(fkeysField); fkMappedNodes[jj] = mapDerivedByChildFragment(distkey, mappedNodes, childTablesPartitionNo[jj], router); fkeysFields[jj] = fkeysField; } // store information for derived fragment mapping for (int kk = 0; kk < numForeignKeys; kk++) { final String fkIdxName = fkIdxNames[kk]; final String fkeysField = fkeysFields[kk]; final GridNode fkMappedNode = fkMappedNodes[kk]; final LRUMap<String, List<NodeWithPartitionNo>> fkCache = fkCaches[kk]; List<NodeWithPartitionNo> storedNodeInfo = fkCache.get(fkeysField); for (final Map.Entry<GridNode, MutableInt> e : mappedNodes.entrySet()) { final GridNode node = e.getKey(); if (node.equals(fkMappedNode)) { continue; // no need to map } int hiddenValue = e.getValue().intValue(); NodeWithPartitionNo nodeInfo = new NodeWithPartitionNo(node, hiddenValue); if (storedNodeInfo == null) { storedNodeInfo = new ArrayList<NodeWithPartitionNo>(8); fkCache.put(fkeysField, storedNodeInfo); } else if (storedNodeInfo.contains(nodeInfo)) {// Note that node has unique hiddenValue to persist continue; } storedNodeInfo.add(nodeInfo); try { InMemoryIndexHelper.writeToFile(fkeysField, nodeInfo, fkIdxName, outputMap, bucketShift); } catch (IOException ioe) { throw new GridException(ioe); } } } } if (mappedNodes.isEmpty()) { throw new IllegalStateException("Could not map records for table: '" + tableName + '\''); } // bind a record to nodes mapRecord(lineBytes, totalRecords, numNodes, nodeAssignMap, mappedNodes, filedSeparator); mappedNodes.clear(); } final ExecutorService sendExecs = ExecutorFactory.newFixedThreadPool(GridXferClient.SENDER_CONCURRENCY, "FileSender", true); final GridXferClient dfsClient = registry.getDfsService().getDFSClient(); final int recvPort = config.getFileReceiverPort(); final Map<GridTask, GridNode> taskmap = new IdentityHashMap<GridTask, GridNode>(numNodes); final Map<GridNode, MutableInt> assignedRecMap = new HashMap<GridNode, MutableInt>(numNodes); for (final Map.Entry<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> e : nodeAssignMap.entrySet()) { GridNode node = e.getKey(); GridNode trgNode = router.resolve(node); Pair<MutableInt, FastByteArrayOutputStream> pair = e.getValue(); MutableInt numRecords = pair.first; assignedRecMap.put(trgNode, numRecords); FastByteArrayOutputStream rows = pair.second; pair.clear(); GridDbUtils.sendfile(sendExecs, dfsClient, csvFileName, rows, trgNode, recvPort); } for (final GridNode node : router.getAllNodes()) { if (!assignedRecMap.containsKey(node)) { assignedRecMap.put(node, new MutableInt(0)); } } this.assignedRecMap = assignedRecMap; ExecutorUtils.shutdownAndAwaitTermination(sendExecs); return taskmap; }
From source file:gridool.db.partitioning.phihash.csv.dist.LocalCsvHashPartitioningJob.java
public Map<GridTask, GridNode> map(final GridRouter router, final Pair<PartitioningJobConf, GridRouter> args) throws GridException { assert (registry != null); final PartitioningJobConf ops = args.getFirst(); final String[] lines = ops.getLines(); final String csvFileName = ops.getFileName(); final DBPartitioningJobConf jobConf = ops.getJobConf(); final GridRouter origRouter = args.getSecond(); // partitioning resources final String tableName; final int tablePartitionNo; final PrimaryKey primaryKey; final Collection<ForeignKey> foreignKeys; final int[] pkeyIndicies; final String[] parentTableFkIndexNames; final int numParentForeignKeys; final boolean hasParentTable; final int numForeignKeys; final String[] fkIdxNames; final int[][] fkPositions; final int[] childTablesPartitionNo; final LRUMap<String, List<NodeWithPartitionNo>>[] fkCaches; {/*from w w w. ja v a 2 s . c o m*/ tableName = jobConf.getTableName(); DistributionCatalog catalog = registry.getDistributionCatalog(); tablePartitionNo = catalog.getTablePartitionNo(tableName, true); Pair<PrimaryKey, Collection<ForeignKey>> primaryForeignKeys = ops.getPrimaryForeignKeys(); primaryKey = primaryForeignKeys.getFirst(); foreignKeys = primaryForeignKeys.getSecond(); pkeyIndicies = primaryKey.getColumnPositions(true); parentTableFkIndexNames = GridDbUtils.getParentTableFkIndexNames(primaryKey); hasParentTable = (parentTableFkIndexNames != null); numParentForeignKeys = hasParentTable ? parentTableFkIndexNames.length : 0; numForeignKeys = foreignKeys.size(); assert (numParentForeignKeys != 0); fkIdxNames = GridDbUtils.getFkIndexNames(foreignKeys, numForeignKeys); fkPositions = GridDbUtils.getFkPositions(foreignKeys, numForeignKeys); childTablesPartitionNo = GridDbUtils.getChildTablesPartitionNo(foreignKeys, numForeignKeys, catalog); fkCaches = GridDbUtils.getFkIndexCaches(numForeignKeys); } // COPY INTO control resources final char filedSeparator = jobConf.getFieldSeparator(); final char quoteChar = jobConf.getStringQuote(); // working resources final ILocalDirectory index = registry.getDirectory(); for (String idxName : fkIdxNames) { configureFkIndex(index, idxName, FK_INDEX_CACHE_SIZE); } final GridNode localNode = config.getLocalNode(); final String[] fields = new String[GridDbUtils.getMaxColumnCount(primaryKey, foreignKeys)]; assert (fields.length > 0); final FixedArrayList<String> fieldList = new FixedArrayList<String>(fields); final Charset charset = Charset.forName("UTF-8"); final StringBuilder strBuf = new StringBuilder(64); final int totalRecords = lines.length; final String[] fkeysFields = new String[numForeignKeys]; final byte[][] distkeys = new byte[numForeignKeys][]; final GridNode[] fkMappedNodes = new GridNode[numForeignKeys]; final int numNodes = origRouter.getGridSize(); final Map<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> nodeAssignMap = new HashMap<GridNode, Pair<MutableInt, FastByteArrayOutputStream>>( numNodes); final Map<GridNode, MutableInt> mappedNodes = new HashMap<GridNode, MutableInt>(numNodes); final Map<GridNode, List<DerivedFragmentInfo>> idxShippingMap = new HashMap<GridNode, List<DerivedFragmentInfo>>( numNodes); for (int i = 0; i < totalRecords; i++) { String line = lines[i]; lines[i] = null; // "primary" fragment mapping mapPrimaryFragment(mappedNodes, tablePartitionNo, localNode); if (hasParentTable) { // "derived by parent" fragment mapping CsvUtils.retrieveFields(line, pkeyIndicies, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String pkeysField = GridDbUtils.combineFields(fields, pkeyIndicies.length, strBuf); final byte[] distkey = StringUtils.getBytes(pkeysField); for (int kk = 0; kk < numParentForeignKeys; kk++) { String idxName = parentTableFkIndexNames[kk]; mapDerivedByParentFragment(distkey, mappedNodes, index, idxName); } } if (numForeignKeys > 0) { // "derived by child" fragment mapping for (int jj = 0; jj < numForeignKeys; jj++) { int[] pos = fkPositions[jj]; CsvUtils.retrieveFields(line, pos, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String fkeysField = GridDbUtils.combineFields(fields, pos.length, strBuf); byte[] distkey = StringUtils.getBytes(fkeysField); fkMappedNodes[jj] = mapDerivedByChildFragment(distkey, mappedNodes, childTablesPartitionNo[jj], origRouter); fkeysFields[jj] = fkeysField; distkeys[jj] = distkey; } // store information for derived fragment mapping for (int kk = 0; kk < numForeignKeys; kk++) { final String fkIdxName = fkIdxNames[kk]; final String fkeysField = fkeysFields[kk]; final byte[] distkey = distkeys[kk]; final GridNode fkMappedNode = fkMappedNodes[kk]; List<DerivedFragmentInfo> shipIdxList = idxShippingMap.get(fkMappedNode); if (shipIdxList == null) { shipIdxList = new ArrayList<DerivedFragmentInfo>(2000); idxShippingMap.put(fkMappedNode, shipIdxList); } final LRUMap<String, List<NodeWithPartitionNo>> fkCache = fkCaches[kk]; List<NodeWithPartitionNo> storedNodeInfo = fkCache.get(fkeysField); for (final Map.Entry<GridNode, MutableInt> e : mappedNodes.entrySet()) { final GridNode node = e.getKey(); //if(node.equals(fkMappedNode)) continue; // no need to map int hiddenValue = e.getValue().intValue(); NodeWithPartitionNo nodeInfo = new NodeWithPartitionNo(node, hiddenValue); if (storedNodeInfo == null) { storedNodeInfo = new ArrayList<NodeWithPartitionNo>(8); fkCache.put(fkeysField, storedNodeInfo); } else if (storedNodeInfo.contains(nodeInfo)) {// Note that node has unique hiddenValue to persist continue; } storedNodeInfo.add(nodeInfo); byte[] value = NodeWithPartitionNo.serialize(node, hiddenValue); // index shipping DerivedFragmentInfo fragInfo = new DerivedFragmentInfo(fkIdxName, distkey, value); shipIdxList.add(fragInfo); } } } if (mappedNodes.isEmpty()) { throw new IllegalStateException("Could not map records for table: '" + tableName + '\''); } // bind a record to nodes byte[] lineBytes = line.getBytes(charset); mapRecord(lineBytes, totalRecords, numNodes, nodeAssignMap, mappedNodes, filedSeparator); mappedNodes.clear(); } final int numTasks = idxShippingMap.size(); final Map<GridTask, GridNode> taskmap = new IdentityHashMap<GridTask, GridNode>(numTasks); for (final Map.Entry<GridNode, List<DerivedFragmentInfo>> e : idxShippingMap.entrySet()) { GridNode node = e.getKey(); List<DerivedFragmentInfo> storeList = e.getValue(); GridTask task = new GridBuildIndexTask(this, storeList); taskmap.put(task, node); } final ExecutorService sendExecs = ExecutorFactory.newFixedThreadPool(GridXferClient.SENDER_CONCURRENCY, "FileSender", true); final GridXferClient dfsClient = registry.getDfsService().getDFSClient(); final int recvPort = config.getFileReceiverPort(); final HashMap<GridNode, MutableInt> assignedRecMap = new HashMap<GridNode, MutableInt>(numNodes); for (final Map.Entry<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> e : nodeAssignMap.entrySet()) { GridNode node = e.getKey(); GridNode trgNode = origRouter.resolve(node); Pair<MutableInt, FastByteArrayOutputStream> pair = e.getValue(); MutableInt numRecords = pair.first; assignedRecMap.put(trgNode, numRecords); FastByteArrayOutputStream rows = pair.second; pair.clear(); GridDbUtils.sendfile(sendExecs, dfsClient, csvFileName, rows, trgNode, recvPort); } for (final GridNode node : origRouter.getAllNodes()) { if (!assignedRecMap.containsKey(node)) { assignedRecMap.put(node, new MutableInt(0)); } } this.assignedRecMap = assignedRecMap; ExecutorUtils.shutdownAndAwaitTermination(sendExecs); return taskmap; }
From source file:gridool.db.partitioning.phihash.csv.pgrace.ParallelGraceLocalCsvHashPartitioningJob.java
public Map<GridTask, GridNode> map(final GridRouter router, final Pair<PartitioningJobConf, GridRouter> args) throws GridException { assert (registry != null); final PartitioningJobConf ops = args.getFirst(); final String[] lines = ops.getLines(); final String csvFileName = ops.getFileName(); final DBPartitioningJobConf jobConf = ops.getJobConf(); final GridRouter origRouter = args.getSecond(); if (lines.length == 0) { throw new GridException("There are no lines to partition"); }//from w ww. j a va 2s .co m final int numBuckets = jobConf.getNumberOfBuckets(); if (numBuckets <= 0) { throw new GridException("Illegal number of buckets for grace hash partitioning: " + numBuckets); } if (!HashUtils.isPowerOfTwo(numBuckets)) { throw new GridException("number of buckets is not power of two: " + numBuckets); } final int bucketShift = HashUtils.shiftsForNextPowerOfTwo(numBuckets); // partitioning resources final String tableName; final int tablePartitionNo; final PrimaryKey primaryKey; final Collection<ForeignKey> foreignKeys; final int[] pkeyIndicies; final String[] parentTableFkIndexNames; final int numParentForeignKeys; final boolean hasParentTable; final int numForeignKeys; final String[] fkIdxNames; final int[][] fkPositions; final int[] childTablesPartitionNo; final LRUMap<String, List<NodeWithPartitionNo>>[] fkCaches; { tableName = jobConf.getTableName(); DistributionCatalog catalog = registry.getDistributionCatalog(); tablePartitionNo = catalog.getTablePartitionNo(tableName, true); Pair<PrimaryKey, Collection<ForeignKey>> primaryForeignKeys = ops.getPrimaryForeignKeys(); primaryKey = primaryForeignKeys.getFirst(); foreignKeys = primaryForeignKeys.getSecond(); pkeyIndicies = primaryKey.getColumnPositions(true); parentTableFkIndexNames = GridDbUtils.getParentTableFkIndexNames(primaryKey); hasParentTable = (parentTableFkIndexNames != null); numParentForeignKeys = hasParentTable ? parentTableFkIndexNames.length : 0; numForeignKeys = foreignKeys.size(); assert (numParentForeignKeys != 0); fkIdxNames = GridDbUtils.getFkIndexNames(foreignKeys, numForeignKeys); fkPositions = GridDbUtils.getFkPositions(foreignKeys, numForeignKeys); childTablesPartitionNo = GridDbUtils.getChildTablesPartitionNo(foreignKeys, numForeignKeys, catalog); fkCaches = GridDbUtils.getFkIndexCaches(numForeignKeys); } // COPY INTO control resources final char filedSeparator = jobConf.getFieldSeparator(); final char quoteChar = jobConf.getStringQuote(); // working resources final GridNode localNode = config.getLocalNode(); final String[] fields = new String[GridDbUtils.getMaxColumnCount(primaryKey, foreignKeys)]; assert (fields.length > 0); final FixedArrayList<String> fieldList = new FixedArrayList<String>(fields); final Charset charset = Charset.forName("UTF-8"); final StringBuilder strBuf = new StringBuilder(64); final int totalRecords = lines.length; final String[] fkeysFields = new String[numForeignKeys]; final byte[][] distkeys = new byte[numForeignKeys][]; final GridNode[] fkMappedNodes = new GridNode[numForeignKeys]; // loading indices for the buckets int bucket = ops.getBucketNumber(); final InMemoryMappingIndex index = hasParentTable ? InMemoryIndexHelper.loadIndex(bucket, parentTableFkIndexNames, registry) : null; final int numNodes = origRouter.getGridSize(); final Map<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> nodeAssignMap = new HashMap<GridNode, Pair<MutableInt, FastByteArrayOutputStream>>( numNodes); final Map<GridNode, MutableInt> mappedNodes = new HashMap<GridNode, MutableInt>(numNodes); final Map<GridNode, FastByteArrayOutputStream> idxShippingMap = new HashMap<GridNode, FastByteArrayOutputStream>( numNodes); for (int i = 0; i < totalRecords; i++) { String line = lines[i]; lines[i] = null; if (numForeignKeys == 0) { // "primary" fragment mapping mapPrimaryFragment(mappedNodes, tablePartitionNo, localNode); } if (hasParentTable) { // "derived by parent" fragment mapping CsvUtils.retrieveFields(line, pkeyIndicies, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String pkeysField = GridDbUtils.combineFields(fields, pkeyIndicies.length, strBuf); for (int kk = 0; kk < numParentForeignKeys; kk++) { String idxName = parentTableFkIndexNames[kk]; mapDerivedByParentFragment(pkeysField, mappedNodes, index, idxName); } } if (numForeignKeys > 0) { // "derived by child" fragment mapping for (int jj = 0; jj < numForeignKeys; jj++) { int[] pos = fkPositions[jj]; CsvUtils.retrieveFields(line, pos, fieldList, filedSeparator, quoteChar); fieldList.trimToZero(); String fkeysField = GridDbUtils.combineFields(fields, pos.length, strBuf); byte[] distkey = StringUtils.getBytes(fkeysField); fkMappedNodes[jj] = mapDerivedByChildFragment(distkey, mappedNodes, childTablesPartitionNo[jj], origRouter); fkeysFields[jj] = fkeysField; distkeys[jj] = distkey; } // store information for derived fragment mapping for (int kk = 0; kk < numForeignKeys; kk++) { final String fkIdxName = fkIdxNames[kk]; final String fkeysField = fkeysFields[kk]; final byte[] distkey = distkeys[kk]; final GridNode fkMappedNode = fkMappedNodes[kk]; FastByteArrayOutputStream shipIdxBuffer = idxShippingMap.get(fkMappedNode); if (shipIdxBuffer == null) { shipIdxBuffer = new FastByteArrayOutputStream(16 * 1024); idxShippingMap.put(fkMappedNode, shipIdxBuffer); } final LRUMap<String, List<NodeWithPartitionNo>> fkCache = fkCaches[kk]; List<NodeWithPartitionNo> storedNodeInfo = fkCache.get(fkeysField); for (final Map.Entry<GridNode, MutableInt> e : mappedNodes.entrySet()) { GridNode node = e.getKey(); int hiddenValue = e.getValue().intValue(); NodeWithPartitionNo nodeInfo = new NodeWithPartitionNo(node, hiddenValue); if (storedNodeInfo == null) { storedNodeInfo = new ArrayList<NodeWithPartitionNo>(8); fkCache.put(fkeysField, storedNodeInfo); } else if (storedNodeInfo.contains(nodeInfo)) {// Note that node has unique hiddenValue to persist continue; } storedNodeInfo.add(nodeInfo); try { InMemoryIndexHelper.writeToStream(distkey, nodeInfo, fkIdxName, shipIdxBuffer, bucketShift); } catch (IOException ioe) { throw new GridException(ioe); } } } } if (mappedNodes.isEmpty()) { throw new IllegalStateException("Could not map records for table: '" + tableName + '\''); } // bind a record to nodes byte[] lineBytes = line.getBytes(charset); mapRecord(lineBytes, totalRecords, numNodes, nodeAssignMap, mappedNodes, filedSeparator); mappedNodes.clear(); } if (numForeignKeys > 0 && idxShippingMap.isEmpty()) { LOG.error("There is no index shipping though numForeignKeys is " + numForeignKeys); } final int numTasks = idxShippingMap.size(); final Map<GridTask, GridNode> taskmap = new IdentityHashMap<GridTask, GridNode>(numTasks); for (final Map.Entry<GridNode, FastByteArrayOutputStream> e : idxShippingMap.entrySet()) { GridNode node = e.getKey(); FastByteArrayOutputStream value = e.getValue(); GridTask task = new ParallelGraceBuildIndexTask(this, value); taskmap.put(task, node); } final ExecutorService sendExecs = ExecutorFactory.newFixedThreadPool(GridXferClient.SENDER_CONCURRENCY, "FileSender", true); final GridXferClient dfsClient = registry.getDfsService().getDFSClient(); final int recvPort = config.getFileReceiverPort(); final HashMap<GridNode, MutableInt> assignedRecMap = new HashMap<GridNode, MutableInt>(numNodes); for (final Map.Entry<GridNode, Pair<MutableInt, FastByteArrayOutputStream>> e : nodeAssignMap.entrySet()) { GridNode node = e.getKey(); GridNode trgNode = origRouter.resolve(node); Pair<MutableInt, FastByteArrayOutputStream> pair = e.getValue(); MutableInt numRecords = pair.first; assignedRecMap.put(trgNode, numRecords); FastByteArrayOutputStream rows = pair.second; pair.clear(); GridDbUtils.sendfile(sendExecs, dfsClient, csvFileName, rows, trgNode, recvPort); } for (final GridNode node : origRouter.getAllNodes()) { if (!assignedRecMap.containsKey(node)) { assignedRecMap.put(node, new MutableInt(0)); } } this.assignedRecMap = assignedRecMap; ExecutorUtils.shutdownAndAwaitTermination(sendExecs); return taskmap; }
From source file:com.google.gwt.emultest.java.util.IdentityHashMapTest.java
public void testHashMapInt() { IdentityHashMap hashMap = new IdentityHashMap(CAPACITY_16); checkEmptyHashMapAssumptions(hashMap); // TODO(mmendez): how do we verify capacity? boolean failed = true; try {// w w w. j a v a 2 s . co m new IdentityHashMap(-SIZE_ONE); } catch (Throwable ex) { if (ex instanceof IllegalArgumentException) { failed = false; } } if (failed) { fail("Failure testing new IdentityHashMap(-1)"); } IdentityHashMap zeroSizedHashMap = new IdentityHashMap(0); assertNotNull(zeroSizedHashMap); }