List of usage examples for java.util ArrayList clear
public void clear()
From source file:net.cbtltd.rest.streamline.A_Handler.java
@SuppressWarnings("unchecked") @Override/*from w w w.j a va 2s.com*/ public void readProducts() { String message = "readProducts Streamline (Altpartyid:" + this.getAltpartyid() + ")"; LOG.debug(message); Date version = new Date(); String responseAllProperties; StringBuilder sbNotKnowLocation = new StringBuilder(); final SqlSession sqlSession = RazorServer.openSession(); try { responseAllProperties = createXMLRequestToStreamline(sqlSession, "GetPropertyList", ""); SAXBuilder builder = new SAXBuilder(); Document document = (Document) builder.build(new StringReader(responseAllProperties)); Element rootNode = document.getRootElement(); List<Element> properties = rootNode.getChild("data").getChildren("property"); for (Element property : properties) { try { String altId = property.getChildText("id"); // LOG.debug("Current AltId="+altId); String countryISO = property.getChildText("country_name"); //data for finding location String state = property.getChildText("state_name"); String city = property.getChildText("city"); String latitudeLocationStr = property.getChildText("location_latitude"); String longitudeLocationStr = property.getChildText("location_longitude"); //Streamline provide us 2 set of coordinates. One for location (city) and one for property. //This coordinates will be used for finding locations, so first we use location coordinates, //and if not exist we use property coordinates if (StringUtils.isEmpty(latitudeLocationStr) && StringUtils.isEmpty(longitudeLocationStr)) { latitudeLocationStr = property.getChildText("latitude"); longitudeLocationStr = property.getChildText("longitude"); } Product product = PartnerService.getProduct(sqlSession, getAltpartyid(), altId); if (product == null) { continue; } if (LOG.isDebugEnabled()) LOG.debug("Processing property : " + product.getName()); Integer roomNumber = 0; Integer bathroomNumber = 0; Integer maxPersonTotalNumber = 0; Integer childNumber = 0; Integer adultsNumber = 0; Double latitude = null; Double longitude = null; try { roomNumber = Integer.parseInt(property.getChildText("bedrooms_number")); } catch (Exception parseExc) { LOG.error("Parse exception: " + parseExc.getMessage()); } try { bathroomNumber = Integer.parseInt(property.getChildText("bathrooms_number")); } catch (Exception parseExc) { LOG.error("Parse exception: " + parseExc.getMessage()); } try { maxPersonTotalNumber = Integer.parseInt(property.getChildText("max_occupants")); } catch (Exception parseExc) { LOG.error("Parse exception: " + parseExc.getMessage()); } try { adultsNumber = Integer.parseInt(property.getChildText("max_adults")); } catch (Exception parseExc) { LOG.error("Parse exception: " + parseExc.getMessage()); } childNumber = maxPersonTotalNumber - adultsNumber; if (childNumber < 0) { childNumber = 0; } try { latitude = Double.valueOf(property.getChildText("latitude")); } catch (Exception parseExc) { } try { longitude = Double.valueOf(property.getChildText("longitude")); } catch (Exception parseExc) { } StringBuilder physicalAddress = new StringBuilder(); StringBuilder physicalAddressForLocation = new StringBuilder(); if (StringUtils.isNotEmpty(property.getChildText("address"))) { physicalAddress.append(property.getChildText("address")).append("\n"); physicalAddressForLocation.append(property.getChildText("address")).append(", "); } if (StringUtils.isNotEmpty(property.getChildText("city"))) { physicalAddress.append(property.getChildText("city")).append("\n"); physicalAddressForLocation.append(property.getChildText("city")).append(", "); } if (StringUtils.isNotEmpty(property.getChildText("state_description"))) { physicalAddress.append(property.getChildText("state_description")).append("\n"); physicalAddressForLocation.append(property.getChildText("state_description")).append(", "); } if (StringUtils.isNotEmpty(property.getChildText("country_name"))) { physicalAddress.append(property.getChildText("country_name")).append("\n"); physicalAddressForLocation.append(property.getChildText("country_name")); } //finding long and lat using physical address and Google Location service //if they do not exist in API response (if exist we do not need to find) //and if we have reason to find - if address is changed or lat or long do not exist in product if ((latitude == null || longitude == null) && (!physicalAddress.toString().equalsIgnoreCase(product.getPhysicaladdress()) || product.getLatitude() == null || product.getLongitude() == null) && StringUtils.isNotEmpty(physicalAddress.toString())) { Location propertyLocation = GoogleLocationProcessor .getGoogleLocation(physicalAddressForLocation.toString()); if (propertyLocation != null) { if (latitude == null) { latitude = propertyLocation.getLatitude(); } if (longitude == null) { longitude = propertyLocation.getLongitude(); } } } String propertyName = property.getChildText("name"); // if(property.getChildText("seo_title")!=null && !property.getChildText("seo_title").equalsIgnoreCase("")){ // propertyName = property.getChildText("seo_title"); // } else{ if (property.getChildText("home_type") != null && !property.getChildText("home_type").equalsIgnoreCase("")) { propertyName += ", " + property.getChildText("home_type"); } if (city != null && !city.equalsIgnoreCase("")) { propertyName += " at " + city; } if (property.getChildText("view_name") != null && !property.getChildText("view_name").equalsIgnoreCase("")) { propertyName += ", with " + property.getChildText("view_name"); } // } if (propertyName.length() > 99) { propertyName = propertyName.substring(0, 99); } product.setCurrency(CURRENCY_IN_STREAMLINE); // product.setName(property.getChildText("name") +", "+ property.getChildText("seo_title") ); product.setName(propertyName); product.setUnit(Unit.DAY); product.setRoom(roomNumber); product.setBathroom(bathroomNumber); product.setQuantity(1); product.setPerson(adultsNumber); product.setChild(0); // product.setChild(childNumber); product.setRank(0.0); product.setPhysicaladdress(physicalAddress.toString()); if (product.getLocationid() == null) { Location location = null; if (locationMap.get(city + state + countryISO) != null) { location = locationMap.get(city + state + countryISO); } else if (StringUtils.isNotEmpty(latitudeLocationStr) && StringUtils.isNotEmpty(longitudeLocationStr)) { location = PartnerService.getLocation(sqlSession, city, state, countryISO, Double.valueOf(latitudeLocationStr), Double.valueOf(longitudeLocationStr)); locationMap.put(city + state + countryISO, location); } else { location = PartnerService.getLocation(sqlSession, city, state, countryISO); locationMap.put(city + state + countryISO, location); } if (location != null) { product.setLocationid(location.getId()); } else { product.setState(Product.SUSPENDED); sbNotKnowLocation.append("\n").append( "Streamline property: " + altId + " country: " + countryISO + " city: " + city); } } product.setLatitude(latitude); product.setLongitude(longitude); product.setWebaddress(getWebaddress()); product.setCommission(getCommission()); product.setDiscount(getDiscount()); product.setRating(5); product.setAltitude(0.0); product.setVersion(version); /* //not used `Options` `Tax` `Code` `Unspsc` `Servicedays` `Toilet` `Infant` `Baby` `Linenchange` `Refresh` `OwnerDiscount` `DynamicPricingEnabled` `AssignedtoManager` `CleaningFee` `SecurityDeposit` */ //description build StringBuilder description = new StringBuilder(); description.append(property.getChildText("seo_description")).append("\n"); description.append(property.getChildText("short_description")).append("\n"); description.append(property.getChildText("description")).append("\n"); ArrayList<String> attributes = new ArrayList<String>(); addType(attributes, property.getChildText("lodging_type_id")); //attributes String otherReqParamsAttributes = "<unit_id>" + altId + "</unit_id>"; String responseAttr = createXMLRequestToStreamline(sqlSession, "GetPropertyAmenities", otherReqParamsAttributes); builder = new SAXBuilder(); Document documentAttr = (Document) builder.build(new StringReader(responseAttr)); Element rootNodeAttributes = documentAttr.getRootElement(); List<Element> propertyAttributes = rootNodeAttributes.getChild("data").getChildren("amenity"); for (Element amenity : propertyAttributes) { addPropertyAttribute(attributes, amenity.getChildText("amenity_name")); } //removing duplicate values from attributes HashSet<String> attributeHashSet = new HashSet<String>(); attributeHashSet.addAll(attributes); attributes.clear(); attributes.addAll(attributeHashSet); sqlSession.getMapper(ProductMapper.class).update(product); product.setPublicText(new Text(product.getPublicId(), product.getName(), Text.Type.HTML, new Date(), description.toString(), Language.EN)); TextService.update(sqlSession, product.getTexts()); RelationService.replace(sqlSession, Relation.PRODUCT_VALUE, product.getId(), product.getValues()); RelationService.create(sqlSession, Relation.PRODUCT_ATTRIBUTE, product.getId(), attributes); RelationService.removeDeprecatedData(sqlSession, Relation.PRODUCT_ATTRIBUTE, product.getId(), attributes); sqlSession.commit(); } catch (Throwable x) { sqlSession.rollback(); LOG.error(x.getMessage()); x.printStackTrace(); } } //print not find attributes HashSet<String> hs = new HashSet<String>(); hs.addAll(PROPERTY_ATTRIBUTES_NOT_FOUND); PROPERTY_ATTRIBUTES_NOT_FOUND.clear(); PROPERTY_ATTRIBUTES_NOT_FOUND.addAll(hs); // LOG.debug("Streamline attributes not find (Altpartyid:"+this.getAltpartyid()+"): "); // for(String tempAttr : PROPERTY_ATTRIBUTES_NOT_FOUND){ // System.out.println(":::"+tempAttr +":::"); // } //canceling product which are not updated Product action = new Product(); action.setAltpartyid(getAltpartyid()); action.setState(Product.CREATED); action.setVersion(version); sqlSession.getMapper(ProductMapper.class).cancelversion(action); sqlSession.commit(); } catch (Throwable x) { sqlSession.rollback(); LOG.error(x.getMessage()); x.printStackTrace(); } finally { sqlSession.close(); } MonitorService.monitor(message, version); }
From source file:com.ferdi2005.secondgram.NotificationsController.java
public void removeDeletedMessagesFromNotifications(final SparseArray<ArrayList<Integer>> deletedMessages) { final ArrayList<MessageObject> popupArray = popupMessages.isEmpty() ? null : new ArrayList<>(popupMessages); notificationsQueue.postRunnable(new Runnable() { @Override/*from w ww.j a v a 2 s . c o m*/ public void run() { int old_unread_count = total_unread_count; SharedPreferences preferences = ApplicationLoader.applicationContext .getSharedPreferences("Notifications", Context.MODE_PRIVATE); for (int a = 0; a < deletedMessages.size(); a++) { int key = deletedMessages.keyAt(a); long dialog_id = -key; ArrayList<Integer> mids = deletedMessages.get(key); Integer currentCount = pushDialogs.get(dialog_id); if (currentCount == null) { currentCount = 0; } Integer newCount = currentCount; for (int b = 0; b < mids.size(); b++) { long mid = mids.get(b); mid |= ((long) key) << 32; MessageObject messageObject = pushMessagesDict.get(mid); if (messageObject != null) { pushMessagesDict.remove(mid); delayedPushMessages.remove(messageObject); pushMessages.remove(messageObject); if (isPersonalMessage(messageObject)) { personal_count--; } if (popupArray != null) { popupArray.remove(messageObject); } newCount--; } } if (newCount <= 0) { newCount = 0; smartNotificationsDialogs.remove(dialog_id); } if (!newCount.equals(currentCount)) { total_unread_count -= currentCount; total_unread_count += newCount; pushDialogs.put(dialog_id, newCount); } if (newCount == 0) { pushDialogs.remove(dialog_id); pushDialogsOverrideMention.remove(dialog_id); if (popupArray != null && pushMessages.isEmpty() && !popupArray.isEmpty()) { popupArray.clear(); } } } if (popupArray != null) { AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { popupMessages = popupArray; } }); } if (old_unread_count != total_unread_count) { if (!notifyCheck) { delayedPushMessages.clear(); showOrUpdateNotification(notifyCheck); } else { scheduleNotificationDelay( lastOnlineFromOtherDevice > ConnectionsManager.getInstance().getCurrentTime()); } } notifyCheck = false; if (preferences.getBoolean("badgeNumber", true)) { setBadge(total_unread_count); } } }); }
From source file:com.ibm.bi.dml.lops.compile.Dag.java
/** * Method to topologically sort lops/*w ww . ja v a 2 s.co m*/ * * @param v */ @SuppressWarnings({ "unchecked", "rawtypes" }) private void doTopologicalSort_strict_order(ArrayList<N> v) { //int numNodes = v.size(); /* * Step 1: compute the level for each node in the DAG. Level for each node is * computed as lops are created. So, this step is need not be performed here. * Step 2: sort the nodes by level, and within a level by node ID. */ // Step1: Performed at the time of creating Lops // Step2: sort nodes by level, and then by node ID Object[] nodearray = v.toArray(); Arrays.sort(nodearray, new LopComparator()); // Copy sorted nodes into "v" and construct a mapping between Lop IDs and sequence of numbers v.clear(); IDMap.clear(); for (int i = 0; i < nodearray.length; i++) { v.add((N) nodearray[i]); IDMap.put(v.get(i).getID(), i); } /* * Compute of All-pair reachability graph (Transitive Closure) of the DAG. * - Perform a depth-first search (DFS) from every node $u$ in the DAG * - and construct the list of reachable nodes from the node $u$ * - store the constructed reachability information in $u$.reachable[] boolean array */ // // for (int i = 0; i < nodearray.length; i++) { boolean[] arr = v.get(i).create_reachable(nodearray.length); Arrays.fill(arr, false); dagDFS(v.get(i), arr); } // print the nodes in sorted order if (LOG.isTraceEnabled()) { for (int i = 0; i < v.size(); i++) { // System.out.print(sortedNodes.get(i).getID() + "(" // + levelmap.get(sortedNodes.get(i).getID()) + "), "); StringBuilder sb = new StringBuilder(); sb.append(v.get(i).getID()); sb.append("("); sb.append(v.get(i).getLevel()); sb.append(") "); sb.append(v.get(i).getType()); sb.append("("); for (Lop vin : v.get(i).getInputs()) { sb.append(vin.getID()); sb.append(","); } sb.append("), "); LOG.trace(sb.toString()); } LOG.trace("topological sort -- done"); } }
From source file:org.apache.hadoop.mapred.NetCDFInputFormatPrunerByFileIndexMultiFile.java
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { FileStatus[] files = listStatus(job); LOG.info("[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] hive query is: " + job.get(HIVE_QUERY, "Kossher")); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] hive query is: " + job.get(HIVE_QUERY, "Kossher")); /* Analyzing Query here */ String hiveQuery = job.get(HIVE_QUERY, "Kossher"); QueryType queryType = QueryType.NOLIMIT; // default mode if (hiveQuery.contains("where") || hiveQuery.contains("WHERE")) { if (hiveQuery.contains("time") || hiveQuery.contains("TIME")) { queryType = QueryType.TIME;/* www .j av a 2 s. c om*/ } else if (hiveQuery.contains("lat") || hiveQuery.contains("LAT")) { queryType = QueryType.LAT; } else if (hiveQuery.contains("lon") || hiveQuery.contains("LON")) { queryType = QueryType.LON; } } float topLimit = -1; float bottomLimit = -1; if (queryType != QueryType.NOLIMIT) { if (hiveQuery.contains("<")) { String[] querySplitted = hiveQuery.split(" "); int i = Arrays.asList(querySplitted).indexOf("<"); topLimit = Float.valueOf(querySplitted[i + 1]); } if (hiveQuery.contains(">")) { String[] querySplitted = hiveQuery.split(" "); int i = Arrays.asList(querySplitted).indexOf(">"); bottomLimit = Float.valueOf(querySplitted[i + 1]); } } //System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndex] QueryType = " + queryType.toString() // +", topLimit = " + topLimit + ", bottomLimit = " + bottomLimit ); //LOG.info("[SAMAN][NetCDFInputFormatPrunerByFileIndex] QueryType = " + queryType.toString() // + ", topLimit = " + topLimit + ", bottomLimit = " + bottomLimit); /* End Analyzing Query here */ System.out.println("[SAMANPruner] beginning of getSplits"); LOG.info("[SAMANPruner] beginning of getSplits"); //System.out.println( "[SAMAN] " + files.length ); //LOG.info( "[SAMAN] " + files.length ); // Save the number of input files in the job-conf job.setLong(NUM_INPUT_FILES, files.length); long totalSize = 0; // compute total size for (FileStatus file : files) { // check we have valid files if (file.isDir()) { throw new IOException("Not a file: " + file.getPath()); } totalSize += file.getLen(); } //long minSize = Math.max(job.getLong("mapred.min.split.size", 1), // minSplitSize); // generate splits ArrayList<NetCDFFileSplit> splits = new ArrayList<NetCDFFileSplit>(numSplits); ArrayList<NetCDFFileSplit> finalSplits = new ArrayList<NetCDFFileSplit>(); NetworkTopology clusterMap = new NetworkTopology(); for (FileStatus file : files) { Path path = file.getPath(); int fileIndex = 0; int dimIndex = 0; if (queryType == QueryType.TIME || queryType == QueryType.NOLIMIT) { if (path.getName().contains("lat") || path.getName().contains("lon")) continue; } else if (queryType == QueryType.LAT) { if (!path.getName().contains("lat")) continue; } else if (queryType == QueryType.LON) { if (!path.getName().contains("lon")) continue; } if (queryType == QueryType.TIME) { String[] parts = path.getName().split("-"); fileIndex = Integer.valueOf(parts[1]); } else if (queryType == QueryType.LAT || queryType == QueryType.LON) { if (path.getName().contains("_")) { String[] parts = path.getName().split("_"); fileIndex = Integer.valueOf(parts[2]); dimIndex = Integer.valueOf(parts[0].substring(7)); } else { //dimIndex = Integer.valueOf(path.getName().substring(7)); String[] parts = path.getName().split("-"); dimIndex = Integer.valueOf(parts[1]); } } //LOG.info("[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] File name is : " + path.getName()); System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] File name is : " + path.getName()); FileSystem fs = path.getFileSystem(job); long length = file.getLen(); BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); if ((length != 0) && isSplitable(fs, path)) { long blockSize = file.getBlockSize(); netInfo = getNetCDFInfo(path, fs, job); long recStart = netInfo.recStart; long[] chunkStarts = netInfo.chunkStarts; long smallSize = netInfo.smallRecSize; long recSize = netInfo.recSize; long splitSize = 0; int chunkIndex = 0; long bytesRemaining = chunkStarts[chunkStarts.length - 1] + recSize - recStart - 2 * smallSize; long thisStart = recStart; // file position long thisChunk = 0; long blockNo = 1; long numChunksPerKey = 0; if (queryType == QueryType.LAT) { long chunkSize = netInfo.timeLength * netInfo.lonLength * 4; numChunksPerKey = blockSize / chunkSize; } else if (queryType == QueryType.LON) { long chunkSize = netInfo.timeLength * netInfo.latLength * 4; numChunksPerKey = blockSize / chunkSize; } System.out.println("[SAMAN][NetCDFInputFormat][getSplits] numChunksPerKey = " + numChunksPerKey); //LOG.info( "[SAMAN] NetCDFInputFormatPruner.getSplits => recStart = " + recStart + ", chunkStarts = " + chunkStarts + // ", smallSize = " + smallSize + ", recSize = " + recSize + ", bytesRemaining = " + bytesRemaining + // ", thisStart = " + thisStart); //System.out.println( "[SAMAN] NetCDFInputFormatPruner.getSplits => recStart = " + recStart + ", chunkStarts = " + chunkStarts + // ", smallSize = " + smallSize + ", recSize = " + recSize + ", bytesRemaining = " + bytesRemaining + // ", thisStart = " + thisStart); while (bytesRemaining > 0) { while (chunkIndex < chunkStarts.length && chunkStarts[chunkIndex] < blockNo * blockSize) { chunkIndex++; } long tempStart = thisStart; long endChunk; if (chunkIndex >= chunkStarts.length) { splitSize = chunkStarts[chunkStarts.length - 1] + recSize - thisStart - smallSize; //bytesRemaining should be 0 after this round } else { splitSize = chunkStarts[chunkIndex] - thisStart - smallSize; thisStart = chunkStarts[chunkIndex]; } endChunk = chunkIndex; blockNo++; //LOG.info( "[SAMAN] NetCDFInputFormatPruner.getSplits => splitSize="+splitSize+", thisStart="+thisStart+ // ", endChunk="+endChunk+", blockNo="+blockNo); System.out.println("[SAMAN] NetCDFInputFormatPruner.getSplits => splitSize=" + splitSize + ", thisStart=" + thisStart + ", endChunk=" + endChunk + ", blockNo=" + blockNo); String[] splitHosts = getSplitHosts(blkLocations, tempStart, splitSize, clusterMap); NetCDFFileSplit split = new NetCDFFileSplit(path, tempStart, splitSize, splitHosts); if (queryType == QueryType.TIME) { if ((topLimit < thisChunk + (fileIndex * netInfo.timeLength)) && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if ((bottomLimit > endChunk + (fileIndex * netInfo.timeLength)) && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } blockToNodes.put(split, splitHosts); // Put the nodes with the specified split into the node to block set System.out.println( "[SAMAN][NetCDFInputFormat][getSplits] Put the nodes with the specified split into the node to block set"); for (int i = 0; i < splitHosts.length; i++) { Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if (splitList == null) { splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put(splitHosts[i], splitList); } splitList.add(split); } System.out.println("[SAMAN][NetCDFInputFormat][getSplits] set start and end!"); split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); } else if (queryType == QueryType.LAT || queryType == QueryType.LON) { //System.out.println( "[SAMAN][NetCDFInputFormat][getSplits] file = " // + path.getName() + ", topLimit = " + topLimit + ", bottomLimit = " + bottomLimit + ", dimIndex = " + dimIndex ); /* if( topLimit < dimIndex*numChunksPerKey && (topLimit != -1) ){ bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if( bottomLimit > dimIndex*numChunksPerKey && (bottomLimit != -1) ){ bytesRemaining -= splitSize; thisChunk = endChunk; continue; }*/ if (topLimit < thisChunk && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if (bottomLimit > endChunk && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } /* if ((topLimit < thisChunk) && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if ((bottomLimit > endChunk) && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } */ //split.getNetCDFFileSplit().endChunk = (long)topLimit; /* split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); */ // Put the block into the block to node set blockToNodes.put(split, splitHosts); // Put the nodes with the specified split into the node to block set for (int i = 0; i < splitHosts.length; i++) { Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if (splitList == null) { splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put(splitHosts[i], splitList); } splitList.add(split); } // For the test, we would assign everything statically. if (bottomLimit > thisChunk && (bottomLimit != -1)) { System.out .println("[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] startChunk = " + bottomLimit); split.getFileSplit().startChunk.add((long) bottomLimit); } else { split.getFileSplit().startChunk.add(thisChunk); } if (topLimit < endChunk && (topLimit != -1)) { System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] endChunk = " + endChunk); split.getFileSplit().endChunk.add((long) topLimit); } else { split.getFileSplit().endChunk.add(endChunk); } } else { if ((topLimit < thisChunk) && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if ((bottomLimit > endChunk) && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } blockToNodes.put(split, splitHosts); // Put the nodes with the specified split into the node to block set for (int i = 0; i < splitHosts.length; i++) { Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if (splitList == null) { splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put(splitHosts[i], splitList); } splitList.add(split); } split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); } splits.add(split); bytesRemaining -= splitSize; thisChunk = endChunk; //LOG.info( "[SAMAN] NetCDFInputFormatPruner.getSplits => bytesRemaining="+bytesRemaining+", thisChunk="+thisChunk ); //System.out.println( "[SAMAN] NetCDFInputFormatPruner.getSplits => bytesRemaining="+bytesRemaining+", thisChunk="+thisChunk ); } } else if (length != 0) { String[] splitHosts = getSplitHosts(blkLocations, 0, length, clusterMap); //splits.add(new FileSplit(path, 0, length, splitHosts)); } else { //Create empty hosts array for zero length files //splits.add(new FileSplit(path, 0, length, new String[0])); } } // Now it's time to merge non-complete splits. // Check if each split has enough space to include another split too Set<String> completedNodes = new HashSet<String>(); ArrayList<NetCDFFileSplit> validBlocks = new ArrayList<NetCDFFileSplit>(); long curSplitSize = 0; Multiset<String> splitsPerNode = HashMultiset.create(); for (Iterator<Map.Entry<String, Set<NetCDFFileSplit>>> iter = nodeToBlocks.entrySet().iterator(); iter .hasNext();) { Map.Entry<String, Set<NetCDFFileSplit>> one = iter.next(); String node = one.getKey(); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] node is = " + node); // Skip the node if it has previously been marked as completed. if (completedNodes.contains(node)) { continue; } Set<NetCDFFileSplit> blocksInCurrentNode = one.getValue(); // for each block, copy it into validBlocks. Delete it from // blockToNodes so that the same block does not appear in // two different splits. Iterator<NetCDFFileSplit> oneBlockIter = blocksInCurrentNode.iterator(); while (oneBlockIter.hasNext()) { NetCDFFileSplit oneblock = oneBlockIter.next(); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + "split is: " + oneblock.getFileSplit().getPath()); // Remove all blocks which may already have been assigned to other // splits. if (!blockToNodes.containsKey(oneblock)) { oneBlockIter.remove(); continue; } validBlocks.add(oneblock); if (queryType == QueryType.LAT) { curSplitSize += (oneblock.getFileSplit().endChunk.get(0) - oneblock.getFileSplit().startChunk.get(0)) * 4 * netInfo.lonLength * netInfo.timeLength; } else if (queryType == QueryType.LON) { curSplitSize += (oneblock.getFileSplit().endChunk.get(0) - oneblock.getFileSplit().startChunk.get(0)) * 4 * netInfo.latLength * netInfo.timeLength; } else if (queryType == QueryType.TIME) { curSplitSize += (oneblock.getFileSplit().endChunk.get(0) - oneblock.getFileSplit().startChunk.get(0)) * 4 * netInfo.latLength * netInfo.lonLength; } else { curSplitSize += (oneblock.getFileSplit().endChunk.get(0) - oneblock.getFileSplit().startChunk.get(0)) * 4 * netInfo.latLength * netInfo.lonLength; } blockToNodes.remove(oneblock); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] curSplitSize = " + curSplitSize); //curSplitSize += singleSplitSize; System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + "Added to valid blocks!"); // if the accumulated split size exceeds the maximum, then // create this split. if (blockSize != 0 && curSplitSize >= blockSize) { // create an input split and add it to the splits array addCreatedSplit(finalSplits, Collections.singleton(node), validBlocks); //totalLength -= curSplitSize; System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + "addCreatedSplit called!"); curSplitSize = 0; splitsPerNode.add(node); // Remove entries from blocksInNode so that we don't walk these // again. //blocksInCurrentNode.removeAll(validBlocks); validBlocks.clear(); // Done creating a single split for this node. Move on to the next // node so that splits are distributed across nodes. //break; } } if (!validBlocks.isEmpty()) { System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] validBlocks not empty!"); addCreatedSplit(finalSplits, Collections.singleton(node), validBlocks); curSplitSize = 0; splitsPerNode.add(node); blocksInCurrentNode.removeAll(validBlocks); validBlocks.clear(); } } Set<NetCDFFileSplit> singleSplitsSet = blockToNodes.keySet(); Iterator itrSingle = singleSplitsSet.iterator(); while (itrSingle.hasNext()) { NetCDFFileSplit temp = (NetCDFFileSplit) itrSingle.next(); addCreatedSingleSplit(finalSplits, temp.getLocations(), temp); } Iterator itr = finalSplits.iterator(); while (itr.hasNext()) { NetCDFFileSplit temp = (NetCDFFileSplit) itr.next(); String[] locations = temp.getFileSplit().getLocations(); String locationsString = ""; for (int i = 0; i < locations.length; i++) locationsString += locations[i]; String pathsString = ""; List<Path> paths = temp.getFileSplit().getPaths(); for (Path path : paths) pathsString += path.getName() + ","; String startsString = ""; List<Long> starts = temp.getFileSplit().startChunk; for (Long start : starts) startsString += (start + ","); String endsString = ""; List<Long> ends = temp.getFileSplit().endChunk; for (Long end : ends) endsString += (end + ","); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + "locations=" + locationsString + "," + "paths=" + pathsString + "," + "starts=" + startsString + "," + "ends=" + endsString + ","); } return finalSplits.toArray(new NetCDFFileSplit[finalSplits.size()]); }
From source file:com.miz.mizuu.fragments.ShowSeasonsFragment.java
private void addAvailableFiles() { ArrayList<FileSource> filesources = MizLib.getFileSources(MizLib.TYPE_SHOWS, true); int tempCount = 0; ArrayList<TvShowEpisode> tempEpisodes = new ArrayList<TvShowEpisode>(allEpisodes); for (int i = 0; i < tempEpisodes.size(); i++) { if (tempEpisodes.get(i).isNetworkFile()) { if (isAdded()) if (MizLib.isWifiConnected(getActivity(), prefsDisableEthernetWiFiCheck)) { FileSource source = null; for (int j = 0; j < filesources.size(); j++) if (tempEpisodes.get(i).getFilepath().contains(filesources.get(j).getFilepath())) { source = filesources.get(j); continue; }// w ww.j a v a2 s. c o m if (source == null) continue; try { final SmbFile file = new SmbFile( MizLib.createSmbLoginString(URLEncoder.encode(source.getDomain(), "utf-8"), URLEncoder.encode(source.getUser(), "utf-8"), URLEncoder.encode(source.getPassword(), "utf-8"), tempEpisodes.get(i).getFilepath(), false)); if (file.exists()) { if (seasonEpisodeCount.get(Integer.valueOf(tempEpisodes.get(i).getSeason())) == 0) { seasonEpisodeCount.append(Integer.valueOf(tempEpisodes.get(i).getSeason()), 1); seasons.add(tempEpisodes.get(i).getSeason()); } else { tempCount = seasonEpisodeCount .get(Integer.valueOf(tempEpisodes.get(i).getSeason())); seasonEpisodeCount.append(Integer.valueOf(tempEpisodes.get(i).getSeason()), tempCount + 1); } shownEpisodes.add(tempEpisodes.get(i)); } } catch (Exception e) { } // Do nothing - the file isn't available (either MalformedURLException or SmbException) } } else if (tempEpisodes.get(i).isUpnpFile()) { if (MizLib.exists(tempEpisodes.get(i).getFilepath())) { if (seasonEpisodeCount.get(Integer.valueOf(tempEpisodes.get(i).getSeason())) == 0) { seasonEpisodeCount.append(Integer.valueOf(tempEpisodes.get(i).getSeason()), 1); seasons.add(tempEpisodes.get(i).getSeason()); } else { tempCount = seasonEpisodeCount.get(Integer.valueOf(tempEpisodes.get(i).getSeason())); seasonEpisodeCount.append(Integer.valueOf(tempEpisodes.get(i).getSeason()), tempCount + 1); } shownEpisodes.add(tempEpisodes.get(i)); } } else { if (new File(tempEpisodes.get(i).getFilepath()).exists()) { if (seasonEpisodeCount.get(Integer.valueOf(tempEpisodes.get(i).getSeason())) == 0) { seasonEpisodeCount.append(Integer.valueOf(tempEpisodes.get(i).getSeason()), 1); seasons.add(tempEpisodes.get(i).getSeason()); } else { tempCount = seasonEpisodeCount.get(Integer.valueOf(tempEpisodes.get(i).getSeason())); seasonEpisodeCount.append(Integer.valueOf(tempEpisodes.get(i).getSeason()), tempCount + 1); } shownEpisodes.add(tempEpisodes.get(i)); } } } tempEpisodes.clear(); tempEpisodes = null; }
From source file:de.ingrid.ibus.comm.Bus.java
public IngridHitDetail[] getDetails(IngridHit[] hits, IngridQuery query, String[] requestedFields) throws Exception { long startGetDetails = 0; if (fLogger.isDebugEnabled()) { startGetDetails = System.currentTimeMillis(); }//from ww w. j a v a2s . c o m if (requestedFields == null) { requestedFields = new String[0]; } // collect requests for plugs HashMap hashMap = new HashMap(); IngridHit hit = null; for (int i = 0; i < hits.length; i++) { hit = hits[i]; // ignore hit if hit is "placeholder" if (hit.isDummyHit()) { if (fLogger.isDebugEnabled()) { fLogger.debug("getDetails: do NOT call iPlug for dummy hit: " + hit); } continue; } ArrayList requestHitList = (ArrayList) hashMap.get(hit.getPlugId()); if (requestHitList == null) { requestHitList = new ArrayList(); hashMap.put(hit.getPlugId(), requestHitList); } requestHitList.add(hit); } // send requests and collect response Iterator iterator = hashMap.keySet().iterator(); IPlug plugProxy; ArrayList resultList = new ArrayList(hits.length); Random random = new Random(System.currentTimeMillis()); long time = 0; while (iterator.hasNext()) { String plugId = (String) iterator.next(); ArrayList requestHitList = (ArrayList) hashMap.get(plugId); if (requestHitList != null) { IngridHit[] requestHits = (IngridHit[]) requestHitList .toArray(new IngridHit[requestHitList.size()]); plugProxy = this.fRegistry.getPlugProxy(plugId); /* * iPlugs with older base-webapp than version 5.0.0 set requested fields "title" and "summary" on detail search by default. * The base-webapp since version 5.0.0 set no fields by default. Requested field "title" and "summary" must set * by the search component like portal. * To make older iPlugs running with duplicate fields (default and requested fields) field "title" and "summary" must * be remove on requested fields. */ PlugDescription pd = this.fRegistry.getPlugDescription(plugId); if (pd != null) { Metadata metadata = pd.getMetadata(); if (metadata != null) { String pdVersion = metadata.getVersion(); if (!IPlugVersionInspector.compareVersion(pdVersion, IPLUG_OLD_VERSION_REMOVE_FIELDS)) { List<String> list = new ArrayList<String>(Arrays.asList(requestedFields)); list.remove("title"); list.remove("summary"); list.remove("content"); requestedFields = list.toArray(new String[0]); } } } if (fLogger.isDebugEnabled()) { fLogger.debug("(search) details start " + plugId + " (" + requestHits.length + ") " + query.hashCode()); } time = System.currentTimeMillis(); IngridHitDetail[] responseDetails = plugProxy.getDetails(requestHits, query, requestedFields); if (fLogger.isDebugEnabled()) { fLogger.debug("(search) details ends (" + responseDetails.length + ")" + plugId + " query:" + query.hashCode() + " within " + (System.currentTimeMillis() - time) + " ms."); } for (int i = 0; i < responseDetails.length; i++) { if (responseDetails[i] == null) { if (fLogger.isErrorEnabled()) { fLogger.error( plugId + ": responded details that are null (set a pseudo responseDetail"); } responseDetails[i] = new IngridHitDetail(plugId, String.valueOf(random.nextInt()), random.nextInt(), 0.0f, "", ""); } responseDetails[i].put(IngridHitDetail.DETAIL_TIMING, (System.currentTimeMillis() - time)); } resultList.addAll(Arrays.asList(responseDetails)); // FIXME: to improve performance we can use an Array instead // of a list here. } if (null != requestHitList) { requestHitList.clear(); requestHitList = null; } } hashMap.clear(); hashMap = null; // int count = resultList.size(); IngridHitDetail[] resultDetails = (IngridHitDetail[]) resultList .toArray(new IngridHitDetail[resultList.size()]); resultList.clear(); resultList = null; // sort to be in the same order as the requested hits. IngridHitDetail[] details = new IngridHitDetail[hits.length]; for (int i = 0; i < hits.length; i++) { // set dummy detail if hit is "placeholder" if (hits[i].isDummyHit()) { details[i] = new IngridHitDetail(hit, "dummy hit", ""); details[i].setDummyHit(true); if (fLogger.isDebugEnabled()) { fLogger.debug("getDetails: dummy hit, add dummy detail: " + details[i]); } continue; } String plugId = hits[i].getPlugId(); String documentId = getDocIdAsString(hits[i]); boolean found = false; // get the details of the hits for (int j = 0; j < resultDetails.length; j++) { IngridHitDetail detail = resultDetails[j]; String detailDocId = getDocIdAsString(detail); if (documentId.equals(detailDocId) && detail.getPlugId().equals(plugId)) { details[i] = detail; pushMetaData(details[i]); // push meta data to details found = true; break; } } if (!found) { if (fLogger.isErrorEnabled()) { fLogger.error("unable to find details getDetails: " + hit.toString()); } details[i] = new IngridHitDetail(hit, "no details found", ""); } } if (fLogger.isDebugEnabled()) { fLogger.debug("TIMING: Create details for Query (" + query.hashCode() + ") in " + (System.currentTimeMillis() - startGetDetails) + "ms."); } return details; }
From source file:org.apache.hadoop.mapred.NetCDFInputFormatPartToMemoryMultiSplit.java
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { FileStatus[] files = listStatus(job); LOG.info("[SAMAN][NetCDFInputFormatPartToMemoryMultiSplit][getSplits] hive query is: " + job.get(HIVE_QUERY, "Kossher")); System.out.println("[SAMAN][NetCDFInputFormatPartToMemoryMultiSplit][getSplits] hive query is: " + job.get(HIVE_QUERY, "Kossher")); /* Analyzing Query here */ String hiveQuery = job.get(HIVE_QUERY, "Kossher"); QueryType queryType = QueryType.NOLIMIT; // default mode if (hiveQuery.contains("where") || hiveQuery.contains("WHERE")) { if (hiveQuery.contains("time") || hiveQuery.contains("TIME")) { queryType = QueryType.TIME;//w ww . j a va 2s .c om } else if (hiveQuery.contains("lat") || hiveQuery.contains("LAT")) { queryType = QueryType.LAT; } else if (hiveQuery.contains("lon") || hiveQuery.contains("LON")) { queryType = QueryType.LON; } } float topLimit = -1; float bottomLimit = -1; if (queryType != QueryType.NOLIMIT) { if (hiveQuery.contains("<")) { String[] querySplitted = hiveQuery.split(" "); int i = Arrays.asList(querySplitted).indexOf("<"); topLimit = Float.valueOf(querySplitted[i + 1]); } if (hiveQuery.contains(">")) { String[] querySplitted = hiveQuery.split(" "); int i = Arrays.asList(querySplitted).indexOf(">"); bottomLimit = Float.valueOf(querySplitted[i + 1]); } } //System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndex] QueryType = " + queryType.toString() // +", topLimit = " + topLimit + ", bottomLimit = " + bottomLimit ); //LOG.info("[SAMAN][NetCDFInputFormatPrunerByFileIndex] QueryType = " + queryType.toString() // + ", topLimit = " + topLimit + ", bottomLimit = " + bottomLimit); /* End Analyzing Query here */ System.out.println("[SAMANPruner] beginning of getSplits"); LOG.info("[SAMANPruner] beginning of getSplits"); //System.out.println( "[SAMAN] " + files.length ); //LOG.info( "[SAMAN] " + files.length ); // Save the number of input files in the job-conf job.setLong(NUM_INPUT_FILES, files.length); long totalSize = 0; // compute total size for (FileStatus file : files) { // check we have valid files if (file.isDir()) { throw new IOException("Not a file: " + file.getPath()); } totalSize += file.getLen(); } //long minSize = Math.max(job.getLong("mapred.min.split.size", 1), // minSplitSize); // generate splits ArrayList<NetCDFFileSplit> splits = new ArrayList<NetCDFFileSplit>(numSplits); ArrayList<NetCDFFileSplit> finalSplits = new ArrayList<NetCDFFileSplit>(); NetworkTopology clusterMap = new NetworkTopology(); for (FileStatus file : files) { Path path = file.getPath(); int fileIndex = 0; int dimIndex = 0; //LOG.info("[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] File name is : " + path.getName()); System.out.println( "[SAMAN][NetCDFInputFormatPartToMemoryMultiSplit][getSplits] File name is : " + path.getName()); FileSystem fs = path.getFileSystem(job); long length = file.getLen(); BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); if ((length != 0) && isSplitable(fs, path)) { long blockSize = file.getBlockSize(); netInfo = getNetCDFInfo(path, fs, job); long recStart = netInfo.recStart; long[] chunkStarts = netInfo.chunkStarts; long smallSize = netInfo.smallRecSize; long recSize = netInfo.recSize; long splitSize = 0; int chunkIndex = 0; long bytesRemaining = chunkStarts[chunkStarts.length - 1] + recSize - recStart - 2 * smallSize; long thisStart = recStart; // file position long thisChunk = 0; long blockNo = 1; long numChunksPerKey = 0; if (queryType == QueryType.LAT) { long chunkSize = netInfo.timeLength * netInfo.lonLength * 4; numChunksPerKey = blockSize / chunkSize; } else if (queryType == QueryType.LON) { long chunkSize = netInfo.timeLength * netInfo.latLength * 4; numChunksPerKey = blockSize / chunkSize; } System.out.println("[SAMAN][NetCDFInputFormat][getSplits] numChunksPerKey = " + numChunksPerKey); //LOG.info( "[SAMAN] NetCDFInputFormatPruner.getSplits => recStart = " + recStart + ", chunkStarts = " + chunkStarts + // ", smallSize = " + smallSize + ", recSize = " + recSize + ", bytesRemaining = " + bytesRemaining + // ", thisStart = " + thisStart); //System.out.println( "[SAMAN] NetCDFInputFormatPruner.getSplits => recStart = " + recStart + ", chunkStarts = " + chunkStarts + // ", smallSize = " + smallSize + ", recSize = " + recSize + ", bytesRemaining = " + bytesRemaining + // ", thisStart = " + thisStart); while (bytesRemaining > 0) { while (chunkIndex < chunkStarts.length && chunkStarts[chunkIndex] < blockNo * blockSize) { chunkIndex++; } long tempStart = thisStart; long endChunk; if (chunkIndex >= chunkStarts.length) { splitSize = chunkStarts[chunkStarts.length - 1] + recSize - thisStart - smallSize; //bytesRemaining should be 0 after this round } else { splitSize = chunkStarts[chunkIndex] - thisStart - smallSize; thisStart = chunkStarts[chunkIndex]; } endChunk = chunkIndex; blockNo++; //LOG.info( "[SAMAN] NetCDFInputFormatPruner.getSplits => splitSize="+splitSize+", thisStart="+thisStart+ // ", endChunk="+endChunk+", blockNo="+blockNo); System.out.println("[SAMAN] NetCDFInputFormatPruner.getSplits => splitSize=" + splitSize + ", thisStart=" + thisStart + ", endChunk=" + endChunk + ", blockNo=" + blockNo); String[] splitHosts = getSplitHosts(blkLocations, tempStart, splitSize, clusterMap); NetCDFFileSplit split = new NetCDFFileSplit(path, tempStart, splitSize, splitHosts); split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); if (queryType == QueryType.TIME) { split.getFileSplit().timeStartLimit.add((long) bottomLimit); split.getFileSplit().timeEndLimit.add((long) topLimit); split.getFileSplit().latStartLimit.add((long) -1); split.getFileSplit().latEndLimit.add((long) -1); split.getFileSplit().lonStartLimit.add((long) -1); split.getFileSplit().lonEndLimit.add((long) -1); } else if (queryType == QueryType.LAT) { split.getFileSplit().timeStartLimit.add((long) -1); split.getFileSplit().timeEndLimit.add((long) -1); split.getFileSplit().latStartLimit.add((long) bottomLimit); split.getFileSplit().latEndLimit.add((long) topLimit); split.getFileSplit().lonStartLimit.add((long) -1); split.getFileSplit().lonEndLimit.add((long) -1); } else if (queryType == QueryType.LON) { split.getFileSplit().timeStartLimit.add((long) -1); split.getFileSplit().timeEndLimit.add((long) -1); split.getFileSplit().latStartLimit.add((long) -1); split.getFileSplit().latEndLimit.add((long) -1); split.getFileSplit().lonStartLimit.add((long) bottomLimit); split.getFileSplit().lonEndLimit.add((long) topLimit); } blockToNodes.put(split, splitHosts); System.out.println( "[SAMAN][NetCDFInputFormat][getSplits] Put the nodes with the specified split into the node to block set"); for (int i = 0; i < splitHosts.length; i++) { Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if (splitList == null) { splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put(splitHosts[i], splitList); } splitList.add(split); } /* if( queryType == QueryType.TIME ) { if ((topLimit < thisChunk + (fileIndex*netInfo.timeLength)) && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if ((bottomLimit > endChunk + (fileIndex*netInfo.timeLength)) && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } blockToNodes.put( split, splitHosts ); // Put the nodes with the specified split into the node to block set System.out.println( "[SAMAN][NetCDFInputFormat][getSplits] Put the nodes with the specified split into the node to block set" ); for( int i = 0; i < splitHosts.length; i++ ){ Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if( splitList == null ){ splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put( splitHosts[i], splitList ); } splitList.add( split ); } System.out.println("[SAMAN][NetCDFInputFormat][getSplits] set start and end!" ); split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); } else if( queryType == QueryType.LAT || queryType == QueryType.LON ){ //System.out.println( "[SAMAN][NetCDFInputFormat][getSplits] file = " // + path.getName() + ", topLimit = " + topLimit + ", bottomLimit = " + bottomLimit + ", dimIndex = " + dimIndex ); */ /* if( topLimit < dimIndex*numChunksPerKey && (topLimit != -1) ){ bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if( bottomLimit > dimIndex*numChunksPerKey && (bottomLimit != -1) ){ bytesRemaining -= splitSize; thisChunk = endChunk; continue; }*/ /* if (topLimit < thisChunk && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if (bottomLimit > endChunk && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } */ /* if ((topLimit < thisChunk) && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if ((bottomLimit > endChunk) && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } */ //split.getNetCDFFileSplit().endChunk = (long)topLimit; /* split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); */ // Put the block into the block to node set /* blockToNodes.put( split, splitHosts ); // Put the nodes with the specified split into the node to block set for( int i = 0; i < splitHosts.length; i++ ){ Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if( splitList == null ){ splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put( splitHosts[i], splitList ); } splitList.add( split ); } // For the test, we would assign everything statically. if( bottomLimit > thisChunk && (bottomLimit != -1) ){ System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] startChunk = " + bottomLimit ); split.getFileSplit().startChunk.add((long)bottomLimit); }else{ split.getFileSplit().startChunk.add(thisChunk); } if( topLimit < endChunk && (topLimit != -1) ){ System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndex][getSplits] endChunk = " + endChunk ); split.getFileSplit().endChunk.add((long)topLimit); }else{ split.getFileSplit().endChunk.add(endChunk); } } else { if ((topLimit < thisChunk) && (topLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } if ((bottomLimit > endChunk) && (bottomLimit != -1)) { bytesRemaining -= splitSize; thisChunk = endChunk; continue; } blockToNodes.put( split, splitHosts ); // Put the nodes with the specified split into the node to block set for( int i = 0; i < splitHosts.length; i++ ){ Set<NetCDFFileSplit> splitList = nodeToBlocks.get(splitHosts[i]); if( splitList == null ){ splitList = new LinkedHashSet<NetCDFFileSplit>(); nodeToBlocks.put( splitHosts[i], splitList ); } splitList.add( split ); } split.getFileSplit().startChunk.add(thisChunk); split.getFileSplit().endChunk.add(endChunk); } */ splits.add(split); bytesRemaining -= splitSize; thisChunk = endChunk; //LOG.info( "[SAMAN] NetCDFInputFormatPruner.getSplits => bytesRemaining="+bytesRemaining+", thisChunk="+thisChunk ); //System.out.println( "[SAMAN] NetCDFInputFormatPruner.getSplits => bytesRemaining="+bytesRemaining+", thisChunk="+thisChunk ); } } else if (length != 0) { String[] splitHosts = getSplitHosts(blkLocations, 0, length, clusterMap); //splits.add(new FileSplit(path, 0, length, splitHosts)); } else { //Create empty hosts array for zero length files //splits.add(new FileSplit(path, 0, length, new String[0])); } } // Now it's time to merge non-complete splits. // Check if each split has enough space to include another split too Set<String> completedNodes = new HashSet<String>(); ArrayList<NetCDFFileSplit> validBlocks = new ArrayList<NetCDFFileSplit>(); long curSplitSize = 0; Multiset<String> splitsPerNode = HashMultiset.create(); for (Iterator<Map.Entry<String, Set<NetCDFFileSplit>>> iter = nodeToBlocks.entrySet().iterator(); iter .hasNext();) { Map.Entry<String, Set<NetCDFFileSplit>> one = iter.next(); String node = one.getKey(); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] node is = " + node); // Skip the node if it has previously been marked as completed. if (completedNodes.contains(node)) { continue; } Set<NetCDFFileSplit> blocksInCurrentNode = one.getValue(); // for each block, copy it into validBlocks. Delete it from // blockToNodes so that the same block does not appear in // two different splits. Iterator<NetCDFFileSplit> oneBlockIter = blocksInCurrentNode.iterator(); while (oneBlockIter.hasNext()) { NetCDFFileSplit oneblock = oneBlockIter.next(); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + "split is: " + oneblock.getFileSplit().getPath()); // Remove all blocks which may already have been assigned to other // splits. if (!blockToNodes.containsKey(oneblock)) { oneBlockIter.remove(); continue; } validBlocks.add(oneblock); if (queryType == QueryType.LAT) { curSplitSize += (oneblock.getFileSplit().latEndLimit.get(0) - oneblock.getFileSplit().latStartLimit.get(0)) * 4 * netInfo.lonLength * netInfo.timeLength; } else if (queryType == QueryType.LON) { curSplitSize += (oneblock.getFileSplit().lonEndLimit.get(0) - oneblock.getFileSplit().lonStartLimit.get(0)) * 4 * netInfo.latLength * netInfo.timeLength; } else if (queryType == QueryType.TIME) { curSplitSize += (oneblock.getFileSplit().timeEndLimit.get(0) - oneblock.getFileSplit().timeStartLimit.get(0)) * 4 * netInfo.latLength * netInfo.lonLength; } else { curSplitSize += (oneblock.getFileSplit().endChunk.get(0) - oneblock.getFileSplit().startChunk.get(0)) * 4 * netInfo.latLength * netInfo.lonLength; } blockToNodes.remove(oneblock); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] curSplitSize = " + curSplitSize); //curSplitSize += singleSplitSize; //System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + // "Added to valid blocks!" ); // if the accumulated split size exceeds the maximum, then // create this split. if (blockSize != 0 && curSplitSize >= blockSize) { // create an input split and add it to the splits array addCreatedSplit(finalSplits, Collections.singleton(node), validBlocks); //totalLength -= curSplitSize; //System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + // "addCreatedSplit called!" ); curSplitSize = 0; splitsPerNode.add(node); // Remove entries from blocksInNode so that we don't walk these // again. //blocksInCurrentNode.removeAll(validBlocks); validBlocks.clear(); // Done creating a single split for this node. Move on to the next // node so that splits are distributed across nodes. //break; } } if (!validBlocks.isEmpty()) { //System.out.println( "[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] validBlocks not empty!" ); addCreatedSplit(finalSplits, Collections.singleton(node), validBlocks); curSplitSize = 0; splitsPerNode.add(node); blocksInCurrentNode.removeAll(validBlocks); validBlocks.clear(); } } Set<NetCDFFileSplit> singleSplitsSet = blockToNodes.keySet(); Iterator itrSingle = singleSplitsSet.iterator(); while (itrSingle.hasNext()) { NetCDFFileSplit temp = (NetCDFFileSplit) itrSingle.next(); addCreatedSingleSplit(finalSplits, temp.getLocations(), temp); } Iterator itr = finalSplits.iterator(); while (itr.hasNext()) { NetCDFFileSplit temp = (NetCDFFileSplit) itr.next(); String[] locations = temp.getFileSplit().getLocations(); String locationsString = ""; for (int i = 0; i < locations.length; i++) locationsString += locations[i]; String pathsString = ""; List<Path> paths = temp.getFileSplit().getPaths(); for (Path path : paths) pathsString += path.getName() + ","; String startsString = ""; List<Long> starts = temp.getFileSplit().startChunk; for (Long start : starts) startsString += (start + ","); String endsString = ""; List<Long> ends = temp.getFileSplit().endChunk; for (Long end : ends) endsString += (end + ","); System.out.println("[SAMAN][NetCDFInputFormatPrunerByFileIndexMultiFile][getSplits] " + "locations=" + locationsString + "," + "paths=" + pathsString + "," + "starts=" + startsString + "," + "ends=" + endsString + ","); } return finalSplits.toArray(new NetCDFFileSplit[finalSplits.size()]); }
From source file:com.sentaroh.android.SMBSync2.SyncTaskUtility.java
public void scanRemoteNetworkDlg(final NotifyEvent p_ntfy, String port_number, boolean scan_start) { //??/* w ww. j av a 2 s. c o m*/ final Dialog dialog = new Dialog(mContext); dialog.requestWindowFeature(Window.FEATURE_NO_TITLE); dialog.setCanceledOnTouchOutside(false); dialog.setContentView(R.layout.scan_remote_ntwk_dlg); LinearLayout ll_dlg_view = (LinearLayout) dialog.findViewById(R.id.scan_remote_ntwk_dlg_view); ll_dlg_view.setBackgroundColor(mGp.themeColorList.dialog_msg_background_color); final LinearLayout title_view = (LinearLayout) dialog.findViewById(R.id.scan_remote_ntwk_title_view); final TextView title = (TextView) dialog.findViewById(R.id.scan_remote_ntwk_title); title_view.setBackgroundColor(mGp.themeColorList.dialog_title_background_color); title.setTextColor(mGp.themeColorList.text_color_dialog_title); final Button btn_scan = (Button) dialog.findViewById(R.id.scan_remote_ntwk_btn_ok); final Button btn_cancel = (Button) dialog.findViewById(R.id.scan_remote_ntwk_btn_cancel); final TextView tvmsg = (TextView) dialog.findViewById(R.id.scan_remote_ntwk_msg); final TextView tv_result = (TextView) dialog.findViewById(R.id.scan_remote_ntwk_scan_result_title); tvmsg.setText(mContext.getString(R.string.msgs_scan_ip_address_press_scan_btn)); tv_result.setVisibility(TextView.GONE); final String from = CommonUtilities.getLocalIpAddress(); String subnet = from.substring(0, from.lastIndexOf(".")); String subnet_o1, subnet_o2, subnet_o3; subnet_o1 = subnet.substring(0, subnet.indexOf(".")); subnet_o2 = subnet.substring(subnet.indexOf(".") + 1, subnet.lastIndexOf(".")); subnet_o3 = subnet.substring(subnet.lastIndexOf(".") + 1, subnet.length()); final EditText baEt1 = (EditText) dialog.findViewById(R.id.scan_remote_ntwk_begin_address_o1); final EditText baEt2 = (EditText) dialog.findViewById(R.id.scan_remote_ntwk_begin_address_o2); final EditText baEt3 = (EditText) dialog.findViewById(R.id.scan_remote_ntwk_begin_address_o3); final EditText baEt4 = (EditText) dialog.findViewById(R.id.scan_remote_ntwk_begin_address_o4); final EditText eaEt4 = (EditText) dialog.findViewById(R.id.scan_remote_ntwk_end_address_o4); baEt1.setText(subnet_o1); baEt2.setText(subnet_o2); baEt3.setText(subnet_o3); baEt4.setText("1"); baEt4.setSelection(1); eaEt4.setText("254"); baEt4.requestFocus(); final CheckedTextView ctv_use_port_number = (CheckedTextView) dialog .findViewById(R.id.scan_remote_ntwk_ctv_use_port); final EditText et_port_number = (EditText) dialog.findViewById(R.id.scan_remote_ntwk_port_number); CommonDialog.setDlgBoxSizeLimit(dialog, true); if (port_number.equals("")) { et_port_number.setEnabled(false); ctv_use_port_number.setChecked(false); } else { et_port_number.setEnabled(true); et_port_number.setText(port_number); ctv_use_port_number.setChecked(true); } ctv_use_port_number.setOnClickListener(new OnClickListener() { @Override public void onClick(View v) { ctv_use_port_number.toggle(); boolean isChecked = ctv_use_port_number.isChecked(); et_port_number.setEnabled(isChecked); } }); final NotifyEvent ntfy_lv_click = new NotifyEvent(mContext); ntfy_lv_click.setListener(new NotifyEventListener() { @Override public void positiveResponse(Context c, Object[] o) { dialog.dismiss(); p_ntfy.notifyToListener(true, o); } @Override public void negativeResponse(Context c, Object[] o) { } }); final ArrayList<ScanAddressResultListItem> ipAddressList = new ArrayList<ScanAddressResultListItem>(); // ScanAddressResultListItem li=new ScanAddressResultListItem(); // li.server_name=mContext.getString(R.string.msgs_ip_address_no_address); // ipAddressList.add(li); final ListView lv = (ListView) dialog.findViewById(R.id.scan_remote_ntwk_scan_result_list); final AdapterScanAddressResultList adap = new AdapterScanAddressResultList(mContext, R.layout.scan_address_result_list_item, ipAddressList, ntfy_lv_click); lv.setAdapter(adap); lv.setScrollingCacheEnabled(false); lv.setScrollbarFadingEnabled(false); //SCAN? btn_scan.setOnClickListener(new View.OnClickListener() { public void onClick(View v) { ipAddressList.clear(); NotifyEvent ntfy = new NotifyEvent(mContext); ntfy.setListener(new NotifyEventListener() { @Override public void positiveResponse(Context c, Object[] o) { if (ipAddressList.size() < 1) { tvmsg.setText(mContext.getString(R.string.msgs_scan_ip_address_not_detected)); tv_result.setVisibility(TextView.GONE); } else { tvmsg.setText(mContext.getString(R.string.msgs_scan_ip_address_select_detected_host)); tv_result.setVisibility(TextView.VISIBLE); } // adap.clear(); // for (int i=0;i<ipAddressList.size();i++) // adap.add(ipAddressList.get(i)); } @Override public void negativeResponse(Context c, Object[] o) { } }); if (auditScanAddressRangeValue(dialog)) { tv_result.setVisibility(TextView.GONE); String ba1 = baEt1.getText().toString(); String ba2 = baEt2.getText().toString(); String ba3 = baEt3.getText().toString(); String ba4 = baEt4.getText().toString(); String ea4 = eaEt4.getText().toString(); String subnet = ba1 + "." + ba2 + "." + ba3; int begin_addr = Integer.parseInt(ba4); int end_addr = Integer.parseInt(ea4); scanRemoteNetwork(dialog, lv, adap, ipAddressList, subnet, begin_addr, end_addr, ntfy); } else { //error } } }); //CANCEL? btn_cancel.setOnClickListener(new View.OnClickListener() { public void onClick(View v) { dialog.dismiss(); p_ntfy.notifyToListener(false, null); } }); // Cancel? dialog.setOnCancelListener(new Dialog.OnCancelListener() { @Override public void onCancel(DialogInterface arg0) { btn_cancel.performClick(); } }); dialog.show(); if (scan_start) btn_scan.performClick(); }
From source file:net.tourbook.export.gpx.DialogExportTour.java
private void doExport() throws IOException { // disable button's getButton(IDialogConstants.OK_ID).setEnabled(false); getButton(IDialogConstants.CANCEL_ID).setEnabled(false); final String completeFilePath = _txtFilePath.getText(); final boolean isOverwriteFiles = _chkOverwriteFiles.getSelection(); final boolean isCamouflageSpeed = _chkCamouflageSpeed.getSelection(); final float[] camouflageSpeed = new float[1]; try {//from w w w .j a va2 s . co m camouflageSpeed[0] = Float.parseFloat(_txtCamouflageSpeed.getText()); } catch (final NumberFormatException e) { camouflageSpeed[0] = 0.1F; } camouflageSpeed[0] *= UI.UNIT_VALUE_DISTANCE / 3.6f; final ArrayList<GarminTrack> trackList = new ArrayList<GarminTrack>(); final ArrayList<GarminWaypoint> wayPointList = new ArrayList<GarminWaypoint>(); final FileCollisionBehavior fileCollisionBehaviour = new FileCollisionBehavior(); if (_tourDataList.size() == 1) { // export one tour final TourData tourData = _tourDataList.get(0); final GarminLap tourLap = getLap(tourData, _chkExportNotes.getSelection()); final GarminTrack track = getTrack(tourData, TourManager.getTourDateTime(tourData), isCamouflageSpeed, camouflageSpeed[0]); if (track != null) { trackList.add(track); } if (_chkExportMarkers.getSelection()) { // get markers when this option is checked getWaypoints(wayPointList, tourData); } doExportTour(tourLap, trackList, wayPointList, completeFilePath, fileCollisionBehaviour, isOverwriteFiles); } else { /* * export multiple tours */ if (_chkMergeAllTours.getSelection()) { /* * merge all tours into one */ _trackStartDateTime = TourManager.getTourDateTime(_tourDataList.get(0)); DateTime trackDateTime; final GarminLap tourLap = new GarminLap(); // create tracklist and lap for (final TourData tourData : _tourDataList) { mergeLap(tourLap, tourData, _chkExportNotes.getSelection()); if (isCamouflageSpeed) { trackDateTime = _trackStartDateTime; } else { trackDateTime = TourManager.getTourDateTime(tourData); } final GarminTrack track = getTrack(tourData, trackDateTime, isCamouflageSpeed, camouflageSpeed[0]); if (track != null) { trackList.add(track); } } doExportTour(tourLap, trackList, wayPointList, completeFilePath, fileCollisionBehaviour, isOverwriteFiles); } else { /* * export each tour separately */ final String exportPathName = getExportPathName(); final boolean addNotes = _chkExportNotes.getSelection(); _progressIndicator.beginTask(_tourDataList.size()); final Job exportJob = new Job("export files") { //$NON-NLS-1$ @Override public IStatus run(final IProgressMonitor monitor) { monitor.beginTask(UI.EMPTY_STRING, _tourDataList.size()); final IPath exportFilePath = new Path(exportPathName).addTrailingSeparator(); for (final TourData tourData : _tourDataList) { // get filepath final IPath filePath = exportFilePath.append(UI.format_yyyymmdd_hhmmss(tourData)) .addFileExtension(_exportExtensionPoint.getFileExtension()); final GarminLap tourLap = getLap(tourData, addNotes); // create tracklist trackList.clear(); final GarminTrack track = getTrack(tourData, TourManager.getTourDateTime(tourData), isCamouflageSpeed, camouflageSpeed[0]); if (track != null) { trackList.add(track); /* * update dialog progress monitor */ Display.getDefault().syncExec(new Runnable() { public void run() { // display exported filepath _lblExportedFilePath.setText(NLS.bind( Messages.dialog_export_lbl_exportFilePath, filePath.toOSString())); // !!! force label update !!! _lblExportedFilePath.update(); _progressIndicator.worked(1); } }); try { doExportTour(tourLap, trackList, wayPointList, filePath.toOSString(), fileCollisionBehaviour, isOverwriteFiles); } catch (final IOException e) { e.printStackTrace(); } } // check if overwrite dialog was canceled if (fileCollisionBehaviour.value == FileCollisionBehavior.DIALOG_IS_CANCELED) { break; } } return Status.OK_STATUS; } }; exportJob.schedule(); try { exportJob.join(); } catch (final InterruptedException e) { e.printStackTrace(); } } } }
From source file:com.igniva.filemanager.activities.MainActivity.java
/** * Returns all available SD-Cards in the system (include emulated) * <p>//w w w. j a v a 2 s. c om * Warning: Hack! Based on Android source code of version 4.3 (API 18) * Because there is no standard way to get it. * TODO: Test on future Android versions 4.4+ * * @return paths to all available SD-Cards in the system (include emulated) */ public List<String> getStorageDirectories() { // Final set of paths final ArrayList<String> rv = new ArrayList<>(); // Primary physical SD-CARD (not emulated) final String rawExternalStorage = System.getenv("EXTERNAL_STORAGE"); // All Secondary SD-CARDs (all exclude primary) separated by ":" final String rawSecondaryStoragesStr = System.getenv("SECONDARY_STORAGE"); // Primary emulated SD-CARD final String rawEmulatedStorageTarget = System.getenv("EMULATED_STORAGE_TARGET"); if (TextUtils.isEmpty(rawEmulatedStorageTarget)) { // Device has physical external storage; use plain paths. if (TextUtils.isEmpty(rawExternalStorage)) { // EXTERNAL_STORAGE undefined; falling back to default. rv.add("/storage/sdcard0"); } else { rv.add(rawExternalStorage); } } else { // Device has emulated storage; external storage paths should have // userId burned into them. final String rawUserId; if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR1) { rawUserId = ""; } else { final String path = Environment.getExternalStorageDirectory().getAbsolutePath(); final String[] folders = DIR_SEPARATOR.split(path); final String lastFolder = folders[folders.length - 1]; boolean isDigit = false; try { Integer.valueOf(lastFolder); isDigit = true; } catch (NumberFormatException ignored) { } rawUserId = isDigit ? lastFolder : ""; } // /storage/emulated/0[1,2,...] if (TextUtils.isEmpty(rawUserId)) { rv.add(rawEmulatedStorageTarget); } else { rv.add(rawEmulatedStorageTarget + File.separator + rawUserId); } } // Add all secondary storages if (!TextUtils.isEmpty(rawSecondaryStoragesStr)) { // All Secondary SD-CARDs splited into array final String[] rawSecondaryStorages = rawSecondaryStoragesStr.split(File.pathSeparator); Collections.addAll(rv, rawSecondaryStorages); } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && checkStoragePermission()) rv.clear(); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { String strings[] = FileUtil.getExtSdCardPathsForActivity(this); for (String s : strings) { File f = new File(s); if (!rv.contains(s) && utils.canListFiles(f)) rv.add(s); } } if (BaseActivity.rootMode) rv.add("/"); File usb = getUsbDrive(); if (usb != null && !rv.contains(usb.getPath())) rv.add(usb.getPath()); return rv; }