List of usage examples for java.util Set size
int size();
From source file:com.lightboxtechnologies.spectrum.SequenceFileExport.java
public static void main(String[] args) throws Exception { final Configuration conf = new Configuration(); final String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); String imageID;/*from w w w .j av a 2s.c om*/ String outpath; String friendlyname; final Set<String> exts = new HashSet<String>(); if ("-f".equals(otherArgs[0])) { if (otherArgs.length != 4) { die(); } // load extensions from file final Path extpath = new Path(otherArgs[1]); InputStream in = null; try { in = extpath.getFileSystem(conf).open(extpath); Reader r = null; try { r = new InputStreamReader(in); BufferedReader br = null; try { br = new BufferedReader(r); String line; while ((line = br.readLine()) != null) { exts.add(line.trim().toLowerCase()); } br.close(); } finally { IOUtils.closeQuietly(br); } r.close(); } finally { IOUtils.closeQuietly(r); } in.close(); } finally { IOUtils.closeQuietly(in); } imageID = otherArgs[2]; friendlyname = otherArgs[3]; outpath = otherArgs[4]; } else { if (otherArgs.length < 3) { die(); } // read extensions from trailing args imageID = otherArgs[0]; friendlyname = otherArgs[1]; outpath = otherArgs[2]; // lowercase all file extensions for (int i = 2; i < otherArgs.length; ++i) { exts.add(otherArgs[i].toLowerCase()); } } conf.setStrings("extensions", exts.toArray(new String[exts.size()])); final Job job = SKJobFactory.createJobFromConf(imageID, friendlyname, "SequenceFileExport", conf); job.setJarByClass(SequenceFileExport.class); job.setMapperClass(SequenceFileExportMapper.class); job.setNumReduceTasks(0); job.setOutputKeyClass(BytesWritable.class); job.setOutputValueClass(MapWritable.class); job.setInputFormatClass(FsEntryHBaseInputFormat.class); FsEntryHBaseInputFormat.setupJob(job, imageID); job.setOutputFormatClass(SequenceFileOutputFormat.class); SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK); FileOutputFormat.setOutputPath(job, new Path(outpath)); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:edu.wpi.khufnagle.webimagemanager.WebImageManager.java
/** * Defines information for the lighthouses, then runs the * photograph-collection process./* www . j a v a 2 s .com*/ * @param args Command-line arguments for this program (not used in this * implementation) */ // Auto-boxing done "on the fly" to show progress of downloading images @SuppressWarnings("boxing") public static void main(final String[] args) { final long startTime = System.nanoTime(); System.out.println("***BEGIN PHOTO TRANSFER PROCESS***"); // Add data for lighthouses (next 375 lines or so) final List<LighthouseInfo> lighthouseData = new ArrayList<LighthouseInfo>(); /* * lighthouseData.add(new LighthouseInfo("Statue of Liberty", 40.689348, * -74.044726)); */// Statue of Liberty = 2080 photos w/out restrictions lighthouseData.add(new LighthouseInfo("Portland Head Light", 43.623104, -70.207867)); lighthouseData.add(new LighthouseInfo("Pemaquid Point Light", 43.836970, -69.505997)); lighthouseData.add(new LighthouseInfo("Five Mile Point (New Haven Harbor) Light", 41.248958, -72.903766)); lighthouseData.add(new LighthouseInfo("Cape Neddick (Nubble) Light", 43.165211, -70.591102)); lighthouseData.add(new LighthouseInfo("Portland Breakwater Light", 43.655516, -70.234813)); lighthouseData.add(new LighthouseInfo("Beavertail Light", 41.449368, -71.399372)); lighthouseData.add(new LighthouseInfo("Bass Harbor Head Light", 44.221976, -68.337214)); lighthouseData.add(new LighthouseInfo("Nobska Point Light", 41.515792, -70.655116)); lighthouseData.add(new LighthouseInfo("Spring Point Ledge Light", 43.652108, -70.223922)); lighthouseData.add(new LighthouseInfo("Gay Head Light", 41.348450, -70.834956)); lighthouseData.add(new LighthouseInfo("Derby Wharf Light", 42.516566, -70.883536)); lighthouseData.add(new LighthouseInfo("Rockland Breakwater Light", 44.104006, -69.077453)); lighthouseData.add(new LighthouseInfo("Sandy Neck Light", 41.722647, -70.280927)); lighthouseData.add(new LighthouseInfo("Marblehead Light", 42.505411, -70.833708)); lighthouseData.add(new LighthouseInfo("Portsmouth Harbor Light", 43.071061, -70.708634)); lighthouseData.add(new LighthouseInfo("Highland Light", 42.039122, -70.062025)); lighthouseData.add(new LighthouseInfo("Cape Elizabeth Light", 43.566058, -70.200042)); lighthouseData.add(new LighthouseInfo("Marshall Point Light", 43.917406, -69.261222)); lighthouseData.add(new LighthouseInfo("Chatham Light", 41.671407, -69.949884)); lighthouseData.add(new LighthouseInfo("Block Island Southeast Light", 41.153412, -71.552117)); lighthouseData.add(new LighthouseInfo("Edgartown Light", 41.390863, -70.503057)); lighthouseData.add(new LighthouseInfo("Watch Hill Light", 41.303884, -71.858575)); lighthouseData.add(new LighthouseInfo("Nauset Light", 41.858305, -69.951631)); lighthouseData .add(new LighthouseInfo("Fayerweather Island (Black Rock Harbor) Light", 41.142380, -73.217409)); lighthouseData.add(new LighthouseInfo("Owls Head Light", 44.092138, -69.044105)); lighthouseData.add(new LighthouseInfo("Point Judith Light", 41.361035, -71.481402)); lighthouseData.add(new LighthouseInfo("Sankaty Head Light", 41.284379, -69.966244)); lighthouseData.add(new LighthouseInfo("Eastern Point Light", 42.580229, -70.664537)); lighthouseData.add(new LighthouseInfo("Fort Pickering Light", 42.526473, -70.866465)); lighthouseData.add(new LighthouseInfo("Wood Island Light", 43.456788, -70.328976)); lighthouseData.add(new LighthouseInfo("Stonington Harbor Light", 41.328780, -71.905486)); lighthouseData.add(new LighthouseInfo("West Quoddy Head Light", 44.815073, -66.950742)); lighthouseData.add(new LighthouseInfo("Fort Point Light", 44.467265, -68.811717)); lighthouseData.add(new LighthouseInfo("Annisquam Light", 42.661874, -70.681488)); lighthouseData.add(new LighthouseInfo("Newport Harbor Light", 41.493299, -71.327038)); lighthouseData.add(new LighthouseInfo("Long Point Light", 42.033117, -70.168651)); lighthouseData.add(new LighthouseInfo("Castle Hill Light", 41.462116, -71.362919)); lighthouseData.add(new LighthouseInfo("Brant Point Light", 41.289918, -70.090287)); lighthouseData.add(new LighthouseInfo("Stratford Point Light", 41.151984, -73.103276)); lighthouseData.add(new LighthouseInfo("Boston Light", 42.327925, -70.890101)); lighthouseData.add(new LighthouseInfo("Lynde Point Light", 41.271452, -72.343142)); lighthouseData.add(new LighthouseInfo("Scituate Light", 42.204748, -70.715814)); lighthouseData.add(new LighthouseInfo("Prospect Harbor Light", 44.403285, -68.012922)); lighthouseData.add(new LighthouseInfo("Wood End Light", 42.021223, -70.193502)); lighthouseData.add(new LighthouseInfo("Rose Island Light", 41.495477, -71.342742)); lighthouseData.add(new LighthouseInfo("Saybrook Breakwater Light", 41.263158, -72.342813)); lighthouseData.add(new LighthouseInfo("Great Point Light", 41.390096, -70.048234)); lighthouseData.add(new LighthouseInfo("Cape Poge Light", 41.418798, -70.451923)); lighthouseData.add(new LighthouseInfo("Monhegan Light", 43.764779, -69.316204)); lighthouseData.add(new LighthouseInfo("Hendricks Head Light", 43.822589, -69.689761)); lighthouseData.add(new LighthouseInfo("Egg Rock Light", 44.354050, -68.138166)); lighthouseData.add(new LighthouseInfo("New London Ledge Light", 41.305826, -72.077448)); lighthouseData.add(new LighthouseInfo("Avery Point Lighthouse", 41.315245, -72.063579)); lighthouseData.add(new LighthouseInfo("Palmers Island Light", 41.626936, -70.909109)); lighthouseData.add(new LighthouseInfo("Cuckolds Light", 43.779663, -69.649982)); lighthouseData.add(new LighthouseInfo("Gull Rocks Light", 41.502451, -71.333140)); lighthouseData.add(new LighthouseInfo("Goat Island Light", 43.357826, -70.425109)); lighthouseData.add(new LighthouseInfo("East Chop Light", 41.470245, -70.567439)); lighthouseData.add(new LighthouseInfo("Neds Point Light", 41.650859, -70.795638)); lighthouseData.add(new LighthouseInfo("Sakonnet Point Light", 41.453090, -71.202382)); lighthouseData.add(new LighthouseInfo("Narrows (Bug) Light", 42.323137, -70.919158)); lighthouseData.add(new LighthouseInfo("Plum Island Light", 42.815119, -70.818981)); lighthouseData.add(new LighthouseInfo("Block Island North Light", 41.227639, -71.575811)); lighthouseData.add(new LighthouseInfo("Mount Desert Rock Light", 43.968582, -68.128306)); lighthouseData.add(new LighthouseInfo("Duxbury Pier Light", 41.987375, -70.648498)); lighthouseData.add(new LighthouseInfo("Long Island Head Light", 42.330197, -70.957712)); lighthouseData.add(new LighthouseInfo("Prudence Island Light", 41.605881, -71.303535)); lighthouseData.add(new LighthouseInfo("Plum Beach Light", 41.530248, -71.405202)); lighthouseData.add(new LighthouseInfo("Doubling Point Light", 43.882503, -69.806792)); lighthouseData.add(new LighthouseInfo("Dice Head Light", 44.382732, -68.819022)); lighthouseData.add(new LighthouseInfo("Ram Island Ledge Light", 43.631457, -70.187366)); lighthouseData.add(new LighthouseInfo("New London Harbor Light", 41.316619, -72.089743)); lighthouseData.add(new LighthouseInfo("Lime Rock Light", 41.477536, -71.325924)); lighthouseData.add(new LighthouseInfo("Ten Pound Island Light", 42.601865, -70.665556)); lighthouseData.add(new LighthouseInfo("Bristol Ferry Light", 41.642842, -71.260319)); lighthouseData.add(new LighthouseInfo("Musselbed Shoals Light", 41.636261, -71.259958)); lighthouseData.add(new LighthouseInfo("Conimicut Light", 41.716969, -71.345106)); lighthouseData.add(new LighthouseInfo("Tongue Point Light", 41.166590, -73.177497)); lighthouseData.add(new LighthouseInfo("Bass River Light", 41.651746, -70.169473)); lighthouseData.add(new LighthouseInfo("Hospital Point Light", 42.546413, -70.856164)); lighthouseData.add(new LighthouseInfo("Newburyport Range Light", 42.811524, -70.864838)); lighthouseData.add(new LighthouseInfo("Dutch Island Light", 41.496702, -71.404299)); lighthouseData.add(new LighthouseInfo("Heron Neck Light", 44.025216, -68.861966)); lighthouseData.add(new LighthouseInfo("Pumpkin Island Light", 44.309166, -68.742876)); lighthouseData.add(new LighthouseInfo("Whaleback Light", 43.058744, -70.696306)); lighthouseData.add(new LighthouseInfo("Hyannis Harbor Light", 41.636267, -70.288439)); lighthouseData.add(new LighthouseInfo("Stage Harbor Light", 41.658692, -69.983689)); lighthouseData.add(new LighthouseInfo("Lovells Island Range Light", 42.332440, -70.930214)); lighthouseData.add(new LighthouseInfo("Hog Island Shoal Light", 41.632338, -71.273198)); lighthouseData.add(new LighthouseInfo("Ram Island Light", 43.803935, -69.599349)); lighthouseData.add(new LighthouseInfo("Bridgeport Harbor Light", 41.156718, -73.179950)); lighthouseData.add(new LighthouseInfo("Straitsmouth Island Light", 42.662236, -70.588157)); lighthouseData.add(new LighthouseInfo("Squirrel Point Light", 43.816520, -69.802402)); lighthouseData.add(new LighthouseInfo("Mayos Beach Light", 41.930755, -70.032097)); lighthouseData.add(new LighthouseInfo("Race Point Light", 42.062314, -70.243084)); lighthouseData.add(new LighthouseInfo("Point Gammon Light", 41.609647, -70.266196)); lighthouseData.add(new LighthouseInfo("Wings Neck Light", 41.680235, -70.661250)); lighthouseData.add(new LighthouseInfo("West Chop Light", 41.480806, -70.599796)); lighthouseData.add(new LighthouseInfo("Bird Island Light", 41.669295, -70.717341)); lighthouseData.add(new LighthouseInfo("Clarks Point Light", 41.593176, -70.901416)); lighthouseData.add(new LighthouseInfo("Thacher Island Light", 42.639168, -70.574759)); lighthouseData.add(new LighthouseInfo("White Island Light", 42.967228, -70.623249)); lighthouseData.add(new LighthouseInfo("Wickford Harbor Light", 41.572618, -71.436831)); lighthouseData.add(new LighthouseInfo("Whale Rock Light", 41.444597, -71.423584)); lighthouseData.add(new LighthouseInfo("Burnt Island Light", 43.825133, -69.640262)); lighthouseData.add(new LighthouseInfo("Rockland Harbor Southwest Light", 44.082720, -69.096310)); lighthouseData.add(new LighthouseInfo("Saddleback Ledge Light", 44.014232, -68.726461)); lighthouseData.add(new LighthouseInfo("Grindle Point Light", 44.281451, -68.942967)); lighthouseData.add(new LighthouseInfo("Winter Harbor Light", 44.361421, -68.087742)); lighthouseData.add(new LighthouseInfo("Peck's Ledge Light", 41.077298, -73.369811)); lighthouseData.add(new LighthouseInfo("Sheffield Island Light", 41.048251, -73.419931)); lighthouseData.add(new LighthouseInfo("Whitlocks Mill Light", 45.162793, -67.227395)); lighthouseData.add(new LighthouseInfo("Boon Island Light", 43.121183, -70.475845)); lighthouseData.add(new LighthouseInfo("Southwest Ledge Light", 41.234443, -72.912092)); lighthouseData.add(new LighthouseInfo("Broad Sound Channel Inner Range Light", 42.326933, -70.984649)); lighthouseData.add(new LighthouseInfo("Spectacle Island Light", 42.326898, -70.984772)); lighthouseData.add(new LighthouseInfo("Deer Island Light", 42.339836, -70.954525)); lighthouseData.add(new LighthouseInfo("Nayatt Point Light", 41.725120, -71.338926)); lighthouseData.add(new LighthouseInfo("Doubling Point Range Lights", 43.882860, -69.795652)); lighthouseData.add(new LighthouseInfo("Burkehaven Light", 43.371669, -72.065869)); lighthouseData.add(new LighthouseInfo("Loon Island Light", 43.392123, -72.059977)); lighthouseData.add(new LighthouseInfo("Curtis Island Light", 44.201372, -69.048865)); lighthouseData.add(new LighthouseInfo("Butler Flats Light", 41.603775, -70.894556)); lighthouseData.add(new LighthouseInfo("Graves Light", 42.365098, -70.869191)); lighthouseData.add(new LighthouseInfo("Stamford Harbor Light", 41.013643, -73.542577)); lighthouseData.add(new LighthouseInfo("Billingsgate Light", 41.871624, -70.068982)); lighthouseData.add(new LighthouseInfo("Monomoy Point Light", 41.559310, -69.993650)); lighthouseData.add(new LighthouseInfo("Bishop & Clerks Light", 41.574154, -70.249963)); lighthouseData.add(new LighthouseInfo("Plymouth Light", 42.003737, -70.600565)); lighthouseData.add(new LighthouseInfo("Cleveland Ledge Light", 41.630927, -70.694201)); lighthouseData.add(new LighthouseInfo("Tarpaulin Cove Light", 41.468822, -70.757514)); lighthouseData.add(new LighthouseInfo("Minots Ledge Light", 42.269678, -70.759136)); lighthouseData.add(new LighthouseInfo("Dumpling Rock Light", 41.538167, -70.921427)); lighthouseData.add(new LighthouseInfo("Bakers Island Light", 42.536470, -70.785995)); lighthouseData.add(new LighthouseInfo("Cuttyhunk Light", 41.414391, -70.949558)); lighthouseData.add(new LighthouseInfo("Egg Rock Light", 42.433346, -70.897386)); lighthouseData.add(new LighthouseInfo("Ipswich Range Light", 42.685360, -70.766128)); lighthouseData.add(new LighthouseInfo("Borden Flats Light", 41.704450, -71.174395)); lighthouseData.add(new LighthouseInfo("Bullocks Point Light", 41.737740, -71.364179)); lighthouseData.add(new LighthouseInfo("Pomham Rocks Light", 41.777618, -71.369594)); lighthouseData.add(new LighthouseInfo("Sabin Point Light", 41.762010, -71.374234)); lighthouseData.add(new LighthouseInfo("Fuller Rock Light", 41.794055, -71.379720)); lighthouseData.add(new LighthouseInfo("Gould Island Light", 41.537826, -71.344804)); lighthouseData.add(new LighthouseInfo("Warwick Light", 41.667111, -71.378413)); lighthouseData.add(new LighthouseInfo("Sassafras Point Light", 41.802496, -71.390272)); lighthouseData.add(new LighthouseInfo("Conanicut Light", 41.573484, -71.371767)); lighthouseData.add(new LighthouseInfo("Poplar Point Light", 41.571053, -71.439189)); lighthouseData.add(new LighthouseInfo("Halfway Rock Light", 43.655873, -70.037402)); lighthouseData.add(new LighthouseInfo("Seguin Island Light", 43.707554, -69.758118)); lighthouseData.add(new LighthouseInfo("Pond Island Light", 43.740031, -69.770273)); lighthouseData.add(new LighthouseInfo("Perkins Island Light", 43.786764, -69.785256)); lighthouseData.add(new LighthouseInfo("Latimer Reef Light", 41.304503, -71.933292)); lighthouseData.add(new LighthouseInfo("Morgan Point Light", 41.316669, -71.989327)); lighthouseData.add(new LighthouseInfo("Franklin Island Light", 43.892184, -69.374842)); lighthouseData.add(new LighthouseInfo("Matinicus Rock Light", 43.783605, -68.854898)); lighthouseData.add(new LighthouseInfo("Tenants Harbor Light", 43.961107, -69.184877)); lighthouseData.add(new LighthouseInfo("Whitehead Light", 43.978706, -69.124285)); lighthouseData.add(new LighthouseInfo("Two Bush Island Light", 43.964239, -69.073942)); lighthouseData.add(new LighthouseInfo("Indian Island Light", 44.165470, -69.061004)); lighthouseData.add(new LighthouseInfo("Browns Head Light", 44.111774, -68.909482)); lighthouseData.add(new LighthouseInfo("Goose Rocks Light", 44.135394, -68.830526)); lighthouseData.add(new LighthouseInfo("Sperry Light", 41.221221, -72.423110)); lighthouseData.add(new LighthouseInfo("Isle au Haut Light", 44.064733, -68.651339)); lighthouseData.add(new LighthouseInfo("Deer Island Thorofare Light", 44.134338, -68.703202)); lighthouseData.add(new LighthouseInfo("Herrick Cove Light", 43.411136, -72.041706)); lighthouseData.add(new LighthouseInfo("Eagle Island Light", 44.217634, -68.767743)); lighthouseData.add(new LighthouseInfo("Burnt Coat Harbor Light", 44.134176, -68.447258)); lighthouseData.add(new LighthouseInfo("Faulkner's Island Light", 41.211612, -72.655088)); lighthouseData.add(new LighthouseInfo("Blue Hill Bay Light", 44.248746, -68.497880)); lighthouseData.add(new LighthouseInfo("Great Duck Island Light", 44.142193, -68.245836)); lighthouseData.add(new LighthouseInfo("Bear Island Light", 44.283485, -68.269858)); lighthouseData.add(new LighthouseInfo("Baker Island Light", 44.241266, -68.198923)); lighthouseData.add(new LighthouseInfo("Crabtree Ledge Light", 44.475613, -68.199383)); lighthouseData.add(new LighthouseInfo("Statford Shoal Light", 41.059557, -73.101394)); lighthouseData.add(new LighthouseInfo("Petit Manan Light", 44.367574, -67.864129)); lighthouseData.add(new LighthouseInfo("Penfield Reef Light", 41.117101, -73.222070)); lighthouseData.add(new LighthouseInfo("Narraguagus Light", 44.462467, -67.837844)); lighthouseData.add(new LighthouseInfo("Nash Island Light", 44.464305, -67.747299)); lighthouseData.add(new LighthouseInfo("Moose Peak Light", 44.474244, -67.533471)); lighthouseData.add(new LighthouseInfo("Green's Ledge Light", 41.041551, -73.443974)); lighthouseData.add(new LighthouseInfo("Libby Island Light", 44.568236, -67.367339)); lighthouseData.add(new LighthouseInfo("Great Captain Island Light", 40.982478, -73.623706)); lighthouseData.add(new LighthouseInfo("Avery Rock Light", 44.654358, -67.344137)); lighthouseData.add(new LighthouseInfo("Little River Light", 44.650873, -67.192325)); lighthouseData.add(new LighthouseInfo("Lubec Channel Light", 44.841955, -66.976731)); lighthouseData.add(new LighthouseInfo("St. Croix River Light", 45.128762, -67.133594)); /* * "Clean out" photo directories before beginning photo transfer process. */ final File photosDir = new File("photos"); final File[] photoLighthouseDirsToDelete = photosDir.listFiles(); if (photoLighthouseDirsToDelete != null) { for (final File photoLighthouseDir : photoLighthouseDirsToDelete) { // Use Apache Commons IO (again) to recursively delete the directory // and all of the files within it if (photoLighthouseDir.exists() && photoLighthouseDir.isDirectory()) { try { FileUtils.deleteDirectory(photoLighthouseDir); System.out.println("Deleted directory \"" + photoLighthouseDir + "\" successfully."); } catch (final IOException ioe) { System.err.println( "Could not delete directory: \"" + photoLighthouseDir + "\" successfully!"); } } } } // Keep track of elapsed time long estimatedTime = System.nanoTime() - startTime; String elapsedTime = WebImageManager.calculateElapsedTime(estimatedTime); System.out.println("Estimated elapsed time: " + elapsedTime + "."); System.out.println(); /* * Keep track of total number of photographs transferred from Flickr * websites to disks across _all_ lighthouses */ int totalNumPhotosTransferred = 0; /* * Keep track of total number of photographs that _should_ be transferred * from Flickr for _all_ lighthouses */ int totalNumPhotos = 0; for (final LighthouseInfo lighthousePieceOfData : lighthouseData) { System.out.println("Processing photos of " + lighthousePieceOfData.getName() + "..."); /* * URL for accessing Flickr APIs. For a given lighthouse, this URL * provides an XML file in response that lists information about every * geotagged, Creative Commons-enabled photograph for that lighthouse * on Flickr. */ // GET Parameter Explanation: // method - Use the "search photos" method for the Flickr APIs // // api_key - A unique key that I use to get the results // // text - Find all lighthouses whose title, tags, or description // contains the word "lighthouse" // // license - Find all photos with a Creative Commons license _except_ // those that do not allow for modification on my part // // content_type - Find photos only (no videos) // // has_geo - Implicitly set to true; implies that all photos are // geotagged // // lat - The latitude of the center of the "search circle" // // lon - The longitude of the center of the "search circle" // // radius - The radius of the "search circle," in _kilometers_ (NOT // miles) // // extras - Also include a URL to the "raw" photo (small version) final String inputURLText = "http://ycpi.api.flickr.com/services/rest/?" + "method=flickr.photos.search" + "&api_key=3ea8366b020383eb91f170c6f41748f5" + "&text=lighthouse" + "&license=1,2,4,5,7" + "&content_type=1" + "&has_geo" + "&lat=" + lighthousePieceOfData.getLatitude() + "&lon=" + lighthousePieceOfData.getLongitude() + "&radius=1" + "&extras=url_s"; // Output file where XML web response will be stored temporarily final String outputFileName = "output.xml"; /* * Convert the name of the lighthouse to a "computer friendly" version * with all lower-case letters and underscores replacing spaces, * apostrophes, and parenthesis */ String lighthouseName = lighthousePieceOfData.getName(); lighthouseName = lighthouseName.toLowerCase(); lighthouseName = lighthouseName.replace(' ', '_'); lighthouseName = lighthouseName.replace('\'', '_'); lighthouseName = lighthouseName.replace('(', '_'); lighthouseName = lighthouseName.replace(')', '_'); // Will contain the textual links to each "raw" photo website Set<String> rawPhotoURLs = new HashSet<String>(); // Make sure file for XML output does not exist at first // (don't want to use an old, incorrect version accidentally) final File outputXMLFile = new File(outputFileName); if (outputXMLFile.exists()) { outputXMLFile.delete(); } System.out.println("Cleaned output XML file containing photo URLs on disk successfully."); /* * Access the list of photographs for a given lighthouse and copy them * to the XML file on disk */ final WebDataExtractor extractor = new WebDataExtractor(inputURLText, outputFileName); System.out.println("Looking for XML file containing lighthosue photo information..."); extractor.transferURLToFile(); System.out.println("Found XML file containing lighthouse photo URLs."); /* * Object for extracting the "raw" URLs from each piece of photo data * in the XML file */ final XMLParser parser = new FlickrXMLOutputParser(outputFileName); // Complete the extraction process rawPhotoURLs = parser.parseFile("//photo/@url_s"); final int numPhotos = rawPhotoURLs.size(); totalNumPhotos += numPhotos; int i = 0; // Counter for keeping track of progress /* * Keep track of photos transferred successfully (which might be less * than the total number of photos defined int the XML output from * Flickr, especially if connection issues occur */ int numPhotosTransferred = 0; for (final String photoURL : rawPhotoURLs) { System.out.print("Transferring photos..."); i++; /* * Go to a website containing a "raw" JPEG image file and save it * accordingly on disk in the photo folder corresponding to the * lighthouse name */ final WebDataExtractor rawPhotoExtractor = new WebDataExtractor(photoURL, "photos/" + lighthouseName + "/lighthouse_photo_" + Integer.toString(i) + ".jpg"); final boolean transferSuccessful = rawPhotoExtractor.transferURLToFile(); if (transferSuccessful) { numPhotosTransferred++; } // Simple progress tracker System.out.printf("%d of %d (%.1f%%) complete.\n", i, numPhotos, i * 1.0 / numPhotos * 100.0); } // Indicate number of photos successfully transferred to disk if (numPhotosTransferred == numPhotos && numPhotos > 0) { System.out.println("All photos transferred to disk successfully!"); } else if (numPhotos == 0) { System.out.println("It appears there are no photos available for this lighthouse..."); } else if (numPhotosTransferred == 1 && numPhotos > 1) { System.out.println("1 photo transferred to disk successfully."); } else if (numPhotosTransferred == 1 && numPhotos == 1) { System.out.println("The photo transferred to disk successfully!"); } else { System.out.println(numPhotosTransferred + " photos transferred to disk successfully."); } // Keep track of elapsed time estimatedTime = System.nanoTime() - startTime; elapsedTime = WebImageManager.calculateElapsedTime(estimatedTime); System.out.println("Estimated elapsed time: " + elapsedTime + "."); // Add extra line in between lighthouses in output stream System.out.println(); /* * Keep track of total number of photos transferred so far across * _all_lighthouses */ totalNumPhotosTransferred += numPhotosTransferred; } // Display "grand" total (which is hopefully greater than 0) System.out.println("***GRAND TOTAL: " + totalNumPhotosTransferred + " OF " + totalNumPhotos + " PHOTOS TRANSFERRED SUCCESSFULLY***"); estimatedTime = System.nanoTime() - startTime; elapsedTime = WebImageManager.calculateElapsedTime(estimatedTime); System.out.println("TOTAL ELAPSED TIME: " + elapsedTime.toUpperCase()); }
From source file:jetbrains.exodus.env.Reflect.java
public static void main(@NotNull final String[] args) throws Exception { if (args.length == 0) { printUsage();/*from w w w. j a v a 2 s . c o m*/ return; } String envPath = null; String envPath2 = null; boolean hasOptions = false; boolean gatherLogStats = false; boolean validateRoots = false; boolean traverse = false; boolean copy = false; boolean traverseAll = false; boolean utilizationInfo = false; final Set<String> files2Clean = new LinkedHashSet<>(); for (final String arg : args) { if (arg.startsWith("-")) { hasOptions = true; if ("-ls".equalsIgnoreCase(arg)) { gatherLogStats = true; } else if ("-r".equalsIgnoreCase(arg)) { validateRoots = true; } else if ("-t".equalsIgnoreCase(arg)) { traverse = true; } else if ("-c".equalsIgnoreCase(arg)) { copy = true; } else if ("-ta".equalsIgnoreCase(arg)) { traverseAll = true; } else if ("-u".equalsIgnoreCase(arg)) { utilizationInfo = true; } else if (arg.toLowerCase().startsWith("-cl")) { files2Clean.add(arg.substring(2)); } else { printUsage(); return; } } else { if (envPath == null) { envPath = arg; } else { envPath2 = arg; break; } } } if (envPath == null || (copy && envPath2 == null)) { printUsage(); return; } System.out.println("Started investigation of " + envPath); final Reflect reflect = new Reflect(new File(envPath)); if (files2Clean.size() > 0) { for (final String file : files2Clean) { reflect.cleanFile(file); } } if (!hasOptions) { reflect.gatherLogStats(); reflect.traverse(); } else { List<DatabaseRoot> roots = null; if (validateRoots || traverseAll) { roots = reflect.roots(); } if (gatherLogStats) { reflect.gatherLogStats(); } if (traverse) { reflect.traverse(); } if (copy) { reflect.copy(new File(envPath2)); } if (utilizationInfo) { reflect.spaceInfoFromUtilization(); } if (traverseAll) { reflect.traverseAll(roots); } } }
From source file:de.uni_rostock.goodod.checker.CheckerApp.java
public static void main(String[] args) throws OWLOntologyCreationException { config = Configuration.getConfiguration(args); String bioTopVariantA = "biotoplite_group_A_TEST.owl"; String bioTopVariantB = "biotoplite_group_B_TEST.owl"; String repoRoot = config.getString("repositoryRoot"); File commonBioTopF = new File(repoRoot + File.separator + config.getString("bioTopLiteSource")); String groupAFile = repoRoot + File.separator + "Results" + File.separator + "GruppeA" + File.separator + bioTopVariantA;//from w w w . j ava2s . c o m String groupBFile = repoRoot + File.separator + "Results" + File.separator + "GruppeB" + File.separator + bioTopVariantB; String testFile = config.getString("testDescription"); IRI bioTopIRI = IRI.create("http://purl.org/biotop/biotoplite.owl"); SimpleIRIMapper bioTopLiteMapper = new SimpleIRIMapper(bioTopIRI, IRI.create(commonBioTopF)); SimpleIRIMapper variantMapperA = new SimpleIRIMapper( IRI.create("http://purl.org/biotop/biotoplite_group_A_TEST.owl"), IRI.create(new File(groupAFile))); SimpleIRIMapper variantMapperB = new SimpleIRIMapper( IRI.create("http://purl.org/biotop/biotoplite_group_B_TEST.owl"), IRI.create(new File(groupBFile))); //logger.info("Loading ontology " + testFile + "."); OWLOntologyManager manager = OWLManager.createOWLOntologyManager(); manager.addIRIMapper(variantMapperA); manager.addIRIMapper(variantMapperB); manager.addIRIMapper(bioTopLiteMapper); FileDocumentSource source = new FileDocumentSource(new File(testFile)); OWLOntology ontology = null; try { ontology = manager.loadOntologyFromOntologyDocument(source); } catch (Throwable e) { logger.fatal("Loading failed", e); System.exit(1); } org.semanticweb.HermiT.Configuration reasonerConfig = new org.semanticweb.HermiT.Configuration(); reasonerConfig.throwInconsistentOntologyException = false; //ReasonerProgressMonitor monitor = new ConsoleProgressMonitor(); reasonerConfig.existentialStrategyType = ExistentialStrategyType.INDIVIDUAL_REUSE; //reasonerConfig.reasonerProgressMonitor = monitor; reasonerConfig.tableauMonitorType = TableauMonitorType.NONE; //reasonerConfig.individualTaskTimeout = 10000; Reasoner reasoner = new Reasoner(reasonerConfig, ontology); reasoner.classifyClasses(); Set<OWLClass> before = reasoner.getUnsatisfiableClasses() .getEntitiesMinus(manager.getOWLDataFactory().getOWLNothing()); //logger.info("Found " + before.size() + " inconsistent classes before import change."); logger.debug(before); reasoner.dispose(); reasoner = null; manager.removeOntology(ontology); ontology = null; Map<IRI, IRI> importMap = new HashMap<IRI, IRI>(); OWLOntologyLoaderConfiguration interimConfig = new OWLOntologyLoaderConfiguration(); for (String str : config.getStringArray("ignoredImports")) { IRI ignoredIRI = IRI.create(str); importMap.put(ignoredIRI, bioTopIRI); interimConfig = interimConfig.addIgnoredImport(ignoredIRI); } interimConfig = interimConfig.setMissingImportHandlingStrategy(MissingImportHandlingStrategy.SILENT); try { ontology = manager.loadOntologyFromOntologyDocument(source, interimConfig); } catch (Throwable e) { logger.fatal("Loading failed", e); System.exit(1); } BasicImportingNormalizerFactory n = new BasicImportingNormalizerFactory(importMap, interimConfig); n.normalize(ontology); reasoner = new Reasoner(reasonerConfig, ontology); reasoner.classifyClasses(); Set<OWLClass> after = reasoner.getUnsatisfiableClasses() .getEntitiesMinus(manager.getOWLDataFactory().getOWLNothing()); //logger.info("Found " + after.size() + " inconsistent classes after import change."); logger.debug(after); /* * We need some tidying afterwards. The after set can contain * inconsistent classes that are inconsistent only because in the new * import, they are subclasses of a class that was already inconsistent before. * Hence we remove them from the after set. */ for (OWLClass c : before) { Set<OWLClass> subclasses = SubClassCollector.collect(c, manager.getImportsClosure(ontology)); for (OWLClass subC : subclasses) { if ((true == after.contains(subC)) && (false == before.contains(subC))) { after.remove(subC); } } } int difference = before.size() - after.size(); if (0 == difference) { logger.info(testFile + ": OK"); } else { logger.warn(testFile + ": Import change is not neutral to inconsistencies (" + before.size() + '/' + after.size() + ")"); } }
From source file:com.act.lcms.db.io.PrintConstructInfo.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());//from w ww . j a va 2s . co m } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } File lcmsDir = new File(cl.getOptionValue(OPTION_DIRECTORY)); if (!lcmsDir.isDirectory()) { System.err.format("File at %s is not a directory\n", lcmsDir.getAbsolutePath()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } try (DB db = DB.openDBFromCLI(cl)) { System.out.print("Loading/updating LCMS scan files into DB\n"); ScanFile.insertOrUpdateScanFilesInDirectory(db, lcmsDir); String construct = cl.getOptionValue(OPTION_CONSTRUCT); List<LCMSWell> lcmsWells = LCMSWell.getInstance().getByConstructID(db, construct); Collections.sort(lcmsWells, new Comparator<LCMSWell>() { @Override public int compare(LCMSWell o1, LCMSWell o2) { return o1.getId().compareTo(o2.getId()); } }); Set<String> uniqueMSIDs = new HashSet<>(); Map<Integer, Plate> platesById = new HashMap<>(); System.out.format("\n\n-- Construct %s --\n\n", construct); List<ChemicalAssociatedWithPathway> pathwayChems = ChemicalAssociatedWithPathway.getInstance() .getChemicalsAssociatedWithPathwayByConstructId(db, construct); System.out.print("Chemicals associated with pathway:\n"); System.out.format(" %-8s%-15s%-45s\n", "index", "kind", "chemical"); for (ChemicalAssociatedWithPathway chem : pathwayChems) { System.out.format(" %-8d%-15s%-45s\n", chem.getIndex(), chem.getKind(), chem.getChemical()); } System.out.print("\nLCMS wells:\n"); System.out.format(" %-15s%-6s%-15s%-15s%-15s\n", "barcode", "well", "msid", "fed", "lcms_count"); for (LCMSWell well : lcmsWells) { uniqueMSIDs.add(well.getMsid()); Plate p = platesById.get(well.getPlateId()); if (p == null) { // TODO: migrate Plate to be a subclass of BaseDBModel. p = Plate.getPlateById(db, well.getPlateId()); platesById.put(p.getId(), p); } String chem = well.getChemical(); List<ScanFile> scanFiles = ScanFile.getScanFileByPlateIDRowAndColumn(db, p.getId(), well.getPlateRow(), well.getPlateColumn()); System.out.format(" %-15s%-6s%-15s%-15s%-15d\n", p.getBarcode(), well.getCoordinatesString(), well.getMsid(), chem == null || chem.isEmpty() ? "--" : chem, scanFiles.size()); System.out.flush(); } List<Integer> plateIds = Arrays.asList(platesById.keySet().toArray(new Integer[platesById.size()])); Collections.sort(plateIds); System.out.print("\nAppears in plates:\n"); for (Integer id : plateIds) { Plate p = platesById.get(id); System.out.format(" %s: %s\n", p.getBarcode(), p.getName()); } List<String> msids = Arrays.asList(uniqueMSIDs.toArray(new String[uniqueMSIDs.size()])); Collections.sort(msids); System.out.format("\nMSIDS: %s\n", StringUtils.join(msids, ", ")); Set<String> availableNegativeControls = new HashSet<>(); for (Map.Entry<Integer, Plate> entry : platesById.entrySet()) { List<LCMSWell> wells = LCMSWell.getInstance().getByPlateId(db, entry.getKey()); for (LCMSWell well : wells) { if (!construct.equals(well.getComposition())) { availableNegativeControls.add(well.getComposition()); } } } // Print available standards for each step w/ plate barcodes and coordinates. System.out.format("\nAvailable Standards:\n"); Map<Integer, Plate> plateCache = new HashMap<>(); for (ChemicalAssociatedWithPathway chem : pathwayChems) { List<StandardWell> matchingWells = StandardWell.getInstance().getStandardWellsByChemical(db, chem.getChemical()); for (StandardWell well : matchingWells) { if (!plateCache.containsKey(well.getPlateId())) { Plate p = Plate.getPlateById(db, well.getPlateId()); plateCache.put(p.getId(), p); } } Map<Integer, List<StandardWell>> standardWellsByPlateId = new HashMap<>(); for (StandardWell well : matchingWells) { List<StandardWell> plateWells = standardWellsByPlateId.get(well.getPlateId()); if (plateWells == null) { plateWells = new ArrayList<>(); standardWellsByPlateId.put(well.getPlateId(), plateWells); } plateWells.add(well); } List<Pair<String, Integer>> plateBarcodes = new ArrayList<>(plateCache.size()); for (Plate p : plateCache.values()) { if (p.getBarcode() == null) { plateBarcodes.add(Pair.of("(no barcode)", p.getId())); } else { plateBarcodes.add(Pair.of(p.getBarcode(), p.getId())); } } Collections.sort(plateBarcodes); System.out.format(" %s:\n", chem.getChemical()); for (Pair<String, Integer> barcodePair : plateBarcodes) { // TODO: hoist this whole sorting/translation step into a utility class. List<StandardWell> wells = standardWellsByPlateId.get(barcodePair.getRight()); if (wells == null) { // Don't print plates that don't apply to this chemical, which can happen because we're caching the plates. continue; } Collections.sort(wells, new Comparator<StandardWell>() { @Override public int compare(StandardWell o1, StandardWell o2) { int c = o1.getPlateRow().compareTo(o2.getPlateRow()); if (c != 0) return c; return o1.getPlateColumn().compareTo(o2.getPlateColumn()); } }); List<String> descriptions = new ArrayList<>(wells.size()); for (StandardWell well : wells) { descriptions.add(String.format("%s in %s%s", well.getCoordinatesString(), well.getMedia(), well.getConcentration() == null ? "" : String.format(" c. %f", well.getConcentration()))); } System.out.format(" %s: %s\n", barcodePair.getLeft(), StringUtils.join(descriptions, ", ")); } } List<String> negativeControlStrains = Arrays .asList(availableNegativeControls.toArray(new String[availableNegativeControls.size()])); Collections.sort(negativeControlStrains); System.out.format("\nAvailable negative controls: %s\n", StringUtils.join(negativeControlStrains, ",")); System.out.print("\n----------\n"); System.out.print("\n\n"); } }
From source file:com.act.lcms.db.analysis.IonSearchAnalysis.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());//from w ww .ja v a2 s . c o m } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } File lcmsDir = new File(cl.getOptionValue(OPTION_DIRECTORY)); if (!lcmsDir.isDirectory()) { System.err.format("File at %s is not a directory\n", lcmsDir.getAbsolutePath()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } Double fontScale = null; if (cl.hasOption("font-scale")) { try { fontScale = Double.parseDouble(cl.getOptionValue("font-scale")); } catch (IllegalArgumentException e) { System.err.format("Argument for font-scale must be a floating point number.\n"); System.exit(1); } } try (DB db = DB.openDBFromCLI(cl)) { Set<String> includeIons = null; if (cl.hasOption("include-ions")) { String[] ionNames = cl.getOptionValues("include-ions"); includeIons = new HashSet<>(Arrays.asList(ionNames)); System.out.format("Including ions in search: %s\n", StringUtils.join(includeIons, ", ")); } Set<String> excludeIons = null; if (cl.hasOption("exclude-ions")) { String[] ionNames = cl.getOptionValues("exclude-ions"); excludeIons = new HashSet<>(Arrays.asList(ionNames)); System.out.format("Excluding ions from search: %s\n", StringUtils.join(excludeIons, ", ")); } Set<Integer> takeSamplesFromPlateIds = null; if (cl.hasOption(OPTION_FILTER_BY_PLATE_BARCODE)) { String[] plateBarcodes = cl.getOptionValues(OPTION_FILTER_BY_PLATE_BARCODE); System.out.format("Considering only sample wells in plates: %s\n", StringUtils.join(plateBarcodes, ", ")); takeSamplesFromPlateIds = new HashSet<>(plateBarcodes.length); for (String plateBarcode : plateBarcodes) { Plate p = Plate.getPlateByBarcode(db, plateBarcode); if (p == null) { System.err.format("WARNING: unable to find plate in DB with barcode %s\n", plateBarcode); } else { takeSamplesFromPlateIds.add(p.getId()); } } // Allow filtering on barcode even if we couldn't find any in the DB. } System.out.format("Loading/updating LCMS scan files into DB\n"); ScanFile.insertOrUpdateScanFilesInDirectory(db, lcmsDir); System.out.format("Processing LCMS scans\n"); Pair<List<LCMSWell>, Set<Integer>> positiveWellsAndPlateIds = Utils.extractWellsAndPlateIds(db, cl.getOptionValues(OPTION_STRAINS), cl.getOptionValues(OPTION_CONSTRUCTS), takeSamplesFromPlateIds, false); List<LCMSWell> positiveWells = positiveWellsAndPlateIds.getLeft(); if (positiveWells.size() == 0) { throw new RuntimeException("Found no LCMS wells for specified strains/constructs"); } // Only take negative samples from the plates where we found the positive samples. Pair<List<LCMSWell>, Set<Integer>> negativeWellsAndPlateIds = Utils.extractWellsAndPlateIds(db, cl.getOptionValues(OPTION_NEGATIVE_STRAINS), cl.getOptionValues(OPTION_NEGATIVE_CONSTRUCTS), positiveWellsAndPlateIds.getRight(), true); List<LCMSWell> negativeWells = negativeWellsAndPlateIds.getLeft(); if (negativeWells == null || negativeWells.size() == 0) { System.err.format("WARNING: no valid negative samples found in same plates as positive samples\n"); } // Extract the reference MZ that will be used in the LCMS trace processing. List<Pair<String, Double>> searchMZs = null; Set<CuratedChemical> standardChemicals = null; List<ChemicalAssociatedWithPathway> pathwayChems = null; if (cl.hasOption(OPTION_SEARCH_MZ)) { // Assume mz can be an FP number of a chemical name. String massStr = cl.getOptionValue(OPTION_SEARCH_MZ); Pair<String, Double> searchMZ = Utils.extractMassFromString(db, massStr); if (searchMZ != null) { searchMZs = Collections.singletonList(searchMZ); } standardChemicals = Utils.extractTargetsForWells(db, positiveWells); } else { CuratedChemical targetChemical = Utils.requireOneTarget(db, positiveWells); if (targetChemical == null) { throw new RuntimeException( "Unable to find a curated chemical entry for specified strains'/constructs' targets. " + "Please specify a chemical name or m/z explicitly or update the curated chemicals list in the DB."); } System.out.format("Using reference M/Z for positive target %s (%f)\n", targetChemical.getName(), targetChemical.getMass()); searchMZs = Collections.singletonList(Pair.of(targetChemical.getName(), targetChemical.getMass())); standardChemicals = Collections.singleton(targetChemical); } // Look up the standard by name, or use the target if none is specified. List<StandardWell> standardWells = null; if (cl.hasOption(OPTION_NO_STANDARD)) { System.err.format("WARNING: skipping standard comparison (no-standard option specified)\n"); standardWells = new ArrayList<>(0); } else if (cl.hasOption(OPTION_STANDARD_WELLS)) { String[] standardCoordinates = cl.getOptionValues(OPTION_STANDARD_WELLS); standardWells = new ArrayList<>(standardCoordinates.length); Plate standardPlate = Plate.getPlateByBarcode(db, cl.getOptionValue(OPTION_STANDARD_PLATE_BARCODE)); List<String> foundCoordinates = new ArrayList<>(standardCoordinates.length); for (String stringCoords : standardCoordinates) { Pair<Integer, Integer> coords = Utils.parsePlateCoordinates(stringCoords); StandardWell well = StandardWell.getInstance().getStandardWellsByPlateIdAndCoordinates(db, standardPlate.getId(), coords.getLeft(), coords.getRight()); if (well == null) { System.err.format("Unable to find standard well at %s [%s]\n", standardPlate.getBarcode(), stringCoords); continue; } standardWells.add(well); foundCoordinates.add(stringCoords); } System.out.format("Using explicitly specified standard wells %s [%s]\n", standardPlate.getBarcode(), StringUtils.join(foundCoordinates, ", ")); } else if (cl.hasOption(OPTION_STANDARD_NAME)) { String standardName = cl.getOptionValue(OPTION_STANDARD_NAME); System.out.format("Using explicitly specified standard %s\n", standardName); standardWells = Collections.singletonList(Utils.extractStandardWellFromPlate(db, cl.getOptionValue(OPTION_STANDARD_PLATE_BARCODE), standardName)); } else if (standardChemicals != null && standardChemicals.size() > 0) { // Default to using the target chemical(s) as a standard if none is specified. standardWells = new ArrayList<>(standardChemicals.size()); for (CuratedChemical c : standardChemicals) { String standardName = c.getName(); System.out.format("Searching for well containing standard %s\n", standardName); standardWells.add(Utils.extractStandardWellFromPlate(db, cl.getOptionValue(OPTION_STANDARD_PLATE_BARCODE), standardName)); } } boolean useFineGrainedMZ = cl.hasOption("fine-grained-mz"); boolean useSNR = cl.hasOption(OPTION_USE_SNR); /* Process the standard, positive, and negative wells, producing ScanData containers that will allow them to be * iterated over for graph writing. */ HashMap<Integer, Plate> plateCache = new HashMap<>(); Pair<List<ScanData<StandardWell>>, Double> allStandardScans = AnalysisHelper.processScans(db, lcmsDir, searchMZs, ScanData.KIND.STANDARD, plateCache, standardWells, useFineGrainedMZ, includeIons, excludeIons, useSNR); Pair<List<ScanData<LCMSWell>>, Double> allPositiveScans = AnalysisHelper.processScans(db, lcmsDir, searchMZs, ScanData.KIND.POS_SAMPLE, plateCache, positiveWells, useFineGrainedMZ, includeIons, excludeIons, useSNR); Pair<List<ScanData<LCMSWell>>, Double> allNegativeScans = AnalysisHelper.processScans(db, lcmsDir, searchMZs, ScanData.KIND.NEG_CONTROL, plateCache, negativeWells, useFineGrainedMZ, includeIons, excludeIons, useSNR); String fmt = "pdf"; String outImg = cl.getOptionValue(OPTION_OUTPUT_PREFIX) + "." + fmt; String outData = cl.getOptionValue(OPTION_OUTPUT_PREFIX) + ".data"; System.err.format("Writing combined scan data to %s and graphs to %s\n", outData, outImg); produceLCMSSearchPlots(lcmsDir, outData, outImg, allStandardScans, allPositiveScans, allNegativeScans, fontScale, useFineGrainedMZ, cl.hasOption(OPTION_USE_HEATMAP), useSNR); } }
From source file:amie.keys.CSAKey.java
public static void main(String[] args) throws IOException, InterruptedException { final Triple<MiningAssistant, Float, String> parsedArgs = parseArguments(args); final Set<Rule> output = new LinkedHashSet<>(); // Helper object that contains the implementation for the calculation // of confidence and support // The file with the non-keys, one per line long timea = System.currentTimeMillis(); List<List<String>> inputNonKeys = Utilities.parseNonKeysFile(parsedArgs.third); System.out.println(inputNonKeys.size() + " input non-keys"); final List<List<String>> nonKeys = pruneBySupport(inputNonKeys, parsedArgs.second, parsedArgs.first.getKb());/* ww w .j a v a 2 s . c om*/ Collections.sort(nonKeys, new Comparator<List<String>>() { @Override public int compare(List<String> o1, List<String> o2) { int r = Integer.compare(o2.size(), o1.size()); if (r == 0) { return Integer.compare(o2.hashCode(), o1.hashCode()); } return r; } }); System.out.println(nonKeys.size() + " non-keys after pruning"); int totalLoad = computeLoad(nonKeys); System.out.println(totalLoad + " is the total load"); int nThreads = Runtime.getRuntime().availableProcessors(); //int batchSize = Math.max(Math.min(maxBatchSize, totalLoad / nThreads), minBatchSize); int batchSize = Math.max(Math.min(maxLoad, totalLoad / nThreads), minLoad); final Queue<int[]> chunks = new PriorityQueue(50, new Comparator<int[]>() { @Override public int compare(int[] o1, int[] o2) { return Integer.compare(o2[2], o1[2]); } }); final HashSet<HashSet<Integer>> nonKeysInt = new HashSet<>(); final HashMap<String, Integer> property2Id = new HashMap<>(); final HashMap<Integer, String> id2Property = new HashMap<>(); final List<Integer> propertiesList = new ArrayList<>(); int support = (int) parsedArgs.second.floatValue(); KB kb = parsedArgs.first.getKb(); buildDictionaries(nonKeys, nonKeysInt, property2Id, id2Property, propertiesList, support, kb); final List<HashSet<Integer>> nonKeysIntList = new ArrayList<>(nonKeysInt); int start = 0; int[] nextIdx = nextIndex(nonKeysIntList, 0, batchSize); int end = nextIdx[0]; int load = nextIdx[1]; while (start < nonKeysIntList.size()) { int[] chunk = new int[] { start, end, load }; chunks.add(chunk); start = end; nextIdx = nextIndex(nonKeysIntList, end, batchSize); end = nextIdx[0]; load = nextIdx[1]; } Thread[] threads = new Thread[Math.min(Runtime.getRuntime().availableProcessors(), chunks.size())]; for (int i = 0; i < threads.length; ++i) { threads[i] = new Thread(new Runnable() { @Override public void run() { while (true) { int[] chunk = null; synchronized (chunks) { if (!chunks.isEmpty()) { chunk = chunks.poll(); } else { break; } } System.out.println("Processing chunk " + Arrays.toString(chunk)); mine(parsedArgs, nonKeysIntList, property2Id, id2Property, propertiesList, chunk[0], chunk[1], output); } } }); threads[i].start(); } for (int i = 0; i < threads.length; ++i) { threads[i].join(); } long timeb = System.currentTimeMillis(); System.out.println("==== Unique C-keys ====="); for (Rule r : output) { System.out.println(Utilities.formatKey(r)); } System.out.println( "VICKEY found " + output.size() + " unique conditional keys in " + (timeb - timea) + " ms"); }
From source file:Main.java
static int countDistinctElements(List source, int position, int count) { List list = source.subList(position, position + count); Set set = new HashSet(); set.addAll(list);//from w w w . j a v a 2 s. c o m return set.size(); }
From source file:Main.java
public static String getTheOnlyElementOf(Set<String> set) { if (set.size() != 1) { throw new RuntimeException("Expected a set of one element"); //$NON-NLS-1$ }// w w w . j a v a 2 s . c o m return set.toArray(new String[1])[0]; }
From source file:Main.java
public static boolean emptySet(Set<?> set) { return (null != set && set.size() <= 0); }