List of usage examples for java.io RandomAccessFile readByte
public final byte readByte() throws IOException
From source file:dk.statsbiblioteket.util.LineReaderTest.java
public void testReadByte() throws Exception { RandomAccessFile ra = new RandomAccessFile(logfile, "r"); LineReader lr = new LineReader(logfile, "r"); int counter = 1; while (!lr.eof()) { assertEquals("Byte #" + counter++ + " should be read correct", ra.readByte(), lr.readByte()); }/*from w w w. j a va 2s . c om*/ ra.close(); lr.close(); }
From source file:translator.logic.AllVendorAnnotationTranslator.java
/** * Reads an edf file and return an array containing start date, time and duration(in seconds) * @param edfFile edf to be read/* www . j ava 2 s .com*/ * @return the string array containing start date, time and duration(in seconds) */ public String[] readEDF(String edfFile) { String[] startDate = new String[2]; @SuppressWarnings("unused") SimpleDateFormat df = new SimpleDateFormat("mm.dd.yyyy hh.mm.ss"); try { RandomAccessFile edf = new RandomAccessFile(new File(edfFile), "r"); edf.seek(168); char[] date = new char[8]; for (int i = 0; i < 8; i++) { date[i] = (char) edf.readByte(); } // edf.read(date); char[] time = new char[8]; for (int i = 0; i < 8; i++) { time[i] = (char) edf.readByte(); } edf.seek(236); char[] numRec = new char[8]; for (int i = 0; i < 8; i++) { numRec[i] = (char) edf.readByte(); //System.out.println(dur[i]); } char[] durRec = new char[8]; for (int i = 0; i < 8; i++) { durRec[i] = (char) edf.readByte(); //System.out.println(dur[i]); } // long numRec = edf.readLong(); // long durRec = edf.readLong(); long duration = Long.parseLong(String.valueOf(durRec).trim()) * Long.parseLong(String.valueOf(numRec).trim()); // long duration = 0; // edf.read(time); startDate[0] = String.valueOf(date) + " " + String.valueOf(time); startDate[1] = String.valueOf(duration); edf.close(); } catch (Exception e) { e.printStackTrace(); StringWriter errors = new StringWriter(); e.printStackTrace(new PrintWriter(errors)); log(errors.toString()); } return startDate; }
From source file:org.commoncrawl.service.crawler.CrawlLog.java
/** * seek out next instance of sync bytes in the file input stream * /* www. j a v a 2 s. c o m*/ * @param file * @throws IOException */ private static boolean seekToNextSyncBytesPos(byte[] syncBytesBuffer, RandomAccessFile file, long maxFileSize) throws IOException { while (file.getFilePointer() < maxFileSize) { try { // read in a sync.length buffer amount file.read(syncBytesBuffer); int syncLen = SYNC_BYTES_SIZE; // start scan for next sync position ... for (int i = 0; file.getFilePointer() < maxFileSize; i++) { int j = 0; for (; j < syncLen; j++) { if (_sync[j] != syncBytesBuffer[(i + j) % syncLen]) break; } if (j == syncLen) { // found matching sync bytes - reset file pos to before sync bytes file.seek(file.getFilePointer() - SYNC_BYTES_SIZE); // position // before // sync return true; } syncBytesBuffer[i % syncLen] = file.readByte(); } } catch (IOException e) { LOG.warn("IOException at:" + file.getFilePointer() + " Exception:" + CCStringUtils.stringifyException(e)); LOG.warn("Skipping to:" + file.getFilePointer() + 4096); file.seek(file.getFilePointer() + 4096); } } return false; }
From source file:big.BigZip.java
/** * Version 2 that permits to extract the text from a compressed file without * creating any file on the disk.//from www . j a v a 2s .c o m * @param filePosition * @return The source code of the compressed file */ public String extractBytesToRAM(final long filePosition) { String result = null; try { // add the signature bytes to our start position long startPosition = filePosition + magicSignature.length(); // enable random access to the BIG file (fast as heck) RandomAccessFile dataBIG = new RandomAccessFile(fileMainBIG, "r"); // jump directly to the position where the file is positioned dataBIG.seek(startPosition); // create a byte array ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); // get the end of this file entry (by brute-force) char test = 0; long endPosition = -1; while (test != -1) { test = dataBIG.readChar(); // if the magic devil number was found.. if (test == 66) { // read the next value for confirmation byte value = dataBIG.readByte(); if (value != 73) { continue; } // we found the next entry endPosition = dataBIG.getFilePointer() - 1; break; } } // rewind back to the start position dataBIG.seek(startPosition); // now we start reading bytes during the mentioned interval while (dataBIG.getFilePointer() < endPosition) { // read a byte from our BIG archive int data = dataBIG.read(); byteOutput.write(data); } // flush data at this point byteOutput.flush(); // now convert the stream from input into an output (to feed the zip stream) ByteArrayInputStream byteInput = new ByteArrayInputStream(byteOutput.toByteArray()); // where we place the decompressed bytes ByteArrayOutputStream textOutput = new ByteArrayOutputStream(); // create the zip streamer final ArchiveInputStream archiveStream; archiveStream = new ArchiveStreamFactory().createArchiveInputStream("zip", byteInput); final ZipArchiveEntry entry = (ZipArchiveEntry) archiveStream.getNextEntry(); // copy all bytes from one location to the other (and decompress the data) IOUtils.copy(archiveStream, textOutput); // flush the results textOutput.flush(); // we've got the result right here! result = textOutput.toString(); // now close all the streams that we have open dataBIG.close(); byteOutput.close(); byteInput.close(); textOutput.close(); archiveStream.close(); } catch (FileNotFoundException ex) { Logger.getLogger(BigZip.class.getName()).log(Level.SEVERE, null, ex); return null; } catch (IOException ex) { Logger.getLogger(BigZip.class.getName()).log(Level.SEVERE, null, ex); return null; } catch (ArchiveException ex) { Logger.getLogger(BigZip.class.getName()).log(Level.SEVERE, null, ex); } return result; }
From source file:com.xperia64.rompatcher.MainActivity.java
public void patch(final boolean c, final boolean d, final boolean r, final String ed) { final ProgressDialog myPd_ring = ProgressDialog.show(MainActivity.this, getResources().getString(R.string.wait), getResources().getString(R.string.wait_desc), true); myPd_ring.setCancelable(false);/* ww w . ja v a2 s .c o m*/ new Thread(new Runnable() { public void run() { if (new File(Globals.patchToApply).exists() && new File(Globals.fileToPatch).exists() && !Globals.fileToPatch.toLowerCase(Locale.US).endsWith(".ecm")) { String msg = getResources().getString(R.string.success); if (!new File(Globals.fileToPatch).canWrite()) { Globals.msg = msg = "Can not write to output file. If you are on KitKat or Lollipop, move the file to your internal storage."; return; } if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".ups")) { int e = upsPatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new", r ? 1 : 0); if (e != 0) { msg = parseError(e, Globals.TYPE_UPS); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".xdelta") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".xdelta3") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".vcdiff")) { RandomAccessFile f = null; try { f = new RandomAccessFile(Globals.patchToApply, "r"); } catch (FileNotFoundException e1) { e1.printStackTrace(); Globals.msg = msg = getResources().getString(R.string.fnf); return; } StringBuilder s = new StringBuilder(); try { if (f.length() >= 9) { for (int i = 0; i < 8; i++) { s.append((char) f.readByte()); f.seek(i + 1); } } } catch (IOException e1) { e1.printStackTrace(); } try { f.close(); } catch (IOException e1) { e1.printStackTrace(); } // Header of xdelta patch determines version if (s.toString().equals("%XDELTA%") || s.toString().equals("%XDZ000%") || s.toString().equals("%XDZ001%") || s.toString().equals("%XDZ002%") || s.toString().equals("%XDZ003%") || s.toString().equals("%XDZ004%")) { int e = xdelta1PatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new"); if (e != 0) { msg = parseError(e, Globals.TYPE_XDELTA1); } } else { int e = xdelta3PatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new"); if (e != 0) { msg = parseError(e, Globals.TYPE_XDELTA3); } } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".bps")) { int e = bpsPatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new", r ? 1 : 0); if (e != 0) { msg = parseError(e, Globals.TYPE_BPS); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".dps")) { int e = dpsPatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new"); if (e != 0) { msg = parseError(e, Globals.TYPE_DPS); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".bsdiff")) { int e = bsdiffPatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new"); if (e != 0) { msg = parseError(e, Globals.TYPE_BSDIFF); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".aps")) { File f = new File(Globals.fileToPatch); File f2 = new File(Globals.fileToPatch + ".bak"); try { Files.copy(f, f2); } catch (IOException e) { e.printStackTrace(); } // Wow. byte[] gbaSig = { 0x41, 0x50, 0x53, 0x31, 0x00 }; byte[] n64Sig = { 0x41, 0x50, 0x53, 0x31, 0x30 }; byte[] realSig = new byte[5]; RandomAccessFile raf = null; System.out.println("APS Patch"); try { raf = new RandomAccessFile(Globals.patchToApply, "r"); } catch (FileNotFoundException e1) { // TODO Auto-generated catch block e1.printStackTrace(); Globals.msg = msg = getResources().getString(R.string.fnf); return; } try { raf.read(realSig); raf.close(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } if (Arrays.equals(realSig, gbaSig)) { System.out.println("GBA APS"); APSGBAPatcher aa = new APSGBAPatcher(); aa.crcTableInit(); int e = 0; try { e = aa.ApplyPatch(Globals.patchToApply, Globals.fileToPatch, r); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); e = -5; } System.out.println("e: " + e); if (e != 0) { msg = parseError(e, Globals.TYPE_APSGBA); } } else if (Arrays.equals(realSig, n64Sig)) { System.out.println("N64 APS"); int e = apsN64PatchRom(Globals.fileToPatch, Globals.patchToApply); if (e != 0) { msg = parseError(e, Globals.TYPE_APSN64); } } else { msg = parseError(-131, -10000); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".ppf")) { File f = new File(Globals.fileToPatch); File f2 = new File(Globals.fileToPatch + ".bak"); try { Files.copy(f, f2); } catch (IOException e) { e.printStackTrace(); } int e = ppfPatchRom(Globals.fileToPatch, Globals.patchToApply, r ? 1 : 0); if (e != 0) { msg = parseError(e, Globals.TYPE_PPF); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".patch")) { int e = xdelta1PatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new"); if (e != 0) { msg = parseError(e, Globals.TYPE_XDELTA1); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".asm")) { File f = new File(Globals.fileToPatch); File f2 = new File(Globals.fileToPatch + ".bak"); try { Files.copy(f, f2); } catch (IOException e) { e.printStackTrace(); } int e; if (Globals.asar) e = asarPatchRom(Globals.fileToPatch, Globals.patchToApply, r ? 1 : 0); else e = asmPatchRom(Globals.fileToPatch, Globals.patchToApply); if (e != 0) { msg = parseError(e, Globals.TYPE_ASM); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".dldi")) { File f = new File(Globals.fileToPatch); File f2 = new File(Globals.fileToPatch + ".bak"); try { Files.copy(f, f2); } catch (IOException e) { e.printStackTrace(); } int e = dldiPatchRom(Globals.fileToPatch, Globals.patchToApply); if (e != 0) { msg = parseError(e, Globals.TYPE_DLDI); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".xpc")) { int e = xpcPatchRom(Globals.fileToPatch, Globals.patchToApply, Globals.fileToPatch + ".new"); if (e != 0) { msg = parseError(e, Globals.TYPE_XPC); } } else if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".nmp")) { String drm = MainActivity.this.getPackageName(); System.out.println("Drm is: " + drm); if (drm.equals("com.xperia64.rompatcher.donation")) { if (c) { File f = new File(Globals.fileToPatch); File f2 = new File(Globals.fileToPatch + ".bak"); try { Files.copy(f, f2); } catch (IOException e) { e.printStackTrace(); } } NitroROMFilesystem fs; try { fs = new NitroROMFilesystem(Globals.fileToPatch); ROM.load(fs); RealPatch.patch(Globals.patchToApply, new Object()); ROM.close(); } catch (Exception e1) { // TODO Auto-generated catch block //e1.printStackTrace(); Globals.msg = msg = String.format(getResources().getString(R.string.nmpDefault), e1.getStackTrace()[0].getFileName(), e1.getStackTrace()[0].getLineNumber()); } if (c && d && !TextUtils.isEmpty(ed)) { File f = new File(Globals.fileToPatch); File f3 = new File(Globals.fileToPatch + ".bak"); File f2 = new File( Globals.fileToPatch.substring(0, Globals.fileToPatch.lastIndexOf('/') + 1) + ed); f.renameTo(f2); f3.renameTo(f); } } else { Globals.msg = msg = getResources().getString(R.string.drmwarning); MainActivity.this.runOnUiThread(new Runnable() { public void run() { AlertDialog.Builder b = new AlertDialog.Builder(MainActivity.this); b.setTitle(getResources().getString(R.string.drmwarning)); b.setIcon( ResourcesCompat.getDrawable(getResources(), R.drawable.icon_pro, null)); b.setMessage(getResources().getString(R.string.drmwarning_desc)); b.setCancelable(false); b.setNegativeButton(getResources().getString(android.R.string.cancel), new OnClickListener() { @Override public void onClick(DialogInterface arg0, int arg1) { } }); b.setPositiveButton(getResources().getString(R.string.nagInfo), new OnClickListener() { @Override public void onClick(DialogInterface arg0, int arg1) { try { startActivity(new Intent(Intent.ACTION_VIEW, Uri.parse( "market://details?id=com.xperia64.rompatcher.donation"))); } catch (android.content.ActivityNotFoundException anfe) { startActivity(new Intent(Intent.ACTION_VIEW, Uri.parse( "http://play.google.com/store/apps/details?id=com.xperia64.rompatcher.donation"))); } } }); b.create().show(); } }); } } else { RandomAccessFile f = null; try { f = new RandomAccessFile(Globals.patchToApply, "r"); } catch (FileNotFoundException e1) { e1.printStackTrace(); Globals.msg = msg = getResources().getString(R.string.fnf); return; } StringBuilder s = new StringBuilder(); try { if (f.length() >= 6) { for (int i = 0; i < 5; i++) { s.append((char) f.readByte()); f.seek(i + 1); } } } catch (IOException e1) { e1.printStackTrace(); } try { f.close(); } catch (IOException e1) { e1.printStackTrace(); } int e; // Two variants of IPS, the normal PATCH type, then this weird one called IPS32 with messily hacked in 32 bit offsets if (s.toString().equals("IPS32")) { e = ips32PatchRom(Globals.fileToPatch, Globals.patchToApply); if (e != 0) { msg = parseError(e, Globals.TYPE_IPS); } } else { e = ipsPatchRom(Globals.fileToPatch, Globals.patchToApply); if (e != 0) { msg = parseError(e, Globals.TYPE_IPS); } } } if (Globals.patchToApply.toLowerCase(Locale.US).endsWith(".ups") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".xdelta") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".xdelta3") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".vcdiff") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".patch") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".bps") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".bsdiff") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".dps") || Globals.patchToApply.toLowerCase(Locale.US).endsWith(".xpc")) { File oldrom = new File(Globals.fileToPatch); File bkrom = new File(Globals.fileToPatch + ".bak"); oldrom.renameTo(bkrom); File newrom = new File(Globals.fileToPatch + ".new"); newrom.renameTo(oldrom); } if (!c) { File f = new File(Globals.fileToPatch + ".bak"); if (f.exists()) { f.delete(); } } else { if (d) { File one = new File(Globals.fileToPatch + ".bak"); File two = new File(Globals.fileToPatch); File three = new File( Globals.fileToPatch.substring(0, Globals.fileToPatch.lastIndexOf('/') + 1) + ed); two.renameTo(three); File four = new File(Globals.fileToPatch); one.renameTo(four); } } Globals.msg = msg; } else if (Globals.fileToPatch.toLowerCase(Locale.US).endsWith(".ecm")) { int e = 0; String msg = getResources().getString(R.string.success); if (c) { if (d) { e = ecmPatchRom(Globals.fileToPatch, Globals.fileToPatch.substring(0, Globals.fileToPatch.lastIndexOf('/')) + ed, 1); } else { //new File(Globals.fileToPatch).renameTo(new File(Globals.fileToPatch+".bak")); e = ecmPatchRom(Globals.fileToPatch, Globals.fileToPatch.substring(0, Globals.fileToPatch.lastIndexOf('.')), 1); } } else { e = ecmPatchRom(Globals.fileToPatch, "", 0); } if (e != 0) { msg = parseError(e, Globals.TYPE_ECM); } Globals.msg = msg; } else { Globals.msg = getResources().getString(R.string.fnf); } } }).start(); new Thread(new Runnable() { @Override public void run() { try { while (Globals.msg.equals("")) { Thread.sleep(25); } ; myPd_ring.dismiss(); runOnUiThread(new Runnable() { public void run() { Toast t = Toast.makeText(staticThis, Globals.msg, Toast.LENGTH_SHORT); t.show(); Globals.msg = ""; } }); if (Globals.msg.equals(getResources().getString(R.string.success))) // Don't annoy people who did something wrong even further { final SharedPreferences prefs = PreferenceManager .getDefaultSharedPreferences(MainActivity.this); int x = prefs.getInt("purchaseNag", 5); if (x != -1) { if ((isPackageInstalled("com.xperia64.timidityae", MainActivity.this) || isPackageInstalled("com.xperia64.rompatcher.donation", MainActivity.this))) { prefs.edit().putInt("purchaseNag", -1); } else { if (x >= 5) { prefs.edit().putInt("purchaseNag", 0).commit(); /*runOnUiThread(new Runnable() { public void run() { AlertDialog.Builder b = new AlertDialog.Builder(MainActivity.this); b.setTitle("Like ROM Patcher?"); b.setIcon(getResources().getDrawable(R.drawable.icon_pro)); b.setMessage(getResources().getString(R.string.nagMsg)); b.setCancelable(false); b.setNegativeButton(getResources().getString(android.R.string.cancel), new OnClickListener(){@Override public void onClick(DialogInterface arg0, int arg1) {}}); b.setPositiveButton(getResources().getString(R.string.nagInfo), new OnClickListener(){ @Override public void onClick(DialogInterface arg0, int arg1) { try { startActivity(new Intent(Intent.ACTION_VIEW, Uri.parse("market://details?id=com.xperia64.rompatcher.donation"))); } catch (android.content.ActivityNotFoundException anfe) { startActivity(new Intent(Intent.ACTION_VIEW, Uri.parse("http://play.google.com/store/apps/details?id=com.xperia64.rompatcher.donation"))); } } }); b.create().show(); } }); // end of UIThread*/ } else { prefs.edit().putInt("purchaseNag", x + 1).commit(); } } } } } catch (Exception e) { } } }).start(); }
From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java
/** * seek out next instance of sync bytes in the file input stream * //from www. java 2 s . c om * @param file * @throws IOException */ private boolean seekToNextSyncBytesPos(RandomAccessFile file) throws IOException { // read in a sync.length buffer amount file.read(_syncByteBuffer); int syncLen = _header._sync.length; // start scan for next sync position ... for (int i = 0; file.getFilePointer() < _header._fileSize; i++) { int j = 0; for (; j < syncLen; j++) { if (_header._sync[j] != _syncByteBuffer[(i + j) % syncLen]) break; } if (j == syncLen) { // found matching sync bytes - reset file pos to before sync bytes file.seek(file.getFilePointer() - LocalLogFileHeader.SYNC_BYTES_SIZE); // position // before // sync return true; } _syncByteBuffer[i % syncLen] = file.readByte(); } return false; }
From source file:au.org.ala.layers.intersect.Grid.java
/** * @return calculated min and max values of a grid file as float [] where [0] is min and [1] is max. *//*from www. j a v a 2 s .com*/ public float[] calculatetMinMax() { float[] ret = new float[2]; ret[0] = Float.MAX_VALUE; ret[1] = Float.MAX_VALUE * -1; long i; int size; byte[] b; RandomAccessFile afile = null; try { //read of random access file can throw an exception File f2 = new File(filename + ".GRI"); if (!f2.exists()) { afile = new RandomAccessFile(filename + ".gri", "r"); } else { afile = new RandomAccessFile(filename + ".GRI", "r"); } long length = ((long) nrows) * ((long) ncols); float f; if (datatype.equalsIgnoreCase("BYTE")) { size = 1; b = new byte[size]; for (i = 0; i < length; i++) { f = afile.readByte(); if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else if (datatype.equalsIgnoreCase("UBYTE")) { size = 1; b = new byte[size]; for (i = 0; i < length; i++) { f = afile.readByte(); if (f < 0) { f += 256; } if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else if (datatype.equalsIgnoreCase("SHORT")) { size = 2; b = new byte[size]; for (i = 0; i < length; i++) { afile.read(b); if (byteorderLSB) { f = (short) (((0xFF & b[1]) << 8) | (b[0] & 0xFF)); } else { f = (short) (((0xFF & b[0]) << 8) | (b[1] & 0xFF)); } if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else if (datatype.equalsIgnoreCase("INT")) { size = 4; b = new byte[size]; for (i = 0; i < length; i++) { afile.read(b); if (byteorderLSB) { f = ((0xFF & b[3]) << 24) | ((0xFF & b[2]) << 16) + ((0xFF & b[1]) << 8) + (b[0] & 0xFF); } else { f = ((0xFF & b[0]) << 24) | ((0xFF & b[1]) << 16) + ((0xFF & b[2]) << 8) + ((0xFF & b[3]) & 0xFF); } if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else if (datatype.equalsIgnoreCase("LONG")) { size = 8; b = new byte[size]; for (i = 0; i < length; i++) { afile.read(b); if (byteorderLSB) { f = ((long) (0xFF & b[7]) << 56) + ((long) (0xFF & b[6]) << 48) + ((long) (0xFF & b[5]) << 40) + ((long) (0xFF & b[4]) << 32) + ((long) (0xFF & b[3]) << 24) + ((long) (0xFF & b[2]) << 16) + ((long) (0xFF & b[1]) << 8) + (0xFF & b[0]); } else { f = ((long) (0xFF & b[0]) << 56) + ((long) (0xFF & b[1]) << 48) + ((long) (0xFF & b[2]) << 40) + ((long) (0xFF & b[3]) << 32) + ((long) (0xFF & b[4]) << 24) + ((long) (0xFF & b[5]) << 16) + ((long) (0xFF & b[6]) << 8) + (0xFF & b[7]); } if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else if (datatype.equalsIgnoreCase("FLOAT")) { size = 4; b = new byte[size]; for (i = 0; i < length; i++) { afile.read(b); ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } f = bb.getFloat(); if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else if (datatype.equalsIgnoreCase("DOUBLE")) { size = 8; b = new byte[8]; for (i = 0; i < length; i++) { afile.read(b); ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } f = (float) bb.getDouble(); if (f != (float) nodatavalue) { ret[0] = Math.min(f * rescale, ret[0]); ret[1] = Math.max(f * rescale, ret[1]); } } } else { logger.error("datatype not supported in Grid.getValues: " + datatype); } } catch (Exception e) { logger.error("error calculating min/max of a grid file", e); } finally { if (afile != null) { try { afile.close(); } catch (Exception e) { logger.error(e.getMessage(), e); } } } return ret; }
From source file:au.org.ala.layers.intersect.Grid.java
/** * @param points input array for longitude and latitude * double[number_of_points][2] * @return array of .gri file values corresponding to the * points provided// ww w . j a v a 2s . c o m */ public float[] getValues(double[][] points) { //confirm inputs since they come from somewhere else if (points == null || points.length == 0) { return null; } //use preloaded grid data if available Grid g = Grid.getLoadedGrid(filename); if (g != null) { return g.getValues2(points); } if (subgrids != null) { return getValues3(points, Math.min(1024 * 1024, 64 * points.length)); } float[] ret = new float[points.length]; int length = points.length; long size; int i, pos; byte[] b; RandomAccessFile afile = null; File f2 = new File(filename + ".GRI"); try { //read of random access file can throw an exception if (!f2.exists()) { afile = new RandomAccessFile(filename + ".gri", "r"); } else { afile = new RandomAccessFile(filename + ".GRI", "r"); } if (datatype.equalsIgnoreCase("BYTE")) { size = 1; b = new byte[(int) size]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); ret[i] = afile.readByte(); } else { ret[i] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("UBYTE")) { size = 1; b = new byte[(int) size]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); ret[i] = afile.readByte(); if (ret[i] < 0) { ret[i] += 256; } } else { ret[i] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("SHORT")) { size = 2; b = new byte[(int) size]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); afile.read(b); if (byteorderLSB) { ret[i] = (short) (((0xFF & b[1]) << 8) | (b[0] & 0xFF)); } else { ret[i] = (short) (((0xFF & b[0]) << 8) | (b[1] & 0xFF)); } //ret[i] = afile.readShort(); } else { ret[i] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("INT")) { size = 4; b = new byte[(int) size]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); afile.read(b); if (byteorderLSB) { ret[i] = ((0xFF & b[3]) << 24) | ((0xFF & b[2]) << 16) + ((0xFF & b[1]) << 8) + (b[0] & 0xFF); } else { ret[i] = ((0xFF & b[0]) << 24) | ((0xFF & b[1]) << 16) + ((0xFF & b[2]) << 8) + ((0xFF & b[3]) & 0xFF); } //ret[i] = afile.readInt(); } else { ret[i] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("LONG")) { size = 8; b = new byte[(int) size]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); afile.read(b); if (byteorderLSB) { ret[i] = ((long) (0xFF & b[7]) << 56) + ((long) (0xFF & b[6]) << 48) + ((long) (0xFF & b[5]) << 40) + ((long) (0xFF & b[4]) << 32) + ((long) (0xFF & b[3]) << 24) + ((long) (0xFF & b[2]) << 16) + ((long) (0xFF & b[1]) << 8) + (0xFF & b[0]); } else { ret[i] = ((long) (0xFF & b[0]) << 56) + ((long) (0xFF & b[1]) << 48) + ((long) (0xFF & b[2]) << 40) + ((long) (0xFF & b[3]) << 32) + ((long) (0xFF & b[4]) << 24) + ((long) (0xFF & b[5]) << 16) + ((long) (0xFF & b[6]) << 8) + (0xFF & b[7]); } //ret[i] = afile.readLong(); } else { ret[i] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("FLOAT")) { size = 4; b = new byte[(int) size]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); afile.read(b); ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } ret[i] = bb.getFloat(); } else { ret[i] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("DOUBLE")) { size = 8; b = new byte[8]; for (i = 0; i < length; i++) { pos = (int) getcellnumber(points[i][0], points[i][1]); if (pos >= 0) { afile.seek(pos * size); afile.read(b); ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } ret[i] = (float) bb.getDouble(); //ret[i] = afile.readFloat(); } else { ret[i] = Float.NaN; } } } else { logger.error("datatype not supported in Grid.getValues: " + datatype); // / should not happen; catch anyway... for (i = 0; i < length; i++) { ret[i] = Float.NaN; } } //replace not a number for (i = 0; i < length; i++) { if ((float) ret[i] == (float) nodatavalue) { ret[i] = Float.NaN; } else { ret[i] *= rescale; } } } catch (Exception e) { logger.error("error getting grid file values", e); } finally { if (afile != null) { try { afile.close(); } catch (Exception e) { logger.error(e.getMessage(), e); } } } return ret; }
From source file:org.commoncrawl.service.listcrawler.HDFSFlusherThread.java
private long generateSequenceFileAndIndex(int itemFlushLimit, RandomAccessFile sourceLogFile, long startPos, long endPos, byte[] syncBytes, SequenceFile.Writer writer, DataOutput indexStreamOut, ArrayList<FingerprintAndOffsetTuple> tupleListOut) throws IOException { byte[] syncCheck = new byte[syncBytes.length]; // and create a list to hold fingerprint / offset information Vector<FingerprintAndOffsetTuple> fpOffsetList = new Vector<FingerprintAndOffsetTuple>(); long currentPos = startPos; LOG.info("Flushing Entries Starting up to offset:" + endPos); CacheItemHeader itemHeader = new CacheItemHeader(); int itemsProcessed = 0; boolean ignoreFlushLimit = false; // start read while (currentPos < endPos) { if ((endPos - currentPos) < LocalLogFileHeader.SYNC_BYTES_SIZE) break; // seek to current position ... sourceLogFile.seek(currentPos);/* w ww . ja va 2 s .com*/ boolean headerLoadFailed = false; try { // read the item header ... assuming things are good so far ... itemHeader.readHeader(sourceLogFile); } catch (IOException e) { CacheManager.LOG.error("### Item Header Load At Position:" + currentPos + " Failed With Exception:" + CCStringUtils.stringifyException(e)); headerLoadFailed = true; } if (headerLoadFailed) { CacheManager.LOG .error("### Item File Corrupt at position:" + currentPos + " Seeking Next Sync Point"); currentPos += LocalLogFileHeader.SYNC_BYTES_SIZE; } // if header sync bytes don't match .. then seek to next sync position ... if (headerLoadFailed || !Arrays.equals(itemHeader._sync, syncBytes)) { CacheManager.LOG .error("### Item File Corrupt at position:" + currentPos + " Seeking Next Sync Point"); // reseek to current pos sourceLogFile.seek(currentPos); // read in a sync.length buffer amount sourceLogFile.readFully(syncCheck); int syncLen = syncBytes.length; // start scan for next sync position ... for (int i = 0; sourceLogFile.getFilePointer() < endPos; i++) { int j = 0; for (; j < syncLen; j++) { if (syncBytes[j] != syncCheck[(i + j) % syncLen]) break; } if (j == syncLen) { sourceLogFile.seek(sourceLogFile.getFilePointer() - LocalLogFileHeader.SYNC_BYTES_SIZE); // position before sync break; } syncCheck[i % syncLen] = sourceLogFile.readByte(); } // whatever, happened file pointer is at current pos currentPos = sourceLogFile.getFilePointer(); if (currentPos < endPos) { CacheManager.LOG.info("### Item Loader Found another sync point at:" + currentPos); } else { CacheManager.LOG.error("### No more sync points found!"); } } else { CacheManager.LOG .info("WritingItem with FP:" + itemHeader._fingerprint + " Pos Is:" + writer.getLength()); // track offset information for index building purposes fpOffsetList.add(new FingerprintAndOffsetTuple(itemHeader._fingerprint, writer.getLength())); // read item data ... CacheItem cacheItem = new CacheItem(); cacheItem.readFields(sourceLogFile); // now read content length int contentLength = sourceLogFile.readInt(); // and if content present... allocate buffer if (contentLength != 0) { // allocate content buffer byte[] contentBuffer = new byte[contentLength]; // read it from disk sourceLogFile.readFully(contentBuffer); // and set content into cache item cacheItem.setContent(new Buffer(contentBuffer)); } CacheManager.LOG.info("Adding to Sequence File Item with URL:" + cacheItem.getUrl()); // write to sequence file ... writer.append(new Text(cacheItem.getUrl()), cacheItem); // now seek past data currentPos += CacheItemHeader.SIZE + itemHeader._dataLength + CacheManager.ITEM_RECORD_TRAILING_BYTES; // increment item count itemsProcessed++; } if (!ignoreFlushLimit && itemsProcessed >= itemFlushLimit) { // ok this gets tricky now ... // figure out how many bytes of data were required to get to flush limit long approxCheckpointSize = currentPos - startPos; // compute a threshold number long bytesThreshold = (long) (approxCheckpointSize * .70); // compute bytes remaining in checkpoint file ... long bytesRemaining = endPos - currentPos; // ok if bytes remaining are less than threshold number then go ahead and gobble // everything up in a single pass (to prevent smaller subsequent index if (bytesRemaining <= bytesThreshold) { // ignore the flush limit and keep on rolling to the end ... ignoreFlushLimit = true; LOG.warn("*****Bytes Remaining:" + bytesRemaining + " less than % of last whole chkpt size:" + approxCheckpointSize + ". Bypassing Flush Limit"); } else { LOG.info("Reached Flush Item Limit:" + itemsProcessed + " Breaking Out"); break; } } } LOG.info("Writing Index"); // ok now build the index file ... HDFSFileIndex.writeIndex(fpOffsetList, indexStreamOut); LOG.info("Done Writing Index. Total Items Written:" + fpOffsetList.size()); // copy offset list into tuple list tupleListOut.addAll(fpOffsetList); return currentPos; }
From source file:org.commoncrawl.service.listcrawler.CacheManager.java
/** * loadCache - load local cache from disk * @param activeLogPath/*from w w w .ja va 2s .co m*/ * @param logFileHeader * @throws IOException */ private synchronized void loadCache(File activeLogPath, LocalLogFileHeader logFileHeader) throws IOException { RandomAccessFile file = new RandomAccessFile(getActiveLogFilePath(), "rw"); byte[] syncCheck = new byte[_header._sync.length]; try { long lastValidPos = LocalLogFileHeader.SIZE; long currentPos = lastValidPos; long endPos = file.length(); CacheItemHeader itemHeader = new CacheItemHeader(); // start read while (currentPos < endPos) { if ((endPos - currentPos) < LocalLogFileHeader.SYNC_BYTES_SIZE) break; // seek to current position ... file.seek(currentPos); boolean headerLoadFailed = false; try { // read the item header ... assuming things are good so far ... itemHeader.readHeader(file); } catch (IOException e) { LOG.error("### Item Header Load Failed With Exception:" + CCStringUtils.stringifyException(e)); headerLoadFailed = true; } if (headerLoadFailed) { LOG.error("### Item File Corrupt at position:" + currentPos + " Seeking Next Sync Point"); currentPos += LocalLogFileHeader.SYNC_BYTES_SIZE; } // if header sync bytes don't match .. then seek to next sync position ... if (headerLoadFailed || !Arrays.equals(itemHeader._sync, _header._sync)) { LOG.error("### Item File Corrupt at position:" + currentPos + " Seeking Next Sync Point"); // reseek to current pos file.seek(currentPos); // read in a sync.length buffer amount file.readFully(syncCheck); int syncLen = _header._sync.length; // start scan for next sync position ... for (int i = 0; file.getFilePointer() < endPos; i++) { int j = 0; for (; j < syncLen; j++) { if (_header._sync[j] != syncCheck[(i + j) % syncLen]) break; } if (j == syncLen) { file.seek(file.getFilePointer() - LocalLogFileHeader.SYNC_BYTES_SIZE); // position before sync break; } syncCheck[i % syncLen] = file.readByte(); } // whatever, happened file pointer is at current pos currentPos = file.getFilePointer(); if (currentPos < endPos) { LOG.info("### Item Loader Found another sync point at:" + currentPos); } else { LOG.error("### No more sync points found!"); } } else { // ok figure out next steps based on header ... // for now, just add item to our list ... _fingerprintToLocalLogPos.put(itemHeader._fingerprint, _localLogStartOffset + currentPos); // now seek past data currentPos += CacheItemHeader.SIZE + itemHeader._dataLength + ITEM_RECORD_TRAILING_BYTES; } } } finally { if (file != null) { file.close(); } } }