List of usage examples for java.io RandomAccessFile seek
public void seek(long pos) throws IOException
From source file:org.apache.hadoop.dfs.TestFsck.java
public void testCorruptBlock() throws Exception { Configuration conf = new Configuration(); conf.setLong("dfs.blockreport.intervalMsec", 1000); FileSystem fs = null;/*from w w w . j a v a 2 s. com*/ DFSClient dfsClient = null; LocatedBlocks blocks = null; int replicaCount = 0; Random random = new Random(); String outStr = null; MiniDFSCluster cluster = new MiniDFSCluster(conf, 3, true, null); cluster.waitActive(); fs = cluster.getFileSystem(); Path file1 = new Path("/testCorruptBlock"); DFSTestUtil.createFile(fs, file1, 1024, (short) 3, 0); // Wait until file replication has completed DFSTestUtil.waitReplication(fs, file1, (short) 3); String block = DFSTestUtil.getFirstBlock(fs, file1).getBlockName(); // Make sure filesystem is in healthy state outStr = runFsck(conf, 0, true, "/"); System.out.println(outStr); assertTrue(outStr.contains("HEALTHY")); // corrupt replicas File baseDir = new File(System.getProperty("test.build.data"), "dfs/data"); for (int i = 0; i < 6; i++) { File blockFile = new File(baseDir, "data" + (i + 1) + "/current/" + block); if (blockFile.exists()) { RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw"); FileChannel channel = raFile.getChannel(); String badString = "BADBAD"; int rand = random.nextInt((int) channel.size() / 2); raFile.seek(rand); raFile.write(badString.getBytes()); raFile.close(); } } // Read the file to trigger reportBadBlocks try { IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), conf, true); } catch (IOException ie) { // Ignore exception } dfsClient = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); blocks = dfsClient.namenode.getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); replicaCount = blocks.get(0).getLocations().length; while (replicaCount != 3) { try { Thread.sleep(100); } catch (InterruptedException ignore) { } blocks = dfsClient.namenode.getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); replicaCount = blocks.get(0).getLocations().length; } assertTrue(blocks.get(0).isCorrupt()); // Check if fsck reports the same outStr = runFsck(conf, 1, true, "/"); System.out.println(outStr); assertTrue(outStr.contains("CORRUPT")); assertTrue(outStr.contains("testCorruptBlock")); cluster.shutdown(); }
From source file:edu.msu.cme.rdp.readseq.readers.core.SFFCore.java
private LinkedHashMap<String, Long> readIndex() throws IOException { if (commonHeader.indexOffset <= commonHeader.headerLength) { throw new IOException("Index offset is not set correctly"); }// www .j av a 2s . co m RandomAccessFile seqFile = super.getRawFile(); long seekBackTo = seqFile.getFilePointer(); seqFile.seek(commonHeader.indexOffset); long dataEnd = seqFile.getFilePointer(); LinkedHashMap<String, Long> seqIndex = new LinkedHashMap(commonHeader.numReads); int magicNumber = seqFile.readInt(); if (magicNumber == mftMagicNumber) { int version = seqFile.readInt(); if (version != v1MagicNumber) { throw new IOException("Can only parse .mft v1.0 indices"); } int xmlSize = seqFile.readInt(); int dataSize = seqFile.readInt(); dataEnd += dataSize; byte[] xml = new byte[xmlSize]; seqFile.read(xml); manifest = new String(xml); } else if (magicNumber == srtMagicNumber) { int version = seqFile.readInt(); if (version != v1MagicNumber) { throw new IOException("Can only parse .srt v1.0 indices"); } if (seqFile.read() != 0) { throw new IOException("Failed to find expected null byte in .srt header"); } dataEnd += commonHeader.indexLength; } else { throw new IOException("No supported index found"); } List<Integer> currIndex = new ArrayList(); while (seqFile.getFilePointer() < dataEnd) { int b = seqFile.readUnsignedByte(); if (b == 0xff) { byte[] nameArray = new byte[currIndex.size() - 5]; long indexLoc = 0; int[] multipliers = new int[] { 0, 16581375, 65025, 255, 1 }; for (int i = 0; i < currIndex.size(); i++) { if (i < nameArray.length) { nameArray[i] = (byte) (currIndex.get(i) & 0xff); } else { int index = i - nameArray.length; indexLoc += currIndex.get(i) * multipliers[index]; } } String name = new String(nameArray); seqIndex.put(name, indexLoc); currIndex.clear(); } else { currIndex.add(b); } } seqFile.seek(seekBackTo); return seqIndex; }
From source file:com.alibaba.otter.node.etl.common.pipe.impl.http.AbstractHttpPipe.java
protected void decodeFile(File file, String key, String crc) { // ??// w w w. jav a 2s. co m RandomAccessFile raf = null; try { raf = new RandomAccessFile(file, "rw"); long totallength = file.length(); int keyLength = ByteUtils.stringToBytes(key).length; int crcLength = ByteUtils.stringToBytes(crc).length; // ? long pos = totallength - keyLength - crcLength; // raf.seek(pos); // ?key byte[] keyBytes = new byte[keyLength]; raf.read(keyBytes, 0, keyLength); String keystr = ByteUtils.bytesToString(keyBytes); if (!key.equals(keystr)) { throw new ChecksumException("unmatch garble key with[" + key + "],[" + keystr + "]"); } // ?? raf.seek(pos + keyLength); byte[] crcBytes = new byte[crcLength]; raf.read(crcBytes, 0, crcLength); String crcStr = ByteUtils.bytesToString(crcBytes); if (!crc.equals(crcStr)) { throw new ChecksumException("unmatch crc with[" + crc + "],[" + crcStr + "]"); } // raf.setLength(pos); } catch (Exception e) { throw new PipeException("read_encrypted_error", e); } finally { IOUtils.closeQuietly(raf); } }
From source file:org.nuxeo.connect.update.task.standalone.commands.Copy.java
/** * @param doOverwrite/* www. j av a 2 s.c o m*/ * @since 5.5 */ protected Command doCopy(Task task, Map<String, String> prefs, File fileToCopy, File dst, boolean doOverwrite) throws PackageException { String dstmd5; File bak = null; CompositeCommand rollbackCommand = new CompositeCommand(); if (fileToCopy.isDirectory()) { if (fileToCopy != file) { dst = new File(dst, fileToCopy.getName()); } dst.mkdirs(); for (File childFile : fileToCopy.listFiles()) { rollbackCommand.addCommand(doCopy(task, prefs, childFile, dst, doOverwrite)); } return rollbackCommand; } if (dst.isDirectory()) { dst = new File(dst, fileToCopy.getName()); } try { FileMatcher filenameMatcher = FileMatcher.getMatcher("{n:.*-}[0-9]+.*\\.jar"); boolean isVersionnedJarFile = filenameMatcher.match(fileToCopy.getName()); if (isVersionnedJarFile) { log.warn(String.format( "Use of the <copy /> command on JAR files is not recommended, prefer using <update /> command to ensure a safe rollback. (%s)", fileToCopy.getName())); } if (isVersionnedJarFile && (overwriteIfNewerVersion || upgradeOnly)) { // Compare source and destination versions set in filename FileVersion fileToCopyVersion, dstVersion = null; String filenameWithoutVersion = filenameMatcher.getValue(); FileMatcher versionMatcher = FileMatcher.getMatcher(filenameWithoutVersion + "{v:[0-9]+.*}\\.jar"); // Get new file version if (versionMatcher.match(fileToCopy.getName())) { fileToCopyVersion = new FileVersion(versionMatcher.getValue()); // Get original file name and version File dir = dst.getParentFile(); File[] list = dir.listFiles(); if (list != null) { for (File f : list) { if (versionMatcher.match(f.getName())) { dst = f; dstVersion = new FileVersion(versionMatcher.getValue()); break; } } } if (dstVersion == null) { if (upgradeOnly) { return null; } } else if (fileToCopyVersion.greaterThan(dstVersion)) { // backup dst and generate rollback command File oldDst = dst; dst = new File(dst.getParentFile(), fileToCopy.getName()); File backup = IOUtils.backup(task.getPackage(), oldDst); rollbackCommand.addCommand(new Copy(backup, oldDst, null, false)); // Delete old dst as its name differs from new version oldDst.delete(); } else if (fileToCopyVersion.isSnapshot() && fileToCopyVersion.equals(dstVersion)) { doOverwrite = true; } else if (!doOverwrite) { log.info("Ignore " + fileToCopy + " because not newer than " + dstVersion + " and 'overwrite' is set to false."); return null; } } } if (dst.exists()) { // backup the destination file if exist. if (!doOverwrite && !append) { // force a rollback throw new PackageException( "Copy command has overwrite flag on false but destination file exists: " + dst); } if (task instanceof UninstallTask) { // no backup for uninstall task } else if (append) { bak = IOUtils.backup(task.getPackage(), fileToCopy); } else { bak = IOUtils.backup(task.getPackage(), dst); } } else { // target file doesn't exists - it will be created dst.getParentFile().mkdirs(); } // copy the file - use getContentToCopy to allow parameterization // for subclasses String content = getContentToCopy(fileToCopy, prefs); if (content != null) { if (append && dst.exists()) { RandomAccessFile rfile = new RandomAccessFile(dst, "r"); try { rfile.seek(dst.length()); if (!"".equals(rfile.readLine())) { content = System.getProperty("line.separator") + content; } } catch (IOException e) { log.error(e); } finally { rfile.close(); } } FileUtils.writeFile(dst, content, append); } else { File tmp = new File(dst.getPath() + ".tmp"); FileUtils.copy(fileToCopy, tmp); if (!tmp.renameTo(dst)) { tmp.delete(); FileUtils.copy(fileToCopy, dst); } } // check whether the copied or restored file was the launcher if (dst.getName().equals(LAUNCHER_JAR) || fileToCopy.getName().equals(LAUNCHER_JAR)) { Environment env = Environment.getDefault(); env.setProperty(LAUNCHER_CHANGED_PROPERTY, "true"); } // get the md5 of the copied file. dstmd5 = IOUtils.createMd5(dst); } catch (IOException e) { throw new PackageException("Failed to copy " + fileToCopy, e); } if (bak == null) { // no file was replaced rollbackCommand.addCommand(new Delete(dst, dstmd5, removeOnExit)); } else if (append) { rollbackCommand.addCommand(new UnAppend(bak, dst)); } else { rollbackCommand.addCommand(new Copy(bak, dst, dstmd5, true)); } return rollbackCommand; }
From source file:com.qingstor.sdk.request.QSOkHttpRequestClient.java
/** * @param method// w ww . j a v a 2 s . c o m * @param bodyContent * @param headParams * @param singedUrl * @throws QSException */ public Request buildStorMultiUpload(final String method, final Map bodyContent, final String singedUrl, final Map headParams, final Map queryParams) throws QSException { Request.Builder builder = new Request.Builder(); String[] sortedHeadersKeys = (String[]) headParams.keySet().toArray(new String[] {}); for (String key : sortedHeadersKeys) { builder.addHeader(key, headParams.get(key) + ""); } if (!headParams.containsKey(QSConstant.PARAM_KEY_USER_AGENT)) { builder.addHeader(QSConstant.PARAM_KEY_USER_AGENT, QSStringUtil.getUserAgent()); } if (bodyContent != null && bodyContent.size() > 0) { String contentType = headParams.get(QSConstant.HEADER_PARAM_KEY_CONTENTTYPE) + ""; MediaType mediaType = MediaType.parse(contentType); RequestBody requestBody = null; Iterator iterator = bodyContent.entrySet().iterator(); int contentLength = Integer.parseInt(headParams.get(QSConstant.PARAM_KEY_CONTENT_LENGTH) + ""); int partNumber = Integer.parseInt(queryParams.get(QSConstant.PARAM_KEY_PART_NUMBER) + ""); while (iterator.hasNext()) { Map.Entry entry = (Map.Entry) iterator.next(); String key = (String) entry.getKey(); Object bodyObj = bodyContent.get(key); if (bodyObj instanceof String) { requestBody = RequestBody.create(mediaType, bodyObj.toString()); } else if (bodyObj instanceof File) { RandomAccessFile rFile = null; try { rFile = new RandomAccessFile((File) bodyObj, "r"); rFile.seek(contentLength * partNumber); requestBody = new MulitFileuploadBody(contentType, rFile, contentLength); } catch (IOException e) { e.printStackTrace(); throw new QSException(e.getMessage()); } } else if (bodyObj instanceof InputStream) { requestBody = new InputStreamUploadBody(contentType, (InputStream) bodyObj, contentLength); } else { String jsonStr = QSStringUtil.objectToJson(key, bodyObj); requestBody = RequestBody.create(mediaType, jsonStr); } } //connection.getOutputStream().write(bodyContent.getBytes()); if (requestBody != null) { builder.method(method, requestBody); } } Request request = builder.url(singedUrl).build(); return request; }
From source file:com.alibaba.otter.node.etl.common.pipe.impl.http.AbstractHttpPipe.java
protected EncryptedData encryptFile(File file) { // ?file path?? EncryptedData encryptedData = null;// www. j av a 2 s . c o m try { encryptedData = EncryptUtils.encrypt(file.getPath().getBytes("UTF-8")); } catch (UnsupportedEncodingException e) { // ignore } // ? RandomAccessFile raf = null; try { raf = new RandomAccessFile(file, "rw"); long origLength = file.length(); int keyLength = ByteUtils.stringToBytes(encryptedData.getKey()).length; int crcLength = ByteUtils.stringToBytes(encryptedData.getCrc()).length; long totalLength = origLength + crcLength + keyLength; raf.setLength(totalLength); raf.seek(origLength); raf.write(ByteUtils.stringToBytes(encryptedData.getKey()), 0, keyLength); raf.seek(origLength + keyLength); raf.write(ByteUtils.stringToBytes(encryptedData.getCrc()), 0, crcLength); } catch (Exception e) { throw new PipeException("write_encrypted_error", e); } finally { IOUtils.closeQuietly(raf); } return encryptedData; }
From source file:edu.tsinghua.lumaqq.qq.Util.java
/** * MD5???10002432//from ww w .j a v a2s . co m * @param file RandomAccessFile * @return MD5 */ public static byte[] getFileMD5(RandomAccessFile file) { try { file.seek(0); byte[] buf = (file.length() > QQ.QQ_MAX_FILE_MD5_LENGTH) ? new byte[QQ.QQ_MAX_FILE_MD5_LENGTH] : new byte[(int) file.length()]; file.readFully(buf); return DigestUtils.md5(buf); } catch (IOException e) { return null; } }
From source file:org.commoncrawl.service.listcrawler.CacheWriterThread.java
@Override public void run() { boolean shutdown = false; while (!shutdown) { try {/*from w ww . j a v a 2s. c o m*/ final CacheWriteRequest request = _writeRequestQueue.take(); switch (request._requestType) { case ExitThreadRequest: { // shutdown condition ... CacheManager.LOG.info("Disk Writer Thread Received Shutdown. Exiting!"); shutdown = true; } break; case WriteRequest: { long timeStart = System.currentTimeMillis(); try { // reset crc calculator (single thread so no worries on synchronization) _crc32Out.reset(); // figure out if we need to compress the item ... if ((request._item.getFlags() & CacheItem.Flags.Flag_IsCompressed) == 0 && request._item.getContent().getCount() != 0) { LOG.info("Incoming Cache Request Content for:" + request._item.getUrl() + " is not compressed. Compressing..."); ByteStream compressedBytesOut = new ByteStream(request._item.getContent().getCount()); ThriftyGZIPOutputStream gzipOutputStream = new ThriftyGZIPOutputStream( compressedBytesOut); gzipOutputStream.write(request._item.getContent().getReadOnlyBytes(), 0, request._item.getContent().getCount()); gzipOutputStream.finish(); LOG.info("Finished Compressing Incoming Content for:" + request._item.getUrl() + " BytesIn:" + request._item.getContent().getCount() + " BytesOut:" + compressedBytesOut.size()); // replace buffer request._item.setContent( new FlexBuffer(compressedBytesOut.getBuffer(), 0, compressedBytesOut.size())); request._item.setFlags((request._item.getFlags() | CacheItem.Flags.Flag_IsCompressed)); } // create streams ... ByteStream bufferOutputStream = new ByteStream(8192); CheckedOutputStream checkedStream = new CheckedOutputStream(bufferOutputStream, _crc32Out); DataOutputStream dataOutputStream = new DataOutputStream(checkedStream); // remember if this item has content ... boolean hasContent = request._item.isFieldDirty(CacheItem.Field_CONTENT); // now mark the content field as clean, so that it will not be serialized in our current serialization attempt ... request._item.setFieldClean(CacheItem.Field_CONTENT); // and go ahead and write out the data to the intermediate buffer while also computing partial checksum request._item.write(dataOutputStream); request._item.setFieldDirty(CacheItem.Field_CONTENT); // ok, now ... write out file header ... CacheItemHeader itemHeader = new CacheItemHeader(_manager.getLocalLogSyncBytes()); itemHeader._status = CacheItemHeader.STATUS_ALIVE; itemHeader._lastAccessTime = System.currentTimeMillis(); itemHeader._fingerprint = request._itemFingerprint; // compute total length ... // first the header bytes in the cacheItem itemHeader._dataLength = bufferOutputStream.size(); // next the content length (encoded - as in size + bytes) ... itemHeader._dataLength += 4 + request._item.getContent().getCount(); // lastly the crc value iteself ... itemHeader._dataLength += 8; // open the log file ... DataOutputBuffer logStream = new DataOutputBuffer(); // ok, go ahead and write the header itemHeader.writeHeader(logStream); // ok now write out the item data minus content... logStream.write(bufferOutputStream.getBuffer(), 0, bufferOutputStream.size()); // now create a checked stream for the content ... CheckedOutputStream checkedStream2 = new CheckedOutputStream(logStream, checkedStream.getChecksum()); dataOutputStream = new DataOutputStream(checkedStream2); // content size dataOutputStream.writeInt(request._item.getContent().getCount()); // now write out the content (via checked stream so that we can calc checksum on content) dataOutputStream.write(request._item.getContent().getReadOnlyBytes(), 0, request._item.getContent().getCount()); // ok ... lastly write out the checksum bytes ... dataOutputStream.writeLong(checkedStream2.getChecksum().getValue()); // and FINALLY, write out the total item bytes (so that we can seek in reverse to read last request log logStream.writeInt(CacheItemHeader.SIZE + itemHeader._dataLength); // ok flush everyting to the memory stream dataOutputStream.flush(); //ok - time to acquire the log semaphore //LOG.info("Acquiring Local Log Semaphore"); _manager.getLocalLogAccessSemaphore().acquireUninterruptibly(); try { // now time to acquire the write semaphore ... _manager.getLocalLogWriteAccessSemaphore().acquireUninterruptibly(); // get the current file position long recordOffset = _manager.getLocalLogFilePos(); try { long ioTimeStart = System.currentTimeMillis(); RandomAccessFile logFile = new RandomAccessFile(_manager.getActiveLogFilePath(), "rw"); try { // seek to our known record offset logFile.seek(recordOffset); // write out the data logFile.write(logStream.getData(), 0, logStream.getLength()); } finally { logFile.close(); } // now we need to update the file header _manager.updateLogFileHeader(_manager.getActiveLogFilePath(), 1, CacheItemHeader.SIZE + itemHeader._dataLength + 4 /*trailing bytes*/); CacheManager.LOG .info("#### Wrote Cache Item in:" + (System.currentTimeMillis() - timeStart) + " iotime:" + (System.currentTimeMillis() - ioTimeStart) + " QueueSize:" + _writeRequestQueue.size()); } finally { // release write semaphore quickly _manager.getLocalLogWriteAccessSemaphore().release(); } // now inform the manager of the completed request ... _manager.writeRequestComplete(request, recordOffset); } finally { //LOG.info("Releasing Local Log Semaphore"); _manager.getLocalLogAccessSemaphore().release(); } } catch (IOException e) { CacheManager.LOG.error("### FUC# BATMAN! - GONNA LOSE THIS REQUEST!!!!:" + CCStringUtils.stringifyException(e)); _manager.writeRequestFailed(request, e); } } break; } } catch (InterruptedException e) { } } }
From source file:com.haulmont.cuba.core.sys.LogControlImpl.java
protected void skipFirstLine(RandomAccessFile logFile) throws IOException { boolean eol = false; while (!eol) { switch (logFile.read()) { case -1:// ww w . j a va 2 s . com case '\n': eol = true; break; case '\r': eol = true; long cur = logFile.getFilePointer(); if ((logFile.read()) != '\n') { logFile.seek(cur); } break; default: break; } } }
From source file:jp.andeb.obbutil.ObbUtilMain.java
private static boolean doAdd(String[] args) { final CommandLine commandLine; try {//from w w w.j a v a2s .co m final CommandLineParser parser = new GnuParser(); commandLine = parser.parse(OPTIONS_FOR_ADD, args); } catch (MissingArgumentException e) { System.err.println("??????: " + e.getOption().getOpt()); printUsage(PROGNAME); return false; } catch (MissingOptionException e) { System.err.println("??????: " + e.getMissingOptions()); printUsage(PROGNAME); return false; } catch (UnrecognizedOptionException e) { System.err.println("????: " + e.getOption()); printUsage(PROGNAME); return false; } catch (ParseException e) { System.err.println(e.getMessage()); printUsage(PROGNAME); return false; } final String pkgName = commandLine.getOptionValue(PACKAGE_NAME.getOpt()); final String versionStr = commandLine.getOptionValue(OBB_VERSION.getOpt()); final Integer version = toInteger(versionStr); if (version == null) { System.err.println("??????: " + versionStr); printUsage(PROGNAME); return false; } final boolean isOverlay = commandLine.hasOption(OVERLAY_FLAG.getOpt()); final String saltStr = commandLine.getOptionValue(SALT.getOpt()); final byte[] salt; if (saltStr == null) { salt = null; } else { salt = toByteArray(saltStr, ObbInfoV1.SALT_LENGTH); if (salt == null) { System.err.println("????: " + saltStr); printUsage(PROGNAME); return false; } } final String[] nonRecognizedArgs = commandLine.getArgs(); if (nonRecognizedArgs.length == 0) { System.err.println("????????"); printUsage(PROGNAME); return false; } if (nonRecognizedArgs.length != 1) { System.err.println("???????"); printUsage(PROGNAME); return false; } final File targetFile = new File(nonRecognizedArgs[0]); final RandomAccessFile targetRaFile; try { targetRaFile = new RandomAccessFile(targetFile, "rw"); } catch (FileNotFoundException e) { System.err.println("????: " + targetFile.getPath()); return false; } try { try { final ObbInfoV1 info = ObbInfoV1.fromFile(targetRaFile); System.err.println( "?? OBB ???????: " + info.toString()); return false; } catch (IOException e) { System.err .println("????????: " + targetFile.getPath()); return false; } catch (NotObbException e) { // } int flag = 0; if (isOverlay) { flag |= ObbInfoV1.FLAG_OVERLAY; } if (salt != null) { flag |= ObbInfoV1.FLAG_SALTED; } final ObbInfoV1 obbInfo = new ObbInfoV1(flag, salt, pkgName, version.intValue()); final ByteBuffer obbInfoBytes = obbInfo.toBytes(); // ??? targetRaFile.setLength(targetRaFile.length() + obbInfoBytes.remaining()); targetRaFile.seek(targetRaFile.length() - obbInfoBytes.remaining()); targetRaFile.write(obbInfoBytes.array(), obbInfoBytes.arrayOffset(), obbInfoBytes.remaining()); } catch (IOException e) { System.err.println("OBB ?????????: " + targetFile.getPath()); return false; } finally { try { targetRaFile.close(); } catch (IOException e) { System.err.println("OBB ?????????: " + targetFile.getPath()); return false; } } System.err.println("OBB ??????????: " + targetFile.getPath()); return true; }