List of usage examples for io.netty.buffer ByteBuf getLong
public abstract long getLong(int index);
From source file:org.apache.bookkeeper.bookie.Journal.java
License:Apache License
/** * record an add entry operation in journal. *//* w ww. java 2 s . c om*/ public void logAddEntry(ByteBuf entry, boolean ackBeforeSync, WriteCallback cb, Object ctx) throws InterruptedException { long ledgerId = entry.getLong(entry.readerIndex() + 0); long entryId = entry.getLong(entry.readerIndex() + 8); logAddEntry(ledgerId, entryId, entry, ackBeforeSync, cb, ctx); }
From source file:org.apache.bookkeeper.bookie.LedgerDescriptorImpl.java
License:Apache License
@Override long addEntry(ByteBuf entry) throws IOException, BookieException { long ledgerId = entry.getLong(entry.readerIndex()); if (ledgerId != this.ledgerId) { throw new IOException("Entry for ledger " + ledgerId + " was sent to " + this.ledgerId); }/* www. jav a 2 s .c o m*/ return ledgerStorage.addEntry(entry); }
From source file:org.apache.bookkeeper.bookie.SortedLedgerStorage.java
License:Apache License
@Override public long addEntry(ByteBuf entry) throws IOException { long ledgerId = entry.getLong(entry.readerIndex() + 0); long entryId = entry.getLong(entry.readerIndex() + 8); long lac = entry.getLong(entry.readerIndex() + 16); memTable.addEntry(ledgerId, entryId, entry.nioBuffer(), this); interleavedLedgerStorage.ledgerCache.updateLastAddConfirmed(ledgerId, lac); return entryId; }
From source file:org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage.java
License:Apache License
@Override public long addEntry(ByteBuf entry) throws IOException, BookieException { long ledgerId = entry.getLong(entry.readerIndex()); return getLedgerSorage(ledgerId).addEntry(entry); }
From source file:org.apache.bookkeeper.bookie.storage.ldb.LocationsIndexRebuildOp.java
License:Apache License
public void initiate() throws IOException { LOG.info("Starting index rebuilding"); // Move locations index to a backup directory String basePath = Bookie.getCurrentDirectory(conf.getLedgerDirs()[0]).toString(); Path currentPath = FileSystems.getDefault().getPath(basePath, "locations"); String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(new Date()); Path backupPath = FileSystems.getDefault().getPath(basePath, "locations.BACKUP-" + timestamp); Files.move(currentPath, backupPath); LOG.info("Created locations index backup at {}", backupPath); long startTime = System.nanoTime(); EntryLogger entryLogger = new EntryLogger(conf, new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()))); Set<Long> entryLogs = entryLogger.getEntryLogsSet(); String locationsDbPath = FileSystems.getDefault().getPath(basePath, "locations").toFile().toString(); Set<Long> activeLedgers = getActiveLedgers(conf, KeyValueStorageRocksDB.factory, basePath); LOG.info("Found {} active ledgers in ledger manager", activeLedgers.size()); KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(locationsDbPath, DbConfigType.Huge, conf);/* w ww . ja v a 2 s . c om*/ int totalEntryLogs = entryLogs.size(); int completedEntryLogs = 0; LOG.info("Scanning {} entry logs", totalEntryLogs); for (long entryLogId : entryLogs) { entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() { @Override public void process(long ledgerId, long offset, ByteBuf entry) throws IOException { long entryId = entry.getLong(8); // Actual location indexed is pointing past the entry size long location = (entryLogId << 32L) | (offset + 4); if (LOG.isDebugEnabled()) { LOG.debug("Rebuilding {}:{} at location {} / {}", ledgerId, entryId, location >> 32, location & (Integer.MAX_VALUE - 1)); } // Update the ledger index page LongPairWrapper key = LongPairWrapper.get(ledgerId, entryId); LongWrapper value = LongWrapper.get(location); newIndex.put(key.array, value.array); } @Override public boolean accept(long ledgerId) { return activeLedgers.contains(ledgerId); } }); ++completedEntryLogs; LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs, totalEntryLogs); } newIndex.sync(); newIndex.close(); LOG.info("Rebuilding index is done. Total time: {}", DurationFormatUtils .formatDurationHMS(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime))); }
From source file:org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.java
License:Apache License
@Override public long addEntry(ByteBuf entry) throws IOException, BookieException { long startTime = MathUtils.nowInNano(); long ledgerId = entry.getLong(entry.readerIndex()); long entryId = entry.getLong(entry.readerIndex() + 8); long lac = entry.getLong(entry.readerIndex() + 16); if (log.isDebugEnabled()) { log.debug("Add entry. {}@{}, lac = {}", ledgerId, entryId, lac); }/*from w ww . j av a2 s. c o m*/ // First we try to do an optimistic locking to get access to the current write cache. // This is based on the fact that the write cache is only being rotated (swapped) every 1 minute. During the // rest of the time, we can have multiple thread using the optimistic lock here without interfering. long stamp = writeCacheRotationLock.tryOptimisticRead(); boolean inserted = false; inserted = writeCache.put(ledgerId, entryId, entry); if (!writeCacheRotationLock.validate(stamp)) { // The write cache was rotated while we were inserting. We need to acquire the proper read lock and repeat // the operation because we might have inserted in a write cache that was already being flushed and cleared, // without being sure about this last entry being flushed or not. stamp = writeCacheRotationLock.readLock(); try { inserted = writeCache.put(ledgerId, entryId, entry); } finally { writeCacheRotationLock.unlockRead(stamp); } } if (!inserted) { triggerFlushAndAddEntry(ledgerId, entryId, entry); } // after successfully insert the entry, update LAC and notify the watchers updateCachedLacIfNeeded(ledgerId, lac); recordSuccessfulEvent(dbLedgerStorageStats.getAddEntryStats(), startTime); return entryId; }
From source file:org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage.java
License:Apache License
private void fillReadAheadCache(long orginalLedgerId, long firstEntryId, long firstEntryLocation) { try {/*from w w w . j a va 2 s .co m*/ long firstEntryLogId = (firstEntryLocation >> 32); long currentEntryLogId = firstEntryLogId; long currentEntryLocation = firstEntryLocation; int count = 0; long size = 0; while (count < readAheadCacheBatchSize && currentEntryLogId == firstEntryLogId) { ByteBuf entry = entryLogger.internalReadEntry(orginalLedgerId, -1, currentEntryLocation); try { long currentEntryLedgerId = entry.getLong(0); long currentEntryId = entry.getLong(8); if (currentEntryLedgerId != orginalLedgerId) { // Found an entry belonging to a different ledger, stopping read-ahead return; } // Insert entry in read cache readCache.put(orginalLedgerId, currentEntryId, entry); count++; size += entry.readableBytes(); currentEntryLocation += 4 + entry.readableBytes(); currentEntryLogId = currentEntryLocation >> 32; } finally { entry.release(); } } dbLedgerStorageStats.getReadAheadBatchCountStats().registerSuccessfulValue(count); dbLedgerStorageStats.getReadAheadBatchSizeStats().registerSuccessfulValue(size); } catch (Exception e) { if (log.isDebugEnabled()) { log.debug("Exception during read ahead for ledger: {}: e", orginalLedgerId, e); } } }
From source file:org.apache.bookkeeper.statelib.impl.mvcc.MVCCRecord.java
License:Apache License
public void setValue(ByteBuf buf, ValueType valueType) { if (null != value) { value.release();/* ww w . j a v a 2s .com*/ } this.value = buf; this.valueType = valueType; if (ValueType.NUMBER == valueType) { this.number = buf.getLong(0); } }
From source file:org.apache.bookkeeper.tools.cli.commands.bookie.ReadLogCommand.java
License:Apache License
/** * Scan over an entry log file for a particular entry. * * @param logId Entry Log File id./*from ww w . ja v a2 s . c om*/ * @param ledgerId id of the ledger * @param entryId entryId of the ledger we are looking for (-1 for all of the entries of the ledger) * @param printMsg Whether printing the entry data. * @throws Exception */ private void scanEntryLogForSpecificEntry(ServerConfiguration conf, long logId, final long ledgerId, final long entryId, final boolean printMsg) throws Exception { System.out.println("Scan entry log " + logId + " (" + Long.toHexString(logId) + ".log)" + " for LedgerId " + ledgerId + ((entryId == -1) ? "" : " for EntryId " + entryId)); final MutableBoolean entryFound = new MutableBoolean(false); scanEntryLog(conf, logId, new EntryLogger.EntryLogScanner() { @Override public boolean accept(long candidateLedgerId) { return ((candidateLedgerId == ledgerId) && ((!entryFound.booleanValue()) || (entryId == -1))); } @Override public void process(long candidateLedgerId, long startPos, ByteBuf entry) { long entrysLedgerId = entry.getLong(entry.readerIndex()); long entrysEntryId = entry.getLong(entry.readerIndex() + 8); if ((candidateLedgerId == entrysLedgerId) && (candidateLedgerId == ledgerId) && ((entrysEntryId == entryId) || (entryId == -1))) { entryFound.setValue(true); FormatUtil.formatEntry(startPos, entry, printMsg, ledgerIdFormatter, entryFormatter); } } }); if (!entryFound.booleanValue()) { System.out.println("LedgerId " + ledgerId + ((entryId == -1) ? "" : " EntryId " + entryId) + " is not available in the entry log " + logId + " (" + Long.toHexString(logId) + ".log)"); } }
From source file:org.dcache.xrootd.protocol.messages.GenericReadRequestMessage.java
License:Open Source License
public GenericReadRequestMessage(ByteBuf buffer, int requestId) { super(buffer, requestId); int alen = buffer.getInt(20); if (alen <= 8) { pathid = -1;/*w w w .ja v a 2s . com*/ readList = new EmbeddedReadRequest[0]; } else { int prefix = 0; if (alen % 16 == 0) { pathid = -1; } else if (alen % 16 != 8) { pathid = -1; LOGGER.warn("invalid readv request: data doesn't start with 8 byte prefix (pathid)"); } else { pathid = buffer.getUnsignedByte(24); prefix = 8; } int numberOfListEntries = (alen - prefix) / 16; readList = new EmbeddedReadRequest[numberOfListEntries]; for (int i = 0; i < numberOfListEntries; i++) { int j = 24 + prefix + i * 16; readList[i] = new EmbeddedReadRequest(buffer.getInt(j), buffer.getInt(j + 4), buffer.getLong(j + 8)); } } }