List of usage examples for java.lang Long SIZE
int SIZE
To view the source code for java.lang Long SIZE.
Click Source Link
From source file:org.apache.kylin.engine.mr.common.CubeStatsReader.java
/** * Estimate the cuboid's size// ww w .j a va 2 s. c o m * * @return the cuboid size in M bytes */ private static double estimateCuboidStorageSize(CubeSegment cubeSegment, long cuboidId, long rowCount, long baseCuboidId, List<Integer> rowKeyColumnLength) { int rowkeyLength = cubeSegment.getRowKeyPreambleSize(); KylinConfig kylinConf = cubeSegment.getConfig(); long mask = Long.highestOneBit(baseCuboidId); long parentCuboidIdActualLength = (long) Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId); for (int i = 0; i < parentCuboidIdActualLength; i++) { if ((mask & cuboidId) > 0) { rowkeyLength += rowKeyColumnLength.get(i); //colIO.getColumnLength(columnList.get(i)); } mask = mask >> 1; } // add the measure length int normalSpace = rowkeyLength; int countDistinctSpace = 0; for (MeasureDesc measureDesc : cubeSegment.getCubeDesc().getMeasures()) { DataType returnType = measureDesc.getFunction().getReturnDataType(); if (measureDesc.getFunction().getExpression().equals(FunctionDesc.FUNC_COUNT_DISTINCT)) { countDistinctSpace += returnType.getStorageBytesEstimate(); } else { normalSpace += returnType.getStorageBytesEstimate(); } } double cuboidSizeRatio = kylinConf.getJobCuboidSizeRatio(); double cuboidSizeMemHungryRatio = kylinConf.getJobCuboidSizeCountDistinctRatio(); double ret = (1.0 * normalSpace * rowCount * cuboidSizeRatio + 1.0 * countDistinctSpace * rowCount * cuboidSizeMemHungryRatio) / (1024L * 1024L); return ret; }
From source file:gobblin.writer.SimpleDataWriterTest.java
/** * Prepend the size to each record and delimit the record. Each record * should be prepended by the size of that record and the bytes written should * include the prepended bytes.//from w w w . ja va2 s .c o m */ @Test public void testPrependSizeWithDelimiter() throws IOException { properties.setProp(ConfigurationKeys.SIMPLE_WRITER_PREPEND_SIZE, true); SimpleDataWriter writer = buildSimpleDataWriter(); byte[] rec1 = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; byte[] rec2 = { 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 }; byte[] rec3 = { 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45 }; byte[][] records = { rec1, rec2, rec3 }; writer.write(rec1); writer.write(rec2); writer.write(rec3); writer.close(); writer.commit(); Assert.assertEquals(writer.recordsWritten(), 3); Assert.assertEquals(writer.bytesWritten(), rec1.length + rec2.length + rec3.length + (Long.SIZE / 8 * 3) + 3); File outputFile = new File(writer.getOutputFilePath()); DataInputStream dis = new DataInputStream(new FileInputStream(outputFile)); for (int i = 0; i < 3; i++) { long size = dis.readLong(); Assert.assertEquals(size, records[i].length + 1); for (int j = 0; j < size - 1; j++) { Assert.assertEquals(dis.readByte(), records[i][j]); } Assert.assertEquals(dis.readByte(), '\n'); } }
From source file:org.apache.hadoop.util.TestStringUtils.java
@Test(timeout = 30000) public void testTraditionalBinaryPrefix() throws Exception { //test string2long(..) String[] symbol = { "k", "m", "g", "t", "p", "e" }; long m = 1024; for (String s : symbol) { assertEquals(0, string2long(0 + s)); assertEquals(m, string2long(1 + s)); m *= 1024;//from w w w .jav a2 s . c o m } assertEquals(0L, string2long("0")); assertEquals(1024L, string2long("1k")); assertEquals(-1024L, string2long("-1k")); assertEquals(1259520L, string2long("1230K")); assertEquals(-1259520L, string2long("-1230K")); assertEquals(104857600L, string2long("100m")); assertEquals(-104857600L, string2long("-100M")); assertEquals(956703965184L, string2long("891g")); assertEquals(-956703965184L, string2long("-891G")); assertEquals(501377302265856L, string2long("456t")); assertEquals(-501377302265856L, string2long("-456T")); assertEquals(11258999068426240L, string2long("10p")); assertEquals(-11258999068426240L, string2long("-10P")); assertEquals(1152921504606846976L, string2long("1e")); assertEquals(-1152921504606846976L, string2long("-1E")); String tooLargeNumStr = "10e"; try { string2long(tooLargeNumStr); fail("Test passed for a number " + tooLargeNumStr + " too large"); } catch (IllegalArgumentException e) { assertEquals(tooLargeNumStr + " does not fit in a Long", e.getMessage()); } String tooSmallNumStr = "-10e"; try { string2long(tooSmallNumStr); fail("Test passed for a number " + tooSmallNumStr + " too small"); } catch (IllegalArgumentException e) { assertEquals(tooSmallNumStr + " does not fit in a Long", e.getMessage()); } String invalidFormatNumStr = "10kb"; char invalidPrefix = 'b'; try { string2long(invalidFormatNumStr); fail("Test passed for a number " + invalidFormatNumStr + " has invalid format"); } catch (IllegalArgumentException e) { assertEquals("Invalid size prefix '" + invalidPrefix + "' in '" + invalidFormatNumStr + "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)", e.getMessage()); } //test long2string(..) assertEquals("0", long2String(0, null, 2)); for (int decimalPlace = 0; decimalPlace < 2; decimalPlace++) { for (int n = 1; n < TraditionalBinaryPrefix.KILO.value; n++) { assertEquals(n + "", long2String(n, null, decimalPlace)); assertEquals(-n + "", long2String(-n, null, decimalPlace)); } assertEquals("1 K", long2String(1L << 10, null, decimalPlace)); assertEquals("-1 K", long2String(-1L << 10, null, decimalPlace)); } assertEquals("8.00 E", long2String(Long.MAX_VALUE, null, 2)); assertEquals("8.00 E", long2String(Long.MAX_VALUE - 1, null, 2)); assertEquals("-8 E", long2String(Long.MIN_VALUE, null, 2)); assertEquals("-8.00 E", long2String(Long.MIN_VALUE + 1, null, 2)); final String[] zeros = { " ", ".0 ", ".00 " }; for (int decimalPlace = 0; decimalPlace < zeros.length; decimalPlace++) { final String trailingZeros = zeros[decimalPlace]; for (int e = 11; e < Long.SIZE - 1; e++) { final TraditionalBinaryPrefix p = TraditionalBinaryPrefix.values()[e / 10 - 1]; { // n = 2^e final long n = 1L << e; final String expected = (n / p.value) + " " + p.symbol; assertEquals("n=" + n, expected, long2String(n, null, 2)); } { // n = 2^e + 1 final long n = (1L << e) + 1; final String expected = (n / p.value) + trailingZeros + p.symbol; assertEquals("n=" + n, expected, long2String(n, null, decimalPlace)); } { // n = 2^e - 1 final long n = (1L << e) - 1; final String expected = ((n + 1) / p.value) + trailingZeros + p.symbol; assertEquals("n=" + n, expected, long2String(n, null, decimalPlace)); } } } assertEquals("1.50 K", long2String(3L << 9, null, 2)); assertEquals("1.5 K", long2String(3L << 9, null, 1)); assertEquals("1.50 M", long2String(3L << 19, null, 2)); assertEquals("2 M", long2String(3L << 19, null, 0)); assertEquals("3 G", long2String(3L << 30, null, 2)); // test byteDesc(..) assertEquals("0 B", StringUtils.byteDesc(0)); assertEquals("-100 B", StringUtils.byteDesc(-100)); assertEquals("1 KB", StringUtils.byteDesc(1024)); assertEquals("1.50 KB", StringUtils.byteDesc(3L << 9)); assertEquals("1.50 MB", StringUtils.byteDesc(3L << 19)); assertEquals("3 GB", StringUtils.byteDesc(3L << 30)); // test formatPercent(..) assertEquals("10%", StringUtils.formatPercent(0.1, 0)); assertEquals("10.0%", StringUtils.formatPercent(0.1, 1)); assertEquals("10.00%", StringUtils.formatPercent(0.1, 2)); assertEquals("1%", StringUtils.formatPercent(0.00543, 0)); assertEquals("0.5%", StringUtils.formatPercent(0.00543, 1)); assertEquals("0.54%", StringUtils.formatPercent(0.00543, 2)); assertEquals("0.543%", StringUtils.formatPercent(0.00543, 3)); assertEquals("0.5430%", StringUtils.formatPercent(0.00543, 4)); }
From source file:it.unimi.di.big.mg4j.tool.PartitionLexically.java
public PartitionLexically(final String inputBasename, final String outputBasename, final LexicalPartitioningStrategy strategy, final String strategyFilename, final int bufferSize, final long logInterval) { this.inputBasename = inputBasename; this.outputBasename = outputBasename; this.strategy = strategy; this.strategyFilename = strategyFilename; this.bufferSize = bufferSize & ~(Long.SIZE - 1); this.logInterval = logInterval; numIndices = strategy.numberOfLocalIndices(); strategyProperties = strategy.properties(); localBasename = new String[numIndices]; for (int i = 0; i < numIndices; i++) localBasename[i] = outputBasename + "-" + i; }
From source file:it.unimi.dsi.sux4j.mph.ZFastTrieDistributorMonotoneMinimalPerfectHashFunction.java
/** Creates a new monotone minimal perfect hash function based on a z-fast trie distributor using the given * keys, transformation strategy and bucket size. * //ww w . java 2s . com * @param keys the keys among which the trie must be able to rank. * @param transform a transformation strategy that must turn the keys into a list of * distinct, prefix-free, lexicographically increasing (in iteration order) bit vectors. * @param log2BucketSize the logarithm of the bucket size, or -1 for the default value. * @param signatureWidth a signature width, or 0 for no signature. * @param tempDir a temporary directory for the store files, or {@code null} for the standard temporary directory. */ protected ZFastTrieDistributorMonotoneMinimalPerfectHashFunction(final Iterable<? extends T> keys, final TransformationStrategy<? super T> transform, final int log2BucketSize, final int signatureWidth, final File tempDir) throws IOException { this.transform = transform; defRetValue = -1; // For the very few cases in which we can decide long maxLength = 0; long totalLength = 0; RandomGenerator r = new XorShift1024StarRandomGenerator(); final ChunkedHashStore<BitVector> chunkedHashStore = new ChunkedHashStore<BitVector>( TransformationStrategies.identity(), tempDir); chunkedHashStore.reset(r.nextLong()); final Iterable<BitVector> bitVectors = TransformationStrategies.wrap(keys, transform); final ProgressLogger pl = new ProgressLogger(LOGGER); pl.displayLocalSpeed = true; pl.displayFreeMemory = true; pl.itemsName = "keys"; pl.start("Scanning collection..."); for (BitVector bv : bitVectors) { maxLength = Math.max(maxLength, bv.length()); totalLength += bv.length(); chunkedHashStore.add(bv); pl.lightUpdate(); } pl.done(); chunkedHashStore.checkAndRetry(bitVectors); size = chunkedHashStore.size(); if (size == 0) { this.log2BucketSize = -1; distributor = null; offset = null; signatureMask = 0; signatures = null; chunkedHashStore.close(); return; } final long averageLength = (totalLength + size - 1) / size; final long forecastBucketSize = (long) Math.ceil( 10.5 + 4.05 * log(averageLength) + 2.43 * log(log(size) + 1) + 2.43 * log(log(averageLength) + 1)); this.log2BucketSize = log2BucketSize == -1 ? Fast.mostSignificantBit(forecastBucketSize) : log2BucketSize; LOGGER.debug("Average length: " + averageLength); LOGGER.debug("Max length: " + maxLength); LOGGER.debug("Bucket size: " + (1L << this.log2BucketSize)); LOGGER.info("Computing z-fast trie distributor..."); distributor = new ZFastTrieDistributor<BitVector>(bitVectors, this.log2BucketSize, TransformationStrategies.identity(), chunkedHashStore); LOGGER.info("Computing offsets..."); offset = new GOV3Function.Builder<BitVector>().store(chunkedHashStore).values(new AbstractLongBigList() { final long bucketSizeMask = (1L << ZFastTrieDistributorMonotoneMinimalPerfectHashFunction.this.log2BucketSize) - 1; public long getLong(long index) { return index & bucketSizeMask; } public long size64() { return size; } }, this.log2BucketSize).indirect().build(); seed = chunkedHashStore.seed(); double logU = averageLength * log(2); LOGGER.info("Forecast bit cost per element: " + 1.0 / forecastBucketSize * (-6 * log2(log(2)) + 5 * log2(logU) + 2 * log2(forecastBucketSize) + log2(log(logU) - log(log(2))) + 6 * GOV3Function.C + 3 * log2(E) + 3 * log2(log(3.0 * size)) + 3 + GOV3Function.C * forecastBucketSize + GOV3Function.C * forecastBucketSize * log2(forecastBucketSize))); LOGGER.info("Actual bit cost per element: " + (double) numBits() / size); if (signatureWidth != 0) { signatureMask = -1L >>> Long.SIZE - signatureWidth; signatures = chunkedHashStore.signatures(signatureWidth, pl); } else { signatureMask = 0; signatures = null; } chunkedHashStore.close(); }
From source file:org.cloudata.core.commitlog.pipe.CommitLogFileChannel.java
private synchronized void cancelLastMark() throws IOException { if (!isOpen()) { return;//from w ww . j a v a 2s. c om } try { channel.truncate(readLastIndex(indexChannel)); } catch (Exception e) { LOG.warn("Fail to truncate channel", e); } long pos = indexChannel.size() - Long.SIZE / 8; if (pos < 0) { pos = 0; } indexChannel.truncate(pos); }
From source file:org.kiji.schema.FormattedEntityId.java
/** * Create an hbase row key, which is a byte array from the given formatted kijiRowKey. * This method requires that the kijiRowKey argument is the correct length for the specified * format./*from w w w. j a va 2 s . co m*/ * The following encoding will be used to ensure correct ordering: * Strings are UTF-8 encoded and terminated by a null byte. Strings cannot contain "\u0000". * Integers are exactly 4 bytes long. * Longs are exactly 8 bytes long. * Both integers and longs have the sign bit flipped so that their values are wrapped around to * create the correct lexicographic ordering. (i.e. after converting to byte array, * MIN_INT < 0 < MAX_INT). * Hashed components are exactly hash_size bytes long and are the first component of * the hbase key. * Except for the first, all components of a kijiRowKey can be null. However, to maintain * ordering, all components to the right of a null component must also be null. Nullable index * in the row key format specifies which component (and hence following components) are nullable. * By default, the hash only uses the first component, but this can be changed using the Range * Scan index. * * @param format The formatted row key format for this table. * @param kijiRowKey An ordered list of Objects of the key components. * @return A byte array representing the encoded Hbase row key. */ private static byte[] makeHbaseRowKey(RowKeyFormat2 format, List<Object> kijiRowKey) { ArrayList<byte[]> hbaseKey = new ArrayList<byte[]>(); final byte zeroDelim = 0; int pos; for (pos = 0; pos < kijiRowKey.size(); pos++) { // we have already done the validation check for null cascades. if (null == kijiRowKey.get(pos)) { continue; } byte[] tempBytes; switch (getType(kijiRowKey.get(pos))) { case STRING: if (((String) kijiRowKey.get(pos)).contains("\u0000")) { throw new EntityIdException("String component cannot contain \u0000"); } try { hbaseKey.add(((String) kijiRowKey.get(pos)).getBytes("UTF-8")); } catch (UnsupportedEncodingException e) { LOG.error(e.toString()); throw new EntityIdException(String.format("UnsupportedEncoding for component %d", pos)); } break; case INTEGER: int temp = (Integer) kijiRowKey.get(pos); tempBytes = ByteBuffer.allocate(Integer.SIZE / Byte.SIZE).putInt(temp).array(); tempBytes[0] = (byte) ((int) tempBytes[0] ^ (int) Byte.MIN_VALUE); hbaseKey.add(tempBytes); break; case LONG: long templong = (Long) kijiRowKey.get(pos); tempBytes = ByteBuffer.allocate(Long.SIZE / Byte.SIZE).putLong(templong).array(); tempBytes[0] = (byte) ((int) tempBytes[0] ^ (int) Byte.MIN_VALUE); hbaseKey.add(tempBytes); break; default: throw new RuntimeException("Invalid code path"); } } // hash stuff int hashUpto = format.getRangeScanStartIndex() - 1; ByteArrayOutputStream tohash = new ByteArrayOutputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); for (pos = 0; pos <= hashUpto && pos < hbaseKey.size(); pos++) { tohash.write(hbaseKey.get(pos), 0, hbaseKey.get(pos).length); } byte[] hashed = Arrays.copyOfRange(Hasher.hash(tohash.toByteArray()), 0, format.getSalt().getHashSize()); baos.write(hashed, 0, hashed.length); // to materialize or not to materialize that is the question if (format.getSalt().getSuppressKeyMaterialization()) { return baos.toByteArray(); } else { for (pos = 0; pos < hbaseKey.size(); pos++) { baos.write(hbaseKey.get(pos), 0, hbaseKey.get(pos).length); if (format.getComponents().get(pos).getType() == ComponentType.STRING || format.getComponents().get(pos) == null) { // empty strings will be encoded as null, hence we need to delimit them too baos.write(zeroDelim); } } return baos.toByteArray(); } }
From source file:com.moz.fiji.schema.FormattedEntityId.java
/** * Create an hbase row key, which is a byte array from the given formatted fijiRowKey. * This method requires that the fijiRowKey argument is the correct length for the specified * format./*www. ja va2 s . c o m*/ * The following encoding will be used to ensure correct ordering: * Strings are UTF-8 encoded and terminated by a null byte. Strings cannot contain "\u0000". * Integers are exactly 4 bytes long. * Longs are exactly 8 bytes long. * Both integers and longs have the sign bit flipped so that their values are wrapped around to * create the correct lexicographic ordering. (i.e. after converting to byte array, * MIN_INT < 0 < MAX_INT). * Hashed components are exactly hash_size bytes long and are the first component of * the hbase key. * Except for the first, all components of a fijiRowKey can be null. However, to maintain * ordering, all components to the right of a null component must also be null. Nullable index * in the row key format specifies which component (and hence following components) are nullable. * By default, the hash only uses the first component, but this can be changed using the Range * Scan index. * * @param format The formatted row key format for this table. * @param fijiRowKey An ordered list of Objects of the key components. * @return A byte array representing the encoded Hbase row key. */ private static byte[] makeHbaseRowKey(RowKeyFormat2 format, List<Object> fijiRowKey) { ArrayList<byte[]> hbaseKey = new ArrayList<byte[]>(); final byte zeroDelim = 0; int pos; for (pos = 0; pos < fijiRowKey.size(); pos++) { // we have already done the validation check for null cascades. if (null == fijiRowKey.get(pos)) { continue; } byte[] tempBytes; switch (getType(fijiRowKey.get(pos))) { case STRING: if (((String) fijiRowKey.get(pos)).contains("\u0000")) { throw new EntityIdException("String component cannot contain \u0000"); } try { hbaseKey.add(((String) fijiRowKey.get(pos)).getBytes("UTF-8")); } catch (UnsupportedEncodingException e) { LOG.error(e.toString()); throw new EntityIdException(String.format("UnsupportedEncoding for component %d", pos)); } break; case INTEGER: int temp = (Integer) fijiRowKey.get(pos); tempBytes = ByteBuffer.allocate(Integer.SIZE / Byte.SIZE).putInt(temp).array(); tempBytes[0] = (byte) ((int) tempBytes[0] ^ (int) Byte.MIN_VALUE); hbaseKey.add(tempBytes); break; case LONG: long templong = (Long) fijiRowKey.get(pos); tempBytes = ByteBuffer.allocate(Long.SIZE / Byte.SIZE).putLong(templong).array(); tempBytes[0] = (byte) ((int) tempBytes[0] ^ (int) Byte.MIN_VALUE); hbaseKey.add(tempBytes); break; default: throw new RuntimeException("Invalid code path"); } } // hash stuff int hashUpto = format.getRangeScanStartIndex() - 1; ByteArrayOutputStream tohash = new ByteArrayOutputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); for (pos = 0; pos <= hashUpto && pos < hbaseKey.size(); pos++) { tohash.write(hbaseKey.get(pos), 0, hbaseKey.get(pos).length); } byte[] hashed = Arrays.copyOfRange(Hasher.hash(tohash.toByteArray()), 0, format.getSalt().getHashSize()); baos.write(hashed, 0, hashed.length); // to materialize or not to materialize that is the question if (format.getSalt().getSuppressKeyMaterialization()) { return baos.toByteArray(); } else { for (pos = 0; pos < hbaseKey.size(); pos++) { baos.write(hbaseKey.get(pos), 0, hbaseKey.get(pos).length); if (format.getComponents().get(pos).getType() == ComponentType.STRING || format.getComponents().get(pos) == null) { // empty strings will be encoded as null, hence we need to delimit them too baos.write(zeroDelim); } } return baos.toByteArray(); } }
From source file:org.mycore.common.content.MCRContent.java
/** * Uses provided parameter to compute simple weak ETag. * //from ww w . j ava2 s. com * @param systemId * != null, {@link #getSystemId()} * @param length * >= 0, {@link #length()} * @param lastModified * >= 0, {@link #lastModified()} * @return null if any preconditions are not met. */ protected String getSimpleWeakETag(String systemId, long length, long lastModified) { if (systemId == null || length < 0 || lastModified < 0) { return null; } StringBuilder b = new StringBuilder(32); b.append("W/\""); long lhash = systemId.hashCode(); byte[] unencodedETag = ByteBuffer.allocate(Long.SIZE / 4).putLong(lastModified ^ lhash) .putLong(length ^ lhash).array(); b.append(Base64.getEncoder().encodeToString(unencodedETag)); b.append('"'); return b.toString(); }
From source file:com.google.cloud.bigtable.hbase.TestIncrement.java
/** * Requirement 6.6/*w w w . j av a 2 s .c o m*/ */ @Test public void testIncrementEightBytes() throws IOException { // Initialize Table table = getConnection().getTable(TABLE_NAME); byte[] rowKey = dataHelper.randomData("testrow-"); byte[] qual = dataHelper.randomData("qual-"); byte[] value = new byte[8]; new Random().nextBytes(value); ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); buffer.put(value); buffer.flip(); long equivalentValue = buffer.getLong(); // Put the bytes in Put put = new Put(rowKey).addColumn(COLUMN_FAMILY, qual, value); table.put(put); // Increment Increment increment = new Increment(rowKey).addColumn(COLUMN_FAMILY, qual, 1L); Result result = table.increment(increment); Assert.assertEquals("Should have incremented the bytes like a long", equivalentValue + 1L, Bytes.toLong(CellUtil.cloneValue(result.getColumnLatestCell(COLUMN_FAMILY, qual)))); }