List of usage examples for java.lang Long SIZE
int SIZE
To view the source code for java.lang Long SIZE.
Click Source Link
From source file:it.unimi.dsi.webgraph.algo.HyperBall.java
private long usedMemory() { long bytes = 0; for (long[] a : bits) bytes += a.length * ((long) Long.SIZE / Byte.SIZE); if (!external) bytes *= 2;//from ww w . j a va 2s .c o m if (sumOfDistances != null) bytes += sumOfDistances.length * ((long) Float.SIZE / Byte.SIZE); if (sumOfInverseDistances != null) bytes += sumOfInverseDistances.length * ((long) Float.SIZE / Byte.SIZE); for (int i = discountFunction.length; i-- != 0;) bytes += discountedCentrality[i].length * ((long) Float.SIZE / Byte.SIZE); // We use bits, so usedMemory() can be called before allocation. if (modifiedCounter != null) bytes += modifiedCounter.length; if (modifiedResultCounter != null) bytes += modifiedResultCounter.length; if (nextMustBeChecked != null) bytes += nextMustBeChecked.length; if (mustBeChecked != null) bytes += mustBeChecked.length; return bytes; }
From source file:efen.parsewiki.WikipediaDocumentSequence.java
public static void main(final String arg[]) throws ParserConfigurationException, SAXException, IOException, JSAPException, ClassNotFoundException { SimpleJSAP jsap = new SimpleJSAP(WikipediaDocumentSequence.class.getName(), "Computes the redirects of a Wikipedia dump and integrate them into an existing virtual document resolver for the dump.", new Parameter[] { new Switch("bzip2", 'b', "bzip2", "The file is compressed with bzip2"), new Switch("iso", 'i', "iso", "Use ISO-8859-1 coding internally (i.e., just use the lower eight bits of each character)."), new FlaggedOption("width", JSAP.INTEGER_PARSER, Integer.toString(Long.SIZE), JSAP.NOT_REQUIRED, 'w', "width", "The width, in bits, of the signatures used to sign the function from URIs to their rank."), new UnflaggedOption("file", JSAP.STRING_PARSER, JSAP.REQUIRED, "The file containing the Wikipedia dump."), new UnflaggedOption("baseURL", JSAP.STRING_PARSER, JSAP.REQUIRED, "The base URL for the collection (e.g., http://en.wikipedia.org/wiki/)."), new UnflaggedOption("uris", JSAP.STRING_PARSER, JSAP.REQUIRED, "The URIs of the documents in the collection (generated by ScanMetadata)."), new UnflaggedOption("vdr", JSAP.STRING_PARSER, JSAP.REQUIRED, "The name of a precomputed virtual document resolver for the collection."), new UnflaggedOption("redvdr", JSAP.STRING_PARSER, JSAP.REQUIRED, "The name of the resulting virtual document resolver.") }); JSAPResult jsapResult = jsap.parse(arg); if (jsap.messagePrinted()) return;/* w w w .jav a 2 s.co m*/ final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final Object2ObjectOpenHashMap<MutableString, String> redirects = new Object2ObjectOpenHashMap<MutableString, String>(); final String baseURL = jsapResult.getString("baseURL"); final ProgressLogger progressLogger = new ProgressLogger(LOGGER); progressLogger.itemsName = "redirects"; progressLogger.start("Extracting redirects..."); final SAXParser parser = saxParserFactory.newSAXParser(); final DefaultHandler handler = new DefaultHandler() { private boolean inTitle; private MutableString title = new MutableString(); @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { inTitle = false; title.length(0); } else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first title element. else if ("redirect".equals(localName) && attributes.getValue("title") != null) { progressLogger.update(); redirects.put(title.copy(), attributes.getValue("title")); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("title".equals(localName)) inTitle = false; } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inTitle) title.append(ch, start, length); } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inTitle) title.append(ch, start, length); } }; InputStream in = new FileInputStream(jsapResult.getString("file")); if (jsapResult.userSpecified("bzip2")) in = new BZip2CompressorInputStream(in); parser.parse(new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); progressLogger.done(); final Object2LongLinkedOpenHashMap<MutableString> resolved = new Object2LongLinkedOpenHashMap<MutableString>(); final VirtualDocumentResolver vdr = (VirtualDocumentResolver) BinIO.loadObject(jsapResult.getString("vdr")); progressLogger.expectedUpdates = redirects.size(); progressLogger.start("Examining redirects..."); for (Map.Entry<MutableString, String> e : redirects.entrySet()) { final MutableString start = new MutableString().append(baseURL) .append(Encoder.encodeTitleToUrl(e.getKey().toString(), true)); final MutableString end = new MutableString().append(baseURL) .append(Encoder.encodeTitleToUrl(e.getValue(), true)); final long s = vdr.resolve(start); if (s == -1) { final long t = vdr.resolve(end); if (t != -1) resolved.put(start.copy(), t); else LOGGER.warn("Failed redirect: " + start + " -> " + end); } else LOGGER.warn("URL " + start + " is already known to the virtual document resolver"); progressLogger.lightUpdate(); } progressLogger.done(); //System.err.println(resolved); final Iterable<MutableString> allURIs = Iterables .concat(new FileLinesCollection(jsapResult.getString("uris"), "UTF-8"), resolved.keySet()); final long numberOfDocuments = vdr.numberOfDocuments(); final TransformationStrategy<CharSequence> transformationStrategy = jsapResult.userSpecified("iso") ? TransformationStrategies.iso() : TransformationStrategies.utf16(); BinIO.storeObject(new URLMPHVirtualDocumentResolver(new SignedRedirectedStringMap(numberOfDocuments, new ShiftAddXorSignedStringMap(allURIs.iterator(), new MWHCFunction.Builder<CharSequence>().keys(allURIs).transform(transformationStrategy) .build(), jsapResult.getInt("width")), resolved.values().toLongArray())), jsapResult.getString("redvdr")); }
From source file:it.unimi.dsi.sux4j.mph.MWHCFunction.java
/** Returns the number of bits used by this structure. * /*from w w w . j a v a 2s. c om*/ * @return the number of bits used by this structure. */ public long numBits() { if (n == 0) return 0; return (marker != null ? rank.numBits() + marker.length() : 0) + (data != null ? data.size64() : 0) * width + seed.length * (long) Long.SIZE + offset.length * (long) Integer.SIZE; }
From source file:it.unimi.di.wikipedia.parsing.NamespacedWikipediaDocumentSequence.java
public static void main(final String arg[]) throws ParserConfigurationException, SAXException, IOException, JSAPException, ClassNotFoundException { SimpleJSAP jsap = new SimpleJSAP(NamespacedWikipediaDocumentSequence.class.getName(), "Computes the redirects of a Wikipedia dump and integrate them into an existing virtual document resolver for the dump.", new Parameter[] { new Switch("bzip2", 'b', "bzip2", "The file is compressed with bzip2"), new Switch("iso", 'i', "iso", "Use ISO-8859-1 coding internally (i.e., just use the lower eight bits of each character)."), new FlaggedOption("width", JSAP.INTEGER_PARSER, Integer.toString(Long.SIZE), JSAP.NOT_REQUIRED, 'w', "width", "The width, in bits, of the signatures used to sign the function from URIs to their rank."), new UnflaggedOption("file", JSAP.STRING_PARSER, JSAP.REQUIRED, "The file containing the Wikipedia dump."), new UnflaggedOption("baseURL", JSAP.STRING_PARSER, JSAP.REQUIRED, "The base URL for the collection (e.g., http://en.wikipedia.org/wiki/)."), new UnflaggedOption("uris", JSAP.STRING_PARSER, JSAP.REQUIRED, "The URIs of the documents in the collection (generated by ScanMetadata)."), new UnflaggedOption("vdr", JSAP.STRING_PARSER, JSAP.REQUIRED, "The name of a precomputed virtual document resolver for the collection."), new UnflaggedOption("redvdr", JSAP.STRING_PARSER, JSAP.REQUIRED, "The name of the resulting virtual document resolver.") }); JSAPResult jsapResult = jsap.parse(arg); if (jsap.messagePrinted()) return;// ww w . j av a2s. c o m final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final Object2ObjectOpenHashMap<MutableString, String> redirects = new Object2ObjectOpenHashMap<MutableString, String>(); final String baseURL = jsapResult.getString("baseURL"); final ProgressLogger progressLogger = new ProgressLogger(LOGGER); progressLogger.itemsName = "redirects"; progressLogger.start("Extracting redirects..."); final SAXParser parser = saxParserFactory.newSAXParser(); final DefaultHandler handler = new DefaultHandler() { private boolean inTitle; private MutableString title = new MutableString(); @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { inTitle = false; title.length(0); } else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first title element. else if ("redirect".equals(localName) && attributes.getValue("title") != null) { progressLogger.update(); redirects.put(title.copy(), attributes.getValue("title")); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("title".equals(localName)) inTitle = false; } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inTitle) title.append(ch, start, length); } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inTitle) title.append(ch, start, length); } }; InputStream in = new FileInputStream(jsapResult.getString("file")); if (jsapResult.userSpecified("bzip2")) in = new BZip2CompressorInputStream(in); parser.parse(new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); progressLogger.done(); final Object2LongLinkedOpenHashMap<MutableString> resolved = new Object2LongLinkedOpenHashMap<MutableString>(); final VirtualDocumentResolver vdr = (VirtualDocumentResolver) BinIO.loadObject(jsapResult.getString("vdr")); progressLogger.expectedUpdates = redirects.size(); progressLogger.start("Examining redirects..."); for (Map.Entry<MutableString, String> e : redirects.entrySet()) { final MutableString start = new MutableString().append(baseURL) .append(Encoder.encodeTitleToUrl(e.getKey().toString(), true)); final MutableString end = new MutableString().append(baseURL) .append(Encoder.encodeTitleToUrl(e.getValue(), true)); final long s = vdr.resolve(start); if (s == -1) { final long t = vdr.resolve(end); if (t != -1) resolved.put(start.copy(), t); else LOGGER.warn("Failed redirect: " + start + " -> " + end); } else LOGGER.warn("URL " + start + " is already known to the virtual document resolver"); progressLogger.lightUpdate(); } progressLogger.done(); //System.err.println(resolved); final Iterable<MutableString> allURIs = Iterables .concat(new FileLinesCollection(jsapResult.getString("uris"), "UTF-8"), resolved.keySet()); final long numberOfDocuments = vdr.numberOfDocuments(); final TransformationStrategy<CharSequence> transformationStrategy = jsapResult.userSpecified("iso") ? TransformationStrategies.iso() : TransformationStrategies.utf16(); BinIO.storeObject(new URLMPHVirtualDocumentResolver(new SignedRedirectedStringMap(numberOfDocuments, new ShiftAddXorSignedStringMap(allURIs.iterator(), new MWHCFunction.Builder<CharSequence>().keys(allURIs).transform(transformationStrategy) .build(), jsapResult.getInt("width")), resolved.values().toLongArray())), jsapResult.getString("redvdr")); }
From source file:it.unimi.dsi.sux4j.mph.GOV3Function.java
/** Low-level access to the output of this function. * * <p>This method makes it possible to build several kind of functions on the same {@link ChunkedHashStore} and * then retrieve the resulting values by generating a single triple of hashes. The method * {@link TwoStepsGOV3Function#getLong(Object)} is a good example of this technique. * * @param triple a triple generated as documented in {@link ChunkedHashStore}. * @return the output of the function.// w ww . j a v a 2 s. c o m */ public long getLongByTriple(final long[] triple) { if (n == 0) return defRetValue; final int[] e = new int[3]; final int chunk = chunkShift == Long.SIZE ? 0 : (int) (triple[0] >>> chunkShift); final long chunkOffset = offsetAndSeed[chunk] & OFFSET_MASK; Linear3SystemSolver.tripleToEquation(triple, offsetAndSeed[chunk] & ~OFFSET_MASK, (int) ((offsetAndSeed[chunk + 1] & OFFSET_MASK) - chunkOffset), e); final long e0 = e[0] + chunkOffset, e1 = e[1] + chunkOffset, e2 = e[2] + chunkOffset; if (e0 == -1) return defRetValue; final long result = rank == null ? data.getLong(e0) ^ data.getLong(e1) ^ data.getLong(e2) : (marker.getBoolean(e0) ? data.getLong(rank.rank(e0)) : 0) ^ (marker.getBoolean(e1) ? data.getLong(rank.rank(e1)) : 0) ^ (marker.getBoolean(e2) ? data.getLong(rank.rank(e2)) : 0); if (signatureMask == 0) return result; if (signatures != null) return result >= n || signatures.getLong(result) != (triple[0] & signatureMask) ? defRetValue : result; else return ((result ^ triple[0]) & signatureMask) != 0 ? defRetValue : 1; }
From source file:it.unimi.dsi.sux4j.io.ChunkedHashStore.java
/** Returns an iterator over the chunks of this chunked hash store. * * @return an iterator over the chunks of this chunked hash store. *///from w ww.ja v a2 s .com public Iterator<Chunk> iterator() { if (closed) throw new IllegalStateException("This " + getClass().getSimpleName() + " has been closed "); for (DataOutputStream d : dos) try { d.flush(); } catch (IOException e) { throw new RuntimeException(e); } int m = 0; for (int i = 0; i < virtualDiskChunks; i++) { int s = 0; for (int j = 0; j < diskChunkStep; j++) s += count[i * diskChunkStep + j]; if (s > m) m = s; } final int maxCount = m; return new AbstractObjectIterator<Chunk>() { private int chunk; private FastBufferedInputStream fbis; private int last; private int chunkSize; private final long[] buffer0 = new long[maxCount]; private final long[] buffer1 = new long[maxCount]; private final long[] buffer2 = new long[maxCount]; private final long[] data = hashMask != 0 ? null : new long[maxCount]; public boolean hasNext() { return chunk < chunks; } @SuppressWarnings("unchecked") public Chunk next() { if (!hasNext()) throw new NoSuchElementException(); final long[] buffer0 = this.buffer0; if (chunk % (chunks / virtualDiskChunks) == 0) { final int diskChunk = (int) (chunk / (chunks / virtualDiskChunks)); final long[] buffer1 = this.buffer1, buffer2 = this.buffer2; chunkSize = 0; try { if (diskChunkStep == 1) { fbis = new FastBufferedInputStream(new FileInputStream(file[diskChunk])); chunkSize = count[diskChunk]; } else { final FileInputStream[] fis = new FileInputStream[diskChunkStep]; for (int i = 0; i < fis.length; i++) { fis[i] = new FileInputStream(file[diskChunk * diskChunkStep + i]); chunkSize += count[diskChunk * diskChunkStep + i]; } fbis = new FastBufferedInputStream(new SequenceInputStream( new IteratorEnumeration(Arrays.asList(fis).iterator()))); } final DataInputStream dis = new DataInputStream(fbis); final long triple[] = new long[3]; int count = 0; for (int j = 0; j < chunkSize; j++) { triple[0] = dis.readLong(); triple[1] = dis.readLong(); triple[2] = dis.readLong(); if (DEBUG) System.err.println("From disk: " + Arrays.toString(triple)); if (filter == null || filter.evaluate(triple)) { buffer0[count] = triple[0]; buffer1[count] = triple[1]; buffer2[count] = triple[2]; if (hashMask == 0) data[count] = dis.readLong(); count++; } else if (hashMask == 0) dis.readLong(); // Discard data } chunkSize = count; dis.close(); } catch (IOException e) { throw new RuntimeException(e); } it.unimi.dsi.fastutil.Arrays.quickSort(0, chunkSize, new AbstractIntComparator() { private static final long serialVersionUID = 0L; public int compare(final int x, final int y) { int t = Long.signum(buffer0[x] - buffer0[y]); if (t != 0) return t; t = Long.signum(buffer1[x] - buffer1[y]); if (t != 0) return t; return Long.signum(buffer2[x] - buffer2[y]); } }, new Swapper() { public void swap(final int x, final int y) { final long e0 = buffer0[x], e1 = buffer1[x], e2 = buffer2[x]; buffer0[x] = buffer0[y]; buffer1[x] = buffer1[y]; buffer2[x] = buffer2[y]; buffer0[y] = e0; buffer1[y] = e1; buffer2[y] = e2; if (hashMask == 0) { final long v = data[x]; data[x] = data[y]; data[y] = v; } } }); if (DEBUG) { for (int i = 0; i < chunkSize; i++) System.err.println(buffer0[i] + ", " + buffer1[i] + ", " + buffer2[i]); } if (!checkedForDuplicates && chunkSize > 1) for (int i = chunkSize - 1; i-- != 0;) if (buffer0[i] == buffer0[i + 1] && buffer1[i] == buffer1[i + 1] && buffer2[i] == buffer2[i + 1]) throw new ChunkedHashStore.DuplicateException(); if (chunk == chunks - 1) checkedForDuplicates = true; last = 0; } final int start = last; while (last < chunkSize && (chunkShift == Long.SIZE ? 0 : buffer0[last] >>> chunkShift) == chunk) last++; chunk++; return new Chunk(buffer0, buffer1, buffer2, data, hashMask, start, last); } }; }
From source file:it.unimi.dsi.sux4j.mph.GOV3Function.java
/** Returns the number of bits used by this structure. * // ww w . ja v a 2 s . co m * @return the number of bits used by this structure. */ public long numBits() { if (n == 0) return 0; return (marker != null ? rank.numBits() + marker.length() : 0) + (data != null ? data.size64() : 0) * width + offsetAndSeed.length * (long) Long.SIZE; }
From source file:eu.europa.esig.dss.DSSUtils.java
public static long toLong(final byte[] bytes) { // Long.valueOf(new String(bytes)).longValue(); ByteBuffer buffer = ByteBuffer.allocate(8); buffer.put(bytes, 0, Long.SIZE / 8); // TODO: (Bob: 2014 Jan 22) To be checked if it is not platform dependent? buffer.flip();//need flip return buffer.getLong(); }
From source file:org.eclipse.dataset.AbstractDataset.java
/** * @param dtype/*from w ww. j a v a 2 s. c o m*/ * @param isize * number of elements in an item * @return length of single item in bytes */ public static int getItemsize(final int dtype, final int isize) { int size; switch (dtype) { case BOOL: size = 1; // How is this defined? break; case INT8: case ARRAYINT8: size = Byte.SIZE / 8; break; case INT16: case ARRAYINT16: case RGB: size = Short.SIZE / 8; break; case INT32: case ARRAYINT32: size = Integer.SIZE / 8; break; case INT64: case ARRAYINT64: size = Long.SIZE / 8; break; case FLOAT32: case ARRAYFLOAT32: case COMPLEX64: size = Float.SIZE / 8; break; case FLOAT64: case ARRAYFLOAT64: case COMPLEX128: size = Double.SIZE / 8; break; default: size = 0; break; } return size * isize; }
From source file:com.alibaba.jstorm.utils.JStormUtils.java
public static byte[] longToBytes(long x) { ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); buffer.putLong(x);//www .j ava2 s . c o m return buffer.array(); }