List of usage examples for java.nio ByteBuffer getInt
public abstract int getInt();
From source file:edu.umass.cs.gigapaxos.paxospackets.RequestPacket.java
public RequestPacket(ByteBuffer bbuf) throws UnsupportedEncodingException, UnknownHostException { super(bbuf);/*from ww w . ja va 2s . com*/ int exactLength = bbuf.position(); this.requestID = bbuf.getLong(); this.stop = bbuf.get() == (byte) 1; exactLength += (8 + 1); // addresses byte[] ca = new byte[4]; bbuf.get(ca); int cport = (int) bbuf.getShort(); cport = cport >= 0 ? cport : cport + 2 * (Short.MAX_VALUE + 1); this.clientAddress = cport != 0 ? new InetSocketAddress(InetAddress.getByAddress(ca), cport) : null; byte[] la = new byte[4]; bbuf.get(la); int lport = (int) bbuf.getShort(); lport = lport >= 0 ? lport : lport + 2 * (Short.MAX_VALUE + 1); this.listenAddress = lport != 0 ? new InetSocketAddress(InetAddress.getByAddress(la), lport) : null; exactLength += (4 + 2 + 4 + 2); // other non-final fields this.entryReplica = bbuf.getInt(); this.entryTime = bbuf.getLong(); this.shouldReturnRequestValue = bbuf.get() == (byte) 1; this.forwardCount = bbuf.getInt(); exactLength += (4 + 8 + 1 + 4); // digest related fields this.broadcasted = bbuf.get() == (byte) 1; int digestLength = bbuf.getInt(); if (digestLength > 0) bbuf.get(this.digest = new byte[digestLength]); // highly variable length fields // requestValue int reqValLen = bbuf.getInt(); byte[] reqValBytes = new byte[reqValLen]; bbuf.get(reqValBytes); this.requestValue = reqValBytes.length > 0 ? new String(reqValBytes, CHARSET) : null; exactLength += (4 + reqValBytes.length); // responseValue int respValLen = bbuf.getInt(); byte[] respValBytes = new byte[respValLen]; bbuf.get(respValBytes); this.responseValue = respValBytes.length > 0 ? new String(respValBytes, CHARSET) : null; exactLength += (4 + respValBytes.length); int numBatched = bbuf.getInt(); if (numBatched == 0) return; // else // batched requests this.batched = new RequestPacket[numBatched]; for (int i = 0; i < numBatched; i++) { int len = bbuf.getInt(); byte[] element = new byte[len]; bbuf.get(element); this.batched[i] = new RequestPacket(element); } assert (exactLength > 0); }
From source file:com.healthmarketscience.jackcess.Column.java
/** * @param lvalDefinition Column value that points to an LVAL record * @return The LVAL data//from w w w .j a v a 2 s . c o m */ private byte[] readLongValue(byte[] lvalDefinition) throws IOException { ByteBuffer def = ByteBuffer.wrap(lvalDefinition).order(PageChannel.DEFAULT_BYTE_ORDER); int lengthWithFlags = def.getInt(); int length = lengthWithFlags & (~LONG_VALUE_TYPE_MASK); byte[] rtn = new byte[length]; byte type = (byte) ((lengthWithFlags & LONG_VALUE_TYPE_MASK) >>> 24); if (type == LONG_VALUE_TYPE_THIS_PAGE) { // inline long value def.getInt(); //Skip over lval_dp def.getInt(); //Skip over unknown def.get(rtn); } else { // long value on other page(s) if (lvalDefinition.length != getFormat().SIZE_LONG_VALUE_DEF) { throw new IOException("Expected " + getFormat().SIZE_LONG_VALUE_DEF + " bytes in long value definition, but found " + lvalDefinition.length); } int rowNum = ByteUtil.getUnsignedByte(def); int pageNum = ByteUtil.get3ByteInt(def, def.position()); ByteBuffer lvalPage = getPageChannel().createPageBuffer(); switch (type) { case LONG_VALUE_TYPE_OTHER_PAGE: { getPageChannel().readPage(lvalPage, pageNum); short rowStart = Table.findRowStart(lvalPage, rowNum, getFormat()); short rowEnd = Table.findRowEnd(lvalPage, rowNum, getFormat()); if ((rowEnd - rowStart) != length) { throw new IOException("Unexpected lval row length"); } lvalPage.position(rowStart); lvalPage.get(rtn); } break; case LONG_VALUE_TYPE_OTHER_PAGES: ByteBuffer rtnBuf = ByteBuffer.wrap(rtn); int remainingLen = length; while (remainingLen > 0) { lvalPage.clear(); getPageChannel().readPage(lvalPage, pageNum); short rowStart = Table.findRowStart(lvalPage, rowNum, getFormat()); short rowEnd = Table.findRowEnd(lvalPage, rowNum, getFormat()); // read next page information lvalPage.position(rowStart); rowNum = ByteUtil.getUnsignedByte(lvalPage); pageNum = ByteUtil.get3ByteInt(lvalPage); // update rowEnd and remainingLen based on chunkLength int chunkLength = (rowEnd - rowStart) - 4; if (chunkLength > remainingLen) { rowEnd = (short) (rowEnd - (chunkLength - remainingLen)); chunkLength = remainingLen; } remainingLen -= chunkLength; lvalPage.limit(rowEnd); rtnBuf.put(lvalPage); } break; default: throw new IOException("Unrecognized long value type: " + type); } } return rtn; }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.dta.DTAFileReader.java
private void decodeExpansionFields(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeExpansionFields(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }//from w w w. java2s . c o m // Added since release 105 // [1-byte byte_field][short(2)/int(4)_field][variable_field whose // length is specified by the previous short/int field] int int_type_expansion_field = constantTable.get("EXPANSION"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("int_type_expansion_field=" + int_type_expansion_field); while (true) { byte[] firstByte = new byte[1]; byte[] lengthBytes = new byte[int_type_expansion_field]; int nbyte = stream.read(firstByte, 0, 1); dbgLog.fine("read 1st byte"); int nbytes = stream.read(lengthBytes, 0, int_type_expansion_field); dbgLog.fine("read next integer"); ByteBuffer bb_field_length = ByteBuffer.wrap(lengthBytes); if (isLittleEndian) { bb_field_length.order(ByteOrder.LITTLE_ENDIAN); dbgLog.fine("byte reversed"); } int field_length; if (int_type_expansion_field == 2) { field_length = bb_field_length.getShort(); } else { field_length = bb_field_length.getInt(); } if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("field_length=" + field_length); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("firstByte[0]=" + firstByte[0]); if ((field_length + (int) firstByte[0]) == 0) { // reached the end of this field break; } else { byte[] stringField = new byte[field_length]; nbyte = stream.read(stringField, 0, field_length); } } dbgLog.fine("***** decodeExpansionFields(): end *****"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.dta.DTAFileReader.java
private void parseValueLabelsReleasel108(BufferedInputStream stream) throws IOException { dbgLog.fine("***** parseValueLabelsRelease108(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }/* w ww .j a v a 2 s. c o m*/ int nvar = (Integer) smd.getFileInformation().get("varQnty"); int length_label_name = constantTable.get("NAME"); int length_value_label_header = value_label_table_length + length_label_name + VALUE_LABEL_HEADER_PADDING_LENGTH; if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("value_label_table_length=" + value_label_table_length); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("length_value_label_header=" + length_value_label_header); /* Seg field byte type 1-1. len_vlt(Seg.2) 4 int 1-2. vlt_name 9/33 char+(\0) == name used in Sec2.part 5 1-3. padding 3 byte ----------------------------------- 16/40 2-1. n(# of vls) 4 int 2-2. m(len_labels) 4 int 2-3. label_offsets 4*n int[] 2-4. values 4*n int[] 2-5. labels m char */ for (int i = 0; i < nvar; i++) { if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("\n\n" + i + "th value-label table header"); byte[] valueLabelHeader = new byte[length_value_label_header]; // Part 1: reading the header of a value-label table if exists int nbytes = stream.read(valueLabelHeader, 0, length_value_label_header); if (nbytes == 0) { throw new IOException("reading value label header: no datum"); } // 1.1 length_value_label_table ByteBuffer bb_value_label_header = ByteBuffer.wrap(valueLabelHeader, 0, value_label_table_length); if (isLittleEndian) { bb_value_label_header.order(ByteOrder.LITTLE_ENDIAN); } int length_value_label_table = bb_value_label_header.getInt(); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("length of this value-label table=" + length_value_label_table); // 1.2 labelName String rawLabelName = new String(Arrays.copyOfRange(valueLabelHeader, value_label_table_length, (value_label_table_length + length_label_name)), "ISO-8859-1"); String labelName = getNullStrippedString(rawLabelName); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("label name = " + labelName + "\n"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine(i + "-th value-label table"); // Part 2: reading the value-label table byte[] valueLabelTable_i = new byte[length_value_label_table]; int noBytes = stream.read(valueLabelTable_i, 0, length_value_label_table); if (noBytes == 0) { throw new IOException("reading value label table: no datum"); } // 2-1. 4-byte-integer: number of units in this table (n) int valueLabelTable_offset = 0; ByteBuffer bb_value_label_pairs = ByteBuffer.wrap(valueLabelTable_i, valueLabelTable_offset, value_label_table_length); if (isLittleEndian) { bb_value_label_pairs.order(ByteOrder.LITTLE_ENDIAN); } int no_value_label_pairs = bb_value_label_pairs.getInt(); valueLabelTable_offset += value_label_table_length; if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("no_value_label_pairs=" + no_value_label_pairs); // 2-2. 4-byte-integer: length of the label section (m bytes) ByteBuffer bb_length_label_segment = ByteBuffer.wrap(valueLabelTable_i, valueLabelTable_offset, value_label_table_length); if (isLittleEndian) { bb_length_label_segment.order(ByteOrder.LITTLE_ENDIAN); } int length_label_segment = bb_length_label_segment.getInt(); valueLabelTable_offset += value_label_table_length; // 2-3. 4-byte-integer array (4xm): offset values for the label sec. // these "label offsets" actually appear to represent the byte // offsets of the label strings, as stored in the next section. // as of now, these are not used for anything, and the code // below assumes that the labels are already in the same // order as the numeric values! -- L.A. int[] label_offsets = new int[no_value_label_pairs]; int byte_offset = valueLabelTable_offset; for (int j = 0; j < no_value_label_pairs; j++) { // note: 4-byte singed, not java's long ByteBuffer bb_label_offset = ByteBuffer.wrap(valueLabelTable_i, byte_offset, value_label_table_length); if (isLittleEndian) { bb_label_offset.order(ByteOrder.LITTLE_ENDIAN); dbgLog.fine("label offset: byte reversed"); } label_offsets[j] = bb_label_offset.getInt(); dbgLog.fine("label offset [" + j + "]: " + label_offsets[j]); byte_offset += value_label_table_length; } // 2-4. 4-byte-integer array (4xm): value array (sorted) dbgLog.fine("value array"); int[] valueList = new int[no_value_label_pairs]; int offset_value = byte_offset; for (int k = 0; k < no_value_label_pairs; k++) { ByteBuffer bb_value_list = ByteBuffer.wrap(valueLabelTable_i, offset_value, value_label_table_length); if (isLittleEndian) { bb_value_list.order(ByteOrder.LITTLE_ENDIAN); } valueList[k] = bb_value_list.getInt(); offset_value += value_label_table_length; } // 2-5. m-byte chars that store label data (m units of labels) String label_segment = new String( Arrays.copyOfRange(valueLabelTable_i, offset_value, (length_label_segment + offset_value)), "ISO-8859-1"); // L.A. -- 2011.2.25: // This assumes that the labels are already stored in the right // order: (see my comment for the section 2.3 above) //String[] labelList = label_segment.split("\0"); // Instead, we should be using the offset values obtained in // the section 2.3 above, and select the corresponding // substrings: String[] labelList = new String[no_value_label_pairs]; for (int l = 0; l < no_value_label_pairs; l++) { String lblString = null; int lblOffset = label_offsets[l]; lblString = label_segment.substring(lblOffset); int nullIndx = lblString.indexOf('\000'); if (nullIndx > -1) { lblString = lblString.substring(0, nullIndx); } labelList[l] = lblString; } // this should work! -- L.A. // (TODO: check the v105 value label parsing method, to see if // something similar applies there) Map<String, String> tmpValueLabelTable = new LinkedHashMap<String, String>(); for (int l = 0; l < no_value_label_pairs; l++) { if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine(l + "-th pair:" + valueList[l] + "[" + labelList[l] + "]"); tmpValueLabelTable.put(Integer.toString(valueList[l]), labelList[l]); } valueLabelTable.put(labelName, tmpValueLabelTable); if (stream.available() == 0) { // reached the end of this file // do exit-processing dbgLog.fine("***** reached the end of the file at " + i + "th value-label Table *****"); break; } } // for loop if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("valueLabelTable:\n" + valueLabelTable); smd.setValueLabelTable(valueLabelTable); dbgLog.fine("***** parseValueLabelsRelease108(): end *****"); }
From source file:srebrinb.compress.sevenzip.SevenZFile.java
private void readUnpackInfo(final ByteBuffer header, final Archive archive) throws IOException { int nid = getUnsignedByte(header); if (nid != NID.kFolder) { throw new IOException("Expected kFolder, got " + nid); }//from w w w.ja va2 s. c om final long numFolders = readUint64(header); final Folder[] folders = new Folder[(int) numFolders]; archive.folders = folders; final int external = getUnsignedByte(header); if (external != 0) { throw new IOException("External unsupported"); } for (int i = 0; i < (int) numFolders; i++) { folders[i] = readFolder(header); } nid = getUnsignedByte(header); if (nid != NID.kCodersUnpackSize) { throw new IOException("Expected kCodersUnpackSize, got " + nid); } for (final Folder folder : folders) { folder.unpackSizes = new long[(int) folder.totalOutputStreams]; for (int i = 0; i < folder.totalOutputStreams; i++) { folder.unpackSizes[i] = readUint64(header); } } nid = getUnsignedByte(header); if (nid == NID.kCRC) { final BitSet crcsDefined = readAllOrBits(header, (int) numFolders); for (int i = 0; i < (int) numFolders; i++) { if (crcsDefined.get(i)) { folders[i].hasCrc = true; folders[i].crc = 0xffffFFFFL & header.getInt(); } else { folders[i].hasCrc = false; } } nid = getUnsignedByte(header); } if (nid != NID.kEnd) { throw new IOException("Badly terminated UnpackInfo"); } }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReader.java
private void decodeExpansionFields(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeExpansionFields(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }/*from w w w .j a v a 2 s . co m*/ // Added since release 105 // [1-byte byte_field][short(2)/int(4)_field][variable_field whose // length is specified by the previous short/int field] int int_type_expansion_field = constantTable.get("EXPANSION"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("int_type_expansion_field=" + int_type_expansion_field); while (true) { byte[] firstByte = new byte[1]; byte[] lengthBytes = new byte[int_type_expansion_field]; int nbyte = stream.read(firstByte, 0, 1); dbgLog.fine("read 1st byte"); int nbytes = stream.read(lengthBytes, 0, int_type_expansion_field); dbgLog.fine("read next integer"); ByteBuffer bb_field_length = ByteBuffer.wrap(lengthBytes); if (isLittleEndian) { bb_field_length.order(ByteOrder.LITTLE_ENDIAN); dbgLog.fine("byte reversed"); } int field_length; if (int_type_expansion_field == 2) { field_length = bb_field_length.getShort(); } else { field_length = bb_field_length.getInt(); } if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("field_length=" + field_length); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("firstByte[0]=" + firstByte[0]); if ((field_length + firstByte[0]) == 0) { // reached the end of this field break; } else { byte[] stringField = new byte[field_length]; nbyte = stream.read(stringField, 0, field_length); } } dbgLog.fine("decodeExpansionFields(): end"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReader.java
private void parseValueLabelsReleasel108(BufferedInputStream stream) throws IOException { dbgLog.fine("parseValueLabelsRelease108(): start"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }/*from ww w .j a va 2s . c om*/ int nvar = dataTable.getVarQuantity().intValue(); int length_label_name = constantTable.get("NAME"); int length_value_label_header = value_label_table_length + length_label_name + VALUE_LABEL_HEADER_PADDING_LENGTH; if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("value_label_table_length=" + value_label_table_length); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("length_value_label_header=" + length_value_label_header); } /* Seg field byte type 1-1. len_vlt(Seg.2) 4 int 1-2. vlt_name 9/33 char+(\0) == name used in Sec2.part 5 1-3. padding 3 byte ----------------------------------- 16/40 2-1. n(# of vls) 4 int 2-2. m(len_labels) 4 int 2-3. label_offsets 4*n int[] 2-4. values 4*n int[] 2-5. labels m char */ // This map will hold a temporary lookup table for all the categorical // value-label groups: // These groups have unique names, and a group *may be shared* between // multiple variables. In the method decodeDescriptorValueLabel above // we have populated a lookup table where variables are linked to the // corresponding value-label groups by name. Thus we must fully populate // the full map of all the variable group, then go through the list // of variables and create the dataverse variable categories from // them. -- L.A. 4.0 Map<String, Map<String, String>> tempValueLabelTable = new LinkedHashMap<>(); for (int i = 0; i < nvar; i++) { if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("\n\n" + i + "th value-label table header"); } byte[] valueLabelHeader = new byte[length_value_label_header]; // Part 1: reading the header of a value-label table if exists int nbytes = stream.read(valueLabelHeader, 0, length_value_label_header); if (nbytes == 0) { throw new IOException("reading value label header: no datum"); } // 1.1 length_value_label_table ByteBuffer bb_value_label_header = ByteBuffer.wrap(valueLabelHeader, 0, value_label_table_length); if (isLittleEndian) { bb_value_label_header.order(ByteOrder.LITTLE_ENDIAN); } int length_value_label_table = bb_value_label_header.getInt(); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("length of this value-label table=" + length_value_label_table); } // 1.2 labelName String rawLabelName = new String(Arrays.copyOfRange(valueLabelHeader, value_label_table_length, (value_label_table_length + length_label_name)), "ISO-8859-1"); String labelName = getNullStrippedString(rawLabelName); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("label name = " + labelName + "\n"); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine(i + "-th value-label table"); } // Part 2: reading the value-label table byte[] valueLabelTable_i = new byte[length_value_label_table]; int noBytes = stream.read(valueLabelTable_i, 0, length_value_label_table); if (noBytes == 0) { throw new IOException("reading value label table: no datum"); } // 2-1. 4-byte-integer: number of units in this table (n) int valueLabelTable_offset = 0; ByteBuffer bb_value_label_pairs = ByteBuffer.wrap(valueLabelTable_i, valueLabelTable_offset, value_label_table_length); if (isLittleEndian) { bb_value_label_pairs.order(ByteOrder.LITTLE_ENDIAN); } int no_value_label_pairs = bb_value_label_pairs.getInt(); valueLabelTable_offset += value_label_table_length; if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("no_value_label_pairs=" + no_value_label_pairs); } // 2-2. 4-byte-integer: length of the label section (m bytes) ByteBuffer bb_length_label_segment = ByteBuffer.wrap(valueLabelTable_i, valueLabelTable_offset, value_label_table_length); if (isLittleEndian) { bb_length_label_segment.order(ByteOrder.LITTLE_ENDIAN); } int length_label_segment = bb_length_label_segment.getInt(); valueLabelTable_offset += value_label_table_length; // 2-3. 4-byte-integer array (4xm): offset values for the label sec. // these "label offsets" actually appear to represent the byte // offsets of the label strings, as stored in the next section. // as of now, these are not used for anything, and the code // below assumes that the labels are already in the same // order as the numeric values! -- L.A. int[] label_offsets = new int[no_value_label_pairs]; int byte_offset = valueLabelTable_offset; for (int j = 0; j < no_value_label_pairs; j++) { // note: 4-byte singed, not java's long ByteBuffer bb_label_offset = ByteBuffer.wrap(valueLabelTable_i, byte_offset, value_label_table_length); if (isLittleEndian) { bb_label_offset.order(ByteOrder.LITTLE_ENDIAN); dbgLog.fine("label offset: byte reversed"); } label_offsets[j] = bb_label_offset.getInt(); dbgLog.fine("label offset [" + j + "]: " + label_offsets[j]); byte_offset += value_label_table_length; } // 2-4. 4-byte-integer array (4xm): value array (sorted) dbgLog.fine("value array"); int[] valueList = new int[no_value_label_pairs]; int offset_value = byte_offset; for (int k = 0; k < no_value_label_pairs; k++) { ByteBuffer bb_value_list = ByteBuffer.wrap(valueLabelTable_i, offset_value, value_label_table_length); if (isLittleEndian) { bb_value_list.order(ByteOrder.LITTLE_ENDIAN); } valueList[k] = bb_value_list.getInt(); offset_value += value_label_table_length; } // 2-5. m-byte chars that store label data (m units of labels) String label_segment = new String( Arrays.copyOfRange(valueLabelTable_i, offset_value, (length_label_segment + offset_value)), "ISO-8859-1"); // L.A. -- 2011.2.25: // This assumes that the labels are already stored in the right // order: (see my comment for the section 2.3 above) //String[] labelList = label_segment.split("\0"); // Instead, we should be using the offset values obtained in // the section 2.3 above, and select the corresponding // substrings: String[] labelList = new String[no_value_label_pairs]; for (int l = 0; l < no_value_label_pairs; l++) { String lblString = null; int lblOffset = label_offsets[l]; lblString = label_segment.substring(lblOffset); int nullIndx = lblString.indexOf('\000'); if (nullIndx > -1) { lblString = lblString.substring(0, nullIndx); } labelList[l] = lblString; } // this should work! -- L.A. // (TODO: check the v105 value label parsing method, to see if // something similar applies there) // Finally, we've reached the actual value-label pairs. We'll go // through them and put them on the temporary lookup map: tempValueLabelTable.put(labelName, new LinkedHashMap<>()); for (int l = 0; l < no_value_label_pairs; l++) { if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine(l + "-th pair:" + valueList[l] + "[" + labelList[l] + "]"); } // TODO: do we need any null/empty string checks here? -- L.A. 4.0 tempValueLabelTable.get(labelName).put(Integer.toString(valueList[l]), labelList[l]); } if (stream.available() == 0) { // reached the end of the file dbgLog.fine("reached the end of the file at " + i + "th value-label Table"); break; } } // for nvar loop // And now we can go through the list of variables, see if any have // value-label groups linked, then build dataverse VariableCategory // objects for them, using the values stored in the temporary map // we've just built: // TODO: this code is duplicated between this, and the "105 version" of // this method, above. Maybe it should be isolated in its own method. // -- L.A. 4.0 for (int i = 0; i < nvar; i++) { if (valueLabelsLookupTable[i] != null) { if (tempValueLabelTable.get(valueLabelsLookupTable[i]) != null) { // What if it is null? -- is it a legit condition, that // a variable was advertised as having categorical values, // but no such cat value group exists under this name? // -- L.A. for (String value : tempValueLabelTable.get(valueLabelsLookupTable[i]).keySet()) { VariableCategory cat = new VariableCategory(); cat.setValue(value); cat.setLabel(tempValueLabelTable.get(valueLabelsLookupTable[i]).get(value)); /* cross-link the variable and category to each other: */ cat.setDataVariable(dataTable.getDataVariables().get(i)); dataTable.getDataVariables().get(i).getCategories().add(cat); } } } } dbgLog.fine("parseValueLabelsRelease108(): end"); }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.dta.DTAFileReader.java
private void decodeData(BufferedInputStream stream) throws IOException { dbgLog.fine("\n***** decodeData(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }//from w ww . j a v a2 s . c om int nvar = (Integer) smd.getFileInformation().get("varQnty"); int nobs = (Integer) smd.getFileInformation().get("caseQnty"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("data diminsion[rxc]=(" + nobs + "," + nvar + ")"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("bytes per row=" + bytes_per_row + " bytes"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("variableTypelList=" + Arrays.deepToString(variableTypelList)); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("StringVariableTable=" + StringVariableTable); FileOutputStream fileOutTab = null; PrintWriter pwout = null; // create a File object to save the tab-delimited data file File tabDelimitedDataFile = File.createTempFile("tempTabfile.", ".tab"); String tabDelimitedDataFileName = tabDelimitedDataFile.getAbsolutePath(); // save the temp file name in the metadata object smd.getFileInformation().put("tabDelimitedDataFileLocation", tabDelimitedDataFileName); fileOutTab = new FileOutputStream(tabDelimitedDataFile); pwout = new PrintWriter(new OutputStreamWriter(fileOutTab, "utf8"), true); // data storage // Object[][] dataTable = new Object[nobs][nvar]; // for later variable-wise calculations of statistics // dataTable2 sotres cut-out data columnwise Object[][] dataTable2 = new Object[nvar][nobs]; String[][] dateFormat = new String[nvar][nobs]; for (int i = 0; i < nobs; i++) { byte[] dataRowBytes = new byte[bytes_per_row]; Object[] dataRow = new Object[nvar]; int nbytes = stream.read(dataRowBytes, 0, bytes_per_row); if (nbytes == 0) { String errorMessage = "reading data: no data were read at(" + i + "th row)"; throw new IOException(errorMessage); } // decoding each row int byte_offset = 0; for (int columnCounter = 0; columnCounter < variableTypelList.length; columnCounter++) { Integer varType = variableTypeMap.get(variableTypelList[columnCounter]); String variableFormat = variableFormats[columnCounter]; boolean isDateTimeDatum = isDateTimeDatumList[columnCounter]; switch (varType != null ? varType : 256) { case -5: // Byte case // note: 1 byte signed byte byte_datum = dataRowBytes[byte_offset]; if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column byte =" + byte_datum); if (byte_datum >= BYTE_MISSING_VALUE) { if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column byte MV=" + byte_datum); dataRow[columnCounter] = MissingValueForTextDataFileNumeric; dataTable2[columnCounter][i] = null; //use null reference to indicate missing value in data that is passed to UNF } else { dataRow[columnCounter] = byte_datum; dataTable2[columnCounter][i] = byte_datum; } byte_offset++; break; case -4: // Stata-int (=java's short: 2byte) case // note: 2-byte signed int, not java's int ByteBuffer int_buffer = ByteBuffer.wrap(dataRowBytes, byte_offset, 2); if (isLittleEndian) { int_buffer.order(ByteOrder.LITTLE_ENDIAN); } short short_datum = int_buffer.getShort(); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column stata int =" + short_datum); if (short_datum >= INT_MISSIG_VALUE) { if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column stata long missing value=" + short_datum); dataTable2[columnCounter][i] = null; //use null reference to indicate missing value in data that is passed to UNF if (isDateTimeDatum) { dataRow[columnCounter] = MissingValueForTextDataFileString; } else { dataRow[columnCounter] = MissingValueForTextDataFileNumeric; } } else { if (isDateTimeDatum) { DecodedDateTime ddt = decodeDateTimeData("short", variableFormat, Short.toString(short_datum)); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row , decodedDateTime " + ddt.decodedDateTime + ", format=" + ddt.format); dataRow[columnCounter] = ddt.decodedDateTime; dateFormat[columnCounter][i] = ddt.format; dataTable2[columnCounter][i] = dataRow[columnCounter]; } else { dataTable2[columnCounter][i] = short_datum; dataRow[columnCounter] = short_datum; } } byte_offset += 2; break; case -3: // stata-Long (= java's int: 4 byte) case // note: 4-byte singed, not java's long dbgLog.fine("DATreader: stata long"); ByteBuffer long_buffer = ByteBuffer.wrap(dataRowBytes, byte_offset, 4); if (isLittleEndian) { long_buffer.order(ByteOrder.LITTLE_ENDIAN); } int int_datum = long_buffer.getInt(); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine(i + "-th row " + columnCounter + "=th column stata long =" + int_datum); if (int_datum >= LONG_MISSING_VALUE) { if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine(i + "-th row " + columnCounter + "=th column stata long missing value=" + int_datum); dataTable2[columnCounter][i] = null; //use null reference to indicate missing value in data that is passed to UNF if (isDateTimeDatum) { dataRow[columnCounter] = MissingValueForTextDataFileString; } else { dataRow[columnCounter] = MissingValueForTextDataFileNumeric; } } else { if (isDateTimeDatum) { DecodedDateTime ddt = decodeDateTimeData("int", variableFormat, Integer.toString(int_datum)); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row , decodedDateTime " + ddt.decodedDateTime + ", format=" + ddt.format); dataRow[columnCounter] = ddt.decodedDateTime; dateFormat[columnCounter][i] = ddt.format; dataTable2[columnCounter][i] = dataRow[columnCounter]; } else { dataTable2[columnCounter][i] = int_datum; dataRow[columnCounter] = int_datum; } } byte_offset += 4; break; case -2: // float case // note: 4-byte ByteBuffer float_buffer = ByteBuffer.wrap(dataRowBytes, byte_offset, 4); if (isLittleEndian) { float_buffer.order(ByteOrder.LITTLE_ENDIAN); } float float_datum = float_buffer.getFloat(); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column float =" + float_datum); if (FLOAT_MISSING_VALUE_SET.contains(float_datum)) { if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column float missing value=" + float_datum); dataTable2[columnCounter][i] = null; //use null reference to indicate missing value in data that is passed to UNF if (isDateTimeDatum) { dataRow[columnCounter] = MissingValueForTextDataFileString; } else { dataRow[columnCounter] = MissingValueForTextDataFileNumeric; } } else { if (isDateTimeDatum) { DecodedDateTime ddt = decodeDateTimeData("float", variableFormat, doubleNumberFormatter.format(float_datum)); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row , decodedDateTime " + ddt.decodedDateTime + ", format=" + ddt.format); dataRow[columnCounter] = ddt.decodedDateTime; dateFormat[columnCounter][i] = ddt.format; dataTable2[columnCounter][i] = dataRow[columnCounter]; } else { dataTable2[columnCounter][i] = float_datum; dataRow[columnCounter] = float_datum; } } byte_offset += 4; break; case -1: // double case // note: 8-byte ByteBuffer double_buffer = ByteBuffer.wrap(dataRowBytes, byte_offset, 8); if (isLittleEndian) { double_buffer.order(ByteOrder.LITTLE_ENDIAN); } double double_datum = double_buffer.getDouble(); if (DOUBLE_MISSING_VALUE_SET.contains(double_datum)) { dataTable2[columnCounter][i] = null; //use null reference to indicate missing value in data that is passed to UNF if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column double missing value=" + double_datum); if (isDateTimeDatum) { dataRow[columnCounter] = MissingValueForTextDataFileString; } else { dataRow[columnCounter] = MissingValueForTextDataFileNumeric; } } else { if (isDateTimeDatum) { DecodedDateTime ddt = decodeDateTimeData("double", variableFormat, doubleNumberFormatter.format(double_datum)); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row , decodedDateTime " + ddt.decodedDateTime + ", format=" + ddt.format); dataRow[columnCounter] = ddt.decodedDateTime; dateFormat[columnCounter][i] = ddt.format; dataTable2[columnCounter][i] = dataRow[columnCounter]; } else { dataTable2[columnCounter][i] = double_datum; dataRow[columnCounter] = doubleNumberFormatter.format(double_datum); } } byte_offset += 8; break; case 0: // String case int strVarLength = StringVariableTable.get(columnCounter); String raw_datum = new String( Arrays.copyOfRange(dataRowBytes, byte_offset, (byte_offset + strVarLength)), "ISO-8859-1"); String string_datum = getNullStrippedString(raw_datum); if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column string =" + string_datum); if (string_datum.equals("")) { if (dbgLog.isLoggable(Level.FINER)) dbgLog.finer(i + "-th row " + columnCounter + "=th column string missing value=" + string_datum); dataRow[columnCounter] = MissingValueForTextDataFileString; dataTable2[columnCounter][i] = null; //use null reference to indicate missing value in data that is passed to UNF } else { String escapedString = string_datum.replaceAll("\"", Matcher.quoteReplacement("\\\"")); /* * Fixing the bug we've had in the Stata reader for * a longest time: new lines and tabs need to * be escaped too - otherwise it breaks our * TAB file structure! -- L.A. */ escapedString = escapedString.replaceAll("\t", Matcher.quoteReplacement("\\t")); escapedString = escapedString.replaceAll("\n", Matcher.quoteReplacement("\\n")); escapedString = escapedString.replaceAll("\r", Matcher.quoteReplacement("\\r")); // the escaped version of the string will be // stored in the tab file: dataRow[columnCounter] = "\"" + escapedString + "\""; // but note that the "raw" version of it is // used for the UNF: dataTable2[columnCounter][i] = string_datum; } byte_offset += strVarLength; break; default: dbgLog.fine("unknown variable type found"); String errorMessage = "unknow variable Type found at data section"; throw new InvalidObjectException(errorMessage); } // switch } // for-columnCounter // dump the row of data to the external file pwout.println(StringUtils.join(dataRow, "\t")); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine(i + "-th row's data={" + StringUtils.join(dataRow, ",") + "};"); } // for- i (row) pwout.close(); if (dbgLog.isLoggable(Level.FINER)) { dbgLog.finer("\ndataTable2(variable-wise):\n"); dbgLog.finer(Arrays.deepToString(dataTable2)); dbgLog.finer("\ndateFormat(variable-wise):\n"); dbgLog.finer(Arrays.deepToString(dateFormat)); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("variableTypelList:\n" + Arrays.deepToString(variableTypelList)); dbgLog.fine("variableTypelListFinal:\n" + Arrays.deepToString(variableTypelListFinal)); } String[] unfValues = new String[nvar]; for (int j = 0; j < nvar; j++) { String variableType_j = variableTypelListFinal[j]; unfValues[j] = getUNF(dataTable2[j], dateFormat[j], variableType_j, unfVersionNumber, j); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine(j + "th unf value" + unfValues[j]); } if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("unf set:\n" + Arrays.deepToString(unfValues)); fileUnfValue = UNF5Util.calculateUNF(unfValues); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("file-unf=" + fileUnfValue); stataDataSection.setUnf(unfValues); stataDataSection.setFileUnf(fileUnfValue); smd.setVariableUNF(unfValues); smd.getFileInformation().put("fileUNF", fileUnfValue); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("unf values:\n" + unfValues); stataDataSection.setData(dataTable2); // close the stream dbgLog.fine("***** decodeData(): end *****\n\n"); }
From source file:com.koda.integ.hbase.storage.FileExtStorage.java
@Override public StorageHandle storeData(ByteBuffer buf) { writeLock.writeLock().lock();/*w ww . jav a 2s. c o m*/ int pos = 0; try { if (activeBuffer.get() == null) { // unlock writeLock.writeLock().unlock(); // Get next buffer from empty queue - blocking call ByteBuffer bbuf = emptyBuffersQueue.take(); // lock again writeLock.writeLock().lock(); if (activeBuffer.get() == null) { activeBuffer.set(bbuf); bufferOffset.set(0); } else { // somebody already set the activeBuffer // repeat call recursively emptyBuffersQueue.offer(bbuf); writeLock.writeLock().unlock(); return storeData(buf); } } pos = buf.position(); long currentFileLength = currentFileOffsetForWrites.get(); if (bufferOffset.get() == 0 && currentFileLength + bufferSize > fileSizeLimit) { // previous buffer was flushed currentFileOffsetForWrites.set(0); maxIdForWrites.incrementAndGet(); } int size = buf.getInt(); long off = bufferOffset.getAndAdd(size + 4); if (off + size + 4 > bufferSize) { // send current buffer to write queue ByteBuffer buff = activeBuffer.get(); //verifyBuffer(buff); writeQueue.offer(buff); activeBuffer.set(null); if (currentFileLength + bufferSize > fileSizeLimit) { currentFileOffsetForWrites.set(0); maxIdForWrites.incrementAndGet(); } // release lock writeLock.writeLock().unlock(); // Get next buffer from empty queue ByteBuffer bbuf = emptyBuffersQueue.take(); // lock again writeLock.writeLock().lock(); if (activeBuffer.get() == null) { activeBuffer.set(bbuf); } else { // some other thread set already the activeBuffer // repeat call recursively emptyBuffersQueue.offer(bbuf); writeLock.writeLock().unlock(); buf.position(pos); return storeData(buf); } bufferOffset.set(size + 4); // Check if need advance file } // We need to keep overall object (key+block) size in a file buf.position(pos); buf.limit(pos + size + 4); activeBuffer.get().put(buf); FileStorageHandle fsh = new FileStorageHandle(maxIdForWrites.get(), (int) (currentFileOffsetForWrites.get()), size); // Increase offset in current file for writes; currentFileOffsetForWrites.addAndGet(size + 4); return fsh; } catch (InterruptedException e) { e.printStackTrace(); writeLock.writeLock().unlock(); buf.position(pos); return storeData(buf); } finally { WriteLock lock = writeLock.writeLock(); if (lock.isHeldByCurrentThread()) { lock.unlock(); } } }
From source file:org.apache.bookkeeper.bookie.BookieShell.java
/** * Format the message into a readable format. * * @param pos//from www.j av a2 s .co m * File offset of the message stored in entry log file * @param recBuff * Entry Data * @param printMsg * Whether printing the message body */ private void formatEntry(long pos, ByteBuffer recBuff, boolean printMsg) { long ledgerId = recBuff.getLong(); long entryId = recBuff.getLong(); int entrySize = recBuff.limit(); System.out.println("--------- Lid=" + ledgerId + ", Eid=" + entryId + ", ByteOffset=" + pos + ", EntrySize=" + entrySize + " ---------"); if (entryId == Bookie.METAENTRY_ID_LEDGER_KEY) { int masterKeyLen = recBuff.getInt(); byte[] masterKey = new byte[masterKeyLen]; recBuff.get(masterKey); System.out.println("Type: META"); System.out.println("MasterKey: " + bytes2Hex(masterKey)); System.out.println(); return; } if (entryId == Bookie.METAENTRY_ID_FENCE_KEY) { System.out.println("Type: META"); System.out.println("Fenced"); System.out.println(); return; } // process a data entry long lastAddConfirmed = recBuff.getLong(); System.out.println("Type: DATA"); System.out.println("LastConfirmed: " + lastAddConfirmed); if (!printMsg) { System.out.println(); return; } // skip digest checking recBuff.position(32 + 8); System.out.println("Data:"); System.out.println(); try { byte[] ret = new byte[recBuff.remaining()]; recBuff.get(ret); formatter.formatEntry(ret); } catch (Exception e) { System.out.println("N/A. Corrupted."); } System.out.println(); }