Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * Modified in IPONWEB, 2014 * */ package net.iponweb.hadoop.streaming.avro; import org.apache.avro.AvroTypeException; import org.apache.avro.Schema; import org.apache.avro.io.ParsingDecoder; import org.apache.avro.io.parsing.JsonGrammarGenerator; import org.apache.avro.io.parsing.Parser; import org.apache.avro.io.parsing.Symbol; import org.apache.avro.util.Utf8; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.codehaus.jackson.*; import java.io.EOFException; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.*; /* This class is based on original JsonDecoder from Avro with attempt to * handle complex schemas. With somewhat limited success it can. Known limitation * is schema with union of two or more named records or other named types. * Simpler unions seem to work well based on experience of running this on prod servers * And we can convert "NaN" back to NaN if it is placed in double or float fields */ public class IOWJsonDecoder extends ParsingDecoder implements Parser.ActionHandler { private JsonParser in; private static JsonFactory jsonFactory = new JsonFactory(); static final String CHARSET = "ISO-8859-1"; ReorderBuffer currentReorderBuffer; Stack<ReorderBuffer> reorderBuffers = new Stack<ReorderBuffer>(); private static final Log LOG = LogFactory.getLog(IOWJsonDecoder.class); private static class ReorderBuffer { public Map<String, List<JsonElement>> savedFields = new HashMap<String, List<JsonElement>>(); public JsonParser origParser = null; } public IOWJsonDecoder(Symbol s, String str) throws IOException { super(s); in = jsonFactory.createJsonParser(str); in.nextToken(); } public IOWJsonDecoder(Schema s, String str) throws IOException { this(getSymbol(s), str); } private static Symbol getSymbol(Schema schema) { if (null == schema) { throw new NullPointerException("Schema cannot be null!"); } return new JsonGrammarGenerator().generate(schema); } @Override public ByteBuffer readBytes(ByteBuffer old) throws IOException { advance(Symbol.BYTES); if (in.getCurrentToken() == JsonToken.VALUE_STRING) { byte[] result = readByteArray(); in.nextToken(); return ByteBuffer.wrap(result); } else { throw error("bytes"); } } private void advance(Symbol symbol) throws IOException { this.parser.processTrailingImplicitActions(); if (in.getCurrentToken() == null && this.parser.depth() == 1) throw new EOFException(); parser.advance(symbol); } @Override public void readNull() throws IOException { advance(Symbol.NULL); if (in.getCurrentToken() == JsonToken.VALUE_NULL) { in.nextToken(); } else { throw error("null"); } } @Override public boolean readBoolean() throws IOException { advance(Symbol.BOOLEAN); JsonToken t = in.getCurrentToken(); if (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE) { in.nextToken(); return t == JsonToken.VALUE_TRUE; } else { String s = in.getText(); if (s.equals("false") || s.equals("FALSE") || s.equals("0")) { in.nextToken(); return false; } else if (s.equals("true") || s.equals("TRUE") || s.equals("1")) { in.nextToken(); return true; } throw error("boolean"); } } @Override public int readInt() throws IOException { advance(Symbol.INT); if (in.getCurrentToken().isNumeric()) { int result = in.getIntValue(); in.nextToken(); return result; } else { try { String s = in.getText(); in.nextToken(); return Integer.parseInt(s); } catch (Exception e) { throw error("int (" + e.getMessage() + ")"); } } } @Override public long readLong() throws IOException { advance(Symbol.LONG); if (in.getCurrentToken().isNumeric()) { long result = in.getLongValue(); in.nextToken(); return result; } else { try { String s = in.getText(); in.nextToken(); return Long.parseLong(s); } catch (Exception e) { throw error("long (" + e.getMessage() + ")"); } } } @Override public float readFloat() throws IOException { advance(Symbol.FLOAT); if (in.getCurrentToken().isNumeric()) { float result = in.getFloatValue(); in.nextToken(); return result; } else { try { String s = in.getText(); in.nextToken(); if (s.equals("NaN")) { return Float.NaN; } else if (s.equals("-Inf")) { return Float.NEGATIVE_INFINITY; } else if (s.equals("+Inf")) { return Float.POSITIVE_INFINITY; } else { return Float.parseFloat(s); } } catch (Exception e) { throw error("float (" + e.getMessage() + ")"); } } } @Override public double readDouble() throws IOException { advance(Symbol.DOUBLE); if (in.getCurrentToken().isNumeric()) { double result = in.getDoubleValue(); in.nextToken(); return result; } else { try { String s = in.getText(); in.nextToken(); if (s.equals("NaN")) { return Double.NaN; } else if (s.equals("-Inf")) { return Double.NEGATIVE_INFINITY; } else if (s.equals("+Inf")) { return Double.POSITIVE_INFINITY; } else { return Double.parseDouble(s); } } catch (Exception e) { throw error("double (" + e.getMessage() + ")"); } } } @Override public Utf8 readString(Utf8 old) throws IOException { return new Utf8(readString()); } @Override public String readString() throws IOException { advance(Symbol.STRING); if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) { parser.advance(Symbol.MAP_KEY_MARKER); if (in.getCurrentToken() != JsonToken.FIELD_NAME) { throw error("map-key"); } } else { if (in.getCurrentToken() != JsonToken.VALUE_STRING) { throw error("string"); } } String result = in.getText(); in.nextToken(); return result; } @Override public void skipString() throws IOException { advance(Symbol.STRING); if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) { parser.advance(Symbol.MAP_KEY_MARKER); if (in.getCurrentToken() != JsonToken.FIELD_NAME) { throw error("map-key"); } } else { if (in.getCurrentToken() != JsonToken.VALUE_STRING) { throw error("string"); } } in.nextToken(); } private byte[] readByteArray() throws IOException { byte[] result = in.getText().getBytes(CHARSET); return result; } @Override public void skipBytes() throws IOException { advance(Symbol.BYTES); if (in.getCurrentToken() == JsonToken.VALUE_STRING) { in.nextToken(); } else { throw error("bytes"); } } private void checkFixed(int size) throws IOException { advance(Symbol.FIXED); Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol(); if (size != top.size) { throw new AvroTypeException("Incorrect length for fixed binary: expected " + top.size + " but received " + size + " bytes."); } } @Override public void readFixed(byte[] bytes, int start, int len) throws IOException { checkFixed(len); if (in.getCurrentToken() == JsonToken.VALUE_STRING) { byte[] result = readByteArray(); in.nextToken(); if (result.length != len) { throw new AvroTypeException("Expected fixed length " + len + ", but got" + result.length); } System.arraycopy(result, 0, bytes, start, len); } else { throw error("fixed"); } } @Override public void skipFixed(int length) throws IOException { checkFixed(length); doSkipFixed(length); } private void doSkipFixed(int length) throws IOException { if (in.getCurrentToken() == JsonToken.VALUE_STRING) { byte[] result = readByteArray(); in.nextToken(); if (result.length != length) { throw new AvroTypeException("Expected fixed length " + length + ", but got" + result.length); } } else { throw error("fixed"); } } @Override protected void skipFixed() throws IOException { advance(Symbol.FIXED); Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol(); doSkipFixed(top.size); } @Override public int readEnum() throws IOException { advance(Symbol.ENUM); Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol(); if (in.getCurrentToken() == JsonToken.VALUE_STRING) { in.getText(); int n = top.findLabel(in.getText()); if (n >= 0) { in.nextToken(); return n; } throw new AvroTypeException("Unknown symbol in enum " + in.getText()); } else { throw error("fixed"); } } @Override public long readArrayStart() throws IOException { advance(Symbol.ARRAY_START); if (in.getCurrentToken() == JsonToken.START_ARRAY) { in.nextToken(); return doArrayNext(); } else { throw error("array-start"); } } @Override public long arrayNext() throws IOException { advance(Symbol.ITEM_END); return doArrayNext(); } private long doArrayNext() throws IOException { if (in.getCurrentToken() == JsonToken.END_ARRAY) { parser.advance(Symbol.ARRAY_END); in.nextToken(); return 0; } else { return 1; } } @Override public long skipArray() throws IOException { advance(Symbol.ARRAY_START); if (in.getCurrentToken() == JsonToken.START_ARRAY) { in.skipChildren(); in.nextToken(); advance(Symbol.ARRAY_END); } else { throw error("array-start"); } return 0; } @Override public long readMapStart() throws IOException { advance(Symbol.MAP_START); if (in.getCurrentToken() == JsonToken.START_OBJECT) { in.nextToken(); return doMapNext(); } else { throw error("map-start"); } } @Override public long mapNext() throws IOException { advance(Symbol.ITEM_END); return doMapNext(); } private long doMapNext() throws IOException { if (in.getCurrentToken() == JsonToken.END_OBJECT) { in.nextToken(); advance(Symbol.MAP_END); return 0; } else { return 1; } } @Override public long skipMap() throws IOException { advance(Symbol.MAP_START); if (in.getCurrentToken() == JsonToken.START_OBJECT) { in.skipChildren(); in.nextToken(); advance(Symbol.MAP_END); } else { throw error("map-start"); } return 0; } @Override public int readIndex() throws IOException { advance(Symbol.UNION); Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol(); int n; JsonToken token = in.getCurrentToken(); if (token == JsonToken.VALUE_NULL) { n = a.findLabel("null"); if (n < 0) throw error("null"); parser.pushSymbol(a.getSymbol(n)); return n; } if (token == JsonToken.START_OBJECT) { // Iterate over union branches. Assume that first SEQUENCE symbol is our guessed branch n = 0; for (Symbol s : a.symbols) { if (s.kind == Symbol.Kind.SEQUENCE) break; n++; } if (n == a.symbols.length) throw new AvroTypeException("Union has no branch of type 'record' or 'array'"); parser.pushSymbol(a.getSymbol(n)); return n; } else { // Token is not an object // Check if we have enum somewhere in our symbols. If so, check our value against enum values // If found, push that enum into parser n = 0; for (Symbol s : a.symbols) { if (s.kind == Symbol.Kind.SEQUENCE) for (Symbol seq : s.production) { if (seq.kind == Symbol.Kind.EXPLICIT_ACTION && seq instanceof org.apache.avro.io.parsing.Symbol.EnumLabelsAction) { Symbol.EnumLabelsAction en = (Symbol.EnumLabelsAction) seq; if (en.findLabel(in.getText()) >= 0) { parser.pushSymbol(a.getSymbol(n)); return n; } } } n++; } // Trying to guess correct branch using token type GUESSING: switch (token) { case START_ARRAY: n = a.findLabel("array"); if (n < 0) throw error("array"); parser.pushSymbol(a.getSymbol(n)); break; case VALUE_NUMBER_FLOAT: final String floats[] = { "float", "double" }; for (String b : floats) { n = a.findLabel(b); if (n >= 0) { parser.pushSymbol(a.getSymbol(n)); break GUESSING; } } throw error("float or double"); case VALUE_NUMBER_INT: final String integers[] = { "int", "long", "float", "double" }; for (String b : integers) { n = a.findLabel(b); if (n >= 0) { parser.pushSymbol(a.getSymbol(n)); break GUESSING; } } throw error("int, long, float or double"); case VALUE_FALSE: case VALUE_TRUE: n = a.findLabel("boolean"); if (n < 0) throw error("boolean"); parser.pushSymbol(a.getSymbol(n)); break; case VALUE_STRING: final String nanCandidates[] = { "float", "double" }; n = a.findLabel("string"); if (n < 0 && (in.getText().equals("NaN") || in.getText().equals("Inf") || in.getText().equals("-Inf") || in.getText().equals("+Inf"))) { // Try to substitute NaN for (String nn : nanCandidates) { if ((n = a.findLabel(nn)) >= 0) { parser.pushSymbol(a.getSymbol(n)); break GUESSING; } } throw error("string (is NaN/Inf, but no string, double or float branches found)"); } else if (n < 0) { // Try to make double of this string String s = in.getText(); try { Double.parseDouble(s); } catch (NumberFormatException e) { throw error("string (not looks like number) and no string branch found"); } // String could be converted to double; Try integer as well try { Integer.parseInt(s); if ((n = a.findLabel("int")) >= 0) { parser.pushSymbol(a.getSymbol(n)); break; } } catch (NumberFormatException e) { } // Now try to find float or double (same as nanCandidates just by coincident for (String nn : nanCandidates) { if ((n = a.findLabel(nn)) >= 0) { parser.pushSymbol(a.getSymbol(n)); break GUESSING; } } throw error("string and also looks like number but no string nor numeric branches found"); } else parser.pushSymbol(a.getSymbol(n)); break; default: throw error("start-union"); } return n; } } @Override public Symbol doAction(Symbol input, Symbol top) throws IOException { if (top instanceof Symbol.FieldAdjustAction) { Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top; String name = fa.fname; if (currentReorderBuffer != null) { List<JsonElement> node = currentReorderBuffer.savedFields.get(name); if (node != null) { currentReorderBuffer.savedFields.remove(name); currentReorderBuffer.origParser = in; in = makeParser(node); return null; } } if (in.getCurrentToken() == JsonToken.FIELD_NAME) { do { String fn = in.getText(); in.nextToken(); if (name.equals(fn)) { return null; } else { if (currentReorderBuffer == null) { currentReorderBuffer = new ReorderBuffer(); } currentReorderBuffer.savedFields.put(fn, getVaueAsTree(in)); } } while (in.getCurrentToken() == JsonToken.FIELD_NAME); throw new AvroTypeException("Expected field name not found: " + fa.fname); } } else if (top == Symbol.FIELD_END) { if (currentReorderBuffer != null && currentReorderBuffer.origParser != null) { in = currentReorderBuffer.origParser; currentReorderBuffer.origParser = null; } } else if (top == Symbol.RECORD_START) { if (in.getCurrentToken() == JsonToken.START_OBJECT) { in.nextToken(); reorderBuffers.push(currentReorderBuffer); currentReorderBuffer = null; } else { throw error("record-start"); } } else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) { if (in.getCurrentToken() == JsonToken.END_OBJECT) { in.nextToken(); if (top == Symbol.RECORD_END) { if (currentReorderBuffer != null && !currentReorderBuffer.savedFields.isEmpty()) { throw error("Unknown fields: " + currentReorderBuffer.savedFields.keySet()); } currentReorderBuffer = reorderBuffers.pop(); } } else { throw error(top == Symbol.RECORD_END ? "record-end" : "union-end"); } } else { throw new AvroTypeException("Unknown action symbol " + top); } return null; } private static class JsonElement { public final JsonToken token; public final String value; public JsonElement(JsonToken t, String value) { this.token = t; this.value = value; } public JsonElement(JsonToken t) { this(t, null); } } private static List<JsonElement> getVaueAsTree(JsonParser in) throws IOException { int level = 0; List<JsonElement> result = new ArrayList<JsonElement>(); do { JsonToken t = in.getCurrentToken(); switch (t) { case START_OBJECT: case START_ARRAY: level++; result.add(new JsonElement(t)); break; case END_OBJECT: case END_ARRAY: level--; result.add(new JsonElement(t)); break; case FIELD_NAME: case VALUE_STRING: case VALUE_NUMBER_INT: case VALUE_NUMBER_FLOAT: case VALUE_TRUE: case VALUE_FALSE: case VALUE_NULL: result.add(new JsonElement(t, in.getText())); break; } in.nextToken(); } while (level != 0); result.add(new JsonElement(null)); return result; } private JsonParser makeParser(final List<JsonElement> elements) throws IOException { return new JsonParser() { int pos = 0; @Override public ObjectCodec getCodec() { throw new UnsupportedOperationException(); } @Override public void setCodec(ObjectCodec c) { throw new UnsupportedOperationException(); } @Override public void close() throws IOException { throw new UnsupportedOperationException(); } @Override public JsonToken nextToken() throws IOException { pos++; return elements.get(pos).token; } @Override public JsonParser skipChildren() throws IOException { int level = 0; do { switch (elements.get(pos++).token) { case START_ARRAY: case START_OBJECT: level++; break; case END_ARRAY: case END_OBJECT: level--; break; } } while (level > 0); return this; } @Override public boolean isClosed() { throw new UnsupportedOperationException(); } @Override public String getCurrentName() throws IOException { throw new UnsupportedOperationException(); } @Override public JsonStreamContext getParsingContext() { throw new UnsupportedOperationException(); } @Override public JsonLocation getTokenLocation() { throw new UnsupportedOperationException(); } @Override public JsonLocation getCurrentLocation() { throw new UnsupportedOperationException(); } @Override public String getText() throws IOException { return elements.get(pos).value; } @Override public char[] getTextCharacters() throws IOException { throw new UnsupportedOperationException(); } @Override public int getTextLength() throws IOException { throw new UnsupportedOperationException(); } @Override public int getTextOffset() throws IOException { throw new UnsupportedOperationException(); } @Override public Number getNumberValue() throws IOException { throw new UnsupportedOperationException(); } @Override public NumberType getNumberType() throws IOException { throw new UnsupportedOperationException(); } @Override public int getIntValue() throws IOException { return Integer.parseInt(getText()); } @Override public long getLongValue() throws IOException { return Long.parseLong(getText()); } @Override public BigInteger getBigIntegerValue() throws IOException { throw new UnsupportedOperationException(); } @Override public float getFloatValue() throws IOException { return Float.parseFloat(getText()); } @Override public double getDoubleValue() throws IOException { return Double.parseDouble(getText()); } @Override public BigDecimal getDecimalValue() throws IOException { throw new UnsupportedOperationException(); } @Override public byte[] getBinaryValue(Base64Variant b64variant) throws IOException { throw new UnsupportedOperationException(); } @Override public JsonToken getCurrentToken() { return elements.get(pos).token; } }; } private AvroTypeException error(String type) { String val; String fld; String loc; String cntx; String lastToken; try { val = in.getText(); } catch (Exception e) { val = "*UNKNOWN*"; } try { fld = in.getCurrentName(); } catch (Exception e) { fld = "*UNKNOWN*"; } try { loc = in.getCurrentLocation().toString(); } catch (Exception e) { loc = "*UNKNOWN*"; } try { lastToken = in.getLastClearedToken().toString(); } catch (Exception e) { lastToken = "*UNKNOWN*"; } try { cntx = in.getParsingContext().getCurrentName(); } catch (Exception e) { cntx = "*UNKNOWN*"; } return new AvroTypeException( "Expected " + type + ". Got " + in.getCurrentToken() + " (" + val + ") at field '" + fld + "'" + ". Location: '" + loc + "', last token = '" + lastToken + "', context: '" + cntx + "'"); } }