org.apache.solr.common.util.JavaBinCodec.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.common.util.JavaBinCodec.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.common.util;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.Predicate;

import org.apache.solr.common.ConditionalKeyMapWriter;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.IteratorWriter.ItemWriter;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.CommonParams;
import org.noggit.CharArr;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.solr.common.util.ByteArrayUtf8CharSequence.convertCharSeq;

/**
 * Defines a space-efficient serialization/deserialization format for transferring data.
 * <p>
 * JavaBinCodec has built in support many commonly used types.  This includes primitive types (boolean, byte,
 * short, double, int, long, float), common Java containers/utilities (Date, Map, Collection, Iterator, String,
 * Object[], byte[]), and frequently used Solr types ({@link NamedList}, {@link SolrDocument},
 * {@link SolrDocumentList}). Each of the above types has a pair of associated methods which read and write
 * that type to a stream.
 * <p>
 * Classes that aren't supported natively can still be serialized/deserialized by providing
 * an {@link JavaBinCodec.ObjectResolver} object that knows how to work with the unsupported class.
 * This allows {@link JavaBinCodec} to be used to marshall/unmarshall arbitrary content.
 * <p>
 * NOTE -- {@link JavaBinCodec} instances cannot be reused for more than one marshall or unmarshall operation.
 */
public class JavaBinCodec implements PushWriter {

    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
    private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();

    public static final byte NULL = 0, BOOL_TRUE = 1, BOOL_FALSE = 2, BYTE = 3, SHORT = 4, DOUBLE = 5, INT = 6,
            LONG = 7, FLOAT = 8, DATE = 9, MAP = 10, SOLRDOC = 11, SOLRDOCLST = 12, BYTEARR = 13, ITERATOR = 14,
            /**
             * this is a special tag signals an end. No value is associated with it
             */
            END = 15,

            SOLRINPUTDOC = 16, MAP_ENTRY_ITER = 17, ENUM_FIELD_VALUE = 18, MAP_ENTRY = 19, UUID = 20, // This is reserved to be used only in LogCodec
            // types that combine tag + length (or other info) in a single byte
            TAG_AND_LEN = (byte) (1 << 5), STR = (byte) (1 << 5), SINT = (byte) (2 << 5), SLONG = (byte) (3 << 5),
            ARR = (byte) (4 << 5), //
            ORDERED_MAP = (byte) (5 << 5), // SimpleOrderedMap (a NamedList subclass, and more common)
            NAMED_LST = (byte) (6 << 5), // NamedList
            EXTERN_STRING = (byte) (7 << 5);

    private static final int MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY = 65536;

    private static byte VERSION = 2;
    private final ObjectResolver resolver;
    protected FastOutputStream daos;
    private StringCache stringCache;
    private WritableDocFields writableDocFields;
    private boolean alreadyMarshalled;
    private boolean alreadyUnmarshalled;
    protected boolean readStringAsCharSeq = false;

    public JavaBinCodec() {
        resolver = null;
        writableDocFields = null;
    }

    public JavaBinCodec setReadStringAsCharSeq(boolean flag) {
        readStringAsCharSeq = flag;
        return this;
    }

    /**
     * Use this to use this as a PushWriter. ensure that close() is called explicitly after use
     *
     * @param os The output stream
     */
    public JavaBinCodec(OutputStream os, ObjectResolver resolver) throws IOException {
        this.resolver = resolver;
        initWrite(os);
    }

    public JavaBinCodec(ObjectResolver resolver) {
        this(resolver, null);
    }

    public JavaBinCodec setWritableDocFields(WritableDocFields writableDocFields) {
        this.writableDocFields = writableDocFields;
        return this;

    }

    public JavaBinCodec(ObjectResolver resolver, StringCache stringCache) {
        this.resolver = resolver;
        this.stringCache = stringCache;
    }

    public ObjectResolver getResolver() {
        return resolver;
    }

    public void marshal(Object nl, OutputStream os) throws IOException {
        try {
            initWrite(os);
            writeVal(nl);
        } finally {
            alreadyMarshalled = true;
            daos.flushBuffer();
        }
    }

    protected void initWrite(OutputStream os) throws IOException {
        assert !alreadyMarshalled;
        init(FastOutputStream.wrap(os));
        daos.writeByte(VERSION);
    }

    /** expert: sets a new output stream */
    public void init(FastOutputStream os) {
        daos = os;
    }

    byte version;

    public Object unmarshal(byte[] buf) throws IOException {
        FastInputStream dis = initRead(buf);
        return readVal(dis);
    }

    public Object unmarshal(InputStream is) throws IOException {
        FastInputStream dis = initRead(is);
        return readVal(dis);
    }

    protected FastInputStream initRead(InputStream is) throws IOException {
        assert !alreadyUnmarshalled;
        FastInputStream dis = FastInputStream.wrap(is);
        return _init(dis);
    }

    protected FastInputStream initRead(byte[] buf) throws IOException {
        assert !alreadyUnmarshalled;
        FastInputStream dis = new FastInputStream(null, buf, 0, buf.length);
        return _init(dis);
    }

    protected FastInputStream _init(FastInputStream dis) throws IOException {
        version = dis.readByte();
        if (version != VERSION) {
            throw new RuntimeException("Invalid version (expected " + VERSION + ", but " + version
                    + ") or the data in not in 'javabin' format");
        }

        alreadyUnmarshalled = true;
        return dis;
    }

    public SimpleOrderedMap<Object> readOrderedMap(DataInputInputStream dis) throws IOException {
        int sz = readSize(dis);
        SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>(sz);
        for (int i = 0; i < sz; i++) {
            String name = (String) readVal(dis);
            Object val = readVal(dis);
            nl.add(name, val);
        }
        return nl;
    }

    public NamedList<Object> readNamedList(DataInputInputStream dis) throws IOException {
        int sz = readSize(dis);
        NamedList<Object> nl = new NamedList<>(sz);
        for (int i = 0; i < sz; i++) {
            String name = (String) readVal(dis);
            Object val = readVal(dis);
            nl.add(name, val);
        }
        return nl;
    }

    public void writeNamedList(NamedList<?> nl) throws IOException {
        writeTag(nl instanceof SimpleOrderedMap ? ORDERED_MAP : NAMED_LST, nl.size());
        for (int i = 0; i < nl.size(); i++) {
            String name = nl.getName(i);
            writeExternString(name);
            Object val = nl.getVal(i);
            writeVal(val);
        }
    }

    public void writeVal(Object val) throws IOException {
        if (writeKnownType(val)) {
            return;
        } else {
            ObjectResolver resolver = null;
            if (val instanceof ObjectResolver) {
                resolver = (ObjectResolver) val;
            } else {
                resolver = this.resolver;
            }
            if (resolver != null) {
                Object tmpVal = resolver.resolve(val, this);
                if (tmpVal == null)
                    return; // null means the resolver took care of it fully
                if (writeKnownType(tmpVal))
                    return;
            }
        }
        // Fallback to do *something*.
        // note: if the user of this codec doesn't want this (e.g. UpdateLog) it can supply an ObjectResolver that does
        //  something else like throw an exception.
        writeVal(val.getClass().getName() + ':' + val.toString());
    }

    protected static final Object END_OBJ = new Object();

    protected byte tagByte;

    public Object readVal(DataInputInputStream dis) throws IOException {
        tagByte = dis.readByte();
        return readObject(dis);
    }

    protected Object readObject(DataInputInputStream dis) throws IOException {
        // if ((tagByte & 0xe0) == 0) {
        // if top 3 bits are clear, this is a normal tag

        // OK, try type + size in single byte
        switch (tagByte >>> 5) {
        case STR >>> 5:
            return readStr(dis, stringCache, readStringAsCharSeq);
        case SINT >>> 5:
            return readSmallInt(dis);
        case SLONG >>> 5:
            return readSmallLong(dis);
        case ARR >>> 5:
            return readArray(dis);
        case ORDERED_MAP >>> 5:
            return readOrderedMap(dis);
        case NAMED_LST >>> 5:
            return readNamedList(dis);
        case EXTERN_STRING >>> 5:
            return readExternString(dis);
        }

        switch (tagByte) {
        case NULL:
            return null;
        case DATE:
            return new Date(dis.readLong());
        case INT:
            return dis.readInt();
        case BOOL_TRUE:
            return Boolean.TRUE;
        case BOOL_FALSE:
            return Boolean.FALSE;
        case FLOAT:
            return dis.readFloat();
        case DOUBLE:
            return dis.readDouble();
        case LONG:
            return dis.readLong();
        case BYTE:
            return dis.readByte();
        case SHORT:
            return dis.readShort();
        case MAP:
            return readMap(dis);
        case SOLRDOC:
            return readSolrDocument(dis);
        case SOLRDOCLST:
            return readSolrDocumentList(dis);
        case BYTEARR:
            return readByteArray(dis);
        case ITERATOR:
            return readIterator(dis);
        case END:
            return END_OBJ;
        case SOLRINPUTDOC:
            return readSolrInputDocument(dis);
        case ENUM_FIELD_VALUE:
            return readEnumFieldValue(dis);
        case MAP_ENTRY:
            return readMapEntry(dis);
        case MAP_ENTRY_ITER:
            return readMapIter(dis);
        }

        throw new RuntimeException("Unknown type " + tagByte);
    }

    public boolean writeKnownType(Object val) throws IOException {
        if (writePrimitive(val))
            return true;
        if (val instanceof NamedList) {
            writeNamedList((NamedList<?>) val);
            return true;
        }
        if (val instanceof SolrDocumentList) { // SolrDocumentList is a List, so must come before List check
            writeSolrDocumentList((SolrDocumentList) val);
            return true;
        }
        if (val instanceof SolrInputField) {
            return writeKnownType(((SolrInputField) val).getValue());
        }
        if (val instanceof IteratorWriter) {
            writeIterator((IteratorWriter) val);
            return true;
        }
        if (val instanceof Collection) {
            writeArray((Collection) val);
            return true;
        }
        if (val instanceof Object[]) {
            writeArray((Object[]) val);
            return true;
        }
        if (val instanceof SolrDocument) {
            //this needs special treatment to know which fields are to be written
            writeSolrDocument((SolrDocument) val);
            return true;
        }
        if (val instanceof SolrInputDocument) {
            writeSolrInputDocument((SolrInputDocument) val);
            return true;
        }
        if (val instanceof MapWriter) {
            writeMap((MapWriter) val);
            return true;
        }
        if (val instanceof Map) {
            writeMap((Map) val);
            return true;
        }
        if (val instanceof Iterator) {
            writeIterator((Iterator) val);
            return true;
        }
        if (val instanceof Path) {
            writeStr(((Path) val).toAbsolutePath().toString());
            return true;
        }
        if (val instanceof Iterable) {
            writeIterator(((Iterable) val).iterator());
            return true;
        }
        if (val instanceof EnumFieldValue) {
            writeEnumFieldValue((EnumFieldValue) val);
            return true;
        }
        if (val instanceof Map.Entry) {
            writeMapEntry((Map.Entry) val);
            return true;
        }
        if (val instanceof MapSerializable) {
            //todo find a better way to reuse the map more efficiently
            writeMap(((MapSerializable) val).toMap(new NamedList().asShallowMap()));
            return true;
        }
        if (val instanceof AtomicInteger) {
            writeInt(((AtomicInteger) val).get());
            return true;
        }
        if (val instanceof AtomicLong) {
            writeLong(((AtomicLong) val).get());
            return true;
        }
        if (val instanceof AtomicBoolean) {
            writeBoolean(((AtomicBoolean) val).get());
            return true;
        }
        return false;
    }

    public class BinEntryWriter implements MapWriter.EntryWriter {
        @Override
        public MapWriter.EntryWriter put(CharSequence k, Object v) throws IOException {
            writeExternString(k);
            JavaBinCodec.this.writeVal(v);
            return this;
        }

        @Override
        public MapWriter.EntryWriter put(CharSequence k, int v) throws IOException {
            writeExternString(k);
            JavaBinCodec.this.writeInt(v);
            return this;
        }

        @Override
        public MapWriter.EntryWriter put(CharSequence k, long v) throws IOException {
            writeExternString(k);
            JavaBinCodec.this.writeLong(v);
            return this;
        }

        @Override
        public MapWriter.EntryWriter put(CharSequence k, float v) throws IOException {
            writeExternString(k);
            JavaBinCodec.this.writeFloat(v);
            return this;
        }

        @Override
        public MapWriter.EntryWriter put(CharSequence k, double v) throws IOException {
            writeExternString(k);
            JavaBinCodec.this.writeDouble(v);
            return this;
        }

        @Override
        public MapWriter.EntryWriter put(CharSequence k, boolean v) throws IOException {
            writeExternString(k);
            writeBoolean(v);
            return this;
        }

        @Override
        public MapWriter.EntryWriter put(CharSequence k, CharSequence v) throws IOException {
            writeExternString(k);
            writeStr(v);
            return this;
        }

        private BiConsumer<CharSequence, Object> biConsumer;

        @Override
        public BiConsumer<CharSequence, Object> getBiConsumer() {
            if (biConsumer == null)
                biConsumer = MapWriter.EntryWriter.super.getBiConsumer();
            return biConsumer;
        }
    }

    public final BinEntryWriter ew = new BinEntryWriter();

    public void writeMap(MapWriter val) throws IOException {
        writeTag(MAP_ENTRY_ITER);
        val.writeMap(ew);
        writeTag(END);
    }

    public void writeTag(byte tag) throws IOException {
        daos.writeByte(tag);
    }

    public void writeTag(byte tag, int size) throws IOException {
        if ((tag & 0xe0) != 0) {
            if (size < 0x1f) {
                daos.writeByte(tag | size);
            } else {
                daos.writeByte(tag | 0x1f);
                writeVInt(size - 0x1f, daos);
            }
        } else {
            daos.writeByte(tag);
            writeVInt(size, daos);
        }
    }

    public void writeByteArray(byte[] arr, int offset, int len) throws IOException {
        writeTag(BYTEARR, len);
        daos.write(arr, offset, len);
    }

    public byte[] readByteArray(DataInputInputStream dis) throws IOException {
        byte[] arr = new byte[readVInt(dis)];
        dis.readFully(arr);
        return arr;
    }

    //use this to ignore the writable interface because , child docs will ignore the fl flag
    // is it a good design?
    private boolean ignoreWritable = false;
    private MapWriter.EntryWriter cew;

    public void writeSolrDocument(SolrDocument doc) throws IOException {
        List<SolrDocument> children = doc.getChildDocuments();
        int fieldsCount = 0;
        if (writableDocFields == null || writableDocFields.wantsAllFields() || ignoreWritable) {
            fieldsCount = doc.size();
        } else {
            for (Entry<String, Object> e : doc) {
                if (toWrite(e.getKey()))
                    fieldsCount++;
            }
        }
        int sz = fieldsCount + (children == null ? 0 : children.size());
        writeTag(SOLRDOC);
        writeTag(ORDERED_MAP, sz);
        if (cew == null)
            cew = new ConditionalKeyMapWriter.EntryWriterWrapper(ew, (k) -> toWrite(k.toString()));
        doc.writeMap(cew);
        if (children != null) {
            try {
                ignoreWritable = true;
                for (SolrDocument child : children) {
                    writeSolrDocument(child);
                }
            } finally {
                ignoreWritable = false;
            }
        }

    }

    protected boolean toWrite(String key) {
        return writableDocFields == null || ignoreWritable || writableDocFields.isWritable(key);
    }

    public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException {
        tagByte = dis.readByte();
        int size = readSize(dis);
        SolrDocument doc = new SolrDocument(new LinkedHashMap<>(size));
        for (int i = 0; i < size; i++) {
            String fieldName;
            Object obj = readVal(dis); // could be a field name, or a child document
            if (obj instanceof SolrDocument) {
                doc.addChildDocument((SolrDocument) obj);
                continue;
            } else {
                fieldName = (String) obj;
            }
            Object fieldVal = readVal(dis);
            doc.setField(fieldName, fieldVal);
        }
        return doc;
    }

    public SolrDocumentList readSolrDocumentList(DataInputInputStream dis) throws IOException {
        SolrDocumentList solrDocs = new SolrDocumentList();
        List list = (List) readVal(dis);
        solrDocs.setNumFound((Long) list.get(0));
        solrDocs.setStart((Long) list.get(1));
        solrDocs.setMaxScore((Float) list.get(2));

        @SuppressWarnings("unchecked")
        List<SolrDocument> l = (List<SolrDocument>) readVal(dis);
        solrDocs.addAll(l);
        return solrDocs;
    }

    public void writeSolrDocumentList(SolrDocumentList docs) throws IOException {
        writeTag(SOLRDOCLST);
        List<Number> l = new ArrayList<>(3);
        l.add(docs.getNumFound());
        l.add(docs.getStart());
        l.add(docs.getMaxScore());
        writeArray(l);
        writeArray(docs);
    }

    public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws IOException {
        int sz = readVInt(dis);
        float docBoost = (Float) readVal(dis);
        if (docBoost != 1f) {
            String message = "Ignoring document boost: " + docBoost
                    + " as index-time boosts are not supported anymore";
            if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) {
                log.warn(message);
            } else {
                log.debug(message);
            }
        }
        SolrInputDocument sdoc = createSolrInputDocument(sz);
        for (int i = 0; i < sz; i++) {
            String fieldName;
            Object obj = readVal(dis); // could be a boost, a field name, or a child document
            if (obj instanceof Float) {
                float boost = (Float) obj;
                if (boost != 1f) {
                    String message = "Ignoring field boost: " + boost
                            + " as index-time boosts are not supported anymore";
                    if (WARNED_ABOUT_INDEX_TIME_BOOSTS.compareAndSet(false, true)) {
                        log.warn(message);
                    } else {
                        log.debug(message);
                    }
                }
                fieldName = (String) readVal(dis);
            } else if (obj instanceof SolrInputDocument) {
                sdoc.addChildDocument((SolrInputDocument) obj);
                continue;
            } else {
                fieldName = (String) obj;
            }
            Object fieldVal = readVal(dis);
            sdoc.setField(fieldName, fieldVal);
        }
        return sdoc;
    }

    protected SolrInputDocument createSolrInputDocument(int sz) {
        return new SolrInputDocument(new LinkedHashMap<>(sz));
    }

    static final Predicate<CharSequence> IGNORECHILDDOCS = it -> !CommonParams.CHILDDOC.equals(it.toString());

    public void writeSolrInputDocument(SolrInputDocument sdoc) throws IOException {
        List<SolrInputDocument> children = sdoc.getChildDocuments();
        int sz = sdoc.size() + (children == null ? 0 : children.size());
        writeTag(SOLRINPUTDOC, sz);
        writeFloat(1f); // document boost
        sdoc.writeMap(new ConditionalKeyMapWriter.EntryWriterWrapper(ew, IGNORECHILDDOCS));
        if (children != null) {
            for (SolrInputDocument child : children) {
                writeSolrInputDocument(child);
            }
        }
    }

    public Map<Object, Object> readMapIter(DataInputInputStream dis) throws IOException {
        Map<Object, Object> m = newMap(-1);
        for (;;) {
            Object key = readVal(dis);
            if (key == END_OBJ)
                break;
            Object val = readVal(dis);
            m.put(key, val);
        }
        return m;
    }

    /**
     * create a new Map object
     * @param size expected size, -1 means unknown size
     */
    protected Map<Object, Object> newMap(int size) {
        return size < 0 ? new LinkedHashMap<>() : new LinkedHashMap<>(size);
    }

    public Map<Object, Object> readMap(DataInputInputStream dis) throws IOException {
        int sz = readVInt(dis);
        return readMap(dis, sz);
    }

    protected Map<Object, Object> readMap(DataInputInputStream dis, int sz) throws IOException {
        Map<Object, Object> m = newMap(sz);
        for (int i = 0; i < sz; i++) {
            Object key = readVal(dis);
            Object val = readVal(dis);
            m.put(key, val);

        }
        return m;
    }

    public final ItemWriter itemWriter = new ItemWriter() {
        @Override
        public ItemWriter add(Object o) throws IOException {
            writeVal(o);
            return this;
        }

        @Override
        public ItemWriter add(int v) throws IOException {
            writeInt(v);
            return this;
        }

        @Override
        public ItemWriter add(long v) throws IOException {
            writeLong(v);
            return this;
        }

        @Override
        public ItemWriter add(float v) throws IOException {
            writeFloat(v);
            return this;
        }

        @Override
        public ItemWriter add(double v) throws IOException {
            writeDouble(v);
            return this;
        }

        @Override
        public ItemWriter add(boolean v) throws IOException {
            writeBoolean(v);
            return this;
        }
    };

    @Override
    public void writeIterator(IteratorWriter val) throws IOException {
        writeTag(ITERATOR);
        val.writeIter(itemWriter);
        writeTag(END);
    }

    public void writeIterator(Iterator iter) throws IOException {
        writeTag(ITERATOR);
        while (iter.hasNext()) {
            writeVal(iter.next());
        }
        writeTag(END);
    }

    public List<Object> readIterator(DataInputInputStream fis) throws IOException {
        ArrayList<Object> l = new ArrayList<>();
        while (true) {
            Object o = readVal(fis);
            if (o == END_OBJ)
                break;
            l.add(o);
        }
        return l;
    }

    public void writeArray(List l) throws IOException {
        writeTag(ARR, l.size());
        for (int i = 0; i < l.size(); i++) {
            writeVal(l.get(i));
        }
    }

    public void writeArray(Collection coll) throws IOException {
        writeTag(ARR, coll.size());
        for (Object o : coll) {
            writeVal(o);
        }

    }

    public void writeArray(Object[] arr) throws IOException {
        writeTag(ARR, arr.length);
        for (int i = 0; i < arr.length; i++) {
            Object o = arr[i];
            writeVal(o);
        }
    }

    public List<Object> readArray(DataInputInputStream dis) throws IOException {
        int sz = readSize(dis);
        return readArray(dis, sz);
    }

    protected List readArray(DataInputInputStream dis, int sz) throws IOException {
        ArrayList<Object> l = new ArrayList<>(sz);
        for (int i = 0; i < sz; i++) {
            l.add(readVal(dis));
        }
        return l;
    }

    /**
     * write {@link EnumFieldValue} as tag+int value+string value
     * @param enumFieldValue to write
     */
    public void writeEnumFieldValue(EnumFieldValue enumFieldValue) throws IOException {
        writeTag(ENUM_FIELD_VALUE);
        writeInt(enumFieldValue.toInt());
        writeStr(enumFieldValue.toString());
    }

    public void writeMapEntry(Map.Entry val) throws IOException {
        writeTag(MAP_ENTRY);
        writeVal(val.getKey());
        writeVal(val.getValue());
    }

    /**
     * read {@link EnumFieldValue} (int+string) from input stream
     * @param dis data input stream
     * @return {@link EnumFieldValue}
     */
    public EnumFieldValue readEnumFieldValue(DataInputInputStream dis) throws IOException {
        Integer intValue = (Integer) readVal(dis);
        String stringValue = (String) convertCharSeq(readVal(dis));
        return new EnumFieldValue(intValue, stringValue);
    }

    public Map.Entry<Object, Object> readMapEntry(DataInputInputStream dis) throws IOException {
        final Object key = readVal(dis);
        final Object value = readVal(dis);
        return new Map.Entry<Object, Object>() {

            @Override
            public Object getKey() {
                return key;
            }

            @Override
            public Object getValue() {
                return value;
            }

            @Override
            public String toString() {
                return "MapEntry[" + key + ":" + value + "]";
            }

            @Override
            public Object setValue(Object value) {
                throw new UnsupportedOperationException();
            }

            @Override
            public int hashCode() {
                int result = 31;
                result *= 31 + getKey().hashCode();
                result *= 31 + getValue().hashCode();
                return result;
            }

            @Override
            public boolean equals(Object obj) {
                if (this == obj) {
                    return true;
                }
                if (!(obj instanceof Entry)) {
                    return false;
                }
                Map.Entry<Object, Object> entry = (Entry<Object, Object>) obj;
                return (this.getKey().equals(entry.getKey()) && this.getValue().equals(entry.getValue()));
            }
        };
    }

    /**
     * write the string as tag+length, with length being the number of UTF-8 bytes
     */
    public void writeStr(CharSequence s) throws IOException {
        if (s == null) {
            writeTag(NULL);
            return;
        }
        if (s instanceof Utf8CharSequence) {
            writeUTF8Str((Utf8CharSequence) s);
            return;
        }
        int end = s.length();
        int maxSize = end * ByteUtils.MAX_UTF8_BYTES_PER_CHAR;

        if (maxSize <= MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY) {
            if (bytes == null || bytes.length < maxSize)
                bytes = new byte[maxSize];
            int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0);
            writeTag(STR, sz);
            daos.write(bytes, 0, sz);
        } else {
            // double pass logic for large strings, see SOLR-7971
            int sz = ByteUtils.calcUTF16toUTF8Length(s, 0, end);
            writeTag(STR, sz);
            if (bytes == null || bytes.length < 8192)
                bytes = new byte[8192];
            ByteUtils.writeUTF16toUTF8(s, 0, end, daos, bytes);
        }
    }

    byte[] bytes;
    CharArr arr = new CharArr();
    private StringBytes bytesRef = new StringBytes(bytes, 0, 0);

    public CharSequence readStr(DataInputInputStream dis) throws IOException {
        return readStr(dis, null, readStringAsCharSeq);
    }

    public CharSequence readStr(DataInputInputStream dis, StringCache stringCache, boolean readStringAsCharSeq)
            throws IOException {
        if (readStringAsCharSeq) {
            return readUtf8(dis);
        }
        int sz = readSize(dis);
        return _readStr(dis, stringCache, sz);
    }

    private CharSequence _readStr(DataInputInputStream dis, StringCache stringCache, int sz) throws IOException {
        if (bytes == null || bytes.length < sz)
            bytes = new byte[sz];
        dis.readFully(bytes, 0, sz);
        if (stringCache != null) {
            return stringCache.get(bytesRef.reset(bytes, 0, sz));
        } else {
            arr.reset();
            ByteUtils.UTF8toUTF16(bytes, 0, sz, arr);
            return arr.toString();
        }
    }

    /////////// code to optimize reading UTF8
    static final int MAX_UTF8_SZ = 1024 * 64;//too big strings can cause too much memory allocation
    private Function<ByteArrayUtf8CharSequence, String> stringProvider;
    private BytesBlock bytesBlock;

    protected CharSequence readUtf8(DataInputInputStream dis) throws IOException {
        int sz = readSize(dis);
        return readUtf8(dis, sz);
    }

    protected CharSequence readUtf8(DataInputInputStream dis, int sz) throws IOException {
        ByteArrayUtf8CharSequence result = new ByteArrayUtf8CharSequence(null, 0, 0);
        if (dis.readDirectUtf8(result, sz)) {
            result.stringProvider = getStringProvider();
            return result;
        }

        if (sz > MAX_UTF8_SZ)
            return _readStr(dis, null, sz);
        if (bytesBlock == null)
            bytesBlock = new BytesBlock(1024 * 4);
        BytesBlock block = this.bytesBlock.expand(sz);
        dis.readFully(block.getBuf(), block.getStartPos(), sz);

        result.reset(block.getBuf(), block.getStartPos(), sz, null);
        result.stringProvider = getStringProvider();
        return result;
    }

    private Function<ByteArrayUtf8CharSequence, String> getStringProvider() {
        if (stringProvider == null) {
            stringProvider = new Function<>() {
                final CharArr charArr = new CharArr(8);

                @Override
                public String apply(ByteArrayUtf8CharSequence butf8cs) {
                    synchronized (charArr) {
                        charArr.reset();
                        ByteUtils.UTF8toUTF16(butf8cs.buf, butf8cs.offset(), butf8cs.size(), charArr);
                        return charArr.toString();
                    }
                }
            };
        }
        return this.stringProvider;
    }

    public void writeInt(int val) throws IOException {
        if (val > 0) {
            int b = SINT | (val & 0x0f);

            if (val >= 0x0f) {
                b |= 0x10;
                daos.writeByte(b);
                writeVInt(val >>> 4, daos);
            } else {
                daos.writeByte(b);
            }

        } else {
            daos.writeByte(INT);
            daos.writeInt(val);
        }
    }

    public int readSmallInt(DataInputInputStream dis) throws IOException {
        int v = tagByte & 0x0F;
        if ((tagByte & 0x10) != 0)
            v = (readVInt(dis) << 4) | v;
        return v;
    }

    public void writeLong(long val) throws IOException {
        if ((val & 0xff00000000000000L) == 0) {
            int b = SLONG | ((int) val & 0x0f);
            if (val >= 0x0f) {
                b |= 0x10;
                daos.writeByte(b);
                writeVLong(val >>> 4, daos);
            } else {
                daos.writeByte(b);
            }
        } else {
            daos.writeByte(LONG);
            daos.writeLong(val);
        }
    }

    public long readSmallLong(DataInputInputStream dis) throws IOException {
        long v = tagByte & 0x0F;
        if ((tagByte & 0x10) != 0)
            v = (readVLong(dis) << 4) | v;
        return v;
    }

    public void writeFloat(float val) throws IOException {
        daos.writeByte(FLOAT);
        daos.writeFloat(val);
    }

    public boolean writePrimitive(Object val) throws IOException {
        if (val == null) {
            daos.writeByte(NULL);
            return true;
        } else if (val instanceof Utf8CharSequence) {
            writeUTF8Str((Utf8CharSequence) val);
            return true;
        } else if (val instanceof CharSequence) {
            writeStr((CharSequence) val);
            return true;
        } else if (val instanceof Number) {

            if (val instanceof Integer) {
                writeInt(((Integer) val).intValue());
                return true;
            } else if (val instanceof Long) {
                writeLong(((Long) val).longValue());
                return true;
            } else if (val instanceof Float) {
                writeFloat(((Float) val).floatValue());
                return true;
            } else if (val instanceof Double) {
                writeDouble(((Double) val).doubleValue());
                return true;
            } else if (val instanceof Byte) {
                daos.writeByte(BYTE);
                daos.writeByte(((Byte) val).intValue());
                return true;
            } else if (val instanceof Short) {
                daos.writeByte(SHORT);
                daos.writeShort(((Short) val).intValue());
                return true;
            }
            return false;

        } else if (val instanceof Date) {
            daos.writeByte(DATE);
            daos.writeLong(((Date) val).getTime());
            return true;
        } else if (val instanceof Boolean) {
            writeBoolean((Boolean) val);
            return true;
        } else if (val instanceof byte[]) {
            writeByteArray((byte[]) val, 0, ((byte[]) val).length);
            return true;
        } else if (val instanceof ByteBuffer) {
            ByteBuffer buf = (ByteBuffer) val;
            writeByteArray(buf.array(), buf.position(), buf.limit() - buf.position());
            return true;
        } else if (val == END_OBJ) {
            writeTag(END);
            return true;
        }
        return false;
    }

    protected void writeBoolean(boolean val) throws IOException {
        if (val)
            daos.writeByte(BOOL_TRUE);
        else
            daos.writeByte(BOOL_FALSE);
    }

    protected void writeDouble(double val) throws IOException {
        daos.writeByte(DOUBLE);
        daos.writeDouble(val);
    }

    public void writeMap(Map<?, ?> val) throws IOException {
        writeTag(MAP, val.size());
        if (val instanceof MapWriter) {
            ((MapWriter) val).writeMap(ew);
            return;
        }
        for (Map.Entry<?, ?> entry : val.entrySet()) {
            Object key = entry.getKey();
            if (key instanceof String) {
                writeExternString((String) key);
            } else {
                writeVal(key);
            }
            writeVal(entry.getValue());
        }
    }

    public int readSize(DataInputInputStream in) throws IOException {
        int sz = tagByte & 0x1f;
        if (sz == 0x1f)
            sz += readVInt(in);
        return sz;
    }

    /**
     * Special method for variable length int (copied from lucene). Usually used for writing the length of a
     * collection/array/map In most of the cases the length can be represented in one byte (length &lt; 127) so it saves 3
     * bytes/object
     *
     * @throws IOException If there is a low-level I/O error.
     */
    public static void writeVInt(int i, FastOutputStream out) throws IOException {
        while ((i & ~0x7F) != 0) {
            out.writeByte((byte) ((i & 0x7f) | 0x80));
            i >>>= 7;
        }
        out.writeByte((byte) i);
    }

    /**
     * The counterpart for {@link #writeVInt(int, FastOutputStream)}
     *
     * @throws IOException If there is a low-level I/O error.
     */
    public static int readVInt(DataInputInputStream in) throws IOException {
        byte b = in.readByte();
        int i = b & 0x7F;
        for (int shift = 7; (b & 0x80) != 0; shift += 7) {
            b = in.readByte();
            i |= (b & 0x7F) << shift;
        }
        return i;
    }

    public static void writeVLong(long i, FastOutputStream out) throws IOException {
        while ((i & ~0x7F) != 0) {
            out.writeByte((byte) ((i & 0x7f) | 0x80));
            i >>>= 7;
        }
        out.writeByte((byte) i);
    }

    public static long readVLong(DataInputInputStream in) throws IOException {
        byte b = in.readByte();
        long i = b & 0x7F;
        for (int shift = 7; (b & 0x80) != 0; shift += 7) {
            b = in.readByte();
            i |= (long) (b & 0x7F) << shift;
        }
        return i;
    }

    private int stringsCount = 0;
    private Map<String, Integer> stringsMap;
    private List<CharSequence> stringsList;

    public void writeExternString(CharSequence s) throws IOException {
        if (s == null) {
            writeTag(NULL);
            return;
        }
        Integer idx = stringsMap == null ? null : stringsMap.get(s);
        if (idx == null)
            idx = 0;
        writeTag(EXTERN_STRING, idx);
        if (idx == 0) {
            writeStr(s);
            if (stringsMap == null)
                stringsMap = new HashMap<>();
            stringsMap.put(s.toString(), ++stringsCount);
        }

    }

    public CharSequence readExternString(DataInputInputStream fis) throws IOException {
        int idx = readSize(fis);
        if (idx != 0) {// idx != 0 is the index of the extern string
            return stringsList.get(idx - 1);
        } else {// idx == 0 means it has a string value
            tagByte = fis.readByte();
            CharSequence s = readStr(fis, stringCache, false);
            if (s != null)
                s = s.toString();
            if (stringsList == null)
                stringsList = new ArrayList<>();
            stringsList.add(s);
            return s;
        }
    }

    public void writeUTF8Str(Utf8CharSequence utf8) throws IOException {
        writeTag(STR, utf8.size());
        daos.writeUtf8CharSeq(utf8);
    }

    public long getTotalBytesWritten() {
        if (daos != null) {
            return daos.written;
        }
        return 0;
    }

    /**
     * Allows extension of {@link JavaBinCodec} to support serialization of arbitrary data types.
     * <p>
     * Implementors of this interface write a method to serialize a given object using an existing {@link JavaBinCodec}
     */
    public interface ObjectResolver {
        /**
         * Examine and attempt to serialize the given object, using a {@link JavaBinCodec} to write it to a stream.
         *
         * @param o     the object that the caller wants serialized.
         * @param codec used to actually serialize {@code o}.
         * @return the object {@code o} itself if it could not be serialized, or {@code null} if the whole object was successfully serialized.
         * @see JavaBinCodec
         */
        Object resolve(Object o, JavaBinCodec codec) throws IOException;
    }

    public interface WritableDocFields {
        boolean isWritable(String name);

        boolean wantsAllFields();
    }

    public static class StringCache {
        private final Cache<StringBytes, String> cache;

        public StringCache(Cache<StringBytes, String> cache) {
            this.cache = cache;
        }

        public String get(StringBytes b) {
            String result = cache.get(b);
            if (result == null) {
                //make a copy because the buffer received may be changed later by the caller
                StringBytes copy = new StringBytes(Arrays.copyOfRange(b.bytes, b.offset, b.offset + b.length), 0,
                        b.length);
                CharArr arr = new CharArr();
                ByteUtils.UTF8toUTF16(b.bytes, b.offset, b.length, arr);
                result = arr.toString();
                cache.put(copy, result);
            }
            return result;
        }
    }

    @Override
    public void close() throws IOException {
        if (daos != null) {
            daos.flushBuffer();
        }
    }
}