Example usage for org.apache.hadoop.io Text readString

List of usage examples for org.apache.hadoop.io Text readString

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. Text
  6. readString

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text readString.

Prototype

public static String readString(DataInput in) throws IOException

Source Link

Document

Read a UTF8 encoded string from in

Usage

From source file:org.apache.nutch.indexer.field.FieldWritable.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    name = Text.readString(in);
    value = Text.readString(in);/*from ww  w  .  j  a  v a2s  . c o  m*/
    type = FieldType.valueOf(Text.readString(in));
    boost = in.readFloat();
    indexed = in.readBoolean();
    stored = in.readBoolean();
    tokenized = in.readBoolean();
}

From source file:org.apache.nutch.indexer.IndexDocument.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    fields.clear();//w ww  . j a  va  2s.  c om
    byte version = in.readByte();
    if (version != VERSION) {
        throw new VersionMismatchException(VERSION, version);
    }
    int size = WritableUtils.readVInt(in);
    for (int i = 0; i < size; i++) {
        String name = Text.readString(in);
        IndexField field = new IndexField();
        field.readFields(in);
        fields.put(name, field);
    }
    weight = in.readFloat();
    documentMeta.readFields(in);
}

From source file:org.apache.nutch.indexer.IndexField.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    weight = in.readFloat();//from  w  ww .ja  va 2  s .co m
    int count = in.readInt();
    values = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        String type = Text.readString(in);

        if (type.equals("java.lang.String")) {
            values.add(Text.readString(in));
        } else if (type.equals("java.lang.Boolean")) {
            values.add(in.readBoolean());
        } else if (type.equals("java.lang.Integer")) {
            values.add(in.readInt());
        } else if (type.equals("java.lang.Float")) {
            values.add(in.readFloat());
        } else if (type.equals("java.lang.Long")) {
            values.add(in.readLong());
        } else if (type.equals("java.util.Date")) {
            values.add(new Date(in.readLong()));
        }
    }
}

From source file:org.apache.nutch.indexer.NutchDocument.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    byte version = in.readByte();
    if (version != VERSION) {
        throw new VersionMismatchException(VERSION, version);
    }/*from  w  w w  .jav a2 s  .  c o  m*/
    int size = WritableUtils.readVInt(in);
    for (int i = 0; i < size; i++) {
        String name = Text.readString(in);
        int numValues = WritableUtils.readVInt(in);
        fields.put(name, new ArrayList<String>());
        for (int j = 0; j < numValues; j++) {
            String value = Text.readString(in);
            addFieldUnprotected(name, value);
        }
    }
    score = in.readFloat();
    documentMeta.readFields(in);
}

From source file:org.apache.nutch.metadata.Metadata.java

License:Apache License

public final void readFields(DataInput in) throws IOException {
    int keySize = in.readInt();
    String key;// w w w  .  j  av  a  2s . c  o  m
    for (int i = 0; i < keySize; i++) {
        key = Text.readString(in);
        int valueSize = in.readInt();
        for (int j = 0; j < valueSize; j++) {
            add(key, Text.readString(in));
        }
    }
}

From source file:org.apache.nutch.parse.Outlink.java

License:Apache License

public void readFields(DataInput in) throws IOException {
    toUrl = Text.readString(in);
    anchor = Text.readString(in);
}

From source file:org.apache.nutch.parse.ParseData.java

License:Apache License

public final void readFields(DataInput in) throws IOException {

    version = in.readByte();/* w w  w  . ja  va2  s  . c  o  m*/
    // incompatible change from UTF8 (version < 5) to Text
    if (version != VERSION)
        throw new VersionMismatchException(VERSION, version);
    status = ParseStatus.read(in);
    title = Text.readString(in); // read title

    int numOutlinks = in.readInt();
    outlinks = new Outlink[numOutlinks];
    for (int i = 0; i < numOutlinks; i++) {
        outlinks[i] = Outlink.read(in);
    }

    if (version < 3) {
        int propertyCount = in.readInt(); // read metadata
        contentMeta.clear();
        for (int i = 0; i < propertyCount; i++) {
            contentMeta.add(Text.readString(in), Text.readString(in));
        }
    } else {
        contentMeta.clear();
        contentMeta.readFields(in);
    }
    if (version > 3) {
        parseMeta.clear();
        parseMeta.readFields(in);
    }
}

From source file:org.apache.nutch.protocol.Content.java

License:Apache License

private final void readFieldsCompressed(DataInput in) throws IOException {
    byte oldVersion = in.readByte();
    switch (oldVersion) {
    case 0://from   w  w  w .j  a  va 2  s.  c om
    case 1:
        url = Text.readString(in); // read url
        base = Text.readString(in); // read base

        content = new byte[in.readInt()]; // read content
        in.readFully(content);

        contentType = Text.readString(in); // read contentType
        // reconstruct metadata
        int keySize = in.readInt();
        String key;
        for (int i = 0; i < keySize; i++) {
            key = Text.readString(in);
            int valueSize = in.readInt();
            for (int j = 0; j < valueSize; j++) {
                metadata.add(key, Text.readString(in));
            }
        }
        break;
    case 2:
        url = Text.readString(in); // read url
        base = Text.readString(in); // read base

        content = new byte[in.readInt()]; // read content
        in.readFully(content);

        contentType = Text.readString(in); // read contentType
        metadata.readFields(in); // read meta data
        break;
    default:
        throw new VersionMismatchException((byte) 2, oldVersion);
    }

}

From source file:org.apache.nutch.protocol.Content.java

License:Apache License

public final void readFields(DataInput in) throws IOException {
    metadata.clear();/*  w  ww.  j  av  a2s .c  o  m*/
    int sizeOrVersion = in.readInt();
    if (sizeOrVersion < 0) { // version
        version = sizeOrVersion;
        switch (version) {
        case VERSION:
            url = Text.readString(in);
            base = Text.readString(in);

            content = new byte[in.readInt()];
            in.readFully(content);

            contentType = Text.readString(in);
            metadata.readFields(in);
            break;
        default:
            throw new VersionMismatchException((byte) VERSION, (byte) version);
        }
    } else { // size
        byte[] compressed = new byte[sizeOrVersion];
        in.readFully(compressed, 0, compressed.length);
        ByteArrayInputStream deflated = new ByteArrayInputStream(compressed);
        DataInput inflater = new DataInputStream(new InflaterInputStream(deflated));
        readFieldsCompressed(inflater);
    }
}

From source file:org.apache.nutch.scoring.ScoreDatum.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    score = in.readFloat();//from   w  w w  .j  a  v a  2  s  . c  o m
    url = Text.readString(in);
    anchor = Text.readString(in);
    distance = WritableUtils.readVInt(in);
    metaData.clear();

    int size = WritableUtils.readVInt(in);
    for (int i = 0; i < size; i++) {
        String key = Text.readString(in);
        byte[] value = Bytes.readByteArray(in);
        metaData.put(key, value);
    }
}