com.cloudera.sqoop.lib.LobRef.java Source code

Java tutorial

Introduction

Here is the source code for com.cloudera.sqoop.lib.LobRef.java

Source

/**
 * Licensed to Cloudera, Inc. under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  Cloudera, Inc. licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.cloudera.sqoop.lib;

import java.io.Closeable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import com.cloudera.sqoop.io.LobFile;
import com.cloudera.sqoop.io.LobReaderCache;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * Abstract base class that holds a reference to a Blob or a Clob.
 * DATATYPE is the type being held (e.g., a byte array).
 * CONTAINERTYPE is the type used to hold this data (e.g., BytesWritable).
 * ACCESSORTYPE is the type used to access this data in a streaming fashion
 *   (either an InputStream or a Reader).
 */
public abstract class LobRef<DATATYPE, CONTAINERTYPE, ACCESSORTYPE> implements Closeable, Writable {

    public static final Log LOG = LogFactory.getLog(LobRef.class.getName());

    protected LobRef() {
        this.fileName = null;
        this.offset = 0;
        this.length = 0;

        this.realData = null;
    }

    protected LobRef(CONTAINERTYPE container) {
        this.fileName = null;
        this.offset = 0;
        this.length = 0;

        this.realData = container;
    }

    protected LobRef(String file, long offset, long length) {
        this.fileName = file;
        this.offset = offset;
        this.length = length;

        this.realData = null;
    }

    // If the data is 'small', it's held directly, here.
    private CONTAINERTYPE realData;

    /** Internal API to retrieve the data object. */
    protected CONTAINERTYPE getDataObj() {
        return realData;
    }

    /** Internal API to set the data object. */
    protected void setDataObj(CONTAINERTYPE data) {
        this.realData = data;
    }

    // If there data is too large to materialize fully, it's written into a file
    // whose path (relative to the rest of the dataset) is recorded here. This
    // takes precedence if the value fof fileName is non-null. These records are
    // currently written into LobFile-formatted files, which hold multiple
    // records. The starting offset and length of the record are recorded here
    // as well.
    private String fileName;
    private long offset;
    private long length;

    // If we've opened a LobFile object, track our reference to it here.
    private LobFile.Reader lobReader;

    @Override
    @SuppressWarnings("unchecked")
    /**
     * Clone the current reference object. data is deep-copied; any open
     * file handle remains with the original only.
     */
    public Object clone() throws CloneNotSupportedException {
        LobRef<DATATYPE, CONTAINERTYPE, ACCESSORTYPE> r = (LobRef<DATATYPE, CONTAINERTYPE, ACCESSORTYPE>) super.clone();

        r.lobReader = null; // Reference to opened reader is not duplicated.
        if (null != realData) {
            r.realData = deepCopyData(realData);
        }

        return r;
    }

    @Override
    protected synchronized void finalize() throws Throwable {
        close();
        super.finalize();
    }

    public void close() throws IOException {
        // Discard any open LobReader.
        if (null != this.lobReader) {
            LobReaderCache.getCache().recycle(this.lobReader);
        }
    }

    /**
     * @return true if the LOB data is in an external file; false if
     * it materialized inline.
     */
    public boolean isExternal() {
        return fileName != null;
    }

    /**
     * Convenience method to access #getDataStream(Configuration, Path)
     * from within a map task that read this LobRef from a file-based
     * InputSplit.
     * @param mapContext the Mapper.Context instance that encapsulates
     * the current map task.
     * @return an object that lazily streams the record to the client.
     * @throws IllegalArgumentException if it cannot find the source
     * path for this LOB based on the MapContext.
     * @throws IOException if it could not read the LOB from external storage.
     */
    public ACCESSORTYPE getDataStream(Mapper.Context mapContext) throws IOException {
        InputSplit split = mapContext.getInputSplit();
        if (split instanceof FileSplit) {
            Path basePath = ((FileSplit) split).getPath().getParent();
            return getDataStream(mapContext.getConfiguration(), basePath);
        } else {
            throw new IllegalArgumentException("Could not ascertain LOB base path from MapContext.");
        }
    }

    /**
     * Get access to the LOB data itself.
     * This method returns a lazy reader of the LOB data, accessing the
     * filesystem for external LOB storage as necessary.
     * @param conf the Configuration used to access the filesystem
     * @param basePath the base directory where the table records are
     * stored.
     * @return an object that lazily streams the record to the client.
     * @throws IOException if it could not read the LOB from external storage.
     */
    public ACCESSORTYPE getDataStream(Configuration conf, Path basePath) throws IOException {
        if (isExternal()) {
            // Read from external storage.
            Path pathToRead = LobReaderCache.qualify(new Path(basePath, fileName), conf);
            LOG.debug("Retreving data stream from external path: " + pathToRead);
            if (lobReader != null) {
                // We already have a reader open to a LobFile. Is it the correct file?
                if (!pathToRead.equals(lobReader.getPath())) {
                    // No. Close this.lobReader and get the correct one.
                    LOG.debug("Releasing previous external reader for " + lobReader.getPath());
                    LobReaderCache.getCache().recycle(lobReader);
                    lobReader = LobReaderCache.getCache().get(pathToRead, conf);
                }
            } else {
                lobReader = LobReaderCache.getCache().get(pathToRead, conf);
            }

            // We now have a LobFile.Reader associated with the correct file. Get to
            // the correct offset and return an InputStream/Reader to the user.
            if (lobReader.tell() != offset) {
                LOG.debug("Seeking to record start offset " + offset);
                lobReader.seek(offset);
            }

            if (!lobReader.next()) {
                throw new IOException("Could not locate record at " + pathToRead + ":" + offset);
            }

            return getExternalSource(lobReader);
        } else {
            // This data is already materialized in memory; wrap it and return.
            return getInternalSource(realData);
        }
    }

    /**
     * Using the LobFile reader, get an accessor InputStream or Reader to the
     * underlying data.
     */
    protected abstract ACCESSORTYPE getExternalSource(LobFile.Reader reader) throws IOException;

    /**
     * Wrap the materialized data in an InputStream or Reader.
     */
    protected abstract ACCESSORTYPE getInternalSource(CONTAINERTYPE data);

    /**
     * @return the materialized data itself.
     */
    protected abstract DATATYPE getInternalData(CONTAINERTYPE data);

    /**
     * Make a copy of the materialized data.
     */
    protected abstract CONTAINERTYPE deepCopyData(CONTAINERTYPE data);

    public DATATYPE getData() {
        if (isExternal()) {
            throw new RuntimeException("External LOBs must be read via getDataStream()");
        }

        return getInternalData(realData);
    }

    @Override
    public String toString() {
        if (isExternal()) {
            return "externalLob(lf," + fileName + "," + Long.toString(offset) + "," + Long.toString(length) + ")";
        } else {
            return realData.toString();
        }
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        // The serialization format for this object is:
        // boolean isExternal
        // if true, then:
        //   a string identifying the external storage type
        //   and external-storage-specific data.
        // if false, then we use readFieldsInternal() to allow BlobRef/ClobRef
        // to serialize as it sees fit.
        //
        // Currently the only external storage supported is LobFile, identified
        // by the string "lf". This serializes with the filename (as a string),
        // followed by a long-valued offset and a long-valued length.

        boolean isExternal = in.readBoolean();
        if (isExternal) {
            this.realData = null;

            String storageType = Text.readString(in);
            if (!storageType.equals("lf")) {
                throw new IOException("Unsupported external LOB storage code: " + storageType);
            }

            // Storage type "lf" is LobFile: filename, offset, length.
            this.fileName = Text.readString(in);
            this.offset = in.readLong();
            this.length = in.readLong();
        } else {
            readFieldsInternal(in);

            this.fileName = null;
            this.offset = 0;
            this.length = 0;
        }
    }

    /**
     * Perform the readFields() operation on a fully-materializable record.
     * @param in the DataInput to deserialize from.
     */
    protected abstract void readFieldsInternal(DataInput in) throws IOException;

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeBoolean(isExternal());
        if (isExternal()) {
            Text.writeString(out, "lf"); // storage type "lf" for LobFile.
            Text.writeString(out, fileName);
            out.writeLong(offset);
            out.writeLong(length);
        } else {
            writeInternal(out);
        }
    }

    /**
     * Perform the write() operation on a fully-materializable record.
     * @param out the DataOutput to deserialize to.
     */
    protected abstract void writeInternal(DataOutput out) throws IOException;

    protected static final ThreadLocal<Matcher> EXTERNAL_MATCHER = new ThreadLocal<Matcher>() {
        @Override
        protected Matcher initialValue() {
            Pattern externalPattern = Pattern.compile("externalLob\\(lf,(.*),([0-9]+),([0-9]+)\\)");
            return externalPattern.matcher("");
        }
    };
}