com.mellanox.r4h.DataXceiverBase.java Source code

Java tutorial

Introduction

Here is the source code for com.mellanox.r4h.DataXceiverBase.java

Source

/*
 ** Copyright (C) 2014 Mellanox Technologies
 **
 ** Licensed under the Apache License, Version 2.0 (the "License");
 ** you may not use this file except in compliance with the License.
 ** You may obtain a copy of the License at:
 **
 ** http://www.apache.org/licenses/LICENSE-2.0
 **
 ** Unless required by applicable law or agreed to in writing, software
 ** distributed under the License is distributed on an "AS IS" BASIS,
 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 ** either express or implied. See the License for the specific language
 ** governing permissions and  limitations under the License.
 **
 */

package com.mellanox.r4h;

import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR;
import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.ERROR_ACCESS_TOKEN;
import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.SUCCESS;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.protocol.datatransfer.Op;
import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.CachingStrategyProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.accelio.jxio.ClientSession;
import org.accelio.jxio.EventName;
import org.accelio.jxio.EventReason;
import org.accelio.jxio.Msg;
import org.accelio.jxio.ServerSession;
import org.accelio.jxio.ServerSession.SessionKey;
import org.apache.hadoop.hdfs.server.datanode.R4HBlockReceiver;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.server.datanode.DataNodeBridge;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;

/**
 * R4H's parallel class to the original
 * org.apache.hadoop.hdfs.server.datanode.DataXceiver
 * 
 * @see org.DataXceiver.hadoop.hdfs.server.datanode.DataXceiver It breaks the
 *      original serial flow of processing HDFS operation into an async flow
 *      with JXIO RDMA library CURRENTLY: only HDFS WRITE operation is supported
 */
abstract class DataXceiverBase {
    private final static Log LOG = LogFactory.getLog(DataXceiverBase.class);
    private static final int NUM_OF_BLOCK_RECEIVER_CREATION_ATTEMPTS = 3;
    private final DataXceiverServer dxcs;
    ClientSession clientSession;
    private final DataNodeBridge dnBridge;
    protected WriteOprHeader oprHeader;
    private R4HBlockReceiver blockReceiver;
    private final ServerPortalWorker spw;
    private final ServerSession serverSession;
    private final String uri;
    private final List<Msg> onFlightMsgs = new LinkedList<Msg>();
    long clientOnFlightNumMsgs = 0;
    private boolean serverSessionClosed = false;
    private boolean clientSessionClosed = false;
    private boolean isFirstRequest = true;
    private boolean isFirstReply = true;
    boolean clientSessionCloseEventExpected = true;
    boolean returnedAuxillaryExecutorToPool = false;
    private final R4HExecutor ioExecutor;
    private final R4HExecutor auxExecutor;
    private final MessageAction msgCallbacks;

    class SSCallbacks implements ServerSession.Callbacks {

        @Override
        public void onRequest(final Msg msg) {
            spw.processReplies();
            try {
                onFlightMsgs.add(msg);

                if (isFirstRequest) {
                    asyncProcessOPRHeaderRequest(msg);
                    isFirstRequest = false;
                } else {
                    asyncProcessPacketRequest(msg);
                }
            } catch (Throwable t) {
                LOG.error("DataXceiver exception during execution of new async request", t);
                if ((DataXceiverBase.this.serverSession != null)
                        && (!DataXceiverBase.this.serverSession.getIsClosing())) {
                    DataXceiverBase.this.serverSession.close();
                }
            }
        }

        @Override
        public boolean onMsgError(Msg msg, EventReason reason) {
            boolean releaseMsg = true; // Always return the message to server's bound pool
            try {
                if (hasPipeline()) {
                    LOG.error(String.format(
                            "Server Msg error: MSG=%s reason=%s ss=%s ss.isClosing=%s cs=%s cs.isClosing=%s", msg,
                            reason, serverSession, serverSession.getIsClosing(), clientSession,
                            clientSession.getIsClosing()));
                } else {
                    LOG.error(String.format("Server Msg error: MSG=%s reason=%s ss=%s ss.isClosing=%s", msg, reason,
                            serverSession, serverSession.getIsClosing()));
                }
            } catch (Throwable t) {
                LOG.error("Got exception during processing of Msg error", t);
                if ((DataXceiverBase.this.serverSession != null)
                        && (!DataXceiverBase.this.serverSession.getIsClosing())) {
                    DataXceiverBase.this.serverSession.close();
                }
            }
            onFlightMsgs.remove(msg);
            DataXceiverBase.this.close();
            return releaseMsg;
        }

        @Override
        public void onSessionEvent(EventName session_event, EventReason reason) {
            String logmsg = String.format("Server Session event: event=%s reason=%s ss=%s uri=%s", session_event,
                    reason, serverSession, uri);
            boolean needAuxThreadInit = false;
            switch (session_event) {
            case SESSION_CLOSED:
                if (onFlightMsgs.size() == 0) {
                    LOG.info(logmsg);
                } else {
                    LOG.error(logmsg);
                    needAuxThreadInit = true;
                }
                break;
            case SESSION_ERROR:
            case SESSION_REJECT:
                LOG.error(logmsg);
                needAuxThreadInit = true;
                break;
            default:
                break;
            }

            // Assuming no other kinds of event for SS except CLOSED,ERROR and REJECT
            serverSessionClosed = true;
            if ((clientSession != null) && (!clientSession.getIsClosing())) {
                clientSessionCloseEventExpected = true;
                clientSession.close();
            }

            if ((clientSessionClosed) || (!hasPipeline())) {
                if (onFlightMsgs.size() > 0) {
                    LOG.warn(String.format("Discarding %d messages for server seesion %s", onFlightMsgs.size(),
                            serverSession));
                    for (Msg m : onFlightMsgs) {
                        serverSession.discardRequest(m);
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Discarding MSG: " + m);
                        }
                    }
                    onFlightMsgs.clear();
                    clientOnFlightNumMsgs = 0;
                }
                if (!returnedAuxillaryExecutorToPool) {
                    returnAuxillaryExecutortoPool(needAuxThreadInit);
                }

            } else if (onFlightMsgs.size() > 0) {
                LOG.warn(
                        "Server session closed but there are still messages on flight for proxy client - waiting for client close event to discard messages and return ServerWorker to pool");
            }

        }

    }

    class CSCallbacks implements ClientSession.Callbacks {

        private void asyncOnResponse(final Msg msg) throws IOException {
            if (isFirstReply) {
                LOG.info("Going to process pipeline reply for OPR header. uri=" + DataXceiverBase.this.uri);
                processOPRHeaderReply(msg);
                isFirstReply = false;
            } else {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Going to process pipeline reply for packet");
                }
                processPacketReply(msg);
            }
        }

        @Override
        public void onResponse(final Msg msg) {
            spw.processReplies();
            clientOnFlightNumMsgs--;
            if (LOG.isTraceEnabled()) {
                LOG.trace("Got reply from pipeline client - " + clientSession);
            }
            auxExecutor.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        if (!clientSession.getIsClosing()) {
                            asyncOnResponse(msg);
                        }
                    } catch (Throwable t) {
                        LOG.error("Failed to process async reply", t);
                        asyncCloseServerSession();
                    }
                }
            });
        }

        @Override
        public void onSessionEstablished() {
            LOG.info("Client session established: " + clientSession + ", uri=" + uri);
        }

        @Override
        public void onMsgError(Msg msg, EventReason reason) {
            LOG.error(String.format(
                    "Client Msg error: MSG=%s reason=%s ss=%s ss.isClosing=%s cs=%s cs.isClosing=%s", msg, reason,
                    serverSession, serverSession.getIsClosing(), clientSession, clientSession.getIsClosing()));
            DataXceiverBase.this.close();
        }

        @Override
        public void onSessionEvent(EventName session_event, EventReason reason) {
            String logmsg = String.format("Client Session event: event=%s reason=%s ss=%s clientOnFlight=%d",
                    session_event, reason, serverSession, clientOnFlightNumMsgs);
            boolean needAuxThreadInit = false;
            switch (session_event) {
            case SESSION_CLOSED:
                if ((DataXceiverBase.this.clientSessionCloseEventExpected) && (clientOnFlightNumMsgs == 0)) {
                    LOG.info(logmsg);
                } else {
                    LOG.error(logmsg);
                    needAuxThreadInit = true;
                }
                break;
            case SESSION_ERROR:
            case SESSION_REJECT:
                LOG.error(logmsg);
                LOG.error("Closing server session due to error event: ss=" + serverSession);
                if ((serverSession != null) && (!serverSession.getIsClosing())) {
                    serverSession.close();
                }
                needAuxThreadInit = true;
                break;
            default:
                break;
            }

            clientSessionCloseEventExpected = false; // refers to close/error/reject
            clientSessionClosed = true;

            if (serverSessionClosed) {
                LOG.debug("Client session closed after server session was closed");
                if (clientOnFlightNumMsgs > 0) {
                    LOG.warn(String.format(
                            "Clinet session closed while still mirror messages on flight. Discarding %d messages ...",
                            onFlightMsgs.size()));
                }
                for (Msg m : onFlightMsgs) {
                    serverSession.discardRequest(m);
                }
                onFlightMsgs.clear();
                clientOnFlightNumMsgs = 0;

                if (!returnedAuxillaryExecutorToPool) {
                    returnAuxillaryExecutortoPool(needAuxThreadInit);
                }
            }

        }

    }

    DataXceiverBase(DataXceiverServer dxcs, ServerPortalWorker spw, SessionKey sKey, R4HExecutor ioExec,
            R4HExecutor auxExecutor) {
        this.dxcs = dxcs;
        this.spw = spw;
        this.dnBridge = dxcs.dnBridge;
        this.ioExecutor = ioExec;
        this.auxExecutor = auxExecutor;
        this.uri = sKey.getUri();
        DataXceiverBase.SSCallbacks ssCbs = this.new SSCallbacks();
        serverSession = new ServerSession(sKey, ssCbs);
        this.msgCallbacks = new MessageAction() {

            @Override
            public void onMessageAction(Msg msg) {
                completePacket(msg);
            }

        };
    }

    private void asyncProcessOPRHeaderRequest(final Msg msg) {
        final Runnable oprHeaderRequestTask = new Runnable() {

            @Override
            public void run() {
                try {
                    processOPRHeaderRequest(msg);
                } catch (Exception e) {
                    LOG.error("Got exception during processing packet request", e);
                    DataXceiverBase.this.close(true);
                }
            }
        };

        auxExecutor.execute(oprHeaderRequestTask);
    }

    private void processOPRHeaderRequest(Msg msg) throws IOException, NoSuchFieldException, NoSuchMethodException,
            IllegalArgumentException, IllegalAccessException {
        if (serverSession.getIsClosing()) {
            LOG.warn("Process OPRHeaderRequest for a closed session, discarding...");
            return;
        }

        if (LOG.isTraceEnabled()) {
            LOG.trace("Processing block header request. uri=" + DataXceiverBase.this.uri);
        }

        msg.getIn().position(0);
        DataInputStream in = new DataInputStream(new ByteBufferInputStream(msg.getIn()));
        final short version = in.readShort();
        if (version != DataTransferProtocol.DATA_TRANSFER_VERSION) {
            in.close();
            throw new IOException("Version Mismatch (Expected: " + DataTransferProtocol.DATA_TRANSFER_VERSION
                    + ", Received: " + version + " )");
        }
        Op op = Op.read(in);
        if (op != Op.WRITE_BLOCK) {
            throw new IOException("Unknown op " + op + " in data stream");
        }

        parseOpWriteBlock(in);

        // check single target for transfer-RBW/Finalized
        if (oprHeader.isTransfer() && oprHeader.getTargets().length > 0) {
            throw new IOException(oprHeader.getStage() + " does not support multiple targets "
                    + Arrays.asList(oprHeader.getTargets()));
        }

        if (LOG.isDebugEnabled()) {
            LOG.debug("uri= " + DataXceiverBase.this.uri + "\nopWriteBlock: stage=" + oprHeader.getStage()
                    + ", clientname=" + oprHeader.getClientName() + "\n  block  =" + oprHeader.getBlock()
                    + ", newGs=" + oprHeader.getLatestGenerationStamp() + ", bytesRcvd=["
                    + oprHeader.getMinBytesRcvd() + ", " + oprHeader.getMaxBytesRcvd() + "]" + "\n  targets="
                    + Arrays.asList(oprHeader.getTargets()) + "; pipelineSize=" + oprHeader.getPipelineSize()
                    + ", srcDataNode=" + oprHeader.getSrcDataNode() + ", isDatanode=" + oprHeader.isDatanode()
                    + ", isClient=" + oprHeader.isClient() + ", isTransfer=" + oprHeader.isTransfer()
                    + ", writeBlock receive buf size " + msg.getIn().limit());
        }

        // We later mutate block's generation stamp and length, but we need to
        // forward the original version of the block to downstream mirrors, so
        // make a copy here.
        final ExtendedBlock originalBlock = new ExtendedBlock(oprHeader.getBlock());
        oprHeader.getBlock().setNumBytes(dnBridge.getEstimateBlockSize());
        LOG.info("Receiving " + oprHeader.getBlock() + " src: " + uri);

        boolean isTokenAccessOk = checkAccess(oprHeader.isClient(), oprHeader.getBlock(), oprHeader.getBlockToken(),
                Op.WRITE_BLOCK, BlockTokenSecretManager.AccessMode.WRITE, msg);

        if (isTokenAccessOk) {

            if (oprHeader.isDatanode() || oprHeader.getStage() != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) {
                // open a block receiver
                createBlockReciver(in, NUM_OF_BLOCK_RECEIVER_CREATION_ATTEMPTS);

                if (LOG.isTraceEnabled()) {
                    LOG.trace("After BlockReceiver creation: " + blockReceiver);
                }

            } else {
                dnBridge.recoverClose(oprHeader.getBlock(), oprHeader.getLatestGenerationStamp(),
                        oprHeader.getMinBytesRcvd());
            }

            //
            // Connect to downstream machine, if appropriate
            //
            if (hasPipeline()) {
                try {
                    blockReceiver.setMirrorOut(new DummyDataOutputStream()); // we send to pipeline with RDMA and then keep using vanila's
                                                                             // original
                                                                             // receivePacket function by modifying mirror stream with dummy
                                                                             // stream to
                                                                             // avoid sending to pipeline from vanila's flow
                                                                             // openPipelineConnection();
                                                                             // sendOprHeaderToPipeline(msg, originalBlock);
                    ClientSession.Callbacks csCBs = DataXceiverBase.this.new CSCallbacks();
                    String clientURI = DataXceiverBase.this.uri.toString();
                    spw.queueAsyncPipelineConnection(csCBs, clientURI, oprHeader, DataXceiverBase.this);
                    /* queue request to send OPR Header to pipeline */
                    Msg mirror = msg.getMirror(false);
                    mirror.getOut().clear();
                    DataOutputStream mirrorOut = new DataOutputStream(new ByteBufferOutputStream(mirror.getOut()));
                    senderWriteBlock(mirrorOut, originalBlock);
                    mirrorOut.flush();
                    spw.queueAsyncRequest(mirror, this);
                } catch (Exception e) {
                    if (oprHeader.isClient()) {
                        replyHeaderAck(msg, ERROR, oprHeader.getTargetByIndex(0).getXferAddr());
                        // NB: Unconditionally using the xfer addr w/o hostname
                        LOG.error(dnBridge.getDN() + ":Exception transfering block " + oprHeader.getBlock()
                                + " to mirror " + oprHeader.getTargetByIndex(0).getInfoAddr() + ": "
                                + StringUtils.stringifyException(e));
                    } else {
                        LOG.info(dnBridge.getDN() + ":Exception transfering " + oprHeader.getBlock() + " to mirror "
                                + oprHeader.getTargetByIndex(0).getInfoAddr() + "- continuing without the mirror",
                                e);
                    }
                }
            } else if (oprHeader.isClient() && !oprHeader.isTransfer()) {
                replyHeaderAck(msg); // async
            }
        }
    }

    private void processOPRHeaderReply(Msg msg) throws IOException {
        // read connect ack (only for clients, not for replication req)
        if (oprHeader.isClient()) {
            msg.getIn().position(0);
            BlockOpResponseProto connectAck = BlockOpResponseProto
                    .parseFrom(PBHelper.vintPrefixed(new ByteBufferInputStream(msg.getIn())));
            Status mirrorInStatus = connectAck.getStatus();
            String firstBadLink = connectAck.getFirstBadLink();
            if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) {
                LOG.info("Datanode " + oprHeader.getTargets().length + " got response for connect ack "
                        + " from downstream datanode with firstbadlink as " + firstBadLink);
            }

            // send connect-ack to source for clients and not transfer-RBW/Finalized
            if (!oprHeader.isTransfer()) {
                Msg origMsg = msg.getMirror(false);
                replyHeaderPipelineAck(origMsg, mirrorInStatus, firstBadLink);
            }
        }
    }

    private void asyncProcessPacketRequest(final Msg msg) {

        final Runnable requestTask = new Runnable() {

            @Override
            public void run() {
                processPacketRequest(msg);
            }
        };

        PacketMessageContext context = new PacketMessageContext();
        msg.setUserContext(context);

        msg.getIn().position(0);
        msg.getIn().clear();

        Msg mirror = msg.getMirror(false);

        if (hasPipeline()) {
            // increment counter for pipeline
            context.incrementReferenceCounter();
        }
        auxExecutor.execute(msg, msgCallbacks, requestTask);

        if (hasPipeline()) {
            sendPktToPipeline(mirror);
        }
    }

    private void processPacketRequest(Msg msg) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("Processing packet request. uri=" + DataXceiverBase.this.uri);
        }
        try {

            blockReceiver.processPacket(msg);

            PacketHeader pkt = blockReceiver.getPacketHeader();
            ByteBuffer curCopyBuff = blockReceiver.getCurCopyBuff();

            if (pkt == null) {
                throw new IOException("Unexpected null packet header after processing the packet");
            }

            if (LOG.isDebugEnabled()) {
                LOG.debug("After processing packet: " + pkt + "\nuri=" + DataXceiverBase.this.uri);
            }

            PacketMessageContext pmc = PacketMessageContext.getPacketMessageContext(msg);
            pmc.setPacketData(pkt, curCopyBuff);

            boolean isLastPkt = pmc.isLastPacketInBlock();
            long seqNo = pmc.getSeqno();

            if (!hasPipeline()) {

                if (isLastPkt) {
                    Status status = ERROR;
                    try {
                        blockReceiver.finalizeBlock();
                        status = SUCCESS;
                    } catch (IOException e) {
                        LOG.error("failed on async packet processing (last execution for packet): "
                                + StringUtils.stringifyException(e));
                    } finally {
                        PipelineAck replyAck = new R4HPipelineAck(seqNo, new Status[] { status });
                        pmc.setMessageAck(replyAck);
                    }
                } else {
                    PipelineAck replyAck = new R4HPipelineAck(seqNo, new Status[] { SUCCESS });
                    pmc.setMessageAck(replyAck);
                }
            }

        } catch (Exception e) {
            LOG.error("Got error during packet procesing", e);
            close(true);
        }
    }

    private void processPacketReply(Msg msg) throws IOException {
        boolean readAckfields = false;
        Msg origMsg = msg.getMirror(false);
        PacketMessageContext pmc = PacketMessageContext.getPacketMessageContext(origMsg);
        if (LOG.isTraceEnabled()) {
            LOG.trace("origMsg isMirror=" + origMsg.getIsMirror());
        }
        long expected = PipelineAck.UNKOWN_SEQNO;
        long seqno = PipelineAck.UNKOWN_SEQNO;
        R4HPipelineAck ack = new R4HPipelineAck();
        try {
            msg.getIn().position(0);
            ack.readFields(new DataInputStream(new ByteBufferInputStream(msg.getIn())));
            readAckfields = true;

            if (LOG.isDebugEnabled()) {
                LOG.debug("PacketResponder " + oprHeader.getNumTargets() + " for block " + blockReceiver.getBlock()
                        + " got " + ack);
            }
            msg.getOut().position(0);
            seqno = ack.getSeqno();
            expected = pmc.getSeqno();
            boolean isLastPkt = pmc.isLastPacketInBlock();

            // verify seqno
            if (seqno != expected) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace(String.format("MSG user context: msg ref=%s , context ref=%s",
                            Integer.toHexString(System.identityHashCode(msg)),
                            Integer.toHexString(System.identityHashCode(msg.getUserContext()))));
                }
                throw new IOException("PacketResponder " + oprHeader.getNumTargets() + " for block "
                        + oprHeader.getBlock() + " expected seqno:" + expected + " received:" + seqno);
            }

            // If this is the last packet in block, then close block
            // file and finalize the block before responding success
            if (isLastPkt) {
                blockReceiver.finalizeBlock();
                asyncCloseClientSession();
            }

            PipelineAck ackReadyForReply = preparePipelineAck(origMsg, expected, ack, SUCCESS);
            pmc.setMessageAck(ackReadyForReply);

        } catch (Throwable e) {
            LOG.error("Failed during processing packet reply: " + blockReceiver.getBlock() + " "
                    + oprHeader.getNumTargets() + " Exception " + StringUtils.stringifyException(e));
            if ((clientSession != null) && (!clientSession.getIsClosing())) {
                asyncCloseClientSession();
            }
        } finally {
            if (serverSession != null) {
                if (!readAckfields) {
                    PipelineAck brokenPipelineError = prepareBrokenPipelineAck(origMsg, expected);
                    pmc.setMessageAck(brokenPipelineError);
                } else if ((clientSession == null) && (!pmc.isLastPacketInBlock())) {
                    PipelineAck errorAck = preparePipelineAck(origMsg, expected, ack, ERROR);
                    pmc.setMessageAck(errorAck);
                }
            }

            int refCount = pmc.decrementReferenceCounter();
            if (refCount == 0) {
                msgCallbacks.onMessageAction(origMsg);
            }
        }
    }

    private PipelineAck preparePipelineAck(Msg origMsg, long expectedSeqno, R4HPipelineAck ack, Status s) {
        // construct my ack message
        Status[] replies = null;
        short ackLen = oprHeader.getNumTargets() == 0 ? 0 : ack.getNumOfReplies();
        replies = new Status[1 + ackLen];
        replies[0] = s;
        for (int i = 0; i < ackLen; i++) {
            replies[i + 1] = ack.getReply(i);
        }
        return new R4HPipelineAck(expectedSeqno, replies,
                0/* TODO: update totalAckTimeNanos for specific packet */);
    }

    private PipelineAck prepareBrokenPipelineAck(Msg origMsg, long expectedSeqno) {
        // construct error ack message: when pipeline ack status isn't known then we send back SUCCESS,ERROR as marker
        Status[] replies = null;
        replies = new Status[2];
        replies[0] = SUCCESS;
        replies[1] = ERROR;
        return new R4HPipelineAck(expectedSeqno, replies);
    }

    private void sendPktToPipeline(Msg mirror) {
        mirror.getOut().position(mirror.getOut().limit());
        R4HProtocol.wrappedSendRequest(clientSession, mirror, LOG);
        clientOnFlightNumMsgs++;
    }

    // queue async reply
    private void replyHeaderAck(Msg msg) throws IOException {
        replyHeaderAck(msg, SUCCESS, "");
    }

    // queue async reply
    private void replyHeaderAck(Msg msg, Status status, String firstBadNode) throws IOException {
        msg.getOut().position(0);
        OutputStream replyOut = new ByteBufferOutputStream(msg.getOut());
        // send connect-ack to source for clients and not transfer-RBW/Finalized
        if (LOG.isDebugEnabled()) {
            LOG.debug("Datanode " + oprHeader.getTargets().length + " forwarding connect ack to upstream");
        }
        BlockOpResponseProto.newBuilder().setStatus(status).setFirstBadLink(firstBadNode).build()
                .writeDelimitedTo(replyOut);
        replyOut.flush();
        LOG.info("sending response for src: " + uri);
        spw.queueAsyncReply(DataXceiverBase.this, msg, onFlightMsgs, true);
    }

    private void replyPacketAck(Msg msg, PipelineAck replyAck, boolean breakEventLoop, boolean lastPacketInBlock)
            throws IOException {
        msg.getOut().clear();
        OutputStream replyOut = new ByteBufferOutputStream(msg.getOut());
        replyAck.write(replyOut);
        replyOut.flush();

        if (LOG.isDebugEnabled()) {
            LOG.debug("queue ack reply for async response : " + replyAck + "\nuri=" + uri);
        }
        spw.queueAsyncReply(DataXceiverBase.this, msg, onFlightMsgs, breakEventLoop);
    }

    private boolean hasPipeline() {
        return (oprHeader != null && oprHeader.getNumTargets() > 0);
    }

    private void completePacket(Msg msg) {
        PacketMessageContext pmc = PacketMessageContext.getPacketMessageContext(msg);
        spw.getIOBufferSupplier().returnBuffer(pmc.getIOBuffer());

        if (pmc.getMessageAck().isSuccess()
                && pmc.getUpdatedOffsetInBlock() > blockReceiver.getReplicaInfo().getBytesAcked()) {
            blockReceiver.getReplicaInfo().setBytesAcked(pmc.getUpdatedOffsetInBlock());
        }

        try {
            replyPacketAck(msg, pmc.getMessageAck(), pmc.isLastPacketInBlock() || pmc.getSyncBlock(),
                    pmc.isLastPacketInBlock());
        } catch (IOException e) {
            LOG.error("Failed on submiting a reply ack: " + StringUtils.stringifyException(e));
            if ((clientSession != null) && (!clientSession.getIsClosing())) {
                asyncCloseClientSession();
            }
        }

        if (pmc.isLastPacketInBlock()) {
            asyncReturnAuxillaryExecutortoPool();
        }

    }

    private void replyHeaderPipelineAck(Msg msg, Status mirrorInStatus, String firstBadLink) throws IOException {
        if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) {
            LOG.info("Datanode " + oprHeader.getTargets().length
                    + " forwarding connect ack to upstream firstbadlink is " + firstBadLink + "\nuri=" + uri);
        }
        msg.getOut().clear();
        BlockOpResponseProto protobuff = BlockOpResponseProto.newBuilder().setStatus(mirrorInStatus)
                .setFirstBadLink(firstBadLink).build();
        protobuff.writeDelimitedTo(new ByteBufferOutputStream(msg.getOut()));
        spw.queueAsyncReply(DataXceiverBase.this, msg, onFlightMsgs, true);
    }

    ServerSession getSessionServer() {
        return this.serverSession;
    }

    void close() {
        close(false);
    }

    void close(boolean async) {
        Runnable closeRunnable = new Runnable() {

            @Override
            public void run() {
                if ((DataXceiverBase.this.clientSession != null)
                        && (!DataXceiverBase.this.clientSession.getIsClosing())) {
                    LOG.info("Closing mirror client session: " + DataXceiverBase.this.clientSession);
                    clientSessionCloseEventExpected = true;
                    DataXceiverBase.this.clientSession.close();
                }

                if ((serverSession != null) && (!serverSession.getIsClosing())) {
                    LOG.info("Closing server session: " + DataXceiverBase.this.serverSession);
                    serverSession.close();
                }
            }
        };

        if (async) {
            spw.queueAsyncRunnable(closeRunnable);
        } else {
            closeRunnable.run();
        }
    }

    @Override
    public String toString() {
        return String.format("DataXceiver{SS='%s', SC='%s'}", serverSession,
                (clientSession == null) ? "-" : clientSession);
    }

    // TODO: this method is copied from original DXC. Need to consider reuse instead, maybe by inheritance+reflection
    private boolean checkAccess(final boolean reply, final ExtendedBlock blk, final Token<BlockTokenIdentifier> t,
            final Op op, final BlockTokenSecretManager.AccessMode mode, final Msg msg) throws IOException {
        if (dnBridge.isBlockTokenEnabled()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Checking block access token for block '" + blk.getBlockId() + "' with mode '" + mode
                        + "'");
            }
            try {
                dnBridge.getBlockPoolTokenSecretManager().checkAccess(t, null, blk, mode);
            } catch (InvalidToken e) {
                if (reply) {
                    final OutputStream replyOut = new ByteBufferOutputStream(msg.getOut());
                    BlockOpResponseProto.Builder resp = BlockOpResponseProto.newBuilder()
                            .setStatus(ERROR_ACCESS_TOKEN);
                    if (mode == BlockTokenSecretManager.AccessMode.WRITE) {
                        DatanodeRegistration dnR = dnBridge.getDNRegistrationForBP(blk.getBlockPoolId());
                        // NB: Unconditionally using the xfer addr w/o hostname
                        resp.setFirstBadLink(dnR.getXferAddr());
                    }
                    resp.build().writeDelimitedTo(replyOut);
                    replyOut.flush();
                    spw.queueAsyncReply(DataXceiverBase.this, msg, onFlightMsgs, true);
                }
                LOG.error("Block token verification failed: op=" + op + ", serverSession=" + serverSession
                        + ", message=" + e.getLocalizedMessage());
                return false;// FAILED
            }
        }
        return true; // SUCCESS
    }

    private void asyncCloseServerSession() {
        spw.queueAsyncRunnable(new Runnable() {

            @Override
            public void run() {
                if ((DataXceiverBase.this.serverSession != null)
                        && (!DataXceiverBase.this.serverSession.getIsClosing())) {
                    DataXceiverBase.this.serverSession.close();
                }
            }
        });
    }

    private void asyncCloseClientSession() {
        final ClientSession cs = DataXceiverBase.this.clientSession;
        if (cs != null) {
            spw.queueAsyncRunnable(new Runnable() {

                @Override
                public void run() {
                    if (!cs.getIsClosing()) {
                        clientSessionCloseEventExpected = true;
                        cs.close();
                    }
                }
            });
        }
    }

    public String getUri() {
        return this.uri;
    }

    private void asyncReturnAuxillaryExecutortoPool() {
        spw.queueAsyncRunnable(new Runnable() {

            @Override
            public void run() {
                if (!returnedAuxillaryExecutorToPool) {
                    returnAuxillaryExecutortoPool(false);
                }
            }

        });
    }

    private void returnAuxillaryExecutortoPool(boolean needAuxThreadInit) {
        returnedAuxillaryExecutorToPool = true;
        spw.decermentSessionsCounter();
        dxcs.returnAuxillaryExecutortoPool(auxExecutor, needAuxThreadInit);
    }

    static protected CachingStrategy getCachingStrategy(CachingStrategyProto strategy) {
        Boolean dropBehind = strategy.hasDropBehind() ? strategy.getDropBehind() : null;
        Long readahead = strategy.hasReadahead() ? strategy.getReadahead() : null;
        return new CachingStrategy(dropBehind, readahead);
    }

    // Sometimes, creating a BlockReciver is accompanied by a failure of the DURefreshThread (du: cannot access error)
    // which lead to expected shutdown of the connection. another attempt to create the BlockRevicer should prevent this shutdown.
    // For more info see HADOOP-8640.
    private void createBlockReciver(DataInputStream in, int attempts) throws NoSuchFieldException,
            NoSuchMethodException, IllegalArgumentException, IllegalAccessException, IOException {
        try {
            blockReceiver = new R4HBlockReceiver(spw.getIOBufferSupplier(), oprHeader, in, serverSession.toString(),
                    dnBridge, ioExecutor, msgCallbacks);
        } catch (IOException e) {
            attempts--;
            LOG.warn(String.format("Got exception in BlockReceiver constructor - %d attempts remaining", attempts),
                    e);
            if (attempts <= 0) {
                LOG.error("Failed on BlockReceiver creation, no attempts remaining");
                throw e;
            } else {
                createBlockReciver(in, attempts);
            }
        }
    }

    abstract void parseOpWriteBlock(DataInputStream in) throws IOException;

    abstract void senderWriteBlock(DataOutputStream out, ExtendedBlock origBlk) throws IOException;
}