Java tutorial
/* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ package bftsmart.tom.core; import bftsmart.communication.ServerCommunicationSystem; import bftsmart.consensus.Decision; import bftsmart.consensus.Epoch; import bftsmart.consensus.Consensus; import bftsmart.consensus.TimestampValuePair; import bftsmart.consensus.messages.ConsensusMessage; import bftsmart.consensus.messages.MessageFactory; import bftsmart.consensus.roles.Acceptor; import bftsmart.reconfiguration.ServerViewController; import bftsmart.statemanagement.StateManager; import bftsmart.tom.core.messages.TOMMessage; import bftsmart.tom.leaderchange.RequestsTimer; import bftsmart.tom.leaderchange.CollectData; import bftsmart.tom.leaderchange.LCManager; import bftsmart.tom.leaderchange.LCMessage; import bftsmart.tom.leaderchange.CertifiedDecision; import bftsmart.tom.util.BatchBuilder; import bftsmart.tom.util.BatchReader; import bftsmart.tom.util.Logger; import bftsmart.tom.util.TOMUtil; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.security.MessageDigest; import java.security.SignedObject; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.logging.Level; import org.apache.commons.codec.binary.Base64; /** * * This class implements the synchronization phase described in * Joao Sousa's 'From Byzantine Consensus to BFT state machine replication: a latency-optimal transformation' (May 2012) * * This class implements all optimizations described at the end of the paper * * @author joao */ public class Synchronizer { // out of context messages related to the leader change are stored here private final HashSet<LCMessage> outOfContextLC; // Manager of the leader change private final LCManager lcManager; //Total order layer private final TOMLayer tom; // Stuff from TOMLayer that this object needs private final RequestsTimer requestsTimer; private final ExecutionManager execManager; private final ServerViewController controller; private final BatchBuilder bb; private final ServerCommunicationSystem communication; private final StateManager stateManager; private final Acceptor acceptor; private final MessageDigest md; // Attributes to temporarely store synchronization info // if state transfer is required for synchronization private int tempRegency = -1; private CertifiedDecision tempLastHighestCID = null; private HashSet<SignedObject> tempSignedCollects = null; private byte[] tempPropose = null; private int tempBatchSize = -1; private boolean tempIAmLeader = false; public Synchronizer(TOMLayer tom) { this.tom = tom; this.requestsTimer = this.tom.requestsTimer; this.execManager = this.tom.execManager; this.controller = this.tom.controller; this.bb = this.tom.bb; this.communication = this.tom.getCommunication(); this.stateManager = this.tom.stateManager; this.acceptor = this.tom.acceptor; this.md = this.tom.md; this.outOfContextLC = new HashSet<>(); this.lcManager = new LCManager(this.tom, this.controller, this.md); } public LCManager getLCManager() { return lcManager; } /** * This method is called when there is a timeout and the request has already * been forwarded to the leader * * @param requestList List of requests that the replica wanted to order but * didn't manage to */ public void triggerTimeout(List<TOMMessage> requestList) { ObjectOutputStream out = null; ByteArrayOutputStream bos = new ByteArrayOutputStream(); int regency = lcManager.getNextReg(); requestsTimer.stopTimer(); requestsTimer.Enabled(false); // still not in the leader change phase? if (lcManager.getNextReg() == lcManager.getLastReg()) { lcManager.setNextReg(lcManager.getLastReg() + 1); // define next timestamp regency = lcManager.getNextReg(); // update variable // store messages to be ordered lcManager.setCurrentRequestTimedOut(requestList); // store information about messages that I'm going to send lcManager.addStop(regency, this.controller.getStaticConf().getProcessId()); //execManager.stop(); // stop consensus execution //Get requests that timed out and the requests received in STOP messages //and add those STOPed requests to the client manager addSTOPedRequestsToClientManager(); List<TOMMessage> messages = getRequestsToRelay(); try { // serialize content to send in STOP message out = new ObjectOutputStream(bos); if (messages != null && messages.size() > 0) { //TODO: If this is null, then there was no timeout nor STOP messages. //What to do? byte[] serialized = bb.makeBatch(messages, 0, 0, controller); out.writeBoolean(true); out.writeObject(serialized); } else { out.writeBoolean(false); System.out.println( "(Synchronizer.triggerTimeout) Strange... did not include any request in my STOP message for regency " + regency); } byte[] payload = bos.toByteArray(); out.flush(); bos.flush(); out.close(); bos.close(); // send STOP-message System.out.println("(Synchronizer.triggerTimeout) sending STOP message to install regency " + regency + " with " + (messages != null ? messages.size() : 0) + " request(s) to relay"); LCMessage stop = new LCMessage(this.controller.getStaticConf().getProcessId(), TOMUtil.STOP, regency, payload); requestsTimer.setSTOP(regency, stop); // make replica re-transmit the stop message until a new regency is installed communication.send(this.controller.getCurrentViewOtherAcceptors(), stop); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } finally { try { out.close(); bos.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } } } processOutOfContextSTOPs(regency); // the replica might have received STOPs // that were out of context at the time they // were received, but now can be processed startSynchronization(regency); // evaluate STOP messages } // Processes STOP messages that were not process upon reception, because they were // ahead of the replica's expected regency private void processOutOfContextSTOPs(int regency) { Logger.println( "(Synchronizer.processOutOfContextSTOPs) Checking if there are out of context STOPs for regency " + regency); Set<LCMessage> stops = getOutOfContextLC(TOMUtil.STOP, regency); if (stops.size() > 0) { System.out.println("(Synchronizer.processOutOfContextSTOPs) Processing " + stops.size() + " out of context STOPs for regency " + regency); } else { Logger.println( "(Synchronizer.processOutOfContextSTOPs) No out of context STOPs for regency " + regency); } for (LCMessage m : stops) { TOMMessage[] requests = deserializeTOMMessages(m.getPayload()); // store requests that came with the STOP message lcManager.addRequestsFromSTOP(requests); // store information about the STOP message lcManager.addStop(regency, m.getSender()); } } // Processes STOPDATA messages that were not process upon reception, because they were // ahead of the replica's expected regency private void processSTOPDATA(LCMessage msg, int regency) { //TODO: It is necessary to verify the proof of the last decided consensus and the signature of the state of the current consensus! CertifiedDecision lastData = null; SignedObject signedCollect = null; int last = -1; byte[] lastValue = null; Set<ConsensusMessage> proof = null; ByteArrayInputStream bis; ObjectInputStream ois; try { // deserialize the content of the message bis = new ByteArrayInputStream(msg.getPayload()); ois = new ObjectInputStream(bis); if (ois.readBoolean()) { // content of the last decided cid last = ois.readInt(); lastValue = (byte[]) ois.readObject(); proof = (Set<ConsensusMessage>) ois.readObject(); //TODO: Proof is missing! } lastData = new CertifiedDecision(msg.getSender(), last, lastValue, proof); lcManager.addLastCID(regency, lastData); signedCollect = (SignedObject) ois.readObject(); ois.close(); bis.close(); lcManager.addCollect(regency, signedCollect); int bizantineQuorum = (controller.getCurrentViewN() + controller.getCurrentViewF()) / 2; int cftQuorum = (controller.getCurrentViewN()) / 2; // Did I already got messages from a Byzantine/Crash quorum, // related to the last cid as well as for the current? boolean conditionBFT = (controller.getStaticConf().isBFT() && lcManager.getLastCIDsSize(regency) > bizantineQuorum && lcManager.getCollectsSize(regency) > bizantineQuorum); boolean conditionCFT = (lcManager.getLastCIDsSize(regency) > cftQuorum && lcManager.getCollectsSize(regency) > cftQuorum); if (conditionBFT || conditionCFT) { catch_up(regency); } } catch (IOException ex) { ex.printStackTrace(System.err); } catch (ClassNotFoundException ex) { ex.printStackTrace(System.err); } } // Processes SYNC messages that were not process upon reception, because they were // ahead of the replica's expected regency private void processSYNC(byte[] payload, int regency) { CertifiedDecision lastHighestCID = null; int currentCID = -1; HashSet<SignedObject> signedCollects = null; byte[] propose = null; int batchSize = -1; ByteArrayInputStream bis; ObjectInputStream ois; try { // deserialization of the message content bis = new ByteArrayInputStream(payload); ois = new ObjectInputStream(bis); lastHighestCID = (CertifiedDecision) ois.readObject(); signedCollects = (HashSet<SignedObject>) ois.readObject(); propose = (byte[]) ois.readObject(); batchSize = ois.readInt(); lcManager.setCollects(regency, signedCollects); currentCID = lastHighestCID.getCID() + 1; // Is the predicate "sound" true? Is the certificate for LastCID valid? if (lcManager.sound(lcManager.selectCollects(regency, currentCID)) && (!controller.getStaticConf().isBFT() || lcManager.hasValidProof(lastHighestCID))) { finalise(regency, lastHighestCID, signedCollects, propose, batchSize, false); } ois.close(); bis.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } catch (ClassNotFoundException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } } // Fetches synchronization messages that were not process upon reception, // because they were ahead of the replica's expected regency private Set<LCMessage> getOutOfContextLC(int type, int regency) { HashSet<LCMessage> result = new HashSet<>(); for (LCMessage m : outOfContextLC) { if (m.getType() == type && m.getReg() == regency) { result.add(m); } } outOfContextLC.removeAll(result); // avoid memory leaks return result; } // Deserializes requests that were included in STOP messages private TOMMessage[] deserializeTOMMessages(byte[] playload) { ByteArrayInputStream bis; ObjectInputStream ois; TOMMessage[] requests = null; try { // deserialize the content of the STOP message bis = new ByteArrayInputStream(playload); ois = new ObjectInputStream(bis); boolean hasReqs = ois.readBoolean(); if (hasReqs) { // Store requests that the other replica did not manage to order //TODO: The requests have to be verified! byte[] temp = (byte[]) ois.readObject(); BatchReader batchReader = new BatchReader(temp, controller.getStaticConf().getUseSignatures() == 1); requests = batchReader.deserialiseRequests(controller); } ois.close(); bis.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } catch (ClassNotFoundException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } return requests; } // Get requests that timed out and the requests received in STOP messages private List<TOMMessage> getRequestsToRelay() { List<TOMMessage> messages = lcManager.getCurrentRequestTimedOut(); if (messages == null) { messages = new LinkedList<>(); } // Include requests from STOP messages in my own STOP message List<TOMMessage> messagesFromSTOP = lcManager.getRequestsFromSTOP(); if (messagesFromSTOP != null) { for (TOMMessage m : messagesFromSTOP) { if (!messages.contains(m)) { messages.add(m); } } } Logger.println("(Synchronizer.getRequestsToRelay) I need to relay " + messages.size() + " requests"); return messages; } //adds requests received via STOP messages to the client manager private void addSTOPedRequestsToClientManager() { List<TOMMessage> messagesFromSTOP = lcManager.getRequestsFromSTOP(); if (messagesFromSTOP != null) { Logger.println( "(Synchronizer.addRequestsToClientManager) Adding to client manager the requests contained in STOP messages"); for (TOMMessage m : messagesFromSTOP) { tom.requestReceived(m); } } } /** * Remove all STOP messages being retransmitted up until * the specified regency * @param regency The regency up to which STOP retransmission should be canceled */ public void removeSTOPretransmissions(int regency) { Set<Integer> timers = requestsTimer.getTimers(); for (int t : timers) { if (t <= regency) requestsTimer.stopSTOP(t); } } // this method is called when a timeout occurs or when a STOP message is recevied private void startSynchronization(int nextReg) { boolean condition; ObjectOutputStream out = null; ByteArrayOutputStream bos = null; if (this.controller.getStaticConf().isBFT()) { condition = lcManager.getStopsSize(nextReg) > this.controller.getCurrentViewF(); } else { condition = lcManager.getStopsSize(nextReg) > 0; } // Ask to start the synchronizations phase if enough messages have been received already if (condition && lcManager.getNextReg() == lcManager.getLastReg()) { Logger.println("(Synchronizer.startSynchronization) initialize synch phase"); requestsTimer.Enabled(false); requestsTimer.stopTimer(); lcManager.setNextReg(lcManager.getLastReg() + 1); // define next timestamp int regency = lcManager.getNextReg(); // store information about message I am going to send lcManager.addStop(regency, this.controller.getStaticConf().getProcessId()); //execManager.stop(); // stop execution of consensus //Get requests that timed out and the requests received in STOP messages //and add those STOPed requests to the client manager addSTOPedRequestsToClientManager(); List<TOMMessage> messages = getRequestsToRelay(); try { // serialize conent to send in the STOP message bos = new ByteArrayOutputStream(); out = new ObjectOutputStream(bos); // Do I have messages to send in the STOP message? if (messages != null && messages.size() > 0) { //TODO: If this is null, there was no timeout nor STOP messages. //What shall be done then? out.writeBoolean(true); byte[] serialized = bb.makeBatch(messages, 0, 0, controller); out.writeObject(serialized); } else { out.writeBoolean(false); System.out.println( "(Synchronizer.startSynchronization) Strange... did not include any request in my STOP message for regency " + regency); } out.flush(); bos.flush(); byte[] payload = bos.toByteArray(); out.close(); bos.close(); // send message STOP System.out.println("(Synchronizer.startSynchronization) sending STOP message to install regency " + regency + " with " + (messages != null ? messages.size() : 0) + " request(s) to relay"); LCMessage stop = new LCMessage(this.controller.getStaticConf().getProcessId(), TOMUtil.STOP, regency, payload); requestsTimer.setSTOP(regency, stop); // make replica re-transmit the stop message until a new regency is installed communication.send(this.controller.getCurrentViewOtherAcceptors(), stop); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } finally { try { out.close(); bos.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } } } if (this.controller.getStaticConf().isBFT()) { condition = lcManager.getStopsSize(nextReg) > (2 * this.controller.getCurrentViewF()); } else { condition = lcManager.getStopsSize(nextReg) > this.controller.getCurrentViewF(); } // Did the synchronization phase really started? //if (lcManager.getStopsSize(nextReg) > this.reconfManager.getQuorum2F() && lcManager.getNextReg() > lcManager.getLastReg()) { if (condition && lcManager.getNextReg() > lcManager.getLastReg()) { if (!execManager.stopped()) execManager.stop(); // stop consensus execution if more than f replicas sent a STOP message Logger.println("(Synchronizer.startSynchronization) installing regency " + lcManager.getNextReg()); lcManager.setLastReg(lcManager.getNextReg()); // define last timestamp int regency = lcManager.getLastReg(); // avoid memory leaks lcManager.removeStops(nextReg); lcManager.clearCurrentRequestTimedOut(); lcManager.clearRequestsFromSTOP(); requestsTimer.Enabled(true); requestsTimer.setShortTimeout(-1); requestsTimer.startTimer(); //int leader = regency % this.reconfManager.getCurrentViewN(); // new leader int leader = lcManager.getNewLeader(); int in = tom.getInExec(); // cid to execute int last = tom.getLastExec(); // last cid decided execManager.setNewLeader(leader); // If I am not the leader, I have to send a STOPDATA message to the elected leader if (leader != this.controller.getStaticConf().getProcessId()) { try { // serialize content of the STOPDATA message bos = new ByteArrayOutputStream(); out = new ObjectOutputStream(bos); Consensus cons = null; // content of the last decided CID if (last > -1) cons = execManager.getConsensus(last); //Do I have info on my last executed consensus? if (cons != null && cons.getDecisionEpoch() != null && cons.getDecisionEpoch().propValue != null) { out.writeBoolean(true); out.writeInt(last); //byte[] decision = exec.getLearner().getDecision(); byte[] decision = cons.getDecisionEpoch().propValue; Set<ConsensusMessage> proof = cons.getDecisionEpoch().getProof(); out.writeObject(decision); out.writeObject(proof); // TODO: WILL BE NECESSARY TO ADD A PROOF!!! } else { out.writeBoolean(false); ////// THIS IS TO CATCH A BUG!!!!! if (last > -1) { System.out.println("[DEBUG INFO FOR LAST CID #1]"); if (cons == null) { if (last > -1) System.out.println("No consensus instance for cid " + last); } else if (cons.getDecisionEpoch() == null) { System.out.println("No decision epoch for cid " + last); } else { System.out.println( "epoch for cid: " + last + ": " + cons.getDecisionEpoch().toString()); if (cons.getDecisionEpoch().propValue == null) { System.out.println("No propose for cid " + last); } else { System.out.println( "Propose hash for cid " + last + ": " + Base64.encodeBase64String( tom.computeHash(cons.getDecisionEpoch().propValue))); } } } } if (in > -1) { // content of cid in execution cons = execManager.getConsensus(in); //cons.incEts(); // make the consensus advance to the next epoch cons.setETS(regency); // make the consensus advance to the next epoch //int ets = cons.getEts(); //cons.createEpoch(ets, controller); cons.createEpoch(regency, controller); //Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + ets); Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + regency); TimestampValuePair quorumWrites; if (cons.getQuorumWrites() != null) { quorumWrites = cons.getQuorumWrites(); } else { quorumWrites = new TimestampValuePair(0, new byte[0]); } HashSet<TimestampValuePair> writeSet = cons.getWriteSet(); //CollectData collect = new CollectData(this.controller.getStaticConf().getProcessId(), in, ets, quorumWrites, writeSet); CollectData collect = new CollectData(this.controller.getStaticConf().getProcessId(), in, regency, quorumWrites, writeSet); SignedObject signedCollect = tom.sign(collect); out.writeObject(signedCollect); } else { cons = execManager.getConsensus(last + 1); //cons.incEts(); // make the consensus advance to the next epoch cons.setETS(regency); // make the consensus advance to the next epoch //int ets = cons.getEts(); //cons.createEpoch(ets, controller); cons.createEpoch(regency, controller); //Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + ets); Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + regency); //CollectData collect = new CollectData(this.controller.getStaticConf().getProcessId(), last + 1, ets, new TimestampValuePair(0, new byte[0]), new HashSet<TimestampValuePair>()); CollectData collect = new CollectData(this.controller.getStaticConf().getProcessId(), last + 1, regency, new TimestampValuePair(0, new byte[0]), new HashSet<TimestampValuePair>()); SignedObject signedCollect = tom.sign(collect); out.writeObject(signedCollect); } out.flush(); bos.flush(); byte[] payload = bos.toByteArray(); out.close(); bos.close(); int[] b = new int[1]; b[0] = leader; System.out .println("(Synchronizer.startSynchronization) sending STOPDATA of regency " + regency); // send message SYNC to the new leader communication.send(b, new LCMessage(this.controller.getStaticConf().getProcessId(), TOMUtil.STOPDATA, regency, payload)); //TODO: Turn on timeout again? } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } finally { try { out.close(); bos.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } } // the replica might have received a SYNC that was out of context at the time it was received, but now can be processed Set<LCMessage> sync = getOutOfContextLC(TOMUtil.SYNC, regency); Logger.println( "(Synchronizer.startSynchronization) Checking if there are out of context SYNC for regency " + regency); if (sync.size() > 0) { System.out.println( "(Synchronizer.startSynchronization) Processing out of context SYNC for regency " + regency); } else { Logger.println( "(Synchronizer.startSynchronization) No out of context SYNC for regency " + regency); } for (LCMessage m : sync) { if (m.getSender() == execManager.getCurrentLeader()) { processSYNC(m.getPayload(), regency); return; // makes no sense to continue, since there is only one SYNC message } } } else { // If leader, I will store information that I would send in a SYNC message Logger.println("(Synchronizer.startSynchronization) I'm the leader for this new regency"); CertifiedDecision lastDec = null; CollectData collect = null; Consensus cons = null; //Content of the last decided CID if (last > -1) cons = execManager.getConsensus(last); //Do I have info on my last executed consensus? if (cons != null && cons.getDecisionEpoch() != null && cons.getDecisionEpoch().propValue != null) { //byte[] decision = exec.getLearner().getDecision(); byte[] decision = cons.getDecisionEpoch().propValue; Set<ConsensusMessage> proof = cons.getDecisionEpoch().getProof(); lastDec = new CertifiedDecision(this.controller.getStaticConf().getProcessId(), last, decision, proof); // TODO: WILL BE NECESSARY TO ADD A PROOF!!!?? } else { lastDec = new CertifiedDecision(this.controller.getStaticConf().getProcessId(), last, null, null); ////// THIS IS TO CATCH A BUG!!!!! if (last > -1) { System.out.println("[DEBUG INFO FOR LAST CID #2]"); if (cons == null) { if (last > -1) System.out.println("No consensus instance for cid " + last); } else if (cons.getDecisionEpoch() == null) { System.out.println("No decision epoch for cid " + last); } else { System.out .println("epoch for cid: " + last + ": " + cons.getDecisionEpoch().toString()); } if (cons.getDecisionEpoch().propValue == null) { System.out.println("No propose for cid " + last); } else { System.out.println("Propose hash for cid " + last + ": " + Base64 .encodeBase64String(tom.computeHash(cons.getDecisionEpoch().propValue))); } } } lcManager.addLastCID(regency, lastDec); if (in > -1) { // content of cid being executed cons = execManager.getConsensus(in); //cons.incEts(); // make the consensus advance to the next epoch cons.setETS(regency); // make the consensus advance to the next epoch //int ets = cons.getEts(); //cons.createEpoch(ets, controller); cons.createEpoch(regency, controller); //Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + ets); Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + regency); TimestampValuePair quorumWrites; if (cons.getQuorumWrites() != null) { quorumWrites = cons.getQuorumWrites(); } else { quorumWrites = new TimestampValuePair(0, new byte[0]); } HashSet<TimestampValuePair> writeSet = cons.getWriteSet(); //collect = new CollectData(this.controller.getStaticConf().getProcessId(), in, ets, quorumWrites, writeSet); collect = new CollectData(this.controller.getStaticConf().getProcessId(), in, regency, quorumWrites, writeSet); } else { cons = execManager.getConsensus(last + 1); //cons.incEts(); // make the consensus advance to the next epoch cons.setETS(regency); // make the consensus advance to the next epoch //int ets = cons.getEts(); //cons.createEpoch(ets, controller); cons.createEpoch(regency, controller); //Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + ets); Logger.println("(Synchronizer.startSynchronization) incrementing ets of consensus " + cons.getId() + " to " + regency); //collect = new CollectData(this.controller.getStaticConf().getProcessId(), last + 1, ets, new TimestampValuePair(0, new byte[0]), new HashSet<TimestampValuePair>()); collect = new CollectData(this.controller.getStaticConf().getProcessId(), last + 1, regency, new TimestampValuePair(0, new byte[0]), new HashSet<TimestampValuePair>()); } SignedObject signedCollect = tom.sign(collect); lcManager.addCollect(regency, signedCollect); // the replica might have received STOPDATAs that were out of context at the time they were received, but now can be processed Set<LCMessage> stopdatas = getOutOfContextLC(TOMUtil.STOPDATA, regency); Logger.println( "(Synchronizer.startSynchronization) Checking if there are out of context STOPDATAs for regency " + regency); if (stopdatas.size() > 0) { System.out.println("(Synchronizer.startSynchronization) Processing " + stopdatas.size() + " out of context STOPDATAs for regency " + regency); } else { Logger.println("(Synchronizer.startSynchronization) No out of context STOPDATAs for regency " + regency); } for (LCMessage m : stopdatas) { processSTOPDATA(m, regency); } } } } /** * This method is called by the MessageHandler each time it received * messages related to the leader change * * @param msg Message received from the other replica */ public void deliverTimeoutRequest(LCMessage msg) { switch (msg.getType()) { case TOMUtil.STOP: { // message STOP System.out.println("(Synchronizer.deliverTimeoutRequest) Last regency: " + lcManager.getLastReg() + ", next regency: " + lcManager.getNextReg()); // this message is for the next leader change? if (msg.getReg() == lcManager.getLastReg() + 1) { Logger.println("(Synchronizer.deliverTimeoutRequest) received regency change request"); TOMMessage[] requests = deserializeTOMMessages(msg.getPayload()); // store requests that came with the STOP message lcManager.addRequestsFromSTOP(requests); // store information about the message STOP lcManager.addStop(msg.getReg(), msg.getSender()); processOutOfContextSTOPs(msg.getReg()); // the replica might have received STOPs // that were out of context at the time they // were received, but now can be processed startSynchronization(msg.getReg()); // evaluate STOP messages } else if (msg.getReg() > lcManager.getLastReg()) { // send STOP to out of context if // it is for a future regency System.out.println( "(Synchronizer.deliverTimeoutRequest) Keeping STOP message as out of context for regency " + msg.getReg()); outOfContextLC.add(msg); } else { System.out.println("(Synchronizer.deliverTimeoutRequest) Discarding STOP message"); } } break; case TOMUtil.STOPDATA: { // STOPDATA messages int regency = msg.getReg(); System.out.println("(Synchronizer.deliverTimeoutRequest) Last regency: " + lcManager.getLastReg() + ", next regency: " + lcManager.getNextReg()); // Am I the new leader, and am I expecting this messages? if (regency == lcManager.getLastReg() && this.controller.getStaticConf().getProcessId() == execManager .getCurrentLeader()/*(regency % this.reconfManager.getCurrentViewN())*/) { Logger.println("(Synchronizer.deliverTimeoutRequest) I'm the new leader and I received a STOPDATA"); processSTOPDATA(msg, regency); } else if (msg.getReg() > lcManager.getLastReg()) { // send STOPDATA to out of context if // it is for a future regency System.out.println( "(Synchronizer.deliverTimeoutRequest) Keeping STOPDATA message as out of context for regency " + msg.getReg()); outOfContextLC.add(msg); } else { System.out.println("(Synchronizer.deliverTimeoutRequest) Discarding STOPDATA message"); } } break; case TOMUtil.SYNC: { // message SYNC int regency = msg.getReg(); System.out.println("(Synchronizer.deliverTimeoutRequest) Last regency: " + lcManager.getLastReg() + ", next regency: " + lcManager.getNextReg()); // I am expecting this sync? boolean isExpectedSync = (regency == lcManager.getLastReg() && regency == lcManager.getNextReg()); // Is this sync what I wanted to get in the previous iteration of the synchoronization phase? boolean islateSync = (regency == lcManager.getLastReg() && regency == (lcManager.getNextReg() - 1)); //Did I already sent a stopdata in this iteration? boolean sentStopdata = (lcManager.getStopsSize(lcManager.getNextReg()) == 0); //if 0, I already purged the stops, //which I only do when I am about to //send the stopdata // I am (or was) waiting for this message, and did I received it from the new leader? if ((isExpectedSync || // Expected case (islateSync && !sentStopdata)) && // might happen if I timeout before receiving the SYNC (msg.getSender() == execManager.getCurrentLeader())) { //if (msg.getReg() == lcManager.getLastReg() && // msg.getReg() == lcManager.getNextReg() && msg.getSender() == lm.getCurrentLeader()/*(regency % this.reconfManager.getCurrentViewN())*/) { processSYNC(msg.getPayload(), regency); } else if (msg.getReg() > lcManager.getLastReg()) { // send SYNC to out of context if // it is for a future regency System.out.println( "(Synchronizer.deliverTimeoutRequest) Keeping SYNC message as out of context for regency " + msg.getReg()); outOfContextLC.add(msg); } else { System.out.println("(Synchronizer.deliverTimeoutRequest) Discarding SYNC message"); } } break; } } // this method is used to verify if the leader can make the message catch-up // and also sends the message private void catch_up(int regency) { Logger.println("(Synchronizer.catch_up) verify STOPDATA info"); ObjectOutputStream out = null; ByteArrayOutputStream bos = null; CertifiedDecision lastHighestCID = lcManager.getHighestLastCID(regency); int currentCID = lastHighestCID.getCID() + 1; HashSet<SignedObject> signedCollects = null; byte[] propose = null; int batchSize = -1; // normalize the collects and apply to them the predicate "sound" if (lcManager.sound(lcManager.selectCollects(regency, currentCID))) { Logger.println("(Synchronizer.catch_up) sound predicate is true"); signedCollects = lcManager.getCollects(regency); // all original collects that the replica has received Decision dec = new Decision(-1); // the only purpose of this object is to obtain the batchsize, // using code inside of createPropose() propose = tom.createPropose(dec); batchSize = dec.batchSize; try { // serialization of the CATCH-UP message bos = new ByteArrayOutputStream(); out = new ObjectOutputStream(bos); out.writeObject(lastHighestCID); //TODO: Missing: serialization of the proof? out.writeObject(signedCollects); out.writeObject(propose); out.writeInt(batchSize); out.flush(); bos.flush(); byte[] payload = bos.toByteArray(); out.close(); bos.close(); System.out.println("(Synchronizer.catch_up) sending SYNC message for regency " + regency); // send the CATCH-UP message communication.send(this.controller.getCurrentViewOtherAcceptors(), new LCMessage( this.controller.getStaticConf().getProcessId(), TOMUtil.SYNC, regency, payload)); finalise(regency, lastHighestCID, signedCollects, propose, batchSize, true); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } finally { try { out.close(); bos.close(); } catch (IOException ex) { ex.printStackTrace(); java.util.logging.Logger.getLogger(TOMLayer.class.getName()).log(Level.SEVERE, null, ex); } } } } //This method is invoked by the state transfer protocol to notify the replica // that it can end synchronization public void resumeLC() { Consensus cons = execManager.getConsensus(tempLastHighestCID.getCID()); Epoch e = cons.getLastEpoch(); int ets = cons.getEts(); if (e == null || e.getTimestamp() != ets) { e = cons.createEpoch(ets, controller); } else { e.clear(); } byte[] hash = tom.computeHash(tempLastHighestCID.getDecision()); e.propValueHash = hash; e.propValue = tempLastHighestCID.getDecision(); e.deserializedPropValue = tom.checkProposedValue(tempLastHighestCID.getDecision(), false); finalise(tempRegency, tempLastHighestCID, tempSignedCollects, tempPropose, tempBatchSize, tempIAmLeader); } // this method is called on all replicas, and serves to verify and apply the // information sent in the catch-up message private void finalise(int regency, CertifiedDecision lastHighestCID, HashSet<SignedObject> signedCollects, byte[] propose, int batchSize, boolean iAmLeader) { int currentCID = lastHighestCID.getCID() + 1; Logger.println("(Synchronizer.finalise) final stage of LC protocol"); int me = this.controller.getStaticConf().getProcessId(); Consensus cons = null; Epoch e = null; if (tom.getLastExec() + 1 < lastHighestCID.getCID()) { // is this a delayed replica? System.out.println( "(Synchronizer.finalise) NEEDING TO USE STATE TRANSFER!! (" + lastHighestCID.getCID() + ")"); tempRegency = regency; tempLastHighestCID = lastHighestCID; tempSignedCollects = signedCollects; tempPropose = propose; tempBatchSize = batchSize; tempIAmLeader = iAmLeader; execManager.getStoppedMsgs().add(acceptor.getFactory().createPropose(currentCID, 0, propose)); stateManager.requestAppState(lastHighestCID.getCID()); return; } /*else if (tom.getLastExec() + 1 == lastHighestCID.getCID()) { // Is this replica still executing the last decided consensus? System.out.println("(Synchronizer.finalise) I'm still at the CID before the most recent one!!! (" + lastHighestCID.getCID() + ")"); cons = execManager.getConsensus(lastHighestCID.getCID()); e = cons.getLastEpoch(); int ets = cons.getEts(); if (e == null || e.getTimestamp() != ets) { e = cons.createEpoch(ets, controller); } else { e.clear(); } byte[] hash = tom.computeHash(lastHighestCID.getCIDDecision()); e.propValueHash = hash; e.propValue = lastHighestCID.getCIDDecision(); e.deserializedPropValue = tom.checkProposedValue(lastHighestCID.getCIDDecision(), false); cons.decided(e, true); // pass the decision to the delivery thread }*/ // install proof of the last decided consensus cons = execManager.getConsensus(lastHighestCID.getCID()); e = null; for (ConsensusMessage cm : lastHighestCID.getConsMessages()) { if (e == null) e = cons.getEpoch(cm.getEpoch(), true, controller); if (e.getTimestamp() != cm.getEpoch()) { System.out.println( "(Synchronizer.finalise) Strange... proof of last decided consensus contains messages from more than just one epoch"); e = cons.getEpoch(cm.getEpoch(), true, controller); } e.addToProof(cm); if (cm.getType() == MessageFactory.ACCEPT) { e.setAccept(cm.getSender(), cm.getValue()); } else if (cm.getType() == MessageFactory.WRITE) { e.setWrite(cm.getSender(), cm.getValue()); } } if (e != null) { System.out.println( "(Synchronizer.finalise) Installed proof of last decided consensus " + lastHighestCID.getCID()); byte[] hash = tom.computeHash(lastHighestCID.getDecision()); e.propValueHash = hash; e.propValue = lastHighestCID.getDecision(); e.deserializedPropValue = tom.checkProposedValue(lastHighestCID.getDecision(), false); // Is this replica still executing the last decided consensus? if (tom.getLastExec() + 1 == lastHighestCID.getCID()) { System.out.println("(Synchronizer.finalise) I'm still at the CID before the most recent one!!! (" + lastHighestCID.getCID() + ")"); cons.decided(e, true); } else { cons.decided(e, false); } } else { System.out.println("(Synchronizer.finalise) I did not install any proof of last decided consensus " + lastHighestCID.getCID()); } cons = null; e = null; // get a value that satisfies the predicate "bind" byte[] tmpval = null; HashSet<CollectData> selectedColls = lcManager.selectCollects(signedCollects, currentCID, regency); tmpval = lcManager.getBindValue(selectedColls); Logger.println("(Synchronizer.finalise) Trying to find a binded value"); // If such value does not exist, obtain the value written by the new leader if (tmpval == null && lcManager.unbound(selectedColls)) { Logger.println("(Synchronizer.finalise) did not found a value that might have already been decided"); tmpval = propose; } else { Logger.println("(Synchronizer.finalise) found a value that might have been decided"); } if (tmpval != null) { // did I manage to get some value? Logger.println("(Synchronizer.finalise) resuming normal phase"); lcManager.removeCollects(regency); // avoid memory leaks // stop the re-transmission of the STOP message for all regencies up to this one removeSTOPretransmissions(regency); cons = execManager.getConsensus(currentCID); e = cons.getLastEpoch(); int ets = cons.getEts(); //Update current consensus with latest ETS. This may be necessary //if I 'jumped' to a consensus instance ahead of the one I was executing //int currentETS = lcManager.getETS(currentCID, selectedColls); //if (currentETS > ets) { if (regency > ets) { //System.out.println("(Synchronizer.finalise) Updating consensus' ETS after SYNC (from " + ets + " to " + currentETS +")"); System.out.println("(Synchronizer.finalise) Updating consensus' ETS after SYNC (from " + ets + " to " + regency + ")"); /*do { cons.incEts(); } while (cons.getEts() != currentETS);*/ cons.setETS(regency); //cons.createEpoch(currentETS, controller); cons.createEpoch(regency, controller); e = cons.getLastEpoch(); } // Make sure the epoch is created /*if (e == null || e.getTimestamp() != ets) { e = cons.createEpoch(ets, controller); } else { e.clear(); }*/ if (e == null || e.getTimestamp() != regency) { e = cons.createEpoch(regency, controller); } else { e.clear(); } /********* LEADER CHANGE CODE ********/ cons.removeWritten(tmpval); cons.addWritten(tmpval); /*************************************/ byte[] hash = tom.computeHash(tmpval); e.propValueHash = hash; e.propValue = tmpval; e.deserializedPropValue = tom.checkProposedValue(tmpval, false); if (cons.getDecision().firstMessageProposed == null) { if (e.deserializedPropValue != null && e.deserializedPropValue.length > 0) { cons.getDecision().firstMessageProposed = e.deserializedPropValue[0]; } else { cons.getDecision().firstMessageProposed = new TOMMessage(); // to avoid null pointer } } if (this.controller.getStaticConf().isBFT()) { e.setWrite(me, hash); } else { e.setAccept(me, hash); /********* LEADER CHANGE CODE ********/ Logger.println("(Synchronizer.finalise) [CFT Mode] Setting consensus " + currentCID + " QuorumWrite tiemstamp to " + e.getConsensus().getEts() + " and value " + Arrays.toString(hash)); e.getConsensus().setQuorumWrites(hash); /*************************************/ } // resume normal operation execManager.restart(); //leaderChanged = true; tom.setInExec(currentCID); if (iAmLeader) { Logger.println("(Synchronizer.finalise) wake up proposer thread"); tom.imAmTheLeader(); } // waik up the thread that propose values in normal operation // send a WRITE/ACCEPT message to the other replicas if (this.controller.getStaticConf().isBFT()) { System.out.println("(Synchronizer.finalise) sending WRITE message for CID " + currentCID + ", timestamp " + e.getTimestamp() + ", value " + Arrays.toString(e.propValueHash)); communication.send(this.controller.getCurrentViewOtherAcceptors(), acceptor.getFactory().createWrite(currentCID, e.getTimestamp(), e.propValueHash)); } else { System.out.println("(Synchronizer.finalise) sending ACCEPT message for CID " + currentCID + ", timestamp " + e.getTimestamp() + ", value " + Arrays.toString(e.propValueHash)); communication.send(this.controller.getCurrentViewOtherAcceptors(), acceptor.getFactory().createAccept(currentCID, e.getTimestamp(), e.propValueHash)); } } else { Logger.println("(Synchronizer.finalise) sync phase failed for regency" + regency); } } }