List of usage examples for java.lang InterruptedException InterruptedException
public InterruptedException()
InterruptedException with no detail message. From source file:LinkedTransferQueue.java
public E take() throws InterruptedException { Object e = xfer(null, WAIT, 0); if (e != null) { return cast(e); }/*from w ww . j av a 2 s. c o m*/ Thread.interrupted(); throw new InterruptedException(); }
From source file:LinkedTransferQueue.java
public E poll(long timeout, TimeUnit unit) throws InterruptedException { Object e = xfer(null, TIMEOUT, unit.toNanos(timeout)); if (e != null || !Thread.interrupted()) { return cast(e); }/*from w ww . j av a 2 s. co m*/ throw new InterruptedException(); }
From source file:canreg.client.dataentry.Import.java
/** * * @param task/* w w w. j a v a2s. c o m*/ * @param doc * @param map * @param file * @param server * @param io * @return * @throws java.sql.SQLException * @throws java.rmi.RemoteException * @throws canreg.server.database.RecordLockedException */ public static boolean importFile(Task<Object, String> task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { //public static boolean importFile(canreg.client.gui.management.CanReg4MigrationInternalFrame.MigrationTask task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { boolean success = false; Set<String> noNeedToLookAtPatientVariables = new TreeSet<String>(); noNeedToLookAtPatientVariables.add(io.getPatientIDVariableName()); noNeedToLookAtPatientVariables.add(io.getPatientRecordIDVariableName()); String firstNameVariableName = io.getFirstNameVariableName(); String sexVariableName = io.getSexVariableName(); CSVParser parser = null; CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(io.getSeparator()); int linesToRead = io.getMaxLines(); HashMap mpCodes = new HashMap(); int numberOfLinesRead = 0; Map<String, Integer> nameSexTable = server.getNameSexTables(); try { // FileInputStream fis = new FileInputStream(file); // BufferedReader bsr = new BufferedReader(new InputStreamReader(fis, io.getFileCharset())); // Logger.getLogger(Import.class.getName()).log(Level.CONFIG, "Name of the character encoding {0}"); int numberOfRecordsInFile = canreg.common.Tools.numberOfLinesInFile(file.getAbsolutePath()); if (linesToRead > 0) { linesToRead = Math.min(numberOfRecordsInFile, linesToRead); } else { linesToRead = numberOfRecordsInFile; } parser = CSVParser.parse(file, io.getFileCharset(), format); for (CSVRecord csvRecord : parser) { numberOfLinesRead++; // We allow for null tasks... boolean needToSavePatientAgain = true; int patientDatabaseRecordID = -1; if (task != null) { task.firePropertyChange("progress", (numberOfLinesRead - 1) * 100 / linesToRead, (numberOfLinesRead) * 100 / linesToRead); } // Build patient part Patient patient = new Patient(); for (int i = 0; i < map.size(); i++) { Relation rel = map.get(i); if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("patient")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { patient.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { patient.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with patient part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // debugOut(patient.toString()); // Build tumour part Tumour tumour = new Tumour(); for (canreg.client.dataentry.Relation rel : map) { if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("tumour")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { tumour.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { tumour.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with tumour part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // Build source part Set<Source> sources = Collections.synchronizedSet(new LinkedHashSet<Source>()); Source source = new Source(); for (canreg.client.dataentry.Relation rel : map) { if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase(Globals.SOURCE_TABLE_NAME)) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { source.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { source.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with source part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } sources.add(source); tumour.setSources(sources); // debugOut(tumour.toString()); // add patient to the database Object patientID = patient.getVariable(io.getPatientIDVariableName()); Object patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); if (patientID == null) { // save the record to get the new patientID; patientDatabaseRecordID = server.savePatient(patient); patient = (Patient) server.getRecord(patientDatabaseRecordID, Globals.PATIENT_TABLE_NAME, false); patientID = patient.getVariable(io.getPatientIDVariableName()); patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); } if (io.isDataFromPreviousCanReg()) { // set update date for the patient the same as for the tumour Object updateDate = tumour.getVariable(io.getTumourUpdateDateVariableName()); patient.setVariable(io.getPatientUpdateDateVariableName(), updateDate); // Set the patientID the same as the tumourID initially // Object tumourSequence = tumour.getVariable(io.getTumourSequenceVariableName()); Object tumourSequence = "1"; String tumourSequenceString = tumourSequence + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; // If this is a multiple primary tumour... String mpCodeString = (String) tumour.getVariable(io.getMultiplePrimaryVariableName()); if (mpCodeString != null && mpCodeString.length() > 0) { patientID = lookUpPatientID(mpCodeString, patientID, mpCodes); // rebuild sequenceNumber Tumour[] tumours = new Tumour[0]; try { tumours = CanRegClientApp.getApplication() .getTumourRecordsBasedOnPatientID(patientID + "", false); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } tumourSequenceString = (tumours.length + 1) + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; Patient[] oldPatients = null; try { oldPatients = CanRegClientApp.getApplication().getPatientRecordsByID((String) patientID, false); } catch (RemoteException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SecurityException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (RecordLockedException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } for (Patient oldPatient : oldPatients) { if (!Tools.newRecordContainsNewInfo(patient, oldPatient, noNeedToLookAtPatientVariables)) { needToSavePatientAgain = false; patient = oldPatient; patientRecordID = oldPatient.getVariable(io.getPatientRecordIDVariableName()); } } } Object tumourID = patientRecordID + "" + tumourSequenceString; // patient.setVariable(io.getPatientIDVariableName(), patientID); tumour.setVariable(io.getTumourIDVariablename(), tumourID); // And store the record ID patient.setVariable(io.getPatientRecordIDVariableName(), patientRecordID); // Set the patient ID number on the tumour tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); // Set the deprecated flag to 0 - no obsolete records from CR4 tumour.setVariable(io.getObsoleteTumourFlagVariableName(), "0"); patient.setVariable(io.getObsoletePatientFlagVariableName(), "0"); } // Set the name in the firstName database String sex = (String) patient.getVariable(sexVariableName); if (sex != null && sex.length() > 0) { Integer sexCode = Integer.parseInt(sex); String firstNames = (String) patient.getVariable(firstNameVariableName); if (firstNames != null) { String[] firstNamesArray = firstNames.split(" "); for (String firstName : firstNamesArray) { if (firstName != null && firstName.trim().length() > 0) { // here we use the locale specific toUpperCase Integer registeredSexCode = nameSexTable.get(firstName); if (registeredSexCode == null) { NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, false); nameSexTable.put(firstName, sexCode); } else if (registeredSexCode != sexCode) { if (registeredSexCode != 9) { sexCode = 9; NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, true); nameSexTable.remove(firstName); nameSexTable.put(firstName, sexCode); } } } } } } if (needToSavePatientAgain) { if (patientDatabaseRecordID > 0) { server.editPatient(patient); } else { patientDatabaseRecordID = server.savePatient(patient); } } if (patient != null && tumour != null) { String icd10 = (String) tumour.getVariable(io.getICD10VariableName()); if (icd10 == null || icd10.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICD10, patient, tumour); tumour.setVariable(io.getICD10VariableName(), conversionResult[0].getValue()); } String iccc = (String) tumour.getVariable(io.getICCCVariableName()); if (iccc == null || iccc.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICCC3, patient, tumour); tumour.setVariable(io.getICCCVariableName(), conversionResult[0].getValue()); } } if (tumour.getVariable(io.getPatientIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); } if (tumour.getVariable(io.getPatientRecordIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); } int tumourDatabaseIDNumber = server.saveTumour(tumour); if (Thread.interrupted()) { //We've been interrupted: no more importing. throw new InterruptedException(); } } task.firePropertyChange("finished", null, null); success = true; } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (InterruptedException ex) { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Interupted on line: " + (numberOfLinesRead + 1) + ". ", ex); success = true; } catch (IndexOutOfBoundsException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } finally { if (parser != null) { try { parser.close(); } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } } } return success; }
From source file:org.kitodo.export.ExportDms.java
/** * Starts copying all directories configured as export folder. * * @param process/* w ww . j av a2 s .c o m*/ * object * @param destination * the destination directory * @throws InterruptedException * if the user clicked stop on the thread running the export DMS * task * */ private void directoryDownload(Process process, URI destination) throws IOException, InterruptedException { Collection<Subfolder> processDirs = process.getProject().getFolders().parallelStream() .filter(Folder::isCopyFolder).map(folder -> new Subfolder(process, folder)) .collect(Collectors.toList()); VariableReplacer variableReplacer = new VariableReplacer(null, null, process, null); for (Subfolder processDir : processDirs) { URI dstDir = destination.resolve(variableReplacer.replace(processDir.getFolder().getRelativePath())); fileService.createDirectories(dstDir); Collection<URI> srcs = processDir.listContents().values(); int progress = 0; for (URI src : srcs) { if (Objects.nonNull(exportDmsTask)) { exportDmsTask.setWorkDetail(fileService.getFileName(src)); } fileService.copyFileToDirectory(src, dstDir); if (Objects.nonNull(exportDmsTask)) { exportDmsTask .setProgress((int) ((progress++ + 1) * 98d / processDirs.size() / srcs.size() + 1)); if (exportDmsTask.isInterrupted()) { throw new InterruptedException(); } } } } }
From source file:net.sourceforge.vulcan.core.support.ProjectBuilderTest.java
public void testRunnablePhaseHandlesInterruptedException() throws Exception { doRunnablePhaseTest(new RunnableCallback() { public void run(BuildContext buildContext) throws Exception { throw new InterruptedException(); }/*from w w w. jav a 2s . c o m*/ }); }
From source file:nl.nn.adapterframework.batch.StreamTransformerPipe.java
private Object transform(String streamId, BufferedReader reader, IPipeLineSession session, ParameterResolutionContext prc) throws PipeRunException { String rawRecord = null;//from w w w . ja v a 2s .c om int linenumber = 0; int counter = 0; StringBuffer sb = null; List prevParsedRecord = null; IRecordHandler prevHandler = null; IRecordHandlerManager currentManager = initialManager.getRecordFactoryUsingFilename(session, streamId); try { openDocument(session, streamId, prc); while ((rawRecord = reader.readLine()) != null) { if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } linenumber++; // remember linenumber for exception handler if (StringUtils.isEmpty(rawRecord)) { continue; // ignore empty line } // get handlers for current line RecordHandlingFlow flow = currentManager.getRecordHandler(session, rawRecord); if (flow == null) { log.debug("<no flow>: " + rawRecord); continue; // ignore line for which no handlers are registered } else { //log.debug("flow ["+flow.getRecordKey()+"] openBlockBeforeLine ["+flow.getOpenBlockBeforeLine()+"]"); } IResultHandler resultHandler = flow.getResultHandler(); closeBlock(session, resultHandler, streamId, flow, flow.getCloseBlockBeforeLine(), "closeBlockBeforeLine of flow [" + flow.getRecordKey() + "]", prc); String obbl = null; if (flow.getOpenBlockBeforeLineNumber() > 0) { if (counter % flow.getOpenBlockBeforeLineNumber() == 0) { obbl = flow.getOpenBlockBeforeLine(); } } else { obbl = flow.getOpenBlockBeforeLine(); } openBlock(session, resultHandler, streamId, flow, obbl, prc); if (isStoreOriginalBlock()) { if (resultHandler instanceof ResultBlock2Sender) { // If session does not contain a previous block, it never existed, or has been removed by closing the block. // In both cases a new block has just started if (!session.containsKey(originalBlockKey)) { sb = new StringBuffer(); } if (sb.length() > 0) { sb.append(System.getProperty("line.separator")); } sb.append(rawRecord); // already put the block in the session, also if the block is not yet complete. session.put(originalBlockKey, sb.toString()); } } IRecordHandler curHandler = flow.getRecordHandler(); if (curHandler != null) { log.debug("manager [" + currentManager.getName() + "] key [" + flow.getRecordKey() + "] record handler [" + curHandler.getName() + "] line [" + linenumber + "]: " + rawRecord); // there is a record handler, so transform the line List parsedRecord = curHandler.parse(session, rawRecord); Object result = curHandler.handleRecord(session, parsedRecord, prc); counter++; // if there is a result handler, write the transformed result if (result != null && resultHandler != null) { boolean recordTypeChanged = curHandler.isNewRecordType(session, curHandler.equals(prevHandler), prevParsedRecord, parsedRecord); log.debug("manager [" + currentManager.getName() + "] key [" + flow.getRecordKey() + "] record handler [" + curHandler.getName() + "] recordTypeChanged [" + recordTypeChanged + "]"); if (recordTypeChanged && prevHandler != null && resultHandler.isBlockByRecordType()) { String prevRecordType = prevHandler.getRecordType(prevParsedRecord); log.debug("record handler [" + prevHandler.getName() + "] result handler [" + resultHandler.getName() + "] closing block for record type [" + prevRecordType + "]"); closeBlock(session, resultHandler, streamId, flow, prevRecordType, "record type change", prc); } // the hasPrefix() call allows users use a suffix without a prefix. // The suffix is then only written at the end of the file. if (recordTypeChanged && resultHandler.hasPrefix()) { if (prevHandler != null) { resultHandler.closeRecordType(session, streamId, prc); } resultHandler.openRecordType(session, streamId, prc); } if (recordTypeChanged && resultHandler.isBlockByRecordType()) { String recordType = curHandler.getRecordType(parsedRecord); log.debug("record handler [" + curHandler.getName() + "] result handler [" + resultHandler.getName() + "] opening block [" + recordType + "]"); openBlock(session, resultHandler, streamId, flow, recordType, prc); } resultHandler.handleResult(session, streamId, flow.getRecordKey(), result, prc); } prevParsedRecord = parsedRecord; prevHandler = curHandler; } else { log.debug("manager [" + currentManager.getName() + "] key [" + flow.getRecordKey() + "], no record handler: " + rawRecord); } closeBlock(session, resultHandler, streamId, flow, flow.getCloseBlockAfterLine(), "closeBlockAfterLine of flow [" + flow.getRecordKey() + "]", prc); openBlock(session, resultHandler, streamId, flow, flow.getOpenBlockAfterLine(), prc); // get the manager for the next record currentManager = flow.getNextRecordHandlerManager(); } return finalizeResult(session, streamId, false, prc); } catch (Exception e) { try { finalizeResult(session, streamId, true, prc); } catch (Throwable t) { log.error("Unexpected error during finalizeResult of [" + streamId + "]", t); } throw new PipeRunException(this, "Error while transforming [" + streamId + "] at or after line [" + linenumber + "]", e); } finally { closeDocument(session, streamId, prc); } }
From source file:de.sub.goobi.export.dms.ExportDms.java
/** * Download image./*from w w w .ja v a 2s. c o m*/ * * @param process * object * @param userHome * File * @param atsPpnBand * String * @param ordnerEndung * String */ public void imageDownload(Process process, URI userHome, String atsPpnBand, final String ordnerEndung) throws IOException, InterruptedException { /* * dann den Ausgangspfad ermitteln */ URI tifOrdner = serviceManager.getProcessService().getImagesTifDirectory(true, process); /* * jetzt die Ausgangsordner in die Zielordner kopieren */ if (fileService.fileExist(tifOrdner) && fileService.getSubUris(tifOrdner).size() > 0) { URI zielTif = userHome.resolve(File.separator + atsPpnBand + ordnerEndung); /* bei Agora-Import einfach den Ordner anlegen */ if (process.getProject().isUseDmsImport()) { if (!fileService.fileExist(zielTif)) { fileService.createDirectory(userHome, atsPpnBand + ordnerEndung); } } else { /* * wenn kein Agora-Import, dann den Ordner mit * Benutzerberechtigung neu anlegen */ User myUser = (User) Helper.getManagedBeanValue("#{LoginForm.myBenutzer}"); try { fileService.createDirectoryForUser(zielTif, myUser.getLogin()); } catch (Exception e) { if (exportDmsTask != null) { exportDmsTask.setException(e); } else { Helper.setFehlerMeldung("Export canceled, error", "could not create destination directory"); } logger.error("could not create destination directory", e); if (e instanceof IOException) { throw (IOException) e; } else if (e instanceof InterruptedException) { throw (InterruptedException) e; } else if (e instanceof RuntimeException) { throw (RuntimeException) e; } else { throw new UndeclaredThrowableException(e); } } } /* jetzt den eigentlichen Kopiervorgang */ ArrayList<URI> dateien = fileService.getSubUris(Helper.dataFilter, tifOrdner); for (int i = 0; i < dateien.size(); i++) { if (exportDmsTask != null) { exportDmsTask.setWorkDetail(fileService.getFileName(dateien.get(i))); } URI meinZiel = zielTif.resolve(File.separator + fileService.getFileName(dateien.get(i))); fileService.copyFile(dateien.get(i), meinZiel); if (exportDmsTask != null) { exportDmsTask.setProgress((int) ((i + 1) * 98d / dateien.size() + 1)); if (exportDmsTask.isInterrupted()) { throw new InterruptedException(); } } } if (exportDmsTask != null) { exportDmsTask.setWorkDetail(null); } } }
From source file:com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketCloudApiClient.java
/** * {@inheritDoc}//from ww w . j a v a 2 s. c om */ @NonNull @Override public List<BitbucketRepositoryHook> getWebHooks() throws IOException, InterruptedException { List<BitbucketRepositoryHook> repositoryHooks = new ArrayList<>(); int pageNumber = 1; UriTemplate template = UriTemplate.fromTemplate(REPO_URL_TEMPLATE + "/hooks{?page,pagelen}") .set("owner", owner).set("repo", repositoryName).set("page", pageNumber).set("pagelen", 50); String url = template.expand(); try { String response = getRequest(url); BitbucketRepositoryHooks page = parsePaginatedRepositoryHooks(response); repositoryHooks.addAll(page.getValues()); while (page.getNext() != null) { if (Thread.interrupted()) { throw new InterruptedException(); } pageNumber++; response = getRequest(url = template.set("page", pageNumber).expand()); page = parsePaginatedRepositoryHooks(response); repositoryHooks.addAll(page.getValues()); } return repositoryHooks; } catch (IOException e) { throw new IOException("I/O error when parsing response from URL: " + url, e); } }
From source file:com.chinamobile.bcbsp.bspcontroller.BSPController.java
/** * BSPcontroller construction method with BSP conf and controller identifier. * @param conf//from ww w.j a va2s . c om * BSP system configuration * @param identifier * BSpcontroller identifier. * @throws IOException * @throws InterruptedException */ @SuppressWarnings("static-access") BSPController(BSPConfiguration conf, String identifier) throws IOException, InterruptedException { this.conf = conf; this.controllerIdentifier = identifier; String zkAddress = conf.get(Constants.ZOOKEEPER_QUORUM) + ":" + conf.getInt(Constants.ZOOKEPER_CLIENT_PORT, Constants.DEFAULT_ZOOKEPER_CLIENT_PORT); // this.zk = new ZooKeeper(zkAddress, Constants.SESSION_TIME_OUT, this); bspzk = new BSPZookeeperImpl(zkAddress, Constants.SESSION_TIME_OUT, this); // determine the role of the BspController this.determineBspControllerRole(); LOG.info("the BspController's role is :" + this.role); this.haLogOperator = new HDFSOperator(); if (this.role.equals(BspControllerRole.ACTIVE)) { LOG.info(" before this.mfl=new MonitorFaultLog();"); this.mfl = new MonitorFaultLog(); this.haLogOperator.deleteFile(conf.get(Constants.BC_BSP_HA_LOG_DIR)); this.haLogOperator.createFile(conf.get(Constants.BC_BSP_HA_LOG_DIR) + Constants.BC_BSP_HA_SUBMIT_LOG); } // Create the scheduler and init scheduler services Class<? extends StaffScheduler> schedulerClass = conf.getClass("bsp.master.taskscheduler", SimpleStaffScheduler.class, StaffScheduler.class); this.staffScheduler = ReflectionUtils.newInstance(schedulerClass, conf); this.staffScheduler.setRole(role); String host = getAddress(conf).getHostName(); int port = getAddress(conf).getPort(); LOG.info("RPC BSPController: host " + host + " port " + port); this.controllerServer = RPC.getServer(this, host, port, conf); infoPort = conf.getInt("bsp.http.infoserver.port", 40026); // infoPort = 40026; infoServer = new HttpServer("bcbsp", host, infoPort, true, conf); infoServer.setAttribute("bcbspController", this); // starting webserver infoServer.start(); BSPController.HEART_BEAT_TIMEOUT = conf.getLong(Constants.HEART_BEAT_TIMEOUT, 5000); BSPController.PAUSE_TIMEOUT = conf.getInt(Constants.SLEEP_TIMEOUT, 10000); this.cto = new CheckTimeOut(); // Baoxing Yang added this.pf = new ProcessFault(); while (!Thread.currentThread().isInterrupted()) { try { if (bspfs == null) { // fs = FileSystem.get(conf); bspfs = new BSPFileSystemImpl(conf); } // Clean up the system dir, which will only work if hdfs is out // of safe mode. if (bspsystemDir == null) { // systemDir = new Path(getSystemDir()); bspsystemDir = new BSPPathImpl(getSystemDir()); } if (role.equals(BspControllerRole.ACTIVE)) { LOG.info("Cleaning up the system directory"); LOG.info(bspsystemDir); // fs.delete(systemDir, true); bspfs.delete(bspsystemDir.getSystemDir(), true); // if (FileSystem.mkdirs(fs, systemDir, new FsPermission( // SYSTEM_DIR_PERMISSION))) if (BSPFileSystemImpl.mkdirs(bspfs.getFs(), bspsystemDir.getSystemDir(), new BSPFspermissionImpl(1).getFp())) { break; } LOG.error("Mkdirs failed to create " + bspsystemDir); } else { break; } } catch (AccessControlException ace) { LOG.warn("Failed to operate on bsp.system.dir (" + bspsystemDir + ") because of permissions."); LOG.warn("Manually delete the bsp.system.dir (" + bspsystemDir + ") and then start the BSPController."); LOG.warn("Bailing out ... "); throw ace; } catch (IOException ie) { //LOG.error("problem cleaning system directory: " + bspsystemDir, ie); throw new RuntimeException("problem cleaning system directory: " + bspsystemDir, ie); } Thread.sleep(FS_ACCESS_RETRY_PERIOD); } if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } //deleteLocalFiles(Constants.BC_BSP_LOCAL_SUBDIR_CONTROLLER); }
From source file:skewtune.mapreduce.STJobTracker.java
@SuppressWarnings("unchecked") STJobTracker(final JobConf conf, String jobtrackerIndentifier) throws IOException, InterruptedException { // find the owner of the process // get the desired principal to load String keytabFilename = conf.get(JTConfig.JT_KEYTAB_FILE); UserGroupInformation.setConfiguration(conf); if (keytabFilename != null) { String desiredUser = conf.get(JTConfig.JT_USER_NAME, System.getProperty("user.name")); UserGroupInformation.loginUserFromKeytab(desiredUser, keytabFilename); mrOwner = UserGroupInformation.getLoginUser(); } else {// www . j av a2s.co m mrOwner = UserGroupInformation.getCurrentUser(); } supergroup = conf.get(MR_SUPERGROUP, "supergroup"); LOG.info("Starting jobtracker with owner as " + mrOwner.getShortUserName() + " and supergroup as " + supergroup); long secretKeyInterval = conf.getLong(MRConfig.DELEGATION_KEY_UPDATE_INTERVAL_KEY, MRConfig.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); long tokenMaxLifetime = conf.getLong(MRConfig.DELEGATION_TOKEN_MAX_LIFETIME_KEY, MRConfig.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); long tokenRenewInterval = conf.getLong(MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); secretManager = new DelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, DELEGATION_TOKEN_GC_INTERVAL); secretManager.startThreads(); // // Grab some static constants // NUM_HEARTBEATS_IN_SECOND = conf.getInt(JT_HEARTBEATS_IN_SECOND, DEFAULT_NUM_HEARTBEATS_IN_SECOND); if (NUM_HEARTBEATS_IN_SECOND < MIN_NUM_HEARTBEATS_IN_SECOND) { NUM_HEARTBEATS_IN_SECOND = DEFAULT_NUM_HEARTBEATS_IN_SECOND; } HEARTBEATS_SCALING_FACTOR = conf.getFloat(JT_HEARTBEATS_SCALING_FACTOR, DEFAULT_HEARTBEATS_SCALING_FACTOR); if (HEARTBEATS_SCALING_FACTOR < MIN_HEARTBEATS_SCALING_FACTOR) { HEARTBEATS_SCALING_FACTOR = DEFAULT_HEARTBEATS_SCALING_FACTOR; } // whether to dump or not every heartbeat message even when DEBUG is enabled dumpHeartbeat = conf.getBoolean(JT_HEARTBEATS_DUMP, false); // This is a directory of temporary submission files. We delete it // on startup, and can delete any files that we're done with this.conf = conf; JobConf jobConf = new JobConf(conf); // Set ports, start RPC servers, setup security policy etc. InetSocketAddress addr = getAddress(conf); this.localMachine = addr.getHostName(); this.port = addr.getPort(); int handlerCount = conf.getInt(JT_IPC_HANDLER_COUNT, 10); this.interTrackerServer = RPC.getServer(SkewTuneClientProtocol.class, this, addr.getHostName(), addr.getPort(), handlerCount, false, conf, secretManager); if (LOG.isDebugEnabled()) { Properties p = System.getProperties(); for (Iterator it = p.keySet().iterator(); it.hasNext();) { String key = (String) it.next(); String val = p.getProperty(key); LOG.debug("Property '" + key + "' is " + val); } } InetSocketAddress infoSocAddr = NetUtils .createSocketAddr(conf.get(JT_HTTP_ADDRESS, String.format("%s:0", this.localMachine))); String infoBindAddress = infoSocAddr.getHostName(); int tmpInfoPort = infoSocAddr.getPort(); this.startTime = System.currentTimeMillis(); infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf); infoServer.setAttribute("job.tracker", this); infoServer.addServlet("jobcompletion", "/completion", JobCompletionServlet.class); infoServer.addServlet("taskspeculation", "/speculation", SpeculationEventServlet.class); infoServer.addServlet("skewreport", "/skew", SkewReportServlet.class); infoServer.addServlet("tasksplit", "/split/*", SplitTaskServlet.class); infoServer.addServlet("tasksplitV2", "/splitV2/*", SplitTaskV2Servlet.class); infoServer.start(); this.trackerIdentifier = jobtrackerIndentifier; // The rpc/web-server ports can be ephemeral ports... // ... ensure we have the correct info this.port = interTrackerServer.getListenerAddress().getPort(); this.conf.set(JT_IPC_ADDRESS, (this.localMachine + ":" + this.port)); LOG.info("JobTracker up at: " + this.port); this.infoPort = this.infoServer.getPort(); this.conf.set(JT_HTTP_ADDRESS, infoBindAddress + ":" + this.infoPort); LOG.info("JobTracker webserver: " + this.infoServer.getPort()); this.defaultNotificationUrl = String.format("http://%s:%d/completion?jobid=$jobId&status=$jobStatus", infoBindAddress, this.infoPort); LOG.info("JobTracker completion URI: " + defaultNotificationUrl); // this.defaultSpeculationEventUrl = String.format("http://%s:%d/speculation?taskid=$taskId&remainTime=$taskRemainTime",infoBindAddress,this.infoPort); this.defaultSpeculationEventUrl = String.format("http://%s:%d/speculation?jobid=$jobId", infoBindAddress, this.infoPort); LOG.info("JobTracker speculation event URI: " + defaultSpeculationEventUrl); this.defaultSkewReportUrl = String.format("http://%s:%d/skew", infoBindAddress, this.infoPort); LOG.info("JobTracker skew report event URI: " + defaultSkewReportUrl); this.trackerHttp = String.format("http://%s:%d", infoBindAddress, this.infoPort); while (!Thread.currentThread().isInterrupted()) { try { // if we haven't contacted the namenode go ahead and do it if (fs == null) { fs = mrOwner.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return FileSystem.get(conf); } }); } // clean up the system dir, which will only work if hdfs is out // of safe mode if (systemDir == null) { systemDir = new Path(getSystemDir()); } try { FileStatus systemDirStatus = fs.getFileStatus(systemDir); if (!systemDirStatus.getOwner().equals(mrOwner.getShortUserName())) { throw new AccessControlException( "The systemdir " + systemDir + " is not owned by " + mrOwner.getShortUserName()); } if (!systemDirStatus.getPermission().equals(SYSTEM_DIR_PERMISSION)) { LOG.warn("Incorrect permissions on " + systemDir + ". Setting it to " + SYSTEM_DIR_PERMISSION); fs.setPermission(systemDir, new FsPermission(SYSTEM_DIR_PERMISSION)); } else { break; } } catch (FileNotFoundException fnf) { } // ignore } catch (AccessControlException ace) { LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir + ") because of permissions."); LOG.warn("Manually delete the " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir + ") and then start the JobTracker."); LOG.warn("Bailing out ... "); throw ace; } catch (IOException ie) { LOG.info("problem cleaning system directory: " + systemDir, ie); } Thread.sleep(FS_ACCESS_RETRY_PERIOD); } if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } // initialize cluster variable cluster = new Cluster(this.conf); // now create a job client proxy jtClient = (ClientProtocol) RPC.getProxy(ClientProtocol.class, ClientProtocol.versionID, JobTracker.getAddress(conf), mrOwner, this.conf, NetUtils.getSocketFactory(conf, ClientProtocol.class)); new SpeculativeScheduler().start(); // initialize task event fetcher new TaskCompletionEventFetcher().start(); // Same with 'localDir' except it's always on the local disk. asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(conf), conf.getLocalDirs()); asyncDiskService.moveAndDeleteFromEachVolume(SUBDIR); // keep at least one asynchronous worker per CPU core int numProcs = Runtime.getRuntime().availableProcessors(); LOG.info("# of available processors = " + numProcs); int maxFactor = conf.getInt(JT_MAX_ASYNC_WORKER_FACTOR, 2); asyncWorkers = new ThreadPoolExecutor(numProcs, numProcs * maxFactor, 30, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(true), new ThreadPoolExecutor.CallerRunsPolicy()); speculativeSplit = conf.getBoolean(JT_SPECULATIVE_SPLIT, false); }