List of usage examples for java.util.concurrent LinkedBlockingQueue LinkedBlockingQueue
public LinkedBlockingQueue(Collection<? extends E> c)
From source file:com.alibaba.cobar.client.CobarSqlMapClientTemplate.java
private ExecutorService createCustomExecutorService(int poolSize, final String method) { int coreSize = Runtime.getRuntime().availableProcessors(); if (poolSize < coreSize) { coreSize = poolSize;//from w w w . j ava 2s . co m } ThreadFactory tf = new ThreadFactory() { public Thread newThread(Runnable r) { Thread t = new Thread(r, "thread created at CobarSqlMapClientTemplate method [" + method + "]"); t.setDaemon(true); return t; } }; BlockingQueue<Runnable> queueToUse = new LinkedBlockingQueue<Runnable>(coreSize); final ThreadPoolExecutor executor = new ThreadPoolExecutor(coreSize, poolSize, 60, TimeUnit.SECONDS, queueToUse, tf, new ThreadPoolExecutor.CallerRunsPolicy()); return executor; }
From source file:com.streamsets.pipeline.stage.origin.jdbc.cdc.oracle.OracleCDCSource.java
@Override public List<ConfigIssue> init() { List<ConfigIssue> issues = super.init(); errorRecordHandler = new DefaultErrorRecordHandler(getContext()); useLocalBuffering = !getContext().isPreview() && configBean.bufferLocally; if (!hikariConfigBean.driverClassName.isEmpty()) { try {/*from w w w .j a v a2 s. com*/ Class.forName(hikariConfigBean.driverClassName); } catch (ClassNotFoundException e) { LOG.error("Hikari Driver class not found.", e); issues.add(getContext().createConfigIssue(Groups.LEGACY.name(), DRIVER_CLASSNAME, JdbcErrors.JDBC_28, e.toString())); } } issues = hikariConfigBean.validateConfigs(getContext(), issues); if (connection == null) { // For tests, we set a mock connection try { dataSource = jdbcUtil.createDataSourceForRead(hikariConfigBean); connection = dataSource.getConnection(); connection.setAutoCommit(false); } catch (StageException | SQLException e) { LOG.error("Error while connecting to DB", e); issues.add( getContext().createConfigIssue(Groups.JDBC.name(), CONNECTION_STR, JDBC_00, e.toString())); return issues; } } recordQueue = new LinkedBlockingQueue<>(2 * configBean.baseConfigBean.maxBatchSize); String container = configBean.pdb; List<SchemaAndTable> schemasAndTables; try { initializeStatements(); alterSession(); } catch (SQLException ex) { LOG.error("Error while creating statement", ex); issues.add(getContext().createConfigIssue(Groups.JDBC.name(), CONNECTION_STR, JDBC_00, hikariConfigBean.getConnectionString())); } zoneId = ZoneId.of(configBean.dbTimeZone); dateTimeColumnHandler = new DateTimeColumnHandler(zoneId); String commitScnField; BigDecimal scn = null; try { scn = getEndingSCN(); switch (configBean.startValue) { case SCN: if (new BigDecimal(configBean.startSCN).compareTo(scn) > 0) { issues.add(getContext().createConfigIssue(CDC.name(), "oracleCDCConfigBean.startSCN", JDBC_47, scn.toPlainString())); } break; case LATEST: // If LATEST is used, use now() as the startDate and proceed as if a startDate was specified configBean.startDate = nowAtDBTz().format(dateTimeColumnHandler.dateFormatter); // fall-through case DATE: try { LocalDateTime startDate = dateTimeColumnHandler.getDate(configBean.startDate); if (startDate.isAfter(nowAtDBTz())) { issues.add(getContext().createConfigIssue(CDC.name(), "oracleCDCConfigBean.startDate", JDBC_48)); } } catch (DateTimeParseException ex) { LOG.error("Invalid date", ex); issues.add( getContext().createConfigIssue(CDC.name(), "oracleCDCConfigBean.startDate", JDBC_49)); } break; default: throw new IllegalStateException("Unknown start value!"); } } catch (SQLException ex) { LOG.error("Error while getting SCN", ex); issues.add(getContext().createConfigIssue(CREDENTIALS.name(), USERNAME, JDBC_42)); } try (Statement reusedStatement = connection.createStatement()) { int majorVersion = getDBVersion(issues); // If version is 12+, then the check for table presence must be done in an alternate container! if (majorVersion == -1) { return issues; } if (majorVersion >= 12) { if (!StringUtils.isEmpty(container)) { String switchToPdb = "ALTER SESSION SET CONTAINER = " + configBean.pdb; try { reusedStatement.execute(switchToPdb); } catch (SQLException ex) { LOG.error("Error while switching to container: " + container, ex); issues.add(getContext().createConfigIssue(Groups.CREDENTIALS.name(), USERNAME, JDBC_40, container)); return issues; } containerized = true; } } schemasAndTables = new ArrayList<>(); for (SchemaTableConfigBean tables : configBean.baseConfigBean.schemaTableConfigs) { tables.schema = configBean.baseConfigBean.caseSensitive ? tables.schema : tables.schema.toUpperCase(); tables.table = configBean.baseConfigBean.caseSensitive ? tables.table : tables.table.toUpperCase(); if (tables.excludePattern != null) { tables.excludePattern = configBean.baseConfigBean.caseSensitive ? tables.excludePattern : tables.excludePattern.toUpperCase(); } Pattern p = StringUtils.isEmpty(tables.excludePattern) ? null : Pattern.compile(tables.excludePattern); try (ResultSet rs = jdbcUtil.getTableAndViewMetadata(connection, tables.schema, tables.table)) { while (rs.next()) { String schemaName = rs.getString(TABLE_METADATA_TABLE_SCHEMA_CONSTANT); String tableName = rs.getString(TABLE_METADATA_TABLE_NAME_CONSTANT); if (p == null || !p.matcher(tableName).matches()) { schemaName = schemaName.trim(); tableName = tableName.trim(); schemasAndTables.add(new SchemaAndTable(schemaName, tableName)); } } } } validateTablePresence(reusedStatement, schemasAndTables, issues); if (!issues.isEmpty()) { return issues; } for (SchemaAndTable schemaAndTable : schemasAndTables) { try { tableSchemas.put(schemaAndTable, getTableSchema(schemaAndTable)); if (scn != null) { tableSchemaLastUpdate.put(schemaAndTable, scn); } } catch (SQLException ex) { LOG.error("Error while switching to container: " + container, ex); issues.add(getContext().createConfigIssue(Groups.CREDENTIALS.name(), USERNAME, JDBC_50)); } } container = CDB_ROOT; if (majorVersion >= 12) { try { switchContainer.execute(); LOG.info("Switched to CDB$ROOT to start LogMiner."); } catch (SQLException ex) { // Fatal only if we switched to a PDB earlier if (containerized) { LOG.error("Error while switching to container: " + container, ex); issues.add(getContext().createConfigIssue(Groups.CREDENTIALS.name(), USERNAME, JDBC_40, container)); return issues; } // Log it anyway LOG.info("Switching containers failed, ignoring since there was no PDB switch", ex); } } commitScnField = majorVersion >= 11 ? "COMMIT_SCN" : "CSCN"; } catch (SQLException ex) { LOG.error("Error while creating statement", ex); issues.add(getContext().createConfigIssue(Groups.JDBC.name(), CONNECTION_STR, JDBC_00, hikariConfigBean.getConnectionString())); return issues; } final String ddlTracking = shouldTrackDDL ? " + DBMS_LOGMNR.DDL_DICT_TRACKING" : ""; final String readCommitted = useLocalBuffering ? "" : "+ DBMS_LOGMNR.COMMITTED_DATA_ONLY"; this.logMinerProcedure = "BEGIN" + " DBMS_LOGMNR.START_LOGMNR(" + " {}," + " {}," + " OPTIONS => DBMS_LOGMNR." + configBean.dictionary.name() + " + DBMS_LOGMNR.CONTINUOUS_MINE" + readCommitted + " + DBMS_LOGMNR.NO_SQL_DELIMITER" + ddlTracking + ");" + " END;"; final String base = "SELECT SCN, USERNAME, OPERATION_CODE, TIMESTAMP, SQL_REDO, TABLE_NAME, " + commitScnField + ", SEQUENCE#, CSF, XIDUSN, XIDSLT, XIDSQN, RS_ID, SSN, SEG_OWNER, ROLLBACK, ROW_ID " + " FROM V$LOGMNR_CONTENTS" + " WHERE "; final String tableCondition = getListOfSchemasAndTables(schemasAndTables); final String commitRollbackCondition = Utils.format("OPERATION_CODE = {} OR OPERATION_CODE = {}", COMMIT_CODE, ROLLBACK_CODE); final String operationsCondition = "OPERATION_CODE IN (" + getSupportedOperations() + ")"; final String restartNonBufferCondition = Utils.format("((" + commitScnField + " = ? AND SEQUENCE# > ?) OR " + commitScnField + " > ?)" + (shouldTrackDDL ? " OR (OPERATION_CODE = {} AND SCN > ?)" : ""), DDL_CODE); if (useLocalBuffering) { selectString = String.format("%s ((%s AND (%s)) OR (%s))", base, tableCondition, operationsCondition, commitRollbackCondition); } else { selectString = base + " (" + tableCondition + " AND (" + operationsCondition + "))" + "AND (" + restartNonBufferCondition + ")"; } try { initializeLogMnrStatements(); } catch (SQLException ex) { LOG.error("Error while creating statement", ex); issues.add(getContext().createConfigIssue(Groups.JDBC.name(), CONNECTION_STR, JDBC_00, hikariConfigBean.getConnectionString())); } if (configBean.dictionary == DictionaryValues.DICT_FROM_REDO_LOGS) { try { startLogMnrForRedoDict(); } catch (Exception ex) { LOG.warn("Error while attempting to start LogMiner to load dictionary", ex); issues.add(getContext().createConfigIssue(Groups.CDC.name(), "oracleCDCConfigBean.dictionary", JDBC_44, ex)); } } if (useLocalBuffering && configBean.bufferLocation == BufferingValues.ON_DISK) { File tmpDir = new File(System.getProperty("java.io.tmpdir")); String relativePath = getContext().getSdcId() + "/" + getContext().getPipelineId() + "/" + getContext().getStageInfo().getInstanceName(); this.txnBufferLocation = new File(tmpDir, relativePath); try { if (txnBufferLocation.exists()) { FileUtils.deleteDirectory(txnBufferLocation); LOG.info("Deleted " + txnBufferLocation.toString()); } Files.createDirectories(txnBufferLocation.toPath()); LOG.info("Created " + txnBufferLocation.toString()); } catch (IOException ex) { Throwables.propagate(ex); } } if (configBean.bufferLocally) { if (configBean.parseQuery) { parsingExecutor = Executors.newFixedThreadPool(configBean.parseThreadPoolSize, new ThreadFactoryBuilder().setNameFormat("Oracle CDC Origin Parse Thread - %d").build()); } else { parsingExecutor = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setNameFormat("Oracle CDC Origin Parse Thread - %d").build()); } } if (configBean.txnWindow >= configBean.logminerWindow) { issues.add(getContext().createConfigIssue(Groups.CDC.name(), "oracleCDCConfigBean.logminerWindow", JDBC_81)); } version = useLocalBuffering ? VERSION_UNCOMMITTED : VERSION_STR; delay = getContext().createGauge("Read Lag (seconds)"); return issues; }
From source file:ipc.Server.java
/** Constructs a server listening on the named port and address. Parameters passed must * be of the named class. The <code>handlerCount</handlerCount> determines * the number of handler threads that will be used to process calls. * /* ww w .j a v a 2s. c o m*/ */ @SuppressWarnings("unchecked") protected Server(String bindAddress, int port, Class<? extends Writable> paramClass, int handlerCount, Configuration conf, String serverName) throws IOException { this.bindAddress = bindAddress; this.conf = conf; this.port = port; this.paramClass = paramClass; this.handlerCount = handlerCount; this.socketSendBufferSize = 0; this.maxQueueSize = handlerCount * conf.getInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT); this.maxRespSize = conf.getInt(CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY, CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT); this.readThreads = conf.getInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_DEFAULT); this.callQueue = new LinkedBlockingQueue<Call>(maxQueueSize); this.maxIdleTime = 2 * conf.getInt("ipc.client.connection.maxidletime", 1000); this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10); this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000); // this.secretManager = (SecretManager<TokenIdentifier>) secretManager; this.authorize = conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false); // this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); this.isSecurityEnabled = false; // Start the listener here and let it bind to the port listener = new Listener(); this.port = listener.getAddress().getPort(); this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false); // Create the responder here responder = new Responder(); }
From source file:com.hortonworks.hbase.replication.bridge.HBaseServer.java
protected HBaseServer(String bindAddress, int port, Class<? extends Writable> paramClass, int handlerCount, int priorityHandlerCount, Configuration conf, String serverName, int highPriorityLevel) throws IOException { this.bindAddress = bindAddress; this.conf = conf; this.port = port; this.paramClass = paramClass; this.handlerCount = handlerCount; this.priorityHandlerCount = priorityHandlerCount; this.socketSendBufferSize = 0; // temporary backward compatibility String oldMaxQueueSize = this.conf.get("ipc.server.max.queue.size"); if (oldMaxQueueSize == null) { this.maxQueueLength = this.conf.getInt("ipc.server.max.callqueue.length", handlerCount * DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER); } else {//from ww w. j a v a 2s. co m LOG.warn("ipc.server.max.queue.size was renamed " + "ipc.server.max.callqueue.length, " + "please update your configuration"); this.maxQueueLength = Integer.getInteger(oldMaxQueueSize); } this.maxQueueSize = this.conf.getInt("ipc.server.max.callqueue.size", DEFAULT_MAX_CALLQUEUE_SIZE); this.readThreads = conf.getInt("ipc.server.read.threadpool.size", 10); this.callQueue = new LinkedBlockingQueue<Call>(maxQueueLength); if (priorityHandlerCount > 0) { this.priorityCallQueue = new LinkedBlockingQueue<Call>(maxQueueLength); // TODO hack on size } else { this.priorityCallQueue = null; } this.highPriorityLevel = highPriorityLevel; this.maxIdleTime = 2 * conf.getInt("ipc.client.connection.maxidletime", 1000); this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10); this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000); this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout", 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT); this.numOfReplicationHandlers = conf.getInt("hbase.regionserver.replication.handler.count", 3); if (numOfReplicationHandlers > 0) { this.replicationQueue = new LinkedBlockingQueue<Call>(maxQueueSize); } // Start the listener here and let it bind to the port listener = new Listener(); this.port = listener.getAddress().getPort(); this.rpcMetrics = new HBaseRpcMetrics(serverName, Integer.toString(this.port)); this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false); this.tcpKeepAlive = conf.getBoolean("ipc.server.tcpkeepalive", true); this.warnDelayedCalls = conf.getInt(WARN_DELAYED_CALLS, DEFAULT_WARN_DELAYED_CALLS); this.delayedCalls = new AtomicInteger(0); this.responseQueuesSizeThrottler = new SizeBasedThrottler( conf.getLong(RESPONSE_QUEUES_MAX_SIZE, DEFAULT_RESPONSE_QUEUES_MAX_SIZE)); // Create the responder here responder = new Responder(); }
From source file:com.github.hrpc.rpc.Server.java
/** * Constructs a server listening on the named port and address. Parameters passed must * be of the named class. The <code>handlerCount</handlerCount> determines * the number of handler threads that will be used to process calls. * If queueSizePerHandler or numReaders are not -1 they will be used instead of parameters * from configuration. Otherwise the configuration will be picked up. * * If rpcRequestClass is null then the rpcRequestClass must have been * registered via {@link #registerProtocolEngine(RpcPayloadHeader.RpcKind, * Class, RPC.RpcInvoker)}/*from ww w.j a v a 2 s . co m*/ * This parameter has been retained for compatibility with existing tests * and usage. */ @SuppressWarnings("unchecked") protected Server(String bindAddress, int port, Class<? extends Writable> rpcRequestClass, int handlerCount, int numReaders, int queueSizePerHandler, Option conf, String serverName, String portRangeConfig) throws IOException { this.bindAddress = bindAddress; this.conf = conf; this.portRangeConfig = portRangeConfig; this.port = port; this.rpcRequestClass = rpcRequestClass; this.handlerCount = handlerCount; this.socketSendBufferSize = 0; this.maxDataLength = conf.getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT); if (queueSizePerHandler != -1) { this.maxQueueSize = queueSizePerHandler; } else { this.maxQueueSize = handlerCount * conf.getInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT); } this.maxRespSize = conf.getInt(CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY, CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT); if (numReaders != -1) { this.readThreads = numReaders; } else { this.readThreads = conf.getInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_DEFAULT); } this.callQueue = new LinkedBlockingQueue<Call>(maxQueueSize); this.maxIdleTime = 2 * conf.getInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT); this.maxConnectionsToNuke = conf.getInt(CommonConfigurationKeysPublic.IPC_CLIENT_KILL_MAX_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_KILL_MAX_DEFAULT); this.thresholdIdleConnections = conf.getInt(CommonConfigurationKeysPublic.IPC_CLIENT_IDLETHRESHOLD_KEY, CommonConfigurationKeysPublic.IPC_CLIENT_IDLETHRESHOLD_DEFAULT); // Start the listener here and let it bind to the port listener = new Listener(); this.port = listener.getAddress().getPort(); this.rpcMetrics = RpcMetrics.create(this); this.tcpNoDelay = conf.getBoolean(CommonConfigurationKeysPublic.IPC_SERVER_TCPNODELAY_KEY, CommonConfigurationKeysPublic.IPC_SERVER_TCPNODELAY_DEFAULT); // Create the responder here responder = new Responder(); this.exceptionsHandler.addTerseExceptions(StandbyException.class); }