Java tutorial
/* * Copyright 2015 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.spring.batch.configuration; import java.util.HashMap; import java.util.Map; import javax.sql.DataSource; import io.spring.batch.domain.ColumnRangePartitioner; import io.spring.batch.domain.Customer; import io.spring.batch.domain.CustomerRowMapper; import org.springframework.amqp.core.AmqpTemplate; import org.springframework.amqp.core.Queue; import org.springframework.amqp.rabbit.connection.ConnectionFactory; import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.partition.PartitionHandler; import org.springframework.batch.integration.partition.BeanFactoryStepLocator; import org.springframework.batch.integration.partition.MessageChannelPartitionHandler; import org.springframework.batch.integration.partition.StepExecutionRequestHandler; import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider; import org.springframework.batch.item.database.JdbcBatchItemWriter; import org.springframework.batch.item.database.JdbcPagingItemReader; import org.springframework.batch.item.database.Order; import org.springframework.batch.item.database.support.MySqlPagingQueryProvider; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.SimpleAsyncTaskExecutor; import org.springframework.integration.amqp.inbound.AmqpInboundGateway; import org.springframework.integration.amqp.outbound.AmqpOutboundEndpoint; import org.springframework.integration.annotation.ServiceActivator; import org.springframework.integration.channel.ExecutorChannel; import org.springframework.integration.channel.NullChannel; import org.springframework.integration.channel.QueueChannel; import org.springframework.integration.core.MessagingTemplate; import org.springframework.integration.dsl.channel.MessageChannels; import org.springframework.integration.scheduling.PollerMetadata; import org.springframework.messaging.PollableChannel; import org.springframework.scheduling.support.PeriodicTrigger; /** * @author Michael Minella */ @Configuration public class JobConfiguration implements ApplicationContextAware { @Autowired public JobBuilderFactory jobBuilderFactory; @Autowired public StepBuilderFactory stepBuilderFactory; @Autowired public DataSource dataSource; @Autowired public JobExplorer jobExplorer; private ApplicationContext applicationContext; private static final int GRID_SIZE = 4; @Bean public PartitionHandler partitionHandler() throws Exception { MessageChannelPartitionHandler partitionHandler = new MessageChannelPartitionHandler(); partitionHandler.setStepName("slaveStep"); partitionHandler.setGridSize(GRID_SIZE); partitionHandler.setMessagingOperations(messageTemplate()); partitionHandler.setPollInterval(5000l); partitionHandler.setJobExplorer(this.jobExplorer); partitionHandler.afterPropertiesSet(); return partitionHandler; } @Bean public MessagingTemplate messageTemplate() { MessagingTemplate messagingTemplate = new MessagingTemplate(outboundRequests()); messagingTemplate.setReceiveTimeout(60000000l); return messagingTemplate; } @Bean public ExecutorChannel outboundRequests() { return MessageChannels.executor("outboundRequests", new SimpleAsyncTaskExecutor()).get(); } @Bean @ServiceActivator(inputChannel = "outboundRequests") public AmqpOutboundEndpoint amqpOutboundEndpoint(AmqpTemplate template) { AmqpOutboundEndpoint endpoint = new AmqpOutboundEndpoint(template); endpoint.setExpectReply(true); endpoint.setOutputChannel(inboundRequests()); endpoint.setRoutingKey("partition.requests"); return endpoint; } @Bean public Queue requestQueue() { return new Queue("partition.requests", false); } @Bean public ColumnRangePartitioner partitioner() { ColumnRangePartitioner columnRangePartitioner = new ColumnRangePartitioner(); columnRangePartitioner.setColumn("id"); columnRangePartitioner.setDataSource(this.dataSource); columnRangePartitioner.setTable("customer"); return columnRangePartitioner; } @Bean public AmqpInboundGateway inbound(SimpleMessageListenerContainer listenerContainer) { AmqpInboundGateway gateway = new AmqpInboundGateway(listenerContainer); gateway.setRequestChannel(inboundRequests()); gateway.setRequestTimeout(60000000l); gateway.afterPropertiesSet(); return gateway; } @Bean public SimpleMessageListenerContainer container(ConnectionFactory connectionFactory) { SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(connectionFactory); container.setQueueNames("partition.requests"); container.setConcurrentConsumers(GRID_SIZE); return container; } @Bean public PollableChannel outboundStaging() { return new NullChannel(); } @Bean public QueueChannel inboundRequests() { return new QueueChannel(); } @Bean @ServiceActivator(inputChannel = "inboundRequests", outputChannel = "outboundStaging") public StepExecutionRequestHandler stepExecutionRequestHandler() { StepExecutionRequestHandler stepExecutionRequestHandler = new StepExecutionRequestHandler(); BeanFactoryStepLocator stepLocator = new BeanFactoryStepLocator(); stepLocator.setBeanFactory(this.applicationContext); stepExecutionRequestHandler.setStepLocator(stepLocator); stepExecutionRequestHandler.setJobExplorer(this.jobExplorer); return stepExecutionRequestHandler; } @Bean(name = PollerMetadata.DEFAULT_POLLER) public PollerMetadata defaultPoller() { PollerMetadata pollerMetadata = new PollerMetadata(); pollerMetadata.setTrigger(new PeriodicTrigger(10)); return pollerMetadata; } @Bean @StepScope public JdbcPagingItemReader<Customer> pagingItemReader( @Value("#{stepExecutionContext['minValue']}") Long minValue, @Value("#{stepExecutionContext['maxValue']}") Long maxValue) { System.out.println("reading " + minValue + " to " + maxValue); JdbcPagingItemReader<Customer> reader = new JdbcPagingItemReader<>(); reader.setDataSource(this.dataSource); reader.setFetchSize(1000); reader.setRowMapper(new CustomerRowMapper()); MySqlPagingQueryProvider queryProvider = new MySqlPagingQueryProvider(); queryProvider.setSelectClause("id, firstName, lastName, birthdate"); queryProvider.setFromClause("from customer"); queryProvider.setWhereClause("where id >= " + minValue + " and id <= " + maxValue); Map<String, Order> sortKeys = new HashMap<>(1); sortKeys.put("id", Order.ASCENDING); queryProvider.setSortKeys(sortKeys); reader.setQueryProvider(queryProvider); return reader; } @Bean @StepScope public JdbcBatchItemWriter<Customer> customerItemWriter() { JdbcBatchItemWriter<Customer> itemWriter = new JdbcBatchItemWriter<>(); itemWriter.setDataSource(this.dataSource); itemWriter.setSql("INSERT INTO NEW_CUSTOMER VALUES (:id, :firstName, :lastName, :birthdate)"); itemWriter.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider()); itemWriter.afterPropertiesSet(); return itemWriter; } @Bean public Step step1() throws Exception { return stepBuilderFactory.get("step1").partitioner(slaveStep().getName(), partitioner()).step(slaveStep()) .partitionHandler(partitionHandler()).build(); } @Bean public Step slaveStep() { return stepBuilderFactory.get("slaveStep").<Customer, Customer>chunk(1000) .reader(pagingItemReader(null, null)).writer(customerItemWriter()).build(); } @Bean public Job job() throws Exception { return jobBuilderFactory.get("job").start(step1()).build(); } @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { this.applicationContext = applicationContext; } }