Java tutorial
package siddur.solidtrust.azure; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.log4j.Logger; import org.core4j.Enumerable; import org.odata4j.core.OEntity; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Controller; import org.springframework.transaction.annotation.Transactional; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.supercsv.io.Tokenizer; import org.supercsv.prefs.CsvPreference; import siddur.solidtrust.ConfigBean; import siddur.solidtrust.FileSystemUtil; import siddur.solidtrust.FreePersister; import siddur.solidtrust.SolidtrustConstants; import siddur.solidtrust.entity.AzureCar; import siddur.solidtrust.entity.UpdateLog; import siddur.solidtrust.util.DateUtil; @Controller @RequestMapping(value = "/azure") public class AzureCarController { private static final Logger log4j = Logger.getLogger(AzureCarController.class); private static final int ONE_HOUR = 1000 * 60 * 60; private static final long AZURE_SHRESHOLD = 3L * 1000 * 1000 * 1000; private boolean running = false; private BlockingQueue<List<AzureCar>> queue; private boolean downloaded = true; @PersistenceContext private EntityManager em; @Autowired private ConfigBean configBean; @Autowired private AzureCarPersister persister; @Autowired private FreePersister freePersister; @Autowired private OpendataService opendataService; @Transactional(readOnly = true) @RequestMapping(value = "/log.html") public String list(Model model) { model.addAttribute("logs", em.createQuery("from UpdateLog l order by l.id desc", UpdateLog.class) .setMaxResults(20).getResultList()); model.addAttribute("useCache", configBean.getValue(SolidtrustConstants.USE_CACHE)); return "azure/log"; } @RequestMapping(value = "/useCache") public String useCache( @RequestParam(value = "useCache", required = false, defaultValue = "false") boolean useCache) { configBean.setValue(SolidtrustConstants.USE_CACHE, useCache + "", true); return "redirect:/azure/log.html"; } @Deprecated @RequestMapping(value = "/stop") public void stop() { running = false; log4j.info("Stop backup or update"); } @Deprecated @RequestMapping(value = "/status") public @ResponseBody Map<String, Object> status() { Map<String, Object> map = new HashMap<String, Object>(); map.put("running", running); if (queue != null) { List<AzureCar> cars = queue.peek(); if (cars != null) { map.put("data", cars.get(0).getLicensePlate() + "-" + cars.get(cars.size() - 1).getLicensePlate()); } } return map; } @Scheduled(cron = "0 0 2 * * *") //2:00 daily public void scheduleBackup() { if (!"true".equals(configBean.getValue(SolidtrustConstants.SYN_AZURE_MONTHLY))) { return; } log4j.info("start to backup azure data"); boolean success = doDownloadAndBackup(); if (success) { log4j.info("Fetch successfully from opendata"); } else { log4j.info("Fail to fetch csv from opendata"); } } private boolean doDownloadAndBackup() { downloaded = false; try { File csv = AzureDownloader.download(); long size = csv.length(); log4j.info("size of csv: " + size); if (size > AZURE_SHRESHOLD) { downloaded = true; doBackup(); updateOtherTables(0); return true; } } catch (Exception e) { log4j.error(e.getMessage(), e); } return false; } @RequestMapping(value = "/download") public @ResponseBody String download() { new Thread(new Runnable() { @Override public void run() { try { AzureDownloader.download(); } catch (IOException e) { log4j.error(e.getMessage(), e); } } }).start(); return "Download program is started, and lasts about 3 hours"; } @RequestMapping(value = "/size") public @ResponseBody long size() { return AzureDownloader.size(); } @RequestMapping(value = "/persist") public @ResponseBody String persist() { new Thread(new Runnable() { @Override public void run() { try { doBackup(); } catch (Exception e) { log4j.warn(e.getMessage(), e); } } }).start(); return "Persist program is started, and lasts about 1 hours"; } @RequestMapping(value = "/fuelType") public @ResponseBody String fuelType() { new Thread(new Runnable() { @Override public void run() { opendataService.update(); } }).start(); return "Update program is started, and lasts about 1 hours"; } /* * Update Register Date from Azure, * Param days: days = 0 means today, days = 1 means yesterday */ @RequestMapping("/updateDateField") @Deprecated public @ResponseBody String updateResiterDate( @RequestParam(value = "days", required = false, defaultValue = "0") int days) { int count = doUpdateDateField(days, AzureCarConstants.DATUMAANVANGTENAAMSTELLING); int count1 = doUpdateDateField(days, AzureCarConstants.VERVALDATUMAPK); return count + " " + AzureCarConstants.DATUMAANVANGTENAAMSTELLING + ", " + count1 + AzureCarConstants.VERVALDATUMAPK; } @RequestMapping("/wok") public @ResponseBody int updateWachtopkeuren() { return doUpdateWachtopkeuren(); } @RequestMapping("/csv") public void downloadAzureCSV(HttpServletResponse resp) { log4j.info("Sync csv is requested"); File f = new File(FileSystemUtil.getTempDir(), "azure.csv"); if (f.isFile()) { long size = f.length(); long modified = f.lastModified(); log4j.info("find azure.csv[size=" + size + ", modified=" + new Date(modified) + "]"); //downloaded within the past 10 hours if (downloaded && size > AZURE_SHRESHOLD && (System.currentTimeMillis() - modified) < ONE_HOUR * 10) { log4j.info("Start to download"); OutputStream os = null; InputStream is = null; try { os = resp.getOutputStream(); is = new FileInputStream(f); IOUtils.copy(is, os); } catch (Exception e) { log4j.warn(e.getMessage(), e); } finally { IOUtils.closeQuietly(is); IOUtils.closeQuietly(os); } } else { log4j.info("Antiquated or imcomplete file, not to download"); } } else { log4j.info("azure.csv not found"); } } private void doBackup() { if (running) { log4j.info("Backup or update is running."); return; } running = true; log4j.info("start to back opendata"); Date start = new Date(); UpdateLog log = new UpdateLog(); log.setType(0); log.setStartAt(start); ExecutorService pool; List<AzureCar> batch; int itemIndex = 0; try { BufferedReader reader = AzureDownloader.getReader(); String title = reader.readLine(); log4j.info("encounter title:" + title); queue = new LinkedBlockingQueue<List<AzureCar>>(200); pool = consume(queue, 10); int batchSize = 1000; batch = new ArrayList<AzureCar>(batchSize); List<String> list = new ArrayList<String>(60); String lastLP = null; Tokenizer tokenizer = new Tokenizer(reader, CsvPreference.STANDARD_PREFERENCE); try { while (running && tokenizer.readColumns(list) && !list.isEmpty()) { itemIndex++; if (itemIndex == 1) { log.setStartLicensePlate(list.get(0)); } AzureCar azureCar = AzureDownloader.convert2(list); azureCar.setScrapedAt(start); azureCar.setUpdated(start); batch.add(azureCar); if (itemIndex % batchSize == 0) { if (batch != null) { queue.put(batch); log4j.debug(MessageFormat.format("have fetched records[{0} - {1}]", itemIndex - batchSize + 1, itemIndex)); } lastLP = batch.get(batch.size() - 1).getLicensePlate(); batch = new ArrayList<AzureCar>(batchSize); } } } catch (Exception e) { log4j.error(e.getMessage(), e); } finally { tokenizer.close(); } if (!batch.isEmpty()) { lastLP = batch.get(batch.size() - 1).getLicensePlate(); queue.put(batch); log4j.info(MessageFormat.format("have fetched records[{0} - {1}]", itemIndex - batch.size() + 1, itemIndex)); Thread.sleep(100); } pool.shutdown(); log.setEndLicensePlate(lastLP); Thread.sleep(10 * 60 * 1000); } catch (Throwable e) { log4j.error(e.getMessage(), e); } finally { running = false; } log4j.info("mark cars removed from rdw"); //mark cars removed from rdw persister.markRemoved(start); log.setEndAt(new Date()); log.setAmount(itemIndex); freePersister.save(log); log4j.info("finish backup records"); opendataService.update(); } @Deprecated private int doUpdateWachtopkeuren() { int total = 0; log4j.info("Start to fetch [Wachtopkeuren=Ja] from AzureCar"); UpdateLog log = new UpdateLog(); log.setType(3); log.setStartAt(new Date()); //1) set wok = false persister.initWachtopkeuren(); //2) update wok = true int pageIndex = 0; try { while (true) { List<String> lpList = persister.getWachtopkeuren(pageIndex++); int count = persister.updateWachtopkeuren(lpList); log4j.info("Persist " + count + " records"); total += count; if (count < AzureConnector.pageSize) { break; } } } catch (Exception e) { log4j.error(e.getMessage(), e); } finally { running = false; } //3) set removed date persister.calcWachtopkeuren(); log.setAmount(total); log4j.info("Fetch " + total + " records totally"); log.setEndAt(new Date()); freePersister.save(log); return total; } @Deprecated private int doUpdateDateField(int days, String datefield) { if (running) { log4j.info("Backup or update is running."); return 0; } running = true; int total = 0; log4j.info("Start to synchronize " + datefield + " from azure"); UpdateLog log = new UpdateLog(); log.setType(2); log.setStartAt(new Date()); int pageIndex = 0; try { while (true) { int count = dealEachPage(days, pageIndex++, datefield); total += count; if (count < AzureConnector.pageSize) { break; } } } catch (Exception e) { log4j.error(e.getMessage(), e); } finally { running = false; } log.setAmount(total); log4j.info("Fetch " + total + " records totally"); log.setEndAt(new Date()); freePersister.save(log); return total; } @Deprecated private int dealEachPage(int days, int pageIndex, String datefield) throws Exception { int count = 0; Enumerable<OEntity> entityList = AzureConnector.updateDateField(days, pageIndex, datefield); if (entityList != null) { count = entityList.count(); log4j.info("Fetch " + count + " records in Page " + pageIndex); persister.updateBatch(entityList); } return count; } @RequestMapping(value = "/mark_repetition") public void markRepetition(@RequestParam("table") String table, @RequestParam("all") int all) { String start = all == 1 ? null : DateUtil.date2String(DateUtils.addDays(new Date(), -10)); log4j.info("Start date: " + start); if (table.equals("Marktplaats")) { persister.markMarktplaatsRepetition(table, start); } else if (table.equals("AutoscoutNl")) { persister.markAutoscoutRepetition(table, start); } } @RequestMapping(value = "/update_others") public void updateOtherTables(@RequestParam(value = "table", required = false, defaultValue = "0") int table) { String start = DateUtil.date2String(DateUtils.addDays(new Date(), -10)); if (table == 0 || table == 1) { log4j.info("update dateRegisted field in Marktplaats"); try { persister.updateMarktplaats(); persister.updateSortedMarktplaats(); persister.markMarktplaatsRepetition("Marktplaats", start); } catch (Exception e) { log4j.error(e.getMessage(), e); } } if (table == 0 || table == 2) { log4j.info("update AutoscoutNl field in AutoscoutNl"); try { persister.updateAutoscoutNl(); persister.markAutoscoutRepetition("AutoscoutNl", start); // AutoscoutNl has not AdDate } catch (Exception e) { log4j.error(e.getMessage(), e); } } if (table == 0 || table == 3) { try { log4j.info("update Wachtopkeuren field in AutoscoutNl"); doUpdateWachtopkeuren(); } catch (Exception e) { log4j.error(e.getMessage(), e); } } } private ExecutorService consume(final BlockingQueue<List<AzureCar>> queue, int count) { Runnable consumer = new Runnable() { @Override public void run() { try { while (running) { List<AzureCar> batch = queue.poll(100, TimeUnit.SECONDS); if (batch != null) { persister.saveBatch(batch); } } } catch (InterruptedException e) { log4j.info(e.getMessage(), e); } } }; ExecutorService pool = Executors.newFixedThreadPool(count); for (int i = 0; i < count; i++) { pool.execute(consumer); } return pool; } public static void main(String[] args) { new Timer().schedule(new TimerTask() { @Override public void run() { System.out.println(Thread.currentThread().getName()); } }, 5000, 1000); } }