Java tutorial
/* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ package com.gumtreescraper.gui; import com.gumtreescraper.model.Gumtree; import com.gumtreescraper.scraper.GumtreeScraper; import com.gumtreescraper.util.GumtreeUtils; import java.util.ArrayList; import java.util.Date; import java.util.List; import javax.swing.JOptionPane; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; /** * * @author duonghung1269 */ public class GumtreeScraperForm extends javax.swing.JFrame { private Logger LOG = Logger.getLogger(this.getClass()); /** * Creates new form GumtreeScraperForm */ public GumtreeScraperForm() { initComponents(); cbbTimeout.setSelectedItem("60"); ftfEditedDate.setText(GumtreeUtils.convertDateToString(new Date())); } /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always * regenerated by the Form Editor. */ @SuppressWarnings("unchecked") // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents private void initComponents() { btnScrape = new javax.swing.JButton(); jLabel1 = new javax.swing.JLabel(); jLabel2 = new javax.swing.JLabel(); tfUsername = new javax.swing.JTextField(); jLabel3 = new javax.swing.JLabel(); tfOutputFileName = new javax.swing.JTextField(); jLabel4 = new javax.swing.JLabel(); tfPassword = new javax.swing.JPasswordField(); cbbUrl = new javax.swing.JComboBox<String>(); cbbTimeout = new javax.swing.JComboBox(); jLabel5 = new javax.swing.JLabel(); jLabel6 = new javax.swing.JLabel(); ftfEditedDate = new javax.swing.JFormattedTextField(); jLabel7 = new javax.swing.JLabel(); setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); setTitle("Gumtree Scraper"); btnScrape.setText("Scrape"); btnScrape.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { btnScrapeActionPerformed(evt); } }); jLabel1.setText("Username"); jLabel2.setText("Password"); jLabel3.setText("Url"); tfOutputFileName.setText("gumtree.csv"); jLabel4.setText("Output File"); cbbUrl.setModel(new javax.swing.DefaultComboBoxModel( new String[] { "http://www.gumtree.com.au/s-land-for-sale/c20031", "http://www.gumtree.com.au/s-property-for-rent/c18364", "http://www.gumtree.com.au/s-property-for-sale/c18367" })); cbbTimeout.setModel(new javax.swing.DefaultComboBoxModel( new String[] { "30", "45", "60", "75", "90", "105", "120", "135", "150", "165", "180", " " })); jLabel5.setText("Timeout"); jLabel6.setText("seconds"); ftfEditedDate.setFormatterFactory(new javax.swing.text.DefaultFormatterFactory( new javax.swing.text.DateFormatter(new java.text.SimpleDateFormat("dd/MM/yyyy")))); jLabel7.setText("Edited Date"); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); getContentPane().setLayout(layout); layout.setHorizontalGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup().addGap(27, 27, 27) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) .addComponent(jLabel1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addComponent(jLabel2) .addComponent(jLabel3, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addComponent(jLabel4, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addComponent(jLabel5).addComponent(jLabel7)) .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(btnScrape) .addComponent(cbbUrl, javax.swing.GroupLayout.PREFERRED_SIZE, 434, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(layout .createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) .addComponent(ftfEditedDate, javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addComponent(cbbTimeout, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap( javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(jLabel6))) .addGroup(layout .createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false) .addComponent(tfPassword, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, 225, Short.MAX_VALUE) .addComponent(tfUsername, javax.swing.GroupLayout.Alignment.LEADING)) .addComponent(tfOutputFileName, javax.swing.GroupLayout.PREFERRED_SIZE, 225, javax.swing.GroupLayout.PREFERRED_SIZE)) .addContainerGap(34, Short.MAX_VALUE))); layout.setVerticalGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING).addGroup( javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup().addGap(36, 36, 36).addGroup(layout .createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE).addComponent(jLabel1) .addComponent(tfUsername, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(jLabel2).addComponent(tfPassword, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGap(28, 28, 28) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(jLabel3).addComponent(cbbUrl, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(tfOutputFileName, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(jLabel4)) .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(cbbTimeout, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(jLabel6).addComponent(jLabel5)) .addGap(18, 18, 18) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(ftfEditedDate, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(jLabel7)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 20, Short.MAX_VALUE) .addComponent(btnScrape).addGap(59, 59, 59))); pack(); }// </editor-fold>//GEN-END:initComponents private void btnScrapeActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btnScrapeActionPerformed String userName = tfUsername.getText().trim(); String password = new String(tfPassword.getPassword()); String url = (String) cbbUrl.getSelectedItem(); String fileName = tfOutputFileName.getText().trim(); // userName = "dangduonghung@gmail.com"; // url = "http://www.gumtree.com.au/s-land-for-sale/c20031"; if (StringUtils.isBlank(userName) || StringUtils.isBlank(password) || StringUtils.isBlank(url) || StringUtils.isBlank(fileName) || StringUtils.isBlank(ftfEditedDate.getText())) { JOptionPane.showMessageDialog(this, "All fields are required!!!"); LOG.error("All fields are required!!!"); return; } LOG.error("TEST ERROR"); GumtreeScraper scraper = null; try { Date lastEditedDate = GumtreeUtils.convertStringToDate(ftfEditedDate.getText().trim()); int timeout = Integer.parseInt((String) cbbTimeout.getSelectedItem()); scraper = new GumtreeScraper(userName, password, url, fileName, lastEditedDate, timeout); boolean validLogin = scraper.login(); if (!validLogin) { JOptionPane.showMessageDialog(this, "Invalid credentials!!!"); LOG.debug("Invalid credentials!!!"); return; } btnScrape.setText("Scraping..."); btnScrape.setEnabled(false); this.repaint(); this.invalidate(); List<Gumtree> gumtrees = new ArrayList<>(); scraper.scrapeWithJSoup(gumtrees, url); scraper.updateGumtreeModel(gumtrees); System.out.print(gumtrees); JOptionPane.showMessageDialog(this, "Scraping completed successfully! Please find ouput file " + tfOutputFileName + " in the same folder."); btnScrape.setText("Scrape"); btnScrape.setEnabled(true); this.repaint(); this.invalidate(); } catch (Exception ex) { // System.out.print(ex); // ex.printStackTrace(); JOptionPane.showMessageDialog(this, "Something wrong!!! \nDetails: " + ex.getMessage()); if (scraper != null) { scraper.closeBrowser(); } // LOG.debug(ex.getCause()); // LOG.info(ex.getMessage()); // LOG.debug("ed", ex); LOG.error("Scrape ERROR: ", ex); btnScrape.setText("Scrape"); btnScrape.setEnabled(true); this.repaint(); this.invalidate(); } }//GEN-LAST:event_btnScrapeActionPerformed /** * @param args the command line arguments */ public static void main(String args[]) { /* Set the Nimbus look and feel */ //<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) "> /* If Nimbus (introduced in Java SE 6) is not available, stay with the default look and feel. * For details see http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html */ try { for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) { if ("Nimbus".equals(info.getName())) { javax.swing.UIManager.setLookAndFeel(info.getClassName()); break; } } } catch (ClassNotFoundException ex) { java.util.logging.Logger.getLogger(GumtreeScraperForm.class.getName()) .log(java.util.logging.Level.SEVERE, null, ex); } catch (InstantiationException ex) { java.util.logging.Logger.getLogger(GumtreeScraperForm.class.getName()) .log(java.util.logging.Level.SEVERE, null, ex); } catch (IllegalAccessException ex) { java.util.logging.Logger.getLogger(GumtreeScraperForm.class.getName()) .log(java.util.logging.Level.SEVERE, null, ex); } catch (javax.swing.UnsupportedLookAndFeelException ex) { java.util.logging.Logger.getLogger(GumtreeScraperForm.class.getName()) .log(java.util.logging.Level.SEVERE, null, ex); } //</editor-fold> String log4jPath = "src/main/java/log4j.properties"; PropertyConfigurator.configure(log4jPath); /* Create and display the form */ java.awt.EventQueue.invokeLater(new Runnable() { public void run() { new GumtreeScraperForm().setVisible(true); } }); } // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton btnScrape; private javax.swing.JComboBox cbbTimeout; private javax.swing.JComboBox<String> cbbUrl; private javax.swing.JFormattedTextField ftfEditedDate; private javax.swing.JLabel jLabel1; private javax.swing.JLabel jLabel2; private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel4; private javax.swing.JLabel jLabel5; private javax.swing.JLabel jLabel6; private javax.swing.JLabel jLabel7; private javax.swing.JTextField tfOutputFileName; private javax.swing.JPasswordField tfPassword; private javax.swing.JTextField tfUsername; // End of variables declaration//GEN-END:variables }