List of usage examples for java.sql PreparedStatement setDouble
void setDouble(int parameterIndex, double x) throws SQLException;
double value. From source file:com.pari.nm.utils.db.InventoryDBHelper.java
public static Customer insertOrUpdateCustomer(Customer customer) { PreparedStatement ps = null; PreparedStatement ps1 = null; PreparedStatement ps2 = null; Connection c = null;//from w w w .j a v a 2 s.com ResultSet rs = null; try { c = DBHelper.getConnection(); boolean insert = true; if (customer.getId() != -1) { String query = "select * from customers where customer_id=" + customer.getId(); rs = DBHelper.executeQuery(query); if (rs.next()) { insert = false; } try { rs.close(); } catch (Exception ee) { } } if (!insert) { ps = c.prepareStatement(DBHelperConstants.CUSTOMER_UPDATE); } else { customer.setId(getNextCustomerId(c)); ps = c.prepareStatement(DBHelperConstants.CUSTOMER_INSERT); ps1 = c.prepareStatement(DBHelperConstants.SHADOW_CUSTOMER_INSERT); ps2 = c.prepareStatement(DBHelperConstants.INSERT_PROFILES_ALL); } if (insert) { ps.setInt(1, customer.getId()); ps.setString(2, customer.getCustomerName()); ps.setString(3, customer.getContactName()); ps.setString(4, customer.getEmailAddresses()); ps.setString(5, customer.getContactPhoneNumber()); ps.setString(6, customer.getLogin()); ps.setString(7, customer.getSecurityKey()); ps.setInt(8, customer.getCreatorId()); ps.setString(9, customer.getCreaterName()); ps.setDouble(10, customer.getNoOfLicenses()); ps1.setInt(1, customer.getId()); ps1.setInt(2, customer.getId()); ps2.setInt(1, customer.getId()); ps2.setString(2, customer.getCustomerName()); // Creating profiles FaultProfile fp = new FaultProfile(); fp.setProfId(customer.getId()); fp.setProfName(customer.getCustomerName()); FaultSpecFactory.getInstance().addFaultProfileOnly(fp); } else { ps.setString(1, customer.getCustomerName()); ps.setString(2, customer.getContactName()); ps.setString(3, customer.getEmailAddresses()); ps.setString(4, customer.getContactPhoneNumber()); ps.setString(5, customer.getLogin()); ps.setString(6, customer.getSecurityKey()); ps.setInt(7, customer.getNoOfLicenses()); ps.setDouble(8, customer.getId()); } ps.executeUpdate(); if (ps1 != null) { ps1.executeUpdate(); } if (ps2 != null) { ps2.executeUpdate(); } return customer; } catch (Exception ee) { logger.warn("Error while inserting or updating the customer", ee); } finally { try { ps.close(); } catch (Exception ee) { } try { ps1.close(); } catch (Exception ee) { } try { ps2.close(); } catch (Exception ee) { } try { DBHelper.releaseConnection(c); } catch (Exception ee) { } try { rs.close(); } catch (Exception ee) { } } return null; }
From source file:HSqlPrimerDesign.java
@SuppressWarnings("Duplicates") public static void locations(Connection connection) throws ClassNotFoundException, SQLException, InstantiationException, IllegalAccessException, IOException { long time = System.nanoTime(); String base = new File("").getAbsolutePath(); DpalLoad.main(new String[0]); Dpal_Inst = DpalLoad.INSTANCE_WIN64; System.out.println(Dpal_Inst); Connection db = connection;/*from www.j a va 2 s .c o m*/ db.setAutoCommit(false); Statement stat = db.createStatement(); PrintWriter log = new PrintWriter(new File("javalog.log")); stat.execute("SET FILES LOG FALSE;"); PreparedStatement st = db.prepareStatement("INSERT INTO Primerdb.MatchedPrimers(" + "Primer, PrimerMatch, Comp,FragAVG,FragVAR,H2SD,L2SD, Cluster, Strain)" + "Values(?,?,?,?,?,?,?,?,?)"); ResultSet call = stat.executeQuery("Select * From Primerdb.Phages;"); List<String[]> phages = new ArrayList<>(); while (call.next()) { String[] r = new String[3]; r[0] = call.getString("Strain"); r[1] = call.getString("Cluster"); r[2] = call.getString("Name"); phages.add(r); // if(strain.equals("-myco")) { // if (r[2].equals("xkcd")) { // strain = r[0]; // } // }else if(strain.equals("-arthro")){ // if (r[2].equals("ArV1")) { // strain = r[0]; // } // } } call.close(); Set<String> strains = phages.stream().map(y -> y[0]).collect(Collectors.toSet()); for (String x : strains) { Set<String> clust = phages.stream().filter(y -> y[0].equals(x)).map(y -> y[1]) .collect(Collectors.toSet()); String[] clusters = clust.toArray(new String[clust.size()]); // String z ="A1"; for (String z : clusters) { System.out.println("Starting:" + z); List<Primer> primers = new ArrayList<>(); Set<Matches> matched = new HashSet<>(); Set<String> clustphage = phages.stream().filter(a -> a[0].equals(x) && a[1].equals(z)) .map(a -> a[2]).collect(Collectors.toSet()); String[] clustphages = clustphage.toArray(new String[clustphage.size()]); if (clustphages.length > 1) { try { ResultSet resultSet = stat .executeQuery("Select * from primerdb.primers" + " where Strain ='" + x + "' and Cluster ='" + z + "' and UniqueP = true" + " and Hairpin = false"); while (resultSet.next()) { Primer primer = new Primer(resultSet.getString("Sequence")); primer.setTm(resultSet.getDouble("Tm")); primers.add(primer); } resultSet.close(); } catch (SQLException e) { e.printStackTrace(); System.out.println("Error occurred at " + x + " " + z); } System.out.println(primers.size()); Set<Primer> primerlist2 = primers.stream().collect(Collectors.toSet()); Primer[] primers2 = primerlist2.toArray(new Primer[primerlist2.size()]); Map<String, Map<CharSequence, List<Integer>>> locations = Collections .synchronizedMap(new HashMap<>()); clustphage.stream().forEach(phage -> { String[] seqs = Fasta.parse(base + "/Fastas/" + phage + ".fasta"); String sequence = seqs[0] + seqs[1]; Map<String, List<Integer>> seqInd = new HashMap<>(); for (int i = 0; i <= sequence.length() - 10; i++) { String sub = sequence.substring(i, i + 10); if (seqInd.containsKey(sub)) { seqInd.get(sub).add(i); } else { List<Integer> list = new ArrayList<>(); list.add(i); seqInd.put(sub, list); } } Map<CharSequence, List<Integer>> alllocs = new HashMap<>(); for (Primer primer : primers2) { List<Integer> locs = new ArrayList<>(); String sequence1 = primer.getSequence(); String frag = sequence1.substring(0, 10); List<Integer> integers = seqInd.get(frag); if (integers != null) { for (Integer i : integers) { if ((sequence1.length() + i) < sequence.length() && sequence.substring(i, sequence1.length() + i).equals(sequence1)) { locs.add(i); } } } alllocs.put(sequence1, locs); } locations.put(phage, alllocs); }); System.out.println("locations found"); System.out.println((System.nanoTime() - time) / Math.pow(10, 9) / 60.0); final int[] k = new int[] { 0 }; primerlist2.parallelStream().forEach(a -> { int matches = 0; int i = 0; while (primers2[i] != a) { i++; } for (int j = i + 1; j < primers2.length; j++) { double[] frags = new double[clustphages.length]; int phageCounter = 0; Primer b = primers2[j]; boolean match = true; if (matches > 0) { break; } if (Math.abs(a.getTm() - b.getTm()) > 5.0 || a.getSequence().equals(b.getSequence())) { continue; } for (String phage : clustphages) { List<Integer> loc1 = locations.get(phage).get(a.getSequence()); List<Integer> loc2 = locations.get(phage).get(b.getSequence()); // if(loc1.size()==0){ // System.out.println(phage+" "+a.getSequence()); // } if (loc1.size() == 0 || loc2.size() == 0) { // if (loc1.size()!=1||loc2.size()!=1){ match = false; break; } boolean found = false; int fragCount = 0; int l1 = loc1.get(0); int l2 = loc2.get(0); int count1 = 0; int count2 = 0; int frag = Math.abs(l1 - l2); while (!found) { if (frag >= 500 && frag <= 2000) { fragCount++; if (++count1 < loc1.size()) l1 = loc1.get(count1); else if (++count2 < loc2.size()) l2 = loc2.get(count2); } else if (l1 < l2 && frag < 500) { count2++; } else if (l1 > l2 && frag < 500) { count1++; } else if (l1 > l2 && frag > 2000) { count2++; } else if (l1 < l2 && frag > 2000) { count1++; } else { break; } if (count1 < loc1.size() && count2 < loc2.size()) { l1 = loc1.get(count1); l2 = loc2.get(count2); frag = Math.abs(l1 - l2); } else { if (fragCount == 1) { found = true; frags[phageCounter++] = frag + 0.0; } else { break; } } } if (!found) { match = false; break; } } if (match) { matches++; matched.add(new Matches(a, b, frags)); } } // k[0]++; // System.out.println(k[0]); }); System.out.println((System.nanoTime() - time) / Math.pow(10, 9) / 60.0); System.out.println("Primers matched"); int c = 0; int i = 0; try { for (Matches primerkey : matched) { c++; String primer1 = primerkey.one.getSequence(); String primer2 = primerkey.two.getSequence(); st.setString(1, primer1); st.setString(2, primer2); st.setDouble(3, complementarity(primer1, primer2, Dpal_Inst)); st.setDouble(4, primerkey.stats.getMean()); st.setDouble(5, primerkey.stats.getVariance()); st.setDouble(6, primerkey.stats.getMean() + 2 * primerkey.stats.getStandardDeviation()); st.setDouble(7, primerkey.stats.getMean() - 2 * primerkey.stats.getStandardDeviation()); st.setString(8, z); st.setString(9, x); st.addBatch(); i++; if (i == 1000) { i = 0; st.executeBatch(); db.commit(); } } if (i > 0) { st.executeBatch(); db.commit(); } } catch (SQLException e) { e.printStackTrace(); System.out.println("Error occurred at " + x + " " + z); } System.out.println(c); } log.println(z); log.flush(); System.gc(); } } stat.execute("SET FILES LOG TRUE;"); st.close(); stat.close(); System.out.println("Matches Submitted"); }
From source file:com.flexive.core.storage.genericSQL.GenericBinarySQLStorage.java
/** * Transfer a binary from the transit to the 'real' binary table * * @param _con open and valid connection * @param binary the binary descriptor//from w w w . ja v a 2s .c o m * @param id desired id * @param version desired version * @param quality desired quality * @return descriptor of final binary * @throws FxDbException on errors looking up the sequencer */ private BinaryDescriptor binaryTransit(Connection _con, BinaryDescriptor binary, long id, int version, int quality) throws FxDbException { PreparedStatement ps = null; BinaryDescriptor created; FileInputStream fis = null; boolean dbTransit; boolean dbStorage; final long dbThreshold; final long dbPreviewThreshold; final int divisionId = FxContext.get().getDivisionId(); try { final DivisionConfigurationEngine divisionConfig = EJBLookup.getDivisionConfigurationEngine(); dbTransit = divisionConfig.get(SystemParameters.BINARY_TRANSIT_DB); if (id >= 0) { dbThreshold = divisionConfig.get(SystemParameters.BINARY_DB_THRESHOLD); dbPreviewThreshold = divisionConfig.get(SystemParameters.BINARY_DB_PREVIEW_THRESHOLD); } else { //force storage of system binaries in the database dbThreshold = -1; dbPreviewThreshold = -1; } dbStorage = dbThreshold < 0 || binary.getSize() < dbThreshold; } catch (FxApplicationException e) { throw e.asRuntimeException(); } Connection con = null; try { con = Database.getNonTXDataSource(divisionId).getConnection(); con.setAutoCommit(false); double resolution = 0.0; int width = 0; int height = 0; boolean isImage = binary.getMimeType().startsWith("image/"); if (isImage) { try { width = Integer .parseInt(defaultString(FxXMLUtils.getElementData(binary.getMetadata(), "width"), "0")); height = Integer.parseInt( defaultString(FxXMLUtils.getElementData(binary.getMetadata(), "height"), "0")); resolution = Double.parseDouble( defaultString(FxXMLUtils.getElementData(binary.getMetadata(), "xResolution"), "0")); } catch (NumberFormatException e) { //ignore LOG.warn(e, e); } } created = new BinaryDescriptor(CacheAdmin.getStreamServers(), id, version, quality, System.currentTimeMillis(), binary.getName(), binary.getSize(), binary.getMetadata(), binary.getMimeType(), isImage, resolution, width, height, binary.getMd5sum()); //we can copy the blob directly into the binary table if the database is used for transit and the final binary is //stored in the filesystem final boolean copyBlob = dbTransit && dbStorage; boolean storePrev1FS = false, storePrev2FS = false, storePrev3FS = false, storePrev4FS = false; long prev1Length = -1, prev2Length = -1, prev3Length = -1, prev4Length = -1; if (dbPreviewThreshold >= 0) { //we have to check if preview should be stored on the filesystem ps = con.prepareStatement(BINARY_TRANSIT_PREVIEW_SIZES); ps.setString(1, binary.getHandle()); ResultSet rs = ps.executeQuery(); if (!rs.next()) throw new FxDbException("ex.content.binary.transitNotFound", binary.getHandle()); rs.getLong(1); //check if previewref is null if (rs.wasNull()) { //if previews are not referenced, check thresholds storePrev1FS = (prev1Length = rs.getLong(2)) >= dbPreviewThreshold && !rs.wasNull(); storePrev2FS = (prev2Length = rs.getLong(3)) >= dbPreviewThreshold && !rs.wasNull(); storePrev3FS = (prev3Length = rs.getLong(4)) >= dbPreviewThreshold && !rs.wasNull(); storePrev4FS = (prev4Length = rs.getLong(5)) >= dbPreviewThreshold && !rs.wasNull(); } } if (ps != null) ps.close(); String previewSelect = (storePrev1FS ? ",NULL" : ",PREV1") + (storePrev2FS ? ",NULL" : ",PREV2") + (storePrev3FS ? ",NULL" : ",PREV3") + (storePrev4FS ? ",NULL" : ",PREV4"); //check if the binary is to be replaced ps = con.prepareStatement( "SELECT COUNT(*) FROM " + TBL_CONTENT_BINARY + " WHERE ID=? AND VER=? AND QUALITY=?"); ps.setLong(1, created.getId()); ps.setInt(2, created.getVersion()); //version ps.setInt(3, created.getQuality()); //quality ResultSet rsExist = ps.executeQuery(); final boolean replaceBinary = rsExist != null && rsExist.next() && rsExist.getLong(1) > 0; ps.close(); int paramIndex = 1; if (replaceBinary) { ps = con.prepareStatement(BINARY_TRANSIT_REPLACE + (copyBlob ? BINARY_TRANSIT_REPLACE_FBLOB_COPY : BINARY_TRANSIT_REPLACE_FBLOB_PARAM) + BINARY_TRANSIT_REPLACE_PARAMS); FxBinaryUtils.removeBinary(divisionId, created.getId()); } else { ps = con.prepareStatement((copyBlob ? BINARY_TRANSIT : BINARY_TRANSIT_FILESYSTEM) + previewSelect + BINARY_TRANSIT_PREVIEW_WHERE); ps.setLong(paramIndex++, created.getId()); ps.setInt(paramIndex++, created.getVersion()); //version ps.setInt(paramIndex++, created.getQuality()); //quality } File binaryTransit = null; boolean removeTransitFile = false; if (dbTransit) { //transit is handled in the database try { if (!dbStorage) { //binaries are stored on the filesystem binaryTransit = getBinaryTransitFileInfo(binary).getBinaryTransitFile(); removeTransitFile = true; //have to clean up afterwards since its a temporary file we get } } catch (FxApplicationException e) { if (e instanceof FxDbException) throw (FxDbException) e; throw new FxDbException(e); } } else { //transit file resides on the local file system binaryTransit = FxBinaryUtils.getTransitFile(divisionId, binary.getHandle()); removeTransitFile = true; // temporary transit file can be removed as well if (binaryTransit == null) throw new FxDbException("ex.content.binary.transitNotFound", binary.getHandle()); } boolean needExplicitBlobInsert = false; if (copyBlob && replaceBinary) ps.setString(paramIndex++, binary.getHandle()); if (!copyBlob) { //we do not perform a simple blob copy operation in the database if (dbStorage) { //binary is stored in the database -> copy it from the transit file (might be a temp. file) if (blobInsertSelectAllowed()) { fis = new FileInputStream(binaryTransit); ps.setBinaryStream(paramIndex++, fis, (int) binaryTransit.length()); } else { ps.setNull(paramIndex++, Types.BINARY); needExplicitBlobInsert = true; } } else { //binary is stored on the filesystem -> move transit file to binary storage file try { if (!FxFileUtils.moveFile(binaryTransit, FxBinaryUtils.createBinaryFile(divisionId, created.getId(), created.getVersion(), created.getQuality(), PreviewSizes.ORIGINAL.getBlobIndex()))) throw new FxDbException(LOG, "ex.content.binary.fsCopyFailed", created.getId()); } catch (IOException e) { throw new FxDbException(LOG, "ex.content.binary.fsCopyFailedError", created.getId(), e.getMessage()); } ps.setNull(paramIndex++, Types.BINARY); } } // int cnt = paramIndex; //copyBlob ? 4 : 5; ps.setString(paramIndex++, created.getName()); ps.setLong(paramIndex++, created.getSize()); setBigString(ps, paramIndex++, created.getMetadata()); ps.setString(paramIndex++, created.getMimeType()); if (replaceBinary) ps.setNull(paramIndex++, java.sql.Types.NUMERIC); //set preview ref to null ps.setBoolean(paramIndex++, created.isImage()); ps.setDouble(paramIndex++, created.getResolution()); ps.setInt(paramIndex++, created.getWidth()); ps.setInt(paramIndex++, created.getHeight()); ps.setString(paramIndex++, created.getMd5sum()); if (replaceBinary) { ps.setLong(paramIndex++, created.getId()); ps.setInt(paramIndex++, created.getVersion()); //version ps.setInt(paramIndex, created.getQuality()); //quality } else ps.setString(paramIndex, binary.getHandle()); ps.executeUpdate(); if (needExplicitBlobInsert) { ps.close(); ps = con.prepareStatement( "UPDATE " + TBL_CONTENT_BINARY + " SET FBLOB=? WHERE ID=? AND VER=? AND QUALITY=?"); fis = new FileInputStream(binaryTransit); ps.setBinaryStream(1, fis, (int) binaryTransit.length()); ps.setLong(2, created.getId()); ps.setInt(3, created.getVersion()); //version ps.setInt(4, created.getQuality()); //quality ps.executeUpdate(); } if (removeTransitFile && binaryTransit != null) { //transit file was a temp. file -> got to clean up FxFileUtils.removeFile(binaryTransit); } if (replaceBinary) { ps.close(); //set all preview entries to the values provided by the transit table ps = con.prepareStatement("UPDATE " + TBL_CONTENT_BINARY + " SET PREV1=NULL,PREV2=NULL,PREV3=NULL,PREV4=NULL WHERE ID=? AND VER=? AND QUALITY=?"); ps.setLong(1, created.getId()); ps.setInt(2, created.getVersion()); //version ps.setInt(3, created.getQuality()); //quality ps.executeUpdate(); ps.close(); ps = con.prepareStatement( "SELECT PREV1_WIDTH,PREV1_HEIGHT,PREV1SIZE,PREV2_WIDTH,PREV2_HEIGHT,PREV2SIZE,PREV3_WIDTH,PREV3_HEIGHT,PREV3SIZE,PREV4_WIDTH,PREV4_HEIGHT,PREV4SIZE FROM " + TBL_BINARY_TRANSIT + " WHERE BKEY=?"); ps.setString(1, binary.getHandle()); ResultSet rsPrev = ps.executeQuery(); if (rsPrev != null && rsPrev.next()) { long[] data = new long[12]; for (int d = 0; d < 12; d++) data[d] = rsPrev.getLong(d + 1); ps.close(); ps = con.prepareStatement("UPDATE " + TBL_CONTENT_BINARY + " SET PREV1_WIDTH=?,PREV1_HEIGHT=?,PREV1SIZE=?,PREV2_WIDTH=?,PREV2_HEIGHT=?,PREV2SIZE=?,PREV3_WIDTH=?,PREV3_HEIGHT=?,PREV3SIZE=?,PREV4_WIDTH=?,PREV4_HEIGHT=?,PREV4SIZE=? WHERE ID=? AND VER=? AND QUALITY=?"); for (int d = 0; d < 12; d++) ps.setLong(d + 1, data[d]); ps.setLong(13, created.getId()); ps.setInt(14, created.getVersion()); //version ps.setInt(15, created.getQuality()); //quality ps.executeUpdate(); } } //finally fetch the preview blobs from transit and store them on the filesystem if required if (storePrev1FS || storePrev2FS || storePrev3FS || storePrev4FS) { ps.close(); previewSelect = (!storePrev1FS ? ",NULL" : ",PREV1") + (!storePrev2FS ? ",NULL" : ",PREV2") + (!storePrev3FS ? ",NULL" : ",PREV3") + (!storePrev4FS ? ",NULL" : ",PREV4"); ps = con.prepareStatement("SELECT " + previewSelect.substring(1) + BINARY_TRANSIT_PREVIEW_WHERE); ps.setString(1, binary.getHandle()); ResultSet rs = ps.executeQuery(); if (!rs.next()) throw new FxDbException("ex.content.binary.transitNotFound", binary.getHandle()); if (storePrev1FS) try { if (!FxFileUtils.copyStream2File(prev1Length, rs.getBinaryStream(1), FxBinaryUtils.createBinaryFile(divisionId, created.getId(), created.getVersion(), created.getQuality(), PreviewSizes.PREVIEW1.getBlobIndex()))) throw new FxDbException(LOG, "ex.content.binary.fsCopyFailed", created.getId() + "[" + PreviewSizes.PREVIEW1.getBlobIndex() + "]"); } catch (IOException e) { throw new FxDbException(LOG, "ex.content.binary.fsCopyFailedError", created.getId() + "[" + PreviewSizes.PREVIEW1.getBlobIndex() + "]", e.getMessage()); } if (storePrev2FS) try { if (!FxFileUtils.copyStream2File(prev2Length, rs.getBinaryStream(2), FxBinaryUtils.createBinaryFile(divisionId, created.getId(), created.getVersion(), created.getQuality(), PreviewSizes.PREVIEW2.getBlobIndex()))) throw new FxDbException(LOG, "ex.content.binary.fsCopyFailed", created.getId() + "[" + PreviewSizes.PREVIEW2.getBlobIndex() + "]"); } catch (IOException e) { throw new FxDbException(LOG, "ex.content.binary.fsCopyFailedError", created.getId() + "[" + PreviewSizes.PREVIEW2.getBlobIndex() + "]", e.getMessage()); } if (storePrev3FS) try { if (!FxFileUtils.copyStream2File(prev3Length, rs.getBinaryStream(3), FxBinaryUtils.createBinaryFile(divisionId, created.getId(), created.getVersion(), created.getQuality(), PreviewSizes.PREVIEW3.getBlobIndex()))) throw new FxDbException(LOG, "ex.content.binary.fsCopyFailed", created.getId() + "[" + PreviewSizes.PREVIEW3.getBlobIndex() + "]"); } catch (IOException e) { throw new FxDbException(LOG, "ex.content.binary.fsCopyFailedError", created.getId() + "[" + PreviewSizes.PREVIEW3.getBlobIndex() + "]", e.getMessage()); } if (storePrev4FS) try { if (!FxFileUtils.copyStream2File(prev4Length, rs.getBinaryStream(4), FxBinaryUtils.createBinaryFile(divisionId, created.getId(), created.getVersion(), created.getQuality(), PreviewSizes.SCREENVIEW.getBlobIndex()))) throw new FxDbException(LOG, "ex.content.binary.fsCopyFailed", created.getId() + "[" + PreviewSizes.SCREENVIEW.getBlobIndex() + "]"); } catch (IOException e) { throw new FxDbException(LOG, "ex.content.binary.fsCopyFailedError", created.getId() + "[" + PreviewSizes.SCREENVIEW.getBlobIndex() + "]", e.getMessage()); } } con.commit(); } catch (SQLException e) { throw new FxDbException(e, "ex.db.sqlError", e.getMessage()); } catch (FileNotFoundException e) { throw new FxDbException(e, "ex.content.binary.IOError", binary.getHandle()); } finally { Database.closeObjects(GenericBinarySQLStorage.class, con, ps); FxSharedUtils.close(fis); } return created; }
From source file:com.gtwm.pb.model.manageData.DataManagement.java
public int importCSV(HttpServletRequest request, TableInfo table, boolean updateExistingRecords, BaseField recordIdentifierField, boolean generateRowIds, char separator, char quotechar, int numHeaderLines, boolean useRelationDisplayValues, boolean importSequenceValues, boolean requireExactRelationMatches, boolean trim, boolean merge, List<FileItem> multipartItems, String csvContent) throws SQLException, InputRecordException, IOException, CantDoThatException, ObjectNotFoundException, DisallowedException, CodingErrorException { if (!FileUpload.isMultipartContent(new ServletRequestContext(request))) { if (csvContent == null) { throw new CantDoThatException( "To import CSV content, a file must be uploaded (form posted as multi-part) or csv_content specified"); }//from w w w .j ava2s .c o m } int numImportedRecords = 0; // get field set to import into. LinkedHashSet to ensure order is // retained so the right values are imported into the right fields LinkedHashSet<BaseField> fields = new LinkedHashSet<BaseField>(table.getFields()); // if row IDs aren't included in the data to import, remove ID from the // field set BaseField primaryKey = table.getPrimaryKey(); if (recordIdentifierField == null) { recordIdentifierField = primaryKey; } if (generateRowIds || (updateExistingRecords && !recordIdentifierField.equals(primaryKey))) { fields.remove(primaryKey); } Map<RelationField, Map<String, String>> relationLookups = new HashMap<RelationField, Map<String, String>>(); // Remove fields which shouldn't be modified during the import // For serial fields, if we need to set serial values explicitly, this // will have to be dealt with later for (BaseField field : table.getFields()) { if (field instanceof SequenceField && (!field.equals(primaryKey)) && (!importSequenceValues)) { fields.remove(field); } else if (field.getHidden()) { if (field.getFieldName().equals(HiddenFields.VIEW_COUNT.getFieldName()) || field.getFieldName().equals(HiddenFields.COMMENTS_FEED.getFieldName())) { fields.remove(field); } else if (updateExistingRecords) { if (field.getFieldName().equals(HiddenFields.DATE_CREATED.getFieldName()) || field.getFieldName().equals(HiddenFields.CREATED_BY.getFieldName())) { fields.remove(field); } } } else if (!field.getFieldCategory().savesData()) { fields.remove(field); } // Also, if importing relations by display value, look up // display/internal value mappings if (useRelationDisplayValues && field instanceof RelationField) { Map<String, String> displayToInternalValue = ((RelationFieldDefn) field).getItems(true, false); relationLookups.put((RelationField) field, displayToInternalValue); } } // Prepare SQL String insertSQLCode = null; String updateSQLCode = null; String logCreationSQLCode = null; // If updating, we'll need a record ID value. Depending on what the // identifier field is, this could be one of a couple of different types String recordIdentifierString = null; Integer recordIdentifierInteger = null; int recordIdentifierFieldNum = 0; DatabaseFieldType identifierFieldDbType = null; if (updateExistingRecords) { identifierFieldDbType = recordIdentifierField.getDbType(); if (!identifierFieldDbType.equals(DatabaseFieldType.VARCHAR) && !identifierFieldDbType.equals(DatabaseFieldType.INTEGER) && !identifierFieldDbType.equals(DatabaseFieldType.SERIAL)) { throw new CantDoThatException("The record identifier field has to be text or a whole number, " + recordIdentifierField + " is a " + identifierFieldDbType); } updateSQLCode = "UPDATE " + table.getInternalTableName() + " SET "; int fieldNum = 0; for (BaseField field : fields) { fieldNum += 1; if (merge) { // Update database only if there's a non-null value from the // spreadsheet updateSQLCode += field.getInternalFieldName() + " = COALESCE(?," + field.getInternalFieldName() + "), "; } else { updateSQLCode += field.getInternalFieldName() + " = ?, "; } if (field.equals(recordIdentifierField)) { recordIdentifierFieldNum = fieldNum; } } if (recordIdentifierFieldNum == 0) { throw new CantDoThatException("Can't find the field specified as record identifier (" + recordIdentifierField + ") in the list of table fields " + fields + " in table " + table); } updateSQLCode = updateSQLCode.substring(0, updateSQLCode.length() - 2); updateSQLCode += " WHERE " + recordIdentifierField.getInternalFieldName() + "=?"; logCreationSQLCode = "UPDATE " + table.getInternalTableName() + " SET " + table.getField(HiddenFields.DATE_CREATED.getFieldName()).getInternalFieldName() + "=?, " + table.getField(HiddenFields.CREATED_BY.getFieldName()).getInternalFieldName() + "=? WHERE " + primaryKey.getInternalFieldName() + "=?"; } insertSQLCode = "INSERT INTO " + table.getInternalTableName() + "("; String placeholders = ""; for (BaseField field : fields) { insertSQLCode += field.getInternalFieldName() + ", "; placeholders += "?, "; } placeholders = placeholders.substring(0, placeholders.length() - 2); insertSQLCode = insertSQLCode.substring(0, insertSQLCode.length() - 2) + ") VALUES (" + placeholders + ")"; // Find content to import Reader inputStreamReader = null; if (csvContent != null) { inputStreamReader = new StringReader(csvContent); } else { for (FileItem item : multipartItems) { // if item is a file if (!item.isFormField()) { if (item.getName().toLowerCase().endsWith(".xls")) { throw new CantDoThatException( "You need to upload as a CSV to import, Excel files can't be imported directly"); } inputStreamReader = new InputStreamReader(item.getInputStream()); break; } } } if (inputStreamReader == null) { throw new CantDoThatException("No file uploaded"); } CSVReader csvReader = new CSVReader(inputStreamReader, separator, quotechar, numHeaderLines); // returns a list of String arrays List<String[]> csvLines = (List<String[]>) csvReader.readAll(); // do db inserts Connection conn = null; PreparedStatement statement = null; // backupInsertStatement is for when an update returns 0 rows affected, // i.e. there's no matching row. In this case, do an insert PreparedStatement backupInsertStatement = null; PreparedStatement logCreationStatement = null; // These two variables used in exception handling int importLine = 0; BaseField fieldImported = null; Timestamp importTime = new Timestamp(System.currentTimeMillis()); AppUserInfo loggedInUser = authManager.getUserByUserName(request, request.getRemoteUser()); String fullname = loggedInUser.getForename() + " " + loggedInUser.getSurname() + " (" + loggedInUser.getUserName() + ")"; try { conn = this.dataSource.getConnection(); conn.setAutoCommit(false); if (updateExistingRecords) { statement = conn.prepareStatement(updateSQLCode); backupInsertStatement = conn.prepareStatement(insertSQLCode); logCreationStatement = conn.prepareStatement(logCreationSQLCode); } else { statement = conn.prepareStatement(insertSQLCode); } CSVLINE: for (String[] csvLineArray : csvLines) { // convert to an object rather than a primitive array - // easier to work with List<String> lineValues = Arrays.asList(csvLineArray); importLine++; // skip blank lines if (lineValues.size() == 1) { if (lineValues.get(0).length() == 0) { continue CSVLINE; } } int fieldNum = 0; for (BaseField field : fields) { fieldImported = field; fieldNum++; if (field.getHidden()) { String fieldName = field.getFieldName(); if (fieldName.equals(HiddenFields.LOCKED.getFieldName())) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else if (fieldName.equals(HiddenFields.DATE_CREATED.getFieldName()) || fieldName.equals(HiddenFields.LAST_MODIFIED.getFieldName())) { statement.setTimestamp(fieldNum, importTime); if (updateExistingRecords) { backupInsertStatement.setTimestamp(fieldNum, importTime); } } else if (fieldName.equals(HiddenFields.CREATED_BY.getFieldName()) || fieldName.equals(HiddenFields.MODIFIED_BY.getFieldName())) { statement.setString(fieldNum, fullname); if (updateExistingRecords) { backupInsertStatement.setString(fieldNum, fullname); } } } else if (fieldNum > lineValues.size()) { // booleans have a not null constraint if (field.getDbType().equals(Types.BOOLEAN)) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else { statement.setNull(fieldNum, Types.NULL); if (updateExistingRecords) { backupInsertStatement.setNull(fieldNum, Types.NULL); } } } else { String lineValue = lineValues.get(fieldNum - 1); if (lineValue != null) { if (trim) { lineValue = lineValue.trim(); } if (lineValue.equals("")) { // booleans have a not null constraint if (field.getDbType().equals(Types.BOOLEAN)) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else { statement.setNull(fieldNum, Types.NULL); if (updateExistingRecords) { backupInsertStatement.setNull(fieldNum, Types.NULL); } } } else { if ((field instanceof FileField) && (generateRowIds)) { throw new CantDoThatException( "Cannot generate row ids when importing file names. See line " + importLine + ", field '" + field.getFieldName() + "' with value '" + lineValue + "'"); } switch (field.getDbType()) { case VARCHAR: statement.setString(fieldNum, lineValue); if (updateExistingRecords) { backupInsertStatement.setString(fieldNum, lineValue); if (field.equals(recordIdentifierField)) { recordIdentifierString = lineValue; } } break; case TIMESTAMP: // deal with month and year // resolution dates exported if (lineValue.matches("^[a-zA-Z]{3}\\s\\d{2,4}$")) { lineValue = "01 " + lineValue; } else if (lineValue.matches("^\\d{2,4}")) { lineValue = "01 Jan " + lineValue; } try { Calendar calValue = CalendarParser.parse(lineValue, CalendarParser.DD_MM_YY); statement.setTimestamp(fieldNum, new Timestamp(calValue.getTimeInMillis())); if (updateExistingRecords) { backupInsertStatement.setTimestamp(fieldNum, new Timestamp(calValue.getTimeInMillis())); } } catch (CalendarParserException cpex) { throw new InputRecordException("Error importing line " + importLine + ", field " + field + ": " + cpex.getMessage(), field, cpex); } break; case FLOAT: lineValue = lineValue.trim().replaceAll("[^\\d\\.\\+\\-eE]", ""); statement.setDouble(fieldNum, Double.valueOf(lineValue)); if (updateExistingRecords) { backupInsertStatement.setDouble(fieldNum, Double.valueOf(lineValue)); } break; case INTEGER: if ((field instanceof RelationField) && (useRelationDisplayValues)) { // find key value for display value RelationField relationField = (RelationField) field; Map<String, String> valueKeyMap = relationLookups.get(relationField); String internalValueString = valueKeyMap.get(lineValue); if (internalValueString == null) { if (!requireExactRelationMatches) { // A very basic fuzzy matching // algorithm String potentialDisplayValue = null; String lineValueLowerCase = lineValue.toLowerCase(); FUZZYMATCH: for (Map.Entry<String, String> entry : valueKeyMap .entrySet()) { potentialDisplayValue = entry.getKey(); if (potentialDisplayValue.toLowerCase() .contains(lineValueLowerCase)) { internalValueString = entry.getValue(); break FUZZYMATCH; } } } if (internalValueString == null) { throw new CantDoThatException("Error importing line " + importLine + ", field " + relationField + ": Can't find a related '" + relationField.getRelatedTable() + "' for " + relationField.getDisplayField() + " '" + lineValue + "'. "); } } int keyValue = Integer.valueOf(internalValueString); statement.setInt(fieldNum, keyValue); if (updateExistingRecords) { backupInsertStatement.setInt(fieldNum, keyValue); if (field.equals(recordIdentifierField)) { recordIdentifierInteger = keyValue; } } } else { lineValue = lineValue.trim().replaceAll("[^\\d\\.\\+\\-eE]", ""); int keyValue = Integer.valueOf(lineValue); statement.setInt(fieldNum, keyValue); if (updateExistingRecords) { backupInsertStatement.setInt(fieldNum, keyValue); if (field.equals(recordIdentifierField)) { recordIdentifierInteger = keyValue; } } } break; case SERIAL: lineValue = lineValue.trim().replaceAll("[^\\d\\.\\+\\-eE]", ""); int keyValue = Integer.valueOf(lineValue); statement.setInt(fieldNum, keyValue); if (updateExistingRecords) { backupInsertStatement.setInt(fieldNum, keyValue); if (field.equals(recordIdentifierField)) { recordIdentifierInteger = keyValue; } } break; case BOOLEAN: boolean filterValueIsTrue = Helpers.valueRepresentsBooleanTrue(lineValue); statement.setBoolean(fieldNum, filterValueIsTrue); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, filterValueIsTrue); } break; } } } else { // booleans have a not null constraint if (field.getDbType().equals(Types.BOOLEAN)) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else { statement.setNull(fieldNum, Types.NULL); if (updateExistingRecords) { backupInsertStatement.setNull(fieldNum, Types.NULL); } } } } } if (updateExistingRecords) { // for potential error messages String recordIdentifierDescription = null; if (identifierFieldDbType.equals(DatabaseFieldType.INTEGER) || identifierFieldDbType.equals(DatabaseFieldType.SERIAL)) { if (recordIdentifierInteger == null) { throw new InputRecordException( "Can't find a record identifier value at line " + importLine, recordIdentifierField); } recordIdentifierDescription = recordIdentifierField.getFieldName() + " = " + recordIdentifierInteger; // Set the 'WHERE recordIdentifier = ?' clause statement.setInt(fields.size() + 1, recordIdentifierInteger); } else { if (recordIdentifierString == null) { throw new InputRecordException( "Can't find a record identifier value at line " + importLine, recordIdentifierField); } recordIdentifierDescription = recordIdentifierField.getFieldName() + " = '" + recordIdentifierString + "'"; // Set the 'WHERE recordIdentifier = ?' clause statement.setString(fields.size() + 1, recordIdentifierString); } int rowsAffected = statement.executeUpdate(); if (rowsAffected == 0) { // If can't find a match to update, insert a record // instead backupInsertStatement.executeUpdate(); // NB Postgres specific code to find Row ID of newly // inserted record, not cross-db compatible String newRowIdSQLCode = "SELECT currval('" + table.getInternalTableName() + "_" + primaryKey.getInternalFieldName() + "_seq')"; PreparedStatement newRowIdStatement = conn.prepareStatement(newRowIdSQLCode); ResultSet newRowIdResults = newRowIdStatement.executeQuery(); if (newRowIdResults.next()) { int newRowId = newRowIdResults.getInt(1); // Add creation metadata to the new row logCreationStatement.setTimestamp(1, importTime); logCreationStatement.setString(2, fullname); logCreationStatement.setInt(3, newRowId); int creationLogRowsAffected = logCreationStatement.executeUpdate(); if (creationLogRowsAffected == 0) { throw new SQLException( "Unable to update creation metadata of newly inserted record, using query " + logCreationStatement); } } else { newRowIdResults.close(); newRowIdStatement.close(); throw new SQLException("Row ID not found for the newly inserted record. '" + newRowIdStatement + "' didn't work"); } newRowIdResults.close(); newRowIdStatement.close(); } else if (rowsAffected > 1) { throw new InputRecordException("Error importing line " + importLine + ". The record identifier field " + recordIdentifierDescription + " should match only 1 record in the database but it actually matches " + rowsAffected, recordIdentifierField); } // reset to null for the next line recordIdentifierString = null; recordIdentifierInteger = null; } else { statement.executeUpdate(); } numImportedRecords += 1; } statement.close(); if (backupInsertStatement != null) { backupInsertStatement.close(); } if (logCreationStatement != null) { logCreationStatement.close(); } // reset the primary key ID sequence so new records can be added resetSequence((SequenceField) primaryKey, conn); // and any other sequence fields if (importSequenceValues) { for (BaseField field : table.getFields()) { if ((!field.equals(primaryKey)) && field instanceof SequenceField) { resetSequence((SequenceField) field, conn); } } } // ANALYZE the table after import if (numImportedRecords > 1000) { Statement analyzeStatement = conn.createStatement(); analyzeStatement.execute("ANALYZE " + table.getInternalTableName()); analyzeStatement.close(); } conn.commit(); } catch (SQLException sqlex) { String databaseErrorMessage = Helpers.replaceInternalNames(sqlex.getMessage(), table.getDefaultReport()); logger.warn("Import failed, statement is " + statement); logger.warn("Backup insert statement is " + backupInsertStatement); String errorMessage = "Error importing CSV line " + importLine; if (!fieldImported.getHidden()) { errorMessage += ", field '" + fieldImported + "'"; } errorMessage += ": " + databaseErrorMessage; throw new InputRecordException(errorMessage, fieldImported, sqlex); } catch (NumberFormatException nfex) { String causeMessage = nfex.getMessage(); causeMessage = causeMessage.replaceAll("For input string", "value"); String errorMessage = "Error parsing number when importing CSV line " + importLine; if (!fieldImported.getHidden()) { errorMessage += ", field '" + fieldImported + "'"; } errorMessage += ": " + causeMessage; throw new InputRecordException(errorMessage, fieldImported, nfex); } finally { if (conn != null) { conn.close(); } } this.logLastDataChangeTime(request); logLastTableDataChangeTime(table); UsageLogger usageLogger = new UsageLogger(this.dataSource); String logMessage = "" + numImportedRecords; if (updateExistingRecords) { logMessage += " records imported"; } else { logMessage += " new records imported"; } if (csvContent != null) { logMessage += " from file"; } usageLogger.logDataChange(loggedInUser, table, null, AppAction.CSV_IMPORT, -1, logMessage); UsageLogger.startLoggingThread(usageLogger); return numImportedRecords; }
From source file:com.l2jfree.gameserver.gameobjects.L2Player.java
private void storeCharBase() { Connection con = null;//from w ww.j av a2 s . c o m try { // Get the exp, level, and sp of base class to store in base table int currentClassIndex = getClassIndex(); _classIndex = 0; long exp = getStat().getExp(); int level = getStat().getLevel(); int sp = getStat().getSp(); _classIndex = currentClassIndex; con = L2DatabaseFactory.getInstance().getConnection(con); // Update base class PreparedStatement statement = con.prepareStatement(UPDATE_CHARACTER); statement.setInt(1, level); statement.setInt(2, getMaxHp()); statement.setDouble(3, getStatus().getCurrentHp()); statement.setInt(4, getMaxCp()); statement.setDouble(5, getStatus().getCurrentCp()); statement.setInt(6, getMaxMp()); statement.setDouble(7, getStatus().getCurrentMp()); statement.setInt(8, getAppearance().getFace()); statement.setInt(9, getAppearance().getHairStyle()); statement.setInt(10, getAppearance().getHairColor()); statement.setInt(11, getAppearance().getSex() ? 1 : 0); statement.setInt(12, getHeading()); statement.setInt(13, _observerMode ? _obsX : getX()); statement.setInt(14, _observerMode ? _obsY : getY()); statement.setInt(15, _observerMode ? _obsZ : getZ()); statement.setLong(16, exp); statement.setLong(17, getExpBeforeDeath()); statement.setInt(18, sp); statement.setInt(19, getKarma()); statement.setInt(20, getFame()); statement.setInt(21, getPvpKills()); statement.setInt(22, getPkKills()); statement.setInt(23, getClanId()); statement.setInt(24, getRace().ordinal()); statement.setInt(25, getClassId().getId()); statement.setLong(26, getDeleteTimer()); statement.setString(27, getTitle()); statement.setInt(28, getAccessLevel()); statement.setInt(29, isOnline()); statement.setInt(30, isIn7sDungeon() ? 1 : 0); statement.setInt(31, getClanPrivileges()); statement.setInt(32, getWantsPeace()); statement.setInt(33, getBaseClass()); statement.setLong(34, getOnlineTime()); statement.setInt(35, isInJail() ? 1 : 0); statement.setLong(36, getJailTimer()); statement.setInt(37, getNewbie()); statement.setInt(38, isNoble() ? 1 : 0); statement.setLong(39, getPledgeRank()); statement.setInt(40, getSubPledgeType()); statement.setInt(41, getLvlJoinedAcademy()); statement.setLong(42, getApprentice()); statement.setLong(43, getSponsor()); statement.setInt(44, getAllianceWithVarkaKetra()); statement.setLong(45, getClanJoinExpiryTime()); statement.setLong(46, getClanCreateExpiryTime()); statement.setLong(47, getBanChatTimer()); statement.setString(48, getName()); statement.setLong(49, getDeathPenaltyBuffLevel()); statement.setInt(50, getVitalityPoints()); statement.setInt(51, getBookMarkSlot()); statement.setInt(52, getObjectId()); statement.execute(); statement.close(); } catch (Exception e) { _log.error("Could not store char base data: ", e); } finally { L2DatabaseFactory.close(con); } }
From source file:com.l2jfree.gameserver.gameobjects.L2Player.java
/** * Create a new player in the characters table of the database.<BR><BR> *///from ww w. j a v a2 s. co m private boolean createDb() { Connection con = null; try { con = L2DatabaseFactory.getInstance().getConnection(con); PreparedStatement statement = con.prepareStatement("INSERT INTO characters " + "(account_name,charId,char_name,level,maxHp,curHp,maxCp,curCp,maxMp,curMp," + "face,hairStyle,hairColor,sex,exp,sp,karma,fame,pvpkills,pkkills,clanid,race," + "classid,deletetime,cancraft,title,accesslevel,online,isin7sdungeon,clan_privs," + "wantspeace,base_class,newbie,nobless,pledge_rank) " + "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"); statement.setString(1, _accountName); statement.setInt(2, getObjectId()); statement.setString(3, getName()); statement.setInt(4, getLevel()); statement.setInt(5, getMaxHp()); statement.setDouble(6, getStatus().getCurrentHp()); statement.setInt(7, getMaxCp()); statement.setDouble(8, getStatus().getCurrentCp()); statement.setInt(9, getMaxMp()); statement.setDouble(10, getStatus().getCurrentMp()); statement.setInt(11, getAppearance().getFace()); statement.setInt(12, getAppearance().getHairStyle()); statement.setInt(13, getAppearance().getHairColor()); statement.setInt(14, getAppearance().getSex() ? 1 : 0); statement.setLong(15, getExp()); statement.setInt(16, getSp()); statement.setInt(17, getKarma()); statement.setInt(18, getFame()); statement.setInt(19, getPvpKills()); statement.setInt(20, getPkKills()); statement.setInt(21, getClanId()); statement.setInt(22, getRace().ordinal()); statement.setInt(23, getClassId().getId()); statement.setLong(24, getDeleteTimer()); statement.setInt(25, hasDwarvenCraft() ? 1 : 0); statement.setString(26, getTitle()); statement.setInt(27, getAccessLevel()); statement.setInt(28, isOnline()); statement.setInt(29, isIn7sDungeon() ? 1 : 0); statement.setInt(30, getClanPrivileges()); statement.setInt(31, getWantsPeace()); statement.setInt(32, getBaseClass()); statement.setInt(33, getNewbie()); statement.setInt(34, isNoble() ? 1 : 0); statement.setLong(35, 0); statement.executeUpdate(); statement.close(); } catch (Exception e) { _log.error("Could not insert char data: ", e); return false; } finally { L2DatabaseFactory.close(con); } return true; }
From source file:com.l2jfrozen.gameserver.model.actor.instance.L2PcInstance.java
/** * Store char base./*from www . j a va2 s. co m*/ */ private synchronized void storeCharBase() { Connection con = null; try { // Get the exp, level, and sp of base class to store in base table final int currentClassIndex = getClassIndex(); _classIndex = 0; final long exp = getStat().getExp(); final int level = getStat().getLevel(); final int sp = getStat().getSp(); _classIndex = currentClassIndex; con = L2DatabaseFactory.getInstance().getConnection(false); PreparedStatement statement; // Update base class statement = con.prepareStatement(UPDATE_CHARACTER); statement.setInt(1, level); statement.setInt(2, getMaxHp()); statement.setDouble(3, getCurrentHp()); statement.setInt(4, getMaxCp()); statement.setDouble(5, getCurrentCp()); statement.setInt(6, getMaxMp()); statement.setDouble(7, getCurrentMp()); statement.setInt(8, getSTR()); statement.setInt(9, getCON()); statement.setInt(10, getDEX()); statement.setInt(11, getINT()); statement.setInt(12, getMEN()); statement.setInt(13, getWIT()); statement.setInt(14, getAppearance().getFace()); statement.setInt(15, getAppearance().getHairStyle()); statement.setInt(16, getAppearance().getHairColor()); statement.setInt(17, getHeading()); statement.setInt(18, _observerMode ? _obsX : getX()); statement.setInt(19, _observerMode ? _obsY : getY()); statement.setInt(20, _observerMode ? _obsZ : getZ()); statement.setLong(21, exp); statement.setLong(22, getExpBeforeDeath()); statement.setInt(23, sp); statement.setInt(24, getKarma()); statement.setInt(25, getPvpKills()); statement.setInt(26, getPkKills()); statement.setInt(27, getRecomHave()); statement.setInt(28, getRecomLeft()); statement.setInt(29, getClanId()); statement.setInt(30, getMaxLoad()); statement.setInt(31, getRace().ordinal()); // if (!isSubClassActive()) // else // statement.setInt(30, getBaseTemplate().race.ordinal()); statement.setInt(32, getClassId().getId()); statement.setLong(33, getDeleteTimer()); statement.setString(34, getTitle()); statement.setInt(35, getAccessLevel().getLevel()); if (_isInOfflineMode || isOnline() == 1) { // in offline mode or online statement.setInt(36, 1); } else statement.setInt(36, isOnline()); // statement.setInt(36, _isOffline ? 0 : isOnline()); statement.setInt(37, isIn7sDungeon() ? 1 : 0); statement.setInt(38, getClanPrivileges()); statement.setInt(39, getWantsPeace()); statement.setInt(40, getBaseClass()); long totalOnlineTime = _onlineTime; if (_onlineBeginTime > 0) { totalOnlineTime += (System.currentTimeMillis() - _onlineBeginTime) / 1000; } statement.setLong(41, totalOnlineTime); // statement.setInt(42, isInJail() ? 1 : 0); // statement.setLong(43, getJailTimer()); statement.setInt(42, getPunishLevel().value()); statement.setLong(43, getPunishTimer()); statement.setInt(44, isNewbie() ? 1 : 0); statement.setInt(45, isNoble() ? 1 : 0); statement.setLong(46, getPowerGrade()); statement.setInt(47, getPledgeType()); statement.setLong(48, getLastRecomUpdate()); statement.setInt(49, getLvlJoinedAcademy()); statement.setLong(50, getApprentice()); statement.setLong(51, getSponsor()); statement.setInt(52, getAllianceWithVarkaKetra()); statement.setLong(53, getClanJoinExpiryTime()); statement.setLong(54, getClanCreateExpiryTime()); statement.setString(55, getName()); statement.setLong(56, getDeathPenaltyBuffLevel()); statement.setInt(57, getPcBangScore()); statement.setString(58, StringToHex(Integer.toHexString(_originalNameColorOffline).toUpperCase())); statement.setString(59, StringToHex(Integer.toHexString(getAppearance().getTitleColor()).toUpperCase())); // TODO allow different colors support to players store // statement.setString(58, StringToHex(Integer.toHexString(getAppearance().getNameColor()).toUpperCase())); // statement.setString(59, StringToHex(Integer.toHexString(getAppearance().getTitleColor()).toUpperCase())); statement.setInt(60, isAio() ? 1 : 0); statement.setLong(61, getAioEndTime()); statement.setInt(62, getObjectId()); statement.execute(); DatabaseUtils.close(statement); statement = null; } catch (final Exception e) { LOGGER.warn("Could not store char base data: "); e.printStackTrace(); } finally { CloseUtil.close(con); } }
From source file:com.l2jfrozen.gameserver.model.actor.instance.L2PcInstance.java
/** * Create a new player in the characters table of the database.<BR> * <BR>/*from www . j a v a 2 s. c om*/ * @return true, if successful */ private boolean createDb() { boolean output = false; Connection con = null; try { con = L2DatabaseFactory.getInstance().getConnection(false); PreparedStatement statement; statement = con.prepareStatement("INSERT INTO characters " + "(account_name,obj_Id,char_name,level,maxHp,curHp,maxCp,curCp,maxMp,curMp," + "acc,crit,evasion,mAtk,mDef,mSpd,pAtk,pDef,pSpd,runSpd,walkSpd," + "str,con,dex,_int,men,wit,face,hairStyle,hairColor,sex," + "movement_multiplier,attack_speed_multiplier,colRad,colHeight," + "exp,sp,karma,pvpkills,pkkills,clanid,maxload,race,classid,deletetime," + "cancraft,title,accesslevel,online,isin7sdungeon,clan_privs,wantspeace," + "base_class,newbie,nobless,power_grade,last_recom_date"/* * , * banchat_time * , */ + ",name_color,title_color,aio,aio_end) " + "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"); statement.setString(1, _accountName); statement.setInt(2, getObjectId()); statement.setString(3, getName()); statement.setInt(4, getLevel()); statement.setInt(5, getMaxHp()); statement.setDouble(6, getCurrentHp()); statement.setInt(7, getMaxCp()); statement.setDouble(8, getCurrentCp()); statement.setInt(9, getMaxMp()); statement.setDouble(10, getCurrentMp()); statement.setInt(11, getAccuracy()); statement.setInt(12, getCriticalHit(null, null)); statement.setInt(13, getEvasionRate(null)); statement.setInt(14, getMAtk(null, null)); statement.setInt(15, getMDef(null, null)); statement.setInt(16, getMAtkSpd()); statement.setInt(17, getPAtk(null)); statement.setInt(18, getPDef(null)); statement.setInt(19, getPAtkSpd()); statement.setInt(20, getRunSpeed()); statement.setInt(21, getWalkSpeed()); statement.setInt(22, getSTR()); statement.setInt(23, getCON()); statement.setInt(24, getDEX()); statement.setInt(25, getINT()); statement.setInt(26, getMEN()); statement.setInt(27, getWIT()); statement.setInt(28, getAppearance().getFace()); statement.setInt(29, getAppearance().getHairStyle()); statement.setInt(30, getAppearance().getHairColor()); statement.setInt(31, getAppearance().getSex() ? 1 : 0); statement.setDouble(32, 1/* getMovementMultiplier() */); statement.setDouble(33, 1/* getAttackSpeedMultiplier() */); statement.setDouble(34, getTemplate().collisionRadius/* getCollisionRadius() */); statement.setDouble(35, getTemplate().collisionHeight/* getCollisionHeight() */); statement.setLong(36, getExp()); statement.setInt(37, getSp()); statement.setInt(38, getKarma()); statement.setInt(39, getPvpKills()); statement.setInt(40, getPkKills()); statement.setInt(41, getClanId()); statement.setInt(42, getMaxLoad()); statement.setInt(43, getRace().ordinal()); statement.setInt(44, getClassId().getId()); statement.setLong(45, getDeleteTimer()); statement.setInt(46, hasDwarvenCraft() ? 1 : 0); statement.setString(47, getTitle()); statement.setInt(48, getAccessLevel().getLevel()); statement.setInt(49, isOnline()); statement.setInt(50, isIn7sDungeon() ? 1 : 0); statement.setInt(51, getClanPrivileges()); statement.setInt(52, getWantsPeace()); statement.setInt(53, getBaseClass()); statement.setInt(54, isNewbie() ? 1 : 0); statement.setInt(55, isNoble() ? 1 : 0); statement.setLong(56, 0); statement.setLong(57, System.currentTimeMillis()); statement.setString(58, StringToHex(Integer.toHexString(getAppearance().getNameColor()).toUpperCase())); statement.setString(59, StringToHex(Integer.toHexString(getAppearance().getTitleColor()).toUpperCase())); statement.setInt(60, isAio() ? 1 : 0); statement.setLong(61, 0); statement.executeUpdate(); DatabaseUtils.close(statement); statement = null; output = true; } catch (final Exception e) { LOGGER.error("Could not insert char data", e); } finally { CloseUtil.close(con); con = null; } if (output) { final String text = "Created new character : " + getName() + " for account: " + _accountName; Log.add(text, "New_chars"); } return output; }
From source file:com.thesmartweb.swebrank.Search_analysis.java
/** * Method to perform the queries to the search engines, get the links and get all the webpage and semantic stats for the links * @param iteration_counter The iteration number of the algorithm (to use it in the id for elasticsearch) * @param directory_save The directory we are going to several files * @param domain The domain that we are searching for (to use it in the id for elasticsearch) * @param enginechoice The search engines that were chosen to be used * @param quer the query we search for// w w w.j av a 2 s .c o m * @param results_number the results number that we are going to get from every search engine * @param top_visible the number of results if we use Visibility score * @param SWebRankSettings the settings for LDA and SwebRank in general (check the ReadInput Class) * @param alpha alpha value of LDA * @param mozMetrics the metrics of choice if Moz is going to be used * @param top_count_moz the amount of results if we use Moz * @param moz_threshold_option flag to show if we are going to use a threshold in Moz metrics or not * @param moz_threshold the moz threshold value * @param ContentSemantics get the choice of Content Semantic Analysis algorithm that we are going to use * @param SensebotConcepts the amount of concepts to be recognized if Sensebot is used * @param config_path the configuration path to get all the api keys * @return a list with the words recognized as important by the content semantic analysis algorithm we have chosen */ public List<String> perform(int iteration_counter, String directory_save, String domain, List<Boolean> enginechoice, String quer, int results_number, int top_visible, List<Double> SWebRankSettings, double alpha, List<Boolean> mozMetrics, int top_count_moz, boolean moz_threshold_option, double moz_threshold, List<Boolean> ContentSemantics, int SensebotConcepts, String config_path) { //=======connect to mysql========= Connection conn = null; PreparedStatement stmt = null; try { ReadInput ri = new ReadInput(); List<String> mysqlAdminSettings = ri.GetKeyFile(config_path, "mysqlAdmin"); String port = mysqlAdminSettings.get(2); String dbname = mysqlAdminSettings.get(3); String url = "jdbc:mysql://localhost:" + port + "/" + dbname + "?zeroDateTimeBehavior=convertToNull"; String user = mysqlAdminSettings.get(0); String password = mysqlAdminSettings.get(1); System.out.println("Connecting to database..."); conn = DriverManager.getConnection(url, user, password); LinksParseAnalysis ld = new LinksParseAnalysis(); //we create the array that are going to store the results from each search engine String[] links_google = new String[results_number]; String[] links_yahoo = new String[results_number]; String[] links_bing = new String[results_number]; //we create the array that is going to store all the results from all the search engines together String[] links_total = new String[(results_number * 3)]; //--------if we have selected to use a Moz metric, then we should set the links_total to be of size of top_count_seomoz*3 since it means that the results_number has been set to its max value (50) if (mozMetrics.get(0)) { links_total = new String[(top_count_moz) * 3]; } int[] nlinks = new int[2]; if (enginechoice.get(0)) { //get bing results BingResults br = new BingResults(); links_bing = br.Get(quer, results_number, directory_save, config_path); } if (enginechoice.get(1)) { //get google results GoogleResults gr = new GoogleResults(); links_google = gr.Get(quer, results_number, directory_save, config_path); } if (enginechoice.get(2)) { //get yahoo results YahooResults yr = new YahooResults(); links_yahoo = yr.Get(quer, results_number, directory_save, config_path); } HashMap<Integer, List<String>> EntitiesMapDBP = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapDBP = new HashMap<>(); HashMap<Integer, List<String>> EntitiesMapDand = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapDand = new HashMap<>(); HashMap<Integer, List<String>> EntitiesMapYahoo = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapYahoo = new HashMap<>(); HashMap<Integer, String> parseOutputList = new HashMap<>(); for (int i = 0; i < results_number * 3; i++) { parseOutputList.put(i, ""); } //************* boolean false_flag = true; if (false_flag) { if (mozMetrics.get(0)) { //we check if moz works Moz moz = new Moz(); boolean checkmoz = moz.check(config_path); if (checkmoz) { //perform if (links_yahoo.length > 0) { links_yahoo = moz.perform(links_yahoo, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } if (links_google.length > 0) { links_google = moz.perform(links_google, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } if (links_bing.length > 0) { links_bing = moz.perform(links_bing, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } } } //we are creating Sindice class in order to get the number of semantic triples of a webpage Sindice striple = new Sindice(); //create htmlparser to get the number of links in a webpage if (mozMetrics.get(0)) { results_number = links_yahoo.length; } WebParser htm = new WebParser(); //create an array that contains all the links together for (int i = 0; i < 3; i++) { try { if (i == 0) { System.arraycopy(links_yahoo, 0, links_total, 0, results_number); } if (i == 1) { System.arraycopy(links_google, 0, links_total, links_yahoo.length, results_number); } if (i == 2) { System.arraycopy(links_bing, 0, links_total, ((links_yahoo.length) + (links_google.length)), results_number); } } catch (ArrayIndexOutOfBoundsException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<String>(); return finalList; } } //merged true => visibility score if (enginechoice.get(3)) { VisibilityScore vb = new VisibilityScore();//we have a merged engine //erase using vb.perform all the duplicate links links_total = vb.perform(links_google, links_yahoo, links_bing); //if we have Moz option set to true we have to get the results rearranged according to the moz metric selected if (mozMetrics.get(0)) { Moz checkMoz = new Moz(); boolean check_seo = checkMoz.check(config_path); if (check_seo) { Moz MOZ = new Moz(); links_total = MOZ.perform(links_total, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } } //here we calculate the visibility score links_total = vb.visibility_score(links_total, links_yahoo, links_bing, links_google, top_visible); } String[][] total_catent = new String[links_total.length][2]; for (int r = 0; r < total_catent.length; r++) { total_catent[r][0] = ""; total_catent[r][1] = ""; } for (int j = 0; j < links_total.length; j++) { if (links_total[j] != null) { String urlString = links_total[j]; if (urlString.length() > 199) { urlString = links_total[j].substring(0, 198); } int rank = -1; int engine = -1;//0 for yahoo,1 for google,2 for bing if (j < results_number) { rank = j; engine = 0; } else if (j < results_number * 2) { rank = j - results_number; engine = 1; } else if (j < results_number * 3) { rank = j - results_number * 2; engine = 2; } try { //we initialize the row in settings table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO SETTINGS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we initialize the row in semantic stats table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO SEMANTICSTATS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we initialize the row in namespaces stats table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO NAMESPACESSTATS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we put the info inside the settings conn = DriverManager.getConnection(url, user, password); StringBuilder settingsStmBuild = new StringBuilder(); settingsStmBuild.append("UPDATE SETTINGS SET "); settingsStmBuild.append("`nTopics`=? , "); settingsStmBuild.append("`alpha`=? , "); settingsStmBuild.append("`beta`=? , "); settingsStmBuild.append("`niters`=? , "); settingsStmBuild.append("`prob_threshold`=? , "); settingsStmBuild.append("`moz`=? , "); settingsStmBuild.append("`top_count_moz`=? , "); settingsStmBuild.append("`moz_threshold`=? , "); settingsStmBuild.append("`moz_threshold_option`=? , "); settingsStmBuild.append("`top_visible`=? , "); settingsStmBuild.append("`Domain_Authority`=? , "); settingsStmBuild.append("`External_MozRank`=? , "); settingsStmBuild.append("`MozRank`=? , "); settingsStmBuild.append("`MozTrust`=? , "); settingsStmBuild.append("`Page_Authority`=? , "); settingsStmBuild.append("`Subdomain_mozRank`=? , "); settingsStmBuild.append("`merged`=? , "); settingsStmBuild.append("`results_number`=? , "); settingsStmBuild.append("`Diffbotflag`=? , "); settingsStmBuild.append("`LDAflag`=? , "); settingsStmBuild.append("`Sensebotflag`=? , "); settingsStmBuild.append("`TFIDFflag`=? , "); settingsStmBuild.append("`SensebotConcepts`=? , "); settingsStmBuild.append("`nTopTopics`=? , "); settingsStmBuild.append("`combinelimit`=? ,"); settingsStmBuild.append("`newtermstocombine`=? ,"); settingsStmBuild.append("`newqueriesmax`=? ,"); settingsStmBuild.append("`ngdthreshold`=? ,"); settingsStmBuild.append("`entitiesconfi`=? ,"); settingsStmBuild.append("`dbpediasup`=? "); settingsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(settingsStmBuild.toString()); stmt.setInt(1, SWebRankSettings.get(1).intValue()); stmt.setDouble(2, alpha); stmt.setDouble(3, SWebRankSettings.get(0)); stmt.setInt(4, SWebRankSettings.get(2).intValue()); stmt.setDouble(5, SWebRankSettings.get(3)); stmt.setBoolean(6, mozMetrics.get(0)); stmt.setInt(7, top_count_moz); stmt.setDouble(8, moz_threshold); stmt.setBoolean(9, moz_threshold_option); stmt.setInt(10, top_visible); stmt.setBoolean(11, mozMetrics.get(1)); stmt.setBoolean(12, mozMetrics.get(2)); stmt.setBoolean(13, mozMetrics.get(3)); stmt.setBoolean(14, mozMetrics.get(4)); stmt.setBoolean(15, mozMetrics.get(5)); stmt.setBoolean(16, mozMetrics.get(6)); stmt.setBoolean(17, enginechoice.get(3)); stmt.setInt(18, results_number); stmt.setBoolean(19, ContentSemantics.get(0)); stmt.setBoolean(20, ContentSemantics.get(1)); stmt.setBoolean(21, ContentSemantics.get(2)); stmt.setBoolean(22, ContentSemantics.get(3)); stmt.setInt(23, SensebotConcepts); stmt.setInt(24, SWebRankSettings.get(11).intValue()); stmt.setInt(25, SWebRankSettings.get(7).intValue()); stmt.setInt(26, SWebRankSettings.get(9).intValue()); stmt.setInt(27, SWebRankSettings.get(10).intValue()); stmt.setDouble(28, SWebRankSettings.get(6)); stmt.setDouble(29, SWebRankSettings.get(12)); stmt.setDouble(30, SWebRankSettings.get(13)); stmt.setString(31, urlString); stmt.setString(32, quer); stmt.setInt(33, engine); stmt.setString(34, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } if (htm.checkconn(links_total[j])) {//if we can connect to the url we continue to update semantics stats and namespaces stats tables nlinks = htm.getnlinks(links_total[j]); StringBuilder webstatsStmBuild = new StringBuilder(); try { conn = DriverManager.getConnection(url, user, password); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`number_links`=? , "); webstatsStmBuild.append("`redirect_links`=? , "); webstatsStmBuild.append("`internal_links`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, nlinks[0]);//total numbers of links stmt.setInt(2, nlinks[0] - nlinks[1]); stmt.setInt(3, nlinks[1]);//internal links stmt.setString(4, urlString); stmt.setString(5, quer); stmt.setInt(6, engine); stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); System.out.println("I am going to get the stats from Sindice\n"); int ntriples = striple.getsindicestats(links_total[j]);//get the amount of semantic triples using Sindice API System.out.println("I am going insert the semantic triples number in the DB\n"); stmt = conn.prepareStatement( "UPDATE SEMANTICSTATS SET `total_semantic_triples`=? WHERE `url` =? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setInt(1, ntriples); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); System.out.println("I inserted the semantic triples number in the DB\n"); //---namespaces----- System.out.println("I am going to insert the namespaces in the DB\n"); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } boolean flagStriple = false; if (flagStriple) { if (striple.namespaces[0]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/vocab/bio/0.1/` = ? WHERE `url` = ? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[1]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/dc/elements/1.1/` =? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[2]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/coo/n` = ? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[3]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://web.resource.org/cc/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[4]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://diligentarguont.ontoware.org/2005/10/arguonto`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[5]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://usefulinc.com/ns/doap`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[6]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://xmlns.com/foaf/0.1/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[7]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/goodrelations/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[8]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/muto/core`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[9]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://webns.net/mvcb/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[10]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/ontology/mo/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[11]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/innovation/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[12]) { try { stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://openguid.net/rdf`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[13]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.slamka.cz/ontologies/diagnostika.owl`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[14]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/ontology/po/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[15]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/net/provenance/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[16]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/rss/1.0/modules/syndication`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[17]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[18]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://madskills.com/public/xml/rss/module/trackback/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[19]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/ns/void`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[20]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.fzi.de/2008/wise/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[21]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://xmlns.com/wot/0.1`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[22]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/1999/02/22-rdf-syntax-ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[23]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `rdf-schema`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[24]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `XMLschema`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[25]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `OWL`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[26]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/dc/terms/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[27]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `VCARD`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[28]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.geonames.org/ontology`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[29]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://search.yahoo.com/searchmonkey/commerce/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[30]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://search.yahoo.com/searchmonkey/media/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[31]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://cb.semsol.org/ns#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[32]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://blogs.yandex.ru/schema/foaf/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[33]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/2003/01/geo/wgs84_pos#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[34]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/ns#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[35]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/types#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[36]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://smw.ontoware.org/2005/smw#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[37]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/rss/1.0/`= ? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[38]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/2004/12/q/contentlabel#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } System.out.println("I inserted the namespaces in the DB\n"); System.out.println("I will get the semantic entities and categories\n"); //get the semantic entities and categories from Yahoo Content Analysis Service YahooEntityCategory yec = new YahooEntityCategory(); yec.connect(links_total[j], quer, false, SWebRankSettings.get(12));//without stemming EntitiesMapYahoo.put(j, yec.GetEntitiesYahoo()); CategoriesMapYahoo.put(j, yec.GetCategoriesYahoo()); double ent_avg_yahoo_score = yec.GetEntitiesYahooScore(); double cat_avg_yahoo_score = yec.GetCategoriesYahooScore(); int cat_cnt = yec.GetCatQuerCnt(); int ent_cnt = yec.GetEntQuerCnt(); int cat_cnt_whole = yec.GetCatQuerCntWhole(); int ent_cnt_whole = yec.GetEntQuerCntWhole(); yec.connect(links_total[j], quer, true, SWebRankSettings.get(12));//with stemming int cat_cnt_stem = yec.GetCatQuerCnt(); int ent_cnt_stem = yec.GetEntQuerCnt(); int cat_cnt_whole_stem = yec.GetCatQuerCntWhole(); int ent_cnt_whole_stem = yec.GetEntQuerCntWhole(); //get the semantic entities and categories from Dandelion Named entity extraction API DandelionEntities dec = new DandelionEntities(); dec.connect(links_total[j], quer, false, config_path, SWebRankSettings.get(12));//without stemming EntitiesMapDand.put(j, dec.GetEntitiesDand()); CategoriesMapDand.put(j, dec.GetCategoriesDand()); double ent_avg_d_score = dec.GetEntitiesScoreDand(); int cat_cnt_dand = dec.getCat(); int ent_cnt_dand = dec.getEnt(); int cat_cnt_dand_whole = dec.getCatWhole(); int ent_cnt_dand_whole = dec.getEntWhole(); dec.connect(links_total[j], quer, true, config_path, SWebRankSettings.get(12));//with stemming int cat_cnt_dand_stem = dec.getCat(); int ent_cnt_dand_stem = dec.getEnt(); int cat_cnt_dand_whole_stem = dec.getCatWhole(); int ent_cnt_dand_whole_stem = dec.getEntWhole(); //get the semantic entities and categories from dbpedia spotlight DBpediaSpotlightClient dbpspot = new DBpediaSpotlightClient(SWebRankSettings.get(12), SWebRankSettings.get(13).intValue()); dbpspot.countEntCat(links_total[j], quer, false);//false is not stemming EntitiesMapDBP.put(j, dbpspot.getEntities()); CategoriesMapDBP.put(j, dbpspot.getCategories()); double ent_avg_dbpspot_score = dbpspot.getEntitiesAvgScore(); double ent_max_dbpspot_score = dbpspot.getEntitiesMaxScore(); double ent_min_dbpspot_score = dbpspot.getEntitiesMinScore(); double ent_median_dbpspot_score = dbpspot.getEntitiesMedianScore(); double ent_std_dbpspot_score = dbpspot.getEntitiesStdScore(); double ent_avg_dbpspot_support = dbpspot.getEntitiesAvgSupport(); double ent_max_dbpspot_support = dbpspot.getEntitiesMaxSupport(); double ent_min_dbpspot_support = dbpspot.getEntitiesMinSupport(); double ent_median_dbpspot_support = dbpspot.getEntitiesMedianSupport(); double ent_std_dbpspot_support = dbpspot.getEntitiesStdSupport(); double ent_avg_dbpspot_dif = dbpspot.getEntitiesAvgDif(); double ent_max_dbpspot_dif = dbpspot.getEntitiesMaxDif(); double ent_min_dbpspot_dif = dbpspot.getEntitiesMinDif(); double ent_median_dbpspot_dif = dbpspot.getEntitiesMedianDif(); double ent_std_dbpspot_dif = dbpspot.getEntitiesStdDif(); double unique_ent_cnt_dbpspot = dbpspot.getUniqueEntCnt(); double unique_ent_scoreSum_dbpspot = dbpspot.getUniqueEntScoreSum(); int cat_cnt_dbpspot = dbpspot.getcountCat(); int ent_cnt_dbpspot = dbpspot.getcountEnt(); int cat_cnt_dbpspot_whole = dbpspot.getcountCatWhole(); int ent_cnt_dbpspot_whole = dbpspot.getcountEntWhole(); double ent_sup_cnt_dbpspot = dbpspot.getcountSupEnt(); double ent_sim_cnt_dbpspot = dbpspot.getcountSimEnt(); double ent_dif_cnt_dbpspot = dbpspot.getcountDifEnt(); double high_precision_content_dbpspot = dbpspot.getHighPrecEntities(); dbpspot.countEntCat(links_total[j], quer, true);//true is for stemming int cat_cnt_dbpspot_stem = dbpspot.getcountCat(); int ent_cnt_dbpspot_stem = dbpspot.getcountEnt(); int cat_cnt_dbpspot_whole_stem = dbpspot.getcountCatWhole(); int ent_cnt_dbpspot_whole_stem = dbpspot.getcountEntWhole(); double ent_sup_cnt_dbpspot_stem = dbpspot.getcountSupEnt(); double ent_sim_cnt_dbpspot_stem = dbpspot.getcountSimEnt(); double ent_dif_cnt_dbpspot_stem = dbpspot.getcountDifEnt(); System.out.println("I insert the semantic entities and categories stats in the DB\n"); StringBuilder entitiesStatementBuilder = new StringBuilder(); try { entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_y_score`=?,"); entitiesStatementBuilder.append("`cat_avg_y_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_yahoo_score); stmt.setDouble(2, cat_avg_yahoo_score); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dand_score`=?,"); entitiesStatementBuilder.append("`ent_avg_dbpspot_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_d_score); stmt.setDouble(2, ent_avg_dbpspot_score); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_max_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_max_dbpspot_score); stmt.setDouble(2, ent_min_dbpspot_score); stmt.setDouble(3, ent_median_dbpspot_score); stmt.setDouble(4, ent_std_dbpspot_score); stmt.setString(5, links_total[j]); stmt.setString(6, quer); if (j < results_number) { stmt.setInt(7, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(7, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(7, 2);//2 for bing } stmt.setString(8, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_max_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_support`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_dbpspot_support); stmt.setDouble(2, ent_max_dbpspot_support); stmt.setDouble(3, ent_min_dbpspot_support); stmt.setDouble(4, ent_median_dbpspot_support); stmt.setDouble(5, ent_std_dbpspot_support); stmt.setString(6, links_total[j]); stmt.setString(7, quer); if (j < results_number) { stmt.setInt(8, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(8, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(8, 2);//2 for bing } stmt.setString(9, domain); System.out.println("avg db support" + ent_avg_dbpspot_support); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_max_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_dif`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_dbpspot_dif); stmt.setDouble(2, ent_max_dbpspot_dif); stmt.setDouble(3, ent_min_dbpspot_dif); stmt.setDouble(4, ent_median_dbpspot_dif); stmt.setDouble(5, ent_std_dbpspot_dif); stmt.setString(6, links_total[j]); stmt.setString(7, quer); if (j < results_number) { stmt.setInt(8, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(8, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(8, 2);//2 for bing } stmt.setString(9, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_sup_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`ent_dif_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`ent_sim_cnt_dbpspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_sup_cnt_dbpspot); stmt.setDouble(2, ent_dif_cnt_dbpspot); stmt.setDouble(3, ent_sim_cnt_dbpspot); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_sup_cnt_dbpspot_stem`=?,"); entitiesStatementBuilder.append("`ent_dif_cnt_dbpspot_stem`=?,"); entitiesStatementBuilder.append("`ent_sim_cnt_dbpspot_stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_sup_cnt_dbpspot_stem); stmt.setDouble(2, ent_dif_cnt_dbpspot_stem); stmt.setDouble(3, ent_sim_cnt_dbpspot_stem); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`unique_ent_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`unique_ent_scoreSum_dbpspot`=?,"); entitiesStatementBuilder.append("`high_precision_content_dbpspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, unique_ent_cnt_dbpspot); stmt.setDouble(2, unique_ent_scoreSum_dbpspot); stmt.setDouble(3, high_precision_content_dbpspot); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y`=?,"); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt); stmt.setInt(2, ent_cnt); stmt.setInt(3, cat_cnt_whole); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_W`=?,"); entitiesStatementBuilder.append("`Categories_Contained_Query_D`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, ent_cnt_whole); stmt.setInt(2, cat_cnt_dand); stmt.setInt(3, ent_cnt_dand); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_W`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_whole); stmt.setInt(2, ent_cnt_dand_whole); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot); stmt.setInt(2, ent_cnt_dbpspot); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_W`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_whole); stmt.setInt(2, ent_cnt_dbpspot_whole); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_stem); stmt.setInt(2, ent_cnt_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_whole_stem); stmt.setInt(2, ent_cnt_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_stem); stmt.setInt(2, ent_cnt_dand_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_whole_stem); stmt.setInt(2, ent_cnt_dand_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_stem); stmt.setInt(2, ent_cnt_dbpspot_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_whole_stem); stmt.setInt(2, ent_cnt_dbpspot_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } System.out.println("I inserted the semantic entities and categories stats in the DB\n"); System.out.println( "I will get the html stats for the " + j + " link:" + links_total[j] + "\n"); boolean flag_htmlstats = htm.gethtmlstats(links_total[j]);//get the semantic stats from the html code if (flag_htmlstats) { System.out.println( "I got the html stats for the " + j + " link:" + links_total[j] + "\n"); int scripts_cnt = htm.scripts_number; int nschem = htm.nschem; int hreln = htm.hreln; int total_micron = htm.total_micron; int micron1 = htm.micron1; int micron2 = htm.micron2; int microd = htm.microd; System.out.println("I will insert webstats in the DB\n"); webstatsStmBuild.setLength(0); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`scripts_cnt`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, scripts_cnt); stmt.setString(2, urlString); stmt.setString(3, quer); if (j < results_number) { stmt.setInt(4, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(4, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(4, 2);//2 for bing } stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); System.out.println("I inserted webstats in the DB\n"); System.out.println("I will insert semantic stats in the DB\n"); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`schema.org_entities`=? , "); semanticstatsStmBuild.append("`hreltags`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, nschem); stmt.setInt(2, hreln); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`total_microformats`=? , "); semanticstatsStmBuild.append("`Microformats-1`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, total_micron); stmt.setInt(2, micron1); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`Microformats-2`=? , "); semanticstatsStmBuild.append("`Microdata`=? , "); semanticstatsStmBuild.append("`FOAF_HTML`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, micron2); stmt.setInt(2, microd); stmt.setInt(3, htm.foaf); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } System.out.println("I inserted semantic stats in the DB\n"); } } } } String[] parse_output; if (ContentSemantics.get(3) || ContentSemantics.get(1)) { //we perform LDA or TFIDF analysis to the links obtained if (!enginechoice.get(3)) { if (enginechoice.get(2)) {//Yahoo parse_output = ld.perform(links_yahoo, domain, "yahoo", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = 0; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } if (enginechoice.get(1)) {//Google parse_output = ld.perform(links_google, domain, "google", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = results_number; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } if (enginechoice.get(0)) {//Bing parse_output = ld.perform(links_bing, domain, "bing", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = results_number * 2; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } } /*else{ System.gc();//links_total parse_output=ld.perform(links_total, domain, "merged", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(),"Merged",ContentSemantics.get(1),ContentSemantics.get(3), config_path); Collections.addAll(parseOutputList, parse_output); System.gc(); }*/ } } System.gc(); List<String> wordList = null; //hashmap for every engine, with topics, words and probability of each word HashMap<String, HashMap<Integer, HashMap<String, Double>>> enginetopicwordprobmap = new HashMap<>(); List<String> lda_output = new ArrayList<>(); if (ContentSemantics.get(3)) { //get the top content from TFIDF System.out.println("i ll try to read the keys"); wordList = ld.return_topWordsTFIDF(); System.out.println("i returned the wordlist to search analysis"); } else if (ContentSemantics.get(0)) {//get the wordlist from Diffbot Diffbot db = new Diffbot(); wordList = db.compute(links_total, directory_save, config_path); } else if (ContentSemantics.get(2)) {//get the wordllist from Sensebot Sensebot sb = new Sensebot(); wordList = sb.compute(links_total, directory_save, SensebotConcepts, config_path); } else { //get the top content from LDA System.out.println("i ll try to read the keys"); LDAtopicsWords rk = new LDAtopicsWords(); enginetopicwordprobmap = rk.readFile(directory_save, SWebRankSettings.get(4), SWebRankSettings.get(3).intValue(), SWebRankSettings.get(1).intValue(), SWebRankSettings.get(11).intValue()); JSONArray ArrayEngineLevel = new JSONArray(); List<String> ids = new ArrayList<>(); //Node node = nodeBuilder().client(true).clusterName("lshrankldacluster").node(); //Client client = node.client(); Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", "lshrankldacluster") .build(); Client client = new TransportClient(settings) .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)); //save in elastic search the produced by LDA distributions of words over topics for every engine for (String engine : enginetopicwordprobmap.keySet()) { HashMap<Integer, HashMap<String, Double>> topicwordprobmap = new HashMap<>(); topicwordprobmap = enginetopicwordprobmap.get(engine); JSONObject objEngineLevel = new JSONObject(); JSONArray ArrayTopicLevel = new JSONArray(); //for every topic get the words and their probability for (Integer topicindex : topicwordprobmap.keySet()) { JSONObject objTopicLevel = new JSONObject(); objTopicLevel.put("topic", topicindex); JSONObject objmap = new JSONObject(topicwordprobmap.get(topicindex)); Set keySet = objmap.keySet(); Iterator iterator = keySet.iterator(); while (iterator.hasNext()) { String word = iterator.next().toString(); if (!lda_output.contains(word)) { lda_output.add(word); } //get the words in a separate list } objTopicLevel.put("wordsmap", objmap);//write the words in elastic search ArrayTopicLevel.add(objTopicLevel); } objEngineLevel.put("engine", engine); objEngineLevel.put("query", quer); objEngineLevel.put("domain", domain); objEngineLevel.put("iteration", iteration_counter); objEngineLevel.put("TopicsWordMap", ArrayTopicLevel); ArrayEngineLevel.add(objEngineLevel); String id = domain + "/" + quer + "/" + engine + "/" + iteration_counter;//create unique id for the elasticsearch document ids.add(id);//add to the ids list which contains the ids of the current round List<String> elasticIndexes = ri.GetKeyFile(config_path, "elasticSearchIndexes"); IndexRequest indexReq = new IndexRequest(elasticIndexes.get(2), "content", id); indexReq.source(objEngineLevel); IndexResponse indexRes = client.index(indexReq).actionGet(); } //node.close(); client.close(); ElasticGetWordList elasticGetwordList = new ElasticGetWordList();//get the wordlist from elastic search for the ids from the current round wordList = elasticGetwordList.get(ids, config_path); DataManipulation datamanipulation = new DataManipulation(); wordList = datamanipulation.clearListString(wordList); System.out.println("i returned the wordlist to search analysis"); } //get some stats regarding the entities, categories and parsed content from each link comparing it to the top words produced by lda for (int j = 0; j < links_total.length; j++) { if (links_total[j] != null) { String urlString = links_total[j]; if (urlString.length() > 199) { urlString = links_total[j].substring(0, 198); } int rank = -1; int engine = -1;//0 for yahoo,1 for google,2 for bing if (j < results_number) { rank = j; engine = 0; } else if (j < results_number * 2) { rank = j - results_number; engine = 1; } else if (j < results_number * 3) { rank = j - results_number * 2; engine = 2; } LDAsemStats ldaSemStats = new LDAsemStats();//get the stats by comparing the top words produced by LDA and the parsed content //check the LDAsemStats class for more StringBuilder webstatsStmBuild = new StringBuilder(); if (!parseOutputList.isEmpty()) { if (!parseOutputList.get(j).equalsIgnoreCase("") && !parseOutputList.get(j).equalsIgnoreCase("null") && (parseOutputList.get(j).length() > 0)) { ldaSemStats.getTopWordsStats(parseOutputList.get(j), lda_output, false);//without stemming int top_words_lda = ldaSemStats.getTopStats(); double top_words_lda_per = ldaSemStats.getTopPercentageStats(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`top_words_lda`=? , "); webstatsStmBuild.append("`top_words_lda_per`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, top_words_lda); stmt.setDouble(2, top_words_lda_per); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getTopWordsStats(parseOutputList.get(j), lda_output, true);//with stemming int top_words_lda_stem = ldaSemStats.getTopStats(); double top_words_lda_per_stem = ldaSemStats.getTopPercentageStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`top_words_lda_stem`=? , "); webstatsStmBuild.append("`top_words_lda_per_stem`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, top_words_lda_stem); stmt.setDouble(2, top_words_lda_per_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } if (EntitiesMapDBP.get(j) != null && CategoriesMapDBP.get(j) != null) { //we are going to check if semantic entities and categories recognized exist in the lda words recognized as prominent //we are going to use DBPEDIA spotligh and Dandelion named Entity Extraction API //and stemming through Snowball Stemmer ldaSemStats.getEntCatStats(EntitiesMapDBP.get(j), CategoriesMapDBP.get(j), lda_output, false); int ent_cnt_dbpspot_lda = ldaSemStats.getEntStats(); int cat_cnt_dbpspot_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dbpspot_lda`=? , "); webstatsStmBuild.append("`cat_cnt_dbpspot_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dbpspot_lda); stmt.setInt(2, cat_cnt_dbpspot_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapDBP.get(j), CategoriesMapDBP.get(j), lda_output, true); int ent_cnt_dbpspot_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_dbpspot_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dbpspot_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_dbpspot_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dbpspot_lda_stem); stmt.setInt(2, cat_cnt_dbpspot_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (EntitiesMapDand.get(j) != null && CategoriesMapDand.get(j) != null) { ldaSemStats.getEntCatStats(EntitiesMapDand.get(j), CategoriesMapDand.get(j), lda_output, false); int ent_cnt_dand_lda = ldaSemStats.getEntStats(); int cat_cnt_dand_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dand_lda`=? , "); webstatsStmBuild.append("`cat_cnt_dand_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dand_lda); stmt.setInt(2, cat_cnt_dand_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapDand.get(j), CategoriesMapDand.get(j), lda_output, true); int ent_cnt_dand_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_dand_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dand_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_dand_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dand_lda_stem); stmt.setInt(2, cat_cnt_dand_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (EntitiesMapYahoo.get(j) != null && CategoriesMapYahoo.get(j) != null) { //we are going to check if semantic entities and categories recognized exist in the lda words recognized as prominent //we are going to use DBPEDIA spotligh and Dandelion named Entity Extraction API //and stemming through Snowball Stemmer ldaSemStats.getEntCatStats(EntitiesMapYahoo.get(j), CategoriesMapYahoo.get(j), lda_output, false); int ent_cnt_y_lda = ldaSemStats.getEntStats(); int cat_cnt_y_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_y_lda`=? , "); webstatsStmBuild.append("`cat_cnt_y_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_y_lda); stmt.setInt(2, cat_cnt_y_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapYahoo.get(j), CategoriesMapYahoo.get(j), lda_output, true); int ent_cnt_y_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_y_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_y_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_y_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_y_lda_stem); stmt.setInt(2, cat_cnt_y_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } } return wordList; } catch (NullPointerException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<>(); return finalList; } catch (SQLException | ElasticsearchException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<>(); return finalList; } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } }