List of usage examples for java.sql PreparedStatement setBoolean
void setBoolean(int parameterIndex, boolean x) throws SQLException;
boolean value. From source file:ca.sqlpower.matchmaker.address.AddressPool.java
/** * Inserts and updates the contents of the result table with the * {@link AddressResult} contents in this {@link AddressPool}. * AddressResults that are marked as {@link StorageState#DIRTY} are assumed * to be already in the database and are updated. AddressResults that are * marked as {@link StorageState#NEW} are assumed to be new entries that do * no yet exist in the database and are inserted. * /*from w ww .j a v a 2 s. c om*/ * It is worth noting that at the moment, new Address results won't have an * output street number yet (since they have not been validated yet) but a * {@link NullPointerException} gets thrown if we try to insert a null * Integer, so for the time being, I've set the 'null' steet number to be * -1, since I don't believe there's anyone with a negative street number, * but if I'm wrong, this will have to be changed. * * @throws SQLException * @throws SQLObjectException */ public void store(Logger engineLogger, boolean useBatchExecute, boolean debug) throws SQLException, SQLObjectException { setStarted(true); setFinished(false); setCancelled(false); setProgress(0); List<AddressResult> dirtyAddresses = new ArrayList<AddressResult>(); List<AddressResult> deleteAddresses = new ArrayList<AddressResult>(); List<AddressResult> newAddresses = new ArrayList<AddressResult>(); for (List<Object> key : addresses.keySet()) { AddressResult result = addresses.get(key); if (result.getStorageState() == StorageState.DELETE) { deleteAddresses.add(result); } else if (result.getStorageState() == StorageState.DIRTY) { dirtyAddresses.add(result); } else if (result.getStorageState() == StorageState.NEW) { newAddresses.add(result); } } setJobSize(deleteAddresses.size() + dirtyAddresses.size() + newAddresses.size()); engineLogger.debug("# of Delete Address Records:" + deleteAddresses.size()); engineLogger.debug("# of Dirty Address Records:" + dirtyAddresses.size()); engineLogger.debug("# of New Address Records:" + newAddresses.size()); Connection con = null; PreparedStatement ps = null; Statement stmt = null; StringBuilder sql = null; AddressResult result = null; try { con = project.createResultTableConnection(); con.setAutoCommit(false); boolean useBatchUpdates = useBatchExecute && con.getMetaData().supportsBatchUpdates(); SQLTable resultTable = project.getResultTable(); int keySize = project.getSourceTableIndex().getChildCount(); if (deleteAddresses.size() > 0) { stmt = con.createStatement(); for (AddressResult currentResult : deleteAddresses) { sql = new StringBuilder("DELETE FROM "); appendFullyQualifiedTableName(sql, resultTable); sql.append(" WHERE "); int j = 0; for (Object keyValue : currentResult.getKeyValues()) { if (j > 0) { sql.append("AND "); } if (keyValue == null) { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(j).append(" is null "); } else if (keyValue instanceof String || keyValue instanceof Character) { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(j) .append("=" + SQL.quote(keyValue.toString()) + " "); } else { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(j).append("=" + keyValue + " "); } j++; } engineLogger.debug("Preparing the following address result to be deleted: " + currentResult); engineLogger.debug("Executing statement " + sql); stmt.execute(sql.toString()); incrementProgress(); } if (stmt != null) stmt.close(); stmt = null; } Map<String, Integer> columnMetaData = this.getColumnMetaData(engineLogger, resultTable); /* For backward compatibility, see if old column names are being used. * NOTE: the database may return column names as upper case. */ boolean usingNewNames = true; if (columnMetaData.containsKey(OLD_OUTPUT_DELIVERY_INSTALLATION_NAME)) { usingNewNames = false; } engineLogger.debug("Using new shorter names? " + usingNewNames); if (dirtyAddresses.size() > 0) { //First, create and UPDATE PreparedStatement to update dirty records sql = new StringBuilder(); sql.append("UPDATE "); appendFullyQualifiedTableName(sql, resultTable); sql.append(" SET "); sql.append(INPUT_ADDRESS_LINE1).append("=?, "); // 1 sql.append(INPUT_ADDRESS_LINE2).append("=?, "); // 2 sql.append(INPUT_MUNICIPALITY).append("=?, "); // 3 sql.append(INPUT_PROVINCE).append("=?, "); // 4 sql.append(INPUT_COUNTRY).append("=?, "); // 5 sql.append(INPUT_POSTAL_CODE).append("=?, "); // 6 sql.append(OUTPUT_COUNTRY).append("=?, "); // 7 if (usingNewNames) { sql.append(OUTPUT_DELIVERY_INSTALLATION_NAME).append("=?, "); // 8 sql.append(OUTPUT_DELIVERY_INSTALLATION_TYPE).append("=?, "); // 9 } else { sql.append(OLD_OUTPUT_DELIVERY_INSTALLATION_NAME).append("=?, "); // 8 sql.append(OLD_OUTPUT_DELIVERY_INSTALLATION_TYPE).append("=?, "); // 9 } sql.append(OUTPUT_DIRECTION_PREFIX).append("=?, "); // 10 sql.append(OUTPUT_FAILED_PARSING_STRING).append("=?, "); // 11 sql.append(OUTPUT_GENERAL_DELIVERY_NAME).append("=?, "); // 12 sql.append(OUTPUT_LOCK_BOX_NUMBER).append("=?, "); // 13 sql.append(OUTPUT_LOCK_BOX_TYPE).append("=?, "); // 14 sql.append(OUTPUT_MUNICIPALITY).append("=?, "); // 15 sql.append(OUTPUT_POSTAL_CODE).append("=?, "); // 16 sql.append(OUTPUT_PROVINCE).append("=?, "); // 17 sql.append(OUTPUT_RURAL_ROUTE_NUMBER).append("=?, "); // 18 sql.append(OUTPUT_RURAL_ROUTE_TYPE).append("=?, "); // 19 sql.append(OUTPUT_STREET_DIRECTION).append("=?, "); // 20 sql.append(OUTPUT_STREET_NAME).append("=?, "); // 21 sql.append(OUTPUT_STREET_NUMBER).append("=?, "); // 22 sql.append(OUTPUT_STREET_NUMBER_SUFFIX).append("=?, "); // 23 if (usingNewNames) { sql.append(OUTPUT_STREET_NUMBER_SUFFIX_SEPARATE).append("=?, "); // 23.5 } else { sql.append(OLD_OUTPUT_STREET_NUMBER_SUFFIX_SEPARATE).append("=?, "); // 23.5 } sql.append(OUTPUT_STREET_TYPE).append("=?, "); // 24 sql.append(OUTPUT_STREET_TYPE_PREFIX).append("=?, "); // 25 sql.append(OUTPUT_SUITE).append("=?, "); // 26 sql.append(OUTPUT_SUITE_PREFIX).append("=?, "); // 27 sql.append(OUTPUT_SUITE_TYPE).append("=?, "); // 28 sql.append(OUTPUT_TYPE).append("=?, "); // 29 sql.append(OUTPUT_UNPARSED_ADDRESS).append("=?, "); // 30 sql.append(OUTPUT_URBAN_BEFORE_RURAL).append("=?, "); // 31 sql.append(OUTPUT_VALID).append("=? "); // 32 sql.append("WHERE "); String baseStatement = sql.toString(); int batchCount = 0; for (int i = 0; i < dirtyAddresses.size(); i++) { sql = new StringBuilder(baseStatement); result = dirtyAddresses.get(i); int j = 0; // I really wish there was a better way to handle this, // but unfortunately in SQL, <column> = null and <column> is // null are not the same thing, and you usually want 'is // null' Why they couldn't just use '= null' is beyond me. // Otherwise, we could just use a single prepared statement // for all the records. The main reason we had to switch // back to using prepared statements is because different RDBMS // platforms handle Booleans differently (some support // boolean explicitly, others use an integer (1 or 0) for (Object keyValue : result.getKeyValues()) { if (j > 0) { sql.append("AND "); } if (keyValue == null) { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(j).append(" is null "); // 18+ } else if (keyValue instanceof String || keyValue instanceof Character) { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(j) .append("=" + SQL.quote(keyValue.toString()) + " "); // 18+ } else { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(j).append("=" + keyValue + " "); // 18+ } j++; } ps = con.prepareStatement(sql.toString()); Address inputAddress = result.getInputAddress(); this.adjustInputAddress(inputAddress, columnMetaData); engineLogger.debug( "Setting input unparsed address line 1 to " + inputAddress.getUnparsedAddressLine1()); ps.setString(1, inputAddress.getUnparsedAddressLine1()); engineLogger.debug( "Setting input unparsed address line 2 to " + inputAddress.getUnparsedAddressLine2()); ps.setString(2, inputAddress.getUnparsedAddressLine2()); engineLogger.debug("Setting input municipality to " + inputAddress.getMunicipality()); ps.setString(3, inputAddress.getMunicipality()); engineLogger.debug("Setting input province to " + inputAddress.getProvince()); ps.setString(4, inputAddress.getProvince()); engineLogger.debug("Setting input country to " + inputAddress.getCountry()); ps.setString(5, inputAddress.getCountry()); engineLogger.debug("Setting input postal code to " + inputAddress.getPostalCode()); ps.setString(6, inputAddress.getPostalCode()); Address outputAddress = result.getOutputAddress(); this.adjustOutputAddress(outputAddress, columnMetaData, usingNewNames); engineLogger.debug("Setting output suite to " + outputAddress.getSuite()); ps.setString(7, outputAddress.getSuite()); engineLogger.debug("Setting output delivery installation name to " + outputAddress.getDeliveryInstallationName()); ps.setString(8, outputAddress.getDeliveryInstallationName()); engineLogger.debug("Setting output delivery nstallation type to " + outputAddress.getDeliveryInstallationType()); ps.setString(9, outputAddress.getDeliveryInstallationType()); engineLogger.debug("Setting output direction prefix to " + outputAddress.isDirectionPrefix()); ps.setBoolean(10, outputAddress.isDirectionPrefix()); engineLogger.debug( "Setting output failed parsing string to " + outputAddress.getFailedParsingString()); ps.setString(11, outputAddress.getFailedParsingString()); engineLogger.debug( "Setting output general delivery name to " + outputAddress.getGeneralDeliveryName()); ps.setString(12, outputAddress.getGeneralDeliveryName()); engineLogger.debug("Setting output lock box number to " + outputAddress.getLockBoxNumber()); ps.setString(13, outputAddress.getLockBoxNumber()); engineLogger.debug("Setting output lock box type to " + outputAddress.getLockBoxType()); ps.setString(14, outputAddress.getLockBoxType()); engineLogger.debug("Setting output municipality to " + outputAddress.getMunicipality()); ps.setString(15, outputAddress.getMunicipality()); engineLogger.debug("Setting output postal code to " + outputAddress.getPostalCode()); ps.setString(16, outputAddress.getPostalCode()); engineLogger.debug("Setting output province to " + outputAddress.getProvince()); ps.setString(17, outputAddress.getProvince()); engineLogger .debug("Setting output rural route number to " + outputAddress.getRuralRouteNumber()); ps.setString(18, outputAddress.getRuralRouteNumber()); engineLogger.debug("Setting output rural route type to " + outputAddress.getRuralRouteType()); ps.setString(19, outputAddress.getRuralRouteType()); engineLogger.debug("Setting output street direciton to " + outputAddress.getStreetDirection()); ps.setString(20, outputAddress.getStreetDirection()); engineLogger.debug("Setting output street to " + outputAddress.getStreet()); ps.setString(21, outputAddress.getStreet()); engineLogger.debug("Setting output street number to " + outputAddress.getStreetNumber()); Integer streetNumber = outputAddress.getStreetNumber(); if (streetNumber == null) { ps.setNull(22, Types.INTEGER); } else { ps.setInt(22, streetNumber); } engineLogger.debug( "Setting output street number suffix to " + outputAddress.getStreetNumberSuffix()); ps.setString(23, outputAddress.getStreetNumberSuffix()); engineLogger.debug("Setting output street number suffix separate to " + outputAddress.isStreetNumberSuffixSeparate()); Boolean isStreetNumberSuffixSeparate = outputAddress.isStreetNumberSuffixSeparate(); if (isStreetNumberSuffixSeparate == null) { ps.setNull(24, Types.BOOLEAN); } else { ps.setBoolean(24, outputAddress.isStreetNumberSuffixSeparate()); } engineLogger.debug("Setting output suite to " + outputAddress.getSuite()); ps.setString(25, outputAddress.getStreetType()); engineLogger.debug("Setting output streetTypePrefix to " + outputAddress.isStreetTypePrefix()); ps.setBoolean(26, outputAddress.isStreetTypePrefix()); engineLogger.debug("Setting output suite to " + outputAddress.getSuite()); ps.setString(27, outputAddress.getSuite()); engineLogger.debug("Setting output suitePrefix to " + outputAddress.isSuitePrefix()); ps.setBoolean(28, outputAddress.isSuitePrefix()); engineLogger.debug("Setting output suiteType to " + outputAddress.getSuiteType()); ps.setString(29, outputAddress.getSuiteType()); engineLogger.debug("Setting output type to " + outputAddress.getType()); RecordType type = outputAddress.getType(); ps.setString(30, type == null ? null : type.toString()); engineLogger.debug( "Setting output unparsedAddressLine1 to " + outputAddress.getUnparsedAddressLine1()); ps.setString(31, outputAddress.getUnparsedAddressLine1()); engineLogger.debug("Setting output urbanBeforeRural to " + outputAddress.isUrbanBeforeRural()); Boolean urbanBeforeRural = outputAddress.isUrbanBeforeRural(); if (urbanBeforeRural == null) { ps.setNull(32, Types.BOOLEAN); } else { ps.setBoolean(32, outputAddress.isUrbanBeforeRural()); } engineLogger.debug("Setting valid to " + result.isValid()); ps.setBoolean(33, result.isValid()); if (useBatchUpdates) { engineLogger.debug("Adding to batch"); ps.addBatch(); batchCount++; // TODO: The batchCount should be user setable if (batchCount > 1000) { engineLogger.debug("Executing batch"); ps.executeBatch(); batchCount = 0; } } else { engineLogger.debug("Executing statement"); ps.execute(); } incrementProgress(); } // Execute remaining batch statements if (batchCount > 0 && useBatchUpdates) { ps.executeBatch(); } if (ps != null) ps.close(); ps = null; } if (newAddresses.size() > 0) { //Next, let's meke an INSERT PreparedStatement to insert new records sql = new StringBuilder(); sql.append("INSERT INTO "); appendFullyQualifiedTableName(sql, resultTable); sql.append("("); for (int i = 0; i < keySize; i++) { sql.append(SOURCE_ADDRESS_KEY_COLUMN_BASE).append(i).append(", "); } sql.append(INPUT_ADDRESS_LINE1).append(", "); sql.append(INPUT_ADDRESS_LINE2).append(", "); sql.append(INPUT_MUNICIPALITY).append(", "); sql.append(INPUT_PROVINCE).append(", "); sql.append(INPUT_COUNTRY).append(", "); sql.append(INPUT_POSTAL_CODE).append(", "); sql.append(OUTPUT_COUNTRY).append(", "); if (usingNewNames) { sql.append(OUTPUT_DELIVERY_INSTALLATION_NAME).append(", "); sql.append(OUTPUT_DELIVERY_INSTALLATION_TYPE).append(", "); } else { sql.append(OLD_OUTPUT_DELIVERY_INSTALLATION_NAME).append(", "); sql.append(OLD_OUTPUT_DELIVERY_INSTALLATION_TYPE).append(", "); } sql.append(OUTPUT_DIRECTION_PREFIX).append(", "); sql.append(OUTPUT_FAILED_PARSING_STRING).append(", "); sql.append(OUTPUT_GENERAL_DELIVERY_NAME).append(", "); sql.append(OUTPUT_LOCK_BOX_NUMBER).append(", "); sql.append(OUTPUT_LOCK_BOX_TYPE).append(", "); sql.append(OUTPUT_MUNICIPALITY).append(", "); sql.append(OUTPUT_POSTAL_CODE).append(", "); sql.append(OUTPUT_PROVINCE).append(", "); sql.append(OUTPUT_RURAL_ROUTE_NUMBER).append(", "); sql.append(OUTPUT_RURAL_ROUTE_TYPE).append(", "); sql.append(OUTPUT_STREET_DIRECTION).append(", "); sql.append(OUTPUT_STREET_NAME).append(", "); sql.append(OUTPUT_STREET_NUMBER).append(", "); sql.append(OUTPUT_STREET_NUMBER_SUFFIX).append(", "); if (usingNewNames) { sql.append(OUTPUT_STREET_NUMBER_SUFFIX_SEPARATE).append(", "); } else { sql.append(OLD_OUTPUT_STREET_NUMBER_SUFFIX_SEPARATE).append(", "); } sql.append(OUTPUT_STREET_TYPE).append(", "); sql.append(OUTPUT_STREET_TYPE_PREFIX).append(", "); sql.append(OUTPUT_SUITE).append(", "); sql.append(OUTPUT_SUITE_PREFIX).append(", "); sql.append(OUTPUT_SUITE_TYPE).append(", "); sql.append(OUTPUT_TYPE).append(", "); sql.append(OUTPUT_UNPARSED_ADDRESS).append(", "); sql.append(OUTPUT_URBAN_BEFORE_RURAL).append(", "); sql.append(OUTPUT_VALID).append(")"); sql.append("VALUES("); for (int i = 0; i < keySize; i++) { sql.append("?, "); } sql.append( "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); ps = con.prepareStatement(sql.toString()); int batchCount = 0; for (int i = 0; i < newAddresses.size(); i++) { result = newAddresses.get(i); int j = 1; for (Object keyValue : result.getKeyValues()) { ps.setObject(j, keyValue); j++; engineLogger.debug("Setting key value " + j + " to " + keyValue); } Address inputAddress = result.getInputAddress(); this.adjustInputAddress(inputAddress, columnMetaData); engineLogger.debug( "Setting input unparsed address line 1 to " + inputAddress.getUnparsedAddressLine1()); ps.setString(j, inputAddress.getUnparsedAddressLine1()); engineLogger.debug( "Setting input unparsed address line 2 to " + inputAddress.getUnparsedAddressLine2()); ps.setString(j + 1, inputAddress.getUnparsedAddressLine2()); engineLogger.debug("Setting input municipality to " + inputAddress.getMunicipality()); ps.setString(j + 2, inputAddress.getMunicipality()); engineLogger.debug("Setting input province to " + inputAddress.getProvince()); ps.setString(j + 3, inputAddress.getProvince()); engineLogger.debug("Setting input country to " + inputAddress.getCountry()); ps.setString(j + 4, inputAddress.getCountry()); engineLogger.debug("Setting input postal code to " + inputAddress.getPostalCode()); ps.setString(j + 5, inputAddress.getPostalCode()); Address outputAddress = result.getOutputAddress(); this.adjustOutputAddress(outputAddress, columnMetaData, usingNewNames); engineLogger.debug("Setting output suite to " + outputAddress.getSuite()); ps.setString(j + 6, outputAddress.getSuite()); engineLogger.debug("Setting output delivery installation name to " + outputAddress.getDeliveryInstallationName()); ps.setString(j + 7, outputAddress.getDeliveryInstallationName()); engineLogger.debug("Setting output delivery nstallation type to " + outputAddress.getDeliveryInstallationType()); ps.setString(j + 8, outputAddress.getDeliveryInstallationType()); engineLogger.debug("Setting output direction prefix to " + outputAddress.isDirectionPrefix()); ps.setBoolean(j + 9, outputAddress.isDirectionPrefix()); engineLogger.debug( "Setting output failed parsing string to " + outputAddress.getFailedParsingString()); ps.setString(j + 10, outputAddress.getFailedParsingString()); engineLogger.debug( "Setting output general delivery name to " + outputAddress.getGeneralDeliveryName()); ps.setString(j + 11, outputAddress.getGeneralDeliveryName()); engineLogger.debug("Setting output lock box number to " + outputAddress.getLockBoxNumber()); ps.setString(j + 12, outputAddress.getLockBoxNumber()); engineLogger.debug("Setting output lock box type to " + outputAddress.getLockBoxType()); ps.setString(j + 13, outputAddress.getLockBoxType()); engineLogger.debug("Setting output municipality to " + outputAddress.getMunicipality()); ps.setString(j + 14, outputAddress.getMunicipality()); engineLogger.debug("Setting output postal code to " + outputAddress.getPostalCode()); ps.setString(j + 15, outputAddress.getPostalCode()); engineLogger.debug("Setting output province to " + outputAddress.getProvince()); ps.setString(j + 16, outputAddress.getProvince()); engineLogger .debug("Setting output rural route number to " + outputAddress.getRuralRouteNumber()); ps.setString(j + 17, outputAddress.getRuralRouteNumber()); engineLogger.debug("Setting output rural route type to " + outputAddress.getRuralRouteType()); ps.setString(j + 18, outputAddress.getRuralRouteType()); engineLogger.debug("Setting output street direciton to " + outputAddress.getStreetDirection()); ps.setString(j + 19, outputAddress.getStreetDirection()); engineLogger.debug("Setting output street to " + outputAddress.getStreet()); ps.setString(j + 20, outputAddress.getStreet()); engineLogger.debug("Setting output street number to " + outputAddress.getStreetNumber()); Integer streetNumber = outputAddress.getStreetNumber(); if (streetNumber == null) { ps.setNull(j + 21, Types.INTEGER); } else { ps.setInt(j + 21, streetNumber); } engineLogger.debug( "Setting output street number suffix to " + outputAddress.getStreetNumberSuffix()); ps.setString(j + 22, outputAddress.getStreetNumberSuffix()); engineLogger.debug("Setting output street number suffix separate to " + outputAddress.isStreetNumberSuffixSeparate()); Boolean isStreetNumberSuffixSeparate = outputAddress.isStreetNumberSuffixSeparate(); if (isStreetNumberSuffixSeparate == null) { ps.setNull(j + 23, Types.BOOLEAN); } else { ps.setBoolean(j + 23, outputAddress.isStreetNumberSuffixSeparate()); } engineLogger.debug("Setting output suite to " + outputAddress.getSuite()); ps.setString(j + 24, outputAddress.getStreetType()); engineLogger.debug("Setting output streetTypePrefix to " + outputAddress.isStreetTypePrefix()); ps.setBoolean(j + 25, outputAddress.isStreetTypePrefix()); engineLogger.debug("Setting output suite to " + outputAddress.getSuite()); ps.setString(j + 26, outputAddress.getSuite()); engineLogger.debug("Setting output suitePrefix to " + outputAddress.isSuitePrefix()); ps.setBoolean(j + 27, outputAddress.isSuitePrefix()); engineLogger.debug("Setting output suiteType to " + outputAddress.getSuiteType()); ps.setString(j + 28, outputAddress.getSuiteType()); engineLogger.debug("Setting output type to " + outputAddress.getType()); RecordType type = outputAddress.getType(); ps.setString(j + 29, type == null ? null : type.toString()); engineLogger.debug( "Setting output unparsedAddressLine1 to " + outputAddress.getUnparsedAddressLine1()); ps.setString(j + 30, outputAddress.getUnparsedAddressLine1()); engineLogger.debug("Setting output urbanBeforeRural to " + outputAddress.isUrbanBeforeRural()); Boolean urbanBeforeRural = outputAddress.isUrbanBeforeRural(); if (urbanBeforeRural == null) { ps.setNull(j + 31, Types.BOOLEAN); } else { ps.setBoolean(j + 31, outputAddress.isUrbanBeforeRural()); } engineLogger.debug("Setting valid to " + result.isValid()); ps.setBoolean(j + 32, result.isValid()); engineLogger.debug("Preparing the following address to be inserted: " + result); if (useBatchUpdates) { engineLogger.debug("Adding to batch"); ps.addBatch(); batchCount++; // TODO: The batchCount should be user setable if (batchCount > 1000) { engineLogger.debug("Executing batch"); ps.executeBatch(); batchCount = 0; } } else { engineLogger.debug("Executing statement"); ps.execute(); } incrementProgress(); } // Execute remaining batch statements if (batchCount > 0 && useBatchUpdates) { ps.executeBatch(); } if (ps != null) ps.close(); ps = null; } if (debug) { engineLogger.debug("Rolling back changes"); con.rollback(); } else { engineLogger.debug("Committing changes"); con.commit(); } for (AddressResult ar : addresses.values()) { ar.markClean(); } } catch (Exception ex) { try { con.rollback(); } catch (SQLException sqlEx) { engineLogger.error("Error while rolling back. " + "Suppressing this exception to prevent it from overshadowing the orginal exception.", sqlEx); } throw new RuntimeException("Unexpected exception while storing address validation results.\n" + "SQL statement: " + ((sql == null) ? "null" : sql.toString()) + "\n" + "Current result: " + ((result == null) ? "null" : "Input Address:\n" + result.getInputAddress() + "\n" + "Output Address:\n" + result.getOutputAddress()), ex); } finally { setFinished(true); if (ps != null) try { ps.close(); } catch (SQLException e) { engineLogger.error("Error while closing PreparedStatement", e); } if (stmt != null) try { stmt.close(); } catch (SQLException e) { engineLogger.error("Error while closing Statement", e); } if (con != null) try { con.close(); } catch (SQLException e) { engineLogger.error("Error while closing Connection", e); } } }
From source file:com.flexive.ejb.beans.structure.AssignmentEngineBean.java
/** * Remove an assignment/* w w w. j a va 2 s.co m*/ * * @param assignmentId assignment to remove * @param removeSubAssignments if assignment is a group, remove all attached properties and groups? * @param removeDerivedAssignments if derivates of this assignment in derived types exist, remove them as well? * @param disableAssignment if false, find all derived assignments, flag them as 'regular' assignments and set them as new base * @param allowDerivedRemoval allow removal of derived assignments * @throws FxApplicationException on errors */ private void removeAssignment(long assignmentId, boolean removeSubAssignments, boolean removeDerivedAssignments, boolean disableAssignment, boolean allowDerivedRemoval) throws FxApplicationException { final UserTicket ticket = FxContext.getUserTicket(); FxPermissionUtils.checkRole(ticket, Role.StructureManagement); FxAssignment assignment; assignment = CacheAdmin.getEnvironment().getAssignment(assignmentId); assert assignment != null : "Assignment retrieved was null"; if (!disableAssignment) { //if removal, check if its a derived assignment which may not be removed if (!allowDerivedRemoval && assignment.isDerivedAssignment()) throw new FxRemoveException("ex.structure.assignment.delete.derived", assignment.getXPath()); } Connection con = null; PreparedStatement ps = null; StringBuilder sql = new StringBuilder(500); try { con = Database.getDbConnection(); List<FxAssignment> affectedAssignments = new ArrayList<FxAssignment>(10); affectedAssignments.add(assignment); if (assignment instanceof FxGroupAssignment && removeSubAssignments) { FxGroupAssignment ga = (FxGroupAssignment) assignment; _addSubAssignments(affectedAssignments, ga); } if (removeDerivedAssignments) { //find all derived assignments sql.append("SELECT ID FROM ").append(TBL_STRUCT_ASSIGNMENTS).append(" WHERE BASE=?"); ps = con.prepareStatement(sql.toString()); long prevSize = 0; while (prevSize != affectedAssignments.size()) { //run until no derived assignments are found prevSize = affectedAssignments.size(); List<FxAssignment> adds = new ArrayList<FxAssignment>(5); for (FxAssignment check : affectedAssignments) { ps.setLong(1, check.getId()); ResultSet rs = ps.executeQuery(); if (rs != null && rs.next()) { FxAssignment derived = CacheAdmin.getEnvironment().getAssignment(rs.getLong(1)); if (!adds.contains(derived) && !affectedAssignments.contains(derived)) adds.add(derived); } } affectedAssignments.addAll(adds); } ps.close(); sql.setLength(0); } else if (!disableAssignment) { //find all (directly) derived assignments, flag them as 'regular' assignments and set them as new base breakAssignmentInheritance(con, sql, affectedAssignments.toArray(new FxAssignment[affectedAssignments.size()])); } //security checks if (!ticket.isGlobalSupervisor()) { //assignment permission StringBuilder assignmentList = new StringBuilder(200); for (FxAssignment check : affectedAssignments) { assignmentList.append(",").append(check.getId()); if (check instanceof FxPropertyAssignment && check.getAssignedType().isUsePropertyPermissions()) { FxPropertyAssignment pa = (FxPropertyAssignment) check; if (!ticket.mayDeleteACL(pa.getACL().getId(), 0/*owner is irrelevant here*/)) throw new FxNoAccessException("ex.acl.noAccess.delete", pa.getACL().getName()); } } //affected content permission sql.append("SELECT DISTINCT O.ACL FROM ").append(TBL_CONTENT) .append(" O WHERE O.ID IN(SELECT D.ID FROM ").append(TBL_CONTENT_DATA) .append(" D WHERE D.ASSIGN IN(").append(assignmentList.substring(1)).append("))"); java.lang.System.out.println("SQL==" + sql.toString()); ps = con.prepareStatement(sql.toString()); sql.setLength(0); ResultSet rs = ps.executeQuery(); while (rs != null && rs.next()) { if (!ticket.mayDeleteACL(rs.getInt(1), 0/*owner is irrelevant here*/)) throw new FxNoAccessException("ex.acl.noAccess.delete", CacheAdmin.getEnvironment().getACL(rs.getInt(1))); } ps.close(); } if (disableAssignment) sql.append("UPDATE ").append(TBL_STRUCT_ASSIGNMENTS).append(" SET ENABLED=? WHERE ID=?"); else sql.append("DELETE FROM ").append(TBL_STRUCT_ASSIGNMENTS).append(" WHERE ID=?"); ps = con.prepareStatement(sql.toString()); //batch remove all multi language entries and content datas PreparedStatement psML = null; PreparedStatement psData = null; PreparedStatement psDataFT = null; PreparedStatement psBinaryGet = null; PreparedStatement psBinaryRemove = null; PreparedStatement psPropertyOptionRemove = null; PreparedStatement psGroupOptionRemove = null; try { sql.setLength(0); sql.append("DELETE FROM ").append(TBL_STRUCT_ASSIGNMENTS).append(ML).append(" WHERE ID=?"); psML = con.prepareStatement(sql.toString()); sql.setLength(0); sql.append("DELETE FROM ").append(TBL_STRUCT_PROPERTY_OPTIONS).append(" WHERE ASSID=?"); psPropertyOptionRemove = con.prepareStatement(sql.toString()); sql.setLength(0); sql.append("DELETE FROM ").append(TBL_STRUCT_GROUP_OPTIONS).append(" WHERE ASSID=?"); psGroupOptionRemove = con.prepareStatement(sql.toString()); sql.setLength(0); sql.append("DELETE FROM ").append(TBL_CONTENT_DATA).append(" WHERE ASSIGN=?"); psData = con.prepareStatement(sql.toString()); sql.setLength(0); sql.append("DELETE FROM ").append(TBL_CONTENT_DATA_FT).append(" WHERE ASSIGN=?"); psDataFT = con.prepareStatement(sql.toString()); sql.setLength(0); sql.append("SELECT DISTINCT FBLOB FROM ").append(TBL_CONTENT_DATA) .append(" WHERE ASSIGN=? AND FBLOB IS NOT NULL"); psBinaryGet = con.prepareStatement(sql.toString()); sql.setLength(0); sql.append("DELETE FROM ").append(TBL_CONTENT_BINARY).append(" WHERE ID=?"); psBinaryRemove = con.prepareStatement(sql.toString()); for (FxAssignment ml : affectedAssignments) { if (!disableAssignment) { psML.setLong(1, ml.getId()); psML.addBatch(); } psData.setLong(1, ml.getId()); psData.addBatch(); if (ml instanceof FxPropertyAssignment) { if (!disableAssignment) { if (((FxPropertyAssignment) ml).isFlatStorageEntry()) FxFlatStorageManager.getInstance().removeAssignmentMappings(con, ml.getId()); } psDataFT.setLong(1, ml.getId()); psDataFT.addBatch(); psPropertyOptionRemove.setLong(1, ml.getId()); psPropertyOptionRemove.addBatch(); //only need to remove binaries if its a binary type... switch (((FxPropertyAssignment) ml).getProperty().getDataType()) { case Binary: psBinaryGet.setLong(1, ml.getId()); ResultSet rs = psBinaryGet.executeQuery(); while (rs != null && rs.next()) { psBinaryRemove.setLong(1, rs.getLong(1)); psBinaryRemove.addBatch(); } } } else if (ml instanceof FxGroupAssignment) { psGroupOptionRemove.setLong(1, ml.getId()); psGroupOptionRemove.addBatch(); } } if (!disableAssignment) { psML.executeBatch(); psPropertyOptionRemove.executeBatch(); psGroupOptionRemove.executeBatch(); psBinaryRemove.executeBatch(); psDataFT.executeBatch(); psData.executeBatch(); } } finally { Database.closeObjects(AssignmentEngineBean.class, null, psML); Database.closeObjects(AssignmentEngineBean.class, null, psData); Database.closeObjects(AssignmentEngineBean.class, null, psDataFT); Database.closeObjects(AssignmentEngineBean.class, null, psBinaryGet); Database.closeObjects(AssignmentEngineBean.class, null, psBinaryRemove); Database.closeObjects(AssignmentEngineBean.class, null, psGroupOptionRemove); Database.closeObjects(AssignmentEngineBean.class, null, psPropertyOptionRemove); } if (disableAssignment) ps.setBoolean(1, false); if (affectedAssignments.size() > 1) affectedAssignments = FxStructureUtils.resolveRemoveDependencies(affectedAssignments); for (FxAssignment rm : affectedAssignments) { ps.setLong(disableAssignment ? 2 : 1, rm.getId()); ps.executeUpdate(); } FxStructureUtils.removeOrphanedProperties(con); FxStructureUtils.removeOrphanedGroups(con); StructureLoader.reload(con); htracker.track(assignment.getAssignedType(), disableAssignment ? "history.assignment.remove" : "history.assignment.disable", assignment.getXPath(), assignmentId, removeSubAssignments, removeDerivedAssignments); } catch (SQLException e) { EJBUtils.rollback(ctx); throw new FxRemoveException(LOG, e, "ex.db.sqlError", e.getMessage()); } catch (FxCacheException e) { EJBUtils.rollback(ctx); throw new FxRemoveException(LOG, e, "ex.cache", e.getMessage()); } catch (FxLoadException e) { EJBUtils.rollback(ctx); throw new FxRemoveException(e); } finally { Database.closeObjects(TypeEngineBean.class, con, ps); } }
From source file:org.apache.cocoon.util.JDBCTypeConversions.java
/** * Set the Statement column so that the results are mapped correctly. * * @param statement the prepared statement * @param position the position of the column * @param value the value of the column/*from w w w .ja v a 2 s . co m*/ */ public static void setColumn(PreparedStatement statement, int position, Object value, Integer typeObject) throws Exception { if (value instanceof String) { value = ((String) value).trim(); } if (typeObject == null) { throw new SQLException("Can't set column because the type is unrecognized"); } if (value == null) { /** If the value is null, set the column value null and return **/ statement.setNull(position, typeObject.intValue()); return; } if ("".equals(value)) { switch (typeObject.intValue()) { case Types.CHAR: case Types.CLOB: case Types.VARCHAR: /** If the value is an empty string and the column is a string type, we can continue **/ break; default: /** If the value is an empty string and the column is something else, we treat it as a null value **/ statement.setNull(position, typeObject.intValue()); return; } } File file = null; int length = -1; InputStream asciiStream = null; //System.out.println("========================================================================"); //System.out.println("JDBCTypeConversions: setting type "+typeObject.intValue()); switch (typeObject.intValue()) { case Types.CLOB: //System.out.println("CLOB"); Clob clob = null; if (value instanceof Clob) { clob = (Clob) value; } else if (value instanceof File) { File asciiFile = (File) value; asciiStream = new BufferedInputStream(new FileInputStream(asciiFile)); length = (int) asciiFile.length(); clob = new ClobHelper(asciiStream, length); } else if (value instanceof Part) { Part anyFile = (Part) value; asciiStream = new BufferedInputStream(anyFile.getInputStream()); length = anyFile.getSize(); clob = new ClobHelper(asciiStream, length); } else if (value instanceof JDBCxlobHelper) { asciiStream = ((JDBCxlobHelper) value).inputStream; length = ((JDBCxlobHelper) value).length; clob = new ClobHelper(asciiStream, length); } else if (value instanceof Source) { asciiStream = ((Source) value).getInputStream(); length = (int) ((Source) value).getContentLength(); clob = new ClobHelper(asciiStream, length); } else { String asciiText = value.toString(); asciiStream = new ByteArrayInputStream(asciiText.getBytes()); length = asciiText.length(); clob = new ClobHelper(asciiStream, length); } statement.setClob(position, clob); break; case Types.CHAR: // simple large object, e.g. Informix's TEXT //System.out.println("CHAR"); if (value instanceof File) { File asciiFile = (File) value; asciiStream = new BufferedInputStream(new FileInputStream(asciiFile)); length = (int) asciiFile.length(); } else if (value instanceof JDBCxlobHelper) { asciiStream = ((JDBCxlobHelper) value).inputStream; length = ((JDBCxlobHelper) value).length; } else if (value instanceof Source) { asciiStream = ((Source) value).getInputStream(); length = (int) ((Source) value).getContentLength(); } else if (value instanceof Part) { Part anyFile = (Part) value; asciiStream = new BufferedInputStream(anyFile.getInputStream()); length = anyFile.getSize(); clob = new ClobHelper(asciiStream, length); } else { String asciiText = value.toString(); asciiStream = new BufferedInputStream(new ByteArrayInputStream(asciiText.getBytes())); length = asciiText.length(); } statement.setAsciiStream(position, asciiStream, length); break; case Types.BIGINT: //System.out.println("BIGINT"); BigDecimal bd = null; if (value instanceof BigDecimal) { bd = (BigDecimal) value; } else if (value instanceof Number) { bd = BigDecimal.valueOf(((Number) value).longValue()); } else { bd = new BigDecimal(value.toString()); } statement.setBigDecimal(position, bd); break; case Types.TINYINT: //System.out.println("TINYINT"); Byte b = null; if (value instanceof Byte) { b = (Byte) value; } else if (value instanceof Number) { b = new Byte(((Number) value).byteValue()); } else { b = new Byte(value.toString()); } statement.setByte(position, b.byteValue()); break; case Types.DATE: //System.out.println("DATE"); Date d = null; if (value instanceof Date) { d = (Date) value; } else if (value instanceof java.util.Date) { d = new Date(((java.util.Date) value).getTime()); } else if (value instanceof Calendar) { d = new Date(((Calendar) value).getTime().getTime()); } else { d = Date.valueOf(value.toString()); } statement.setDate(position, d); break; case Types.DOUBLE: //System.out.println("DOUBLE"); double db; if (value instanceof Number) { db = (((Number) value).doubleValue()); } else { db = Double.parseDouble(value.toString()); } statement.setDouble(position, db); break; case Types.FLOAT: //System.out.println("FLOAT"); float f; if (value instanceof Number) { f = (((Number) value).floatValue()); } else { f = Float.parseFloat(value.toString()); } statement.setFloat(position, f); break; case Types.NUMERIC: //System.out.println("NUMERIC"); long l; if (value instanceof Number) { l = (((Number) value).longValue()); } else { l = Long.parseLong(value.toString()); } statement.setLong(position, l); break; case Types.SMALLINT: //System.out.println("SMALLINT"); Short s = null; if (value instanceof Short) { s = (Short) value; } else if (value instanceof Number) { s = new Short(((Number) value).shortValue()); } else { s = new Short(value.toString()); } statement.setShort(position, s.shortValue()); break; case Types.TIME: //System.out.println("TIME"); Time t = null; if (value instanceof Time) { t = (Time) value; } else if (value instanceof java.util.Date) { t = new Time(((java.util.Date) value).getTime()); } else { t = Time.valueOf(value.toString()); } statement.setTime(position, t); break; case Types.TIMESTAMP: //System.out.println("TIMESTAMP"); Timestamp ts = null; if (value instanceof Time) { ts = (Timestamp) value; } else if (value instanceof java.util.Date) { ts = new Timestamp(((java.util.Date) value).getTime()); } else { ts = Timestamp.valueOf(value.toString()); } statement.setTimestamp(position, ts); break; case Types.ARRAY: //System.out.println("ARRAY"); statement.setArray(position, (Array) value); // no way to convert string to array break; case Types.STRUCT: //System.out.println("STRUCT"); case Types.OTHER: //System.out.println("OTHER"); statement.setObject(position, value); break; case Types.LONGVARBINARY: //System.out.println("LONGVARBINARY"); statement.setTimestamp(position, new Timestamp((new java.util.Date()).getTime())); break; case Types.VARCHAR: //System.out.println("VARCHAR"); statement.setString(position, value.toString()); break; case Types.BLOB: //System.out.println("BLOB"); if (value instanceof JDBCxlobHelper) { statement.setBinaryStream(position, ((JDBCxlobHelper) value).inputStream, ((JDBCxlobHelper) value).length); } else if (value instanceof Source) { statement.setBinaryStream(position, ((Source) value).getInputStream(), (int) ((Source) value).getContentLength()); } else { Blob blob = null; if (value instanceof Blob) { blob = (Blob) value; } else if (value instanceof File) { file = (File) value; blob = new BlobHelper(new FileInputStream(file), (int) file.length()); } else if (value instanceof String) { file = new File((String) value); blob = new BlobHelper(new FileInputStream(file), (int) file.length()); } else if (value instanceof Part) { Part anyFile = (Part) value; blob = new BlobHelper(new BufferedInputStream(anyFile.getInputStream()), anyFile.getSize()); } else { throw new SQLException("Invalid type for blob: " + value.getClass().getName()); } //InputStream input = new BufferedInputStream(new FileInputStream(file)); statement.setBlob(position, blob); } break; case Types.VARBINARY: //System.out.println("VARBINARY"); if (value instanceof JDBCxlobHelper) { statement.setBinaryStream(position, ((JDBCxlobHelper) value).inputStream, ((JDBCxlobHelper) value).length); } else if (value instanceof Source) { statement.setBinaryStream(position, ((Source) value).getInputStream(), (int) ((Source) value).getContentLength()); } else if (value instanceof Part) { statement.setBinaryStream(position, ((Part) value).getInputStream(), ((Part) value).getSize()); } else { if (value instanceof File) { file = (File) value; } else if (value instanceof String) { file = new File((String) value); } else { throw new SQLException("Invalid type for blob: " + value.getClass().getName()); } //InputStream input = new BufferedInputStream(new FileInputStream(file)); FileInputStream input = new FileInputStream(file); statement.setBinaryStream(position, input, (int) file.length()); } break; case Types.INTEGER: //System.out.println("INTEGER"); Integer i = null; if (value instanceof Integer) { i = (Integer) value; } else if (value instanceof Number) { i = new Integer(((Number) value).intValue()); } else { i = new Integer(value.toString()); } statement.setInt(position, i.intValue()); break; case Types.BIT: //System.out.println("BIT"); Boolean bo = null; if (value instanceof Boolean) { bo = (Boolean) value; } else if (value instanceof Number) { bo = BooleanUtils.toBooleanObject(((Number) value).intValue() == 1); } else { bo = BooleanUtils.toBooleanObject(value.toString()); } statement.setBoolean(position, bo.booleanValue()); break; default: //System.out.println("default"); throw new SQLException("Impossible exception - invalid type "); } //System.out.println("========================================================================"); }
From source file:com.portfolio.data.provider.MysqlDataProvider.java
@Override public String postRights(int userId, String uuid, String role, NodeRight rights) { if (!credential.isAdmin(userId)) throw new RestWebApplicationException(Status.FORBIDDEN, "No admin right"); try {//from w ww . j ava2 s . com ArrayList<String[]> args = new ArrayList<String[]>(); if (rights.read != null) { String[] arg = { "gr.RD", rights.read.toString() }; args.add(arg); } if (rights.write != null) { String[] arg = { "gr.WR", rights.write.toString() }; args.add(arg); } if (rights.delete != null) { String[] arg = { "gr.DL", rights.delete.toString() }; args.add(arg); } if (rights.submit != null) { String[] arg = { "gr.SB", rights.submit.toString() }; args.add(arg); } if (args.isEmpty()) return ""; String[] arg = args.get(0); String sql = "UPDATE group_info gi, group_rights gr SET " + arg[0] + "=?"; for (int i = 1; i < args.size(); ++i) { arg = args.get(i); sql += ", " + arg[0] + "=?"; } sql += " WHERE gi.grid=gr.grid AND gi.label=? AND gr.id=uuid2bin(?)"; PreparedStatement st = connection.prepareStatement(sql); int i = 1; do { arg = args.get(i - 1); st.setBoolean(i, Boolean.parseBoolean(arg[1])); ++i; } while (i <= args.size()); st.setString(i, role); ++i; st.setString(i, uuid); st.executeUpdate(); st.close(); } catch (Exception e) { e.printStackTrace(); } finally { } return "ok"; }
From source file:com.portfolio.data.provider.MysqlDataProvider.java
@Deprecated @Override//from w w w .j a v a 2s .com public String putMacroAction(int userId, Integer macro, String role, String data) { if (!credential.isAdmin(userId)) throw new RestWebApplicationException(Status.FORBIDDEN, "No admin right"); String sql = ""; PreparedStatement st; Integer output = 0; try { DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); InputSource is = new InputSource(new StringReader(data)); Document doc = documentBuilder.parse(is); Element root = doc.getDocumentElement(); boolean rd = "true".equals(root.getAttribute("read")) ? true : false; boolean wr = "true".equals(root.getAttribute("write")) ? true : false; boolean dl = "true".equals(root.getAttribute("delete")) ? true : false; boolean ad = "true".equals(root.getAttribute("add")) ? true : false; Node typesNode = doc.getElementsByTagName("types").item(0); Node types = typesNode.getFirstChild(); String typesText = ""; if (types != null) typesText = types.getNodeValue(); Node rulesNode = doc.getElementsByTagName("rules").item(0); Node rules = rulesNode.getFirstChild(); String rulesText = ""; if (rules != null) rulesText = rules.getNodeValue(); sql = "UPDATE rule_table SET RD=?, WR=?, DL=?, AD=?, " + "types_id=?, rules_id=? " + "WHERE rule_id=? AND role=?"; st = connection.prepareStatement(sql); st.setBoolean(1, rd); st.setBoolean(2, wr); st.setBoolean(3, dl); st.setBoolean(4, ad); st.setString(5, typesText); st.setString(6, rulesText); st.setInt(7, macro); st.setString(8, role); output = st.executeUpdate(); st.close(); } catch (Exception e) { e.printStackTrace(); } return output.toString(); }
From source file:com.portfolio.data.provider.MysqlDataProvider.java
@Deprecated @Override//from w w w. j a v a2s.c o m public String postAddAction(int userId, Integer macro, String role, String data) { if (!credential.isAdmin(userId)) throw new RestWebApplicationException(Status.FORBIDDEN, "No admin right"); String sql = ""; PreparedStatement st; Integer output = 0; try { DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); InputSource is = new InputSource(new StringReader(data)); Document doc = documentBuilder.parse(is); Element root = doc.getDocumentElement(); boolean rd = "true".equals(root.getAttribute("read")) ? true : false; boolean wr = "true".equals(root.getAttribute("write")) ? true : false; boolean dl = "true".equals(root.getAttribute("delete")) ? true : false; boolean ad = "true".equals(root.getAttribute("add")) ? true : false; Node typesNode = doc.getElementsByTagName("types").item(0); Node types = typesNode.getFirstChild(); String typesText = ""; if (types != null) typesText = types.getNodeValue(); Node rulesNode = doc.getElementsByTagName("rules").item(0); Node rules = rulesNode.getFirstChild(); String rulesText = ""; if (rules != null) rulesText = rules.getNodeValue(); sql = "INSERT INTO rule_table(rule_id, role, RD, WR, DL, AD, types_id, rules_id) " + "VALUE(?,?,?,?,?,?,?,?)"; st = connection.prepareStatement(sql); st.setInt(1, macro); st.setString(2, role); st.setBoolean(3, rd); st.setBoolean(4, wr); st.setBoolean(5, dl); st.setBoolean(6, ad); st.setString(7, typesText); st.setString(8, rulesText); output = st.executeUpdate(); st.close(); } catch (Exception e) { e.printStackTrace(); } return output.toString(); }
From source file:com.flexive.ejb.beans.structure.TypeEngineBean.java
/** * Update an existing type/*from ww w .ja va 2 s . c o m*/ * * @param type the type to update * @return id of the type * @throws FxApplicationException on errors */ private long update(FxTypeEdit type) throws FxApplicationException { if (!type.isChanged()) return type.getId(); final UserTicket ticket = FxContext.getUserTicket(); FxPermissionUtils.checkRole(ticket, Role.StructureManagement); final FxEnvironment environment = CacheAdmin.getEnvironment(); if (StringUtils.isEmpty(type.getName())) throw new FxInvalidParameterException("NAME", "ex.structure.update.nameMissing"); //security checks start if (!ticket.mayEditACL(type.getACL().getId(), 0)) throw new FxNoAccessException("ex.acl.noAccess.edit", type.getACL().getName()); //security checks end boolean needReload = false; //full reload only needed if assignments have changed Connection con = null; PreparedStatement ps = null; FxType orgType = environment.getType(type.getId()); StringBuilder sql = new StringBuilder(500); try { con = Database.getDbConnection(); long instanceCount = -1; //number of instances //start name change if (!orgType.getName().equals(type.getName())) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET NAME=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setString(1, type.getName()); ps.setLong(2, type.getId()); ps.executeUpdate(); //update all xpaths affected ps.close(); sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_ASSIGNMENTS) .append(" SET XPATH=REPLACE(XPATH, ?, ?) WHERE TYPEDEF=? AND ") .append(StorageManager.getRegExpLikeOperator("XPATH", "?")); ps = con.prepareStatement(sql.toString()); ps.setString(1, orgType.getName() + "/"); ps.setString(2, type.getName() + "/"); ps.setLong(3, type.getId()); ps.setString(4, "^" + orgType.getName() + "/"); int changed = ps.executeUpdate(); if (changed > 0) needReload = true; htracker.track(orgType, "history.type.update.name", orgType.getName(), type.getName(), type.getId(), changed); } //end name change //start description change if (!orgType.getLabel().equals(type.getLabel())) { Database.storeFxString(type.getLabel(), con, TBL_STRUCT_TYPES, "DESCRIPTION", "ID", type.getId()); htracker.track(orgType, "history.type.update.description", orgType.getLabel(), type.getLabel()); } //end description change //start type mode changes if (type.getMode().getId() != orgType.getMode().getId()) { if (instanceCount < 0) instanceCount = getInstanceCount(con, type.getId()); //allow relation => content (removing all relation specific entries) but content => relation requires 0 instances! if ((type.getMode() == TypeMode.Relation && orgType.getMode() == TypeMode.Content && instanceCount > 0) || orgType.getMode() == TypeMode.Preload || type.getMode() == TypeMode.Preload) throw new FxUpdateException("ex.structure.type.typeMode.notUpgradeable", orgType.getMode(), type.getMode(), orgType.getName()); if (type.getMode() == TypeMode.Content) { if (type.getRelations().size() > 0) { //TODO: remove relation mappings throw new FxUpdateException("ex.notImplemented", "Remove all relation mappings for type"); } if (instanceCount > 0) { //TODO: remove all relation specific entries for existing contents throw new FxUpdateException("ex.notImplemented", "Remove all relation specific entries for existing contents"); } } sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET TYPE_MODE=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getMode().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.typeMode", orgType.getMode(), type.getMode()); } //end type mode changes //start relation changes if (type.getAddedRelations().size() > 0) { sql.setLength(0); sql.append("INSERT INTO ").append(TBL_STRUCT_TYPERELATIONS) .append(" (TYPEDEF,TYPESRC,TYPEDST,MAXSRC,MAXDST)VALUES(?,?,?,?,?)"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getId()); for (FxTypeRelation rel : type.getAddedRelations()) { if (rel.getSource().isRelation()) throw new FxInvalidParameterException("ex.structure.type.relation.wrongTarget", type.getName(), rel.getSource().getName()); if (rel.getDestination().isRelation()) throw new FxInvalidParameterException("ex.structure.type.relation.wrongTarget", type.getName(), rel.getDestination().getName()); ps.setLong(2, rel.getSource().getId()); ps.setLong(3, rel.getDestination().getId()); ps.setLong(4, rel.getMaxSource()); ps.setLong(5, rel.getMaxDestination()); ps.addBatch(); htracker.track(type, "history.type.update.relation.add", type.getName(), rel.getSource().getName(), rel.getMaxSource(), rel.getDestination().getName(), rel.getMaxDestination()); } ps.executeBatch(); } if (type.getUpdatedRelations().size() > 0) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPERELATIONS). // 1 2 3 4 5 append(" SET MAXSRC=?,MAXDST=? WHERE TYPEDEF=? AND TYPESRC=? AND TYPEDST=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(3, type.getId()); for (FxTypeRelation rel : type.getUpdatedRelations()) { if (rel.getSource().isRelation()) throw new FxInvalidParameterException("ex.structure.type.relation.wrongTarget", type.getName(), rel.getSource().getName()); if (rel.getDestination().isRelation()) throw new FxInvalidParameterException("ex.structure.type.relation.wrongTarget", type.getName(), rel.getDestination().getName()); //TODO: check if maxSource/maxDestination is not violated if > 0 ps.setLong(4, rel.getSource().getId()); ps.setLong(5, rel.getDestination().getId()); ps.setLong(1, rel.getMaxSource()); ps.setLong(2, rel.getMaxDestination()); ps.addBatch(); htracker.track(type, "history.type.update.relation.update", type.getName(), rel.getSource().getName(), rel.getMaxSource(), rel.getDestination().getName(), rel.getMaxDestination()); } ps.executeBatch(); } if (type.getRemovedRelations().size() > 0) { sql.setLength(0); sql.append("DELETE FROM ").append(TBL_STRUCT_TYPERELATIONS). // 1 2 3 append(" WHERE TYPEDEF=? AND TYPESRC=? AND TYPEDST=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getId()); for (FxTypeRelation rel : type.getRemovedRelations()) { if (rel.getSource().isRelation()) throw new FxInvalidParameterException("ex.structure.type.relation.wrongTarget", type.getName(), rel.getSource().getName()); if (rel.getDestination().isRelation()) throw new FxInvalidParameterException("ex.structure.type.relation.wrongTarget", type.getName(), rel.getDestination().getName()); int[] rels = getRelationCount(con, type.getId(), rel.getSource().getId(), rel.getDestination().getId()); if (!type.isRemoveInstancesWithRelationTypes() && rels[0] > 0) throw new FxRemoveException("ex.structure.type.relation.relationsExist", type.getName(), rel.getSource().getName(), rel.getDestination().getName(), rels[0]); else if (type.isRemoveInstancesWithRelationTypes() && rels[0] > 0) { removeRelationEntries(con, type.getId(), rel.getSource().getId(), rel.getDestination().getId()); } ps.setLong(2, rel.getSource().getId()); ps.setLong(3, rel.getDestination().getId()); ps.addBatch(); htracker.track(type, "history.type.update.relation.remove", type.getName(), rel.getSource().getName(), rel.getMaxSource(), rel.getDestination().getName(), rel.getMaxDestination()); } ps.executeBatch(); } //end relation changes //start ACL changes if (!type.getACL().equals(orgType.getACL())) { if (type.getACL().getCategory() != ACLCategory.STRUCTURE) throw new FxInvalidParameterException("ACL", "ex.acl.category.invalid", type.getACL().getCategory(), ACLCategory.STRUCTURE); sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET ACL=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getACL().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.acl", orgType.getACL(), type.getACL()); } //end ACL changes //start default instance ACL changes if (type.hasDefaultInstanceACL() != orgType.hasDefaultInstanceACL() || !type.getDefaultInstanceACL().equals(orgType.getDefaultInstanceACL())) { if (type.hasDefaultInstanceACL() && type.getDefaultInstanceACL().getCategory() != ACLCategory.INSTANCE) throw new FxInvalidParameterException("DEFACL", "ex.acl.category.invalid", type.getDefaultInstanceACL().getCategory(), ACLCategory.INSTANCE); sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET DEFACL=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); if (type.hasDefaultInstanceACL()) ps.setLong(1, type.getDefaultInstanceACL().getId()); else ps.setNull(1, java.sql.Types.INTEGER); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.defacl", orgType.hasDefaultInstanceACL() ? orgType.getACL() : "-", type.hasDefaultInstanceACL() ? type.getDefaultInstanceACL() : "-"); } //end default instance ACL changes //start Workflow changes if (!type.getWorkflow().equals(orgType.getWorkflow())) { if (instanceCount < 0) instanceCount = getInstanceCount(con, type.getId()); if (instanceCount > 0) { //Workflow can not be changed with existing instances -> there is no way to reliably //map steps of one workflow to steps of another (even not using stepdefinitions since they //can be used multiple times). A possible solution would be to provide a mapping when changing //workflows but this should be to seldom the case to bother implementing it throw new FxUpdateException("ex.notImplemented", "Workflow changes with existing instance"); } sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET WORKFLOW=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getWorkflow().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.workflow", orgType.getWorkflow(), type.getWorkflow()); } //end Workflow changes //start Category changes if (!type.getCategory().equals(orgType.getCategory())) { if (!ticket.isGlobalSupervisor()) throw new FxUpdateException("ex.structure.type.category.notSupervisor", orgType.getCategory(), type.getCategory(), orgType.getName()); sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET CATEGORY=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setInt(1, type.getCategory().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); } //end Category changes //start language mode changes if (!type.getLanguage().equals(orgType.getLanguage())) { if (instanceCount < 0) instanceCount = getInstanceCount(con, type.getId()); if (instanceCount <= 0 || orgType.getLanguage().isUpgradeable(type.getLanguage())) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET LANG_MODE=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setInt(1, type.getLanguage().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.languageMode", orgType.getLanguage().name(), type.getLanguage().name()); } else throw new FxUpdateException("ex.structure.type.languageMode.notUpgradeable", orgType.getLanguage(), type.getLanguage(), orgType.getName()); } //end language mode changes //start state changes if (type.getState().getId() != orgType.getState().getId()) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET TYPE_STATE=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setInt(1, type.getState().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.state", orgType.getState().name(), type.getState().name()); } //end state changes //start permission changes if (type.getBitCodedPermissions() != orgType.getBitCodedPermissions()) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET SECURITY_MODE=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setInt(1, type.getBitCodedPermissions()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.perm", FxPermissionUtils.toString(orgType.getBitCodedPermissions()), FxPermissionUtils.toString(type.getBitCodedPermissions())); } //end permission changes //start multiple ACL setting changes if (type.isMultipleContentACLs() != orgType.isMultipleContentACLs()) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET MULTIPLE_CONTENT_ACLS=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setBoolean(1, type.isMultipleContentACLs()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.multipleContentACLs", orgType.isMultipleContentACLs(), type.isMultipleContentACLs()); } //end multiple ACL setting changes //start isIncludedInSupertypeQueries setting changes if (type.isIncludedInSupertypeQueries() != orgType.isIncludedInSupertypeQueries()) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET INSUPERTYPEQUERY=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setBoolean(1, type.isIncludedInSupertypeQueries()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.inSupertypeQueries", orgType.isIncludedInSupertypeQueries(), type.isIncludedInSupertypeQueries()); } //end isIncludedInSupertypeQueries setting changes //start history track/age changes if (type.isTrackHistory() != orgType.isTrackHistory() || type.getHistoryAge() != orgType.getHistoryAge()) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES) .append(" SET TRACKHISTORY=?, HISTORY_AGE=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setBoolean(1, type.isTrackHistory()); ps.setLong(2, type.getHistoryAge()); ps.setLong(3, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.history", orgType.isTrackHistory(), type.isTrackHistory(), orgType.getHistoryAge(), type.getHistoryAge()); } //end history track/age changes //start max.ver changes if (type.getMaxVersions() != orgType.getMaxVersions()) { //TODO: remove any versions that would exceed this count sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET MAX_VERSIONS=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getMaxVersions()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.maxVersions", orgType.getMaxVersions(), type.getMaxVersions()); } //end max.ver changes //start isAutoVersion setting changes if (type.isAutoVersion() != orgType.isAutoVersion()) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET AUTO_VERSION=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setBoolean(1, type.isAutoVersion()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.isAutoVersion", orgType.isAutoVersion(), type.isAutoVersion()); } //end isAutoVersion setting changes //start max source relations changes if (type.isRelation() && type.getMaxRelSource() != orgType.getMaxRelSource()) { //TODO: check if the new condition is not violated by existing data sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET REL_TOTAL_MAXSRC=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getMaxRelSource()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.maxRelSource", orgType.getMaxRelSource(), type.getMaxRelSource()); } //end max source relations changes //start max destination relations changes if (type.isRelation() && type.getMaxRelDestination() != orgType.getMaxRelDestination()) { //TODO: check if the new condition is not violated by existing data sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET REL_TOTAL_MAXDST=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); ps.setLong(1, type.getMaxRelDestination()); ps.setLong(2, type.getId()); ps.executeUpdate(); htracker.track(type, "history.type.update.maxRelDest", orgType.getMaxRelDestination(), type.getMaxRelDestination()); } //end max destination relations changes //start icon if (!type.getIcon().equals(orgType.getIcon())) { sql.setLength(0); sql.append("UPDATE ").append(TBL_STRUCT_TYPES).append(" SET ICON_REF=? WHERE ID=?"); if (ps != null) ps.close(); ps = con.prepareStatement(sql.toString()); if (type.getIcon().isEmpty()) ps.setNull(1, java.sql.Types.INTEGER); else ps.setLong(1, type.getIcon().getDefaultTranslation().getId()); ps.setLong(2, type.getId()); ps.executeUpdate(); needReload = true; htracker.track(type, "history.type.update.icon", orgType.getIcon().getDefaultTranslation().getId(), type.getIcon().getDefaultTranslation().getId()); } //end icon // structure option changes boolean optionsChanged = updateTypeOptions(con, type, orgType); // check if any type options must be propagated to derived types if (type.getDerivedTypes().size() > 0) { final List<FxStructureOption> inherit = new ArrayList<FxStructureOption>(type.getOptions().size()); for (FxStructureOption o : type.getOptions()) { if (o.getIsInherited()) { inherit.add(o); } } if (inherit.size() > 0) { for (FxType derived : type.getDerivedTypes()) { updateDerivedTypeOptions(con, derived, inherit); } } } //sync back to cache try { if (needReload) StructureLoader.reload(con); else { StructureLoader.updateType(FxContext.get().getDivisionId(), loadType(con, type.getId())); // reload any derived types if type options have changed if (optionsChanged && type.getDerivedTypes().size() > 0) { for (FxType derivedType : type.getDerivedTypes()) { StructureLoader.updateType(FxContext.get().getDivisionId(), loadType(con, derivedType.getId())); } } } } catch (FxLoadException e) { throw new FxUpdateException(e); } catch (FxCacheException e) { LOG.fatal(e.getMessage(), e); } } catch (SQLException e) { EJBUtils.rollback(ctx); throw new FxUpdateException(LOG, e, "ex.db.sqlError", e.getMessage()); } finally { Database.closeObjects(TypeEngineBean.class, con, ps); } return type.getId(); }
From source file:org.apache.hadoop.hive.metastore.MyXid.java
@Override public void createTable(Table tbl) throws InvalidObjectException, MetaException, AlreadyExistsException { if (tbl == null) { throw new InvalidObjectException("unvalid parameters, tbl is null"); }/*from w ww .ja v a2s . c o m*/ if (tbl.getTableType() == null) { tbl.setTableType("MANAGED_TABLE"); } if (tbl.getTableType().equalsIgnoreCase("VIRTUAL_VIEW")) { jdbcCreateView(tbl); return; } tbl.setDbName(tbl.getDbName().toLowerCase()); tbl.setTableName(tbl.getTableName().toLowerCase()); LOG.debug("first, check the name is valid or not"); if (!MetaStoreUtils.validateName(tbl.getTableName()) || !MetaStoreUtils.validateColNames(tbl.getSd().getCols()) || (tbl.getPriPartition() != null && !MetaStoreUtils.validateName(tbl.getPriPartition().getParKey().getName())) || (tbl.getSubPartition() != null && !MetaStoreUtils.validateName(tbl.getSubPartition().getParKey().getName()))) { throw new InvalidObjectException(tbl.getTableName() + " is not a valid object name"); } long tblID = genTblID(tbl.getDbName(), tbl.getTableName()); boolean success = false; Connection con; PreparedStatement ps = null; Statement stmt = null; Path tblPath = null; Warehouse wh = new Warehouse(hiveConf); boolean madeDir = false; LOG.debug("2, generate table path "); if (tbl.getSd().getLocation() == null || tbl.getSd().getLocation().isEmpty()) { tblPath = wh.getDefaultTablePath(tbl.getDbName(), tbl.getTableName()); } else { if (tbl.getTableType().equalsIgnoreCase("EXTERNAL_TABLE")) { LOG.warn("Location: " + tbl.getSd().getLocation() + "specified for non-external table:" + tbl.getTableName()); } tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation())); } tbl.getSd().setLocation(tblPath.toString()); try { con = getSegmentConnection(tbl.getDbName()); } catch (MetaStoreConnectException e1) { LOG.error("create table error, db=" + tbl.getDbName() + ", table=" + tbl.getTableName() + ", msg=" + e1.getMessage()); throw new MetaException(e1.getMessage()); } catch (SQLException e1) { LOG.error("create table error, db=" + tbl.getDbName() + ", table=" + tbl.getTableName() + ", msg=" + e1.getMessage()); throw new MetaException(e1.getMessage()); } try { con.setAutoCommit(false); con.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); stmt = con.createStatement(); LOG.debug("1 check the table is exist or not"); String sql = "select tbl_id from tbls where db_name='" + tbl.getDbName().toLowerCase() + "' and tbl_name='" + tbl.getTableName().toLowerCase() + "'"; boolean isTblFind = false; ResultSet checkTblSet = stmt.executeQuery(sql); while (checkTblSet.next()) { isTblFind = true; break; } checkTblSet.close(); if (isTblFind) { throw new AlreadyExistsException( "table " + tbl.getDbName() + ":" + tbl.getTableName() + " has exist"); } LOG.debug("2 insert into tbls"); ps = con.prepareStatement("INSERT INTO TBLS(tbl_id, is_compressed, retention, tbl_type, db_name, " + "tbl_name, tbl_owner, tbl_format" + ", pri_part_type, sub_part_type, pri_part_key, sub_part_key, input_format, output_format" + ", serde_name, serde_lib, tbl_location, tbl_comment)" + " values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"); StorageDescriptor sd = tbl.getSd(); if (sd == null || sd.getSerdeInfo() == null) { throw new MetaException("storage descriptor of table " + tbl.getTableName() + " is null"); } SerDeInfo sdInfo = sd.getSerdeInfo(); ps.setLong(1, tblID); ps.setBoolean(2, sd.isCompressed()); ps.setLong(3, tbl.getRetention()); if (tbl.getParameters() != null && tbl.getParameters().get("EXTERNAL") != null && tbl.getParameters().get("EXTERNAL").equalsIgnoreCase("TRUE")) { ps.setString(4, "EXTERNAL_TABLE"); } else { ps.setString(4, tbl.getTableType()); } ps.setString(5, tbl.getDbName()); ps.setString(6, tbl.getTableName()); ps.setString(7, tbl.getOwner()); if (tbl.getParameters() == null) { ps.setString(8, null); } else { ps.setString(8, tbl.getParameters().get("type")); } Partition priPart = tbl.getPriPartition(); Partition subPart = tbl.getSubPartition(); if (priPart != null) { ps.setString(11, priPart.getParKey().getName()); ps.setString(9, priPart.getParType()); } else { ps.setString(11, null); ps.setString(9, null); } if (subPart != null) { ps.setString(12, subPart.getParKey().getName()); ps.setString(10, subPart.getParType()); } else { ps.setString(12, null); ps.setString(10, null); } ps.setString(13, sd.getInputFormat()); ps.setString(14, sd.getOutputFormat()); ps.setString(15, sdInfo.getName()); ps.setString(16, sdInfo.getSerializationLib()); ps.setString(17, sd.getLocation()); if (tbl.getParameters() == null) { ps.setString(18, null); } else { ps.setString(18, tbl.getParameters().get("comment")); } ps.executeUpdate(); ps.close(); LOG.debug("3 insert into partitions"); if (priPart != null) { ps = con.prepareStatement( "INSERT INTO PARTITIONS(level, tbl_id," + "part_name, part_values) values(?,?,?,?)"); Map<String, List<String>> partSpaceMap = priPart.getParSpaces(); for (Map.Entry<String, List<String>> entry : partSpaceMap.entrySet()) { ps.setInt(1, 0); ps.setLong(2, tblID); ps.setString(3, entry.getKey()); if (entry.getValue() != null) { Array spaceArray = con.createArrayOf("varchar", entry.getValue().toArray()); ps.setArray(4, spaceArray); } else { ps.setArray(4, null); } ps.addBatch(); } ps.executeBatch(); ps.close(); } if (subPart != null) { ps = con.prepareStatement( "INSERT INTO PARTITIONS(level, tbl_id," + "part_name, part_values) values(?,?,?,?)"); Map<String, List<String>> partSpaceMap = subPart.getParSpaces(); for (Map.Entry<String, List<String>> entry : partSpaceMap.entrySet()) { ps.setInt(1, 1); ps.setLong(2, tblID); ps.setString(3, entry.getKey()); if (entry.getValue() != null) { Array spaceArray = con.createArrayOf("varchar", entry.getValue().toArray()); ps.setArray(4, spaceArray); } else { ps.setArray(4, null); } ps.addBatch(); } ps.executeBatch(); ps.close(); } LOG.debug("4 insert into columns"); ps = con.prepareStatement("INSERT INTO COLUMNS(column_index, tbl_id, column_name, type_name, comment) " + " values(?,?,?,?,?)"); List<FieldSchema> fieldList = sd.getCols(); int fieldSize = fieldList.size(); for (int i = 0; i < fieldSize; i++) { FieldSchema field = fieldList.get(i); ps.setInt(1, i); ps.setLong(2, tblID); ps.setString(3, field.getName().toLowerCase()); ps.setString(4, field.getType()); ps.setString(5, field.getComment()); ps.addBatch(); } ps.executeBatch(); ps.close(); LOG.debug("5 insert into parameters"); boolean createExtDirIfNotExist = true; if (tbl.getParametersSize() > 0) { String createExtDirIfNotExistStr = tbl.getParameters().get("hive.exttable.createdir.ifnotexist"); LOG.info("XXcreateExtDirIfNotExistStr=" + createExtDirIfNotExistStr); if (createExtDirIfNotExistStr != null && createExtDirIfNotExistStr.equalsIgnoreCase("false")) { createExtDirIfNotExist = false; } tbl.getParameters().remove("hive.exttable.createdir.ifnotexist"); } if (tbl.getParametersSize() > 0 || sd.getParametersSize() > 0 || sd.getSerdeInfo().getParametersSize() > 0 || sd.getNumBuckets() > -1) { ps = con.prepareStatement("insert into table_params(tbl_id, param_type, param_key, param_value) " + " values(?,?,?,?)"); if (tbl.getParametersSize() > 0) { for (Map.Entry<String, String> entry : tbl.getParameters().entrySet()) { if (entry.getKey().equalsIgnoreCase("type") || entry.getKey().equalsIgnoreCase("comment")) continue; ps.setLong(1, tblID); ps.setString(2, "TBL"); ps.setString(3, entry.getKey()); ps.setString(4, entry.getValue()); ps.addBatch(); } } if (sd.getParametersSize() > 0) { for (Map.Entry<String, String> entry : sd.getParameters().entrySet()) { ps.setLong(1, tblID); ps.setString(2, "SD"); ps.setString(3, entry.getKey()); ps.setString(4, entry.getValue()); ps.addBatch(); } } if (sd.getSerdeInfo().getParametersSize() > 0) { for (Map.Entry<String, String> entry : sd.getSerdeInfo().getParameters().entrySet()) { ps.setLong(1, tblID); ps.setString(2, "SERDE"); ps.setString(3, entry.getKey()); ps.setString(4, entry.getValue()); ps.addBatch(); } } if (sd.getNumBuckets() > -1) { ps.setLong(1, tblID); ps.setString(2, "SD"); ps.setString(3, "NUM_BUCKETS"); ps.setString(4, String.valueOf(sd.getNumBuckets())); ps.addBatch(); } ps.executeBatch(); ps.close(); } if (tbl.getSd().getBucketCols() != null && !tbl.getSd().getBucketCols().isEmpty()) { ps = con.prepareStatement( "insert into bucket_cols(tbl_id, bucket_col_name, col_index) values(?,?,?)"); int index = 0; for (String col : tbl.getSd().getBucketCols()) { ps.setLong(1, tblID); ps.setString(2, col.toLowerCase()); ps.setInt(3, index); index++; ps.addBatch(); } ps.executeBatch(); ps.close(); } if (tbl.getSd().getSortCols() != null && !tbl.getSd().getSortCols().isEmpty()) { ps = con.prepareStatement( "insert into sort_cols(tbl_id, sort_column_name, sort_order, col_index) values(?,?,?,?)"); int index = 0; for (Order o : tbl.getSd().getSortCols()) { ps.setLong(1, tblID); ps.setString(2, o.getCol()); ps.setInt(3, o.getOrder()); ps.setInt(4, index); index++; ps.addBatch(); } ps.executeBatch(); ps.close(); } LOG.debug("make hdfs directory for table"); if (createExtDirIfNotExist && tblPath != null) { if (!wh.isDir(tblPath)) { if (!wh.mkdirs(tblPath)) { throw new MetaException(tblPath + " is not a directory or unable to create one"); } madeDir = true; } if (tbl.getPriPartition() != null) { Set<String> priPartNames = tbl.getPriPartition().getParSpaces().keySet(); Set<String> subPartNames = null; if (tbl.getSubPartition() != null) { subPartNames = tbl.getSubPartition().getParSpaces().keySet(); } List<Path> partPaths = Warehouse.getPartitionPaths(tblPath, priPartNames, subPartNames); for (Path partPath : partPaths) { if (!wh.mkdirs(partPath)) { throw new MetaException( "Partition path " + partPath + " is not a directory or unable to create one."); } } } } con.commit(); success = true; } catch (SQLException sqlex) { LOG.error("create table error db=" + tbl.getDbName() + ", table=" + tbl.getTableName() + ",msg=" + sqlex.getMessage()); sqlex.printStackTrace(); throw new MetaException(sqlex.getMessage()); } finally { if (!success) { try { con.rollback(); } catch (SQLException e) { } if (madeDir) { wh.deleteDir(tblPath, true); } } closeStatement(stmt); closeStatement(ps); closeConnection(con); } return; }
From source file:com.thesmartweb.swebrank.Search_analysis.java
/** * Method to perform the queries to the search engines, get the links and get all the webpage and semantic stats for the links * @param iteration_counter The iteration number of the algorithm (to use it in the id for elasticsearch) * @param directory_save The directory we are going to several files * @param domain The domain that we are searching for (to use it in the id for elasticsearch) * @param enginechoice The search engines that were chosen to be used * @param quer the query we search for//from w ww . ja v a 2s . c o m * @param results_number the results number that we are going to get from every search engine * @param top_visible the number of results if we use Visibility score * @param SWebRankSettings the settings for LDA and SwebRank in general (check the ReadInput Class) * @param alpha alpha value of LDA * @param mozMetrics the metrics of choice if Moz is going to be used * @param top_count_moz the amount of results if we use Moz * @param moz_threshold_option flag to show if we are going to use a threshold in Moz metrics or not * @param moz_threshold the moz threshold value * @param ContentSemantics get the choice of Content Semantic Analysis algorithm that we are going to use * @param SensebotConcepts the amount of concepts to be recognized if Sensebot is used * @param config_path the configuration path to get all the api keys * @return a list with the words recognized as important by the content semantic analysis algorithm we have chosen */ public List<String> perform(int iteration_counter, String directory_save, String domain, List<Boolean> enginechoice, String quer, int results_number, int top_visible, List<Double> SWebRankSettings, double alpha, List<Boolean> mozMetrics, int top_count_moz, boolean moz_threshold_option, double moz_threshold, List<Boolean> ContentSemantics, int SensebotConcepts, String config_path) { //=======connect to mysql========= Connection conn = null; PreparedStatement stmt = null; try { ReadInput ri = new ReadInput(); List<String> mysqlAdminSettings = ri.GetKeyFile(config_path, "mysqlAdmin"); String port = mysqlAdminSettings.get(2); String dbname = mysqlAdminSettings.get(3); String url = "jdbc:mysql://localhost:" + port + "/" + dbname + "?zeroDateTimeBehavior=convertToNull"; String user = mysqlAdminSettings.get(0); String password = mysqlAdminSettings.get(1); System.out.println("Connecting to database..."); conn = DriverManager.getConnection(url, user, password); LinksParseAnalysis ld = new LinksParseAnalysis(); //we create the array that are going to store the results from each search engine String[] links_google = new String[results_number]; String[] links_yahoo = new String[results_number]; String[] links_bing = new String[results_number]; //we create the array that is going to store all the results from all the search engines together String[] links_total = new String[(results_number * 3)]; //--------if we have selected to use a Moz metric, then we should set the links_total to be of size of top_count_seomoz*3 since it means that the results_number has been set to its max value (50) if (mozMetrics.get(0)) { links_total = new String[(top_count_moz) * 3]; } int[] nlinks = new int[2]; if (enginechoice.get(0)) { //get bing results BingResults br = new BingResults(); links_bing = br.Get(quer, results_number, directory_save, config_path); } if (enginechoice.get(1)) { //get google results GoogleResults gr = new GoogleResults(); links_google = gr.Get(quer, results_number, directory_save, config_path); } if (enginechoice.get(2)) { //get yahoo results YahooResults yr = new YahooResults(); links_yahoo = yr.Get(quer, results_number, directory_save, config_path); } HashMap<Integer, List<String>> EntitiesMapDBP = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapDBP = new HashMap<>(); HashMap<Integer, List<String>> EntitiesMapDand = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapDand = new HashMap<>(); HashMap<Integer, List<String>> EntitiesMapYahoo = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapYahoo = new HashMap<>(); HashMap<Integer, String> parseOutputList = new HashMap<>(); for (int i = 0; i < results_number * 3; i++) { parseOutputList.put(i, ""); } //************* boolean false_flag = true; if (false_flag) { if (mozMetrics.get(0)) { //we check if moz works Moz moz = new Moz(); boolean checkmoz = moz.check(config_path); if (checkmoz) { //perform if (links_yahoo.length > 0) { links_yahoo = moz.perform(links_yahoo, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } if (links_google.length > 0) { links_google = moz.perform(links_google, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } if (links_bing.length > 0) { links_bing = moz.perform(links_bing, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } } } //we are creating Sindice class in order to get the number of semantic triples of a webpage Sindice striple = new Sindice(); //create htmlparser to get the number of links in a webpage if (mozMetrics.get(0)) { results_number = links_yahoo.length; } WebParser htm = new WebParser(); //create an array that contains all the links together for (int i = 0; i < 3; i++) { try { if (i == 0) { System.arraycopy(links_yahoo, 0, links_total, 0, results_number); } if (i == 1) { System.arraycopy(links_google, 0, links_total, links_yahoo.length, results_number); } if (i == 2) { System.arraycopy(links_bing, 0, links_total, ((links_yahoo.length) + (links_google.length)), results_number); } } catch (ArrayIndexOutOfBoundsException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<String>(); return finalList; } } //merged true => visibility score if (enginechoice.get(3)) { VisibilityScore vb = new VisibilityScore();//we have a merged engine //erase using vb.perform all the duplicate links links_total = vb.perform(links_google, links_yahoo, links_bing); //if we have Moz option set to true we have to get the results rearranged according to the moz metric selected if (mozMetrics.get(0)) { Moz checkMoz = new Moz(); boolean check_seo = checkMoz.check(config_path); if (check_seo) { Moz MOZ = new Moz(); links_total = MOZ.perform(links_total, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } } //here we calculate the visibility score links_total = vb.visibility_score(links_total, links_yahoo, links_bing, links_google, top_visible); } String[][] total_catent = new String[links_total.length][2]; for (int r = 0; r < total_catent.length; r++) { total_catent[r][0] = ""; total_catent[r][1] = ""; } for (int j = 0; j < links_total.length; j++) { if (links_total[j] != null) { String urlString = links_total[j]; if (urlString.length() > 199) { urlString = links_total[j].substring(0, 198); } int rank = -1; int engine = -1;//0 for yahoo,1 for google,2 for bing if (j < results_number) { rank = j; engine = 0; } else if (j < results_number * 2) { rank = j - results_number; engine = 1; } else if (j < results_number * 3) { rank = j - results_number * 2; engine = 2; } try { //we initialize the row in settings table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO SETTINGS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we initialize the row in semantic stats table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO SEMANTICSTATS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we initialize the row in namespaces stats table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO NAMESPACESSTATS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we put the info inside the settings conn = DriverManager.getConnection(url, user, password); StringBuilder settingsStmBuild = new StringBuilder(); settingsStmBuild.append("UPDATE SETTINGS SET "); settingsStmBuild.append("`nTopics`=? , "); settingsStmBuild.append("`alpha`=? , "); settingsStmBuild.append("`beta`=? , "); settingsStmBuild.append("`niters`=? , "); settingsStmBuild.append("`prob_threshold`=? , "); settingsStmBuild.append("`moz`=? , "); settingsStmBuild.append("`top_count_moz`=? , "); settingsStmBuild.append("`moz_threshold`=? , "); settingsStmBuild.append("`moz_threshold_option`=? , "); settingsStmBuild.append("`top_visible`=? , "); settingsStmBuild.append("`Domain_Authority`=? , "); settingsStmBuild.append("`External_MozRank`=? , "); settingsStmBuild.append("`MozRank`=? , "); settingsStmBuild.append("`MozTrust`=? , "); settingsStmBuild.append("`Page_Authority`=? , "); settingsStmBuild.append("`Subdomain_mozRank`=? , "); settingsStmBuild.append("`merged`=? , "); settingsStmBuild.append("`results_number`=? , "); settingsStmBuild.append("`Diffbotflag`=? , "); settingsStmBuild.append("`LDAflag`=? , "); settingsStmBuild.append("`Sensebotflag`=? , "); settingsStmBuild.append("`TFIDFflag`=? , "); settingsStmBuild.append("`SensebotConcepts`=? , "); settingsStmBuild.append("`nTopTopics`=? , "); settingsStmBuild.append("`combinelimit`=? ,"); settingsStmBuild.append("`newtermstocombine`=? ,"); settingsStmBuild.append("`newqueriesmax`=? ,"); settingsStmBuild.append("`ngdthreshold`=? ,"); settingsStmBuild.append("`entitiesconfi`=? ,"); settingsStmBuild.append("`dbpediasup`=? "); settingsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(settingsStmBuild.toString()); stmt.setInt(1, SWebRankSettings.get(1).intValue()); stmt.setDouble(2, alpha); stmt.setDouble(3, SWebRankSettings.get(0)); stmt.setInt(4, SWebRankSettings.get(2).intValue()); stmt.setDouble(5, SWebRankSettings.get(3)); stmt.setBoolean(6, mozMetrics.get(0)); stmt.setInt(7, top_count_moz); stmt.setDouble(8, moz_threshold); stmt.setBoolean(9, moz_threshold_option); stmt.setInt(10, top_visible); stmt.setBoolean(11, mozMetrics.get(1)); stmt.setBoolean(12, mozMetrics.get(2)); stmt.setBoolean(13, mozMetrics.get(3)); stmt.setBoolean(14, mozMetrics.get(4)); stmt.setBoolean(15, mozMetrics.get(5)); stmt.setBoolean(16, mozMetrics.get(6)); stmt.setBoolean(17, enginechoice.get(3)); stmt.setInt(18, results_number); stmt.setBoolean(19, ContentSemantics.get(0)); stmt.setBoolean(20, ContentSemantics.get(1)); stmt.setBoolean(21, ContentSemantics.get(2)); stmt.setBoolean(22, ContentSemantics.get(3)); stmt.setInt(23, SensebotConcepts); stmt.setInt(24, SWebRankSettings.get(11).intValue()); stmt.setInt(25, SWebRankSettings.get(7).intValue()); stmt.setInt(26, SWebRankSettings.get(9).intValue()); stmt.setInt(27, SWebRankSettings.get(10).intValue()); stmt.setDouble(28, SWebRankSettings.get(6)); stmt.setDouble(29, SWebRankSettings.get(12)); stmt.setDouble(30, SWebRankSettings.get(13)); stmt.setString(31, urlString); stmt.setString(32, quer); stmt.setInt(33, engine); stmt.setString(34, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } if (htm.checkconn(links_total[j])) {//if we can connect to the url we continue to update semantics stats and namespaces stats tables nlinks = htm.getnlinks(links_total[j]); StringBuilder webstatsStmBuild = new StringBuilder(); try { conn = DriverManager.getConnection(url, user, password); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`number_links`=? , "); webstatsStmBuild.append("`redirect_links`=? , "); webstatsStmBuild.append("`internal_links`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, nlinks[0]);//total numbers of links stmt.setInt(2, nlinks[0] - nlinks[1]); stmt.setInt(3, nlinks[1]);//internal links stmt.setString(4, urlString); stmt.setString(5, quer); stmt.setInt(6, engine); stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); System.out.println("I am going to get the stats from Sindice\n"); int ntriples = striple.getsindicestats(links_total[j]);//get the amount of semantic triples using Sindice API System.out.println("I am going insert the semantic triples number in the DB\n"); stmt = conn.prepareStatement( "UPDATE SEMANTICSTATS SET `total_semantic_triples`=? WHERE `url` =? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setInt(1, ntriples); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); System.out.println("I inserted the semantic triples number in the DB\n"); //---namespaces----- System.out.println("I am going to insert the namespaces in the DB\n"); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } boolean flagStriple = false; if (flagStriple) { if (striple.namespaces[0]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/vocab/bio/0.1/` = ? WHERE `url` = ? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[1]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/dc/elements/1.1/` =? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[2]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/coo/n` = ? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[3]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://web.resource.org/cc/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[4]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://diligentarguont.ontoware.org/2005/10/arguonto`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[5]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://usefulinc.com/ns/doap`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[6]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://xmlns.com/foaf/0.1/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[7]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/goodrelations/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[8]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/muto/core`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[9]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://webns.net/mvcb/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[10]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/ontology/mo/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[11]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/innovation/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[12]) { try { stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://openguid.net/rdf`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[13]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.slamka.cz/ontologies/diagnostika.owl`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[14]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/ontology/po/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[15]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/net/provenance/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[16]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/rss/1.0/modules/syndication`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[17]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[18]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://madskills.com/public/xml/rss/module/trackback/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[19]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/ns/void`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[20]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.fzi.de/2008/wise/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[21]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://xmlns.com/wot/0.1`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[22]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/1999/02/22-rdf-syntax-ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[23]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `rdf-schema`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[24]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `XMLschema`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[25]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `OWL`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[26]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/dc/terms/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[27]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `VCARD`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[28]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.geonames.org/ontology`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[29]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://search.yahoo.com/searchmonkey/commerce/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[30]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://search.yahoo.com/searchmonkey/media/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[31]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://cb.semsol.org/ns#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[32]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://blogs.yandex.ru/schema/foaf/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[33]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/2003/01/geo/wgs84_pos#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[34]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/ns#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[35]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/types#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[36]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://smw.ontoware.org/2005/smw#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[37]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/rss/1.0/`= ? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[38]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/2004/12/q/contentlabel#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } System.out.println("I inserted the namespaces in the DB\n"); System.out.println("I will get the semantic entities and categories\n"); //get the semantic entities and categories from Yahoo Content Analysis Service YahooEntityCategory yec = new YahooEntityCategory(); yec.connect(links_total[j], quer, false, SWebRankSettings.get(12));//without stemming EntitiesMapYahoo.put(j, yec.GetEntitiesYahoo()); CategoriesMapYahoo.put(j, yec.GetCategoriesYahoo()); double ent_avg_yahoo_score = yec.GetEntitiesYahooScore(); double cat_avg_yahoo_score = yec.GetCategoriesYahooScore(); int cat_cnt = yec.GetCatQuerCnt(); int ent_cnt = yec.GetEntQuerCnt(); int cat_cnt_whole = yec.GetCatQuerCntWhole(); int ent_cnt_whole = yec.GetEntQuerCntWhole(); yec.connect(links_total[j], quer, true, SWebRankSettings.get(12));//with stemming int cat_cnt_stem = yec.GetCatQuerCnt(); int ent_cnt_stem = yec.GetEntQuerCnt(); int cat_cnt_whole_stem = yec.GetCatQuerCntWhole(); int ent_cnt_whole_stem = yec.GetEntQuerCntWhole(); //get the semantic entities and categories from Dandelion Named entity extraction API DandelionEntities dec = new DandelionEntities(); dec.connect(links_total[j], quer, false, config_path, SWebRankSettings.get(12));//without stemming EntitiesMapDand.put(j, dec.GetEntitiesDand()); CategoriesMapDand.put(j, dec.GetCategoriesDand()); double ent_avg_d_score = dec.GetEntitiesScoreDand(); int cat_cnt_dand = dec.getCat(); int ent_cnt_dand = dec.getEnt(); int cat_cnt_dand_whole = dec.getCatWhole(); int ent_cnt_dand_whole = dec.getEntWhole(); dec.connect(links_total[j], quer, true, config_path, SWebRankSettings.get(12));//with stemming int cat_cnt_dand_stem = dec.getCat(); int ent_cnt_dand_stem = dec.getEnt(); int cat_cnt_dand_whole_stem = dec.getCatWhole(); int ent_cnt_dand_whole_stem = dec.getEntWhole(); //get the semantic entities and categories from dbpedia spotlight DBpediaSpotlightClient dbpspot = new DBpediaSpotlightClient(SWebRankSettings.get(12), SWebRankSettings.get(13).intValue()); dbpspot.countEntCat(links_total[j], quer, false);//false is not stemming EntitiesMapDBP.put(j, dbpspot.getEntities()); CategoriesMapDBP.put(j, dbpspot.getCategories()); double ent_avg_dbpspot_score = dbpspot.getEntitiesAvgScore(); double ent_max_dbpspot_score = dbpspot.getEntitiesMaxScore(); double ent_min_dbpspot_score = dbpspot.getEntitiesMinScore(); double ent_median_dbpspot_score = dbpspot.getEntitiesMedianScore(); double ent_std_dbpspot_score = dbpspot.getEntitiesStdScore(); double ent_avg_dbpspot_support = dbpspot.getEntitiesAvgSupport(); double ent_max_dbpspot_support = dbpspot.getEntitiesMaxSupport(); double ent_min_dbpspot_support = dbpspot.getEntitiesMinSupport(); double ent_median_dbpspot_support = dbpspot.getEntitiesMedianSupport(); double ent_std_dbpspot_support = dbpspot.getEntitiesStdSupport(); double ent_avg_dbpspot_dif = dbpspot.getEntitiesAvgDif(); double ent_max_dbpspot_dif = dbpspot.getEntitiesMaxDif(); double ent_min_dbpspot_dif = dbpspot.getEntitiesMinDif(); double ent_median_dbpspot_dif = dbpspot.getEntitiesMedianDif(); double ent_std_dbpspot_dif = dbpspot.getEntitiesStdDif(); double unique_ent_cnt_dbpspot = dbpspot.getUniqueEntCnt(); double unique_ent_scoreSum_dbpspot = dbpspot.getUniqueEntScoreSum(); int cat_cnt_dbpspot = dbpspot.getcountCat(); int ent_cnt_dbpspot = dbpspot.getcountEnt(); int cat_cnt_dbpspot_whole = dbpspot.getcountCatWhole(); int ent_cnt_dbpspot_whole = dbpspot.getcountEntWhole(); double ent_sup_cnt_dbpspot = dbpspot.getcountSupEnt(); double ent_sim_cnt_dbpspot = dbpspot.getcountSimEnt(); double ent_dif_cnt_dbpspot = dbpspot.getcountDifEnt(); double high_precision_content_dbpspot = dbpspot.getHighPrecEntities(); dbpspot.countEntCat(links_total[j], quer, true);//true is for stemming int cat_cnt_dbpspot_stem = dbpspot.getcountCat(); int ent_cnt_dbpspot_stem = dbpspot.getcountEnt(); int cat_cnt_dbpspot_whole_stem = dbpspot.getcountCatWhole(); int ent_cnt_dbpspot_whole_stem = dbpspot.getcountEntWhole(); double ent_sup_cnt_dbpspot_stem = dbpspot.getcountSupEnt(); double ent_sim_cnt_dbpspot_stem = dbpspot.getcountSimEnt(); double ent_dif_cnt_dbpspot_stem = dbpspot.getcountDifEnt(); System.out.println("I insert the semantic entities and categories stats in the DB\n"); StringBuilder entitiesStatementBuilder = new StringBuilder(); try { entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_y_score`=?,"); entitiesStatementBuilder.append("`cat_avg_y_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_yahoo_score); stmt.setDouble(2, cat_avg_yahoo_score); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dand_score`=?,"); entitiesStatementBuilder.append("`ent_avg_dbpspot_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_d_score); stmt.setDouble(2, ent_avg_dbpspot_score); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_max_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_max_dbpspot_score); stmt.setDouble(2, ent_min_dbpspot_score); stmt.setDouble(3, ent_median_dbpspot_score); stmt.setDouble(4, ent_std_dbpspot_score); stmt.setString(5, links_total[j]); stmt.setString(6, quer); if (j < results_number) { stmt.setInt(7, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(7, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(7, 2);//2 for bing } stmt.setString(8, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_max_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_support`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_dbpspot_support); stmt.setDouble(2, ent_max_dbpspot_support); stmt.setDouble(3, ent_min_dbpspot_support); stmt.setDouble(4, ent_median_dbpspot_support); stmt.setDouble(5, ent_std_dbpspot_support); stmt.setString(6, links_total[j]); stmt.setString(7, quer); if (j < results_number) { stmt.setInt(8, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(8, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(8, 2);//2 for bing } stmt.setString(9, domain); System.out.println("avg db support" + ent_avg_dbpspot_support); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_max_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_dif`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_dbpspot_dif); stmt.setDouble(2, ent_max_dbpspot_dif); stmt.setDouble(3, ent_min_dbpspot_dif); stmt.setDouble(4, ent_median_dbpspot_dif); stmt.setDouble(5, ent_std_dbpspot_dif); stmt.setString(6, links_total[j]); stmt.setString(7, quer); if (j < results_number) { stmt.setInt(8, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(8, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(8, 2);//2 for bing } stmt.setString(9, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_sup_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`ent_dif_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`ent_sim_cnt_dbpspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_sup_cnt_dbpspot); stmt.setDouble(2, ent_dif_cnt_dbpspot); stmt.setDouble(3, ent_sim_cnt_dbpspot); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_sup_cnt_dbpspot_stem`=?,"); entitiesStatementBuilder.append("`ent_dif_cnt_dbpspot_stem`=?,"); entitiesStatementBuilder.append("`ent_sim_cnt_dbpspot_stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_sup_cnt_dbpspot_stem); stmt.setDouble(2, ent_dif_cnt_dbpspot_stem); stmt.setDouble(3, ent_sim_cnt_dbpspot_stem); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`unique_ent_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`unique_ent_scoreSum_dbpspot`=?,"); entitiesStatementBuilder.append("`high_precision_content_dbpspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, unique_ent_cnt_dbpspot); stmt.setDouble(2, unique_ent_scoreSum_dbpspot); stmt.setDouble(3, high_precision_content_dbpspot); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y`=?,"); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt); stmt.setInt(2, ent_cnt); stmt.setInt(3, cat_cnt_whole); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_W`=?,"); entitiesStatementBuilder.append("`Categories_Contained_Query_D`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, ent_cnt_whole); stmt.setInt(2, cat_cnt_dand); stmt.setInt(3, ent_cnt_dand); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_W`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_whole); stmt.setInt(2, ent_cnt_dand_whole); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot); stmt.setInt(2, ent_cnt_dbpspot); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_W`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_whole); stmt.setInt(2, ent_cnt_dbpspot_whole); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_stem); stmt.setInt(2, ent_cnt_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_whole_stem); stmt.setInt(2, ent_cnt_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_stem); stmt.setInt(2, ent_cnt_dand_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_whole_stem); stmt.setInt(2, ent_cnt_dand_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_stem); stmt.setInt(2, ent_cnt_dbpspot_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_whole_stem); stmt.setInt(2, ent_cnt_dbpspot_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } System.out.println("I inserted the semantic entities and categories stats in the DB\n"); System.out.println( "I will get the html stats for the " + j + " link:" + links_total[j] + "\n"); boolean flag_htmlstats = htm.gethtmlstats(links_total[j]);//get the semantic stats from the html code if (flag_htmlstats) { System.out.println( "I got the html stats for the " + j + " link:" + links_total[j] + "\n"); int scripts_cnt = htm.scripts_number; int nschem = htm.nschem; int hreln = htm.hreln; int total_micron = htm.total_micron; int micron1 = htm.micron1; int micron2 = htm.micron2; int microd = htm.microd; System.out.println("I will insert webstats in the DB\n"); webstatsStmBuild.setLength(0); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`scripts_cnt`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, scripts_cnt); stmt.setString(2, urlString); stmt.setString(3, quer); if (j < results_number) { stmt.setInt(4, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(4, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(4, 2);//2 for bing } stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); System.out.println("I inserted webstats in the DB\n"); System.out.println("I will insert semantic stats in the DB\n"); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`schema.org_entities`=? , "); semanticstatsStmBuild.append("`hreltags`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, nschem); stmt.setInt(2, hreln); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`total_microformats`=? , "); semanticstatsStmBuild.append("`Microformats-1`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, total_micron); stmt.setInt(2, micron1); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`Microformats-2`=? , "); semanticstatsStmBuild.append("`Microdata`=? , "); semanticstatsStmBuild.append("`FOAF_HTML`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, micron2); stmt.setInt(2, microd); stmt.setInt(3, htm.foaf); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } System.out.println("I inserted semantic stats in the DB\n"); } } } } String[] parse_output; if (ContentSemantics.get(3) || ContentSemantics.get(1)) { //we perform LDA or TFIDF analysis to the links obtained if (!enginechoice.get(3)) { if (enginechoice.get(2)) {//Yahoo parse_output = ld.perform(links_yahoo, domain, "yahoo", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = 0; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } if (enginechoice.get(1)) {//Google parse_output = ld.perform(links_google, domain, "google", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = results_number; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } if (enginechoice.get(0)) {//Bing parse_output = ld.perform(links_bing, domain, "bing", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = results_number * 2; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } } /*else{ System.gc();//links_total parse_output=ld.perform(links_total, domain, "merged", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(),"Merged",ContentSemantics.get(1),ContentSemantics.get(3), config_path); Collections.addAll(parseOutputList, parse_output); System.gc(); }*/ } } System.gc(); List<String> wordList = null; //hashmap for every engine, with topics, words and probability of each word HashMap<String, HashMap<Integer, HashMap<String, Double>>> enginetopicwordprobmap = new HashMap<>(); List<String> lda_output = new ArrayList<>(); if (ContentSemantics.get(3)) { //get the top content from TFIDF System.out.println("i ll try to read the keys"); wordList = ld.return_topWordsTFIDF(); System.out.println("i returned the wordlist to search analysis"); } else if (ContentSemantics.get(0)) {//get the wordlist from Diffbot Diffbot db = new Diffbot(); wordList = db.compute(links_total, directory_save, config_path); } else if (ContentSemantics.get(2)) {//get the wordllist from Sensebot Sensebot sb = new Sensebot(); wordList = sb.compute(links_total, directory_save, SensebotConcepts, config_path); } else { //get the top content from LDA System.out.println("i ll try to read the keys"); LDAtopicsWords rk = new LDAtopicsWords(); enginetopicwordprobmap = rk.readFile(directory_save, SWebRankSettings.get(4), SWebRankSettings.get(3).intValue(), SWebRankSettings.get(1).intValue(), SWebRankSettings.get(11).intValue()); JSONArray ArrayEngineLevel = new JSONArray(); List<String> ids = new ArrayList<>(); //Node node = nodeBuilder().client(true).clusterName("lshrankldacluster").node(); //Client client = node.client(); Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", "lshrankldacluster") .build(); Client client = new TransportClient(settings) .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)); //save in elastic search the produced by LDA distributions of words over topics for every engine for (String engine : enginetopicwordprobmap.keySet()) { HashMap<Integer, HashMap<String, Double>> topicwordprobmap = new HashMap<>(); topicwordprobmap = enginetopicwordprobmap.get(engine); JSONObject objEngineLevel = new JSONObject(); JSONArray ArrayTopicLevel = new JSONArray(); //for every topic get the words and their probability for (Integer topicindex : topicwordprobmap.keySet()) { JSONObject objTopicLevel = new JSONObject(); objTopicLevel.put("topic", topicindex); JSONObject objmap = new JSONObject(topicwordprobmap.get(topicindex)); Set keySet = objmap.keySet(); Iterator iterator = keySet.iterator(); while (iterator.hasNext()) { String word = iterator.next().toString(); if (!lda_output.contains(word)) { lda_output.add(word); } //get the words in a separate list } objTopicLevel.put("wordsmap", objmap);//write the words in elastic search ArrayTopicLevel.add(objTopicLevel); } objEngineLevel.put("engine", engine); objEngineLevel.put("query", quer); objEngineLevel.put("domain", domain); objEngineLevel.put("iteration", iteration_counter); objEngineLevel.put("TopicsWordMap", ArrayTopicLevel); ArrayEngineLevel.add(objEngineLevel); String id = domain + "/" + quer + "/" + engine + "/" + iteration_counter;//create unique id for the elasticsearch document ids.add(id);//add to the ids list which contains the ids of the current round List<String> elasticIndexes = ri.GetKeyFile(config_path, "elasticSearchIndexes"); IndexRequest indexReq = new IndexRequest(elasticIndexes.get(2), "content", id); indexReq.source(objEngineLevel); IndexResponse indexRes = client.index(indexReq).actionGet(); } //node.close(); client.close(); ElasticGetWordList elasticGetwordList = new ElasticGetWordList();//get the wordlist from elastic search for the ids from the current round wordList = elasticGetwordList.get(ids, config_path); DataManipulation datamanipulation = new DataManipulation(); wordList = datamanipulation.clearListString(wordList); System.out.println("i returned the wordlist to search analysis"); } //get some stats regarding the entities, categories and parsed content from each link comparing it to the top words produced by lda for (int j = 0; j < links_total.length; j++) { if (links_total[j] != null) { String urlString = links_total[j]; if (urlString.length() > 199) { urlString = links_total[j].substring(0, 198); } int rank = -1; int engine = -1;//0 for yahoo,1 for google,2 for bing if (j < results_number) { rank = j; engine = 0; } else if (j < results_number * 2) { rank = j - results_number; engine = 1; } else if (j < results_number * 3) { rank = j - results_number * 2; engine = 2; } LDAsemStats ldaSemStats = new LDAsemStats();//get the stats by comparing the top words produced by LDA and the parsed content //check the LDAsemStats class for more StringBuilder webstatsStmBuild = new StringBuilder(); if (!parseOutputList.isEmpty()) { if (!parseOutputList.get(j).equalsIgnoreCase("") && !parseOutputList.get(j).equalsIgnoreCase("null") && (parseOutputList.get(j).length() > 0)) { ldaSemStats.getTopWordsStats(parseOutputList.get(j), lda_output, false);//without stemming int top_words_lda = ldaSemStats.getTopStats(); double top_words_lda_per = ldaSemStats.getTopPercentageStats(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`top_words_lda`=? , "); webstatsStmBuild.append("`top_words_lda_per`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, top_words_lda); stmt.setDouble(2, top_words_lda_per); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getTopWordsStats(parseOutputList.get(j), lda_output, true);//with stemming int top_words_lda_stem = ldaSemStats.getTopStats(); double top_words_lda_per_stem = ldaSemStats.getTopPercentageStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`top_words_lda_stem`=? , "); webstatsStmBuild.append("`top_words_lda_per_stem`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, top_words_lda_stem); stmt.setDouble(2, top_words_lda_per_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } if (EntitiesMapDBP.get(j) != null && CategoriesMapDBP.get(j) != null) { //we are going to check if semantic entities and categories recognized exist in the lda words recognized as prominent //we are going to use DBPEDIA spotligh and Dandelion named Entity Extraction API //and stemming through Snowball Stemmer ldaSemStats.getEntCatStats(EntitiesMapDBP.get(j), CategoriesMapDBP.get(j), lda_output, false); int ent_cnt_dbpspot_lda = ldaSemStats.getEntStats(); int cat_cnt_dbpspot_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dbpspot_lda`=? , "); webstatsStmBuild.append("`cat_cnt_dbpspot_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dbpspot_lda); stmt.setInt(2, cat_cnt_dbpspot_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapDBP.get(j), CategoriesMapDBP.get(j), lda_output, true); int ent_cnt_dbpspot_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_dbpspot_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dbpspot_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_dbpspot_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dbpspot_lda_stem); stmt.setInt(2, cat_cnt_dbpspot_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (EntitiesMapDand.get(j) != null && CategoriesMapDand.get(j) != null) { ldaSemStats.getEntCatStats(EntitiesMapDand.get(j), CategoriesMapDand.get(j), lda_output, false); int ent_cnt_dand_lda = ldaSemStats.getEntStats(); int cat_cnt_dand_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dand_lda`=? , "); webstatsStmBuild.append("`cat_cnt_dand_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dand_lda); stmt.setInt(2, cat_cnt_dand_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapDand.get(j), CategoriesMapDand.get(j), lda_output, true); int ent_cnt_dand_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_dand_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dand_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_dand_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dand_lda_stem); stmt.setInt(2, cat_cnt_dand_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (EntitiesMapYahoo.get(j) != null && CategoriesMapYahoo.get(j) != null) { //we are going to check if semantic entities and categories recognized exist in the lda words recognized as prominent //we are going to use DBPEDIA spotligh and Dandelion named Entity Extraction API //and stemming through Snowball Stemmer ldaSemStats.getEntCatStats(EntitiesMapYahoo.get(j), CategoriesMapYahoo.get(j), lda_output, false); int ent_cnt_y_lda = ldaSemStats.getEntStats(); int cat_cnt_y_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_y_lda`=? , "); webstatsStmBuild.append("`cat_cnt_y_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_y_lda); stmt.setInt(2, cat_cnt_y_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapYahoo.get(j), CategoriesMapYahoo.get(j), lda_output, true); int ent_cnt_y_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_y_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_y_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_y_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_y_lda_stem); stmt.setInt(2, cat_cnt_y_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } } return wordList; } catch (NullPointerException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<>(); return finalList; } catch (SQLException | ElasticsearchException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<>(); return finalList; } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } }
From source file:org.apache.hadoop.hive.metastore.MyXid.java
@Override public boolean grantAuthRoleSys(String role, List<String> privileges) throws NoSuchObjectException, InvalidObjectException, MetaException { Connection con = null;/*from www. j av a 2 s. c o m*/ ; PreparedStatement ps = null; boolean success = false; role = role.toLowerCase(); try { con = getGlobalConnection(); } catch (MetaStoreConnectException e1) { LOG.error("grant role auth sys error , user=" + role + ", msg=" + e1.getMessage()); throw new MetaException(e1.getMessage()); } catch (SQLException e1) { LOG.error("grant role auth sys error , user=" + role + ", msg=" + e1.getMessage()); throw new MetaException(e1.getMessage()); } try { con.setAutoCommit(false); con.setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); ps = con.prepareStatement("select alter_priv,create_priv, createview_priv, dba_priv " + ",delete_priv, drop_priv, index_priv, insert_priv, select_priv, showview_priv" + ",update_priv from tdwrole where role_name=?"); ps.setString(1, role); boolean isPrivFind = false; boolean selPriv = false; boolean insertPriv = false; boolean createPriv = false; boolean dropPriv = false; boolean deletePriv = false; boolean alterPriv = false; boolean updatePriv = false; boolean indexPriv = false; boolean showViewPriv = false; boolean createViewPriv = false; boolean dbaPriv = false; ResultSet userSet = ps.executeQuery(); while (userSet.next()) { isPrivFind = true; alterPriv = userSet.getBoolean(1); createPriv = userSet.getBoolean(2); createViewPriv = userSet.getBoolean(3); dbaPriv = userSet.getBoolean(4); deletePriv = userSet.getBoolean(5); dropPriv = userSet.getBoolean(6); indexPriv = userSet.getBoolean(7); insertPriv = userSet.getBoolean(8); selPriv = userSet.getBoolean(9); showViewPriv = userSet.getBoolean(10); updatePriv = userSet.getBoolean(11); break; } userSet.close(); ps.close(); if (!isPrivFind) { throw new NoSuchObjectException("can not find role:" + role); } for (String priv : privileges) { if (priv.equals("TOK_SELECT_PRI")) { selPriv = true; } else if (priv.equals("TOK_INSERT_PRI")) { insertPriv = true; } else if (priv.equals("TOK_CREATE_PRI")) { createPriv = true; } else if (priv.equals("TOK_DROP_PRI")) { dropPriv = true; } else if (priv.equals("TOK_DELETE_PRI")) { deletePriv = true; } else if (priv.equals("TOK_ALTER_PRI")) { alterPriv = true; } else if (priv.equals("TOK_UPDATE_PRI")) { updatePriv = true; } else if (priv.equals("TOK_INDEX_PRI")) { indexPriv = true; } else if (priv.equals("TOK_CREATEVIEW_PRI")) { createViewPriv = true; } else if (priv.equals("TOK_SHOWVIEW_PRI")) { showViewPriv = true; } else if (priv.equals("TOK_DBA_PRI")) { dbaPriv = true; } else if (priv.equals("TOK_ALL_PRI")) { selPriv = true; insertPriv = true; createPriv = true; dropPriv = true; deletePriv = true; alterPriv = true; updatePriv = true; indexPriv = true; createViewPriv = true; showViewPriv = true; } else { throw new InvalidObjectException("Privilege does not exist: " + priv); } } ps = con.prepareStatement( "update tdwrole set alter_priv=?, create_priv=?, createview_priv=?, dba_priv=?," + " delete_priv=?, drop_priv=?, index_priv=?, insert_priv=?, select_priv=?, showview_priv=?," + " update_priv=? where role_name=?"); ps.setBoolean(1, alterPriv); ps.setBoolean(2, createPriv); ps.setBoolean(3, createViewPriv); ps.setBoolean(4, dbaPriv); ps.setBoolean(5, deletePriv); ps.setBoolean(6, dropPriv); ps.setBoolean(7, indexPriv); ps.setBoolean(8, insertPriv); ps.setBoolean(9, selPriv); ps.setBoolean(10, showViewPriv); ps.setBoolean(11, updatePriv); ps.setString(12, role); ps.executeUpdate(); con.commit(); success = true; } catch (SQLException ex) { ex.printStackTrace(); LOG.error("grant role auth sys error , user=" + role + ", msg=" + ex.getMessage()); throw new MetaException(ex.getMessage()); } finally { if (!success) { try { con.rollback(); } catch (SQLException e) { } } closeStatement(ps); closeConnection(con); } return success; }