List of usage examples for java.util HashMap isEmpty
public boolean isEmpty()
From source file:net.siveo.virtualization.vmware.Main.java
public ArrayList<Counter> getHostPerformanceCounters(HashMap<String, String> mapOfCounter) { ArrayList<Counter> listOfCounters = new ArrayList<Counter>(); Counter counter = null;//from w w w . j a v a 2s . c om //************************************** //************************************** if (mapOfCounter != null && !mapOfCounter.isEmpty()) { for (Map.Entry<String, String> counterMap : mapOfCounter.entrySet()) { counter = new Counter(); counter.setFullName(counterMap.getKey()); counter.setInstance(counterMap.getValue()); listOfCounters.add(counter); } } return listOfCounters; }
From source file:org.telegram.android.MessagesController.java
public boolean processUpdateArray(ArrayList<TLRPC.Update> updates, final ArrayList<TLRPC.User> usersArr, final ArrayList<TLRPC.Chat> chatsArr) { if (updates.isEmpty()) { return true; }/*from www . j a v a 2 s . com*/ long currentTime = System.currentTimeMillis(); final HashMap<Long, ArrayList<MessageObject>> messages = new HashMap<>(); final HashMap<Long, TLRPC.WebPage> webPages = new HashMap<>(); final ArrayList<MessageObject> pushMessages = new ArrayList<>(); final ArrayList<TLRPC.Message> messagesArr = new ArrayList<>(); final HashMap<Integer, Integer> markAsReadMessagesInbox = new HashMap<>(); final HashMap<Integer, Integer> markAsReadMessagesOutbox = new HashMap<>(); final ArrayList<Integer> markAsReadMessages = new ArrayList<>(); final HashMap<Integer, Integer> markAsReadEncrypted = new HashMap<>(); final ArrayList<Integer> deletedMessages = new ArrayList<>(); boolean printChanged = false; final ArrayList<TLRPC.ChatParticipants> chatInfoToUpdate = new ArrayList<>(); final ArrayList<TLRPC.Update> updatesOnMainThread = new ArrayList<>(); final ArrayList<TLRPC.TL_updateEncryptedMessagesRead> tasks = new ArrayList<>(); final ArrayList<Integer> contactsIds = new ArrayList<>(); boolean checkForUsers = true; ConcurrentHashMap<Integer, TLRPC.User> usersDict; ConcurrentHashMap<Integer, TLRPC.Chat> chatsDict; if (usersArr != null) { usersDict = new ConcurrentHashMap<>(); for (TLRPC.User user : usersArr) { usersDict.put(user.id, user); } } else { checkForUsers = false; usersDict = users; } if (chatsArr != null) { chatsDict = new ConcurrentHashMap<>(); for (TLRPC.Chat chat : chatsArr) { chatsDict.put(chat.id, chat); } } else { checkForUsers = false; chatsDict = chats; } if (usersArr != null || chatsArr != null) { AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { putUsers(usersArr, false); putChats(chatsArr, false); } }); } int interfaceUpdateMask = 0; for (TLRPC.Update update : updates) { if (update instanceof TLRPC.TL_updateNewMessage) { TLRPC.TL_updateNewMessage upd = (TLRPC.TL_updateNewMessage) update; if (checkForUsers) { TLRPC.User user = getUser(upd.message.from_id); if (usersDict.get(upd.message.from_id) == null && user == null || upd.message.to_id.chat_id != 0 && chatsDict.get(upd.message.to_id.chat_id) == null && getChat(upd.message.to_id.chat_id) == null) { return false; } if (user != null && user.status != null && user.status.expires <= 0) { onlinePrivacy.put(upd.message.from_id, ConnectionsManager.getInstance().getCurrentTime()); interfaceUpdateMask |= UPDATE_MASK_STATUS; } } messagesArr.add(upd.message); ImageLoader.saveMessageThumbs(upd.message); MessageObject obj = new MessageObject(upd.message, usersDict, true); if (obj.type == 11) { interfaceUpdateMask |= UPDATE_MASK_CHAT_AVATAR; } else if (obj.type == 10) { interfaceUpdateMask |= UPDATE_MASK_CHAT_NAME; } long uid; if (upd.message.to_id.chat_id != 0) { uid = -upd.message.to_id.chat_id; } else { if (upd.message.to_id.user_id == UserConfig.getClientUserId()) { upd.message.to_id.user_id = upd.message.from_id; } uid = upd.message.to_id.user_id; } ArrayList<MessageObject> arr = messages.get(uid); if (arr == null) { arr = new ArrayList<>(); messages.put(uid, arr); } arr.add(obj); if (!obj.isOut() && obj.isUnread()) { pushMessages.add(obj); } } else if (update instanceof TLRPC.TL_updateReadMessagesContents) { markAsReadMessages.addAll(update.messages); } else if (update instanceof TLRPC.TL_updateReadHistoryInbox) { TLRPC.Peer peer = ((TLRPC.TL_updateReadHistoryInbox) update).peer; if (peer.chat_id != 0) { markAsReadMessagesInbox.put(-peer.chat_id, update.max_id); } else { markAsReadMessagesInbox.put(peer.user_id, update.max_id); } } else if (update instanceof TLRPC.TL_updateReadHistoryOutbox) { TLRPC.Peer peer = ((TLRPC.TL_updateReadHistoryOutbox) update).peer; if (peer.chat_id != 0) { markAsReadMessagesOutbox.put(-peer.chat_id, update.max_id); } else { markAsReadMessagesOutbox.put(peer.user_id, update.max_id); } } else if (update instanceof TLRPC.TL_updateDeleteMessages) { deletedMessages.addAll(update.messages); } else if (update instanceof TLRPC.TL_updateUserTyping || update instanceof TLRPC.TL_updateChatUserTyping) { if (update.user_id != UserConfig.getClientUserId()) { long uid = -update.chat_id; if (uid == 0) { uid = update.user_id; } ArrayList<PrintingUser> arr = printingUsers.get(uid); if (update.action instanceof TLRPC.TL_sendMessageCancelAction) { if (arr != null) { for (int a = 0; a < arr.size(); a++) { PrintingUser pu = arr.get(a); if (pu.userId == update.user_id) { arr.remove(a); printChanged = true; break; } } if (arr.isEmpty()) { printingUsers.remove(uid); } } } else { if (arr == null) { arr = new ArrayList<>(); printingUsers.put(uid, arr); } boolean exist = false; for (PrintingUser u : arr) { if (u.userId == update.user_id) { exist = true; u.lastTime = currentTime; u.action = update.action; break; } } if (!exist) { PrintingUser newUser = new PrintingUser(); newUser.userId = update.user_id; newUser.lastTime = currentTime; newUser.action = update.action; arr.add(newUser); printChanged = true; } } onlinePrivacy.put(update.user_id, ConnectionsManager.getInstance().getCurrentTime()); } } else if (update instanceof TLRPC.TL_updateChatParticipants) { interfaceUpdateMask |= UPDATE_MASK_CHAT_MEMBERS; chatInfoToUpdate.add(update.participants); } else if (update instanceof TLRPC.TL_updateUserStatus) { interfaceUpdateMask |= UPDATE_MASK_STATUS; updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateUserName) { interfaceUpdateMask |= UPDATE_MASK_NAME; updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateUserPhoto) { interfaceUpdateMask |= UPDATE_MASK_AVATAR; MessagesStorage.getInstance().clearUserPhotos(update.user_id); updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateUserPhone) { interfaceUpdateMask |= UPDATE_MASK_PHONE; updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateContactRegistered) { if (enableJoined && usersDict.containsKey(update.user_id)) { TLRPC.TL_messageService newMessage = new TLRPC.TL_messageService(); newMessage.action = new TLRPC.TL_messageActionUserJoined(); newMessage.local_id = newMessage.id = UserConfig.getNewMessageId(); UserConfig.saveConfig(false); newMessage.flags = TLRPC.MESSAGE_FLAG_UNREAD; newMessage.date = update.date; newMessage.from_id = update.user_id; newMessage.to_id = new TLRPC.TL_peerUser(); newMessage.to_id.user_id = UserConfig.getClientUserId(); newMessage.dialog_id = update.user_id; messagesArr.add(newMessage); MessageObject obj = new MessageObject(newMessage, usersDict, true); ArrayList<MessageObject> arr = messages.get(newMessage.dialog_id); if (arr == null) { arr = new ArrayList<>(); messages.put(newMessage.dialog_id, arr); } arr.add(obj); pushMessages.add(obj); } } else if (update instanceof TLRPC.TL_updateContactLink) { if (update.my_link instanceof TLRPC.TL_contactLinkContact) { int idx = contactsIds.indexOf(-update.user_id); if (idx != -1) { contactsIds.remove(idx); } if (!contactsIds.contains(update.user_id)) { contactsIds.add(update.user_id); } } else { int idx = contactsIds.indexOf(update.user_id); if (idx != -1) { contactsIds.remove(idx); } if (!contactsIds.contains(update.user_id)) { contactsIds.add(-update.user_id); } } } else if (update instanceof TLRPC.TL_updateNewAuthorization) { AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { NotificationCenter.getInstance() .postNotificationName(NotificationCenter.newSessionReceived); } }); TLRPC.TL_messageService newMessage = new TLRPC.TL_messageService(); newMessage.action = new TLRPC.TL_messageActionLoginUnknownLocation(); newMessage.action.title = update.device; newMessage.action.address = update.location; newMessage.local_id = newMessage.id = UserConfig.getNewMessageId(); UserConfig.saveConfig(false); newMessage.flags = TLRPC.MESSAGE_FLAG_UNREAD; newMessage.date = update.date; newMessage.from_id = 777000; newMessage.to_id = new TLRPC.TL_peerUser(); newMessage.to_id.user_id = UserConfig.getClientUserId(); newMessage.dialog_id = 777000; messagesArr.add(newMessage); MessageObject obj = new MessageObject(newMessage, usersDict, true); ArrayList<MessageObject> arr = messages.get(newMessage.dialog_id); if (arr == null) { arr = new ArrayList<>(); messages.put(newMessage.dialog_id, arr); } arr.add(obj); pushMessages.add(obj); } else if (update instanceof TLRPC.TL_updateNewGeoChatMessage) { //DEPRECATED } else if (update instanceof TLRPC.TL_updateNewEncryptedMessage) { ArrayList<TLRPC.Message> decryptedMessages = SecretChatHelper.getInstance() .decryptMessage(((TLRPC.TL_updateNewEncryptedMessage) update).message); if (decryptedMessages != null && !decryptedMessages.isEmpty()) { int cid = ((TLRPC.TL_updateNewEncryptedMessage) update).message.chat_id; long uid = ((long) cid) << 32; ArrayList<MessageObject> arr = messages.get(uid); if (arr == null) { arr = new ArrayList<>(); messages.put(uid, arr); } for (TLRPC.Message message : decryptedMessages) { ImageLoader.saveMessageThumbs(message); messagesArr.add(message); MessageObject obj = new MessageObject(message, usersDict, true); arr.add(obj); pushMessages.add(obj); } } } else if (update instanceof TLRPC.TL_updateEncryptedChatTyping) { TLRPC.EncryptedChat encryptedChat = getEncryptedChatDB(update.chat_id); if (encryptedChat != null) { update.user_id = encryptedChat.user_id; long uid = ((long) update.chat_id) << 32; ArrayList<PrintingUser> arr = printingUsers.get(uid); if (arr == null) { arr = new ArrayList<>(); printingUsers.put(uid, arr); } boolean exist = false; for (PrintingUser u : arr) { if (u.userId == update.user_id) { exist = true; u.lastTime = currentTime; u.action = new TLRPC.TL_sendMessageTypingAction(); break; } } if (!exist) { PrintingUser newUser = new PrintingUser(); newUser.userId = update.user_id; newUser.lastTime = currentTime; newUser.action = new TLRPC.TL_sendMessageTypingAction(); arr.add(newUser); printChanged = true; } onlinePrivacy.put(update.user_id, ConnectionsManager.getInstance().getCurrentTime()); } } else if (update instanceof TLRPC.TL_updateEncryptedMessagesRead) { markAsReadEncrypted.put(update.chat_id, Math.max(update.max_date, update.date)); tasks.add((TLRPC.TL_updateEncryptedMessagesRead) update); } else if (update instanceof TLRPC.TL_updateChatParticipantAdd) { MessagesStorage.getInstance().updateChatInfo(update.chat_id, update.user_id, false, update.inviter_id, update.version); } else if (update instanceof TLRPC.TL_updateChatParticipantDelete) { MessagesStorage.getInstance().updateChatInfo(update.chat_id, update.user_id, true, 0, update.version); } else if (update instanceof TLRPC.TL_updateDcOptions) { ConnectionsManager.getInstance().updateDcSettings(0); } else if (update instanceof TLRPC.TL_updateEncryption) { SecretChatHelper.getInstance().processUpdateEncryption((TLRPC.TL_updateEncryption) update, usersDict); } else if (update instanceof TLRPC.TL_updateUserBlocked) { final TLRPC.TL_updateUserBlocked finalUpdate = (TLRPC.TL_updateUserBlocked) update; if (finalUpdate.blocked) { ArrayList<Integer> ids = new ArrayList<>(); ids.add(finalUpdate.user_id); MessagesStorage.getInstance().putBlockedUsers(ids, false); } else { MessagesStorage.getInstance().deleteBlockedUser(finalUpdate.user_id); } MessagesStorage.getInstance().getStorageQueue().postRunnable(new Runnable() { @Override public void run() { AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { if (finalUpdate.blocked) { if (!blockedUsers.contains(finalUpdate.user_id)) { blockedUsers.add(finalUpdate.user_id); } } else { blockedUsers.remove((Integer) finalUpdate.user_id); } NotificationCenter.getInstance() .postNotificationName(NotificationCenter.blockedUsersDidLoaded); } }); } }); } else if (update instanceof TLRPC.TL_updateNotifySettings) { updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateServiceNotification) { TLRPC.TL_message newMessage = new TLRPC.TL_message(); newMessage.local_id = newMessage.id = UserConfig.getNewMessageId(); UserConfig.saveConfig(false); newMessage.flags = TLRPC.MESSAGE_FLAG_UNREAD; newMessage.date = ConnectionsManager.getInstance().getCurrentTime(); newMessage.from_id = 777000; newMessage.to_id = new TLRPC.TL_peerUser(); newMessage.to_id.user_id = UserConfig.getClientUserId(); newMessage.dialog_id = 777000; newMessage.media = update.media; newMessage.message = ((TLRPC.TL_updateServiceNotification) update).message; messagesArr.add(newMessage); MessageObject obj = new MessageObject(newMessage, usersDict, true); ArrayList<MessageObject> arr = messages.get(newMessage.dialog_id); if (arr == null) { arr = new ArrayList<>(); messages.put(newMessage.dialog_id, arr); } arr.add(obj); pushMessages.add(obj); } else if (update instanceof TLRPC.TL_updatePrivacy) { updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateWebPage) { webPages.put(update.webpage.id, update.webpage); } } if (!messages.isEmpty()) { for (HashMap.Entry<Long, ArrayList<MessageObject>> pair : messages.entrySet()) { Long key = pair.getKey(); ArrayList<MessageObject> value = pair.getValue(); if (updatePrintingUsersWithNewMessages(key, value)) { printChanged = true; } } } if (printChanged) { updatePrintingStrings(); } final int interfaceUpdateMaskFinal = interfaceUpdateMask; final boolean printChangedArg = printChanged; if (!contactsIds.isEmpty()) { ContactsController.getInstance().processContactsUpdates(contactsIds, usersDict); } MessagesStorage.getInstance().getStorageQueue().postRunnable(new Runnable() { @Override public void run() { AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { if (!pushMessages.isEmpty()) { NotificationsController.getInstance().processNewMessages(pushMessages, true); } } }); } }); if (!messagesArr.isEmpty()) { MessagesStorage.getInstance().putMessages(messagesArr, true, true, false, MediaController.getInstance().getAutodownloadMask()); } AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { int updateMask = interfaceUpdateMaskFinal; boolean avatarsUpdate = false; if (!updatesOnMainThread.isEmpty()) { ArrayList<TLRPC.User> dbUsers = new ArrayList<>(); ArrayList<TLRPC.User> dbUsersStatus = new ArrayList<>(); SharedPreferences.Editor editor = null; for (TLRPC.Update update : updatesOnMainThread) { final TLRPC.User toDbUser = new TLRPC.User(); toDbUser.id = update.user_id; final TLRPC.User currentUser = getUser(update.user_id); if (update instanceof TLRPC.TL_updatePrivacy) { if (update.key instanceof TLRPC.TL_privacyKeyStatusTimestamp) { ContactsController.getInstance().setPrivacyRules(update.rules); } } else if (update instanceof TLRPC.TL_updateUserStatus) { if (update.status instanceof TLRPC.TL_userStatusRecently) { update.status.expires = -100; } else if (update.status instanceof TLRPC.TL_userStatusLastWeek) { update.status.expires = -101; } else if (update.status instanceof TLRPC.TL_userStatusLastMonth) { update.status.expires = -102; } if (currentUser != null) { currentUser.id = update.user_id; currentUser.status = update.status; } toDbUser.status = update.status; dbUsersStatus.add(toDbUser); if (update.user_id == UserConfig.getClientUserId()) { NotificationsController.getInstance() .setLastOnlineFromOtherDevice(update.status.expires); } } else if (update instanceof TLRPC.TL_updateUserName) { if (currentUser != null) { if (!(currentUser instanceof TLRPC.TL_userContact)) { currentUser.first_name = update.first_name; currentUser.last_name = update.last_name; } if (currentUser.username != null && currentUser.username.length() > 0) { usersByUsernames.remove(currentUser.username); } if (update.username != null && update.username.length() > 0) { usersByUsernames.put(update.username, currentUser); } currentUser.username = update.username; } toDbUser.first_name = update.first_name; toDbUser.last_name = update.last_name; toDbUser.username = update.username; dbUsers.add(toDbUser); } else if (update instanceof TLRPC.TL_updateUserPhoto) { if (currentUser != null) { currentUser.photo = update.photo; } avatarsUpdate = true; toDbUser.photo = update.photo; dbUsers.add(toDbUser); } else if (update instanceof TLRPC.TL_updateUserPhone) { if (currentUser != null) { currentUser.phone = update.phone; Utilities.phoneBookQueue.postRunnable(new Runnable() { @Override public void run() { ContactsController.getInstance().addContactToPhoneBook(currentUser, true); } }); } toDbUser.phone = update.phone; dbUsers.add(toDbUser); } else if (update instanceof TLRPC.TL_updateNotifySettings) { if (update.notify_settings instanceof TLRPC.TL_peerNotifySettings && update.peer instanceof TLRPC.TL_notifyPeer) { if (editor == null) { SharedPreferences preferences = ApplicationLoader.applicationContext .getSharedPreferences("Notifications", Activity.MODE_PRIVATE); editor = preferences.edit(); } long dialog_id = update.peer.peer.user_id; if (dialog_id == 0) { dialog_id = -update.peer.peer.chat_id; } TLRPC.TL_dialog dialog = dialogs_dict.get(dialog_id); if (dialog != null) { dialog.notify_settings = update.notify_settings; } if (update.notify_settings.mute_until > ConnectionsManager.getInstance() .getCurrentTime()) { int until = 0; if (update.notify_settings.mute_until > ConnectionsManager.getInstance() .getCurrentTime() + 60 * 60 * 24 * 365) { editor.putInt("notify2_" + dialog_id, 2); if (dialog != null) { dialog.notify_settings.mute_until = Integer.MAX_VALUE; } } else { until = update.notify_settings.mute_until; editor.putInt("notify2_" + dialog_id, 3); editor.putInt("notifyuntil_" + dialog_id, update.notify_settings.mute_until); if (dialog != null) { dialog.notify_settings.mute_until = until; } } MessagesStorage.getInstance().setDialogFlags(dialog_id, ((long) until << 32) | 1); } else { if (dialog != null) { dialog.notify_settings.mute_until = 0; } editor.remove("notify2_" + dialog_id); MessagesStorage.getInstance().setDialogFlags(dialog_id, 0); } } /* else if (update.peer instanceof TLRPC.TL_notifyChats) { disable global settings sync if (editor == null) { SharedPreferences preferences = ApplicationLoader.applicationContext.getSharedPreferences("Notifications", Activity.MODE_PRIVATE); editor = preferences.edit(); } editor.putBoolean("EnableGroup", update.notify_settings.mute_until == 0); editor.putBoolean("EnablePreviewGroup", update.notify_settings.show_previews); } else if (update.peer instanceof TLRPC.TL_notifyUsers) { if (editor == null) { SharedPreferences preferences = ApplicationLoader.applicationContext.getSharedPreferences("Notifications", Activity.MODE_PRIVATE); editor = preferences.edit(); } editor.putBoolean("EnableAll", update.notify_settings.mute_until == 0); editor.putBoolean("EnablePreviewAll", update.notify_settings.show_previews); }*/ } } if (editor != null) { editor.commit(); NotificationCenter.getInstance() .postNotificationName(NotificationCenter.notificationsSettingsUpdated); } MessagesStorage.getInstance().updateUsers(dbUsersStatus, true, true, true); MessagesStorage.getInstance().updateUsers(dbUsers, false, true, true); } if (!webPages.isEmpty()) { NotificationCenter.getInstance() .postNotificationName(NotificationCenter.didReceivedWebpagesInUpdates, webPages); } if (!messages.isEmpty()) { for (HashMap.Entry<Long, ArrayList<MessageObject>> entry : messages.entrySet()) { Long key = entry.getKey(); ArrayList<MessageObject> value = entry.getValue(); updateInterfaceWithMessages(key, value); } NotificationCenter.getInstance().postNotificationName(NotificationCenter.dialogsNeedReload); } if (printChangedArg) { updateMask |= UPDATE_MASK_USER_PRINT; } if (!contactsIds.isEmpty()) { updateMask |= UPDATE_MASK_NAME; updateMask |= UPDATE_MASK_USER_PHONE; } if (!chatInfoToUpdate.isEmpty()) { for (TLRPC.ChatParticipants info : chatInfoToUpdate) { MessagesStorage.getInstance().updateChatInfo(info.chat_id, info, true); NotificationCenter.getInstance().postNotificationName(NotificationCenter.chatInfoDidLoaded, info.chat_id, info); } } if (updateMask != 0) { NotificationCenter.getInstance().postNotificationName(NotificationCenter.updateInterfaces, updateMask); } } }); MessagesStorage.getInstance().getStorageQueue().postRunnable(new Runnable() { @Override public void run() { AndroidUtilities.runOnUIThread(new Runnable() { @Override public void run() { int updateMask = 0; if (!markAsReadMessagesInbox.isEmpty() || !markAsReadMessagesOutbox.isEmpty()) { NotificationCenter.getInstance().postNotificationName(NotificationCenter.messagesRead, markAsReadMessagesInbox, markAsReadMessagesOutbox); NotificationsController.getInstance().processReadMessages(markAsReadMessagesInbox, 0, 0, 0, false); for (HashMap.Entry<Integer, Integer> entry : markAsReadMessagesInbox.entrySet()) { TLRPC.TL_dialog dialog = dialogs_dict.get((long) entry.getKey()); if (dialog != null && dialog.top_message <= entry.getValue()) { MessageObject obj = dialogMessage.get(dialog.top_message); if (obj != null) { obj.setIsRead(); updateMask |= UPDATE_MASK_READ_DIALOG_MESSAGE; } } } for (HashMap.Entry<Integer, Integer> entry : markAsReadMessagesOutbox.entrySet()) { TLRPC.TL_dialog dialog = dialogs_dict.get((long) entry.getKey()); if (dialog != null && dialog.top_message <= entry.getValue()) { MessageObject obj = dialogMessage.get(dialog.top_message); if (obj != null) { obj.setIsRead(); updateMask |= UPDATE_MASK_READ_DIALOG_MESSAGE; } } } } if (!markAsReadEncrypted.isEmpty()) { for (HashMap.Entry<Integer, Integer> entry : markAsReadEncrypted.entrySet()) { NotificationCenter.getInstance().postNotificationName( NotificationCenter.messagesReadEncrypted, entry.getKey(), entry.getValue()); long dialog_id = (long) (entry.getKey()) << 32; TLRPC.TL_dialog dialog = dialogs_dict.get(dialog_id); if (dialog != null) { MessageObject message = dialogMessage.get(dialog.top_message); if (message != null && message.messageOwner.date <= entry.getValue()) { message.setIsRead(); updateMask |= UPDATE_MASK_READ_DIALOG_MESSAGE; } } } } if (!markAsReadMessages.isEmpty()) { NotificationCenter.getInstance().postNotificationName( NotificationCenter.messagesReadContent, markAsReadMessages); } if (!deletedMessages.isEmpty()) { NotificationCenter.getInstance() .postNotificationName(NotificationCenter.messagesDeleted, deletedMessages); for (Integer id : deletedMessages) { MessageObject obj = dialogMessage.get(id); if (obj != null) { obj.deleted = true; } } } if (updateMask != 0) { NotificationCenter.getInstance() .postNotificationName(NotificationCenter.updateInterfaces, updateMask); } } }); } }); if (!webPages.isEmpty()) { MessagesStorage.getInstance().putWebPages(webPages); } if (!markAsReadMessagesInbox.isEmpty() || !markAsReadMessagesOutbox.isEmpty() || !markAsReadEncrypted.isEmpty()) { if (!markAsReadMessagesInbox.isEmpty() || !markAsReadMessagesOutbox.isEmpty()) { MessagesStorage.getInstance().updateDialogsWithReadedMessages(markAsReadMessagesInbox, true); } MessagesStorage.getInstance().markMessagesAsRead(markAsReadMessagesInbox, markAsReadMessagesOutbox, markAsReadEncrypted, true); } if (!markAsReadMessages.isEmpty()) { MessagesStorage.getInstance().markMessagesContentAsRead(markAsReadMessages); } if (!deletedMessages.isEmpty()) { MessagesStorage.getInstance().markMessagesAsDeleted(deletedMessages, true); } if (!deletedMessages.isEmpty()) { MessagesStorage.getInstance().updateDialogsWithDeletedMessages(deletedMessages, true); } if (!tasks.isEmpty()) { for (TLRPC.TL_updateEncryptedMessagesRead update : tasks) { MessagesStorage.getInstance().createTaskForSecretChat(update.chat_id, update.max_date, update.date, 1, null); } } return true; }
From source file:org.telegram.messenger.MessagesController.java
public boolean processUpdateArray(ArrayList<TLRPC.Update> updates, final ArrayList<TLRPC.User> usersArr, final ArrayList<TLRPC.Chat> chatsArr) { if (updates.isEmpty()) { return true; }/*from w w w . jav a 2 s . co m*/ long currentTime = System.currentTimeMillis(); final HashMap<Long, ArrayList<MessageObject>> messages = new HashMap<Long, ArrayList<MessageObject>>(); final ArrayList<TLRPC.Message> messagesArr = new ArrayList<TLRPC.Message>(); final ArrayList<Integer> markAsReadMessages = new ArrayList<Integer>(); final HashMap<Integer, Integer> markAsReadEncrypted = new HashMap<Integer, Integer>(); final ArrayList<Integer> deletedMessages = new ArrayList<Integer>(); final ArrayList<Long> printChanges = new ArrayList<Long>(); final ArrayList<TLRPC.ChatParticipants> chatInfoToUpdate = new ArrayList<TLRPC.ChatParticipants>(); final ArrayList<TLRPC.Update> updatesOnMainThread = new ArrayList<TLRPC.Update>(); final ArrayList<TLRPC.TL_updateEncryptedMessagesRead> tasks = new ArrayList<TLRPC.TL_updateEncryptedMessagesRead>(); final ArrayList<Integer> contactsIds = new ArrayList<Integer>(); MessageObject lastMessage = null; boolean checkForUsers = true; ConcurrentHashMap<Integer, TLRPC.User> usersDict; ConcurrentHashMap<Integer, TLRPC.Chat> chatsDict; if (usersArr != null) { usersDict = new ConcurrentHashMap<Integer, TLRPC.User>(); for (TLRPC.User user : usersArr) { usersDict.put(user.id, user); } } else { checkForUsers = false; usersDict = users; } if (chatsArr != null) { chatsDict = new ConcurrentHashMap<Integer, TLRPC.Chat>(); for (TLRPC.Chat chat : chatsArr) { chatsDict.put(chat.id, chat); } } else { checkForUsers = false; chatsDict = chats; } if (usersArr != null || chatsArr != null) { Utilities.RunOnUIThread(new Runnable() { @Override public void run() { if (usersArr != null) { for (TLRPC.User user : usersArr) { users.put(user.id, user); if (user.id == UserConfig.clientUserId) { UserConfig.currentUser = user; } } } if (chatsArr != null) { for (TLRPC.Chat chat : chatsArr) { chats.put(chat.id, chat); } } } }); } int interfaceUpdateMask = 0; for (TLRPC.Update update : updates) { if (update instanceof TLRPC.TL_updateNewMessage) { TLRPC.TL_updateNewMessage upd = (TLRPC.TL_updateNewMessage) update; if (checkForUsers) { if (usersDict.get(upd.message.from_id) == null && users.get(upd.message.from_id) == null || upd.message.to_id.chat_id != 0 && chatsDict.get(upd.message.to_id.chat_id) == null && chats.get(upd.message.to_id.chat_id) == null) { return false; } } messagesArr.add(upd.message); MessageObject obj = new MessageObject(upd.message, usersDict); if (obj.type == 11) { interfaceUpdateMask |= UPDATE_MASK_CHAT_AVATAR; } else if (obj.type == 10) { interfaceUpdateMask |= UPDATE_MASK_CHAT_NAME; } long uid; if (upd.message.to_id.chat_id != 0) { uid = -upd.message.to_id.chat_id; } else { if (upd.message.to_id.user_id == UserConfig.clientUserId) { upd.message.to_id.user_id = upd.message.from_id; } uid = upd.message.to_id.user_id; } ArrayList<MessageObject> arr = messages.get(uid); if (arr == null) { arr = new ArrayList<MessageObject>(); messages.put(uid, arr); } arr.add(obj); MessagesStorage.lastPtsValue = update.pts; if (upd.message.from_id != UserConfig.clientUserId && upd.message.to_id != null) { if (uid != openned_dialog_id || ApplicationLoader.lastPauseTime != 0) { lastMessage = obj; } } } else if (update instanceof TLRPC.TL_updateMessageID) { //can't be here } else if (update instanceof TLRPC.TL_updateReadMessages) { markAsReadMessages.addAll(update.messages); MessagesStorage.lastPtsValue = update.pts; } else if (update instanceof TLRPC.TL_updateDeleteMessages) { deletedMessages.addAll(update.messages); MessagesStorage.lastPtsValue = update.pts; } else if (update instanceof TLRPC.TL_updateRestoreMessages) { MessagesStorage.lastPtsValue = update.pts; } else if (update instanceof TLRPC.TL_updateUserTyping || update instanceof TLRPC.TL_updateChatUserTyping) { if (update.user_id != UserConfig.clientUserId) { long uid = -update.chat_id; if (uid == 0) { uid = update.user_id; } ArrayList<PrintingUser> arr = printingUsers.get(uid); if (arr == null) { arr = new ArrayList<PrintingUser>(); printingUsers.put(uid, arr); } boolean exist = false; for (PrintingUser u : arr) { if (u.userId == update.user_id) { exist = true; u.lastTime = currentTime; break; } } if (!exist) { PrintingUser newUser = new PrintingUser(); newUser.userId = update.user_id; newUser.lastTime = currentTime; arr.add(newUser); if (!printChanges.contains(uid)) { printChanges.add(uid); } } } } else if (update instanceof TLRPC.TL_updateChatParticipants) { interfaceUpdateMask |= UPDATE_MASK_CHAT_MEMBERS; chatInfoToUpdate.add(update.participants); } else if (update instanceof TLRPC.TL_updateUserStatus) { interfaceUpdateMask |= UPDATE_MASK_STATUS; updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateUserName) { interfaceUpdateMask |= UPDATE_MASK_NAME; updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateUserPhoto) { interfaceUpdateMask |= UPDATE_MASK_AVATAR; MessagesStorage.Instance.clearUserPhotos(update.user_id); /*if (!(update.photo instanceof TLRPC.TL_userProfilePhotoEmpty)) { DEPRECATED if (usersDict.containsKey(update.user_id)) { TLRPC.TL_messageService newMessage = new TLRPC.TL_messageService(); newMessage.action = new TLRPC.TL_messageActionUserUpdatedPhoto(); newMessage.action.newUserPhoto = update.photo; newMessage.local_id = newMessage.id = UserConfig.getNewMessageId(); UserConfig.saveConfig(false); newMessage.unread = true; newMessage.date = update.date; newMessage.from_id = update.user_id; newMessage.to_id = new TLRPC.TL_peerUser(); newMessage.to_id.user_id = UserConfig.clientUserId; newMessage.out = false; newMessage.dialog_id = update.user_id; messagesArr.add(newMessage); MessageObject obj = new MessageObject(newMessage, usersDict); ArrayList<MessageObject> arr = messages.get(newMessage.dialog_id); if (arr == null) { arr = new ArrayList<MessageObject>(); messages.put(newMessage.dialog_id, arr); } arr.add(obj); if (newMessage.from_id != UserConfig.clientUserId && newMessage.to_id != null) { if (newMessage.dialog_id != openned_dialog_id || ApplicationLoader.lastPauseTime != 0) { lastMessage = obj; } } } }*/ updatesOnMainThread.add(update); } else if (update instanceof TLRPC.TL_updateContactRegistered) { if (enableJoined && usersDict.containsKey(update.user_id)) { TLRPC.TL_messageService newMessage = new TLRPC.TL_messageService(); newMessage.action = new TLRPC.TL_messageActionUserJoined(); newMessage.local_id = newMessage.id = UserConfig.getNewMessageId(); UserConfig.saveConfig(false); newMessage.unread = true; newMessage.date = update.date; newMessage.from_id = update.user_id; newMessage.to_id = new TLRPC.TL_peerUser(); newMessage.to_id.user_id = UserConfig.clientUserId; newMessage.out = false; newMessage.dialog_id = update.user_id; messagesArr.add(newMessage); MessageObject obj = new MessageObject(newMessage, usersDict); ArrayList<MessageObject> arr = messages.get(newMessage.dialog_id); if (arr == null) { arr = new ArrayList<MessageObject>(); messages.put(newMessage.dialog_id, arr); } arr.add(obj); if (newMessage.from_id != UserConfig.clientUserId && newMessage.to_id != null) { if (newMessage.dialog_id != openned_dialog_id || ApplicationLoader.lastPauseTime != 0) { lastMessage = obj; } } } // if (!contactsIds.contains(update.user_id)) { // contactsIds.add(update.user_id); // } } else if (update instanceof TLRPC.TL_updateContactLink) { if (update.my_link instanceof TLRPC.TL_contacts_myLinkContact || update.my_link instanceof TLRPC.TL_contacts_myLinkRequested && update.my_link.contact) { int idx = contactsIds.indexOf(-update.user_id); if (idx != -1) { contactsIds.remove(idx); } if (!contactsIds.contains(update.user_id)) { contactsIds.add(update.user_id); } } else { int idx = contactsIds.indexOf(update.user_id); if (idx != -1) { contactsIds.remove(idx); } if (!contactsIds.contains(update.user_id)) { contactsIds.add(-update.user_id); } } } else if (update instanceof TLRPC.TL_updateActivation) { //DEPRECATED } else if (update instanceof TLRPC.TL_updateNewAuthorization) { TLRPC.TL_messageService newMessage = new TLRPC.TL_messageService(); newMessage.action = new TLRPC.TL_messageActionLoginUnknownLocation(); newMessage.action.title = update.device; newMessage.action.address = update.location; newMessage.local_id = newMessage.id = UserConfig.getNewMessageId(); UserConfig.saveConfig(false); newMessage.unread = true; newMessage.date = update.date; newMessage.from_id = 333000; newMessage.to_id = new TLRPC.TL_peerUser(); newMessage.to_id.user_id = UserConfig.clientUserId; newMessage.out = false; newMessage.dialog_id = 333000; messagesArr.add(newMessage); MessageObject obj = new MessageObject(newMessage, usersDict); ArrayList<MessageObject> arr = messages.get(newMessage.dialog_id); if (arr == null) { arr = new ArrayList<MessageObject>(); messages.put(newMessage.dialog_id, arr); } arr.add(obj); if (newMessage.from_id != UserConfig.clientUserId && newMessage.to_id != null) { if (newMessage.dialog_id != openned_dialog_id || ApplicationLoader.lastPauseTime != 0) { lastMessage = obj; } } } else if (update instanceof TLRPC.TL_updateNewGeoChatMessage) { //DEPRECATED } else if (update instanceof TLRPC.TL_updateNewEncryptedMessage) { MessagesStorage.lastQtsValue = update.qts; TLRPC.Message message = decryptMessage(((TLRPC.TL_updateNewEncryptedMessage) update).message); if (message != null) { int cid = ((TLRPC.TL_updateNewEncryptedMessage) update).message.chat_id; messagesArr.add(message); MessageObject obj = new MessageObject(message, usersDict); long uid = ((long) cid) << 32; ArrayList<MessageObject> arr = messages.get(uid); if (arr == null) { arr = new ArrayList<MessageObject>(); messages.put(uid, arr); } arr.add(obj); if (message.from_id != UserConfig.clientUserId && message.to_id != null) { if (uid != openned_dialog_id || ApplicationLoader.lastPauseTime != 0) { lastMessage = obj; } } } } else if (update instanceof TLRPC.TL_updateEncryptedChatTyping) { long uid = ((long) update.chat_id) << 32; ArrayList<PrintingUser> arr = printingUsers.get(uid); if (arr == null) { arr = new ArrayList<PrintingUser>(); printingUsers.put(uid, arr); } boolean exist = false; for (PrintingUser u : arr) { if (u.userId == update.user_id) { exist = true; u.lastTime = currentTime; break; } } if (!exist) { PrintingUser newUser = new PrintingUser(); newUser.userId = update.user_id; newUser.lastTime = currentTime; arr.add(newUser); if (!printChanges.contains(uid)) { printChanges.add(uid); } } } else if (update instanceof TLRPC.TL_updateEncryptedMessagesRead) { markAsReadEncrypted.put(update.chat_id, Math.max(update.max_date, update.date)); tasks.add((TLRPC.TL_updateEncryptedMessagesRead) update); } else if (update instanceof TLRPC.TL_updateChatParticipantAdd) { MessagesStorage.Instance.updateChatInfo(update.chat_id, update.user_id, false, update.inviter_id, update.version); } else if (update instanceof TLRPC.TL_updateChatParticipantDelete) { MessagesStorage.Instance.updateChatInfo(update.chat_id, update.user_id, true, 0, update.version); } else if (update instanceof TLRPC.TL_updateDcOptions) { ConnectionsManager.Instance.updateDcSettings(); } else if (update instanceof TLRPC.TL_updateEncryption) { final TLRPC.EncryptedChat newChat = update.chat; long dialog_id = ((long) newChat.id) << 32; TLRPC.EncryptedChat existingChat = encryptedChats.get(newChat.id); if (existingChat == null) { Semaphore semaphore = new Semaphore(0); ArrayList<TLObject> result = new ArrayList<TLObject>(); MessagesStorage.Instance.getEncryptedChat(newChat.id, semaphore, result); try { semaphore.acquire(); } catch (Exception e) { FileLog.e("tmessages", e); } if (result.size() == 2) { existingChat = (TLRPC.EncryptedChat) result.get(0); TLRPC.User user = (TLRPC.User) result.get(1); users.putIfAbsent(user.id, user); } } if (newChat instanceof TLRPC.TL_encryptedChatRequested && existingChat == null) { int user_id = newChat.participant_id; if (user_id == UserConfig.clientUserId) { user_id = newChat.admin_id; } TLRPC.User user = users.get(user_id); if (user == null) { user = usersDict.get(user_id); } newChat.user_id = user_id; final TLRPC.TL_dialog dialog = new TLRPC.TL_dialog(); dialog.id = dialog_id; dialog.unread_count = 0; dialog.top_message = 0; dialog.last_message_date = update.date; Utilities.RunOnUIThread(new Runnable() { @Override public void run() { dialogs_dict.put(dialog.id, dialog); dialogs.add(dialog); dialogsServerOnly.clear(); encryptedChats.put(newChat.id, newChat); Collections.sort(dialogs, new Comparator<TLRPC.TL_dialog>() { @Override public int compare(TLRPC.TL_dialog tl_dialog, TLRPC.TL_dialog tl_dialog2) { if (tl_dialog.last_message_date == tl_dialog2.last_message_date) { return 0; } else if (tl_dialog.last_message_date < tl_dialog2.last_message_date) { return 1; } else { return -1; } } }); for (TLRPC.TL_dialog d : dialogs) { if ((int) d.id != 0) { dialogsServerOnly.add(d); } } NotificationCenter.Instance.postNotificationName(dialogsNeedReload); } }); MessagesStorage.Instance.putEncryptedChat(newChat, user, dialog); acceptSecretChat(newChat); } else if (newChat instanceof TLRPC.TL_encryptedChat) { if (existingChat != null && existingChat instanceof TLRPC.TL_encryptedChatWaiting && (existingChat.auth_key == null || existingChat.auth_key.length == 1)) { newChat.a_or_b = existingChat.a_or_b; newChat.user_id = existingChat.user_id; processAcceptedSecretChat(newChat); } } else { final TLRPC.EncryptedChat exist = existingChat; Utilities.RunOnUIThread(new Runnable() { @Override public void run() { if (exist != null) { newChat.user_id = exist.user_id; newChat.auth_key = exist.auth_key; encryptedChats.put(newChat.id, newChat); } MessagesStorage.Instance.updateEncryptedChat(newChat); NotificationCenter.Instance.postNotificationName(encryptedChatUpdated, newChat); } }); } } } if (!messages.isEmpty()) { for (HashMap.Entry<Long, ArrayList<MessageObject>> pair : messages.entrySet()) { Long key = pair.getKey(); ArrayList<MessageObject> value = pair.getValue(); boolean printChanged = updatePrintingUsersWithNewMessages(key, value); if (printChanged && !printChanges.contains(key)) { printChanges.add(key); } } } if (!printChanges.isEmpty()) { updatePrintingStrings(); } final MessageObject lastMessageArg = lastMessage; final int interfaceUpdateMaskFinal = interfaceUpdateMask; if (!contactsIds.isEmpty()) { ContactsController.Instance.processContactsUpdates(contactsIds, usersDict); } if (!messagesArr.isEmpty()) { MessagesStorage.Instance.putMessages(messagesArr, true, true); } if (!messages.isEmpty() || !markAsReadMessages.isEmpty() || !deletedMessages.isEmpty() || !printChanges.isEmpty() || !chatInfoToUpdate.isEmpty() || !updatesOnMainThread.isEmpty() || !markAsReadEncrypted.isEmpty() || !contactsIds.isEmpty()) { Utilities.RunOnUIThread(new Runnable() { @Override public void run() { int updateMask = interfaceUpdateMaskFinal; boolean avatarsUpdate = false; if (!updatesOnMainThread.isEmpty()) { ArrayList<TLRPC.User> dbUsers = new ArrayList<TLRPC.User>(); ArrayList<TLRPC.User> dbUsersStatus = new ArrayList<TLRPC.User>(); for (TLRPC.Update update : updatesOnMainThread) { TLRPC.User toDbUser = new TLRPC.User(); toDbUser.id = update.user_id; TLRPC.User currentUser = users.get(update.user_id); if (update instanceof TLRPC.TL_updateUserStatus) { if (!(update.status instanceof TLRPC.TL_userStatusEmpty)) { if (currentUser != null) { currentUser.id = update.user_id; currentUser.status = update.status; if (update.status instanceof TLRPC.TL_userStatusOnline) { currentUser.status.was_online = update.status.expires; } else if (update.status instanceof TLRPC.TL_userStatusOffline) { currentUser.status.expires = update.status.was_online; } else { currentUser.status.was_online = 0; currentUser.status.expires = 0; } } toDbUser.status = update.status; dbUsersStatus.add(toDbUser); } } else if (update instanceof TLRPC.TL_updateUserName) { if (currentUser != null) { currentUser.first_name = update.first_name; currentUser.last_name = update.last_name; } toDbUser.first_name = update.first_name; toDbUser.last_name = update.last_name; dbUsers.add(toDbUser); } else if (update instanceof TLRPC.TL_updateUserPhoto) { if (currentUser != null) { currentUser.photo = update.photo; } avatarsUpdate = true; toDbUser.photo = update.photo; dbUsers.add(toDbUser); } } MessagesStorage.Instance.updateUsers(dbUsersStatus, true, true, true); MessagesStorage.Instance.updateUsers(dbUsers, false, true, true); } if (!messages.isEmpty()) { for (HashMap.Entry<Long, ArrayList<MessageObject>> entry : messages.entrySet()) { Long key = entry.getKey(); ArrayList<MessageObject> value = entry.getValue(); updateInterfaceWithMessages(key, value); } NotificationCenter.Instance.postNotificationName(dialogsNeedReload); } if (!markAsReadMessages.isEmpty()) { for (Integer id : markAsReadMessages) { MessageObject obj = dialogMessage.get(id); if (obj != null) { obj.messageOwner.unread = false; } } NotificationCenter.Instance.postNotificationName(messagesReaded, markAsReadMessages); } if (!markAsReadEncrypted.isEmpty()) { for (HashMap.Entry<Integer, Integer> entry : markAsReadEncrypted.entrySet()) { NotificationCenter.Instance.postNotificationName(messagesReadedEncrypted, entry.getKey(), entry.getValue()); long dialog_id = (long) (entry.getKey()) << 32; TLRPC.TL_dialog dialog = dialogs_dict.get(dialog_id); if (dialog != null) { MessageObject message = dialogMessage.get(dialog.top_message); if (message != null && message.messageOwner.date <= entry.getValue()) { message.messageOwner.unread = false; } } } } if (!deletedMessages.isEmpty()) { NotificationCenter.Instance.postNotificationName(messagesDeleted, deletedMessages); for (Integer id : deletedMessages) { MessageObject obj = dialogMessage.get(id); if (obj != null) { obj.deleted = true; } } } if (!printChanges.isEmpty()) { updateMask |= UPDATE_MASK_USER_PRINT; } if (!contactsIds.isEmpty()) { updateMask |= UPDATE_MASK_NAME; updateMask |= UPDATE_MASK_USER_PHONE; } if (!chatInfoToUpdate.isEmpty()) { for (TLRPC.ChatParticipants info : chatInfoToUpdate) { MessagesStorage.Instance.updateChatInfo(info.chat_id, info, true); NotificationCenter.Instance.postNotificationName(chatInfoDidLoaded, info.chat_id, info); } } if (updateMask != 0) { NotificationCenter.Instance.postNotificationName(updateInterfaces, updateMask); } if (lastMessageArg != null) { showInAppNotification(lastMessageArg); } } }); } if (!markAsReadMessages.isEmpty() || !markAsReadEncrypted.isEmpty()) { MessagesStorage.Instance.markMessagesAsRead(markAsReadMessages, markAsReadEncrypted, true); } if (!deletedMessages.isEmpty()) { MessagesStorage.Instance.markMessagesAsDeleted(deletedMessages, true); } if (!deletedMessages.isEmpty()) { MessagesStorage.Instance.updateDialogsWithDeletedMessages(deletedMessages, true); } if (!markAsReadMessages.isEmpty()) { MessagesStorage.Instance.updateDialogsWithReadedMessages(markAsReadMessages, true); } if (!tasks.isEmpty()) { for (TLRPC.TL_updateEncryptedMessagesRead update : tasks) { MessagesStorage.Instance.createTaskForDate(update.chat_id, update.max_date, update.date, 1); } } return true; }
From source file:com.thesmartweb.swebrank.Search_analysis.java
/** * Method to perform the queries to the search engines, get the links and get all the webpage and semantic stats for the links * @param iteration_counter The iteration number of the algorithm (to use it in the id for elasticsearch) * @param directory_save The directory we are going to several files * @param domain The domain that we are searching for (to use it in the id for elasticsearch) * @param enginechoice The search engines that were chosen to be used * @param quer the query we search for/*from w ww . j a va2 s . c o m*/ * @param results_number the results number that we are going to get from every search engine * @param top_visible the number of results if we use Visibility score * @param SWebRankSettings the settings for LDA and SwebRank in general (check the ReadInput Class) * @param alpha alpha value of LDA * @param mozMetrics the metrics of choice if Moz is going to be used * @param top_count_moz the amount of results if we use Moz * @param moz_threshold_option flag to show if we are going to use a threshold in Moz metrics or not * @param moz_threshold the moz threshold value * @param ContentSemantics get the choice of Content Semantic Analysis algorithm that we are going to use * @param SensebotConcepts the amount of concepts to be recognized if Sensebot is used * @param config_path the configuration path to get all the api keys * @return a list with the words recognized as important by the content semantic analysis algorithm we have chosen */ public List<String> perform(int iteration_counter, String directory_save, String domain, List<Boolean> enginechoice, String quer, int results_number, int top_visible, List<Double> SWebRankSettings, double alpha, List<Boolean> mozMetrics, int top_count_moz, boolean moz_threshold_option, double moz_threshold, List<Boolean> ContentSemantics, int SensebotConcepts, String config_path) { //=======connect to mysql========= Connection conn = null; PreparedStatement stmt = null; try { ReadInput ri = new ReadInput(); List<String> mysqlAdminSettings = ri.GetKeyFile(config_path, "mysqlAdmin"); String port = mysqlAdminSettings.get(2); String dbname = mysqlAdminSettings.get(3); String url = "jdbc:mysql://localhost:" + port + "/" + dbname + "?zeroDateTimeBehavior=convertToNull"; String user = mysqlAdminSettings.get(0); String password = mysqlAdminSettings.get(1); System.out.println("Connecting to database..."); conn = DriverManager.getConnection(url, user, password); LinksParseAnalysis ld = new LinksParseAnalysis(); //we create the array that are going to store the results from each search engine String[] links_google = new String[results_number]; String[] links_yahoo = new String[results_number]; String[] links_bing = new String[results_number]; //we create the array that is going to store all the results from all the search engines together String[] links_total = new String[(results_number * 3)]; //--------if we have selected to use a Moz metric, then we should set the links_total to be of size of top_count_seomoz*3 since it means that the results_number has been set to its max value (50) if (mozMetrics.get(0)) { links_total = new String[(top_count_moz) * 3]; } int[] nlinks = new int[2]; if (enginechoice.get(0)) { //get bing results BingResults br = new BingResults(); links_bing = br.Get(quer, results_number, directory_save, config_path); } if (enginechoice.get(1)) { //get google results GoogleResults gr = new GoogleResults(); links_google = gr.Get(quer, results_number, directory_save, config_path); } if (enginechoice.get(2)) { //get yahoo results YahooResults yr = new YahooResults(); links_yahoo = yr.Get(quer, results_number, directory_save, config_path); } HashMap<Integer, List<String>> EntitiesMapDBP = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapDBP = new HashMap<>(); HashMap<Integer, List<String>> EntitiesMapDand = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapDand = new HashMap<>(); HashMap<Integer, List<String>> EntitiesMapYahoo = new HashMap<>(); HashMap<Integer, List<String>> CategoriesMapYahoo = new HashMap<>(); HashMap<Integer, String> parseOutputList = new HashMap<>(); for (int i = 0; i < results_number * 3; i++) { parseOutputList.put(i, ""); } //************* boolean false_flag = true; if (false_flag) { if (mozMetrics.get(0)) { //we check if moz works Moz moz = new Moz(); boolean checkmoz = moz.check(config_path); if (checkmoz) { //perform if (links_yahoo.length > 0) { links_yahoo = moz.perform(links_yahoo, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } if (links_google.length > 0) { links_google = moz.perform(links_google, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } if (links_bing.length > 0) { links_bing = moz.perform(links_bing, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } } } //we are creating Sindice class in order to get the number of semantic triples of a webpage Sindice striple = new Sindice(); //create htmlparser to get the number of links in a webpage if (mozMetrics.get(0)) { results_number = links_yahoo.length; } WebParser htm = new WebParser(); //create an array that contains all the links together for (int i = 0; i < 3; i++) { try { if (i == 0) { System.arraycopy(links_yahoo, 0, links_total, 0, results_number); } if (i == 1) { System.arraycopy(links_google, 0, links_total, links_yahoo.length, results_number); } if (i == 2) { System.arraycopy(links_bing, 0, links_total, ((links_yahoo.length) + (links_google.length)), results_number); } } catch (ArrayIndexOutOfBoundsException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<String>(); return finalList; } } //merged true => visibility score if (enginechoice.get(3)) { VisibilityScore vb = new VisibilityScore();//we have a merged engine //erase using vb.perform all the duplicate links links_total = vb.perform(links_google, links_yahoo, links_bing); //if we have Moz option set to true we have to get the results rearranged according to the moz metric selected if (mozMetrics.get(0)) { Moz checkMoz = new Moz(); boolean check_seo = checkMoz.check(config_path); if (check_seo) { Moz MOZ = new Moz(); links_total = MOZ.perform(links_total, top_count_moz, moz_threshold, moz_threshold_option, mozMetrics, config_path); } } //here we calculate the visibility score links_total = vb.visibility_score(links_total, links_yahoo, links_bing, links_google, top_visible); } String[][] total_catent = new String[links_total.length][2]; for (int r = 0; r < total_catent.length; r++) { total_catent[r][0] = ""; total_catent[r][1] = ""; } for (int j = 0; j < links_total.length; j++) { if (links_total[j] != null) { String urlString = links_total[j]; if (urlString.length() > 199) { urlString = links_total[j].substring(0, 198); } int rank = -1; int engine = -1;//0 for yahoo,1 for google,2 for bing if (j < results_number) { rank = j; engine = 0; } else if (j < results_number * 2) { rank = j - results_number; engine = 1; } else if (j < results_number * 3) { rank = j - results_number * 2; engine = 2; } try { //we initialize the row in settings table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO SETTINGS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we initialize the row in semantic stats table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO SEMANTICSTATS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we initialize the row in namespaces stats table conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "INSERT INTO NAMESPACESSTATS (url,query,search_engine,search_engine_rank,domain) VALUES (?,?,?,?,?) ON DUPLICATE KEY UPDATE url=VALUES(url),query=VALUES(query),search_engine=VALUES(search_engine),domain=VALUES(domain)"); stmt.setString(1, urlString); stmt.setString(2, quer); stmt.setInt(3, engine); stmt.setInt(4, rank); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { //we put the info inside the settings conn = DriverManager.getConnection(url, user, password); StringBuilder settingsStmBuild = new StringBuilder(); settingsStmBuild.append("UPDATE SETTINGS SET "); settingsStmBuild.append("`nTopics`=? , "); settingsStmBuild.append("`alpha`=? , "); settingsStmBuild.append("`beta`=? , "); settingsStmBuild.append("`niters`=? , "); settingsStmBuild.append("`prob_threshold`=? , "); settingsStmBuild.append("`moz`=? , "); settingsStmBuild.append("`top_count_moz`=? , "); settingsStmBuild.append("`moz_threshold`=? , "); settingsStmBuild.append("`moz_threshold_option`=? , "); settingsStmBuild.append("`top_visible`=? , "); settingsStmBuild.append("`Domain_Authority`=? , "); settingsStmBuild.append("`External_MozRank`=? , "); settingsStmBuild.append("`MozRank`=? , "); settingsStmBuild.append("`MozTrust`=? , "); settingsStmBuild.append("`Page_Authority`=? , "); settingsStmBuild.append("`Subdomain_mozRank`=? , "); settingsStmBuild.append("`merged`=? , "); settingsStmBuild.append("`results_number`=? , "); settingsStmBuild.append("`Diffbotflag`=? , "); settingsStmBuild.append("`LDAflag`=? , "); settingsStmBuild.append("`Sensebotflag`=? , "); settingsStmBuild.append("`TFIDFflag`=? , "); settingsStmBuild.append("`SensebotConcepts`=? , "); settingsStmBuild.append("`nTopTopics`=? , "); settingsStmBuild.append("`combinelimit`=? ,"); settingsStmBuild.append("`newtermstocombine`=? ,"); settingsStmBuild.append("`newqueriesmax`=? ,"); settingsStmBuild.append("`ngdthreshold`=? ,"); settingsStmBuild.append("`entitiesconfi`=? ,"); settingsStmBuild.append("`dbpediasup`=? "); settingsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(settingsStmBuild.toString()); stmt.setInt(1, SWebRankSettings.get(1).intValue()); stmt.setDouble(2, alpha); stmt.setDouble(3, SWebRankSettings.get(0)); stmt.setInt(4, SWebRankSettings.get(2).intValue()); stmt.setDouble(5, SWebRankSettings.get(3)); stmt.setBoolean(6, mozMetrics.get(0)); stmt.setInt(7, top_count_moz); stmt.setDouble(8, moz_threshold); stmt.setBoolean(9, moz_threshold_option); stmt.setInt(10, top_visible); stmt.setBoolean(11, mozMetrics.get(1)); stmt.setBoolean(12, mozMetrics.get(2)); stmt.setBoolean(13, mozMetrics.get(3)); stmt.setBoolean(14, mozMetrics.get(4)); stmt.setBoolean(15, mozMetrics.get(5)); stmt.setBoolean(16, mozMetrics.get(6)); stmt.setBoolean(17, enginechoice.get(3)); stmt.setInt(18, results_number); stmt.setBoolean(19, ContentSemantics.get(0)); stmt.setBoolean(20, ContentSemantics.get(1)); stmt.setBoolean(21, ContentSemantics.get(2)); stmt.setBoolean(22, ContentSemantics.get(3)); stmt.setInt(23, SensebotConcepts); stmt.setInt(24, SWebRankSettings.get(11).intValue()); stmt.setInt(25, SWebRankSettings.get(7).intValue()); stmt.setInt(26, SWebRankSettings.get(9).intValue()); stmt.setInt(27, SWebRankSettings.get(10).intValue()); stmt.setDouble(28, SWebRankSettings.get(6)); stmt.setDouble(29, SWebRankSettings.get(12)); stmt.setDouble(30, SWebRankSettings.get(13)); stmt.setString(31, urlString); stmt.setString(32, quer); stmt.setInt(33, engine); stmt.setString(34, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } if (htm.checkconn(links_total[j])) {//if we can connect to the url we continue to update semantics stats and namespaces stats tables nlinks = htm.getnlinks(links_total[j]); StringBuilder webstatsStmBuild = new StringBuilder(); try { conn = DriverManager.getConnection(url, user, password); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`number_links`=? , "); webstatsStmBuild.append("`redirect_links`=? , "); webstatsStmBuild.append("`internal_links`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, nlinks[0]);//total numbers of links stmt.setInt(2, nlinks[0] - nlinks[1]); stmt.setInt(3, nlinks[1]);//internal links stmt.setString(4, urlString); stmt.setString(5, quer); stmt.setInt(6, engine); stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); System.out.println("I am going to get the stats from Sindice\n"); int ntriples = striple.getsindicestats(links_total[j]);//get the amount of semantic triples using Sindice API System.out.println("I am going insert the semantic triples number in the DB\n"); stmt = conn.prepareStatement( "UPDATE SEMANTICSTATS SET `total_semantic_triples`=? WHERE `url` =? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setInt(1, ntriples); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); System.out.println("I inserted the semantic triples number in the DB\n"); //---namespaces----- System.out.println("I am going to insert the namespaces in the DB\n"); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } boolean flagStriple = false; if (flagStriple) { if (striple.namespaces[0]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/vocab/bio/0.1/` = ? WHERE `url` = ? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[1]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/dc/elements/1.1/` =? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[2]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/coo/n` = ? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[3]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://web.resource.org/cc/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[4]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://diligentarguont.ontoware.org/2005/10/arguonto`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[5]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://usefulinc.com/ns/doap`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[6]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://xmlns.com/foaf/0.1/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[7]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/goodrelations/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[8]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/muto/core`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[9]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://webns.net/mvcb/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[10]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/ontology/mo/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[11]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/innovation/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[12]) { try { stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://openguid.net/rdf`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[13]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.slamka.cz/ontologies/diagnostika.owl`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[14]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/ontology/po/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[15]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/net/provenance/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[16]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/rss/1.0/modules/syndication`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[17]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[18]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://madskills.com/public/xml/rss/module/trackback/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[19]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/ns/void`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[20]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.fzi.de/2008/wise/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[21]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://xmlns.com/wot/0.1`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[22]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/1999/02/22-rdf-syntax-ns`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[23]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `rdf-schema`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[24]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `XMLschema`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[25]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `OWL`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[26]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/dc/terms/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[27]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `VCARD`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[28]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.geonames.org/ontology`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[29]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://search.yahoo.com/searchmonkey/commerce/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[30]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://search.yahoo.com/searchmonkey/media/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[31]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://cb.semsol.org/ns#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[32]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://blogs.yandex.ru/schema/foaf/`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[33]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/2003/01/geo/wgs84_pos#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[34]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/ns#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[35]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://rdfs.org/sioc/types#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[36]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://smw.ontoware.org/2005/smw#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[37]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://purl.org/rss/1.0/`= ? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (striple.namespaces[38]) { try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement( "UPDATE NAMESPACESSTATS SET `http://www.w3.org/2004/12/q/contentlabel#`=? WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt.setBoolean(1, true); stmt.setString(2, urlString); stmt.setString(3, quer); stmt.setInt(4, engine); stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } System.out.println("I inserted the namespaces in the DB\n"); System.out.println("I will get the semantic entities and categories\n"); //get the semantic entities and categories from Yahoo Content Analysis Service YahooEntityCategory yec = new YahooEntityCategory(); yec.connect(links_total[j], quer, false, SWebRankSettings.get(12));//without stemming EntitiesMapYahoo.put(j, yec.GetEntitiesYahoo()); CategoriesMapYahoo.put(j, yec.GetCategoriesYahoo()); double ent_avg_yahoo_score = yec.GetEntitiesYahooScore(); double cat_avg_yahoo_score = yec.GetCategoriesYahooScore(); int cat_cnt = yec.GetCatQuerCnt(); int ent_cnt = yec.GetEntQuerCnt(); int cat_cnt_whole = yec.GetCatQuerCntWhole(); int ent_cnt_whole = yec.GetEntQuerCntWhole(); yec.connect(links_total[j], quer, true, SWebRankSettings.get(12));//with stemming int cat_cnt_stem = yec.GetCatQuerCnt(); int ent_cnt_stem = yec.GetEntQuerCnt(); int cat_cnt_whole_stem = yec.GetCatQuerCntWhole(); int ent_cnt_whole_stem = yec.GetEntQuerCntWhole(); //get the semantic entities and categories from Dandelion Named entity extraction API DandelionEntities dec = new DandelionEntities(); dec.connect(links_total[j], quer, false, config_path, SWebRankSettings.get(12));//without stemming EntitiesMapDand.put(j, dec.GetEntitiesDand()); CategoriesMapDand.put(j, dec.GetCategoriesDand()); double ent_avg_d_score = dec.GetEntitiesScoreDand(); int cat_cnt_dand = dec.getCat(); int ent_cnt_dand = dec.getEnt(); int cat_cnt_dand_whole = dec.getCatWhole(); int ent_cnt_dand_whole = dec.getEntWhole(); dec.connect(links_total[j], quer, true, config_path, SWebRankSettings.get(12));//with stemming int cat_cnt_dand_stem = dec.getCat(); int ent_cnt_dand_stem = dec.getEnt(); int cat_cnt_dand_whole_stem = dec.getCatWhole(); int ent_cnt_dand_whole_stem = dec.getEntWhole(); //get the semantic entities and categories from dbpedia spotlight DBpediaSpotlightClient dbpspot = new DBpediaSpotlightClient(SWebRankSettings.get(12), SWebRankSettings.get(13).intValue()); dbpspot.countEntCat(links_total[j], quer, false);//false is not stemming EntitiesMapDBP.put(j, dbpspot.getEntities()); CategoriesMapDBP.put(j, dbpspot.getCategories()); double ent_avg_dbpspot_score = dbpspot.getEntitiesAvgScore(); double ent_max_dbpspot_score = dbpspot.getEntitiesMaxScore(); double ent_min_dbpspot_score = dbpspot.getEntitiesMinScore(); double ent_median_dbpspot_score = dbpspot.getEntitiesMedianScore(); double ent_std_dbpspot_score = dbpspot.getEntitiesStdScore(); double ent_avg_dbpspot_support = dbpspot.getEntitiesAvgSupport(); double ent_max_dbpspot_support = dbpspot.getEntitiesMaxSupport(); double ent_min_dbpspot_support = dbpspot.getEntitiesMinSupport(); double ent_median_dbpspot_support = dbpspot.getEntitiesMedianSupport(); double ent_std_dbpspot_support = dbpspot.getEntitiesStdSupport(); double ent_avg_dbpspot_dif = dbpspot.getEntitiesAvgDif(); double ent_max_dbpspot_dif = dbpspot.getEntitiesMaxDif(); double ent_min_dbpspot_dif = dbpspot.getEntitiesMinDif(); double ent_median_dbpspot_dif = dbpspot.getEntitiesMedianDif(); double ent_std_dbpspot_dif = dbpspot.getEntitiesStdDif(); double unique_ent_cnt_dbpspot = dbpspot.getUniqueEntCnt(); double unique_ent_scoreSum_dbpspot = dbpspot.getUniqueEntScoreSum(); int cat_cnt_dbpspot = dbpspot.getcountCat(); int ent_cnt_dbpspot = dbpspot.getcountEnt(); int cat_cnt_dbpspot_whole = dbpspot.getcountCatWhole(); int ent_cnt_dbpspot_whole = dbpspot.getcountEntWhole(); double ent_sup_cnt_dbpspot = dbpspot.getcountSupEnt(); double ent_sim_cnt_dbpspot = dbpspot.getcountSimEnt(); double ent_dif_cnt_dbpspot = dbpspot.getcountDifEnt(); double high_precision_content_dbpspot = dbpspot.getHighPrecEntities(); dbpspot.countEntCat(links_total[j], quer, true);//true is for stemming int cat_cnt_dbpspot_stem = dbpspot.getcountCat(); int ent_cnt_dbpspot_stem = dbpspot.getcountEnt(); int cat_cnt_dbpspot_whole_stem = dbpspot.getcountCatWhole(); int ent_cnt_dbpspot_whole_stem = dbpspot.getcountEntWhole(); double ent_sup_cnt_dbpspot_stem = dbpspot.getcountSupEnt(); double ent_sim_cnt_dbpspot_stem = dbpspot.getcountSimEnt(); double ent_dif_cnt_dbpspot_stem = dbpspot.getcountDifEnt(); System.out.println("I insert the semantic entities and categories stats in the DB\n"); StringBuilder entitiesStatementBuilder = new StringBuilder(); try { entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_y_score`=?,"); entitiesStatementBuilder.append("`cat_avg_y_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_yahoo_score); stmt.setDouble(2, cat_avg_yahoo_score); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dand_score`=?,"); entitiesStatementBuilder.append("`ent_avg_dbpspot_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_d_score); stmt.setDouble(2, ent_avg_dbpspot_score); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_max_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_score`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_score`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_max_dbpspot_score); stmt.setDouble(2, ent_min_dbpspot_score); stmt.setDouble(3, ent_median_dbpspot_score); stmt.setDouble(4, ent_std_dbpspot_score); stmt.setString(5, links_total[j]); stmt.setString(6, quer); if (j < results_number) { stmt.setInt(7, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(7, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(7, 2);//2 for bing } stmt.setString(8, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_max_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_support`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_support`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_dbpspot_support); stmt.setDouble(2, ent_max_dbpspot_support); stmt.setDouble(3, ent_min_dbpspot_support); stmt.setDouble(4, ent_median_dbpspot_support); stmt.setDouble(5, ent_std_dbpspot_support); stmt.setString(6, links_total[j]); stmt.setString(7, quer); if (j < results_number) { stmt.setInt(8, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(8, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(8, 2);//2 for bing } stmt.setString(9, domain); System.out.println("avg db support" + ent_avg_dbpspot_support); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_avg_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_max_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_min_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_median_dbpspot_dif`=?,"); entitiesStatementBuilder.append("`ent_std_dbpspot_dif`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_avg_dbpspot_dif); stmt.setDouble(2, ent_max_dbpspot_dif); stmt.setDouble(3, ent_min_dbpspot_dif); stmt.setDouble(4, ent_median_dbpspot_dif); stmt.setDouble(5, ent_std_dbpspot_dif); stmt.setString(6, links_total[j]); stmt.setString(7, quer); if (j < results_number) { stmt.setInt(8, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(8, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(8, 2);//2 for bing } stmt.setString(9, domain); stmt.executeUpdate(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_sup_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`ent_dif_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`ent_sim_cnt_dbpspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_sup_cnt_dbpspot); stmt.setDouble(2, ent_dif_cnt_dbpspot); stmt.setDouble(3, ent_sim_cnt_dbpspot); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`ent_sup_cnt_dbpspot_stem`=?,"); entitiesStatementBuilder.append("`ent_dif_cnt_dbpspot_stem`=?,"); entitiesStatementBuilder.append("`ent_sim_cnt_dbpspot_stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, ent_sup_cnt_dbpspot_stem); stmt.setDouble(2, ent_dif_cnt_dbpspot_stem); stmt.setDouble(3, ent_sim_cnt_dbpspot_stem); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`unique_ent_cnt_dbpspot`=?,"); entitiesStatementBuilder.append("`unique_ent_scoreSum_dbpspot`=?,"); entitiesStatementBuilder.append("`high_precision_content_dbpspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setDouble(1, unique_ent_cnt_dbpspot); stmt.setDouble(2, unique_ent_scoreSum_dbpspot); stmt.setDouble(3, high_precision_content_dbpspot); stmt.setString(4, links_total[j]); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y`=?,"); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt); stmt.setInt(2, ent_cnt); stmt.setInt(3, cat_cnt_whole); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_W`=?,"); entitiesStatementBuilder.append("`Categories_Contained_Query_D`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, ent_cnt_whole); stmt.setInt(2, cat_cnt_dand); stmt.setInt(3, ent_cnt_dand); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_W`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_whole); stmt.setInt(2, ent_cnt_dand_whole); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot); stmt.setInt(2, ent_cnt_dbpspot); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_W`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_W`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_whole); stmt.setInt(2, ent_cnt_dbpspot_whole); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_stem); stmt.setInt(2, ent_cnt_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_Y_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_Y_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_whole_stem); stmt.setInt(2, ent_cnt_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_stem); stmt.setInt(2, ent_cnt_dand_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_D_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_D_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dand_whole_stem); stmt.setInt(2, ent_cnt_dand_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_stem); stmt.setInt(2, ent_cnt_dbpspot_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); entitiesStatementBuilder = new StringBuilder(); entitiesStatementBuilder.append("UPDATE SEMANTICSTATS SET "); entitiesStatementBuilder.append("`Categories_Contained_Query_DBPspot_W_Stem`=?,"); entitiesStatementBuilder.append("`Entities_Contained_Query_DBPspot_W_Stem`=? "); entitiesStatementBuilder .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(entitiesStatementBuilder.toString()); stmt.setInt(1, cat_cnt_dbpspot_whole_stem); stmt.setInt(2, ent_cnt_dbpspot_whole_stem); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } System.out.println("I inserted the semantic entities and categories stats in the DB\n"); System.out.println( "I will get the html stats for the " + j + " link:" + links_total[j] + "\n"); boolean flag_htmlstats = htm.gethtmlstats(links_total[j]);//get the semantic stats from the html code if (flag_htmlstats) { System.out.println( "I got the html stats for the " + j + " link:" + links_total[j] + "\n"); int scripts_cnt = htm.scripts_number; int nschem = htm.nschem; int hreln = htm.hreln; int total_micron = htm.total_micron; int micron1 = htm.micron1; int micron2 = htm.micron2; int microd = htm.microd; System.out.println("I will insert webstats in the DB\n"); webstatsStmBuild.setLength(0); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`scripts_cnt`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, scripts_cnt); stmt.setString(2, urlString); stmt.setString(3, quer); if (j < results_number) { stmt.setInt(4, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(4, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(4, 2);//2 for bing } stmt.setString(5, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); System.out.println("I inserted webstats in the DB\n"); System.out.println("I will insert semantic stats in the DB\n"); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`schema.org_entities`=? , "); semanticstatsStmBuild.append("`hreltags`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, nschem); stmt.setInt(2, hreln); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`total_microformats`=? , "); semanticstatsStmBuild.append("`Microformats-1`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, total_micron); stmt.setInt(2, micron1); stmt.setString(3, urlString); stmt.setString(4, quer); if (j < results_number) { stmt.setInt(5, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(5, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(5, 2);//2 for bing } stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } try { conn = DriverManager.getConnection(url, user, password); StringBuilder semanticstatsStmBuild = new StringBuilder(); semanticstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); semanticstatsStmBuild.append("`Microformats-2`=? , "); semanticstatsStmBuild.append("`Microdata`=? , "); semanticstatsStmBuild.append("`FOAF_HTML`=? "); semanticstatsStmBuild.append( "WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); stmt = conn.prepareStatement(semanticstatsStmBuild.toString()); stmt.setInt(1, micron2); stmt.setInt(2, microd); stmt.setInt(3, htm.foaf); stmt.setString(4, urlString); stmt.setString(5, quer); if (j < results_number) { stmt.setInt(6, 0);//0 for yahoo } else if (j < results_number * 2) { stmt.setInt(6, 1);//1 for google } else if (j < results_number * 3) { stmt.setInt(6, 2);//2 for bing } stmt.setString(7, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } System.out.println("I inserted semantic stats in the DB\n"); } } } } String[] parse_output; if (ContentSemantics.get(3) || ContentSemantics.get(1)) { //we perform LDA or TFIDF analysis to the links obtained if (!enginechoice.get(3)) { if (enginechoice.get(2)) {//Yahoo parse_output = ld.perform(links_yahoo, domain, "yahoo", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = 0; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } if (enginechoice.get(1)) {//Google parse_output = ld.perform(links_google, domain, "google", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = results_number; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } if (enginechoice.get(0)) {//Bing parse_output = ld.perform(links_bing, domain, "bing", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(), ContentSemantics.get(1), ContentSemantics.get(3), config_path); int j = results_number * 2; for (String s : parse_output) { parseOutputList.put(j, s); j++; } System.gc(); } } /*else{ System.gc();//links_total parse_output=ld.perform(links_total, domain, "merged", directory_save, quer, SWebRankSettings.get(1).intValue(), alpha, SWebRankSettings.get(0).doubleValue(), SWebRankSettings.get(2).intValue(), SWebRankSettings.get(3).intValue(),"Merged",ContentSemantics.get(1),ContentSemantics.get(3), config_path); Collections.addAll(parseOutputList, parse_output); System.gc(); }*/ } } System.gc(); List<String> wordList = null; //hashmap for every engine, with topics, words and probability of each word HashMap<String, HashMap<Integer, HashMap<String, Double>>> enginetopicwordprobmap = new HashMap<>(); List<String> lda_output = new ArrayList<>(); if (ContentSemantics.get(3)) { //get the top content from TFIDF System.out.println("i ll try to read the keys"); wordList = ld.return_topWordsTFIDF(); System.out.println("i returned the wordlist to search analysis"); } else if (ContentSemantics.get(0)) {//get the wordlist from Diffbot Diffbot db = new Diffbot(); wordList = db.compute(links_total, directory_save, config_path); } else if (ContentSemantics.get(2)) {//get the wordllist from Sensebot Sensebot sb = new Sensebot(); wordList = sb.compute(links_total, directory_save, SensebotConcepts, config_path); } else { //get the top content from LDA System.out.println("i ll try to read the keys"); LDAtopicsWords rk = new LDAtopicsWords(); enginetopicwordprobmap = rk.readFile(directory_save, SWebRankSettings.get(4), SWebRankSettings.get(3).intValue(), SWebRankSettings.get(1).intValue(), SWebRankSettings.get(11).intValue()); JSONArray ArrayEngineLevel = new JSONArray(); List<String> ids = new ArrayList<>(); //Node node = nodeBuilder().client(true).clusterName("lshrankldacluster").node(); //Client client = node.client(); Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", "lshrankldacluster") .build(); Client client = new TransportClient(settings) .addTransportAddress(new InetSocketTransportAddress("localhost", 9300)); //save in elastic search the produced by LDA distributions of words over topics for every engine for (String engine : enginetopicwordprobmap.keySet()) { HashMap<Integer, HashMap<String, Double>> topicwordprobmap = new HashMap<>(); topicwordprobmap = enginetopicwordprobmap.get(engine); JSONObject objEngineLevel = new JSONObject(); JSONArray ArrayTopicLevel = new JSONArray(); //for every topic get the words and their probability for (Integer topicindex : topicwordprobmap.keySet()) { JSONObject objTopicLevel = new JSONObject(); objTopicLevel.put("topic", topicindex); JSONObject objmap = new JSONObject(topicwordprobmap.get(topicindex)); Set keySet = objmap.keySet(); Iterator iterator = keySet.iterator(); while (iterator.hasNext()) { String word = iterator.next().toString(); if (!lda_output.contains(word)) { lda_output.add(word); } //get the words in a separate list } objTopicLevel.put("wordsmap", objmap);//write the words in elastic search ArrayTopicLevel.add(objTopicLevel); } objEngineLevel.put("engine", engine); objEngineLevel.put("query", quer); objEngineLevel.put("domain", domain); objEngineLevel.put("iteration", iteration_counter); objEngineLevel.put("TopicsWordMap", ArrayTopicLevel); ArrayEngineLevel.add(objEngineLevel); String id = domain + "/" + quer + "/" + engine + "/" + iteration_counter;//create unique id for the elasticsearch document ids.add(id);//add to the ids list which contains the ids of the current round List<String> elasticIndexes = ri.GetKeyFile(config_path, "elasticSearchIndexes"); IndexRequest indexReq = new IndexRequest(elasticIndexes.get(2), "content", id); indexReq.source(objEngineLevel); IndexResponse indexRes = client.index(indexReq).actionGet(); } //node.close(); client.close(); ElasticGetWordList elasticGetwordList = new ElasticGetWordList();//get the wordlist from elastic search for the ids from the current round wordList = elasticGetwordList.get(ids, config_path); DataManipulation datamanipulation = new DataManipulation(); wordList = datamanipulation.clearListString(wordList); System.out.println("i returned the wordlist to search analysis"); } //get some stats regarding the entities, categories and parsed content from each link comparing it to the top words produced by lda for (int j = 0; j < links_total.length; j++) { if (links_total[j] != null) { String urlString = links_total[j]; if (urlString.length() > 199) { urlString = links_total[j].substring(0, 198); } int rank = -1; int engine = -1;//0 for yahoo,1 for google,2 for bing if (j < results_number) { rank = j; engine = 0; } else if (j < results_number * 2) { rank = j - results_number; engine = 1; } else if (j < results_number * 3) { rank = j - results_number * 2; engine = 2; } LDAsemStats ldaSemStats = new LDAsemStats();//get the stats by comparing the top words produced by LDA and the parsed content //check the LDAsemStats class for more StringBuilder webstatsStmBuild = new StringBuilder(); if (!parseOutputList.isEmpty()) { if (!parseOutputList.get(j).equalsIgnoreCase("") && !parseOutputList.get(j).equalsIgnoreCase("null") && (parseOutputList.get(j).length() > 0)) { ldaSemStats.getTopWordsStats(parseOutputList.get(j), lda_output, false);//without stemming int top_words_lda = ldaSemStats.getTopStats(); double top_words_lda_per = ldaSemStats.getTopPercentageStats(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`top_words_lda`=? , "); webstatsStmBuild.append("`top_words_lda_per`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, top_words_lda); stmt.setDouble(2, top_words_lda_per); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getTopWordsStats(parseOutputList.get(j), lda_output, true);//with stemming int top_words_lda_stem = ldaSemStats.getTopStats(); double top_words_lda_per_stem = ldaSemStats.getTopPercentageStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`top_words_lda_stem`=? , "); webstatsStmBuild.append("`top_words_lda_per_stem`=? "); webstatsStmBuild .append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, top_words_lda_stem); stmt.setDouble(2, top_words_lda_per_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } if (EntitiesMapDBP.get(j) != null && CategoriesMapDBP.get(j) != null) { //we are going to check if semantic entities and categories recognized exist in the lda words recognized as prominent //we are going to use DBPEDIA spotligh and Dandelion named Entity Extraction API //and stemming through Snowball Stemmer ldaSemStats.getEntCatStats(EntitiesMapDBP.get(j), CategoriesMapDBP.get(j), lda_output, false); int ent_cnt_dbpspot_lda = ldaSemStats.getEntStats(); int cat_cnt_dbpspot_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dbpspot_lda`=? , "); webstatsStmBuild.append("`cat_cnt_dbpspot_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dbpspot_lda); stmt.setInt(2, cat_cnt_dbpspot_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapDBP.get(j), CategoriesMapDBP.get(j), lda_output, true); int ent_cnt_dbpspot_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_dbpspot_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dbpspot_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_dbpspot_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dbpspot_lda_stem); stmt.setInt(2, cat_cnt_dbpspot_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (EntitiesMapDand.get(j) != null && CategoriesMapDand.get(j) != null) { ldaSemStats.getEntCatStats(EntitiesMapDand.get(j), CategoriesMapDand.get(j), lda_output, false); int ent_cnt_dand_lda = ldaSemStats.getEntStats(); int cat_cnt_dand_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dand_lda`=? , "); webstatsStmBuild.append("`cat_cnt_dand_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dand_lda); stmt.setInt(2, cat_cnt_dand_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapDand.get(j), CategoriesMapDand.get(j), lda_output, true); int ent_cnt_dand_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_dand_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_dand_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_dand_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_dand_lda_stem); stmt.setInt(2, cat_cnt_dand_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } if (EntitiesMapYahoo.get(j) != null && CategoriesMapYahoo.get(j) != null) { //we are going to check if semantic entities and categories recognized exist in the lda words recognized as prominent //we are going to use DBPEDIA spotligh and Dandelion named Entity Extraction API //and stemming through Snowball Stemmer ldaSemStats.getEntCatStats(EntitiesMapYahoo.get(j), CategoriesMapYahoo.get(j), lda_output, false); int ent_cnt_y_lda = ldaSemStats.getEntStats(); int cat_cnt_y_lda = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_y_lda`=? , "); webstatsStmBuild.append("`cat_cnt_y_lda`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_y_lda); stmt.setInt(2, cat_cnt_y_lda); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } ldaSemStats.getEntCatStats(EntitiesMapYahoo.get(j), CategoriesMapYahoo.get(j), lda_output, true); int ent_cnt_y_lda_stem = ldaSemStats.getEntStats(); int cat_cnt_y_lda_stem = ldaSemStats.getCategoryStats(); webstatsStmBuild = new StringBuilder(); webstatsStmBuild.append("UPDATE SEMANTICSTATS SET "); webstatsStmBuild.append("`ent_cnt_y_lda_stem`=? , "); webstatsStmBuild.append("`cat_cnt_y_lda_stem`=? "); webstatsStmBuild.append("WHERE `url`=? AND `query`=? AND `search_engine`=? AND `domain`=?"); try { conn = DriverManager.getConnection(url, user, password); stmt = conn.prepareStatement(webstatsStmBuild.toString()); stmt.setInt(1, ent_cnt_y_lda_stem); stmt.setInt(2, cat_cnt_y_lda_stem); stmt.setString(3, urlString); stmt.setString(4, quer); stmt.setInt(5, engine); stmt.setString(6, domain); stmt.executeUpdate(); } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } } } } return wordList; } catch (NullPointerException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<>(); return finalList; } catch (SQLException | ElasticsearchException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); ArrayList<String> finalList = new ArrayList<>(); return finalList; } finally { try { if (stmt != null) stmt.close(); if (conn != null) conn.close(); } catch (SQLException ex) { Logger.getLogger(Search_analysis.class.getName()).log(Level.SEVERE, null, ex); } } }