List of usage examples for org.apache.commons.lang3 StringEscapeUtils unescapeXml
public static final String unescapeXml(final String input)
Unescapes a string containing XML entity escapes to a string containing the actual Unicode characters corresponding to the escapes.
Supports only the five basic XML entities (gt, lt, quot, amp, apos).
From source file:net.java.sip.communicator.impl.protocol.jabber.OperationSetServerStoredAccountInfoJabberImpl.java
/** * Saves the list of details for this account that were ready to be stored * online on the server. This method performs the actual saving of details * online on the server and is supposed to be invoked after addDetail(), * replaceDetail() and/or removeDetail(). * <p>//from w w w. j av a2s. co m * @throws OperationFailedException with code Network Failure if putting the * new values back online has failed. */ public void save() throws OperationFailedException { assertConnected(); List<GenericDetail> details = infoRetreiver.getContactDetails(uin); VCardXEP0153 vCard = new VCardXEP0153(); for (GenericDetail detail : details) { if (detail instanceof ImageDetail) { byte[] avatar = ((ImageDetail) detail).getBytes(); if (avatar == null) vCard.setAvatar(new byte[0]); else vCard.setAvatar(avatar); fireServerStoredDetailsChangeEvent(jabberProvider, ServerStoredDetailsChangeEvent.DETAIL_ADDED, null, detail); } else if (detail.getClass().equals(FirstNameDetail.class)) { vCard.setFirstName((String) detail.getDetailValue()); } else if (detail.getClass().equals(MiddleNameDetail.class)) { vCard.setMiddleName((String) detail.getDetailValue()); } else if (detail.getClass().equals(LastNameDetail.class)) { vCard.setLastName((String) detail.getDetailValue()); } else if (detail.getClass().equals(NicknameDetail.class)) vCard.setNickName((String) detail.getDetailValue()); else if (detail.getClass().equals(URLDetail.class)) { if (detail.getDetailValue() != null) vCard.setField("URL", ((URL) detail.getDetailValue()).toString()); } else if (detail.getClass().equals(BirthDateDetail.class)) { if (detail.getDetailValue() != null) { Calendar c = ((BirthDateDetail) detail).getCalendar(); DateFormat dateFormat = new SimpleDateFormat( JabberActivator.getResources().getI18NString("plugin.accountinfo.BDAY_FORMAT")); String strdate = dateFormat.format(c.getTime()); vCard.setField("BDAY", strdate); } } else if (detail.getClass().equals(AddressDetail.class)) vCard.setAddressFieldHome("STREET", (String) detail.getDetailValue()); else if (detail.getClass().equals(CityDetail.class)) vCard.setAddressFieldHome("LOCALITY", (String) detail.getDetailValue()); else if (detail.getClass().equals(ProvinceDetail.class)) vCard.setAddressFieldHome("REGION", (String) detail.getDetailValue()); else if (detail.getClass().equals(PostalCodeDetail.class)) vCard.setAddressFieldHome("PCODE", (String) detail.getDetailValue()); else if (detail.getClass().equals(CountryDetail.class)) vCard.setAddressFieldHome("CTRY", (String) detail.getDetailValue()); else if (detail.getClass().equals(PhoneNumberDetail.class)) vCard.setPhoneHome("VOICE", (String) detail.getDetailValue()); else if (detail.getClass().equals(WorkPhoneDetail.class)) vCard.setPhoneWork("VOICE", (String) detail.getDetailValue()); else if (detail.getClass().equals(MobilePhoneDetail.class)) vCard.setPhoneHome("CELL", (String) detail.getDetailValue()); else if (detail.getClass().equals(VideoDetail.class)) vCard.setPhoneHome("VIDEO", (String) detail.getDetailValue()); else if (detail.getClass().equals(WorkVideoDetail.class)) vCard.setPhoneWork("VIDEO", (String) detail.getDetailValue()); else if (detail.getClass().equals(EmailAddressDetail.class)) vCard.setEmailHome((String) detail.getDetailValue()); else if (detail.getClass().equals(WorkEmailAddressDetail.class)) vCard.setEmailWork((String) detail.getDetailValue()); else if (detail.getClass().equals(WorkOrganizationNameDetail.class)) vCard.setOrganization((String) detail.getDetailValue()); else if (detail.getClass().equals(JobTitleDetail.class)) vCard.setField("TITLE", (String) detail.getDetailValue()); else if (detail.getClass().equals(AboutMeDetail.class)) vCard.setField("ABOUTME", (String) detail.getDetailValue()); } //Fix the display name detail String tmp; tmp = infoRetreiver.checkForFullName(vCard); if (tmp != null) { DisplayNameDetail displayNameDetail = new DisplayNameDetail(StringEscapeUtils.unescapeXml(tmp)); Iterator<GenericDetail> detailIt = infoRetreiver.getDetails(uin, DisplayNameDetail.class); while (detailIt.hasNext()) { infoRetreiver.getCachedContactDetails(uin).remove(detailIt.next()); } infoRetreiver.getCachedContactDetails(uin).add(displayNameDetail); } try { vCard.save(jabberProvider.getConnection()); } catch (XMPPException xmppe) { logger.error("Error loading/saving vcard: ", xmppe); throw new OperationFailedException("Error loading/saving vcard: ", 1, xmppe); } }
From source file:android.databinding.tool.store.LayoutFileParser.java
private static String escapeQuotes(String textWithQuotes, boolean unescapeValue) { char first = textWithQuotes.charAt(0); int start = 0, end = textWithQuotes.length(); if (first == '"' || first == '\'') { start = 1;// w ww . ja v a2 s .c om } char last = textWithQuotes.charAt(textWithQuotes.length() - 1); if (last == '"' || last == '\'') { end -= 1; } String val = textWithQuotes.substring(start, end); if (unescapeValue) { return StringEscapeUtils.unescapeXml(val); } else { return val; } }
From source file:net.java.sip.communicator.impl.history.HistoryReaderImpl.java
/** * If there is keyword restriction and doesn't match the conditions * return null. Otherwise return the HistoryRecord corresponding the * given nodes.//from w w w . j av a 2 s . c o m * * @param propertyNodes NodeList * @param timestamp Date * @param keywords String[] * @param field String * @param caseSensitive boolean * @return HistoryRecord */ static HistoryRecord filterByKeyword(NodeList propertyNodes, Date timestamp, String[] keywords, String field, boolean caseSensitive) { ArrayList<String> nameVals = new ArrayList<String>(); int len = propertyNodes.getLength(); boolean targetNodeFound = false; for (int j = 0; j < len; j++) { Node propertyNode = propertyNodes.item(j); if (propertyNode.getNodeType() == Node.ELEMENT_NODE) { String nodeName = propertyNode.getNodeName(); Node nestedNode = propertyNode.getFirstChild(); if (nestedNode == null) continue; // Get nested TEXT node's value String nodeValue = nestedNode.getNodeValue(); // unescape xml chars, we have escaped when writing values nodeValue = StringEscapeUtils.unescapeXml(nodeValue); if (field != null && field.equals(nodeName)) { targetNodeFound = true; if (!matchKeyword(nodeValue, keywords, caseSensitive)) return null; // doesn't match the given keyword(s) // so return nothing } nameVals.add(nodeName); // Get nested TEXT node's value nameVals.add(nodeValue); } } // if we need to find a particular record but the target node is not // present skip this record if (keywords != null && keywords.length > 0 && !targetNodeFound) { return null; } String[] propertyNames = new String[nameVals.size() / 2]; String[] propertyValues = new String[propertyNames.length]; for (int j = 0; j < propertyNames.length; j++) { propertyNames[j] = nameVals.get(j * 2); propertyValues[j] = nameVals.get(j * 2 + 1); } return new HistoryRecord(propertyNames, propertyValues, timestamp); }
From source file:com.rockagen.commons.util.CommUtil.java
/** * unescape XML see StringEscapeUtils.unescapeXml(str) * * @param str value//from w w w . jav a 2 s. c o m * @return string */ public static String unescapeXml(String str) { if (isBlank(str)) { return str; } return StringEscapeUtils.unescapeXml(str); }
From source file:com.jaeksoft.searchlib.parser.HtmlParser.java
@Override protected void parseContent(StreamLimiter streamLimiter, LanguageEnum forcedLang) throws IOException, SearchLibException { titleBoost = getFloatProperty(ClassPropertyEnum.TITLE_BOOST); boostTagMap = new TreeMap<String, BoostTag>(); boostTagMap.put("h1", new BoostTag(ClassPropertyEnum.H1_BOOST)); boostTagMap.put("h2", new BoostTag(ClassPropertyEnum.H2_BOOST)); boostTagMap.put("h3", new BoostTag(ClassPropertyEnum.H3_BOOST)); boostTagMap.put("h4", new BoostTag(ClassPropertyEnum.H4_BOOST)); boostTagMap.put("h5", new BoostTag(ClassPropertyEnum.H5_BOOST)); boostTagMap.put("h6", new BoostTag(ClassPropertyEnum.H6_BOOST)); ignoreMetaNoIndex = getBooleanProperty(ClassPropertyEnum.IGNORE_META_NOINDEX); ignoreMetaNoFollow = getBooleanProperty(ClassPropertyEnum.IGNORE_META_NOFOLLOW); ignoreLinkNoFollow = getBooleanProperty(ClassPropertyEnum.IGNORE_LINK_NOFOLLOW); ignoreUntitledDocuments = getBooleanProperty(ClassPropertyEnum.IGNORE_UNTITLED_DOCUMENTS); ignoreNonCanonical = getBooleanProperty(ClassPropertyEnum.IGNORE_NON_CANONICAL); String currentCharset = null; String headerCharset = null;//w w w.jav a 2s . c o m String detectedCharset = null; IndexDocument sourceDocument = getSourceDocument(); if (sourceDocument != null) { FieldValueItem fieldValueItem = sourceDocument .getFieldValue(UrlItemFieldEnum.INSTANCE.contentTypeCharset.getName(), 0); if (fieldValueItem != null) headerCharset = fieldValueItem.getValue(); if (headerCharset == null) { fieldValueItem = sourceDocument.getFieldValue(UrlItemFieldEnum.INSTANCE.contentEncoding.getName(), 0); if (fieldValueItem != null) headerCharset = fieldValueItem.getValue(); } currentCharset = headerCharset; } if (currentCharset == null) { detectedCharset = streamLimiter.getDetectedCharset(); currentCharset = detectedCharset; } if (currentCharset == null) { currentCharset = getProperty(ClassPropertyEnum.DEFAULT_CHARSET).getValue(); } String xPathExclusions = getProperty(ClassPropertyEnum.XPATH_EXCLUSION).getValue(); Set<Object> xPathExclusionsSet = null; if (!StringUtils.isEmpty(xPathExclusions)) xPathExclusionsSet = new HashSet<Object>(); HtmlParserEnum htmlParserEnum = HtmlParserEnum.find(getProperty(ClassPropertyEnum.HTML_PARSER).getValue()); HtmlDocumentProvider htmlProvider = getHtmlDocumentProvider(htmlParserEnum, currentCharset, streamLimiter, xPathExclusions, xPathExclusionsSet); if (htmlProvider == null) return; URL currentURL = htmlProvider.getBaseHref(); IndexDocument srcDoc = getSourceDocument(); try { if (currentURL == null) currentURL = LinkUtils.newEncodedURL(streamLimiter.getOriginURL()); if (currentURL == null) { FieldValueItem fvi = srcDoc.getFieldValue(UrlItemFieldEnum.INSTANCE.url.getName(), 0); if (fvi != null) currentURL = LinkUtils.newEncodedURL(fvi.getValue()); } } catch (URISyntaxException e) { throw new IOException(e); } URL canonicalURL = htmlProvider.getCanonicalLink(currentURL); if (canonicalURL != null) { String canUrl = canonicalURL.toExternalForm(); addDetectedLink(canUrl); if (ignoreNonCanonical) { String curUrl = currentURL.toExternalForm(); if (!canUrl.equals(curUrl)) { isCanonical = false; return; } } } isCanonical = true; String title = htmlProvider.getTitle(); if (ignoreUntitledDocuments) if (title == null || title.length() == 0) return; ParserResultItem result = getNewParserResultItem(); addFieldTitle(result, title); result.addField(ParserFieldEnum.htmlProvider, htmlProvider.getName()); // Check ContentType charset in meta http-equiv String metaCharset = htmlProvider.getMetaCharset(); String selectedCharset = selectCharset(headerCharset, detectedCharset, metaCharset); if (selectedCharset != null) { if (!selectedCharset.equals(currentCharset)) { currentCharset = selectedCharset; htmlProvider = getHtmlDocumentProvider(htmlParserEnum, currentCharset, streamLimiter, xPathExclusions, xPathExclusionsSet); } } StringWriter writer = new StringWriter(); IOUtils.copy(streamLimiter.getNewInputStream(), writer, currentCharset); result.addField(ParserFieldEnum.htmlSource, writer.toString()); writer.close(); HtmlNodeAbstract<?> rootNode = htmlProvider.getRootNode(); if (rootNode == null) return; for (HtmlNodeAbstract<?> metaNode : htmlProvider.getMetas()) { String metaName = metaNode.getAttributeText("name"); if (metaName != null && metaName.startsWith(OPENSEARCHSERVER_FIELD)) { String field = metaName.substring(OPENSEARCHSERVER_FIELD_LENGTH); String[] fields = field.split("\\."); if (fields != null) { String content = metaNode.getAttributeText("content"); result.addDirectFields(fields, content); } } } result.addField(ParserFieldEnum.charset, currentCharset); String metaRobots = null; String metaDcLanguage = null; String metaContentLanguage = null; for (HtmlNodeAbstract<?> node : htmlProvider.getMetas()) { String attr_name = node.getAttributeText("name"); String attr_http_equiv = node.getAttributeText("http-equiv"); if ("keywords".equalsIgnoreCase(attr_name)) result.addField(ParserFieldEnum.meta_keywords, HtmlDocumentProvider.getMetaContent(node)); else if ("description".equalsIgnoreCase(attr_name)) result.addField(ParserFieldEnum.meta_description, HtmlDocumentProvider.getMetaContent(node)); else if ("robots".equalsIgnoreCase(attr_name)) metaRobots = HtmlDocumentProvider.getMetaContent(node); else if ("dc.language".equalsIgnoreCase(attr_name)) metaDcLanguage = HtmlDocumentProvider.getMetaContent(node); else if ("content-language".equalsIgnoreCase(attr_http_equiv)) metaContentLanguage = HtmlDocumentProvider.getMetaContent(node); } boolean metaRobotsFollow = true; boolean metaRobotsNoIndex = false; if (metaRobots != null) { metaRobots = metaRobots.toLowerCase(); if (metaRobots.contains("noindex") && !ignoreMetaNoIndex) { metaRobotsNoIndex = true; result.addField(ParserFieldEnum.meta_robots, "noindex"); } if (metaRobots.contains("nofollow") && !ignoreMetaNoFollow) { metaRobotsFollow = false; result.addField(ParserFieldEnum.meta_robots, "nofollow"); } } UrlFilterItem[] urlFilterList = getUrlFilterList(); boolean removeFragment = ClassPropertyEnum.KEEP_REMOVE_LIST[1] .equalsIgnoreCase(getProperty(ClassPropertyEnum.URL_FRAGMENT).getValue()); List<HtmlNodeAbstract<?>> nodes = rootNode.getAllNodes("a", "frame", "img"); if (srcDoc != null && nodes != null && metaRobotsFollow) { for (HtmlNodeAbstract<?> node : nodes) { String href = null; String rel = null; String nodeName = node.getNodeName(); if ("a".equals(nodeName)) { href = node.getAttributeText("href"); rel = node.getAttributeText("rel"); } else if ("frame".equals(nodeName) || "img".equals(nodeName)) { href = node.getAttributeText("src"); } boolean follow = true; if (rel != null) if (rel.contains("nofollow") && !ignoreLinkNoFollow) follow = false; URL newUrl = null; if (href != null) if (!href.startsWith("javascript:")) if (currentURL != null) { href = StringEscapeUtils.unescapeXml(href); newUrl = LinkUtils.getLink(currentURL, href, urlFilterList, removeFragment); } if (newUrl != null) { ParserFieldEnum field = null; if (newUrl.getHost().equalsIgnoreCase(currentURL.getHost())) { if (follow) field = ParserFieldEnum.internal_link; else field = ParserFieldEnum.internal_link_nofollow; } else { if (follow) field = ParserFieldEnum.external_link; else field = ParserFieldEnum.external_link_nofollow; } String link = newUrl.toExternalForm(); result.addField(field, link); if (follow) addDetectedLink(link); } } } if (!metaRobotsNoIndex) { nodes = rootNode.getNodes("html", "body"); if (nodes == null || nodes.size() == 0) nodes = rootNode.getNodes("html"); if (nodes != null && nodes.size() > 0) { StringBuilder sb = new StringBuilder(); getBodyTextContent(result, sb, nodes.get(0), true, null, 1024, xPathExclusionsSet); result.addField(ParserFieldEnum.body, sb); } } // Identification de la langue: Locale lang = null; String langMethod = null; String[] pathHtml = { "html" }; nodes = rootNode.getNodes(pathHtml); if (nodes != null && nodes.size() > 0) { langMethod = "html lang attribute"; String l = nodes.get(0).getAttributeText("lang"); if (l != null) lang = Lang.findLocaleISO639(l); } if (lang == null && metaContentLanguage != null) { langMethod = "meta http-equiv content-language"; lang = Lang.findLocaleISO639(metaContentLanguage); } if (lang == null && metaDcLanguage != null) { langMethod = "meta dc.language"; lang = Lang.findLocaleISO639(metaDcLanguage); } if (lang != null) { result.addField(ParserFieldEnum.lang, lang.getLanguage()); result.addField(ParserFieldEnum.lang_method, langMethod); } else if (!metaRobotsNoIndex) lang = result.langDetection(10000, ParserFieldEnum.body); if (getFieldMap().isMapped(ParserFieldEnum.generated_title)) { StringBuilder sb = new StringBuilder(); try { sb.append(new URI(streamLimiter.getOriginURL()).getHost()); } catch (URISyntaxException e) { Logging.error(e); } String generatedTitle = null; for (Map.Entry<String, BoostTag> entry : boostTagMap.entrySet()) { BoostTag boostTag = entry.getValue(); if (boostTag.firstContent != null) { generatedTitle = boostTag.firstContent; break; } } if (generatedTitle == null) { final String FIELD_TITLE = "contents"; MemoryIndex bodyMemoryIndex = new MemoryIndex(); Analyzer bodyAnalyzer = new WhitespaceAnalyzer(Version.LUCENE_36); String bodyText = result.getMergedBodyText(100000, " ", ParserFieldEnum.body); bodyMemoryIndex.addField(FIELD_TITLE, bodyText, bodyAnalyzer); IndexSearcher indexSearcher = bodyMemoryIndex.createSearcher(); IndexReader indexReader = indexSearcher.getIndexReader(); MoreLikeThis mlt = new MoreLikeThis(indexReader); mlt.setAnalyzer(bodyAnalyzer); mlt.setFieldNames(new String[] { FIELD_TITLE }); mlt.setMinWordLen(3); mlt.setMinTermFreq(1); mlt.setMinDocFreq(1); String[] words = mlt.retrieveInterestingTerms(0); if (words != null && words.length > 0) generatedTitle = words[0]; } if (generatedTitle != null) { if (sb.length() > 0) sb.append(" - "); sb.append(generatedTitle); } if (sb.length() > 67) { int pos = sb.indexOf(" ", 60); if (pos == -1) pos = 67; sb.setLength(pos); sb.append("..."); } result.addField(ParserFieldEnum.generated_title, sb.toString()); } }
From source file:mServer.crawler.sender.MediathekBr.java
private String checkThema(String thema) { thema = StringEscapeUtils.unescapeXml(thema.trim()); thema = StringEscapeUtils.unescapeHtml4(thema.trim()); for (String s : listeAlleThemen) { if (thema.equalsIgnoreCase(s)) { return s; }//from ww w . j a va 2s . c om } for (String s : listeAlleThemen) { if (thema.toLowerCase().startsWith(s.toLowerCase())) { return s; } } return getSendername(); }
From source file:gov.llnl.ontology.text.corpora.NYTCorpusDocument.java
/** * Setter for the body property./*from w w w . j ava 2 s . c o m*/ * * @param body * the body to set */ public void setBody(String body) { this.body = StringEscapeUtils.unescapeXml(body); }
From source file:com.hygenics.parser.ParseJSoup.java
/** * Runs the Program/*w ww . j a v a2 s . c o m*/ */ public void run() { int its = 0; this.select = Properties.getProperty(this.select); this.extracondition = Properties.getProperty(this.extracondition); this.column = Properties.getProperty(this.column); createTables(); log.info("Starting Parse via JSoup @ " + Calendar.getInstance().getTime().toString()); ForkJoinPool fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procs); Set<Callable<ArrayList<String>>> collection; List<Future<ArrayList<String>>> futures; ArrayList<String> data = new ArrayList<String>((commitsize + 10)); ArrayList<String> outdata = new ArrayList<String>(((commitsize + 10) * 3)); int offenderhash = offset; boolean run = true; int iteration = 0; int currpos = 0; do { collection = new HashSet<Callable<ArrayList<String>>>(qnums); log.info("Getting Data"); // get data currpos = iteration * commitsize + offset; iteration += 1; String query = select; if (extracondition != null) { query += " " + extracondition; } if (extracondition != null) { query += " WHERE " + extracondition + " AND "; } else { query += " WHERE "; } for (int i = 0; i < qnums; i++) { if (currpos + (Math.round(commitsize / qnums * (i + 1))) < currpos + commitsize) { collection.add(new SplitQuery((query + pullid + " >= " + Integer.toString(currpos + (Math.round(commitsize / qnums * (i)))) + " AND " + pullid + " < " + Integer.toString(currpos + (Math.round(commitsize / qnums * (i + 1))))))); } else { collection.add(new SplitQuery((query + pullid + " >= " + Integer.toString(currpos + (Math.round(commitsize / qnums * (i)))) + " AND " + pullid + " < " + Integer.toString(currpos + commitsize)))); } } if (collection.size() > 0) { futures = fjp.invokeAll(collection); int w = 0; while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) { w++; } for (Future<ArrayList<String>> f : futures) { try { // TODO Get Pages to Parse data.addAll(f.get()); } catch (NullPointerException e) { log.info("Some Data Returned Null"); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } collection = new HashSet<Callable<ArrayList<String>>>(data.size()); // checkstring if (data.size() == 0 && checkstring != null && its <= maxchecks) { its++; collection.add(new SplitQuery(checkstring)); futures = fjp.invokeAll(collection); int w = 0; while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) { w++; } for (Future<ArrayList<String>> f : futures) { try { // TODO Get Pages to Parse data.addAll(f.get()); } catch (NullPointerException e) { log.info("Some Data Returned Null"); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } if (data.size() == 0) { // set to stop if size is0 log.info("No Pages to Parse. Will Terminate"); run = false; } else { // parse log.info("Starting JSoup Parse @ " + Calendar.getInstance().getTime().toString()); for (String json : data) { // faster json reader is minimal json but faster parser is // Simple Json Map<String, Json> jMap = Json.read(json).asJsonMap(); if (jMap.containsKey("offenderhash")) { // string to int in case it is a string and has some // extra space offenderhash = Integer.parseInt(jMap.get("offenderhash").asString().trim()); } boolean allow = true; if (mustcontain != null) { if (jMap.get(column).asString().contains(mustcontain) == false) { allow = false; } } if (cannotcontain != null) { if (jMap.get(column).asString().contains(cannotcontain)) { allow = false; } } // this is the fastest way. I was learning before and will // rewrite when time permits. if (allow == true) { if (jMap.containsKey("offenderhash")) { if (this.singlepaths != null) { collection.add(new ParseSingle(Integer.toString(offenderhash), header, footer, pagenarrow, singlepaths, StringEscapeUtils.unescapeXml(jMap.get(column).asString()), replace, replaceSequence)); } if (this.multipaths != null) { collection.add(new ParseRows(Integer.toString(offenderhash), header, footer, pagenarrow, multipaths, StringEscapeUtils.unescapeXml(jMap.get(column).asString()), replace, replaceSequence)); } if (this.recordpaths != null) { collection.add(new ParseLoop(Integer.toString(offenderhash), header, footer, pagenarrow, recordpaths, StringEscapeUtils.unescapeXml(jMap.get(column).asString()), replace, replaceSequence)); } } } offenderhash += 1; } // complete parse log.info("Waiting for Parsing to Complete."); if (collection.size() > 0) { futures = fjp.invokeAll(collection); int w = 0; while (fjp.isQuiescent() && fjp.getActiveThreadCount() > 0) { w++; } log.info("Waited for " + Integer.toString(w) + " Cycles!"); for (Future<ArrayList<String>> f : futures) { try { outdata.addAll(f.get()); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } log.info("Finished Parsing @ " + Calendar.getInstance().getTime().toString()); int cp = 0; // post data log.info("Posting Data @ " + Calendar.getInstance().getTime().toString()); if (outdata.size() > 0) { for (int i = 0; i < qnums; i++) { ArrayList<String> od = new ArrayList<String>( ((cp + (Math.round(outdata.size() / qnums) - cp)))); if (cp + (Math.round(outdata.size() / qnums)) < outdata.size()) { od.addAll(outdata.subList(cp, (cp + (Math.round(outdata.size() / qnums))))); } else { od.addAll(outdata.subList(cp, (outdata.size() - 1))); } fjp.execute(new SplitPost(template, od)); cp += Math.round(outdata.size() / qnums); } int w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + Integer.toString(w) + " cycles!"); } log.info("Finished Posting to DB @ " + Calendar.getInstance().getTime().toString()); // size should remain same with 10 slot buffer room data.clear(); outdata.clear(); } // my favorite really desperate attempt to actually invoke garbage // collection because of MASSIVE STRINGS System.gc(); Runtime.getRuntime().gc(); } while (run); log.info("Shutting Down FJP"); // shutdown fjp if (fjp.isShutdown() == false) { fjp.shutdownNow(); } log.info("Finished Parsing @ " + Calendar.getInstance().getTime().toString()); }
From source file:com.foo.manager.commonManager.thread.HttpHandleThread.java
private String handleXml_sn_receipt(String jsonString) throws CommonException { // String jsonReturnString = ""; SimpleDateFormat sf = CommonUtil.getDateFormatter(CommonDefine.COMMON_FORMAT_1); System.out.println("---------------------------?FPAPI_SN_RECEIPT-------------------------------"); // /*w ww. ja v a 2 s .c o m*/ JSONObject jsonObject = JSONObject.fromObject(jsonString); JSONObject orderInfo = (JSONObject) jsonObject.get("orderInfo"); JSONObject senderInfo = (JSONObject) orderInfo.get("senderInfo"); JSONArray orderItemList = (JSONArray) orderInfo.get("orderItemList"); // JSONArray customerInfo = (JSONArray) jsonObject.get("customerInfo"); // JSONArray barCodeInfo = (JSONArray) jsonObject.get("barCodeInfo"); // ?? ResourceBundle bundle = CommonUtil.getMessageMappingResource("CEB_SN"); JSONObject result = new JSONObject(); //ORDER_CODEt_sn_order.ORDER_CODE?????? List orderDataList = commonManagerMapper.selectTableListByCol("T_SN_ORDER", "ORDER_CODE", orderInfo.get("orderCode"), null, null); if (orderDataList.size() > 0) { result.put("success", "true"); result.put("errorCode", ""); result.put("errorMsg", ""); result.put("orderCode", orderInfo.get("orderCode")); return result.toString(); } JSONArray resultArray = new JSONArray(); // T_SN_RECEIPT Map primary = new HashMap(); primary.put("primaryId", null); Map data = new HashMap(); for (Object key : orderInfo.keySet()) { if (bundle.containsKey("SN_RECEIPT_" + key.toString().toUpperCase())) { if (orderInfo.get(key) == null || orderInfo.get(key).toString().isEmpty()) { data.put(bundle.getObject("SN_RECEIPT_" + key.toString().toUpperCase()), null); } else { data.put(bundle.getObject("SN_RECEIPT_" + key.toString().toUpperCase()), orderInfo.get(key)); } } } for (Object key : senderInfo.keySet()) { if (bundle.containsKey("SN_RECEIPT_" + key.toString().toUpperCase())) { if (senderInfo.get(key) == null || senderInfo.get(key).toString().isEmpty()) { data.put(bundle.getObject("SN_RECEIPT_" + key.toString().toUpperCase()), null); } else { data.put(bundle.getObject("SN_RECEIPT_" + key.toString().toUpperCase()), senderInfo.get(key)); } } } data.put("SENDER_CODE", orderInfo.get("ownerUserId")); data.put("RECEIPT_STATUS", "0"); data.put("CREAT_DATE", sf.format(new Date())); data.put("CREAT_TIME", new Date()); commonManagerMapper.insertTableByNVList("T_SN_RECEIPT", new ArrayList<String>(data.keySet()), new ArrayList<Object>(data.values()), primary); //???? List<Map> orderItemListForCJ = new ArrayList<Map>(); for (Iterator it = orderItemList.iterator(); it.hasNext();) { JSONObject item = (JSONObject) it.next(); Map primary_sub = new HashMap(); primary_sub.put("primaryId", null); Map dataSub = new HashMap(); Map dataSubForCJ = new HashMap(); for (Object key : item.keySet()) { if (bundle.containsKey("SN_RECEIPT_DETAIL_" + key.toString().toUpperCase())) { if (item.get(key) == null || item.get(key).toString().isEmpty()) { dataSub.put(bundle.getObject("SN_RECEIPT_DETAIL_" + key.toString().toUpperCase()), null); } else { dataSub.put(bundle.getObject("SN_RECEIPT_DETAIL_" + key.toString().toUpperCase()), item.get(key)); } } } // ? dataSub.put("RECEIPT_ID", primary.get("primaryId")); // data.put("CREAT_DATE", sf.format(new Date())); dataSub.put("CREAT_TIME", new Date()); commonManagerMapper.insertTableByNVList("T_SN_RECEIPT_DETAIL", new ArrayList<String>(dataSub.keySet()), new ArrayList<Object>(dataSub.values()), primary_sub); //????? dataSubForCJ.put("ORDER_ITEM_ID", dataSub.get("ORDER_ITEM_ID")); dataSubForCJ.put("SKU", dataSub.get("SKU")); dataSubForCJ.put("ITEM_NAME", dataSub.get("ITEM_NAME")); dataSubForCJ.put("EXPECT_QTY", dataSub.get("EXPECT_QTY")); dataSubForCJ.put("PRODUCE_CODE", dataSub.get("PRODUCE_CODE")); orderItemListForCJ.add(dataSubForCJ); } //???? Map order = new HashMap(); order.put("OWNER", data.get("OWNER")); order.put("ORDER_CODE", data.get("ORDER_CODE")); order.put("ORDER_TYPE", data.get("ORDER_TYPE")); order.put("EXPECT_START_TIME", data.get("EXPECT_START_TIME")); order.put("EXPECT_END_TIME", data.get("EXPECT_END_TIME")); order.put("TMS_ORDER_CODE", data.get("TMS_ORDER_CODE")); order.put("SENDER_ADDRESS", data.get("SENDER_ADDRESS")); order.put("SENDER_CODE", data.get("SENDER_CODE")); order.put("SENDER_NAME", data.get("SENDER_NAME")); order.put("SENDER_MOBILE", data.get("SENDER_MOBILE")); order.put("SENDER_PHONE", data.get("SENDER_PHONE")); order.put("CUST", CommonUtil.getSystemConfigProperty("CJ_cust")); String xmlStringData = XmlUtil.generalCommonXml_CJ("DATA", order, orderItemListForCJ); String requestXmlData = XmlUtil.generalSoapXml_CJ(xmlStringData, "sendInStockOrder"); System.out.println(requestXmlData); //?????? String response = HttpUtil.sendHttpCMD_CJ(requestXmlData, CommonUtil.getSystemConfigProperty("CJ_sendInStockOrder_requestUrl").toString()); //?? // String returnXmlData = XmlUtil // .getResponseFromXmlString_CJ(response); String returnXmlData = XmlUtil.getTotalMidValue(StringEscapeUtils.unescapeXml(response), "<ns:return>", "</ns:return>"); // // String returnXmlData = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><DATA><ORDER><ORDER_CODE>3434222e333</ORDER_CODE><CD>OK</CD><INFO><![CDATA[]]></INFO><ITEMS><ITEM><ORDER_ITEM_ID>420000002xxxxxx</ORDER_ITEM_ID><SKU>P0000KMM</SKU><ACTUAL_QTY>1</ACTUAL_QTY><ACTUAL_QTY_DEFECT>5590</ACTUAL_QTY_DEFECT></ITEM><ITEM><ORDER_ITEM_ID>1234567891</ORDER_ITEM_ID><SKU>P0001KMM</SKU><ACTUAL_QTY>1</ACTUAL_QTY><ACTUAL_QTY_DEFECT>5591</ACTUAL_QTY_DEFECT></ITEM></ITEMS></ORDER></DATA>"; // // String returnXmlData = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><DATA><ORDER><ORDER_CODE>W107xxxxxx</ORDER_CODE><CD>102</CD><INFO>??</INFO></ORDER></DATA>"; //? // Map orderResult = XmlUtil.parseXmlFPAPI_SingleNodes(returnXmlData, "//DATA/ORDER/child::*"); // if (orderResult.containsKey("CD") && "OK".equals(orderResult.get("CD").toString())) { //?? result.put("success", "true"); result.put("errorCode", ""); result.put("errorMsg", ""); } else { // //????B0007,? result.put("success", "false"); result.put("errorCode", "B0007"); result.put("errorMsg", orderResult.get("INFO")); } result.put("orderCode", orderResult.get("ORDER_CODE")); return result.toString(); }
From source file:com.foo.manager.commonManager.thread.HttpHandleThread.java
private JSONObject send2CJ_deliverGoodsNotify(JSONObject orderInfo, JSONObject receiverInfo, JSONArray orderItemList) throws CommonException { // ????// w ww .j av a2s . c om List<Map> orderItemListForCJ = new ArrayList<Map>(); Map dataSubForCJ = new HashMap(); for (Iterator it = orderItemList.iterator(); it.hasNext();) { JSONObject item = (JSONObject) it.next(); // ????? dataSubForCJ.put("ORDER_ITEM_ID", item.get("orderItemId")); dataSubForCJ.put("SKU", item.get("itemId")); dataSubForCJ.put("ITEM_NAME", item.get("itemName")); dataSubForCJ.put("QTY", item.get("itemQuantity")); orderItemListForCJ.add(dataSubForCJ); } String skuString = ""; if (orderItemListForCJ.size() > 0 && orderItemListForCJ.get(0).get("SKU") != null) { skuString = orderItemListForCJ.get(0).get("SKU").toString(); } List<Map<String, Object>> skuList = commonManagerMapper.selectTableListByCol("t_sn_sku", "SKU", skuString, null, null); Map<String, Object> sku = null; if (skuList != null && skuList.size() > 0) { sku = skuList.get(0); } // ????? Map order = new HashMap(); order.put("OWNER", orderInfo.get("ownerUserId")); order.put("ORDER_CODE", orderInfo.get("bol")); order.put("ORDER_TYPE", orderInfo.get("orderType")); order.put("CUSTOMS_MODE", orderInfo.get("customsMode")); order.put("TOTAL_WEIGHT", orderInfo.get("totalWeight")); order.put("WAY_BILLS", orderInfo.get("wayBills")); order.put("PAY_AMOUNT", orderInfo.get("payAmount")); order.put("DEST_CODE", orderInfo.get("destcode")); // ???sku?t_sn_sku.GRP order.put("CMMDTY_GRP", sku != null ? sku.get("GRP") : ""); order.put("TMS_ORDER_CODE", orderInfo.get("tmsOrderCode")); order.put("RECEIVER_ADDRESS", receiverInfo.get("receiverAddress")); order.put("RECEIVER_NAME", receiverInfo.get("receiverName")); order.put("RECEIVER_MOBILE", receiverInfo.get("receiverMobile")); order.put("RECEIVER_PHONE", receiverInfo.get("receiverPhone")); order.put("CUST", CommonUtil.getSystemConfigProperty("CJ_cust")); String xmlStringData = XmlUtil.generalCommonXml_CJ("DATA", order, orderItemListForCJ); String requestXmlData = XmlUtil.generalSoapXml_CJ(xmlStringData, "sendOutStockOrder"); System.out.println(requestXmlData); // ?????? String response = HttpUtil.sendHttpCMD_CJ(requestXmlData, CommonUtil.getSystemConfigProperty("CJ_sendOrder_requestUrl").toString()); // ?? // String returnXmlData = XmlUtil // .getResponseFromXmlString_CJ(response); String returnXmlData = XmlUtil.getTotalMidValue(StringEscapeUtils.unescapeXml(response), "<ns:return>", "</ns:return>"); // // String returnXmlData = // "<?xml version=\"1.0\" encoding=\"UTF-8\"?><DATA><ORDER><ORDER_CODE>3434222e333</ORDER_CODE><CD>OK</CD><INFO><![CDATA[]]></INFO><ITEMS><ITEM><ORDER_ITEM_ID>420000002xxxxxx</ORDER_ITEM_ID><SKU>P0000KMM</SKU><ACTUAL_QTY>1</ACTUAL_QTY><ACTUAL_QTY_DEFECT>5590</ACTUAL_QTY_DEFECT></ITEM><ITEM><ORDER_ITEM_ID>1234567891</ORDER_ITEM_ID><SKU>P0001KMM</SKU><ACTUAL_QTY>1</ACTUAL_QTY><ACTUAL_QTY_DEFECT>5591</ACTUAL_QTY_DEFECT></ITEM></ITEMS></ORDER></DATA>"; // // String returnXmlData = // "<?xml version=\"1.0\" encoding=\"UTF-8\"?><DATA><ORDER><ORDER_CODE>W107xxxxxx</ORDER_CODE><CD>102</CD><INFO>??</INFO></ORDER></DATA>"; // ? // Map orderResult = XmlUtil.parseXmlFPAPI_SingleNodes(returnXmlData, "//DATA/ORDER/child::*"); JSONObject result = new JSONObject(); // if (orderResult.containsKey("CD") && "OK".equals(orderResult.get("CD").toString())) { // ?? result.put("success", "true"); result.put("errorCode", ""); result.put("errorMsg", ""); } else { // // ????B0007,? result.put("success", "false"); result.put("errorCode", "B0007"); result.put("errorMsg", orderResult.get("INFO")); } result.put("orderCode", orderResult.get("ORDER_CODE")); return result; }