List of usage examples for javax.xml.parsers SAXParser parse
public void parse(InputSource is, DefaultHandler dh) throws SAXException, IOException
From source file:cm.aptoide.pt.MainActivity.java
@Override protected void onCreate(Bundle savedInstanceState) { AptoideThemePicker.setAptoideTheme(this); super.onCreate(savedInstanceState); serviceDownloadManagerIntent = new Intent(this, ServiceDownloadManager.class); startService(serviceDownloadManagerIntent); mContext = this; File sdcard_file = new File(SDCARD); if (!sdcard_file.exists() || !sdcard_file.canWrite()) { View simpleView = LayoutInflater.from(mContext).inflate(R.layout.dialog_simple_layout, null); Builder dialogBuilder = new AlertDialog.Builder(mContext).setView(simpleView); final AlertDialog noSDDialog = dialogBuilder.create(); noSDDialog.setTitle(getText(R.string.remote_in_noSD_title)); noSDDialog.setIcon(android.R.drawable.ic_dialog_alert); TextView message = (TextView) simpleView.findViewById(R.id.dialog_message); message.setText(getText(R.string.remote_in_noSD)); noSDDialog.setCancelable(false); noSDDialog.setButton(Dialog.BUTTON_NEUTRAL, getString(android.R.string.ok), new Dialog.OnClickListener() { @Override// w w w . j a va 2 s .c om public void onClick(DialogInterface arg0, int arg1) { finish(); } }); noSDDialog.show(); } else { StatFs stat = new StatFs(sdcard_file.getPath()); long blockSize = stat.getBlockSize(); long totalBlocks = stat.getBlockCount(); long availableBlocks = stat.getAvailableBlocks(); long total = (blockSize * totalBlocks) / 1024 / 1024; long avail = (blockSize * availableBlocks) / 1024 / 1024; Log.d("Aptoide", "* * * * * * * * * *"); Log.d("Aptoide", "Total: " + total + " Mb"); Log.d("Aptoide", "Available: " + avail + " Mb"); if (avail < 10) { Log.d("Aptoide", "No space left on SDCARD..."); Log.d("Aptoide", "* * * * * * * * * *"); View simpleView = LayoutInflater.from(this).inflate(R.layout.dialog_simple_layout, null); Builder dialogBuilder = new AlertDialog.Builder(this).setView(simpleView); final AlertDialog noSpaceDialog = dialogBuilder.create(); noSpaceDialog.setIcon(android.R.drawable.ic_dialog_alert); TextView message = (TextView) simpleView.findViewById(R.id.dialog_message); message.setText(getText(R.string.remote_in_noSDspace)); noSpaceDialog.setButton(Dialog.BUTTON_NEUTRAL, getText(android.R.string.ok), new Dialog.OnClickListener() { @Override public void onClick(DialogInterface arg0, int arg1) { finish(); } }); noSpaceDialog.show(); } else { SharedPreferences sPref = PreferenceManager.getDefaultSharedPreferences(mContext); editor = PreferenceManager.getDefaultSharedPreferences(mContext).edit(); if (!sPref.contains("matureChkBox")) { editor.putBoolean("matureChkBox", ApplicationAptoide.MATURECONTENTSWITCHVALUE); SharedPreferences sPrefOld = getSharedPreferences("aptoide_prefs", MODE_PRIVATE); if (sPrefOld.getString("app_rating", "none").equals("Mature")) { editor.putBoolean("matureChkBox", false); } } if (!sPref.contains("version")) { ApplicationAptoide.setRestartLauncher(true); try { editor.putInt("version", getPackageManager().getPackageInfo(getPackageName(), 0).versionCode); } catch (NameNotFoundException e) { e.printStackTrace(); } } if (sPref.getString("myId", null) == null) { String rand_id = UUID.randomUUID().toString(); editor.putString("myId", rand_id); } if (sPref.getInt("scW", 0) == 0 || sPref.getInt("scH", 0) == 0) { DisplayMetrics dm = new DisplayMetrics(); getWindowManager().getDefaultDisplay().getMetrics(dm); editor.putInt("scW", dm.widthPixels); editor.putInt("scH", dm.heightPixels); } editor.commit(); File file = new File(LOCAL_PATH + "/apks"); if (!file.exists()) { file.mkdirs(); } new Thread(new Runnable() { @Override public void run() { // Note the L that tells the compiler to interpret the // number as a Long final long MAXFILEAGE = 2678400000L; // 1 month in // milliseconds // Get file handle to the directory. In this case the // application files dir File dir = new File(LOCAL_PATH + "/apks"); // Optain list of files in the directory. // listFiles() returns a list of File objects to each // file found. File[] files = dir.listFiles(); // Loop through all files for (File f : files) { // Get the last modified date. Miliseconds since // 1970 long lastmodified = f.lastModified(); // Do stuff here to deal with the file.. // For instance delete files older than 1 month if (lastmodified + MAXFILEAGE < System.currentTimeMillis()) { f.delete(); } } } }).start(); db = Database.getInstance(); Intent i = new Intent(mContext, MainService.class); startService(i); bindService(i, conn, Context.BIND_AUTO_CREATE); order = Order.values()[PreferenceManager.getDefaultSharedPreferences(mContext).getInt("order_list", 0)]; registerReceiver(updatesReceiver, new IntentFilter("update")); registerReceiver(statusReceiver, new IntentFilter("status")); registerReceiver(loginReceiver, new IntentFilter("login")); registerReceiver(storePasswordReceiver, new IntentFilter("401")); registerReceiver(redrawInstalledReceiver, new IntentFilter("pt.caixamagica.aptoide.REDRAW")); if (!ApplicationAptoide.MULTIPLESTORES) { registerReceiver(parseFailedReceiver, new IntentFilter("PARSE_FAILED")); } registerReceiver(newRepoReceiver, new IntentFilter("pt.caixamagica.aptoide.NEWREPO")); registered = true; categoriesStrings = new HashMap<String, Integer>(); // categoriesStrings.put("Applications", R.string.applications); boolean serversFileIsEmpty = true; if (sPref.getBoolean("firstrun", true)) { // Intent shortcutIntent = new Intent(Intent.ACTION_MAIN); // shortcutIntent.setClassName("cm.aptoide.pt", // "cm.aptoide.pt.Start"); // final Intent intent = new Intent(); // intent.putExtra(Intent.EXTRA_SHORTCUT_INTENT, // shortcutIntent); // // intent.putExtra(Intent.EXTRA_SHORTCUT_NAME, // getString(R.string.app_name)); // Parcelable iconResource = // Intent.ShortcutIconResource.fromContext(this, // R.drawable.ic_launcher); // // intent.putExtra(Intent.EXTRA_SHORTCUT_ICON_RESOURCE, // iconResource); // intent.putExtra("duplicate", false); // intent.setAction("com.android.launcher.action.INSTALL_SHORTCUT"); // sendBroadcast(intent); if (new File(LOCAL_PATH + "/servers.xml").exists() && ApplicationAptoide.DEFAULTSTORENAME == null) { try { SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser sp = spf.newSAXParser(); MyappHandler handler = new MyappHandler(); sp.parse(new File(LOCAL_PATH + "/servers.xml"), handler); ArrayList<String> server = handler.getServers(); if (server.isEmpty()) { serversFileIsEmpty = true; } else { getIntent().putExtra("newrepo", server); } } catch (Exception e) { e.printStackTrace(); } } editor.putBoolean("firstrun", false); editor.putBoolean("orderByCategory", true); editor.commit(); } if (getIntent().hasExtra("newrepo")) { ArrayList<String> repos = (ArrayList<String>) getIntent().getSerializableExtra("newrepo"); for (final String uri2 : repos) { View simpleView = LayoutInflater.from(mContext).inflate(R.layout.dialog_simple_layout, null); Builder dialogBuilder = new AlertDialog.Builder(mContext).setView(simpleView); final AlertDialog addNewRepoDialog = dialogBuilder.create(); addNewRepoDialog.setTitle(getString(R.string.add_store)); addNewRepoDialog.setIcon(android.R.drawable.ic_menu_add); TextView message = (TextView) simpleView.findViewById(R.id.dialog_message); message.setText((getString(R.string.newrepo_alrt) + uri2 + " ?")); addNewRepoDialog.setCancelable(false); addNewRepoDialog.setButton(Dialog.BUTTON_POSITIVE, getString(android.R.string.yes), new Dialog.OnClickListener() { @Override public void onClick(DialogInterface arg0, int arg1) { dialogAddStore(uri2, null, null); } }); addNewRepoDialog.setButton(Dialog.BUTTON_NEGATIVE, getString(android.R.string.no), new Dialog.OnClickListener() { @Override public void onClick(DialogInterface dialog, int arg1) { dialog.cancel(); } }); addNewRepoDialog.show(); } } else if (db.getStores(false).getCount() == 0 && ApplicationAptoide.DEFAULTSTORENAME == null && serversFileIsEmpty) { View simpleView = LayoutInflater.from(mContext).inflate(R.layout.dialog_simple_layout, null); Builder dialogBuilder = new AlertDialog.Builder(mContext).setView(simpleView); final AlertDialog addAppsRepoDialog = dialogBuilder.create(); addAppsRepoDialog.setTitle(getString(R.string.add_store)); addAppsRepoDialog.setIcon(android.R.drawable.ic_menu_add); TextView message = (TextView) simpleView.findViewById(R.id.dialog_message); message.setText(getString(R.string.myrepo_alrt) + "\n" + "http://apps.store.aptoide.com/"); addAppsRepoDialog.setCancelable(false); addAppsRepoDialog.setButton(Dialog.BUTTON_POSITIVE, getString(android.R.string.yes), new Dialog.OnClickListener() { @Override public void onClick(DialogInterface arg0, int arg1) { dialogAddStore("http://apps.store.aptoide.com", null, null); } }); addAppsRepoDialog.setButton(Dialog.BUTTON_NEGATIVE, getString(android.R.string.no), new Dialog.OnClickListener() { @Override public void onClick(DialogInterface dialog, int arg1) { dialog.cancel(); } }); addAppsRepoDialog.show(); } new Thread(new Runnable() { @Override public void run() { try { getUpdateParameters(); if (getPackageManager().getPackageInfo(getPackageName(), 0).versionCode < Integer .parseInt(updateParams.get("versionCode"))) { runOnUiThread(new Runnable() { @Override public void run() { requestUpdateSelf(); } }); } } catch (Exception e) { e.printStackTrace(); } } }).start(); } featuredView = LayoutInflater.from(mContext).inflate(R.layout.page_featured, null); availableView = LayoutInflater.from(mContext).inflate(R.layout.page_available, null); updateView = LayoutInflater.from(mContext).inflate(R.layout.page_updates, null); banner = (LinearLayout) availableView.findViewById(R.id.banner); breadcrumbs = (LinearLayout) LayoutInflater.from(mContext).inflate(R.layout.breadcrumb_container, null); installedView = new ListView(mContext); updatesListView = (ListView) updateView.findViewById(R.id.updates_list); availableListView = (ListView) availableView.findViewById(R.id.available_list); joinStores = (CheckBox) availableView.findViewById(R.id.join_stores); availableAdapter = new AvailableListAdapter(mContext, null, CursorAdapter.FLAG_REGISTER_CONTENT_OBSERVER); installedAdapter = new InstalledAdapter(mContext, null, CursorAdapter.FLAG_REGISTER_CONTENT_OBSERVER, db); updatesAdapter = new UpdatesAdapter(mContext, null, CursorAdapter.FLAG_REGISTER_CONTENT_OBSERVER); pb = (TextView) availableView.findViewById(R.id.loading_pb); addStoreButton = availableView.findViewById(R.id.add_store); bannerStoreAvatar = (ImageView) banner.findViewById(R.id.banner_store_avatar); bannerStoreName = (TextView) banner.findViewById(R.id.banner_store_name); bannerStoreDescription = (AutoScaleTextView) banner.findViewById(R.id.banner_store_description); } }
From source file:ee.sk.digidoc.factory.SAXDigiDocFactory.java
/** * Reads in a DigiDoc file. One of fname or isSdoc must be given. * @param fname signed doc filename// w w w.j a va 2 s. co m * @param isSdoc opened stream with DigiDoc data * The user must open and close it. * @param errs list of errors to fill with parsing errors. If given * then attempt is made to continue parsing on errors and return them in this list. * If not given (null) then the first error found will be thrown. * @return signed document object if successfully parsed */ private SignedDoc readSignedDocOfType(String fname, InputStream isSdoc, boolean isBdoc, List errs) throws DigiDocException { // Use an instance of ourselves as the SAX event handler SAXDigiDocFactory handler = this; m_errs = errs; DigiDocVerifyFactory.initProvider(); SAXParserFactory factory = SAXParserFactory.newInstance(); if (m_logger.isDebugEnabled()) m_logger.debug("Start reading ddoc/bdoc " + ((fname != null) ? "from file: " + fname : "from stream") + " bdoc: " + isBdoc); if (fname == null && isSdoc == null) { throw new DigiDocException(DigiDocException.ERR_READ_FILE, "No input file", null); } if (fname != null) { File inFile = new File(fname); if (!inFile.canRead() || inFile.length() == 0) { throw new DigiDocException(DigiDocException.ERR_READ_FILE, "Empty or unreadable input file", null); } } ZipFile zf = null; ZipArchiveInputStream zis = null; ZipArchiveEntry ze = null; InputStream isEntry = null; File fTmp = null; try { factory.setFeature("http://xml.org/sax/features/external-general-entities", false); factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); if (isBdoc) { // bdoc parsing // must be a bdoc document ? m_doc = new SignedDoc(); m_doc.setVersion(SignedDoc.BDOC_VERSION_1_0); m_doc.setFormat(SignedDoc.FORMAT_BDOC); Enumeration eFiles = null; if (fname != null) { zf = new ZipFile(fname, "UTF-8"); eFiles = zf.getEntries(); } else if (isSdoc != null) { zis = new ZipArchiveInputStream(isSdoc, "UTF-8", true, true); } ArrayList lSigFnames = new ArrayList(); ArrayList lDataFnames = new ArrayList(); // read all entries boolean bHasMimetype = false, bManifest1 = false; int nFil = 0; while ((zf != null && eFiles.hasMoreElements()) || (zis != null && ((ze = zis.getNextZipEntry()) != null))) { nFil++; // read entry if (zf != null) { // ZipFile ze = (ZipArchiveEntry) eFiles.nextElement(); isEntry = zf.getInputStream(ze); } else { // ZipArchiveInputStream int n = 0, nTot = 0; if ((ze.getName().equals(FILE_MIMETYPE) || ze.getName().equals(FILE_MANIFEST) || (ze.getName().startsWith(FILE_SIGNATURES) && ze.getName().endsWith(".xml"))) || (nMaxBdocFilCached <= 0 || (ze.getSize() < nMaxBdocFilCached && ze.getSize() >= 0))) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); byte[] data = new byte[2048]; while ((n = zis.read(data)) > 0) { bos.write(data, 0, n); nTot += n; } if (m_logger.isDebugEnabled()) m_logger.debug("Read: " + nTot + " bytes from zip"); data = bos.toByteArray(); bos = null; isEntry = new ByteArrayInputStream(data); } else { File fCacheDir = new File(ConfigManager.instance().getStringProperty( "DIGIDOC_DF_CACHE_DIR", System.getProperty("java.io.tmpdir"))); fTmp = File.createTempFile("bdoc-data", ".tmp", fCacheDir); FileOutputStream fos = new FileOutputStream(fTmp); byte[] data = new byte[2048]; while ((n = zis.read(data)) > 0) { fos.write(data, 0, n); nTot += n; } if (m_logger.isDebugEnabled()) m_logger.debug("Read: " + nTot + " bytes from zip to: " + fTmp.getAbsolutePath()); fos.close(); isEntry = new FileInputStream(fTmp); } } if (m_logger.isDebugEnabled()) m_logger.debug("Entry: " + ze.getName() + " nlen: " + ze.getName().length() + " size: " + ze.getSize() + " dir: " + ze.isDirectory() + " comp-size: " + ze.getCompressedSize()); // mimetype file if (ze.getName().equals(FILE_MIMETYPE)) { if (m_logger.isDebugEnabled()) m_logger.debug("Check mimetype!"); checkBdocMimetype(isEntry); bHasMimetype = true; m_doc.setComment(ze.getComment()); if (nFil != 1) { m_logger.error("mimetype file is " + nFil + " file but must be first"); handleError(new DigiDocException(DigiDocException.ERR_DIGIDOC_BADXML, "mimetype file is not first zip entry", null)); } } else if (ze.getName().equals(FILE_MANIFEST)) { // manifest.xml file if (m_logger.isDebugEnabled()) m_logger.debug("Read manifest"); if (!bManifest1 && isEntry != null) { bManifest1 = true; BdocManifestParser mfparser = new BdocManifestParser(m_doc); mfparser.readManifest(isEntry); } else { m_logger.error("Found multiple manifest.xml files!"); throw new DigiDocException(DigiDocException.ERR_MULTIPLE_MANIFEST_FILES, "Found multiple manifest.xml files!", null); } } else if (ze.getName().startsWith(FILE_SIGNATURES) && ze.getName().endsWith(".xml")) { // some signature m_fileName = ze.getName(); if (m_logger.isDebugEnabled()) m_logger.debug("Reading bdoc siganture: " + m_fileName); boolean bExists = false; for (int j = 0; j < lSigFnames.size(); j++) { String s1 = (String) lSigFnames.get(j); if (s1.equals(m_fileName)) bExists = true; } if (bExists) { m_logger.error("Duplicate signature filename: " + m_fileName); handleError(new DigiDocException(DigiDocException.ERR_DIGIDOC_BADXML, "Duplicate signature filename: " + m_fileName, null)); } else lSigFnames.add(m_fileName); SAXParser saxParser = factory.newSAXParser(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); int n = 0; byte[] data = new byte[2048]; while ((n = isEntry.read(data)) > 0) bos.write(data, 0, n); data = bos.toByteArray(); bos = null; if (m_logger.isDebugEnabled()) m_logger.debug( "Parsing bdoc: " + m_fileName + " size: " + ((data != null) ? data.length : 0)); saxParser.parse(new SignatureInputStream(new ByteArrayInputStream(data)), this); if (m_logger.isDebugEnabled()) m_logger.debug("Parsed bdoc: " + m_fileName); Signature sig1 = m_doc.getLastSignature(); m_sigComment = ze.getComment(); if (sig1 != null) { sig1.setPath(m_fileName); sig1.setComment(ze.getComment()); } } else { // probably a data file if (m_logger.isDebugEnabled()) m_logger.debug("Read data file: " + ze.getName()); if (!ze.isDirectory()) { boolean bExists = false; for (int j = 0; j < lDataFnames.size(); j++) { String s1 = (String) lDataFnames.get(j); if (s1.equals(ze.getName())) bExists = true; } if (bExists) { m_logger.error("Duplicate datafile filename: " + ze.getName()); handleError(new DigiDocException(DigiDocException.ERR_DIGIDOC_BADXML, "Duplicate datafile filename: " + ze.getName(), null)); } else lDataFnames.add(ze.getName()); DataFile df = m_doc.findDataFileById(ze.getName()); if (df != null) { if (ze.getSize() > 0) df.setSize(ze.getSize()); df.setContentType(DataFile.CONTENT_BINARY); df.setFileName(ze.getName()); } else { df = new DataFile(ze.getName(), DataFile.CONTENT_BINARY, ze.getName(), "application/binary", m_doc); if (m_doc.getDataFiles() == null) m_doc.setDataFiles(new ArrayList()); m_doc.getDataFiles().add(df); //m_doc.addDataFile(df); // this does some intiailization work unnecessary here } // enable caching if requested if (isEntry != null) df.setOrCacheBodyAndCalcHashes(isEntry); df.setComment(ze.getComment()); df.setLastModDt(new Date(ze.getTime())); // fix mime type according to DataObjectFormat Signature sig1 = m_doc.getLastSignature(); if (sig1 != null) { Reference dRef = sig1.getSignedInfo().getReferenceForDataFile(df); if (dRef != null) { DataObjectFormat dof = sig1.getSignedInfo() .getDataObjectFormatForReference(dRef); if (dof != null) { df.setMimeType(dof.getMimeType()); } } } } } if (fTmp != null) { fTmp.delete(); fTmp = null; } } // while zip entries if (!bHasMimetype) { m_logger.error("No mimetype file"); handleError(new DigiDocException(DigiDocException.ERR_DIGIDOC_BADXML, "Not a BDOC format file! No mimetype file!", null)); } // if no signatures exist then copy mime-type from manifest.xml to DataFile -s if (m_doc.countSignatures() == 0) { for (int i = 0; i < m_doc.countDataFiles(); i++) { DataFile df = m_doc.getDataFile(i); if (m_doc.getManifest() != null) { for (int j = 0; j < m_doc.getManifest().getNumFileEntries(); j++) { ManifestFileEntry mfe = m_doc.getManifest().getFileEntry(j); if (mfe.getFullPath() != null && mfe.getFullPath().equals(df.getFileName())) { df.setMimeType(mfe.getMediaType()); } // if fullpath } // for } // if } // for i } } else { // ddoc parsing if (m_logger.isDebugEnabled()) m_logger.debug("Reading ddoc: " + fname + " file: " + m_fileName); m_fileName = fname; SAXParser saxParser = factory.newSAXParser(); if (fname != null) saxParser.parse(new SignatureInputStream(new FileInputStream(fname)), this); else if (isSdoc != null) saxParser.parse(isSdoc, this); } } catch (org.xml.sax.SAXParseException ex) { m_logger.error("SAX Error: " + ex); handleError(ex); } catch (Exception ex) { m_logger.error("Error reading3: " + ex); ex.printStackTrace(); /*if(ex instanceof DigiDocException){ DigiDocException dex = (DigiDocException)ex; m_logger.error("Dex: " + ex); if(dex.getNestedException() != null) { dex.getNestedException().printStackTrace(); m_logger.error("Trace: "); } }*/ handleError(ex); } finally { // cleanup try { if (isEntry != null) { isEntry.close(); isEntry = null; } if (zis != null) zis.close(); if (zf != null) zf.close(); if (fTmp != null) { fTmp.delete(); fTmp = null; } } catch (Exception ex) { m_logger.error("Error closing streams and files: " + ex); } } // compare Manifest and DataFiles boolean bErrList = (errs != null); if (errs == null) errs = new ArrayList(); boolean bOk = DigiDocVerifyFactory.verifyManifestEntries(m_doc, errs); if (m_doc == null) { m_logger.error("Error reading4: doc == null"); handleError(new DigiDocException(DigiDocException.ERR_DIGIDOC_BADXML, "This document is not in ddoc or bdoc format", null)); } if (!bErrList && errs.size() > 0) { // if error list was not used then we have to throw exception. So we will throw the first one since we can only do it once DigiDocException ex = (DigiDocException) errs.get(0); throw ex; } return m_doc; }
From source file:io.dataapps.chlorine.mask.MaskFactory.java
public MaskFactory(FinderEngine fEngine, InputStream in) { engine = fEngine;//from ww w.j a v a2 s .c o m try { SAXParserFactory factory = SAXParserFactory.newInstance(); SAXParser saxParser = factory.newSAXParser(); DefaultHandler handler = new DefaultHandler() { boolean bName = false; boolean bClass = false; boolean bConfiguration = false; boolean bDefault = false; String name = ""; String className = ""; String configuration = ""; String defaultStr; public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if (qName.equalsIgnoreCase("NAME")) { bName = true; } else if (qName.equalsIgnoreCase("CLASS")) { bClass = true; } else if (qName.equalsIgnoreCase("CONFIGURATION")) { bConfiguration = true; } else if (qName.equalsIgnoreCase("DEFAULT")) { bDefault = true; } } public void endElement(String uri, String localName, String qName) throws SAXException { if (qName.equalsIgnoreCase("NAME")) { bName = false; name = name.trim(); } else if (qName.equalsIgnoreCase("CLASS")) { bClass = false; className = className.trim(); } else if (qName.equalsIgnoreCase("CONFIGURATION")) { bConfiguration = false; configuration = configuration.trim(); } else if (qName.equalsIgnoreCase("DEFAULT")) { bDefault = false; defaultMasker = defaultStr.trim(); } else if (qName.equalsIgnoreCase("MASKER")) { if (!name.isEmpty() && !className.isEmpty() && !configuration.isEmpty()) { try { Class<?> klass = Thread.currentThread().getContextClassLoader() .loadClass(className); Masker masker = (Masker) klass.newInstance(); masker.init(engine, configuration); maskers.put(name, masker); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { LOG.error(e); } } else { if (name.isEmpty()) { LOG.error("The name for a masker cannot be empty"); } if (className.isEmpty()) { LOG.error("The class name for a masker cannot be empty"); } if (configuration.isEmpty()) { LOG.error("The configuration for a masker cannot be empty"); } } name = ""; configuration = ""; className = ""; defaultStr = ""; } } public void characters(char ch[], int start, int length) throws SAXException { if (bName) { name += new String(ch, start, length); } else if (bClass) { className += new String(ch, start, length); } else if (bConfiguration) { configuration += new String(ch, start, length); } else if (bDefault) { defaultStr += new String(ch, start, length); } } }; saxParser.parse(in, handler); } catch (Exception e) { LOG.error(e); } }
From source file:cm.aptoide.pt.MainActivity.java
private void loadRecommended() { if (Login.isLoggedIn(mContext)) { ((TextView) featuredView.findViewById(R.id.recommended_text)).setVisibility(View.GONE); } else {/*from ww w . j a va 2 s. com*/ ((TextView) featuredView.findViewById(R.id.recommended_text)).setVisibility(View.VISIBLE); } new Thread(new Runnable() { private ArrayList<HashMap<String, String>> valuesRecommended; public void run() { loadUIRecommendedApps(); File f = null; try { SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser sp = spf.newSAXParser(); NetworkUtils utils = new NetworkUtils(); BufferedInputStream bis = new BufferedInputStream( utils.getInputStream("http://webservices.aptoide.com/webservices/listUserBasedApks/" + Login.getToken(mContext) + "/10/xml", null, null, mContext), 8 * 1024); f = File.createTempFile("abc", "abc"); OutputStream out = new FileOutputStream(f); byte buf[] = new byte[1024]; int len; while ((len = bis.read(buf)) > 0) out.write(buf, 0, len); out.close(); bis.close(); String hash = Md5Handler.md5Calc(f); ViewApk parent_apk = new ViewApk(); parent_apk.setApkid("recommended"); if (!hash.equals(db.getItemBasedApksHash(parent_apk.getApkid()))) { // Database.database.beginTransaction(); db.deleteItemBasedApks(parent_apk); sp.parse(f, new HandlerItemBased(parent_apk)); db.insertItemBasedApkHash(hash, parent_apk.getApkid()); // Database.database.setTransactionSuccessful(); // Database.database.endTransaction(); loadUIRecommendedApps(); } } catch (Exception e) { e.printStackTrace(); } if (f != null) f.delete(); } private void loadUIRecommendedApps() { valuesRecommended = db.getItemBasedApksRecommended("recommended"); runOnUiThread(new Runnable() { public void run() { LinearLayout ll = (LinearLayout) featuredView.findViewById(R.id.recommended_container); ll.removeAllViews(); LinearLayout llAlso = new LinearLayout(MainActivity.this); llAlso.setLayoutParams(new LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, LinearLayout.LayoutParams.WRAP_CONTENT)); llAlso.setOrientation(LinearLayout.HORIZONTAL); if (valuesRecommended.isEmpty()) { if (Login.isLoggedIn(mContext)) { TextView tv = new TextView(mContext); tv.setText(R.string.no_recommended_apps); tv.setTextAppearance(mContext, android.R.attr.textAppearanceMedium); tv.setPadding(10, 10, 10, 10); ll.addView(tv); } } else { for (int i = 0; i != valuesRecommended.size(); i++) { LinearLayout txtSamItem = (LinearLayout) getLayoutInflater() .inflate(R.layout.row_grid_item, null); ((TextView) txtSamItem.findViewById(R.id.name)) .setText(valuesRecommended.get(i).get("name")); ImageLoader.getInstance().displayImage(valuesRecommended.get(i).get("icon"), (ImageView) txtSamItem.findViewById(R.id.icon)); float stars = 0f; try { stars = Float.parseFloat(valuesRecommended.get(i).get("rating")); } catch (Exception e) { stars = 0f; } ((RatingBar) txtSamItem.findViewById(R.id.rating)).setIsIndicator(true); ((RatingBar) txtSamItem.findViewById(R.id.rating)).setRating(stars); txtSamItem.setPadding(10, 0, 0, 0); // ((TextView) // txtSamItem.findViewById(R.id.version)) // .setText(getString(R.string.version) +" "+ // valuesRecommended.get(i).get("vername")); ((TextView) txtSamItem.findViewById(R.id.downloads)) .setText("(" + valuesRecommended.get(i).get("downloads") + " " + getString(R.string.downloads) + ")"); txtSamItem.setTag(valuesRecommended.get(i).get("_id")); txtSamItem.setLayoutParams(new LinearLayout.LayoutParams( LinearLayout.LayoutParams.MATCH_PARENT, 100, 1)); // txtSamItem.setOnClickListener(featuredListener); txtSamItem.setOnClickListener(new OnClickListener() { @Override public void onClick(View arg0) { Intent i = new Intent(MainActivity.this, ApkInfo.class); long id = Long.parseLong((String) arg0.getTag()); i.putExtra("_id", id); i.putExtra("top", true); i.putExtra("category", Category.ITEMBASED.ordinal()); startActivity(i); } }); txtSamItem.measure(0, 0); if (i % 2 == 0) { ll.addView(llAlso); llAlso = new LinearLayout(MainActivity.this); llAlso.setLayoutParams(new LinearLayout.LayoutParams( LinearLayout.LayoutParams.MATCH_PARENT, 100)); llAlso.setOrientation(LinearLayout.HORIZONTAL); llAlso.addView(txtSamItem); } else { llAlso.addView(txtSamItem); } } ll.addView(llAlso); } } }); } }).start(); }
From source file:io.dataapps.chlorine.finder.DefaultFinderProvider.java
DefaultFinderProvider(InputStream in, final boolean ignoreEnabledFlag) { try {//from w w w.j av a 2 s . c om SAXParserFactory factory = SAXParserFactory.newInstance(); SAXParser saxParser = factory.newSAXParser(); DefaultHandler handler = new DefaultHandler() { boolean bName = false; boolean bPattern = false; boolean bClass = false; boolean bEnabled = false; boolean bFlags = false; String name = ""; String pattern = ""; String className = ""; String strFlags = ""; int flags = RegexFinder.DEFAULT_FLAGS; String strEnabled = ""; boolean enabled = true; public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if (qName.equalsIgnoreCase("NAME")) { bName = true; } else if (qName.equalsIgnoreCase("PATTERN")) { bPattern = true; } else if (qName.equalsIgnoreCase("CLASS")) { bClass = true; } else if (qName.equalsIgnoreCase("FLAGS")) { bFlags = true; } else if (qName.equalsIgnoreCase("ENABLED")) { bEnabled = true; } } public void endElement(String uri, String localName, String qName) throws SAXException { if (qName.equalsIgnoreCase("NAME")) { bName = false; name = name.trim(); } else if (qName.equalsIgnoreCase("PATTERN")) { bPattern = false; pattern = pattern.trim(); } else if (qName.equalsIgnoreCase("CLASS")) { bClass = false; className = className.trim(); } else if (qName.equalsIgnoreCase("FLAGS")) { bFlags = false; flags = Integer.parseInt(strFlags.trim()); } else if (qName.equalsIgnoreCase("ENABLED")) { bEnabled = false; enabled = Boolean.parseBoolean(strEnabled.trim()); } else if (qName.equalsIgnoreCase("FINDER")) { if (ignoreEnabledFlag || enabled) { if (!className.isEmpty()) { try { Class<?> klass = Thread.currentThread().getContextClassLoader() .loadClass(className); finders.add((Finder) klass.newInstance()); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { LOG.error(e); } } else if (flags != RegexFinder.DEFAULT_FLAGS) { finders.add(new RegexFinder(name, pattern, flags)); } else { finders.add(new RegexFinder(name, pattern)); } } name = ""; pattern = ""; className = ""; flags = RegexFinder.DEFAULT_FLAGS; strFlags = ""; enabled = true; strEnabled = ""; } } public void characters(char ch[], int start, int length) throws SAXException { if (bName) { name += new String(ch, start, length); } else if (bPattern) { pattern += new String(ch, start, length); } else if (bClass) { className += new String(ch, start, length); } else if (bEnabled) { strEnabled += new String(ch, start, length); } else if (bFlags) { strFlags += new String(ch, start, length); } } }; saxParser.parse(in, handler); } catch (Exception e) { LOG.error(e); } }
From source file:cm.aptoide.pt.MainActivity.java
private void loadFeatured() { new Thread(new Runnable() { public void run() { loadUIEditorsApps();/* www. j a va 2 s.co m*/ File f = null; try { SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser sp = spf.newSAXParser(); NetworkUtils utils = new NetworkUtils(); String url; f = File.createTempFile("tempFile", ""); String countryCode = Geolocation.getCountryCode(mContext); Log.d("Aptoide-Geolocation", "Using countrycode: " + countryCode); if (ApplicationAptoide.CUSTOMEDITORSCHOICE) { url = getEditorsChoiceURL(ApplicationAptoide.DEFAULTSTORENAME, countryCode); if (((HttpURLConnection) new URL(url).openConnection()).getResponseCode() != 200) { url = getEditorsChoiceURL("apps", countryCode); } } else { url = getEditorsChoiceURL("apps", countryCode); } long date = utils.getLastModified(new URL(url)); long cachedDate = db.getEditorsChoiceHash(); Log.d("Getting", "Date is " + date); Log.d("Getting", "CachedDate is " + cachedDate); if (cachedDate < date) { BufferedInputStream bis = new BufferedInputStream( utils.getInputStream(url, null, null, mContext), 8 * 1024); OutputStream out = new FileOutputStream(f); byte buf[] = new byte[1024]; int len; while ((len = bis.read(buf)) > 0) out.write(buf, 0, len); out.close(); bis.close(); Server server = new Server(); // Database.database.beginTransaction(); db.deleteEditorsChoice(); sp.parse(f, new HandlerEditorsChoice(server)); db.insertEditorsChoiceHash(date); // Database.database.setTransactionSuccessful(); // Database.database.endTransaction(); loadUIEditorsApps(); } } catch (SAXException e) { // Database.database.setTransactionSuccessful(); // Database.database.endTransaction(); } catch (Exception e) { e.printStackTrace(); } if (f != null) f.delete(); } private String getEditorsChoiceURL(String store, String countryCode) { if (countryCode.length() > 0) { return "http://" + store + ".store.aptoide.com/editors.xml?country=" + countryCode; } return "http://" + store + ".store.aptoide.com/editors.xml"; } }).start(); new Thread(new Runnable() { public void run() { loadUItopapps(); File f = null; try { SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser sp = spf.newSAXParser(); Server server = new Server(); server.id = 1; NetworkUtils utils = new NetworkUtils(); String url; if (ApplicationAptoide.CUSTOMEDITORSCHOICE) { url = "http://" + ApplicationAptoide.DEFAULTSTORENAME + ".store.aptoide.com/top.xml"; if (((HttpURLConnection) new URL(url).openConnection()).getResponseCode() != 200) { url = "http://apps.store.aptoide.com/top.xml"; } } else { url = "http://apps.store.aptoide.com/top.xml"; } long date = utils.getLastModified(new URL(url)); long cachedDate; try { cachedDate = Long.parseLong(db.getRepoHash(server.id, Category.TOPFEATURED)); } catch (Exception e) { cachedDate = 0; } if (cachedDate < date) { BufferedInputStream bis = new BufferedInputStream( utils.getInputStream(url, null, null, mContext), 8 * 1024); f = File.createTempFile("tempFile", ""); OutputStream out = new FileOutputStream(f); byte buf[] = new byte[1024]; int len; while ((len = bis.read(buf)) > 0) out.write(buf, 0, len); out.close(); bis.close(); // Database.database.beginTransaction(); db.deleteFeaturedTopApps(); sp.parse(f, new HandlerFeaturedTop(server)); db.insertFeaturedTopHash(date); loadUItopapps(); f.delete(); } } catch (ParserConfigurationException e) { e.printStackTrace(); } catch (SAXException e) { } catch (IOException e) { e.printStackTrace(); } if (f != null) f.delete(); } }).start(); }
From source file:org.hil.webservice.mobile.impl.ChildrenWebServiceImpl.java
private void parseXml(String xml) { list = new ArrayList<Children>(); tmpAuth = ""; tmpAuthor = ""; SAXParserFactory factory = SAXParserFactory.newInstance(); try {/* w w w .j a v a 2 s . c o m*/ SAXParser saxParser = factory.newSAXParser(); DefaultHandler handler = new DefaultHandler() { Children tempChild; String tempVal = ""; public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if (qName.equalsIgnoreCase("Children")) { tmpAuth = attributes.getValue("sessionAuth"); if (!tmpAuth.equalsIgnoreCase("sessionAuth")) return; tmpAuthor = attributes.getValue("author"); force = Boolean.parseBoolean(attributes.getValue("force")); } tempVal = ""; if (qName.equalsIgnoreCase("Child")) { tempChild = new Children(); } } public void endElement(String uri, String localName, String qName) throws SAXException { if (qName.equalsIgnoreCase("Child")) { //add it to the list list.add(tempChild); } else if (qName.equalsIgnoreCase("id")) { if (tempVal.length() > 0) tempChild.setId(Long.parseLong(tempVal)); } else if (qName.equalsIgnoreCase("fullName")) { if (tempVal.length() > 0) tempChild.setFullName(tempVal); } else if (qName.equalsIgnoreCase("dateOfBirth")) { if (tempVal.length() > 0) { SimpleDateFormat format = new SimpleDateFormat("dd/MM/yyyy"); try { Date bdate = format.parse(tempVal); tempChild.setDateOfBirth(bdate); } catch (ParseException e) { e.printStackTrace(); } } } else if (qName.equalsIgnoreCase("gender")) { if (tempVal.length() > 0) tempChild.setGender(Boolean.parseBoolean(tempVal)); } else if (qName.equalsIgnoreCase("childCode")) { if (tempVal.length() > 0) tempChild.setChildCode(tempVal); } else if (qName.equalsIgnoreCase("fatherName")) { if (tempVal.length() > 0) tempChild.setFatherName(tempVal); } else if (qName.equalsIgnoreCase("fatherBirthYear")) { if (tempVal.length() > 0) tempChild.setFatherBirthYear(Integer.parseInt(tempVal)); } else if (qName.equalsIgnoreCase("fatherID")) { if (tempVal.length() > 0) tempChild.setFatherID(tempVal); } else if (qName.equalsIgnoreCase("fatherMobile")) { if (tempVal.length() > 0) tempChild.setFatherMobile(tempVal); } else if (qName.equalsIgnoreCase("motherName")) { if (tempVal.length() > 0) tempChild.setMotherName(tempVal); } else if (qName.equalsIgnoreCase("motherBirthYear")) { if (tempVal.length() > 0) tempChild.setMotherBirthYear(Integer.parseInt(tempVal)); } else if (qName.equalsIgnoreCase("motherID")) { if (tempVal.length() > 0) tempChild.setMotherID(tempVal); } else if (qName.equalsIgnoreCase("motherMobile")) { if (tempVal.length() > 0) tempChild.setMotherMobile(tempVal); } else if (qName.equalsIgnoreCase("caretakerName")) { if (tempVal.length() > 0) tempChild.setCaretakerName(tempVal); } else if (qName.equalsIgnoreCase("caretakerBirthYear")) { if (tempVal.length() > 0) tempChild.setCaretakerBirthYear(Integer.parseInt(tempVal)); } else if (qName.equalsIgnoreCase("caretakerID")) { if (tempVal.length() > 0) tempChild.setCaretakerID(tempVal); } else if (qName.equalsIgnoreCase("caretakerMobile")) { if (tempVal.length() > 0) tempChild.setCaretakerMobile(tempVal); } else if (qName.equalsIgnoreCase("currentCaretaker")) { if (tempVal.length() > 0) tempChild.setCurrentCaretaker(Short.parseShort(tempVal)); } else if (qName.equalsIgnoreCase("villageId")) { if (tempVal.length() > 0) { tempChild.setVillage(villageDao.get(Long.parseLong(tempVal))); } } } public void characters(char ch[], int start, int length) throws SAXException { tempVal = new String(ch, start, length); } }; saxParser.parse(new InputSource(new StringReader(xml)), handler); } catch (ParserConfigurationException e) { e.printStackTrace(); } catch (SAXException e) { e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
From source file:it.unimi.di.big.mg4j.document.WikipediaDocumentSequence.java
@Override public DocumentIterator iterator() throws IOException { final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final MutableString nameSpaceAccumulator = new MutableString(); final ObjectOpenHashSet<MutableString> nameSpacesAccumulator = new ObjectOpenHashSet<MutableString>(); final ArrayBlockingQueue<DocumentFactory> freeFactories = new ArrayBlockingQueue<DocumentFactory>(16); for (int i = freeFactories.remainingCapacity(); i-- != 0;) freeFactories.add(this.factory.copy()); final ArrayBlockingQueue<DocumentAndFactory> readyDocumentsAndFactories = new ArrayBlockingQueue<DocumentAndFactory>( freeFactories.size());/* w w w . ja v a 2 s .co m*/ final SAXParser parser; try { parser = saxParserFactory.newSAXParser(); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } final DefaultHandler handler = new DefaultHandler() { private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); private boolean inText; private boolean inTitle; private boolean inId; private boolean inTimestamp; private boolean inNamespaceDef; private boolean redirect; private MutableString text = new MutableString(); private MutableString title = new MutableString(); private MutableString id = new MutableString(); private MutableString timestamp = new MutableString(); private final Reference2ObjectMap<Enum<?>, Object> metadata = new Reference2ObjectOpenHashMap<Enum<?>, Object>(); { metadata.put(PropertyBasedDocumentFactory.MetadataKeys.ENCODING, "UTF-8"); metadata.put(MetadataKeys.REDIRECT, redirectAnchors); } @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { redirect = inText = inTitle = inId = inTimestamp = false; text.length(0); title.length(0); id.length(0); timestamp.length(0); } else if ("text".equals(localName)) inText = true; else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first id/title elements. else if ("id".equals(localName) && id.length() == 0) inId = true; else if ("timestamp".equals(localName) && timestamp.length() == 0) inTimestamp = true; else if ("redirect".equals(localName)) { redirect = true; if (attributes.getValue("title") != null) // Accumulate the title of the page as virtual text of the redirect page. synchronized (redirectAnchors) { final String link = Encoder.encodeTitleToUrl(attributes.getValue("title"), true); redirectAnchors.add( new AnchorExtractor.Anchor(new MutableString(baseURL.length() + link.length()) .append(baseURL).append(link), title.copy())); } } else if ("namespace".equals(localName)) { // Found a new namespace inNamespaceDef = true; nameSpaceAccumulator.length(0); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("namespace".equals(localName)) { // Collecting a namespace if (nameSpaceAccumulator.length() != 0) nameSpacesAccumulator.add(nameSpaceAccumulator.copy().toLowerCase()); return; } if ("namespaces".equals(localName)) { // All namespaces collected nameSpaces = ImmutableSet.copyOf(nameSpacesAccumulator); return; } if (!redirect) { if ("title".equals(localName)) { // Set basic metadata for the page metadata.put(PropertyBasedDocumentFactory.MetadataKeys.TITLE, title.copy()); String link = Encoder.encodeTitleToUrl(title.toString(), true); metadata.put(PropertyBasedDocumentFactory.MetadataKeys.URI, new MutableString(baseURL.length() + link.length()).append(baseURL).append(link)); inTitle = false; } else if ("id".equals(localName)) { metadata.put(MetadataKeys.ID, Long.valueOf(id.toString())); inId = false; } else if ("timestamp".equals(localName)) { try { metadata.put(MetadataKeys.LASTEDIT, dateFormat.parse(timestamp.toString())); } catch (ParseException e) { throw new RuntimeException(e.getMessage(), e); } inTimestamp = false; } else if ("text".equals(localName)) { inText = false; if (!keepNamespaced) { // Namespaces are case-insensitive and language-dependent final int pos = title.indexOf(':'); if (pos != -1 && nameSpaces.contains(title.substring(0, pos).toLowerCase())) return; } try { final MutableString html = new MutableString(); DocumentFactory freeFactory; try { freeFactory = freeFactories.take(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } if (parseText) { if (DISAMBIGUATION.search(text) != -1) { // It's a disambiguation page. /* Roi's hack: duplicate links using the page title, so the generic name will end up as anchor text. */ final MutableString newLinks = new MutableString(); for (int start = 0, end; (start = BRACKETS_OPEN.search(text, start)) != -1; start = end) { end = start; final int endOfLink = text.indexOfAnyOf(END_OF_DISAMBIGUATION_LINK, start); // Note that we don't escape title because we are working at the Wikipedia raw text level. if (endOfLink != -1) { newLinks.append(text.array(), start, endOfLink - start).append('|') .append(title).append("]]\n"); end = endOfLink; } end++; } text.append(newLinks); } // We separate categories by OXOXO, so we don't get overflowing phrases. final MutableString category = new MutableString(); for (int start = 0, end; (start = CATEGORY_START.search(text, start)) != -1; start = end) { end = BRACKETS_CLOSED.search(text, start += CATEGORY_START.length()); if (end != -1) category.append(text.subSequence(start, end)).append(" OXOXO "); else break; } metadata.put(MetadataKeys.CATEGORY, category); // Heuristics to get the first paragraph metadata.put(MetadataKeys.FIRSTPAR, new MutableString()); String plainText = wikiModel.render(new PlainTextConverter(true), text.toString()); for (int start = 0; start < plainText.length(); start++) { //System.err.println("Examining " + plainText.charAt( start ) ); if (Character.isWhitespace(plainText.charAt(start))) continue; if (plainText.charAt(start) == '{') { //System.err.print( "Braces " + start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\" -> " ); start = BRACES_CLOSED.search(plainText, start); //System.err.println( start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\"" ); if (start == -1) break; start++; } else if (plainText.charAt(start) == '[') { start = BRACKETS_CLOSED.search(plainText, start); if (start == -1) break; start++; } else { final int end = plainText.indexOf('\n', start); if (end != -1) metadata.put(MetadataKeys.FIRSTPAR, new MutableString(plainText.substring(start, end))); break; } } try { wikiModel.render(new HTMLConverter(), text.toString(), html, false, true); final Map<String, String> categories = wikiModel.getCategories(); // Put back category links in the page (they have been parsed by bliki and to not appear anymore in the HTML rendering) for (Entry<String, String> entry : categories.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue().trim(); if (value.length() != 0) // There are empty such things html.append("\n<a href=\"").append(baseURL).append("Category:") .append(Encoder.encodeTitleToUrl(key, true)).append("\">") .append(HtmlEscapers.htmlEscaper().escape(key)) .append("</a>\n"); } } catch (Exception e) { LOGGER.error("Unexpected exception while parsing " + title, e); } } readyDocumentsAndFactories.put(new DocumentAndFactory( freeFactory.getDocument(IOUtils.toInputStream(html, Charsets.UTF_8), new Reference2ObjectOpenHashMap<Enum<?>, Object>(metadata)), freeFactory)); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } } } } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); if (inId) id.append(ch, start, length); if (inTimestamp) timestamp.append(ch, start, length); if (inNamespaceDef) { nameSpaceAccumulator.append(ch, start, length); inNamespaceDef = false; // Dirty, but it works } } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); } }; final Thread parsingThread = new Thread() { public void run() { try { InputStream in = new FileInputStream(wikipediaXmlDump); if (bzipped) in = new BZip2CompressorInputStream(in); parser.parse( new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); readyDocumentsAndFactories.put(END); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } }; parsingThread.start(); return new AbstractDocumentIterator() { private DocumentFactory lastFactory; @Override public Document nextDocument() throws IOException { try { final DocumentAndFactory documentAndFactory = readyDocumentsAndFactories.take(); if (lastFactory != null) freeFactories.put(lastFactory); if (documentAndFactory == END) return null; lastFactory = documentAndFactory.factory; return documentAndFactory.document; } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } }; }
From source file:efen.parsewiki.WikipediaDocumentSequence.java
@Override public DocumentIterator iterator() throws IOException { final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final MutableString nameSpaceAccumulator = new MutableString(); final ObjectOpenHashSet<MutableString> nameSpacesAccumulator = new ObjectOpenHashSet<MutableString>(); final ArrayBlockingQueue<DocumentFactory> freeFactories = new ArrayBlockingQueue<DocumentFactory>(16); for (int i = freeFactories.remainingCapacity(); i-- != 0;) freeFactories.add(this.factory.copy()); final ArrayBlockingQueue<DocumentAndFactory> readyDocumentsAndFactories = new ArrayBlockingQueue<DocumentAndFactory>( freeFactories.size());// w w w . j a va 2 s.c om final SAXParser parser; try { parser = saxParserFactory.newSAXParser(); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } final DefaultHandler handler = new DefaultHandler() { private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); private boolean inText; private boolean inTitle; private boolean inId; private boolean inTimestamp; private boolean inNamespaceDef; private boolean redirect; private MutableString text = new MutableString(); private MutableString title = new MutableString(); private MutableString id = new MutableString(); private MutableString timestamp = new MutableString(); private final Reference2ObjectMap<Enum<?>, Object> metadata = new Reference2ObjectOpenHashMap<Enum<?>, Object>(); { metadata.put(PropertyBasedDocumentFactory.MetadataKeys.ENCODING, "UTF-8"); metadata.put(MetadataKeys.REDIRECT, redirectAnchors); } @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { redirect = inText = inTitle = inId = inTimestamp = false; text.length(0); title.length(0); id.length(0); timestamp.length(0); } else if ("text".equals(localName)) inText = true; else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first id/title elements. else if ("id".equals(localName) && id.length() == 0) inId = true; else if ("timestamp".equals(localName) && timestamp.length() == 0) inTimestamp = true; else if ("redirect".equals(localName)) { redirect = true; if (attributes.getValue("title") != null) // Accumulate the title of the page as virtual text of the redirect page. synchronized (redirectAnchors) { final String link = Encoder.encodeTitleToUrl(attributes.getValue("title"), true); redirectAnchors.add( new AnchorExtractor.Anchor(new MutableString(baseURL.length() + link.length()) .append(baseURL).append(link), title.copy())); } } else if ("namespace".equals(localName)) { // Found a new namespace inNamespaceDef = true; nameSpaceAccumulator.length(0); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("namespace".equals(localName)) { // Collecting a namespace if (nameSpaceAccumulator.length() != 0) nameSpacesAccumulator.add(nameSpaceAccumulator.copy().toLowerCase()); return; } if ("namespaces".equals(localName)) { // All namespaces collected nameSpaces = ImmutableSet.copyOf(nameSpacesAccumulator); return; } if (!redirect) { if ("title".equals(localName)) { // Set basic metadata for the page metadata.put(PropertyBasedDocumentFactory.MetadataKeys.TITLE, title.copy()); String link = Encoder.encodeTitleToUrl(title.toString(), true); metadata.put(PropertyBasedDocumentFactory.MetadataKeys.URI, new MutableString(baseURL.length() + link.length()).append(baseURL).append(link)); inTitle = false; } else if ("id".equals(localName)) { metadata.put(MetadataKeys.ID, Long.valueOf(id.toString())); inId = false; } else if ("timestamp".equals(localName)) { try { metadata.put(MetadataKeys.LASTEDIT, dateFormat.parse(timestamp.toString())); } catch (ParseException e) { throw new RuntimeException(e.getMessage(), e); } inTimestamp = false; } else if ("text".equals(localName)) { inText = false; if (!keepNamespaced) { // Namespaces are case-insensitive and language-dependent final int pos = title.indexOf(':'); if (pos != -1 && isATrueNamespace(title.substring(0, pos))) return; } try { final MutableString html = new MutableString(); DocumentFactory freeFactory; try { freeFactory = freeFactories.take(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } if (parseText) { if (DISAMBIGUATION.search(text) != -1) { // It's a disambiguation page. /* Roi's hack: duplicate links using the page title, so the generic name will end up as anchor text. */ final MutableString newLinks = new MutableString(); for (int start = 0, end; (start = BRACKETS_OPEN.search(text, start)) != -1; start = end) { end = start; final int endOfLink = text.indexOfAnyOf(END_OF_DISAMBIGUATION_LINK, start); // Note that we don't escape title because we are working at the Wikipedia raw text level. if (endOfLink != -1) { newLinks.append(text.array(), start, endOfLink - start).append('|') .append(title).append("]]\n"); end = endOfLink; } end++; } text.append(newLinks); } // We separate categories by OXOXO, so we don't get overflowing phrases. final MutableString category = new MutableString(); for (int start = 0, end; (start = CATEGORY_START.search(text, start)) != -1; start = end) { end = BRACKETS_CLOSED.search(text, start += CATEGORY_START.length()); if (end != -1) category.append(text.subSequence(start, end)).append(" OXOXO "); else break; } metadata.put(MetadataKeys.CATEGORY, category); // Heuristics to get the first paragraph metadata.put(MetadataKeys.FIRSTPAR, new MutableString()); String plainText = new WikiModel(imageBaseURL, linkBaseURL) .render(new PlainTextConverter(true), text.toString()); for (int start = 0; start < plainText.length(); start++) { //System.err.println("Examining " + plainText.charAt( start ) ); if (Character.isWhitespace(plainText.charAt(start))) continue; if (plainText.charAt(start) == '{') { //System.err.print( "Braces " + start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\" -> " ); start = BRACES_CLOSED.search(plainText, start); //System.err.println( start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\"" ); if (start == -1) break; start++; } else if (plainText.charAt(start) == '[') { start = BRACKETS_CLOSED.search(plainText, start); if (start == -1) break; start++; } else { final int end = plainText.indexOf('\n', start); if (end != -1) metadata.put(MetadataKeys.FIRSTPAR, new MutableString(plainText.substring(start, end)));//new MutableString( new WikiModel( imageBaseURL, linkBaseURL ).render( new PlainTextConverter( true ), text.substring( start, end ).toString() ) ) ); break; } } try { WikiModel wikiModel = new WikiModel(imageBaseURL, linkBaseURL); wikiModel.render(new HTMLConverter(), text.toString(), html, false, false); final Map<String, String> categories = wikiModel.getCategories(); // Put back category links in the page (they have been parsed by bliki and to not appear anymore in the HTML rendering) for (Entry<String, String> entry : categories.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue().trim(); if (value.length() != 0) // There are empty such things html.append("\n<a href=\"").append(baseURL).append("Category:") .append(Encoder.encodeTitleToUrl(key, true)).append("\">") .append(HtmlEscapers.htmlEscaper().escape(key)) .append("</a>\n"); } } catch (Exception e) { LOGGER.error("Unexpected exception while parsing " + title, e); } } readyDocumentsAndFactories.put(new DocumentAndFactory( freeFactory.getDocument(IOUtils.toInputStream(html, Charsets.UTF_8), new Reference2ObjectOpenHashMap<Enum<?>, Object>(metadata)), freeFactory)); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } } } } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); if (inId) id.append(ch, start, length); if (inTimestamp) timestamp.append(ch, start, length); if (inNamespaceDef) { nameSpaceAccumulator.append(ch, start, length); inNamespaceDef = false; // Dirty, but it works } } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); } }; final Thread parsingThread = new Thread() { public void run() { try { InputStream in = new FileInputStream(wikipediaXmlDump); if (bzipped) in = new BZip2CompressorInputStream(in); parser.parse( new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); readyDocumentsAndFactories.put(END); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } }; parsingThread.start(); return new AbstractDocumentIterator() { private DocumentFactory lastFactory; @Override public Document nextDocument() throws IOException { try { final DocumentAndFactory documentAndFactory = readyDocumentsAndFactories.take(); if (lastFactory != null) freeFactories.put(lastFactory); if (documentAndFactory == END) return null; lastFactory = documentAndFactory.factory; return documentAndFactory.document; } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } }; }
From source file:com.flexive.core.storage.GenericDivisionImporter.java
/** * Import flat storages to the hierarchical storage * * @param con an open and valid connection to store imported data * @param zip zip file containing the data * @throws Exception on errors//from w ww . j ava 2s.c o m */ protected void importFlatStoragesHierarchical(Connection con, ZipFile zip) throws Exception { //mapping: storage->level->columnname->assignment id final Map<String, Map<Integer, Map<String, Long>>> flatAssignmentMapping = new HashMap<String, Map<Integer, Map<String, Long>>>( 5); //mapping: assignment id->position index final Map<Long, Integer> assignmentPositions = new HashMap<Long, Integer>(100); //mapping: flatstorage->column sizes [string,bigint,double,select,text] final Map<String, Integer[]> flatstoragesColumns = new HashMap<String, Integer[]>(5); ZipEntry zeMeta = getZipEntry(zip, FILE_FLATSTORAGE_META); DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Document document = builder.parse(zip.getInputStream(zeMeta)); XPath xPath = XPathFactory.newInstance().newXPath(); //calculate column sizes NodeList nodes = (NodeList) xPath.evaluate("/flatstorageMeta/storageMeta", document, XPathConstants.NODESET); Node currNode; for (int i = 0; i < nodes.getLength(); i++) { currNode = nodes.item(i); int cbigInt = Integer.parseInt(currNode.getAttributes().getNamedItem("bigInt").getNodeValue()); int cdouble = Integer.parseInt(currNode.getAttributes().getNamedItem("double").getNodeValue()); int cselect = Integer.parseInt(currNode.getAttributes().getNamedItem("select").getNodeValue()); int cstring = Integer.parseInt(currNode.getAttributes().getNamedItem("string").getNodeValue()); int ctext = Integer.parseInt(currNode.getAttributes().getNamedItem("text").getNodeValue()); String tableName = null; if (currNode.hasChildNodes()) { for (int j = 0; j < currNode.getChildNodes().getLength(); j++) if (currNode.getChildNodes().item(j).getNodeName().equals("name")) { tableName = currNode.getChildNodes().item(j).getTextContent(); } } if (tableName != null) { flatstoragesColumns.put(tableName, new Integer[] { cstring, cbigInt, cdouble, cselect, ctext }); } } //parse mappings nodes = (NodeList) xPath.evaluate("/flatstorageMeta/mapping", document, XPathConstants.NODESET); for (int i = 0; i < nodes.getLength(); i++) { currNode = nodes.item(i); long assignment = Long.valueOf(currNode.getAttributes().getNamedItem("assid").getNodeValue()); int level = Integer.valueOf(currNode.getAttributes().getNamedItem("lvl").getNodeValue()); String storage = null; String columnname = null; final NodeList childNodes = currNode.getChildNodes(); for (int c = 0; c < childNodes.getLength(); c++) { Node child = childNodes.item(c); if ("tblname".equals(child.getNodeName())) storage = child.getTextContent(); else if ("colname".equals(child.getNodeName())) columnname = child.getTextContent(); } if (storage == null || columnname == null) throw new Exception("Invalid flatstorage export: could not read storage or column name!"); if (!flatAssignmentMapping.containsKey(storage)) flatAssignmentMapping.put(storage, new HashMap<Integer, Map<String, Long>>(20)); Map<Integer, Map<String, Long>> levelMap = flatAssignmentMapping.get(storage); if (!levelMap.containsKey(level)) levelMap.put(level, new HashMap<String, Long>(30)); Map<String, Long> columnMap = levelMap.get(level); if (!columnMap.containsKey(columnname)) columnMap.put(columnname, assignment); //calculate position assignmentPositions.put(assignment, getAssignmentPosition(flatstoragesColumns.get(storage), columnname)); } if (flatAssignmentMapping.size() == 0) { LOG.warn("No flatstorage assignments found to process!"); return; } ZipEntry zeData = getZipEntry(zip, FILE_DATA_FLAT); final String xpathStorage = "flatstorages/storage"; final String xpathData = "flatstorages/storage/data"; final PreparedStatement psGetAssInfo = con.prepareStatement( "SELECT DISTINCT a.APROPERTY,a.XALIAS,p.DATATYPE FROM " + DatabaseConst.TBL_STRUCT_ASSIGNMENTS + " a, " + DatabaseConst.TBL_STRUCT_PROPERTIES + " p WHERE a.ID=? AND p.ID=a.APROPERTY"); final Map<Long, Object[]> assignmentPropAlias = new HashMap<Long, Object[]>(assignmentPositions.size()); final String insert1 = "INSERT INTO " + DatabaseConst.TBL_CONTENT_DATA + //1 2 3 4 5 6 =1 =1 =1 =1 7 8 9 "(ID,VER,POS,LANG,TPROP,ASSIGN,XDEPTH,XMULT,XINDEX,PARENTXMULT,ISMAX_VER,ISLIVE_VER,ISMLDEF,"; final String insert2 = "(?,?,?,?,1,?,?,1,1,1,?,?,?,"; final PreparedStatement psString = con .prepareStatement(insert1 + "FTEXT1024,UFTEXT1024,FSELECT,FINT)VALUES" + insert2 + "?,?,0,?)"); final PreparedStatement psText = con .prepareStatement(insert1 + "FCLOB,UFCLOB,FSELECT,FINT)VALUES" + insert2 + "?,?,0,?)"); final PreparedStatement psDouble = con .prepareStatement(insert1 + "FDOUBLE,FSELECT,FINT)VALUES" + insert2 + "?,0,?)"); final PreparedStatement psNumber = con .prepareStatement(insert1 + "FINT,FSELECT,FBIGINT)VALUES" + insert2 + "?,0,?)"); final PreparedStatement psLargeNumber = con .prepareStatement(insert1 + "FBIGINT,FSELECT,FINT)VALUES" + insert2 + "?,0,?)"); final PreparedStatement psFloat = con .prepareStatement(insert1 + "FFLOAT,FSELECT,FINT)VALUES" + insert2 + "?,0,?)"); final PreparedStatement psBoolean = con .prepareStatement(insert1 + "FBOOL,FSELECT,FINT)VALUES" + insert2 + "?,0,?)"); final PreparedStatement psReference = con .prepareStatement(insert1 + "FREF,FSELECT,FINT)VALUES" + insert2 + "?,0,?)"); final PreparedStatement psSelectOne = con .prepareStatement(insert1 + "FSELECT,FINT)VALUES" + insert2 + "?,?)"); try { final SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); final DefaultHandler handler = new DefaultHandler() { private String currentElement = null; private String currentStorage = null; private Map<String, String> data = new HashMap<String, String>(10); private StringBuilder sbData = new StringBuilder(10000); boolean inTag = false; boolean inElement = false; List<String> path = new ArrayList<String>(10); StringBuilder currPath = new StringBuilder(100); int insertCount = 0; /** * {@inheritDoc} */ @Override public void startDocument() throws SAXException { inTag = false; inElement = false; path.clear(); currPath.setLength(0); sbData.setLength(0); data.clear(); currentElement = null; currentStorage = null; insertCount = 0; } /** * {@inheritDoc} */ @Override public void endDocument() throws SAXException { LOG.info("Imported [" + insertCount + "] flatstorage entries into the hierarchical storage"); } /** * {@inheritDoc} */ @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { pushPath(qName, attributes); if (currPath.toString().equals(xpathData)) { inTag = true; data.clear(); for (int i = 0; i < attributes.getLength(); i++) { String name = attributes.getLocalName(i); if (StringUtils.isEmpty(name)) name = attributes.getQName(i); data.put(name, attributes.getValue(i)); } } else if (currPath.toString().equals(xpathStorage)) { currentStorage = attributes.getValue("name"); LOG.info("Processing storage: " + currentStorage); } else { currentElement = qName; } inElement = true; sbData.setLength(0); } /** * Push a path element from the stack * * @param qName element name to push * @param att attributes */ @SuppressWarnings({ "UnusedDeclaration" }) private void pushPath(String qName, Attributes att) { path.add(qName); buildPath(); } /** * Pop the top path element from the stack */ private void popPath() { path.remove(path.size() - 1); buildPath(); } /** * Rebuild the current path */ private synchronized void buildPath() { currPath.setLength(0); for (String s : path) currPath.append(s).append('/'); if (currPath.length() > 1) currPath.delete(currPath.length() - 1, currPath.length()); // System.out.println("currPath: " + currPath); } /** * {@inheritDoc} */ @Override public void endElement(String uri, String localName, String qName) throws SAXException { if (currPath.toString().equals(xpathData)) { // LOG.info("Insert [" + xpathData + "]: [" + data + "]"); inTag = false; processData(); /*try { if (insertMode) { if (executeInsertPhase) { processColumnSet(insertColumns, psInsert); counter += psInsert.executeUpdate(); } } else { if (executeUpdatePhase) { if (processColumnSet(updateSetColumns, psUpdate)) { processColumnSet(updateClauseColumns, psUpdate); counter += psUpdate.executeUpdate(); } } } } catch (SQLException e) { throw new SAXException(e); } catch (ParseException e) { throw new SAXException(e); }*/ } else { if (inTag) { data.put(currentElement, sbData.toString()); } currentElement = null; } popPath(); inElement = false; sbData.setLength(0); } void processData() { // System.out.println("processing " + currentStorage + " -> " + data); final String[] cols = { "string", "bigint", "double", "select", "text" }; for (String column : data.keySet()) { if (column.endsWith("_mld")) continue; for (String check : cols) { if (column.startsWith(check)) { if ("select".equals(check) && "0".equals(data.get(column))) continue; //dont insert 0-referencing selects try { insertData(column); } catch (SQLException e) { //noinspection ThrowableInstanceNeverThrown throw new FxDbException(e, "ex.db.sqlError", e.getMessage()) .asRuntimeException(); } } } } } private void insertData(String column) throws SQLException { final int level = Integer.parseInt(data.get("lvl")); long assignment = flatAssignmentMapping.get(currentStorage).get(level) .get(column.toUpperCase()); int pos = FxArrayUtils.getIntElementAt(data.get("positions"), ',', assignmentPositions.get(assignment)); String _valueData = data.get("valuedata"); Integer valueData = _valueData == null ? null : FxArrayUtils.getHexIntElementAt(data.get("valuedata"), ',', assignmentPositions.get(assignment)); Object[] propXP = getPropertyXPathDataType(assignment); long prop = (Long) propXP[0]; String xpath = (String) propXP[1]; FxDataType dataType; try { dataType = FxDataType.getById((Long) propXP[2]); } catch (FxNotFoundException e) { throw e.asRuntimeException(); } long id = Long.parseLong(data.get("id")); int ver = Integer.parseInt(data.get("ver")); long lang = Integer.parseInt(data.get("lang")); boolean isMaxVer = "1".equals(data.get("ismax_ver")); boolean isLiveVer = "1".equals(data.get("islive_ver")); boolean mlDef = "1".equals(data.get(column + "_mld")); PreparedStatement ps; int vdPos; switch (dataType) { case String1024: ps = psString; ps.setString(10, data.get(column)); ps.setString(11, data.get(column).toUpperCase()); vdPos = 12; break; case Text: case HTML: ps = psText; ps.setString(10, data.get(column)); ps.setString(11, data.get(column).toUpperCase()); vdPos = 12; break; case Number: ps = psNumber; ps.setLong(10, Long.valueOf(data.get(column))); vdPos = 11; break; case LargeNumber: ps = psLargeNumber; ps.setLong(10, Long.valueOf(data.get(column))); vdPos = 11; break; case Reference: ps = psReference; ps.setLong(10, Long.valueOf(data.get(column))); vdPos = 11; break; case Float: ps = psFloat; ps.setFloat(10, Float.valueOf(data.get(column))); vdPos = 11; break; case Double: ps = psDouble; ps.setDouble(10, Double.valueOf(data.get(column))); vdPos = 11; break; case Boolean: ps = psBoolean; ps.setBoolean(10, "1".equals(data.get(column))); vdPos = 11; break; case SelectOne: ps = psSelectOne; ps.setLong(10, Long.valueOf(data.get(column))); vdPos = 11; break; default: //noinspection ThrowableInstanceNeverThrown throw new FxInvalidParameterException("assignment", "ex.structure.flatstorage.datatype.unsupported", dataType.name()) .asRuntimeException(); } ps.setLong(1, id); ps.setInt(2, ver); ps.setInt(3, pos); ps.setLong(4, lang); ps.setLong(5, prop); ps.setLong(6, assignment); ps.setBoolean(7, isMaxVer); ps.setBoolean(8, isLiveVer); ps.setBoolean(9, mlDef); if (valueData == null) ps.setNull(vdPos, java.sql.Types.NUMERIC); else ps.setInt(vdPos, valueData); ps.executeUpdate(); insertCount++; } /** * Get property id, xpath and data type for an assignment * * @param assignment assignment id * @return Object[] {propertyId, xpath, datatype} */ private Object[] getPropertyXPathDataType(long assignment) { if (assignmentPropAlias.get(assignment) != null) return assignmentPropAlias.get(assignment); try { psGetAssInfo.setLong(1, assignment); ResultSet rs = psGetAssInfo.executeQuery(); if (rs != null && rs.next()) { Object[] data = new Object[] { rs.getLong(1), rs.getString(2), rs.getLong(3) }; assignmentPropAlias.put(assignment, data); return data; } } catch (SQLException e) { throw new IllegalArgumentException( "Could not load data for assignment " + assignment + ": " + e.getMessage()); } throw new IllegalArgumentException("Could not load data for assignment " + assignment + "!"); } /** * {@inheritDoc} */ @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inElement) sbData.append(ch, start, length); } }; parser.parse(zip.getInputStream(zeData), handler); } finally { Database.closeObjects(GenericDivisionImporter.class, psGetAssInfo, psString, psBoolean, psDouble, psFloat, psLargeNumber, psNumber, psReference, psSelectOne, psText); } }